ci(semgrep): add more rules, r/python.correctness (#12876)

* ci(semgrep): add more rules, r/python.correctness

- Added file for defining rules as per frappe data model:
frappe_correctness.yml
- Add rule for SQLi, with WARNING only for now
- Add rule file for UX
- WARNING | INFO do not fail the build now

* ci(semgrep): on_cancel, on_submit correctness rule

* ci(semgrep): split workflow in steps

* ci(semgrep): catch line breaks in _()

* chore: fix sider issue

.github/helper/semgrep_rules/frappe_correctness.py

@@ -0,0 +1,28 @@
+import frappe
+from frappe import _, flt
+
+from frappe.model.document import Document
+
+
+def on_submit(self):
+	if self.value_of_goods == 0:
+		frappe.throw(_('Value of goods cannot be 0'))
+	# ruleid: frappe-modifying-after-submit
+	self.status = 'Submitted'
+
+def on_submit(self):   # noqa
+	if flt(self.per_billed) < 100:
+		self.update_billing_status()
+	else:
+		# todook: frappe-modifying-after-submit
+		self.status = "Completed"
+		self.db_set("status", "Completed")
+
+class TestDoc(Document):
+	pass
+
+	def validate(self):
+		#ruleid: frappe-modifying-child-tables-while-iterating
+		for item in self.child_table:
+			if item.value < 0:
+				self.remove(item)

.github/helper/semgrep_rules/frappe_correctness.yml

@@ -0,0 +1,135 @@
+# This file specifies rules for correctness according to how frappe doctype data model works.
+
+rules:
+- id: frappe-modifying-but-not-comitting
+  patterns:
+    - pattern: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+          ...
+          self.db_set(..., self.$ATTR, ...)
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.db_set(..., $SOME_VAR, ...)
+    - pattern-not: |
+        def $METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.save()
+    - metavariable-regex:
+        metavariable: '$ATTR'
+        # this is negative look-ahead, add more attrs to ignore like (ignore|ignore_this_too|ignore_me)
+        regex: '^(?!ignore_linked_doctypes|status_updater)(.*)$'
+    - metavariable-regex:
+        metavariable: "$METHOD"
+        regex: "(on_submit|on_cancel)"
+  message: |
+    DocType modified in self.$METHOD. Please check if modification of self.$ATTR is commited to database.
+  languages: [python]
+  severity: ERROR
+
+- id: frappe-modifying-but-not-comitting-other-method
+  patterns:
+  - pattern: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+          ...
+          self.db_set(..., self.$ATTR, ...)
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = $SOME_VAR
+          ...
+          self.db_set(..., $SOME_VAR, ...)
+  - pattern-not: |
+      class $DOCTYPE(...):
+        def $METHOD(self, ...):
+          ...
+          self.$ANOTHER_METHOD()
+          ...
+          self.save()
+        def $ANOTHER_METHOD(self, ...):
+          ...
+          self.$ATTR = ...
+  - metavariable-regex:
+      metavariable: "$METHOD"
+      regex: "(on_submit|on_cancel)"
+  message: |
+    self.$ANOTHER_METHOD is called from self.$METHOD, check if changes to self.$ATTR are commited to database.
+  languages: [python]
+  severity: ERROR
+
+- id: frappe-print-function-in-doctypes
+  pattern: print(...)
+  message: |
+      Did you mean to leave this print statement in? Consider using msgprint or logger instead of print statement.
+  languages: [python]
+  severity: WARNING
+  paths:
+      exclude:
+        - test_*.py
+      include:
+        - "*/**/doctype/*"
+
+- id: frappe-modifying-child-tables-while-iterating
+  pattern-either:
+    - pattern: |
+        for $ROW in self.$TABLE:
+            ...
+            self.remove(...)
+    - pattern: |
+        for $ROW in self.$TABLE:
+            ...
+            self.append(...)
+  message: |
+      Child table being modified while iterating on it.
+  languages: [python]
+  severity: ERROR
+  paths:
+      include:
+        - "*/**/doctype/*"
+
+- id: frappe-same-key-assigned-twice
+  pattern-either:
+    - pattern: |
+        {..., $X: $A, ..., $X: $B, ...}
+    - pattern: |
+        dict(..., ($X, $A), ..., ($X, $B), ...)
+    - pattern: |
+        _dict(..., ($X, $A), ..., ($X, $B), ...)
+  message: |
+      key `$X` is uselessly assigned twice. This could be a potential bug.
+  languages: [python]
+  severity: ERROR

.github/helper/semgrep_rules/security.yml

@@ -12,3 +12,18 @@ rules:
     exclude:
       - frappe/__init__.py
       - frappe/commands/utils.py
+
+- id: frappe-sqli-format-strings
+  patterns:
+    - pattern-inside: |
+        @frappe.whitelist()
+        def $FUNC(...):
+            ...
+    - pattern-either:
+        - pattern: frappe.db.sql("..." % ...)
+        - pattern: frappe.db.sql(f"...", ...)
+        - pattern: frappe.db.sql("...".format(...), ...)
+  message: |
+      Detected use of raw string formatting for SQL queries. This can lead to sql injection vulnerabilities. Refer security guidelines - https://github.com/frappe/erpnext/wiki/Code-Security-Guidelines
+  languages: [python]
+  severity: WARNING

.github/helper/semgrep_rules/translate.yml

@@ -44,7 +44,8 @@ rules:
   pattern-either:
   - pattern: _(...) + ... + _(...)
   - pattern: _("..." + "...")
-  - pattern-regex: '_\([^\)]*\\\s*'
+  - pattern-regex: '_\([^\)]*\\\s*'    # lines broken by `\`
+  - pattern-regex: '_\(\s*\n'          # line breaks allowed by python for using ( )
   message: |
     Do not split strings inside translate function. Do not concatenate using translate functions.
     Please refer: https://frappeframework.com/docs/user/en/translations

.github/helper/semgrep_rules/ux.py

@@ -0,0 +1,31 @@
+import frappe
+from frappe import msgprint, throw, _
+
+
+# ruleid: frappe-missing-translate-function
+throw("Error Occured")
+
+# ruleid: frappe-missing-translate-function
+frappe.throw("Error Occured")
+
+# ruleid: frappe-missing-translate-function
+frappe.msgprint("Useful message")
+
+# ruleid: frappe-missing-translate-function
+msgprint("Useful message")
+
+
+# ok: frappe-missing-translate-function
+translatedmessage = _("Hello")
+
+# ok: frappe-missing-translate-function
+throw(translatedmessage)
+
+# ok: frappe-missing-translate-function
+msgprint(translatedmessage)
+
+# ok: frappe-missing-translate-function
+msgprint(_("Helpful message"))
+
+# ok: frappe-missing-translate-function
+frappe.throw(_("Error occured"))

.github/helper/semgrep_rules/ux.yml

@@ -0,0 +1,15 @@
+rules:
+- id: frappe-missing-translate-function
+  pattern-either:
+  - patterns:
+      - pattern: frappe.msgprint("...", ...)
+      - pattern-not: frappe.msgprint(_("..."), ...)
+      - pattern-not: frappe.msgprint(__("..."), ...)
+  - patterns:
+      - pattern: frappe.throw("...", ...)
+      - pattern-not: frappe.throw(_("..."), ...)
+      - pattern-not: frappe.throw(__("..."), ...)
+  message: |
+      All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations
+  languages: [python, javascript, json]
+  severity: ERROR

.github/workflows/semgrep.yml

@@ -14,9 +14,19 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
-    - name: Run semgrep
+
+    - name: Setup semgrep
       run: |
         python -m pip install -q semgrep
         git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q
+
+    - name: Semgrep errors
+      run: |
         files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
-        [[ -d .github/helper/semgrep_rules ]] && semgrep --config=.github/helper/semgrep_rules --quiet --error $files
+        [[ -d .github/helper/semgrep_rules ]] && semgrep --severity ERROR --config=.github/helper/semgrep_rules --quiet --error $files
+        semgrep --config="r/python.lang.correctness" --quiet --error $files
+
+    - name: Semgrep warnings
+      run: |
+        files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
+        [[ -d .github/helper/semgrep_rules ]] && semgrep --severity WARNING --severity INFO --config=.github/helper/semgrep_rules --quiet $files