libsast 3.0.2__tar.gz → 3.1.1__tar.gz

{libsast-3.0.2 → libsast-3.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: libsast
-Version: 3.0.2
+Version: 3.1.1
 Summary: A generic SAST library built on top of semgrep and regex
 Home-page: https://github.com/ajinabraham/libsast
 License: LGPL-3.0-or-later
@@ -22,7 +22,7 @@ Classifier: Topic :: Security
 Classifier: Topic :: Software Development :: Quality Assurance
 Requires-Dist: pyyaml (>=6.0)
 Requires-Dist: requests
-Requires-Dist: semgrep (==0.117.0) ; sys_platform != "win32"
+Requires-Dist: semgrep (==1.86.0) ; sys_platform != "win32"
 Project-URL: Bug Tracker, https://github.com/ajinabraham/libsast/issues
 Project-URL: Documentation, https://github.com/ajinabraham/libsast/blob/master/README.md
 Project-URL: Repository, https://github.com/ajinabraham/libsast

{libsast-3.0.2 → libsast-3.1.1}/libsast/__init__.py

@@ -12,7 +12,7 @@ year = str(datetime.now().year)
 __title__ = 'libsast'
 __authors__ = 'Ajin Abraham'
 __copyright__ = f'Copyright {year} Ajin Abraham, opensecurity.in'
-__version__ = '3.0.2'
+__version__ = '3.1.1'
 __version_info__ = tuple(int(i) for i in __version__.split('.'))
 __all__ = [
     'Scanner',

{libsast-3.0.2 → libsast-3.1.1}/libsast/core_matcher/choice_matcher.py

@@ -31,14 +31,18 @@ class ChoiceMatcher:
 
     def scan(self, paths: list) -> dict:
         """Scan file(s) or directory per rule."""
-        if not (self.scan_rules and paths):
-            return
-        self.validate_rules()
-
         if self.show_progress:
             pbar = common.ProgressBar('Choice Match', len(self.scan_rules))
             self.scan_rules = pbar.progress_loop(self.scan_rules)
 
+        file_contents = self.read_file_contents(paths)
+        return self.regex_scan(file_contents)
+
+    def read_file_contents(self, paths: list) -> list:
+        """Load file(s) content."""
+        if not (self.scan_rules and paths):
+            return
+        self.validate_rules()
         choice_args = []
         for rule in self.scan_rules:
             scan_paths = paths
@@ -46,25 +50,29 @@ class ChoiceMatcher:
                 # Scan only alternative path
                 scan_paths = [Path(self.alternative_path)]
             choice_args.append((scan_paths, rule))
+        if not choice_args:
+            return []
 
-        # Use ThreadPoolExecutor for reading file contents and
-        # ProcessPoolExecutor for processing regex
-        with ThreadPoolExecutor() as io_executor, ProcessPoolExecutor(
-                max_workers=self.cpu) as cpu_executor:
+        # Use ThreadPoolExecutor for file reading
+        with ThreadPoolExecutor() as io_executor:
+            # Submit file reading tasks and wait for results
             futures = []
             for args_tuple in choice_args:
-                # Submit each read task and store the future along with the args
                 future = io_executor.submit(
                     self._read_file_contents, args_tuple)
-                futures.append((future, args_tuple))
+                futures.append(future)
+            return [future.result() for future in futures]
+
+    def regex_scan(self, file_contents) -> list:
+        """Process regex matches on the file contents."""
+        # Use ProcessPoolExecutor for regex processing
+        with ProcessPoolExecutor(max_workers=self.cpu) as cpu_executor:
 
             results = []
-            for future, _ in futures:
-                file_contents = future.result()
-                # This will block until the file reading is done
-                # Process the file contents with ProcessPoolExecutor
+            for content in file_contents:
+                # Process Choice Matcher on the file contents
                 process_future = cpu_executor.submit(
-                    self.choice_matcher, file_contents)
+                    self.choice_matcher, content)
                 results.append(process_future.result())
 
         self.add_finding(results)

{libsast-3.0.2 → libsast-3.1.1}/libsast/core_matcher/pattern_matcher.py

@@ -31,27 +31,39 @@ class PatternMatcher:
 
     def scan(self, paths: list) -> dict:
         """Scan file(s) or directory."""
-        if not (self.scan_rules and paths):
-            return
-        self.validate_rules()
-
         if self.show_progress:
             pbar = common.ProgressBar('Pattern Match', len(paths))
             paths = pbar.progress_loop(paths)
 
+        file_contents = self.read_file_contents(paths)
+        return self.regex_scan(file_contents)
+
+    def read_file_contents(self, paths: list) -> list:
+        """Load file(s) content."""
+        if not (self.scan_rules and paths):
+            return
+        self.validate_rules()
+
         # Filter files by extension and size, prepare list for processing
         files_to_scan = {
             sfile for sfile in paths
             if is_file_valid(sfile, self.exts, 5)
         }
+        if not files_to_scan:
+            return []
 
-        # Use a ThreadPool for file reading, and ProcessPool for CPU-bound regex
-        with ThreadPoolExecutor() as io_executor, ProcessPoolExecutor(
-                max_workers=self.cpu) as cpu_executor:
+        # Use a ThreadPool for file reading
+        with ThreadPoolExecutor() as io_executor:
 
             # Read all files
             file_contents = list(io_executor.map(
                 self._read_file_content, files_to_scan))
+            return file_contents
+
+    def regex_scan(self, file_contents: list) -> dict:
+        """Scan file(s) content."""
+        # Use a ProcessPool for CPU-bound regex
+        with ProcessPoolExecutor(max_workers=self.cpu) as cpu_executor:
 
             # Run regex on file data
             results = cpu_executor.map(

libsast-3.1.1/libsast/core_sgrep/helpers.py

@@ -0,0 +1,29 @@
+# -*- coding: utf_8 -*-
+"""Semantic Grep Helpers."""
+import json
+import platform
+import subprocess
+
+
+def invoke_semgrep(paths, scan_rules):
+    if platform.system() == 'Windows':
+        return None
+    ps = [pt.as_posix() for pt in paths]
+    command = [
+        'semgrep',
+        '--metrics=off',
+        '--no-rewrite-rule-ids',
+        '--json',
+        '-q',
+        '--config',
+        scan_rules,
+        *ps,
+    ]
+    try:
+        result = subprocess.run(command, capture_output=True, text=True, check=True)
+        return json.loads(result.stdout)
+    except subprocess.CalledProcessError as e:
+        try:
+            return json.loads(e.output)
+        except json.JSONDecodeError:
+            return {'errors': e.output}

{libsast-3.0.2 → libsast-3.1.1}/libsast/core_sgrep/semantic_sgrep.py

@@ -43,9 +43,11 @@ class SemanticGrep:
 
     def format_output(self, results):
         """Format sgrep results."""
-        errs = self.findings.get('errors')
+        errs = results.get('errors')
         if errs:
             self.findings['errors'] = errs
+        if not results.get('results'):
+            return
         smatches = self.findings['matches']
         for find in results['results']:
             file_details = {
@@ -54,7 +56,7 @@ class SemanticGrep:
                 'match_lines': (find['start']['line'], find['end']['line']),
                 'match_string': find['extra']['lines'],
             }
-            rule_id = find['check_id'].rsplit('.', 1)[1]
+            rule_id = find['check_id']
             if rule_id in smatches:
                 smatches[rule_id]['files'].append(file_details)
             else:

{libsast-3.0.2 → libsast-3.1.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "libsast"
-version = "3.0.2"
+version = "3.1.1"
 description = "A generic SAST library built on top of semgrep and regex"
 keywords = ["libsast", "SAST", "Python SAST", "SAST API", "Regex SAST", "Pattern Matcher"]
 authors = ["Ajin Abraham <ajin@opensecurity.in>"]
@@ -26,7 +26,7 @@ libsast = "libsast.__main__:main"
 python = "^3.8"
 requests = "*"
 pyyaml = ">=6.0"
-semgrep = {version = "0.117.0", markers = "sys_platform != 'win32'"}
+semgrep = {version = "1.86.0", markers = "sys_platform != 'win32'"}
 
 [tool.poetry.group.dev.dependencies]
 bandit = "*"

libsast-3.0.2/libsast/core_sgrep/helpers.py

@@ -1,68 +0,0 @@
-# -*- coding: utf_8 -*-
-"""Semantic Grep Helpers."""
-import json
-import logging
-import platform
-import multiprocessing
-
-
-def invoke_semgrep(paths, scan_rules, **kwargs):
-    """Call Semgrep."""
-    if platform.system() == 'Windows':
-        return None
-    from semgrep import semgrep_main
-    from semgrep.state import get_state
-    from semgrep.constants import OutputFormat
-    from semgrep.output import OutputHandler, OutputSettings
-    try:
-        cpu_count = multiprocessing.cpu_count()
-    except NotImplementedError:
-        cpu_count = 1  # CPU count is not implemented on Windows
-    # Semgrep output formatting
-    state = get_state()
-    state.terminal.configure(
-        verbose=False,
-        debug=False,
-        quiet=True,
-        force_color=False,
-    )
-    logging.getLogger('semgrep').propagate = False
-    output_settings = OutputSettings(
-        output_format=OutputFormat.JSON,
-        output_destination=None,
-        output_per_finding_max_lines_limit=None,
-        output_per_line_max_chars_limit=None,
-        error_on_findings=False,
-        verbose_errors=False,
-        strict=False,
-        timeout_threshold=3,
-    )
-    output_handler = OutputHandler(output_settings)
-    (
-        filtered_matches_by_rule,
-        _,
-        _,
-        _,
-        _,
-        _,
-        _,
-        _,
-        _,
-        _,
-        _,
-        _,
-    ) = semgrep_main.main(
-        output_handler=output_handler,
-        target=[pt.as_posix() for pt in paths],
-        jobs=cpu_count,
-        pattern=None,
-        lang=None,
-        configs=[scan_rules],
-        timeout=5,
-        timeout_threshold=3,
-        **kwargs,
-    )
-    output_handler.rule_matches = [
-        m for ms in filtered_matches_by_rule.values() for m in ms
-    ]
-    return json.loads(output_handler._build_output())