libsast 3.1.0__tar.gz → 3.1.2__tar.gz

{libsast-3.1.0 → libsast-3.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: libsast
-Version: 3.1.0
+Version: 3.1.2
 Summary: A generic SAST library built on top of semgrep and regex
 Home-page: https://github.com/ajinabraham/libsast
 License: LGPL-3.0-or-later
@@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Security
 Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Dist: billiard (>=4.2.1,<5.0.0)
 Requires-Dist: pyyaml (>=6.0)
 Requires-Dist: requests
 Requires-Dist: semgrep (==1.86.0) ; sys_platform != "win32"

{libsast-3.1.0 → libsast-3.1.2}/libsast/__init__.py

@@ -12,7 +12,7 @@ year = str(datetime.now().year)
 __title__ = 'libsast'
 __authors__ = 'Ajin Abraham'
 __copyright__ = f'Copyright {year} Ajin Abraham, opensecurity.in'
-__version__ = '3.1.0'
+__version__ = '3.1.2'
 __version_info__ = tuple(int(i) for i in __version__.split('.'))
 __all__ = [
     'Scanner',

{libsast-3.1.0 → libsast-3.1.2}/libsast/core_matcher/choice_matcher.py

@@ -24,6 +24,7 @@ class ChoiceMatcher:
         self.scan_rules = get_rules(options.get('choice_rules'))
         self.show_progress = options.get('show_progress')
         self.cpu = options.get('cpu_core')
+        self.queue = options.get('queue')
         self.alternative_path = options.get('alternative_path')
         exts = options.get('choice_extensions')
         self.exts = [ext.lower() for ext in exts] if exts else []
@@ -31,14 +32,18 @@ class ChoiceMatcher:
 
     def scan(self, paths: list) -> dict:
         """Scan file(s) or directory per rule."""
-        if not (self.scan_rules and paths):
-            return
-        self.validate_rules()
-
         if self.show_progress:
             pbar = common.ProgressBar('Choice Match', len(self.scan_rules))
             self.scan_rules = pbar.progress_loop(self.scan_rules)
 
+        file_contents = self.read_file_contents(paths)
+        return self.regex_scan(file_contents)
+
+    def read_file_contents(self, paths: list) -> list:
+        """Load file(s) content."""
+        if not (self.scan_rules and paths):
+            return
+        self.validate_rules()
         choice_args = []
         for rule in self.scan_rules:
             scan_paths = paths
@@ -46,26 +51,35 @@ class ChoiceMatcher:
                 # Scan only alternative path
                 scan_paths = [Path(self.alternative_path)]
             choice_args.append((scan_paths, rule))
+        if not choice_args:
+            return []
 
-        # Use ThreadPoolExecutor for reading file contents and
-        # ProcessPoolExecutor for processing regex
-        with ThreadPoolExecutor() as io_executor, ProcessPoolExecutor(
-                max_workers=self.cpu) as cpu_executor:
+        # Use ThreadPoolExecutor for file reading
+        with ThreadPoolExecutor() as io_executor:
+            # Submit file reading tasks and wait for results
             futures = []
             for args_tuple in choice_args:
-                # Submit each read task and store the future along with the args
                 future = io_executor.submit(
                     self._read_file_contents, args_tuple)
-                futures.append((future, args_tuple))
-
-            results = []
-            for future, _ in futures:
-                file_contents = future.result()
-                # This will block until the file reading is done
-                # Process the file contents with ProcessPoolExecutor
-                process_future = cpu_executor.submit(
-                    self.choice_matcher, file_contents)
-                results.append(process_future.result())
+                futures.append(future)
+            return [future.result() for future in futures]
+
+    def regex_scan(self, file_contents) -> list:
+        """Process regex matches on the file contents."""
+        if self.queue:
+            # Use billiard's pool for regex (support queues)
+            from billiard import Pool
+            with Pool(processes=self.cpu) as pool:
+                # Run regex on file data
+                results = pool.map(
+                    self.choice_matcher,
+                    file_contents)
+        else:
+            # Use ProcessPoolExecutor for regex processing
+            with ProcessPoolExecutor(max_workers=self.cpu) as cpu_executor:
+                results = list(cpu_executor.map(
+                    self.choice_matcher,
+                    file_contents))
 
         self.add_finding(results)
         return self.findings

{libsast-3.1.0 → libsast-3.1.2}/libsast/core_matcher/pattern_matcher.py

@@ -25,39 +25,61 @@ class PatternMatcher:
         self.scan_rules = get_rules(options.get('match_rules'))
         self.show_progress = options.get('show_progress')
         self.cpu = options.get('cpu_core')
+        self.queue = options.get('queue')
         exts = options.get('match_extensions')
         self.exts = [ext.lower() for ext in exts] if exts else []
         self.findings = {}
 
     def scan(self, paths: list) -> dict:
         """Scan file(s) or directory."""
-        if not (self.scan_rules and paths):
-            return
-        self.validate_rules()
-
         if self.show_progress:
             pbar = common.ProgressBar('Pattern Match', len(paths))
             paths = pbar.progress_loop(paths)
 
+        file_contents = self.read_file_contents(paths)
+        return self.regex_scan(file_contents)
+
+    def read_file_contents(self, paths: list) -> list:
+        """Load file(s) content."""
+        if not (self.scan_rules and paths):
+            return
+        self.validate_rules()
+
         # Filter files by extension and size, prepare list for processing
         files_to_scan = {
             sfile for sfile in paths
             if is_file_valid(sfile, self.exts, 5)
         }
+        if not files_to_scan:
+            return []
 
-        # Use a ThreadPool for file reading, and ProcessPool for CPU-bound regex
-        with ThreadPoolExecutor() as io_executor, ProcessPoolExecutor(
-                max_workers=self.cpu) as cpu_executor:
+        # Use a ThreadPool for file reading
+        with ThreadPoolExecutor() as io_executor:
 
             # Read all files
             file_contents = list(io_executor.map(
                 self._read_file_content, files_to_scan))
-
-            # Run regex on file data
-            results = cpu_executor.map(
-                self.pattern_matcher,
-                file_contents,
-            )
+            return file_contents
+
+    def regex_scan(self, file_contents: list) -> dict:
+        """Scan file(s) content."""
+        if self.queue:
+            # Use billiard's pool for CPU-bound regex (support queues)
+            from billiard import Pool
+            with Pool(processes=self.cpu) as cpu_executor:
+                # Run regex on file data
+                results = cpu_executor.map(
+                    self.pattern_matcher,
+                    file_contents,
+                )
+        else:
+            # Use a ProcessPool for CPU-bound regex
+            with ProcessPoolExecutor(max_workers=self.cpu) as cpu_executor:
+                # Run regex on file data
+                results = cpu_executor.map(
+                    self.pattern_matcher,
+                    file_contents,
+                )
 
         # Compile findings
         self.add_finding(results)

{libsast-3.1.0 → libsast-3.1.2}/libsast/scanner.py

@@ -26,6 +26,7 @@ class Scanner:
             'ignore_paths': [],
             'show_progress': False,
             'cpu_core': 1,
+            'queue': False,
             # Overwrite with options from invocation
             **(options or {}),
         }

{libsast-3.1.0 → libsast-3.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "libsast"
-version = "3.1.0"
+version = "3.1.2"
 description = "A generic SAST library built on top of semgrep and regex"
 keywords = ["libsast", "SAST", "Python SAST", "SAST API", "Regex SAST", "Pattern Matcher"]
 authors = ["Ajin Abraham <ajin@opensecurity.in>"]
@@ -27,6 +27,7 @@ python = "^3.8"
 requests = "*"
 pyyaml = ">=6.0"
 semgrep = {version = "1.86.0", markers = "sys_platform != 'win32'"}
+billiard = "^4.2.1"
 
 [tool.poetry.group.dev.dependencies]
 bandit = "*"