libsast 2.0.2__tar.gz → 3.0.1__tar.gz

{libsast-2.0.2 → libsast-3.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: libsast
-Version: 2.0.2
+Version: 3.0.1
 Summary: A generic SAST library built on top of semgrep and regex
 Home-page: https://github.com/ajinabraham/libsast
 License: LGPL-3.0-or-later
@@ -16,10 +16,12 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Security
 Classifier: Topic :: Software Development :: Quality Assurance
 Requires-Dist: pyyaml (>=6.0)
-Requires-Dist: requests (>=2.27.1)
+Requires-Dist: requests
 Requires-Dist: semgrep (==0.117.0) ; sys_platform != "win32"
 Project-URL: Bug Tracker, https://github.com/ajinabraham/libsast/issues
 Project-URL: Documentation, https://github.com/ajinabraham/libsast/blob/master/README.md
@@ -59,13 +61,13 @@ usage: libsast [-h] [-o OUTPUT] [-p PATTERN_FILE] [-s SGREP_PATTERN_FILE]
                [--ignore-filenames IGNORE_FILENAMES [IGNORE_FILENAMES ...]]
                [--ignore-extensions IGNORE_EXTENSIONS [IGNORE_EXTENSIONS ...]]
                [--ignore-paths IGNORE_PATHS [IGNORE_PATHS ...]]
-               [--show-progress] [-v]
-               [path [path ...]]
+               [--show-progress] [--cpu-core CPU_CORE] [-v]
+               [path ...]
 
 positional arguments:
   path                  Path can be file(s) or directories
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   -o OUTPUT, --output OUTPUT
                         Output filename to save JSON report.
@@ -74,7 +76,8 @@ optional arguments:
   -s SGREP_PATTERN_FILE, --sgrep-pattern-file SGREP_PATTERN_FILE
                         sgrep rules directory
   --sgrep-file-extensions SGREP_FILE_EXTENSIONS [SGREP_FILE_EXTENSIONS ...]
-                        File extensions that should be scanned with sgrep
+                        File extensions that should be scanned with semantic
+                        grep
   --file-extensions FILE_EXTENSIONS [FILE_EXTENSIONS ...]
                         File extensions that should be scanned with pattern
                         matcher
@@ -85,6 +88,7 @@ optional arguments:
   --ignore-paths IGNORE_PATHS [IGNORE_PATHS ...]
                         Path(s) to ignore
   --show-progress       Show scan progress
+  --cpu-core CPU_CORE   No of CPU cores to use. Use all cores by default
   -v, --version         Show libsast version
 ```
 

{libsast-2.0.2 → libsast-3.0.1}/README.md

@@ -31,13 +31,13 @@ usage: libsast [-h] [-o OUTPUT] [-p PATTERN_FILE] [-s SGREP_PATTERN_FILE]
                [--ignore-filenames IGNORE_FILENAMES [IGNORE_FILENAMES ...]]
                [--ignore-extensions IGNORE_EXTENSIONS [IGNORE_EXTENSIONS ...]]
                [--ignore-paths IGNORE_PATHS [IGNORE_PATHS ...]]
-               [--show-progress] [-v]
-               [path [path ...]]
+               [--show-progress] [--cpu-core CPU_CORE] [-v]
+               [path ...]
 
 positional arguments:
   path                  Path can be file(s) or directories
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   -o OUTPUT, --output OUTPUT
                         Output filename to save JSON report.
@@ -46,7 +46,8 @@ optional arguments:
   -s SGREP_PATTERN_FILE, --sgrep-pattern-file SGREP_PATTERN_FILE
                         sgrep rules directory
   --sgrep-file-extensions SGREP_FILE_EXTENSIONS [SGREP_FILE_EXTENSIONS ...]
-                        File extensions that should be scanned with sgrep
+                        File extensions that should be scanned with semantic
+                        grep
   --file-extensions FILE_EXTENSIONS [FILE_EXTENSIONS ...]
                         File extensions that should be scanned with pattern
                         matcher
@@ -57,6 +58,7 @@ optional arguments:
   --ignore-paths IGNORE_PATHS [IGNORE_PATHS ...]
                         Path(s) to ignore
   --show-progress       Show scan progress
+  --cpu-core CPU_CORE   No of CPU cores to use. Use all cores by default
   -v, --version         Show libsast version
 ```
 

{libsast-2.0.2 → libsast-3.0.1}/libsast/__init__.py

@@ -1,14 +1,18 @@
 #!/usr/bin/env python
 # -*- coding: utf_8 -*-
+from datetime import datetime
+
 from .core_matcher.pattern_matcher import PatternMatcher
 from .core_matcher.choice_matcher import ChoiceMatcher
 from .core_sgrep.semantic_sgrep import SemanticGrep
 from .scanner import Scanner
 
+
+year = str(datetime.now().year)
 __title__ = 'libsast'
 __authors__ = 'Ajin Abraham'
-__copyright__ = 'Copyright 2020 Ajin Abraham, OpenSecurity'
-__version__ = '2.0.2'
+__copyright__ = f'Copyright {year} Ajin Abraham, opensecurity.in'
+__version__ = '3.0.1'
 __version_info__ = tuple(int(i) for i in __version__.split('.'))
 __all__ = [
     'Scanner',

{libsast-2.0.2 → libsast-3.0.1}/libsast/__main__.py

@@ -51,7 +51,7 @@ def main():
     parser.add_argument('--sgrep-file-extensions',
                         nargs='+',
                         help=('File extensions that should be scanned'
-                              ' with sgrep'),
+                              ' with semantic grep'),
                         required=False)
     parser.add_argument('--file-extensions',
                         nargs='+',
@@ -74,6 +74,10 @@ def main():
                         help='Show scan progress',
                         required=False,
                         action='store_true')
+    parser.add_argument('--cpu-core',
+                        help='No of CPU cores to use. Use all cores by default',
+                        type=int,
+                        required=False)
     parser.add_argument('-v', '--version',
                         help='Show libsast version',
                         required=False,
@@ -89,6 +93,7 @@ def main():
             'ignore_extensions': args.ignore_extensions,
             'ignore_paths': args.ignore_paths,
             'show_progress': args.show_progress,
+            'cpu_core': args.cpu_core,
         }
         result = Scanner(options, args.path).scan()
         output(args.output, result)

{libsast-2.0.2 → libsast-3.0.1}/libsast/common.py

@@ -26,7 +26,7 @@ class ProgressBar:
         self.output.write(f'- {self.prefix} {prog} {index}\r')
         self.output.flush()
 
-    def progrees_loop(self, iterator):
+    def progress_loop(self, iterator):
         """Show progress for loop."""
         self.progress_print(0)
         for index, item in enumerate(iterator):
@@ -74,15 +74,18 @@ def read_yaml(file_obj, text=False):
 
 def get_worker_count():
     """Get worker count for pool."""
+    libsast_workers = os.getenv('LIBSAST_WORKERS')
+    if libsast_workers:
+        try:
+            return int(libsast_workers)
+        except ValueError:
+            return 1
     try:
         worker_count = os.cpu_count()
-        if not worker_count:
-            worker_count = 1
-        if worker_count != 1 and sys.platform == 'win32':
-            # Work around https://bugs.python.org/issue26903
-            worker_count = min(worker_count, 61)
-        if os.getenv('LIBSAST_WORKERS'):
-            worker_count = int(os.getenv('LIBSAST_WORKERS'))
     except Exception:
-        worker_count = 16
+        worker_count = 1
+
+    # Adjust worker count for Windows
+    if sys.platform == 'win32':
+        worker_count = min(worker_count, 61)
     return worker_count

libsast-3.0.1/libsast/core_matcher/choice_matcher.py

@@ -0,0 +1,174 @@
+# -*- coding: utf_8 -*-
+"""Choice Matcher."""
+import re
+from pathlib import Path
+from concurrent.futures import (
+    ProcessPoolExecutor,
+    ThreadPoolExecutor,
+)
+from functools import lru_cache
+
+from libsast.core_matcher.helpers import (
+    get_rules,
+    is_file_valid,
+    strip_comments,
+)
+from libsast import (
+    common,
+    exceptions,
+)
+
+
+class ChoiceMatcher:
+    def __init__(self, options: dict) -> None:
+        self.scan_rules = get_rules(options.get('choice_rules'))
+        self.show_progress = options.get('show_progress')
+        self.cpu = options.get('cpu_core')
+        self.alternative_path = options.get('alternative_path')
+        exts = options.get('choice_extensions')
+        self.exts = [ext.lower() for ext in exts] if exts else []
+        self.findings = {}
+
+    def scan(self, paths: list) -> dict:
+        """Scan file(s) or directory per rule."""
+        if not (self.scan_rules and paths):
+            return
+        self.validate_rules()
+
+        if self.show_progress:
+            pbar = common.ProgressBar('Choice Match', len(self.scan_rules))
+            self.scan_rules = pbar.progress_loop(self.scan_rules)
+
+        choice_args = []
+        for rule in self.scan_rules:
+            scan_paths = paths
+            if rule['type'] != 'code' and self.alternative_path:
+                # Scan only alternative path
+                scan_paths = [Path(self.alternative_path)]
+            choice_args.append((scan_paths, rule))
+
+        # Use ThreadPoolExecutor for reading file contents and
+        # ProcessPoolExecutor for processing regex
+        with ThreadPoolExecutor() as io_executor, ProcessPoolExecutor(
+                max_workers=self.cpu) as cpu_executor:
+            futures = []
+            for args_tuple in choice_args:
+                # Submit each read task and store the future along with the args
+                future = io_executor.submit(
+                    self._read_file_contents, args_tuple)
+                futures.append((future, args_tuple))
+
+            results = []
+            for future, _ in futures:
+                file_contents = future.result()
+                # This will block until the file reading is done
+                # Process the file contents with ProcessPoolExecutor
+                process_future = cpu_executor.submit(
+                    self.choice_matcher, file_contents)
+                results.append(process_future.result())
+
+        self.add_finding(results)
+        return self.findings
+
+    def validate_rules(self):
+        """Validate Rules before scanning."""
+        for rule in self.scan_rules:
+            if not isinstance(rule, dict):
+                raise exceptions.InvalidRuleFormatError(
+                    'Choice Matcher Rule format is invalid.')
+            required_keys = [
+                'id',
+                'type',
+                'choice_type',
+                'selection',
+                'choice']
+            for key in required_keys:
+                if not rule.get(key):
+                    raise exceptions.PatternKeyMissingError(
+                        f'The rule is missing the key "{key}"')
+
+    def _read_file_contents(self, args_tuple):
+        """Read file contents for the given paths and rule."""
+        scan_paths, rule = args_tuple
+        results = []
+        for sfile in scan_paths:
+            if not is_file_valid(sfile, self.exts, 5):
+                continue
+            try:
+                data = self._format_content(
+                    sfile.read_text('utf-8', 'ignore'),
+                    sfile.suffix.lower())
+                results.append((data, rule))
+            except Exception as e:
+                raise exceptions.RuleProcessingError(
+                    'Error reading file: {}'.format(sfile)) from e
+        return results
+
+    def find_choices(self, data, rule):
+        """Find Choices."""
+        all_matches = set()
+        for idx, choice in enumerate(rule['choice']):
+            typ = rule['choice_type']
+            if typ == 'and':
+                # Return on first and choice
+                if all(re.compile(and_regx).search(data) for and_regx in choice[0]):
+                    return (all_matches, [idx])
+            elif re.compile(choice[0]).search(data):
+                if typ == 'or':
+                    # Return on first or choice
+                    return (all_matches, [idx])
+                elif typ == 'all':
+                    # Extract all choice(s) from all files
+                    all_matches.add(choice[1])
+        return (all_matches, None)  # None means no matches found.
+
+    def choice_matcher(self, file_contents):
+        """Process regex matches on the file contents."""
+        results = []
+        for data, rule in file_contents:
+            match = self.find_choices(data, rule)
+            results.append({
+                'rule': rule,
+                'matches': match[1] if isinstance(match, tuple) else set(),
+                'all_matches': match[0] if isinstance(match, tuple) else match,
+            })
+        return results
+
+    @staticmethod
+    @lru_cache(maxsize=128)
+    def _format_content(data, file_suffix):
+        return strip_comments(data, file_suffix)
+
+    def add_finding(self, results):
+        """Add Choice Findings and generate metadata."""
+        for res_list in results:
+            if not res_list:
+                continue
+            for match_dict in res_list:
+                rule = match_dict['rule']
+                all_matches = match_dict['all_matches']
+                matches = match_dict['matches']
+
+                # Determine selection string
+                if all_matches:
+                    selection = rule['selection'].format(list(all_matches))
+                elif matches:
+                    select = rule['choice'][min(matches)][1]
+                    selection = rule['selection'].format(select)
+                else:
+                    selection = rule['selection'].format(rule.get('else', ''))
+
+                # Create metadata dictionary
+                meta_dict = {
+                    'choice': selection,
+                    'description': rule['message'],
+                    **{key: rule[key] for key in rule if key not in {
+                        'choice',
+                        'message',
+                        'id',
+                        'type',
+                        'choice_type',
+                        'selection',
+                        'else'}}}
+
+                self.findings[rule['id']] = meta_dict

{libsast-2.0.2 → libsast-3.0.1}/libsast/core_matcher/helpers.py

@@ -85,7 +85,7 @@ def comment_replacer(matches, data):
     return data
 
 
-def strip_comments(data):
+def strip_comments1(data):
     """Remove Comments.
 
     Replace multiline comments first and
@@ -128,3 +128,36 @@ def get_match_lines(content, pos):
         if filepos >= pos[1] and skip:
             # Multiline march, find end line
             return (start_line, idx)
+
+
+def is_file_valid(sfile, allowed_extensions=None, max_size_mb=5):
+    """Check if the file is valid based on its extension and size.
+
+    Args:
+        sfile (Path): The file to check.
+        allowed_extensions (set): A set of allowed file extensions.
+        max_size_mb (int): The maximum file size in MB.
+
+    Returns:
+        bool: True if the file is valid, False otherwise.
+    """
+    # Get the file extension in lowercase
+    ext = sfile.suffix.lower()
+
+    # Check if the file extension is allowed
+    if allowed_extensions and ext not in allowed_extensions:
+        return False
+
+    # Check if the file size exceeds the maximum limit (in bytes)
+    max_size_bytes = max_size_mb * 1024 * 1024  # Convert MB to bytes
+    if sfile.stat().st_size > max_size_bytes:
+        return False
+
+    return True
+
+
+def strip_comments(data, ext):
+    """Format content by stripping comments based on file type."""
+    if ext in ('.html', '.xml'):
+        return strip_comments2(data)
+    return strip_comments1(data)

libsast-3.0.1/libsast/core_matcher/matchers.py

@@ -0,0 +1,104 @@
+# -*- coding: utf_8 -*-
+"""Optimized Algorithms for Pattern Matching with Caching."""
+import re
+from abc import ABC, abstractmethod
+from functools import lru_cache
+
+from libsast.core_matcher.helpers import get_match_lines
+
+
+def get_pos(match):
+    """Adjust zero index in match span."""
+    start, end = match.span()
+    return (1 if start == 0 else start, end)
+
+
+# Cache compiled regex patterns
+@lru_cache(maxsize=128)
+def get_compiled_pattern(pattern):
+    """Compile and cache regex patterns."""
+    return re.compile(pattern)
+
+
+class MatchCommand:
+    def __init__(self):
+        self.patterns = {}
+
+    def _find_match(self, pattern_name, content, rule):
+        pattern_class = self.patterns.get(pattern_name) or globals()[pattern_name]()
+        self.patterns.setdefault(pattern_name, pattern_class)
+        return pattern_class._perform_search(content, rule)
+
+
+class MatchStrategy(ABC):
+    @abstractmethod
+    def _perform_search(self, content, rule):
+        """Search for instances of a pattern match in content."""
+
+
+class Regex(MatchStrategy):
+    def _perform_search(self, content, rule):
+        pattern = get_compiled_pattern(rule['pattern'])
+        return self._find_matches(content, pattern)
+
+    @staticmethod
+    def _find_matches(content, pattern):
+        """Helper to find all matches in content and extract details."""
+        matches = set()
+        for match in pattern.finditer(content):
+            if match.group():
+                match_pos = get_pos(match)
+                match_lines = get_match_lines(content, match_pos)
+                matches.add((match.group(), match_pos, match_lines))
+        return matches
+
+
+class RegexAnd(MatchStrategy):
+    def _perform_search(self, content, rule):
+        patterns = rule['pattern'] if isinstance(
+            rule['pattern'], list) else [rule['pattern']]
+        all_matches = set()
+        for regex in patterns:
+            pattern = get_compiled_pattern(regex)
+            matches = Regex._find_matches(content, pattern)
+            if not matches:
+                return False
+            all_matches.update(matches)
+        return all_matches
+
+
+class RegexOr(MatchStrategy):
+    def _perform_search(self, content, rule):
+        patterns = rule['pattern'] if isinstance(
+            rule['pattern'], list) else [rule['pattern']]
+        matches = set()
+        for regex in patterns:
+            pattern = get_compiled_pattern(regex)
+            matches.update(Regex._find_matches(content, pattern))
+        return matches
+
+
+class RegexAndNot(MatchStrategy):
+    def _perform_search(self, content, rule):
+        present_pattern, not_pattern = rule['pattern']
+        not_found = get_compiled_pattern(not_pattern).search(content) is None
+        if not not_found:
+            return False
+        present_matches = Regex._find_matches(
+            content, get_compiled_pattern(present_pattern))
+        return present_matches if present_matches else False
+
+
+class RegexAndOr(MatchStrategy):
+    def _perform_search(self, content, rule):
+        and_pattern, or_patterns = rule['pattern']
+        and_matches = Regex._find_matches(content, get_compiled_pattern(and_pattern))
+        or_matches = set()
+        for regex in or_patterns:
+            or_pattern = get_compiled_pattern(regex)
+            or_matches = Regex._find_matches(content, or_pattern)
+            if or_matches:
+                break
+        if and_matches and or_matches:
+            return and_matches.union(or_matches)
+        return False

libsast-3.0.1/libsast/core_matcher/pattern_matcher.py

@@ -0,0 +1,157 @@
+# -*- coding: utf_8 -*-
+"""Pattern Matcher."""
+from operator import itemgetter
+from concurrent.futures import (
+    ProcessPoolExecutor,
+    ThreadPoolExecutor,
+)
+from functools import lru_cache
+
+from libsast.core_matcher.helpers import (
+    get_rules,
+    is_file_valid,
+    strip_comments,
+)
+from libsast.core_matcher import matchers
+from libsast import (
+    common,
+    exceptions,
+)
+
+
+class PatternMatcher:
+    def __init__(self, options: dict) -> None:
+        self.matcher = matchers.MatchCommand()
+        self.scan_rules = get_rules(options.get('match_rules'))
+        self.show_progress = options.get('show_progress')
+        self.cpu = options.get('cpu_core')
+        exts = options.get('match_extensions')
+        self.exts = [ext.lower() for ext in exts] if exts else []
+        self.findings = {}
+
+    def scan(self, paths: list) -> dict:
+        """Scan file(s) or directory."""
+        if not (self.scan_rules and paths):
+            return
+        self.validate_rules()
+
+        if self.show_progress:
+            pbar = common.ProgressBar('Pattern Match', len(paths))
+            paths = pbar.progress_loop(paths)
+
+        # Filter files by extension and size, prepare list for processing
+        files_to_scan = {
+            sfile for sfile in paths
+            if is_file_valid(sfile, self.exts, 5)
+        }
+
+        # Use a ThreadPool for file reading, and ProcessPool for CPU-bound regex
+        with ThreadPoolExecutor() as io_executor, ProcessPoolExecutor(
+                max_workers=self.cpu) as cpu_executor:
+
+            # Read all files
+            file_contents = list(io_executor.map(
+                self._read_file_content, files_to_scan))
+
+            # Run regex on file data
+            results = cpu_executor.map(
+                self.pattern_matcher,
+                file_contents,
+            )
+
+        # Compile findings
+        self.add_finding(results)
+        return self.findings
+
+    def validate_rules(self):
+        """Validate Rules before scanning."""
+        available_matchers = {m for m in dir(matchers) if m.startswith('R')}
+        required_keys = ['type', 'pattern']
+
+        for rule in self.scan_rules:
+            if not isinstance(rule, dict):
+                raise exceptions.InvalidRuleFormatError(
+                    'Pattern Matcher Rule format is invalid.')
+
+            # Check for missing required keys
+            missing_keys = [key for key in required_keys if key not in rule]
+            if missing_keys:
+                mkeys = ', '.join(missing_keys)
+                raise exceptions.PatternKeyMissingError(
+                    f'The rule is missing the keys: {mkeys}')
+
+            pattern_name = rule['type']
+            if pattern_name not in available_matchers:
+                supported = ', '.join(available_matchers)
+                raise exceptions.MatcherNotFoundError(
+                    f'Matcher {pattern_name} is not supported.'
+                    f' Available matchers are {supported}.')
+
+    @staticmethod
+    def _read_file_content(file_path):
+        """Read file content with encoding handling."""
+        try:
+            return file_path, file_path.read_text('utf-8', 'ignore')
+        except Exception as e:
+            print(f'Error reading {file_path}: {e}')
+            return file_path, ''
+
+    def pattern_matcher(self, file_data):
+        """Static Analysis Pattern Matcher."""
+        file_path, data = file_data
+        results = []
+        try:
+            fmt_data = self._format_content(data, file_path.suffix.lower())
+            for rule in self.scan_rules:
+                case = rule.get('input_case')
+                if case == 'lower':
+                    fmt_data = fmt_data.lower()
+                elif case == 'upper':
+                    fmt_data = fmt_data.upper()
+                matches = self.matcher._find_match(rule['type'], fmt_data, rule)
+                if matches:
+                    results.append({
+                        'file': file_path.as_posix(),
+                        'rule': rule,
+                        'matches': matches,
+                    })
+        except Exception as e:
+            msg = f'Error processing rule for {file_path}: {e}'
+            raise exceptions.RuleProcessingError(msg)
+        return results
+
+    @staticmethod
+    @lru_cache(maxsize=128)
+    def _format_content(data, file_suffix):
+        return strip_comments(data, file_suffix)
+
+    def add_finding(self, results):
+        """Add Code Analysis Findings."""
+        for res_list in results:
+            if not res_list:
+                continue
+            for match_dict in res_list:
+                rule = match_dict['rule']
+                rule_id = rule['id']
+                files = self.findings.setdefault(
+                    rule_id, {'files': [], 'metadata': {}})['files']
+
+                for match in match_dict['matches']:
+                    file_details = {
+                        'file_path': match_dict['file'],
+                        'match_string': match[0],
+                        'match_position': match[1],
+                        'match_lines': match[2],
+                    }
+                    files.append(file_details)
+
+                if not self.findings[rule_id]['metadata']:
+                    meta = rule.get('metadata', {})
+                    # message & severity are mandatory
+                    meta['description'] = rule['message']
+                    meta['severity'] = rule['severity']
+                    self.findings[rule_id]['metadata'] = meta
+
+                # Sort files by specified criteria
+                self.findings[rule_id]['files'].sort(
+                    key=itemgetter('file_path', 'match_string', 'match_lines'))