isnad-scan 0.3.0__py3-none-any.whl

isnad_scan/patterns.py

@@ -0,0 +1,573 @@
+"""Pattern definitions for security scanning."""
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import List, Tuple, Pattern
+
+
+class Severity(Enum):
+    DANGER = "danger"
+    WARN = "warn"
+    INFO = "info"
+
+
+@dataclass
+class Finding:
+    severity: Severity
+    pattern_id: str
+    description: str
+    file: str
+    line: int
+    match: str
+    context: str = ""
+
+
+@dataclass
+class PatternDef:
+    pattern: Pattern
+    pattern_id: str
+    severity: Severity
+    description: str
+
+
+def _compile(patterns: List[Tuple[str, str, Severity, str]]) -> List[PatternDef]:
+    """Compile pattern definitions."""
+    return [
+        PatternDef(
+            pattern=re.compile(p, re.IGNORECASE | re.MULTILINE),
+            pattern_id=pid,
+            severity=sev,
+            description=desc,
+        )
+        for p, pid, sev, desc in patterns
+    ]
+
+
+PATTERNS = _compile([
+    # ============================================================
+    # DANGER: Immediate threats - high confidence malicious
+    # ============================================================
+    
+    # --- Hidden content (skill injection vectors) ---
+    
+    (r'<!--[\s\S]*?(https?://(?!example\.com|localhost|127\.0\.0\.1)[^\s<>]+)[\s\S]*?-->',
+     'hidden_url_html', Severity.DANGER,
+     'Hidden URL in HTML comment - classic skill injection vector'),
+    
+    (r'<!--[\s\S]*?(curl|wget|fetch|exec|eval|system)[\s\S]*?-->',
+     'hidden_command_html', Severity.DANGER,
+     'Hidden command in HTML comment - skill injection vector'),
+    
+    # Prompt injection patterns in markdown/docs
+    (r'<!--[\s\S]*?(ignore|bypass|override|disable|skip)[\s\S]*?(security|warning|scan|check|filter|restrict)[\s\S]*?-->',
+     'prompt_injection_html', Severity.DANGER,
+     'Potential prompt injection - attempts to override security'),
+    
+    (r'(IMPORTANT|SYSTEM|OVERRIDE|INSTRUCTION)[:\s]*(ignore|bypass|skip|disable)',
+     'prompt_injection_keyword', Severity.DANGER,
+     'Prompt injection keyword pattern - attempts to manipulate LLM'),
+    
+    # Prompt injection suppression - require hidden context (HTML comments, etc.)
+    (r'<!--[^>]*(do not|don\'t|never)\s+(report|flag|warn|alert|scan|check)[^>]*-->',
+     'prompt_injection_suppress', Severity.DANGER,
+     'Hidden instruction to suppress security reporting'),
+    
+    # --- Code execution ---
+    
+    # eval/exec - function calls
+    (r'\beval\s*\([^)]+\)',
+     'eval_usage', Severity.DANGER,
+     'eval() usage - arbitrary code execution risk'),
+    
+    (r'\bexec\s*\([^)]+\)',
+     'exec_usage', Severity.DANGER,
+     'exec() usage - arbitrary code execution risk'),
+    
+    (r'\bcompile\s*\([^)]+["\'][^"\']*["\']\s*,\s*["\']exec["\']\s*\)',
+     'compile_exec', Severity.DANGER,
+     'compile() with exec mode - code execution'),
+    
+    # Evasion-resistant: getattr-based eval/exec
+    (r'getattr\s*\([^)]*["\']+(eval|exec|compile|system)["\']',
+     'getattr_dangerous', Severity.DANGER,
+     'Dynamic access to dangerous function via getattr'),
+    
+    (r'__builtins__\s*[\.\[]',
+     'builtins_access', Severity.DANGER,
+     'Direct __builtins__ access - likely evasion attempt'),
+    
+    # String concatenation to build dangerous calls
+    (r'["\']ev["\']\s*\+\s*["\']al["\']\s*|["\']ex["\']\s*\+\s*["\']ec["\']',
+     'string_concat_evasion', Severity.DANGER,
+     'String concatenation building eval/exec - evasion attempt'),
+    
+    # Nested getattr (double evasion)
+    (r'getattr\s*\(\s*getattr\s*\(',
+     'nested_getattr', Severity.DANGER,
+     'Nested getattr - likely evasion attempt'),
+    
+    # Lambda with dangerous calls inside
+    (r'lambda[^:]*:\s*(eval|exec|compile|system|popen)\s*\(',
+     'lambda_dangerous', Severity.DANGER,
+     'Lambda containing dangerous function call'),
+    
+    # __dict__ access to builtins
+    (r'__dict__\s*\[\s*["\']+(eval|exec|system|compile)["\']',
+     'dict_dangerous_access', Severity.DANGER,
+     'Dictionary access to dangerous function'),
+    
+    # Unicode homoglyph evasion (common lookalikes for 'eval')
+    (r'[ℯеｅ][ｖѵν][ａаɑ][ｌⅼ]\s*\(|[ｅеℯ][ｘхχ][ｅеℯ][ｃсς]\s*\(',
+     'unicode_homoglyph', Severity.DANGER,
+     'Unicode homoglyph evasion - looks like eval/exec'),
+    
+    # --- Shell injection ---
+    
+    (r'subprocess\.(run|call|Popen)\s*\([^)]*shell\s*=\s*True',
+     'shell_injection', Severity.DANGER,
+     'Shell=True in subprocess - command injection risk'),
+    
+    (r'os\.system\s*\(',
+     'os_system', Severity.DANGER,
+     'os.system() - command injection risk'),
+    
+    (r'os\.popen\s*\(',
+     'os_popen', Severity.DANGER,
+     'os.popen() - command injection risk'),
+    
+    (r'commands\.(getoutput|getstatusoutput)\s*\(',
+     'commands_exec', Severity.DANGER,
+     'commands module execution - deprecated and dangerous'),
+    
+    # Node.js
+    (r'child_process\.(exec|execSync|spawn|spawnSync)\s*\(',
+     'node_exec', Severity.DANGER,
+     'child_process execution - command injection risk'),
+    
+    (r'new\s+Function\s*\(',
+     'js_function_constructor', Severity.DANGER,
+     'Function constructor - equivalent to eval()'),
+    
+    (r'setTimeout\s*\(\s*["\']',
+     'settimeout_string', Severity.DANGER,
+     'setTimeout with string argument - implicit eval'),
+    
+    (r'setInterval\s*\(\s*["\']',
+     'setinterval_string', Severity.DANGER,
+     'setInterval with string argument - implicit eval'),
+    
+    # Shell backtick command substitution (in actual shell scripts, not docs)
+    # Only flag in .sh files or when clearly executing
+    (r'^\s*[A-Z_]+=`[^`]*\$\([^)]+\)[^`]*`',
+     'shell_backtick_subst', Severity.DANGER,
+     'Shell variable assignment with nested command substitution'),
+    
+    # --- Data exfiltration (require variable interpolation, not placeholders) ---
+    
+    # Curl/wget with actual variable expansion (not documentation examples)
+    (r'(curl|wget)\s+[^|&;\n]*?(\$\{?[A-Z_]*?(API.?KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL|PRIVATE)[A-Z_]*\}?|\$\([^)]*?(api.?key|secret|token|password)\))',
+     'credential_exfil_curl', Severity.DANGER,
+     'Potential credential exfiltration via curl/wget with variable expansion'),
+    
+    # Python requests with variable (not string literal placeholder)
+    (r'requests\.(get|post|put)\s*\([^)]*?[\+\{].*?(api.?key|secret|token|password)',
+     'credential_exfil_requests', Severity.DANGER,
+     'Potential credential exfiltration via requests with dynamic credential'),
+    
+    # Any network call sending os.environ values
+    (r'(curl|wget|requests|fetch|http).*?os\.environ',
+     'credential_exfil_env', Severity.DANGER,
+     'Network call with environment variable access'),
+    
+    # DNS exfiltration
+    (r'socket\.gethostbyname\s*\([^)]*\+',
+     'dns_exfil_gethostbyname', Severity.DANGER,
+     'Dynamic DNS lookup - potential data exfiltration'),
+    
+    (r'dns\.resolver\.(resolve|query)\s*\([^)]*\+',
+     'dns_exfil_resolver', Severity.DANGER,
+     'Dynamic DNS resolution - potential data exfiltration'),
+    
+    # Tor hidden service in actual URL (not regex pattern or documentation)
+    (r'https?://[a-z0-9]+\.onion',
+     'tor_hidden_service', Severity.DANGER,
+     'Tor hidden service URL - suspicious'),
+    
+    # --- Obfuscation ---
+    
+    (r'base64\.(b64decode|decodebytes)\s*\(',
+     'base64_decode', Severity.DANGER,
+     'Base64 decoding - often used to hide malicious payloads'),
+    
+    (r'codecs\.(decode|encode)\s*\([^)]*rot',
+     'rot_encoding', Severity.DANGER,
+     'ROT encoding - obfuscation technique'),
+    
+    (r'\\x[0-9a-fA-F]{2}(\\x[0-9a-fA-F]{2}){10,}',
+     'hex_encoded_string', Severity.DANGER,
+     'Long hex-encoded string - potential obfuscation'),
+    
+    (r'\\u[0-9a-fA-F]{4}(\\u[0-9a-fA-F]{4}){10,}',
+     'unicode_encoded_string', Severity.DANGER,
+     'Long unicode-encoded string - potential obfuscation'),
+    
+    (r'bytes\.fromhex\s*\(',
+     'bytes_fromhex', Severity.DANGER,
+     'bytes.fromhex() - often used for obfuscation'),
+    
+    (r'chr\s*\(\s*\d+\s*\)\s*\+\s*chr\s*\(',
+     'chr_concatenation', Severity.DANGER,
+     'chr() concatenation - string obfuscation'),
+    
+    # --- Path traversal (only in actual code contexts) ---
+    
+    # Path traversal in function calls (open, read, etc.)
+    (r'(open|read|Path|os\.path|shutil)\s*\([^)]*\.\./\.\.',
+     'path_traversal_code', Severity.DANGER,
+     'Path traversal in code - directory escape attempt'),
+    
+    # Path traversal in string that's being used (assigned or passed)
+    (r'=\s*["\'][^"\']*\.\./\.\.[^"\']*["\']',
+     'path_traversal_assigned', Severity.DANGER,
+     'Path traversal string assignment'),
+    
+    # --- Raw network (unusual for skills) ---
+    
+    (r'socket\.socket\s*\(',
+     'raw_socket', Severity.DANGER,
+     'Raw socket creation - unusual for agent skills'),
+    
+    # --- Deserialization ---
+    
+    (r'pickle\.(load|loads)\s*\(',
+     'pickle_load', Severity.DANGER,
+     'Pickle deserialization - RCE if untrusted data'),
+    
+    (r'marshal\.(load|loads)\s*\(',
+     'marshal_load', Severity.DANGER,
+     'Marshal deserialization - code execution risk'),
+    
+    (r'yaml\.(load|unsafe_load)\s*\([^)]*(?!Loader)',
+     'yaml_unsafe_load', Severity.DANGER,
+     'Unsafe YAML load - potential code execution'),
+    
+    # Reverse shell patterns
+    (r'(bash|sh|nc|ncat|netcat)\s+.*?(-e|exec)\s+.*?(bash|sh|\/bin)',
+     'reverse_shell', Severity.DANGER,
+     'Potential reverse shell command'),
+    
+    (r'python[23]?\s+.*?-c\s+["\'].*?(socket|subprocess|pty)',
+     'python_reverse_shell', Severity.DANGER,
+     'Python one-liner with network/shell - possible reverse shell'),
+    
+    # Attempt to disable security features
+    (r'PYTHONDONTWRITEBYTECODE|PYTHONNOUSERSITE|--trusted-host|--disable-pip-version-check',
+     'security_bypass_env', Severity.WARN,
+     'Environment variable that may weaken security'),
+    
+    # Steganography libraries (hiding data in images)
+    (r'(steganography|stegano|stepic)\.',
+     'steganography', Severity.DANGER,
+     'Steganography library - may hide malicious data in images'),
+    
+    # ============================================================
+    # WARN: Suspicious but context-dependent
+    # ============================================================
+    
+    # --- Network access ---
+    
+    (r'requests\.(get|post|put|delete|patch)\s*\(',
+     'network_request', Severity.WARN,
+     'HTTP request - verify destination is expected'),
+    
+    (r'urllib\.request\.(urlopen|urlretrieve)',
+     'urllib_request', Severity.WARN,
+     'URL fetch - verify destination is expected'),
+    
+    (r'http\.client\.(HTTPConnection|HTTPSConnection)',
+     'http_client', Severity.WARN,
+     'HTTP client connection - verify destination'),
+    
+    (r'aiohttp\.ClientSession',
+     'aiohttp_session', Severity.WARN,
+     'Async HTTP client - verify destination'),
+    
+    (r'websockets?\.(connect|serve)',
+     'websocket', Severity.WARN,
+     'WebSocket connection - verify destination'),
+    
+    (r'fetch\s*\(\s*["\']https?://',
+     'fetch_request', Severity.WARN,
+     'Fetch request - verify destination is expected'),
+    
+    # --- File system ---
+    
+    (r'open\s*\([^)]*["\']w',
+     'file_write', Severity.WARN,
+     'File write operation - verify path is expected'),
+    
+    (r'\.write_text\s*\(|\.write_bytes\s*\(',
+     'pathlib_write', Severity.WARN,
+     'File write via pathlib - verify path is expected'),
+    
+    (r'shutil\.(copy|move|rmtree)',
+     'shutil_operations', Severity.WARN,
+     'File system operations - verify paths'),
+    
+    (r'os\.(remove|unlink|rmdir)\s*\(',
+     'file_deletion', Severity.WARN,
+     'File deletion - verify path is expected'),
+    
+    # --- Environment ---
+    
+    (r'os\.environ\s*[\[\.]|os\.getenv\s*\(',
+     'env_access', Severity.WARN,
+     'Environment variable access - verify which vars'),
+    
+    (r'process\.env\.',
+     'node_env_access', Severity.WARN,
+     'Node.js environment access - verify which vars'),
+    
+    (r'dotenv|load_dotenv',
+     'dotenv_usage', Severity.WARN,
+     'dotenv usage - loads environment from file'),
+    
+    # --- Dynamic behavior ---
+    
+    (r'__import__\s*\(',
+     'dynamic_import', Severity.WARN,
+     'Dynamic import - harder to audit statically'),
+    
+    (r'importlib\.import_module\s*\(',
+     'importlib_import', Severity.WARN,
+     'Dynamic import via importlib'),
+    
+    (r'globals\s*\(\s*\)\s*\[|locals\s*\(\s*\)\s*\[',
+     'globals_locals_access', Severity.WARN,
+     'Dynamic access via globals()/locals()'),
+    
+    (r'setattr\s*\([^)]*,\s*[^"\']+,',
+     'dynamic_setattr', Severity.WARN,
+     'Dynamic setattr with variable name'),
+    
+    # --- Crypto (not dangerous but notable) ---
+    
+    (r'Crypto\.|cryptography\.',
+     'crypto_library', Severity.WARN,
+     'Cryptographic operations - verify implementation'),
+    
+    # --- Path traversal (warn level - needs context) ---
+    
+    # Path traversal in string literals (might be documentation)
+    (r'["\'][^"\']*\.\./\.\.[^"\']*["\']',
+     'path_traversal_string', Severity.WARN,
+     'Path traversal in string - verify if documentation or actual code'),
+    
+    # ============================================================
+    # INFO: Notable but usually fine
+    # ============================================================
+    
+    (r'subprocess\.(run|call|Popen)\s*\(\s*\[',
+     'subprocess_list', Severity.INFO,
+     'Subprocess with list args (safer than shell=True)'),
+    
+    (r'open\s*\([^)]*["\']r["\']',
+     'file_read', Severity.INFO,
+     'File read operation'),
+    
+    (r'hashlib\.',
+     'hashlib_usage', Severity.INFO,
+     'Hashing operations'),
+    
+    (r'logging\.(debug|info|warning|error)',
+     'logging_usage', Severity.INFO,
+     'Logging statements'),
+])
+
+
+# Dependency vulnerability patterns (for requirements.txt, package.json)
+DANGEROUS_PACKAGES = {
+    # Known typosquats (real attacks from PyPI history)
+    'reqeusts', 'requets', 'request', 'requsts', 'reequests',  # requests
+    'crytpography', 'cryptograpy', 'crypotgraphy',  # cryptography
+    'djago', 'djnago', 'dajngo',  # django
+    'flaask', 'flassk',  # flask
+    'coloarama', 'colorma',  # colorama
+    'urllib', 'urllib2',  # stdlib impersonation
+    'setup-tools', 'setuptols',  # setuptools
+    
+    # Explicitly malicious package names
+    'keylogger', 'rat', 'trojan', 'backdoor', 'exploit', 'malware',
+    'credential-stealer', 'password-stealer', 'token-grabber',
+    
+    # Offensive security tools (may indicate malicious skill)
+    'pwntools', 'impacket', 'mitmproxy', 'scapy',
+}
+
+SUSPICIOUS_PACKAGE_PATTERNS = [
+    r'^git\+https?://(?!github\.com/(python|pypa|psf)/)',  # Non-standard git deps
+    r'^git\+ssh://',  # SSH git deps
+    r'@[a-f0-9]{40}$',  # Pinned to specific commit (could be malicious)
+]
+
+
+def is_documentation_context(line: str, filename: str) -> bool:
+    """Check if a line appears to be documentation rather than code."""
+    line_lower = line.lower().strip()
+    
+    # Markdown and documentation files - be more lenient
+    if filename.endswith(('.md', '.markdown', '.rst', '.txt')):
+        # All findings in markdown reference files are likely documentation
+        if '/references/' in filename or '/examples/' in filename or '/docs/' in filename:
+            return True
+        
+        # Bullet point describing a pattern
+        if line_lower.startswith(('-', '*', '•', '>')):
+            return True
+        # Inside a documentation list
+        if '`' in line and ('pattern' in line_lower or 'example' in line_lower or 'detect' in line_lower):
+            return True
+        # API documentation with placeholder values
+        if 'your_api_key' in line_lower or 'your_token' in line_lower or '<api' in line_lower:
+            return True
+        # Curl examples in documentation (common pattern)
+        if line_lower.startswith('curl ') and 'authorization' in line_lower:
+            return True
+        # Security documentation showing bad examples
+        if 'vulnerable' in line_lower or 'dangerous' in line_lower or 'insecure' in line_lower:
+            return True
+        if 'bad' in line_lower and ('example' in line_lower or 'practice' in line_lower):
+            return True
+        # Table rows (|...|)
+        if '|' in line and line.count('|') >= 2:
+            return True
+    
+    # YAML config files (semgrep rules, etc.) - rule definitions, not code
+    if filename.endswith(('.yaml', '.yml')):
+        if line_lower.startswith('- pattern'):
+            return True
+        if 'semgrep' in filename.lower() or '/rules/' in filename.lower():
+            return True
+    
+    # Comment explaining what to look for
+    if line_lower.startswith('#') and ('detect' in line_lower or 'pattern' in line_lower or 'check' in line_lower):
+        return True
+    
+    return False
+
+
+def is_in_code_block(content: str, position: int, filename: str) -> bool:
+    """Check if a position is inside a markdown code block (``` ... ```)."""
+    if not filename.endswith(('.md', '.markdown')):
+        return False
+    
+    # Count code block delimiters before this position
+    before = content[:position]
+    fence_count = before.count('```')
+    
+    # Odd count means we're inside a code block
+    return fence_count % 2 == 1
+
+
+def is_inside_string_literal(context: str, match: str) -> bool:
+    """Check if a match appears to be inside a string literal (dict key, etc.)."""
+    match_start = match[:15].replace('(', r'\(').replace(')', r'\)')
+    
+    # Look for patterns like "eval(": or 'marshal.load':
+    # Dict key pattern: "function_name": 
+    if re.search(rf'["\'][^"\']*?{re.escape(match_start[:10])}[^"\']*?["\']\s*:', context):
+        return True
+    
+    # String containing the function name (description, etc.)
+    if re.search(rf':\s*["\'][^"\']*?{re.escape(match_start[:10])}', context):
+        return True
+    
+    return False
+
+
+def scan_content(content: str, filename: str) -> List[Finding]:
+    """Scan content against all patterns."""
+    findings = []
+    lines = content.split('\n')
+    
+    for pattern_def in PATTERNS:
+        for match in pattern_def.pattern.finditer(content):
+            line_start = content.count('\n', 0, match.start()) + 1
+            
+            if line_start <= len(lines):
+                context = lines[line_start - 1].strip()
+            else:
+                context = ""
+            
+            # Check if this is documentation/string literal context - downgrade severity
+            severity = pattern_def.severity
+            is_docs = (
+                is_documentation_context(context, filename) or
+                is_in_code_block(content, match.start(), filename)
+            )
+            is_string = is_inside_string_literal(context, match.group(0))
+            
+            if (is_docs or is_string) and severity == Severity.DANGER:
+                severity = Severity.INFO  # Downgrade documentation/string mentions
+                    
+            note = ""
+            if is_docs:
+                note = " (in documentation)"
+            elif is_string:
+                note = " (in string literal)"
+                    
+            findings.append(Finding(
+                severity=severity,
+                pattern_id=pattern_def.pattern_id,
+                description=pattern_def.description + note,
+                file=filename,
+                line=line_start,
+                match=match.group(0)[:100],
+                context=context[:200],
+            ))
+    
+    return findings
+
+
+def scan_dependencies(content: str, filename: str) -> List[Finding]:
+    """Scan dependency files for suspicious packages."""
+    findings = []
+    lines = content.split('\n')
+    
+    for i, line in enumerate(lines, 1):
+        line_clean = line.strip().lower()
+        
+        # Skip comments and empty lines
+        if not line_clean or line_clean.startswith('#') or line_clean.startswith('//'):
+            continue
+        
+        # Extract package name
+        pkg_name = line_clean.split('==')[0].split('>=')[0].split('<=')[0].split('[')[0].strip()
+        
+        # Check against dangerous packages
+        if pkg_name in DANGEROUS_PACKAGES:
+            findings.append(Finding(
+                severity=Severity.DANGER,
+                pattern_id='dangerous_package',
+                description=f'Potentially dangerous or typosquatted package: {pkg_name}',
+                file=filename,
+                line=i,
+                match=line.strip()[:100],
+                context=line.strip(),
+            ))
+        
+        # Check suspicious patterns
+        for pattern in SUSPICIOUS_PACKAGE_PATTERNS:
+            if re.search(pattern, line_clean):
+                findings.append(Finding(
+                    severity=Severity.WARN,
+                    pattern_id='suspicious_dependency',
+                    description='Suspicious dependency source',
+                    file=filename,
+                    line=i,
+                    match=line.strip()[:100],
+                    context=line.strip(),
+                ))
+                break
+    
+    return findings