lambda-watcher 0.1.0__py3-none-any.whl

lambda_watcher/analysis/inventory.py

@@ -0,0 +1,118 @@
+"""Walk an extracted package and record every file."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from ..utils import count_lines, is_probably_text, language_for, matches_any, sha256_file, tree_hash
+
+
+@dataclass
+class FileEntry:
+    path: str          # posix relative path
+    size: int
+    sha256: str
+    mode: int
+    is_text: bool
+    is_vendor: bool
+    lang: str
+    lines: int
+
+    def as_dict(self) -> dict:
+        return {
+            "path": self.path,
+            "size": self.size,
+            "sha256": self.sha256,
+            "mode": self.mode,
+            "is_text": self.is_text,
+            "is_vendor": self.is_vendor,
+            "lang": self.lang,
+            "lines": self.lines,
+        }
+
+
+@dataclass
+class Inventory:
+    files: list[FileEntry] = field(default_factory=list)
+    tree_hash: str = ""
+    total_size: int = 0
+    code_size: int = 0
+    code_lines: int = 0
+
+    @property
+    def file_count(self) -> int:
+        return len(self.files)
+
+    @property
+    def code_files(self) -> list[FileEntry]:
+        """First-party files: everything that is not vendored."""
+        return [f for f in self.files if not f.is_vendor]
+
+    @property
+    def code_file_count(self) -> int:
+        return len(self.code_files)
+
+    def by_path(self) -> dict[str, FileEntry]:
+        return {f.path: f for f in self.files}
+
+    def language_breakdown(self) -> dict[str, int]:
+        counts: dict[str, int] = {}
+        for f in self.code_files:
+            counts[f.lang] = counts.get(f.lang, 0) + 1
+        return dict(sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])))
+
+
+def build_inventory(
+    root: Path,
+    vendor_globs: list[str],
+    max_scan_file_kb: int = 2048,
+) -> Inventory:
+    """Hash and classify every file under ``root``."""
+    inventory = Inventory()
+    hashes: list[tuple[str, str]] = []
+    max_scan_bytes = max_scan_file_kb * 1024
+
+    for path in sorted(root.rglob("*")):
+        if path.is_dir():
+            continue
+        if path.is_symlink() and not path.exists():
+            continue
+        try:
+            stat_result = path.stat()
+        except OSError:
+            continue
+
+        rel = path.relative_to(root).as_posix()
+        digest = sha256_file(path)
+        is_vendor = matches_any(rel, vendor_globs)
+        lang = language_for(rel)
+        size = stat_result.st_size
+
+        # Only sniff and count lines for files we might actually diff.
+        text = False
+        lines = 0
+        if size <= max_scan_bytes and lang != "binary":
+            text = is_probably_text(path)
+            if text:
+                lines = count_lines(path)
+
+        entry = FileEntry(
+            path=rel,
+            size=size,
+            sha256=digest,
+            mode=stat_result.st_mode & 0o777,
+            is_text=text,
+            is_vendor=is_vendor,
+            lang=lang,
+            lines=lines,
+        )
+        inventory.files.append(entry)
+        hashes.append((rel, digest))
+        inventory.total_size += size
+        if not is_vendor:
+            inventory.code_size += size
+            inventory.code_lines += lines
+
+    inventory.tree_hash = tree_hash(hashes)
+    return inventory

lambda_watcher/analysis/runtime.py

@@ -0,0 +1,117 @@
+"""Guess the Lambda runtime from the contents of the package."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import PurePosixPath
+
+from .inventory import Inventory
+
+
+@dataclass
+class RuntimeGuess:
+    runtime: str = "unknown"
+    confidence: str = "low"
+    evidence: list[str] = field(default_factory=list)
+    all_scores: dict[str, int] = field(default_factory=dict)
+
+    def as_dict(self) -> dict:
+        return {
+            "runtime": self.runtime,
+            "confidence": self.confidence,
+            "evidence": self.evidence,
+            "scores": self.all_scores,
+        }
+
+
+# Strong signals: a specific filename at any depth.
+_MARKER_FILES: dict[str, tuple[str, int]] = {
+    "lambda_function.py": ("python", 40),
+    "requirements.txt": ("python", 15),
+    "pyproject.toml": ("python", 10),
+    "package.json": ("nodejs", 20),
+    "package-lock.json": ("nodejs", 10),
+    "yarn.lock": ("nodejs", 8),
+    "index.js": ("nodejs", 25),
+    "index.mjs": ("nodejs", 30),
+    "app.js": ("nodejs", 12),
+    "pom.xml": ("java", 20),
+    "build.gradle": ("java", 15),
+    "go.mod": ("go", 30),
+    "bootstrap": ("provided", 25),
+    "gemfile": ("ruby", 20),
+    "gemfile.lock": ("ruby", 15),
+}
+
+_EXT_SCORES: dict[str, tuple[str, int]] = {
+    ".py": ("python", 3),
+    ".js": ("nodejs", 3),
+    ".mjs": ("nodejs", 3),
+    ".cjs": ("nodejs", 3),
+    ".ts": ("nodejs", 2),
+    ".java": ("java", 3),
+    ".class": ("java", 2),
+    ".jar": ("java", 6),
+    ".go": ("go", 3),
+    ".rb": ("ruby", 3),
+    ".cs": ("dotnet", 3),
+    ".dll": ("dotnet", 4),
+}
+
+
+def detect_runtime(inventory: Inventory) -> RuntimeGuess:
+    scores: dict[str, int] = {}
+    evidence: list[str] = []
+
+    def bump(lang: str, points: int, why: str) -> None:
+        scores[lang] = scores.get(lang, 0) + points
+        if why not in evidence:
+            evidence.append(why)
+
+    for entry in inventory.files:
+        name = PurePosixPath(entry.path).name.lower()
+        depth = entry.path.count("/")
+
+        marker = _MARKER_FILES.get(name)
+        if marker:
+            lang, points = marker
+            # A marker at the package root is far more meaningful than one
+            # buried inside a vendored dependency.
+            if entry.is_vendor:
+                points = max(1, points // 10)
+            elif depth > 1:
+                points = max(2, points // 3)
+            bump(lang, points, f"{entry.path}")
+
+        if entry.is_vendor:
+            continue
+        ext = PurePosixPath(entry.path).suffix.lower()
+        ext_hit = _EXT_SCORES.get(ext)
+        if ext_hit:
+            lang, points = ext_hit
+            bump(lang, points, f"*{ext}")
+
+    # A .NET deployment is recognisable by its runtime config file.
+    for entry in inventory.files:
+        if entry.path.endswith(".runtimeconfig.json") or entry.path.endswith(".deps.json"):
+            bump("dotnet", 25, entry.path)
+
+    if not scores:
+        return RuntimeGuess()
+
+    ranked = sorted(scores.items(), key=lambda kv: -kv[1])
+    top, top_score = ranked[0]
+    runner_up = ranked[1][1] if len(ranked) > 1 else 0
+    if top_score >= 30 and top_score >= runner_up * 2:
+        confidence = "high"
+    elif top_score >= 10:
+        confidence = "medium"
+    else:
+        confidence = "low"
+
+    return RuntimeGuess(
+        runtime=top,
+        confidence=confidence,
+        evidence=evidence[:12],
+        all_scores=dict(ranked),
+    )

lambda_watcher/analysis/secrets.py

@@ -0,0 +1,178 @@
+"""Flag credentials and risky calls that shouldn't be sitting in a zip.
+
+This is a lightweight tripwire, not a security scanner. It exists because a
+deployment package is exactly the place a hardcoded key survives unnoticed, and
+because "a new AWS key appeared between v7 and v8" is worth seeing in a diff.
+Matched values are stored redacted; the secret itself is never written to the
+index.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..utils import read_text
+from .inventory import Inventory
+
+
+@dataclass
+class Finding:
+    kind: str
+    severity: str  # high | medium | low
+    path: str
+    line: int
+    detail: str
+    is_vendor: bool = False
+
+    def as_dict(self) -> dict:
+        return {
+            "kind": self.kind,
+            "severity": self.severity,
+            "path": self.path,
+            "line": self.line,
+            "detail": self.detail,
+            "is_vendor": self.is_vendor,
+        }
+
+
+@dataclass(frozen=True)
+class Rule:
+    kind: str
+    severity: str
+    pattern: re.Pattern[str]
+    group: int = 0
+
+
+SECRET_RULES: list[Rule] = [
+    Rule("aws-access-key-id", "high", re.compile(r"\b((?:AKIA|ASIA|AIDA|AROA)[0-9A-Z]{16})\b"), 1),
+    Rule(
+        "aws-secret-access-key", "high",
+        re.compile(r"""(?i)aws_?secret_?access_?key\s*[=:]\s*['"]?([A-Za-z0-9/+=]{40})['"]?"""), 1,
+    ),
+    Rule(
+        "private-key", "high",
+        re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY-----"),
+    ),
+    Rule("github-token", "high", re.compile(r"\b((?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36})\b"), 1),
+    Rule("slack-token", "high", re.compile(r"\b(xox[baprs]-[A-Za-z0-9-]{10,})"), 1),
+    Rule("stripe-key", "high", re.compile(r"\b((?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,})\b"), 1),
+    Rule("google-api-key", "high", re.compile(r"\b(AIza[0-9A-Za-z_-]{35})\b"), 1),
+    Rule(
+        "jwt", "medium",
+        re.compile(r"\b(eyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,})"), 1,
+    ),
+    Rule(
+        "connection-string", "high",
+        re.compile(
+            r"\b((?:mongodb(?:\+srv)?|postgres(?:ql)?|mysql|redis|amqp)"
+            r"://[^\s'\"]+:[^\s'\"@]+@[^\s'\"]+)"
+        ), 1,
+    ),
+    Rule(
+        "hardcoded-credential", "medium",
+        re.compile(
+            r"""(?i)\b(?:password|passwd|pwd|secret|api_?key|apikey|auth_?token|access_?token|client_?secret)\b"""
+            r"""\s*[=:]\s*['"]([^'"\n]{8,})['"]"""
+        ), 1,
+    ),
+]
+
+RISK_RULES: list[Rule] = [
+    Rule("dynamic-exec", "medium", re.compile(r"\b(?:eval|exec)\s*\(")),
+    Rule("shell-injection-risk", "medium", re.compile(r"shell\s*=\s*True|child_process\.exec\s*\(")),
+    Rule("pickle-load", "medium", re.compile(r"\bpickle\.loads?\s*\(")),
+    Rule("tls-verification-off", "high", re.compile(r"verify\s*=\s*False|rejectUnauthorized\s*:\s*false")),
+    Rule("debug-flag", "low", re.compile(r"(?i)\bDEBUG\s*=\s*True\b")),
+]
+
+# Values that are obviously placeholders rather than live credentials.
+_PLACEHOLDER = re.compile(
+    r"(?i)^(?:x{3,}|\*{3,}|\.{3,}|<[^>]*>|\{\{.*\}\}|\$\{.*\}|%s|none|null|todo|"
+    r"change[_-]?me|your[_-].*|my[_-]?(?:secret|password|key).*|example.*|dummy.*|"
+    r"test[_-]?(?:key|secret|token|password)?|sample.*|placeholder.*|redacted.*|"
+    r"fake.*|insert.*|replace.*|password|secret|123456\d*|abc123)$"
+)
+
+_SCANNABLE = {
+    "python", "javascript", "typescript", "java", "ruby", "csharp", "go", "shell",
+    "json", "yaml", "toml", "ini", "dotenv", "text", "xml", "powershell",
+}
+
+
+def _shannon_entropy(value: str) -> float:
+    if not value:
+        return 0.0
+    counts: dict[str, int] = {}
+    for char in value:
+        counts[char] = counts.get(char, 0) + 1
+    length = len(value)
+    return -sum((c / length) * math.log2(c / length) for c in counts.values())
+
+
+def _redact(value: str) -> str:
+    value = value.strip()
+    if len(value) <= 8:
+        return "*" * len(value)
+    return f"{value[:4]}…{value[-2:]} ({len(value)} chars)"
+
+
+def _is_placeholder(value: str) -> bool:
+    stripped = value.strip()
+    if not stripped or _PLACEHOLDER.match(stripped):
+        return True
+    if stripped.startswith(("os.environ", "process.env", "${", "{{", "$(")):
+        return True
+    # Low-entropy strings are usually words, not keys.
+    return len(stripped) >= 16 and _shannon_entropy(stripped) < 2.5
+
+
+def scan(
+    root: Path,
+    inventory: Inventory,
+    include_vendor: bool = False,
+    check_secrets: bool = True,
+    max_files: int = 3000,
+) -> list[Finding]:
+    findings: list[Finding] = []
+    entries = inventory.files if include_vendor else inventory.code_files
+    rules = (SECRET_RULES if check_secrets else []) + RISK_RULES
+    scanned = 0
+
+    for entry in entries:
+        if scanned >= max_files:
+            break
+        if not entry.is_text or entry.lang not in _SCANNABLE:
+            continue
+        if entry.size > 2 * 1024 * 1024:
+            continue
+        text = read_text(root / entry.path, max_bytes=2 * 1024 * 1024)
+        if not text:
+            continue
+        scanned += 1
+
+        seen_in_file: set[tuple[str, str]] = set()
+        for line_no, line in enumerate(text.splitlines(), start=1):
+            if len(line) > 4000:  # minified bundles produce nothing but noise
+                continue
+            for rule in rules:
+                match = rule.pattern.search(line)
+                if not match:
+                    continue
+                value = match.group(rule.group) if rule.group else match.group(0)
+                if rule in SECRET_RULES and rule.group and _is_placeholder(value):
+                    continue
+                detail = _redact(value) if rule.group else value.strip()[:80]
+                key = (rule.kind, detail)
+                if key in seen_in_file:
+                    continue
+                seen_in_file.add(key)
+                findings.append(
+                    Finding(rule.kind, rule.severity, entry.path, line_no, detail, entry.is_vendor)
+                )
+
+    order = {"high": 0, "medium": 1, "low": 2}
+    findings.sort(key=lambda f: (order.get(f.severity, 3), f.path, f.line))
+    return findings

lambda_watcher/analysis/services.py

@@ -0,0 +1,76 @@
+"""Detect which AWS services the package talks to."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..utils import read_text
+from .inventory import Inventory
+
+_PATTERNS = [
+    # boto3.client("dynamodb") / boto3.resource('s3')
+    re.compile(r"""boto3\.(?:client|resource)\s*\(\s*['"]([a-z0-9-]+)['"]"""),
+    re.compile(r"""session\.(?:client|resource)\s*\(\s*['"]([a-z0-9-]+)['"]"""),
+    # @aws-sdk/client-dynamodb
+    re.compile(r"""['"]@aws-sdk/client-([a-z0-9-]+)['"]"""),
+    # require('aws-sdk').S3 / new AWS.DynamoDB(
+    re.compile(r"""new\s+AWS\.([A-Za-z0-9]+)\s*\("""),
+    # software.amazon.awssdk.services.s3
+    re.compile(r"""software\.amazon\.awssdk\.services\.([a-z0-9]+)"""),
+    re.compile(r"""com\.amazonaws\.services\.([a-z0-9]+)"""),
+]
+
+_SCANNABLE = {"python", "javascript", "typescript", "java", "ruby", "csharp", "go"}
+
+# Normalise the different spellings onto one service id.
+_ALIASES = {
+    "dynamodbdocument": "dynamodb",
+    "dynamodbstreams": "dynamodb-streams",
+    "secretsmanager": "secretsmanager",
+    "ssm": "ssm",
+    "sfn": "stepfunctions",
+    "states": "stepfunctions",
+    "cloudwatchlogs": "logs",
+}
+
+
+@dataclass
+class ServiceRef:
+    service: str
+    path: str
+    line: int
+
+    def as_dict(self) -> dict:
+        return {"service": self.service, "path": self.path, "line": self.line}
+
+
+def detect_services(
+    root: Path, inventory: Inventory, max_files: int = 2000
+) -> list[ServiceRef]:
+    refs: list[ServiceRef] = []
+    seen: set[tuple[str, str]] = set()
+    scanned = 0
+
+    for entry in inventory.code_files:
+        if scanned >= max_files:
+            break
+        if not entry.is_text or entry.lang not in _SCANNABLE:
+            continue
+        text = read_text(root / entry.path, max_bytes=1024 * 1024)
+        if not text:
+            continue
+        scanned += 1
+        for line_no, line in enumerate(text.splitlines(), start=1):
+            for pattern in _PATTERNS:
+                for match in pattern.finditer(line):
+                    service = match.group(1).lower()
+                    service = _ALIASES.get(service, service)
+                    key = (service, entry.path)
+                    if key in seen:
+                        continue
+                    seen.add(key)
+                    refs.append(ServiceRef(service, entry.path, line_no))
+    refs.sort(key=lambda r: (r.service, r.path))
+    return refs