lambda-watcher 0.1.0__py3-none-any.whl

lambda_watcher/__init__.py

@@ -0,0 +1,4 @@
+"""Watch a downloads folder for AWS Lambda deployment packages and version them."""
+
+__version__ = "0.1.0"
+__all__ = ["__version__"]

lambda_watcher/__main__.py

@@ -0,0 +1,4 @@
+from lambda_watcher.cli import main
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

lambda_watcher/analysis/__init__.py

@@ -0,0 +1,115 @@
+"""Analysis pipeline: turn an extracted package into a structured manifest."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from ..config import AnalysisConfig
+from .deps import Dependency, detect_dependencies
+from .envvars import EnvVarRef, detect_env_vars
+from .handler import HandlerCandidate, detect_handlers
+from .inventory import FileEntry, Inventory, build_inventory
+from .runtime import RuntimeGuess, detect_runtime
+from .secrets import Finding, scan
+from .services import ServiceRef, detect_services
+
+MANIFEST_SCHEMA = 1
+
+__all__ = [
+    "Analysis",
+    "Dependency",
+    "EnvVarRef",
+    "FileEntry",
+    "Finding",
+    "HandlerCandidate",
+    "Inventory",
+    "RuntimeGuess",
+    "ServiceRef",
+    "analyse",
+    "MANIFEST_SCHEMA",
+]
+
+
+@dataclass
+class Analysis:
+    """Everything we learned about one extracted deployment package."""
+
+    inventory: Inventory
+    runtime: RuntimeGuess
+    handlers: list[HandlerCandidate] = field(default_factory=list)
+    dependencies: list[Dependency] = field(default_factory=list)
+    env_vars: list[EnvVarRef] = field(default_factory=list)
+    services: list[ServiceRef] = field(default_factory=list)
+    findings: list[Finding] = field(default_factory=list)
+
+    @property
+    def primary_handler(self) -> str | None:
+        return self.handlers[0].handler if self.handlers else None
+
+    @property
+    def vendor_file_count(self) -> int:
+        return sum(1 for f in self.inventory.files if f.is_vendor)
+
+    @property
+    def vendor_size(self) -> int:
+        return sum(f.size for f in self.inventory.files if f.is_vendor)
+
+    def unique_env_vars(self, include_reserved: bool = False) -> list[str]:
+        names = {
+            ref.name for ref in self.env_vars if include_reserved or not ref.is_reserved
+        }
+        return sorted(names)
+
+    def unique_services(self) -> list[str]:
+        return sorted({ref.service for ref in self.services})
+
+    def totals(self) -> dict[str, int]:
+        return {
+            "file_count": self.inventory.file_count,
+            "total_size": self.inventory.total_size,
+            "code_file_count": self.inventory.code_file_count,
+            "code_size": self.inventory.code_size,
+            "code_lines": self.inventory.code_lines,
+            "vendor_file_count": self.vendor_file_count,
+            "vendor_size": self.vendor_size,
+        }
+
+    def to_manifest(self, extra: dict[str, Any] | None = None) -> dict[str, Any]:
+        manifest: dict[str, Any] = {
+            "schema": MANIFEST_SCHEMA,
+            "tree_hash": self.inventory.tree_hash,
+            "runtime": self.runtime.as_dict(),
+            "handlers": [h.as_dict() for h in self.handlers],
+            "totals": self.totals(),
+            "languages": self.inventory.language_breakdown(),
+            "dependencies": [d.as_dict() for d in self.dependencies],
+            "env_vars": [e.as_dict() for e in self.env_vars],
+            "services": [s.as_dict() for s in self.services],
+            "findings": [f.as_dict() for f in self.findings],
+            "files": [f.as_dict() for f in self.inventory.files],
+        }
+        if extra:
+            manifest.update(extra)
+        return manifest
+
+
+def analyse(root: Path, cfg: AnalysisConfig) -> Analysis:
+    """Run every analyser over the extracted tree at ``root``."""
+    inventory = build_inventory(root, cfg.vendor_globs, cfg.max_scan_file_kb)
+    runtime = detect_runtime(inventory)
+    handlers = detect_handlers(root, inventory)
+    dependencies = detect_dependencies(root, inventory)
+    env_vars = detect_env_vars(root, inventory) if cfg.scan_env_vars else []
+    services = detect_services(root, inventory) if cfg.scan_aws_services else []
+    findings = scan(root, inventory, check_secrets=cfg.scan_secrets)
+    return Analysis(
+        inventory=inventory,
+        runtime=runtime,
+        handlers=handlers,
+        dependencies=dependencies,
+        env_vars=env_vars,
+        services=services,
+        findings=findings,
+    )

lambda_watcher/analysis/deps.py

@@ -0,0 +1,291 @@
+"""Extract dependencies from a deployment package.
+
+Two kinds of dependency are collected and kept apart:
+
+``declared``
+    What a manifest says the function wants (``requirements.txt``,
+    ``package.json``, ``go.mod`` ...).
+``installed``
+    What is actually vendored inside the zip (``*.dist-info/METADATA``,
+    ``node_modules/*/package.json``). These are the versions that really ran,
+    and they are usually the ones worth diffing.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+
+from ..utils import LOG, read_text
+from .inventory import Inventory
+
+try:  # Python 3.11+
+    import tomllib  # type: ignore[import-not-found]
+except ModuleNotFoundError:  # pragma: no cover
+    try:
+        import tomli as tomllib  # type: ignore[no-redef]
+    except ModuleNotFoundError:  # pragma: no cover
+        tomllib = None  # type: ignore[assignment]
+
+
+@dataclass(frozen=True)
+class Dependency:
+    manager: str      # pip | npm | go | maven | gem
+    name: str
+    version: str | None
+    source: str       # the file it came from
+    is_declared: bool  # False => vendored/installed
+
+    def key(self) -> tuple[str, str]:
+        return (self.manager, self.name.lower())
+
+    def as_dict(self) -> dict:
+        return {
+            "manager": self.manager,
+            "name": self.name,
+            "version": self.version,
+            "source": self.source,
+            "is_declared": self.is_declared,
+        }
+
+
+_REQ_LINE = re.compile(
+    r"^\s*(?P<name>[A-Za-z0-9._-]+)\s*(?:\[[^\]]*\])?\s*"
+    r"(?P<op>==|>=|<=|~=|!=|>|<|===)?\s*(?P<version>[^\s;#]+)?"
+)
+
+
+def _parse_requirements(text: str, source: str) -> list[Dependency]:
+    deps: list[Dependency] = []
+    for raw in text.splitlines():
+        line = raw.split("#", 1)[0].strip()
+        if not line or line.startswith("-"):
+            continue  # -r includes, -e editables, --index-url
+        if line.startswith(("git+", "http://", "https://")):
+            name = re.sub(r"[#?].*$", "", line).rstrip("/").split("/")[-1]
+            deps.append(Dependency("pip", name or line, None, source, True))
+            continue
+        if " @ " in line:  # PEP 508 direct reference
+            name = line.split(" @ ", 1)[0].strip()
+            deps.append(Dependency("pip", name, line.split(" @ ", 1)[1].strip(), source, True))
+            continue
+        match = _REQ_LINE.match(line)
+        if not match or not match.group("name"):
+            continue
+        version = match.group("version") if match.group("op") else None
+        deps.append(Dependency("pip", match.group("name"), version, source, True))
+    return deps
+
+
+def _parse_pyproject(text: str, source: str) -> list[Dependency]:
+    if tomllib is None:
+        return []
+    try:
+        data = tomllib.loads(text)
+    except Exception:  # noqa: BLE001 - malformed manifests are not fatal
+        return []
+    deps: list[Dependency] = []
+    for spec in data.get("project", {}).get("dependencies", []) or []:
+        if not isinstance(spec, str):
+            continue
+        match = _REQ_LINE.match(spec)
+        if match and match.group("name"):
+            deps.append(
+                Dependency(
+                    "pip", match.group("name"),
+                    match.group("version") if match.group("op") else None, source, True,
+                )
+            )
+    poetry = data.get("tool", {}).get("poetry", {}).get("dependencies", {}) or {}
+    for name, spec in poetry.items():
+        if name.lower() == "python":
+            continue
+        version = spec if isinstance(spec, str) else (spec or {}).get("version")
+        deps.append(Dependency("pip", name, version, source, True))
+    return deps
+
+
+def _parse_package_json(text: str, source: str, declared: bool = True) -> list[Dependency]:
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        return []
+    deps: list[Dependency] = []
+    if not declared:
+        # A vendored node_modules/<pkg>/package.json: the resolved version.
+        name = data.get("name")
+        version = data.get("version")
+        if isinstance(name, str):
+            return [Dependency("npm", name, version if isinstance(version, str) else None, source, False)]
+        return []
+    for section in ("dependencies", "devDependencies", "optionalDependencies"):
+        for name, version in (data.get(section) or {}).items():
+            deps.append(
+                Dependency("npm", name, version if isinstance(version, str) else None, source, True)
+            )
+    return deps
+
+
+def _parse_package_lock(text: str, source: str) -> list[Dependency]:
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        return []
+    deps: list[Dependency] = []
+    packages = data.get("packages")
+    if isinstance(packages, dict):  # lockfile v2 / v3
+        for path, meta in packages.items():
+            if not path or not isinstance(meta, dict):
+                continue
+            name = meta.get("name") or path.split("node_modules/")[-1]
+            version = meta.get("version")
+            if name:
+                deps.append(Dependency("npm", name, version, source, False))
+    elif isinstance(data.get("dependencies"), dict):  # lockfile v1
+        for name, meta in data["dependencies"].items():
+            version = meta.get("version") if isinstance(meta, dict) else None
+            deps.append(Dependency("npm", name, version, source, False))
+    return deps
+
+
+_YARN_ENTRY = re.compile(r'^"?([^@\s"][^@\s"]*)@[^\n:]*:\s*$\n(?:.*\n)*?\s+version\s+"([^"]+)"', re.MULTILINE)
+
+
+def _parse_yarn_lock(text: str, source: str) -> list[Dependency]:
+    deps: list[Dependency] = []
+    for match in _YARN_ENTRY.finditer(text):
+        deps.append(Dependency("npm", match.group(1), match.group(2), source, False))
+    return deps
+
+
+_GO_REQUIRE_BLOCK = re.compile(r"require\s*\(([^)]*)\)", re.DOTALL)
+_GO_REQUIRE_LINE = re.compile(r"^\s*([^\s/]+\S*)\s+(v\S+)", re.MULTILINE)
+
+
+def _parse_go_mod(text: str, source: str) -> list[Dependency]:
+    deps: list[Dependency] = []
+    for block in _GO_REQUIRE_BLOCK.findall(text):
+        for name, version in _GO_REQUIRE_LINE.findall(block):
+            deps.append(Dependency("go", name, version, source, True))
+    for line in text.splitlines():
+        line = line.strip()
+        if line.startswith("require ") and "(" not in line:
+            parts = line.split()
+            if len(parts) >= 3:
+                deps.append(Dependency("go", parts[1], parts[2], source, True))
+    return deps
+
+
+def _parse_pom(text: str, source: str) -> list[Dependency]:
+    deps: list[Dependency] = []
+    for block in re.findall(r"<dependency>(.*?)</dependency>", text, re.DOTALL):
+        group = re.search(r"<groupId>(.*?)</groupId>", block, re.DOTALL)
+        artifact = re.search(r"<artifactId>(.*?)</artifactId>", block, re.DOTALL)
+        version = re.search(r"<version>(.*?)</version>", block, re.DOTALL)
+        if artifact:
+            artifact_id = artifact.group(1).strip()
+            name = f"{group.group(1).strip()}:{artifact_id}" if group else artifact_id
+            resolved = version.group(1).strip() if version else None
+            deps.append(Dependency("maven", name, resolved, source, True))
+    return deps
+
+
+_GEMFILE_LOCK = re.compile(r"^\s{4}([a-zA-Z0-9_-]+)\s+\(([^)]+)\)", re.MULTILINE)
+
+
+def _parse_gemfile_lock(text: str, source: str) -> list[Dependency]:
+    return [
+        Dependency("gem", name, version, source, False)
+        for name, version in _GEMFILE_LOCK.findall(text)
+    ]
+
+
+def _parse_metadata(text: str, source: str) -> list[Dependency]:
+    """``*.dist-info/METADATA`` or ``*.egg-info/PKG-INFO``: an installed package."""
+    name = version = None
+    for line in text.splitlines():
+        if line.startswith("Name:") and name is None:
+            name = line.split(":", 1)[1].strip()
+        elif line.startswith("Version:") and version is None:
+            version = line.split(":", 1)[1].strip()
+        if name and version:
+            break
+        if not line.strip():  # headers end at the first blank line
+            break
+    if name:
+        return [Dependency("pip", name, version, source, False)]
+    return []
+
+
+# filename (lowercase) -> parser
+_MANIFEST_PARSERS = {
+    "requirements.txt": _parse_requirements,
+    "requirements-prod.txt": _parse_requirements,
+    "requirements_prod.txt": _parse_requirements,
+    "pyproject.toml": _parse_pyproject,
+    "package.json": _parse_package_json,
+    "package-lock.json": _parse_package_lock,
+    "yarn.lock": _parse_yarn_lock,
+    "go.mod": _parse_go_mod,
+    "pom.xml": _parse_pom,
+    "gemfile.lock": _parse_gemfile_lock,
+}
+
+
+def detect_dependencies(root: Path, inventory: Inventory) -> list[Dependency]:
+    """Collect declared and installed dependencies from an extracted package."""
+    found: list[Dependency] = []
+
+    for entry in inventory.files:
+        pure = PurePosixPath(entry.path)
+        name = pure.name.lower()
+        parent = pure.parent.name.lower()
+
+        try:
+            # Installed Python distributions.
+            if name == "metadata" and parent.endswith(".dist-info"):
+                text = read_text(root / entry.path, max_bytes=64 * 1024)
+                if text:
+                    found.extend(_parse_metadata(text, entry.path))
+                continue
+            if name == "pkg-info" and parent.endswith(".egg-info"):
+                text = read_text(root / entry.path, max_bytes=64 * 1024)
+                if text:
+                    found.extend(_parse_metadata(text, entry.path))
+                continue
+
+            # Installed npm packages: node_modules/<pkg>/package.json, or
+            # node_modules/@scope/<pkg>/package.json for scoped ones.
+            if name == "package.json" and "node_modules/" in entry.path:
+                tail = entry.path.rsplit("node_modules/", 1)[1]
+                max_depth = 2 if tail.startswith("@") else 1
+                if tail.count("/") <= max_depth:  # skip foo/dist/package.json and friends
+                    text = read_text(root / entry.path, max_bytes=256 * 1024)
+                    if text:
+                        found.extend(_parse_package_json(text, entry.path, declared=False))
+                continue
+
+            if entry.is_vendor:
+                continue
+
+            parser = _MANIFEST_PARSERS.get(name)
+            if parser is None:
+                continue
+            if entry.size > 8 * 1024 * 1024:
+                continue
+            text = read_text(root / entry.path, max_bytes=8 * 1024 * 1024)
+            if text:
+                found.extend(parser(text, entry.path))
+        except Exception as exc:  # noqa: BLE001 - one bad manifest must not fail ingest
+            LOG.debug("dependency parse failed for %s: %s", entry.path, exc)
+
+    # Deduplicate, preferring installed versions over declared ranges.
+    best: dict[tuple[str, str, bool], Dependency] = {}
+    for dep in found:
+        key = (dep.manager, dep.name.lower(), dep.is_declared)
+        current = best.get(key)
+        if current is None or (current.version is None and dep.version is not None):
+            best[key] = dep
+    return sorted(best.values(), key=lambda d: (d.manager, d.is_declared, d.name.lower()))

lambda_watcher/analysis/envvars.py

@@ -0,0 +1,80 @@
+"""Collect the environment variables the code reads.
+
+Environment variables are configuration that lives outside the zip, so a diff
+that shows a new ``os.environ["TABLE_NAME"]`` is a strong signal that the
+deployment also needs a config change.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..utils import read_text
+from .inventory import Inventory
+
+_PATTERNS = [
+    re.compile(r"""os\.environ\s*\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\]"""),
+    re.compile(r"""os\.environ\.get\s*\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]"""),
+    re.compile(r"""os\.getenv\s*\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]"""),
+    re.compile(r"""process\.env\.([A-Za-z_][A-Za-z0-9_]*)"""),
+    re.compile(r"""process\.env\s*\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\]"""),
+    re.compile(r"""System\.getenv\s*\(\s*"([A-Za-z_][A-Za-z0-9_]*)"""),
+    re.compile(r"""ENV\s*\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\]"""),
+    re.compile(r"""Environment\.GetEnvironmentVariable\s*\(\s*"([A-Za-z_][A-Za-z0-9_]*)"""),
+]
+
+# Runtime-provided variables are noise in a diff.
+_AWS_RESERVED = {
+    "AWS_REGION", "AWS_DEFAULT_REGION", "AWS_EXECUTION_ENV", "AWS_LAMBDA_FUNCTION_NAME",
+    "AWS_LAMBDA_FUNCTION_VERSION", "AWS_LAMBDA_FUNCTION_MEMORY_SIZE", "AWS_LAMBDA_LOG_GROUP_NAME",
+    "AWS_LAMBDA_LOG_STREAM_NAME", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY",
+    "AWS_SESSION_TOKEN", "LAMBDA_TASK_ROOT", "LAMBDA_RUNTIME_DIR", "_HANDLER", "_X_AMZN_TRACE_ID",
+    "TZ", "PATH", "HOME", "NODE_PATH", "PYTHONPATH", "LANG",
+}
+
+_SCANNABLE = {"python", "javascript", "typescript", "java", "ruby", "csharp", "go", "shell"}
+
+
+@dataclass
+class EnvVarRef:
+    name: str
+    path: str
+    line: int
+    is_reserved: bool = False
+
+    def as_dict(self) -> dict:
+        return {"name": self.name, "path": self.path, "line": self.line, "is_reserved": self.is_reserved}
+
+
+def detect_env_vars(
+    root: Path, inventory: Inventory, include_vendor: bool = False, max_files: int = 2000
+) -> list[EnvVarRef]:
+    refs: list[EnvVarRef] = []
+    seen: set[tuple[str, str, int]] = set()
+    entries = inventory.files if include_vendor else inventory.code_files
+    scanned = 0
+
+    for entry in entries:
+        if scanned >= max_files:
+            break
+        if not entry.is_text or entry.lang not in _SCANNABLE:
+            continue
+        text = read_text(root / entry.path, max_bytes=1024 * 1024)
+        if not text:
+            continue
+        scanned += 1
+        for line_no, line in enumerate(text.splitlines(), start=1):
+            for pattern in _PATTERNS:
+                for match in pattern.finditer(line):
+                    name = match.group(1)
+                    key = (name, entry.path, line_no)
+                    if key in seen:
+                        continue
+                    seen.add(key)
+                    refs.append(
+                        EnvVarRef(name, entry.path, line_no, is_reserved=name in _AWS_RESERVED)
+                    )
+    refs.sort(key=lambda r: (r.name, r.path, r.line))
+    return refs

lambda_watcher/analysis/handler.py

@@ -0,0 +1,111 @@
+"""Find the likely Lambda entry point(s) in an extracted package."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+
+from ..utils import read_text
+from .inventory import Inventory
+
+# def handler(event, context)  /  async def handler(event, context)
+_PY_HANDLER = re.compile(
+    r"^\s*(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\(\s*[A-Za-z_]\w*\s*(?::[^,)]+)?\s*,"
+    r"\s*[A-Za-z_]\w*",
+    re.MULTILINE,
+)
+_JS_HANDLER = re.compile(
+    r"(?:^|\n)\s*(?:module\.)?exports\.([A-Za-z_$][\w$]*)\s*=|"
+    r"(?:^|\n)\s*export\s+(?:async\s+)?(?:const|function)\s+([A-Za-z_$][\w$]*)",
+)
+_JAVA_HANDLER = re.compile(r"implements\s+RequestHandler|public\s+\w+\s+handleRequest\s*\(")
+
+# Filenames AWS itself defaults to, in preference order.
+_PREFERRED = (
+    "lambda_function.py", "index.mjs", "index.js", "index.py", "app.py",
+    "main.py", "handler.py", "lambda_handler.py", "app.js", "handler.js",
+    "index.cjs", "main.go", "bootstrap",
+)
+
+
+@dataclass
+class HandlerCandidate:
+    path: str
+    symbol: str
+    handler: str  # "module.function", the value you paste into the console
+    score: int
+
+    def as_dict(self) -> dict:
+        return {"path": self.path, "symbol": self.symbol, "handler": self.handler, "score": self.score}
+
+
+def _module_name(path: str) -> str:
+    pure = PurePosixPath(path)
+    stem = pure.stem
+    parts = list(pure.parts[:-1]) + [stem]
+    return ".".join(parts)
+
+
+def detect_handlers(root: Path, inventory: Inventory, max_files: int = 400) -> list[HandlerCandidate]:
+    """Return handler candidates, best guess first."""
+    candidates: list[HandlerCandidate] = []
+    scanned = 0
+
+    entries = [
+        f
+        for f in inventory.code_files
+        if f.is_text and f.lang in {"python", "javascript", "typescript", "java"}
+    ]
+    # Shallow files first: the real handler is rarely six directories deep.
+    entries.sort(key=lambda f: (f.path.count("/"), len(f.path)))
+
+    for entry in entries:
+        if scanned >= max_files:
+            break
+        scanned += 1
+        text = read_text(root / entry.path, max_bytes=512 * 1024)
+        if not text:
+            continue
+
+        name = PurePosixPath(entry.path).name
+        base_score = 0
+        if name in _PREFERRED:
+            base_score += 50 - _PREFERRED.index(name)
+        if entry.path.count("/") == 0:
+            base_score += 20
+
+        if entry.lang == "python":
+            for match in _PY_HANDLER.finditer(text):
+                symbol = match.group(1)
+                score = base_score + (30 if symbol in {"lambda_handler", "handler"} else 0)
+                if symbol.startswith("_"):
+                    score -= 15
+                candidates.append(
+                    HandlerCandidate(entry.path, symbol, f"{_module_name(entry.path)}.{symbol}", score)
+                )
+        elif entry.lang in {"javascript", "typescript"}:
+            for match in _JS_HANDLER.finditer(text):
+                symbol = match.group(1) or match.group(2)
+                if not symbol:
+                    continue
+                score = base_score + (30 if symbol == "handler" else 0)
+                candidates.append(
+                    HandlerCandidate(entry.path, symbol, f"{_module_name(entry.path)}.{symbol}", score)
+                )
+        elif entry.lang == "java":
+            if _JAVA_HANDLER.search(text):
+                candidates.append(
+                    HandlerCandidate(entry.path, "handleRequest", _module_name(entry.path), base_score + 20)
+                )
+
+    candidates.sort(key=lambda c: -c.score)
+    # Deduplicate on the handler string while preserving order.
+    seen: set[str] = set()
+    unique: list[HandlerCandidate] = []
+    for candidate in candidates:
+        if candidate.handler in seen:
+            continue
+        seen.add(candidate.handler)
+        unique.append(candidate)
+    return unique[:10]