lambda-watcher 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

lambda_watcher/__init__.py

@@ -1,4 +1,4 @@
 """Watch a downloads folder for AWS Lambda deployment packages and version them."""
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 __all__ = ["__version__"]

lambda_watcher/analysis/__init__.py

@@ -46,26 +46,57 @@ class Analysis:
 
     @property
     def primary_handler(self) -> str | None:
+        """The handler AWS would most likely invoke, or None if none was found.
+
+        :func:`~.handler.detect_handlers` returns its candidates best-first, so this
+        is simply the top one — something like ``lambda_function.lambda_handler``.
+        """
         return self.handlers[0].handler if self.handlers else None
 
     @property
     def vendor_file_count(self) -> int:
+        """How many files came from ``node_modules``, ``site-packages`` and friends.
+
+        Usually most of the package. Reported separately so a diff can say "1 file
+        you wrote changed, 4,812 vendored files came along with it" instead of
+        burying the first number in the second.
+        """
         return sum(1 for f in self.inventory.files if f.is_vendor)
 
     @property
     def vendor_size(self) -> int:
+        """Total bytes of vendored files, the companion to :attr:`vendor_file_count`."""
         return sum(f.size for f in self.inventory.files if f.is_vendor)
 
     def unique_env_vars(self, include_reserved: bool = False) -> list[str]:
+        """The distinct environment variable names the code reads, sorted.
+
+        Collapses the per-reference list, which can name the same variable from a
+        dozen lines. Runtime-provided names (``AWS_REGION``, ``PATH``) are left out
+        unless ``include_reserved`` is set, because they are the same in every
+        package and only ever add noise to a diff.
+        """
         names = {
             ref.name for ref in self.env_vars if include_reserved or not ref.is_reserved
         }
         return sorted(names)
 
     def unique_services(self) -> list[str]:
+        """The distinct AWS service ids the code talks to, sorted.
+
+        ``["dynamodb", "s3", "sqs"]`` — one entry per service no matter how many
+        call sites mention it.
+        """
         return sorted({ref.service for ref in self.services})
 
     def totals(self) -> dict[str, int]:
+        """The headline counts for this package, as a flat dict.
+
+        The numbers a summary line is built from: how many files, how many bytes,
+        and how much of each is first-party code rather than vendored dependency.
+        Kept as one dict because it goes straight into the manifest and into the
+        index as a row.
+        """
         return {
             "file_count": self.inventory.file_count,
             "total_size": self.inventory.total_size,
@@ -77,6 +108,17 @@ class Analysis:
         }
 
     def to_manifest(self, extra: dict[str, Any] | None = None) -> dict[str, Any]:
+        """Render this analysis as the ``manifest.json`` written beside the version.
+
+        The manifest is the source of truth on disk: the SQLite index is rebuilt
+        from these files by :mod:`~lambda_watcher.reindex`, so anything the index
+        needs has to be here. ``extra`` carries the fields only the ingest knows —
+        the function name, the sequence number, the originating zip — which are
+        merged in over the top.
+
+        Bumping the shape of what this returns means bumping ``MANIFEST_SCHEMA``,
+        and old manifests still have to reindex.
+        """
         manifest: dict[str, Any] = {
             "schema": MANIFEST_SCHEMA,
             "tree_hash": self.inventory.tree_hash,

lambda_watcher/analysis/deps.py

@@ -32,6 +32,14 @@ except ModuleNotFoundError:  # pragma: no cover
 
 @dataclass(frozen=True)
 class Dependency:
+    """One dependency, either declared in a manifest or installed in the zip.
+
+    ``is_declared`` is the important flag. A declared entry came from
+    ``requirements.txt`` and may be a range (``boto3>=1.34``); an installed
+    entry came from ``site-packages`` and is an exact version that really
+    shipped (``boto3 1.34.0``). Frozen so it can go in a set.
+    """
+
     manager: str      # pip | npm | go | maven | gem
     name: str
     version: str | None
@@ -39,9 +47,16 @@ class Dependency:
     is_declared: bool  # False => vendored/installed
 
     def key(self) -> tuple[str, str]:
+        """Identity across versions: ``(manager, lowercased name)``.
+
+        Deliberately excludes the version, because this is what a diff groups on to
+        notice that ``boto3`` went from 1.34.0 to 1.35.20 rather than reporting one
+        package removed and a different one added.
+        """
         return (self.manager, self.name.lower())
 
     def as_dict(self) -> dict:
+        """This dependency as plain JSON-ready data, for the manifest."""
         return {
             "manager": self.manager,
             "name": self.name,
@@ -58,6 +73,16 @@ _REQ_LINE = re.compile(
 
 
 def _parse_requirements(text: str, source: str) -> list[Dependency]:
+    """Parse a ``requirements.txt`` into declared pip dependencies.
+
+    Handles the ordinary ``boto3==1.34.0`` form plus extras (``requests[security]``),
+    direct URLs and ``git+`` references (the trailing path segment becomes the
+    name), and PEP 508 ``name @ url`` entries. Comments and the flag lines that
+    start with ``-`` (``-r base.txt``, ``-e .``, ``--index-url``) are skipped.
+
+    A bare ``boto3`` with no comparison operator records a None version — the
+    file asked for the package but not for any particular release.
+    """
     deps: list[Dependency] = []
     for raw in text.splitlines():
         line = raw.split("#", 1)[0].strip()
@@ -80,6 +105,16 @@ def _parse_requirements(text: str, source: str) -> list[Dependency]:
 
 
 def _parse_pyproject(text: str, source: str) -> list[Dependency]:
+    """Parse a ``pyproject.toml`` into declared pip dependencies.
+
+    Reads both the standard ``[project] dependencies`` list and Poetry's
+    ``[tool.poetry.dependencies]`` table, whose values may be a bare version
+    string or a table with a ``version`` key. Poetry's ``python`` entry is
+    dropped: it constrains the interpreter, not the package set.
+
+    Returns nothing if TOML cannot be parsed — on Python 3.10 ``tomli`` may be
+    absent, and a malformed file is not worth failing an ingest over.
+    """
     if tomllib is None:
         return []
     try:
@@ -108,6 +143,15 @@ def _parse_pyproject(text: str, source: str) -> list[Dependency]:
 
 
 def _parse_package_json(text: str, source: str, declared: bool = True) -> list[Dependency]:
+    """Parse a ``package.json``, either as a manifest or as an installed package.
+
+    The same filename means two different things depending on where it sits.
+    At the package root it is a manifest, and ``declared=True`` reads the
+    ``dependencies``/``devDependencies``/``optionalDependencies`` tables, whose
+    values are ranges like ``^4.17.21``. Inside ``node_modules/<pkg>/`` it
+    describes one installed package, and ``declared=False`` takes the file's own
+    ``name`` and ``version`` — the exact release that shipped.
+    """
     try:
         data = json.loads(text)
     except json.JSONDecodeError:
@@ -129,6 +173,13 @@ def _parse_package_json(text: str, source: str, declared: bool = True) -> list[D
 
 
 def _parse_package_lock(text: str, source: str) -> list[Dependency]:
+    """Parse a ``package-lock.json`` into installed npm dependencies.
+
+    Supports both lockfile layouts: v2/v3 keep a flat ``packages`` map keyed by
+    path, where the name has to be recovered from the key when the entry omits
+    it, while v1 keeps a ``dependencies`` map keyed by name. Both give resolved
+    versions, so entries are recorded as installed rather than declared.
+    """
     try:
         data = json.loads(text)
     except json.JSONDecodeError:
@@ -154,6 +205,12 @@ _YARN_ENTRY = re.compile(r'^"?([^@\s"][^@\s"]*)@[^\n:]*:\s*$\n(?:.*\n)*?\s+versi
 
 
 def _parse_yarn_lock(text: str, source: str) -> list[Dependency]:
+    """Parse a ``yarn.lock`` into installed npm dependencies.
+
+    Yarn's format is not JSON, so this matches each ``name@range:`` header
+    against the indented ``version "1.2.3"`` line that follows it and takes the
+    resolved version.
+    """
     deps: list[Dependency] = []
     for match in _YARN_ENTRY.finditer(text):
         deps.append(Dependency("npm", match.group(1), match.group(2), source, False))
@@ -165,6 +222,11 @@ _GO_REQUIRE_LINE = re.compile(r"^\s*([^\s/]+\S*)\s+(v\S+)", re.MULTILINE)
 
 
 def _parse_go_mod(text: str, source: str) -> list[Dependency]:
+    """Parse a ``go.mod`` into declared Go dependencies.
+
+    Covers both spellings: the grouped ``require ( ... )`` block and the
+    single-line ``require example.com/mod v1.2.3`` form.
+    """
     deps: list[Dependency] = []
     for block in _GO_REQUIRE_BLOCK.findall(text):
         for name, version in _GO_REQUIRE_LINE.findall(block):
@@ -179,6 +241,16 @@ def _parse_go_mod(text: str, source: str) -> list[Dependency]:
 
 
 def _parse_pom(text: str, source: str) -> list[Dependency]:
+    """Parse a Maven ``pom.xml`` into declared Java dependencies.
+
+    Each ``<dependency>`` becomes one entry named ``groupId:artifactId``, the
+    coordinate Maven itself uses. A ``<version>`` that is a property reference
+    (``${aws.sdk.version}``) is recorded verbatim, since resolving it would mean
+    evaluating the build.
+
+    Regex rather than an XML parser because this only needs the common shape and
+    must not fail an ingest over an unusual document.
+    """
     deps: list[Dependency] = []
     for block in re.findall(r"<dependency>(.*?)</dependency>", text, re.DOTALL):
         group = re.search(r"<groupId>(.*?)</groupId>", block, re.DOTALL)
@@ -196,6 +268,11 @@ _GEMFILE_LOCK = re.compile(r"^\s{4}([a-zA-Z0-9_-]+)\s+\(([^)]+)\)", re.MULTILINE
 
 
 def _parse_gemfile_lock(text: str, source: str) -> list[Dependency]:
+    """Parse a ``Gemfile.lock`` into installed Ruby gems.
+
+    The four-space-indented ``name (1.2.3)`` lines under ``specs:`` are the
+    resolved versions, which is why these are recorded as installed.
+    """
     return [
         Dependency("gem", name, version, source, False)
         for name, version in _GEMFILE_LOCK.findall(text)

lambda_watcher/analysis/envvars.py

@@ -39,18 +39,43 @@ _SCANNABLE = {"python", "javascript", "typescript", "java", "ruby", "csharp", "g
 
 @dataclass
 class EnvVarRef:
+    """One place in the code where an environment variable is read.
+
+    The same variable read from three files is three refs; collapse them with
+    :meth:`~lambda_watcher.analysis.Analysis.unique_env_vars` when you want the
+    set of names rather than the call sites.
+    """
+
     name: str
     path: str
     line: int
     is_reserved: bool = False
 
     def as_dict(self) -> dict:
+        """This reference as plain JSON-ready data, for the manifest."""
         return {"name": self.name, "path": self.path, "line": self.line, "is_reserved": self.is_reserved}
 
 
 def detect_env_vars(
     root: Path, inventory: Inventory, include_vendor: bool = False, max_files: int = 2000
 ) -> list[EnvVarRef]:
+    """Find every environment variable the code reads, across languages.
+
+    Scans each text file line by line for the idioms that read configuration —
+    ``os.environ["X"]`` and ``os.getenv("X")`` in Python, ``process.env.X`` in
+    JavaScript, ``System.getenv("X")`` in Java, and the Ruby and C# equivalents
+    — and records the name, file and line of each hit.
+
+    Vendored dependencies are skipped by default: they read hundreds of
+    variables that have nothing to do with this function. ``max_files`` caps how
+    many files are opened so a package with an enormous tree cannot make an
+    ingest crawl. Results are deduplicated by ``(name, path, line)`` and sorted,
+    so re-analysing an unchanged tree produces an identical list.
+
+    This is textual pattern matching, not parsing: a name built at runtime
+    (``os.environ[prefix + "_URL"]``) is invisible to it, and one inside a
+    comment still counts.
+    """
     refs: list[EnvVarRef] = []
     seen: set[tuple[str, str, int]] = set()
     entries = inventory.files if include_vendor else inventory.code_files

lambda_watcher/analysis/handler.py

@@ -31,16 +31,30 @@ _PREFERRED = (
 
 @dataclass
 class HandlerCandidate:
+    """One possible Lambda entry point, with a score saying how likely it is.
+
+    ``handler`` is the string you would actually paste into the AWS console —
+    ``lambda_function.lambda_handler`` — assembled from the file's module path
+    and the function's name.
+    """
+
     path: str
     symbol: str
     handler: str  # "module.function", the value you paste into the console
     score: int
 
     def as_dict(self) -> dict:
+        """This candidate as plain JSON-ready data, for the manifest."""
         return {"path": self.path, "symbol": self.symbol, "handler": self.handler, "score": self.score}
 
 
 def _module_name(path: str) -> str:
+    """Turn a file path into the dotted module path AWS expects.
+
+    ``src/app/handler.py`` -> ``src.app.handler``. The extension is dropped and
+    every directory separator becomes a dot, which is the form the handler
+    setting takes.
+    """
     pure = PurePosixPath(path)
     stem = pure.stem
     parts = list(pure.parts[:-1]) + [stem]

lambda_watcher/analysis/inventory.py

@@ -10,6 +10,14 @@ from ..utils import count_lines, is_probably_text, language_for, matches_any, sh
 
 @dataclass
 class FileEntry:
+    """One file inside an extracted package, hashed and classified.
+
+    ``path`` is always posix-style and relative to the package root, so the same
+    tree hashes identically on Windows and Linux. ``is_vendor`` is the flag most
+    of the tool keys off: it separates the handful of files somebody wrote from
+    the thousands that came out of ``pip install``.
+    """
+
     path: str          # posix relative path
     size: int
     sha256: str
@@ -20,6 +28,7 @@ class FileEntry:
     lines: int
 
     def as_dict(self) -> dict:
+        """This entry as plain JSON-ready data, for the manifest."""
         return {
             "path": self.path,
             "size": self.size,
@@ -34,6 +43,14 @@ class FileEntry:
 
 @dataclass
 class Inventory:
+    """Every file in one extracted package, plus the totals worth caching.
+
+    ``tree_hash`` is the identity of the whole tree and what decides whether an
+    ingest has found a new version. The size and line counts are accumulated
+    during the walk rather than recomputed, since they are wanted on every
+    summary screen.
+    """
+
     files: list[FileEntry] = field(default_factory=list)
     tree_hash: str = ""
     total_size: int = 0
@@ -42,6 +59,7 @@ class Inventory:
 
     @property
     def file_count(self) -> int:
+        """How many files the package contains, vendored ones included."""
         return len(self.files)
 
     @property
@@ -51,12 +69,24 @@ class Inventory:
 
     @property
     def code_file_count(self) -> int:
+        """How many first-party files there are — the length of :attr:`code_files`."""
         return len(self.code_files)
 
     def by_path(self) -> dict[str, FileEntry]:
+        """The files as a ``{path: entry}`` lookup.
+
+        Diffing two versions means asking "was this path in the other one too?"
+        thousands of times, which wants a dict rather than a scan of the list.
+        """
         return {f.path: f for f in self.files}
 
     def language_breakdown(self) -> dict[str, int]:
+        """Count first-party files per language, most common first.
+
+        ``{"python": 12, "json": 3, "markdown": 1}``. Vendored files are excluded
+        deliberately — counting them would report the language of the dependencies
+        rather than of the function.
+        """
         counts: dict[str, int] = {}
         for f in self.code_files:
             counts[f.lang] = counts.get(f.lang, 0) + 1

lambda_watcher/analysis/runtime.py

@@ -10,12 +10,20 @@ from .inventory import Inventory
 
 @dataclass
 class RuntimeGuess:
+    """Which Lambda runtime this package looks like, and why we think so.
+
+    ``evidence`` names the files and extensions that drove the guess, so a
+    surprising answer can be argued with rather than just disbelieved, and
+    ``all_scores`` keeps the runners-up for the same reason.
+    """
+
     runtime: str = "unknown"
     confidence: str = "low"
     evidence: list[str] = field(default_factory=list)
     all_scores: dict[str, int] = field(default_factory=dict)
 
     def as_dict(self) -> dict:
+        """This guess as plain JSON-ready data, for the manifest."""
         return {
             "runtime": self.runtime,
             "confidence": self.confidence,
@@ -60,10 +68,30 @@ _EXT_SCORES: dict[str, tuple[str, int]] = {
 
 
 def detect_runtime(inventory: Inventory) -> RuntimeGuess:
+    """Score the package's files to decide which runtime it targets.
+
+    Two kinds of signal are added up. Marker filenames are strong and specific
+    — ``lambda_function.py`` is worth 40 points towards Python, ``go.mod`` 30
+    towards Go — while file extensions are weak and cumulative, a few points
+    each. The language with the highest total wins.
+
+    Where a marker sits matters as much as which marker it is. One at the
+    package root is what the author intended; the same name inside
+    ``node_modules`` is somebody else's ``package.json`` and is worth a tenth as
+    much, and one buried a few directories down a third. Extensions inside
+    vendored trees are ignored outright, since a Python package that vendors a
+    JavaScript build tool should still read as Python.
+
+    Confidence is about the margin, not the total: ``high`` needs both a decisive
+    score and twice the runner-up, so a package that genuinely looks like two
+    runtimes says so instead of picking one and sounding certain. An empty or
+    unrecognisable tree returns the default ``unknown``/``low`` guess.
+    """
     scores: dict[str, int] = {}
     evidence: list[str] = []
 
     def bump(lang: str, points: int, why: str) -> None:
+        """Add ``points`` to a language's score, recording ``why`` once."""
         scores[lang] = scores.get(lang, 0) + points
         if why not in evidence:
             evidence.append(why)

lambda_watcher/analysis/secrets.py

@@ -20,6 +20,12 @@ from .inventory import Inventory
 
 @dataclass
 class Finding:
+    """One flagged line: a possible credential or a risky call.
+
+    ``detail`` is safe to print and store. For a matched secret it is already
+    redacted by :func:`_redact`; the value itself never leaves this module.
+    """
+
     kind: str
     severity: str  # high | medium | low
     path: str
@@ -28,6 +34,7 @@ class Finding:
     is_vendor: bool = False
 
     def as_dict(self) -> dict:
+        """This finding as plain JSON-ready data, for the manifest."""
         return {
             "kind": self.kind,
             "severity": self.severity,
@@ -40,6 +47,14 @@ class Finding:
 
 @dataclass(frozen=True)
 class Rule:
+    """One pattern to look for, and how loudly to complain when it matches.
+
+    ``group`` names the capture group holding the interesting value. A rule with
+    ``group=0`` has nothing to extract — ``-----BEGIN PRIVATE KEY-----`` is the
+    whole finding — and that difference decides both whether the placeholder
+    filter runs and whether the detail is redacted.
+    """
+
     kind: str
     severity: str
     pattern: re.Pattern[str]
@@ -103,6 +118,12 @@ _SCANNABLE = {
 
 
 def _shannon_entropy(value: str) -> float:
+    """Bits of entropy per character, used to tell keys from words.
+
+    A real credential draws on the whole alphabet fairly evenly and scores high;
+    an English word or a repeated placeholder scores low. Used only as one input
+    to :func:`_is_placeholder`, never on its own.
+    """
     if not value:
         return 0.0
     counts: dict[str, int] = {}
@@ -113,6 +134,13 @@ def _shannon_entropy(value: str) -> float:
 
 
 def _redact(value: str) -> str:
+    """Render a matched value so it can be stored without storing the secret.
+
+    ``AKIAIOSFODNN7EXAMPLE`` -> ``AKIA…LE (20 chars)``. Enough to recognise the
+    value again and to see it change between versions, not enough to use. Short
+    values are replaced by stars entirely, since four of eight characters would
+    give too much away.
+    """
     value = value.strip()
     if len(value) <= 8:
         return "*" * len(value)
@@ -120,6 +148,18 @@ def _redact(value: str) -> str:
 
 
 def _is_placeholder(value: str) -> bool:
+    """True when a matched value is obviously not a live credential.
+
+    The scanner would be useless if every ``password = "changeme"`` in an
+    example file produced a high-severity finding. Three things disqualify a
+    match: a known placeholder shape (``xxxx``, ``<your-key>``, ``${VAR}``,
+    ``TODO``), a value that is plainly an environment lookup rather than a
+    literal, and a long string whose entropy is too low to be a key — see
+    :func:`_shannon_entropy`.
+
+    Tuned to under-report. A missed secret is a gap in a tripwire; a wall of
+    false positives is a feature people switch off.
+    """
     stripped = value.strip()
     if not stripped or _PLACEHOLDER.match(stripped):
         return True
@@ -136,6 +176,25 @@ def scan(
     check_secrets: bool = True,
     max_files: int = 3000,
 ) -> list[Finding]:
+    """Scan the package for credentials and risky calls, worst first.
+
+    Runs two rule sets over every scannable text file: :data:`SECRET_RULES`,
+    which look for things that should never be in a zip (AWS keys, private
+    keys, GitHub and Slack tokens, connection strings with passwords in them),
+    and :data:`RISK_RULES`, which look for patterns worth a second glance
+    (``eval(``, ``shell=True``, ``verify=False``). Setting ``check_secrets``
+    False keeps only the second set.
+
+    Several limits keep this cheap and quiet rather than exhaustive. Vendored
+    files are skipped by default, files over 2 MB are not opened, ``max_files``
+    caps the walk, and lines longer than 4,000 characters are ignored because a
+    minified bundle matches everything and means nothing. Secret matches are
+    filtered through :func:`_is_placeholder`, and each ``(kind, value)`` pair is
+    reported once per file rather than once per occurrence.
+
+    Findings come back sorted by severity then location, so the first row is
+    the one worth reading. Values are redacted before they are returned.
+    """
     findings: list[Finding] = []
     entries = inventory.files if include_vendor else inventory.code_files
     rules = (SECRET_RULES if check_secrets else []) + RISK_RULES

lambda_watcher/analysis/services.py

@@ -38,17 +38,37 @@ _ALIASES = {
 
 @dataclass
 class ServiceRef:
+    """One place in the code where an AWS service is used.
+
+    Recorded per ``(service, file)`` rather than per line — the first mention in
+    a file is enough to say the file talks to DynamoDB.
+    """
+
     service: str
     path: str
     line: int
 
     def as_dict(self) -> dict:
+        """This reference as plain JSON-ready data, for the manifest."""
         return {"service": self.service, "path": self.path, "line": self.line}
 
 
 def detect_services(
     root: Path, inventory: Inventory, max_files: int = 2000
 ) -> list[ServiceRef]:
+    """Find which AWS services the code calls, across SDK dialects.
+
+    Recognises the shapes each SDK uses to name a service: ``boto3.client("s3")``
+    in Python, the ``@aws-sdk/client-dynamodb`` import and ``new AWS.S3(`` in
+    JavaScript, and the ``com.amazonaws.services.*`` and
+    ``software.amazon.awssdk.services.*`` package paths in Java. The captured
+    name is lowercased and mapped through ``_ALIASES`` so the spellings converge
+    — ``sfn`` and ``states`` both become ``stepfunctions``.
+
+    Only first-party code is scanned, and only the first hit per
+    ``(service, file)`` is kept, so vendoring the AWS SDK does not make a
+    function look like it calls every service Amazon sells.
+    """
     refs: list[ServiceRef] = []
     seen: set[tuple[str, str]] = set()
     scanned = 0