lambda-watcher 0.1.0__py3-none-any.whl

lambda_watcher/store.py

@@ -0,0 +1,209 @@
+"""On-disk layout of the archive.
+
+::
+
+    <root>/
+      config.yaml
+      index.db
+      logs/watcher.log
+      reports/
+      quarantine/            # archives that failed to extract
+      repos/
+        <slug>/              # optional git mirror, one commit per version
+      functions/
+        <slug>/
+          versions/
+            0001-a1b2c3d4/
+              code/          # the extracted tree
+              manifest.json  # full analysis for this version
+              package.zip    # the original download (optional)
+
+The directories are the source of truth. ``index.db`` is a rebuildable index.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .config import Config
+from .utils import LOG, rmtree, short_hash
+
+
+#: Where the mirror used to sit, inside the function directory. Both spellings
+#: are migrated: ``git/`` shipped, ``repo/`` was a brief step on the way here.
+LEGACY_REPO_DIRNAMES = ("git", "repo")
+
+
+@dataclass
+class VersionPaths:
+    root: Path
+
+    @property
+    def code(self) -> Path:
+        return self.root / "code"
+
+    @property
+    def manifest(self) -> Path:
+        return self.root / "manifest.json"
+
+    @property
+    def package(self) -> Path:
+        return self.root / "package.zip"
+
+
+class Store:
+    def __init__(self, cfg: Config) -> None:
+        self.cfg = cfg
+        cfg.ensure_dirs()
+
+    # -- paths -----------------------------------------------------------
+    def function_dir(self, slug: str) -> Path:
+        return self.cfg.functions_dir / slug
+
+    def versions_dir(self, slug: str) -> Path:
+        return self.function_dir(slug) / "versions"
+
+    def repo_dir(self, slug: str) -> Path:
+        """The function's git mirror: a real working tree you can open in an editor.
+
+        It sits at ``repos/<slug>/`` rather than inside the function directory
+        for one blunt reason — an editor names the window after the folder you
+        opened, and every function opening as "repo" would be useless. Here the
+        folder is called ``order-processor``, which is what you want to read in
+        the sidebar. Generated per-function output already lives this way:
+        ``reports/<slug>/`` is the same shape.
+
+        Older archives kept it under the function directory. Those are moved
+        here on first access, so a store from a previous version keeps working
+        without a reindex.
+        """
+        repo = self.cfg.repos_dir / slug
+        if repo.exists():
+            return repo
+        for name in LEGACY_REPO_DIRNAMES:
+            legacy = self.function_dir(slug) / name
+            if not (legacy / ".git").is_dir():
+                continue
+            try:
+                repo.parent.mkdir(parents=True, exist_ok=True)
+                legacy.rename(repo)
+                LOG.info("moved the git mirror from %s to %s", legacy, repo)
+            except OSError as exc:
+                LOG.warning("could not move %s to %s: %s", legacy, repo, exc)
+                return legacy
+            break
+        return repo
+
+    def version_dirname(self, seq: int, tree_hash: str) -> str:
+        return f"{seq:04d}-{short_hash(tree_hash)}"
+
+    def version_paths(self, slug: str, seq: int, tree_hash: str) -> VersionPaths:
+        return VersionPaths(self.versions_dir(slug) / self.version_dirname(seq, tree_hash))
+
+    def resolve_version_dir(self, stored_dir: str) -> Path:
+        """Version dirs are stored relative to the root so the store can move."""
+        path = Path(stored_dir)
+        return path if path.is_absolute() else self.cfg.root / path
+
+    def relative(self, path: Path) -> str:
+        try:
+            return str(path.relative_to(self.cfg.root))
+        except ValueError:
+            return str(path)
+
+    # -- staging ---------------------------------------------------------
+    def new_staging_dir(self, token: str) -> Path:
+        """A scratch directory on the same filesystem as the final location."""
+        staging = self.cfg.root / ".staging" / token
+        rmtree(staging)
+        staging.mkdir(parents=True, exist_ok=True)
+        return staging
+
+    def clear_staging(self) -> None:
+        rmtree(self.cfg.root / ".staging")
+
+    # -- manifests -------------------------------------------------------
+    def write_manifest(self, paths: VersionPaths, manifest: dict[str, Any]) -> None:
+        paths.manifest.parent.mkdir(parents=True, exist_ok=True)
+        paths.manifest.write_text(
+            json.dumps(manifest, indent=2, sort_keys=False, ensure_ascii=False),
+            encoding="utf-8",
+        )
+
+    def read_manifest(self, version_dir: Path) -> dict[str, Any] | None:
+        manifest = Path(version_dir) / "manifest.json"
+        if not manifest.exists():
+            return None
+        try:
+            return json.loads(manifest.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError) as exc:
+            LOG.warning("could not read %s: %s", manifest, exc)
+            return None
+
+    # -- archive handling ------------------------------------------------
+    def keep_original(self, zip_path: Path, paths: VersionPaths) -> Path | None:
+        """Copy or move the download next to its extracted version."""
+        mode = self.cfg.store.on_ingest
+        if not self.cfg.store.keep_zip or mode == "leave":
+            return None
+        paths.root.mkdir(parents=True, exist_ok=True)
+        target = paths.package
+        try:
+            if mode == "move":
+                shutil.move(str(zip_path), str(target))
+            else:
+                shutil.copy2(str(zip_path), str(target))
+        except OSError as exc:
+            LOG.warning("could not %s %s into the store: %s", mode, zip_path.name, exc)
+            return None
+        return target
+
+    def discard_original(self, zip_path: Path) -> None:
+        """Delete a download whose content is already archived, in ``move`` mode.
+
+        Whether this particular file may be removed at all is the caller's
+        decision, not this one's: see ``Ingestor.ingest``.
+        """
+        if self.cfg.store.on_ingest != "move":
+            return
+        try:
+            zip_path.unlink()
+        except OSError as exc:
+            LOG.warning("could not remove duplicate download %s: %s", zip_path, exc)
+
+    def quarantine(self, zip_path: Path, reason: str) -> Path | None:
+        """Park an archive we could not process, with a note about why."""
+        target_dir = self.cfg.quarantine_dir
+        target_dir.mkdir(parents=True, exist_ok=True)
+        target = target_dir / zip_path.name
+        counter = 1
+        while target.exists():
+            target = target_dir / f"{zip_path.stem}-{counter}{zip_path.suffix}"
+            counter += 1
+        try:
+            shutil.copy2(str(zip_path), str(target))
+            target.with_suffix(target.suffix + ".reason.txt").write_text(
+                f"{zip_path}\n{reason}\n", encoding="utf-8"
+            )
+            return target
+        except OSError as exc:
+            LOG.warning("could not quarantine %s: %s", zip_path, exc)
+            return None
+
+    # -- retention -------------------------------------------------------
+    def prune(self, slug: str, keep: int) -> list[Path]:
+        """Delete the oldest version directories beyond ``keep``."""
+        if keep <= 0:
+            return []
+        versions = sorted(
+            (p for p in self.versions_dir(slug).glob("*") if p.is_dir()), key=lambda p: p.name
+        )
+        removed: list[Path] = []
+        for path in versions[:-keep] if len(versions) > keep else []:
+            rmtree(path)
+            removed.append(path)
+        return removed

lambda_watcher/templates.py

@@ -0,0 +1,124 @@
+"""The annotated config file written by ``lambda-watcher init``."""
+
+from __future__ import annotations
+
+import json
+
+from .config import DEFAULT_HOME, default_download_dirs
+
+
+def _yaml_str(value: object) -> str:
+    """Quote a value as a YAML scalar.
+
+    Windows paths are the reason this exists: ``C:\\Users\\you`` inside a
+    double-quoted YAML scalar makes ``\\U`` an escape sequence, and the config
+    ``init`` had just written could not be read back. JSON string syntax is a
+    subset of YAML's double-quoted style, so ``json.dumps`` escapes the
+    backslashes correctly on every platform.
+    """
+    return json.dumps(str(value))
+
+
+_DIRS = "\n".join(f"    - {_yaml_str(d)}" for d in default_download_dirs())
+
+DEFAULT_CONFIG_YAML = f"""# lambda-watcher configuration
+# Every setting below is optional; the values shown are the defaults.
+
+watch:
+  # Folders to watch. Add more if you download from several places.
+  dirs:
+{_DIRS}
+  extensions: [".zip"]
+  # A file must stop changing for this many seconds before it is read, so a
+  # half-finished download is never archived.
+  stable_seconds: 2.0
+  recursive: false
+  # Switch on if your Downloads folder is a network share, a VM mount or WSL,
+  # where native filesystem events are unreliable.
+  force_polling: false
+  # Windows reports a file as "modified" when an antivirus scan, the search
+  # indexer or OneDrive touches it. Anything whose contents were last written
+  # longer ago than this is not treated as a new arrival: the event is ignored,
+  # and `on_ingest: move` never deletes it. 0 turns the check off.
+  arrival_max_age_seconds: 300
+  # Pick up zips that arrived while the watcher was not running.
+  scan_on_start: true
+  scan_on_start_max_age_hours: 24
+
+store:
+  root: {_yaml_str(DEFAULT_HOME)}
+  # copy  = leave the download in place (default)
+  # move  = take it out of Downloads once archived, keeping that folder clean
+  # leave = archive only the extracted tree, never the .zip
+  on_ingest: copy
+  keep_zip: true
+  max_uncompressed_mb: 2048
+  max_files: 200000
+  # 0 keeps every version forever.
+  max_versions_per_function: 0
+
+naming:
+  # Explicit filename -> function name rules, tried first. `name` may use \\1 etc.
+  # rules:
+  #   - pattern: "^prod[-_](.+?)[-_]deploy"
+  #     name: "\\\\1"
+  #   - pattern: "orders"
+  #     name: "order-processor"
+  case_insensitive: true
+  infer_from_zip: true
+
+analysis:
+  scan_secrets: true
+  scan_env_vars: true
+  scan_aws_services: true
+  max_scan_file_kb: 2048
+  # Paths treated as third-party rather than your code. Diffs hide these by
+  # default and summarise them as dependency changes instead.
+  vendor_globs:
+    - "node_modules/**"
+    - "**/node_modules/**"
+    - "**/site-packages/**"
+    - "**/*.dist-info/**"
+    - "**/*.egg-info/**"
+    - "vendor/**"
+    - "**/__pycache__/**"
+
+diff:
+  ignore_vendor: true
+  context_lines: 3
+  max_diff_file_kb: 512
+  max_diff_lines: 2000
+  ignore_globs: ["**/*.pyc", "**/*.so", "**/*.map"]
+
+git_mirror:
+  # Keeps one git repo per function under functions/<name>/repo/, one commit
+  # per version, tagged v0001... so `git diff v0002 v0010`, `lw open` and any
+  # git GUI just work.
+  enabled: true
+  author_name: lambda-watcher
+  author_email: lambda-watcher@localhost
+  include_vendor: true
+  tag_prefix: v
+
+notify:
+  enabled: true
+  # Only say something when the code actually differs from the last version.
+  only_on_change: true
+  # Put what changed in the notification - "2 modified, +24/-5 lines, 1 new env
+  # var" - rather than just the file count and size.
+  summarise_changes: true
+
+report:
+  # Render the comparison against the previous version as each one is archived,
+  # so the answer to "what changed?" is already written by the time the
+  # notification about it appears. Each lands in reports/<function>/, alongside
+  # a latest.html that always points at the newest comparison.
+  auto_diff: true
+  include_vendor: false
+
+# What `lw open` launches on a folder. Left empty, it looks for VS Code and
+# friends on PATH. $LAMBDA_WATCHER_EDITOR overrides this.
+editor: ""
+
+log_level: INFO
+"""

lambda_watcher/utils.py

@@ -0,0 +1,314 @@
+"""Small shared helpers: hashing, text detection, path matching, formatting."""
+
+from __future__ import annotations
+
+import fnmatch
+import hashlib
+import logging
+import os
+import re
+import unicodedata
+from datetime import datetime, timezone
+from pathlib import Path, PurePosixPath
+
+LOG = logging.getLogger("lambda_watcher")
+
+_CHUNK = 1 << 20  # 1 MiB
+
+# Extension -> language label, used for reporting and syntax hints.
+LANG_BY_EXT: dict[str, str] = {
+    ".py": "python", ".pyi": "python", ".js": "javascript", ".mjs": "javascript",
+    ".cjs": "javascript", ".jsx": "javascript", ".ts": "typescript", ".tsx": "typescript",
+    ".java": "java", ".kt": "kotlin", ".go": "go", ".rb": "ruby", ".rs": "rust",
+    ".cs": "csharp", ".php": "php", ".sh": "shell", ".bash": "shell", ".ps1": "powershell",
+    ".json": "json", ".yaml": "yaml", ".yml": "yaml", ".toml": "toml", ".ini": "ini",
+    ".cfg": "ini", ".xml": "xml", ".html": "html", ".css": "css", ".sql": "sql",
+    ".md": "markdown", ".txt": "text", ".csv": "csv", ".env": "dotenv",
+    ".jar": "binary", ".so": "binary", ".dll": "binary", ".dylib": "binary",
+    ".pyc": "binary", ".zip": "binary", ".png": "binary", ".jpg": "binary",
+    ".gz": "binary", ".whl": "binary", ".class": "binary",
+}
+
+BINARY_LANGS = {"binary"}
+
+
+def sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(_CHUNK), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def tree_hash(entries: list[tuple[str, str]]) -> str:
+    """Content hash of a whole extracted tree.
+
+    ``entries`` is a list of ``(relative_path, file_sha256)``. Deliberately
+    ignores timestamps, file order and zip metadata so that re-downloading an
+    unchanged function produces the same hash.
+    """
+    payload = "\n".join(f"{digest}  {path}" for path, digest in sorted(entries))
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def short_hash(digest: str, length: int = 8) -> str:
+    return digest[:length]
+
+
+def is_probably_text(path: Path, sniff_bytes: int = 8192) -> bool:
+    """Cheap binary check: NUL bytes or undecodable content means binary."""
+    try:
+        with path.open("rb") as fh:
+            chunk = fh.read(sniff_bytes)
+    except OSError:
+        return False
+    if not chunk:
+        return True
+    if b"\x00" in chunk:
+        return False
+    try:
+        chunk.decode("utf-8")
+    except UnicodeDecodeError:
+        # Latin-1 fallback keeps logs and odd encodings readable.
+        printable = sum(1 for b in chunk if 32 <= b < 127 or b in (9, 10, 13))
+        return printable / len(chunk) > 0.85
+    return True
+
+
+def language_for(path: str) -> str:
+    ext = PurePosixPath(path).suffix.lower()
+    if ext:
+        return LANG_BY_EXT.get(ext, ext.lstrip("."))
+    name = PurePosixPath(path).name.lower()
+    if name in {"dockerfile", "makefile", "gemfile", "rakefile", "procfile"}:
+        return name
+    return "text"
+
+
+def read_text(path: Path, max_bytes: int | None = None) -> str | None:
+    """Read a file as text, returning None if it is not decodable."""
+    try:
+        data = path.read_bytes() if max_bytes is None else path.read_bytes()[:max_bytes]
+    except OSError:
+        return None
+    if b"\x00" in data[:8192]:
+        return None
+    for encoding in ("utf-8", "utf-8-sig", "latin-1"):
+        try:
+            return data.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+    return None
+
+
+def count_lines(path: Path) -> int:
+    """Number of lines, counting a trailing partial line."""
+    total = 0
+    last = b""
+    try:
+        with path.open("rb") as fh:
+            for chunk in iter(lambda: fh.read(_CHUNK), b""):
+                total += chunk.count(b"\n")
+                last = chunk
+    except OSError:
+        return 0
+    if last and not last.endswith(b"\n"):
+        total += 1
+    return total
+
+
+def slugify(name: str, fallback: str = "unnamed") -> str:
+    """Filesystem-safe directory name that still resembles the function name."""
+    normalised = unicodedata.normalize("NFKD", name)
+    ascii_only = normalised.encode("ascii", "ignore").decode("ascii")
+    slug = re.sub(r"[^A-Za-z0-9._-]+", "-", ascii_only).strip("-._")
+    slug = re.sub(r"-{2,}", "-", slug)
+    if not slug:
+        slug = fallback
+    # Windows reserves a handful of device names.
+    if slug.upper().split(".")[0] in {
+        "CON", "PRN", "AUX", "NUL", *(f"COM{i}" for i in range(1, 10)),
+        *(f"LPT{i}" for i in range(1, 10)),
+    }:
+        slug = f"_{slug}"
+    return slug[:120]
+
+
+# The ref a source-archive directory is named after: `myrepo-1.2.3`,
+# `myrepo-main`, `myrepo-a1b2c3d`. A bare `-v2` is deliberately absent, for the
+# same reason NamingConfig.strip_patterns leaves it alone: it is far more often
+# part of a real name than a version tag.
+_REF_SUFFIX = re.compile(
+    r"[-_](?P<ref>"
+    r"v?\d+\.\d+(?:\.\d+)?(?:[-.][0-9A-Za-z.]+)?"   # 1.2.3, v1.2.3, 1.2.3-rc1
+    r"|main|master|develop|trunk"
+    r"|[0-9a-f]{7,40}"                                # commit sha, short or full
+    r")$"
+)
+
+
+def ref_from_dirname(name: str) -> str | None:
+    """Pull the version ref out of a source-archive directory name.
+
+    ``myrepo-1.2.3`` -> ``v1.2.3``; ``myrepo-main`` -> ``main``. Returns None
+    when the name carries no ref, which is the common case for a deployment
+    package. This only ever produces a label, never an identity, so a name that
+    happens to end in something hex-shaped costs a cosmetic mislabel and
+    nothing more.
+    """
+    match = _REF_SUFFIX.search(name.strip())
+    if not match:
+        return None
+    ref = match.group("ref")
+    # Normalise a bare `1.2.3` to `v1.2.3`; leave `main` and shas as they are.
+    return f"v{ref}" if ref[0].isdigit() else ref
+
+
+def matches_any(path: str, patterns: list[str]) -> bool:
+    """True when a posix-style relative path matches any glob pattern."""
+    for pattern in patterns:
+        if fnmatch.fnmatch(path, pattern):
+            return True
+        # ``foo/**`` should also match ``foo/bar`` on platforms where fnmatch
+        # does not treat ** specially.
+        if pattern.endswith("/**") and (path == pattern[:-3] or path.startswith(pattern[:-2])):
+            return True
+        if pattern.startswith("**/") and fnmatch.fnmatch(path, pattern[3:]):
+            return True
+    return False
+
+
+def human_size(num_bytes: float) -> str:
+    step = 1024.0
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if abs(num_bytes) < step:
+            return f"{num_bytes:,.0f} {unit}" if unit == "B" else f"{num_bytes:,.1f} {unit}"
+        num_bytes /= step
+    return f"{num_bytes:,.1f} PB"
+
+
+def rename_label(old: str, new: str) -> tuple[str, str, str, str]:
+    """A rename split into ``(shared prefix, was, is now, shared suffix)``.
+
+    ``site-packages/boto3-1.34.0.dist-info/METADATA`` becoming
+    ``site-packages/boto3-1.35.20.dist-info/METADATA`` is one version number
+    moving, but written out in full twice it is 90 characters of near-identical
+    path and the reader has to diff it by eye. Naming only the part that moved
+    is what git does, and for the same reason.
+
+    The split lands on separator boundaries, so the middle is always whole path
+    segments or whole ``-``/``.``-delimited pieces of a name rather than a cut
+    through the middle of a word. When the two paths share nothing, the middle
+    is simply both of them entire.
+    """
+    separators = "/-_."
+    head = 0
+    for i, (a, b) in enumerate(zip(old, new, strict=False)):
+        if a != b:
+            break
+        if a in separators:
+            head = i + 1
+    else:
+        head = min(len(old), len(new))
+
+    tail = 0
+    for i, (a, b) in enumerate(
+        zip(reversed(old[head:]), reversed(new[head:]), strict=False)
+    ):
+        if a != b:
+            break
+        if a in separators:
+            tail = i + 1
+
+    stop_old, stop_new = len(old) - tail, len(new) - tail
+    return old[:head], old[head:stop_old], new[head:stop_new], old[stop_old:]
+
+
+def signed(num: int) -> str:
+    return f"+{num}" if num > 0 else str(num)
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def parse_iso(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        return datetime.fromisoformat(value)
+    except ValueError:
+        return None
+
+
+def format_ts(value: str | None) -> str:
+    dt = parse_iso(value)
+    if dt is None:
+        return "-"
+    return dt.astimezone().strftime("%Y-%m-%d %H:%M")
+
+
+def relative_ts(value: str | None) -> str:
+    """"20 minutes ago", "yesterday", "3 weeks ago" — a timestamp read at a glance.
+
+    The dashboard answers "did it catch my last deploy?", and a wall-clock stamp
+    makes the reader do the subtraction themselves. Past a couple of months the
+    exact day starts mattering more than the distance, so it falls back to the
+    date.
+    """
+    dt = parse_iso(value)
+    if dt is None:
+        return "-"
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    seconds = int((datetime.now(timezone.utc) - dt).total_seconds())
+    if seconds < 60:
+        return "just now"                       # also covers a little clock skew
+    minutes = seconds // 60
+    if minutes < 60:
+        return f"{minutes} minute{'s' if minutes != 1 else ''} ago"
+    hours = minutes // 60
+    if hours < 24:
+        return f"{hours} hour{'s' if hours != 1 else ''} ago"
+    days = hours // 24
+    if days == 1:
+        return "yesterday"
+    if days < 14:
+        return f"{days} days ago"
+    if days < 60:
+        weeks = days // 7
+        return f"{weeks} week{'s' if weeks != 1 else ''} ago"
+    return dt.astimezone().strftime("%Y-%m-%d")
+
+
+def rmtree(path: Path) -> None:
+    """Delete a tree, tolerating read-only files (common inside zips)."""
+    import shutil
+    import stat
+
+    def _onerror(func, target, _exc):  # pragma: no cover - platform specific
+        try:
+            os.chmod(target, stat.S_IWRITE)
+            func(target)
+        except OSError:
+            pass
+
+    if path.exists():
+        shutil.rmtree(path, onerror=_onerror)
+
+
+def setup_logging(level: str = "INFO", log_file: Path | None = None) -> logging.Logger:
+    LOG.setLevel(getattr(logging, str(level).upper(), logging.INFO))
+    LOG.handlers.clear()
+    LOG.propagate = False
+    fmt = logging.Formatter("%(asctime)s %(levelname)-7s %(message)s", "%Y-%m-%d %H:%M:%S")
+    if log_file is not None:
+        log_file.parent.mkdir(parents=True, exist_ok=True)
+        fh = logging.FileHandler(log_file, encoding="utf-8")
+        fh.setFormatter(fmt)
+        LOG.addHandler(fh)
+    return LOG