lambda-watcher 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. lambda_watcher/__init__.py +4 -0
  2. lambda_watcher/__main__.py +4 -0
  3. lambda_watcher/analysis/__init__.py +115 -0
  4. lambda_watcher/analysis/deps.py +291 -0
  5. lambda_watcher/analysis/envvars.py +80 -0
  6. lambda_watcher/analysis/handler.py +111 -0
  7. lambda_watcher/analysis/inventory.py +118 -0
  8. lambda_watcher/analysis/runtime.py +117 -0
  9. lambda_watcher/analysis/secrets.py +178 -0
  10. lambda_watcher/analysis/services.py +76 -0
  11. lambda_watcher/cli.py +1406 -0
  12. lambda_watcher/config.py +324 -0
  13. lambda_watcher/db.py +466 -0
  14. lambda_watcher/diffing/__init__.py +14 -0
  15. lambda_watcher/diffing/build.py +51 -0
  16. lambda_watcher/diffing/compare.py +525 -0
  17. lambda_watcher/diffing/highlight.py +312 -0
  18. lambda_watcher/diffing/icons.py +132 -0
  19. lambda_watcher/diffing/intraline.py +162 -0
  20. lambda_watcher/diffing/render_html.py +697 -0
  21. lambda_watcher/diffing/render_text.py +198 -0
  22. lambda_watcher/extract.py +227 -0
  23. lambda_watcher/gitmirror.py +151 -0
  24. lambda_watcher/identify.py +201 -0
  25. lambda_watcher/ingest.py +480 -0
  26. lambda_watcher/notify.py +59 -0
  27. lambda_watcher/reindex.py +158 -0
  28. lambda_watcher/service.py +553 -0
  29. lambda_watcher/store.py +209 -0
  30. lambda_watcher/templates.py +124 -0
  31. lambda_watcher/utils.py +314 -0
  32. lambda_watcher/watcher.py +241 -0
  33. lambda_watcher-0.1.0.dist-info/METADATA +409 -0
  34. lambda_watcher-0.1.0.dist-info/RECORD +38 -0
  35. lambda_watcher-0.1.0.dist-info/WHEEL +5 -0
  36. lambda_watcher-0.1.0.dist-info/entry_points.txt +3 -0
  37. lambda_watcher-0.1.0.dist-info/licenses/LICENSE +201 -0
  38. lambda_watcher-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,201 @@
1
+ """Work out which Lambda function a downloaded archive belongs to.
2
+
3
+ Downloads arrive with whatever name the browser gave them, so identification
4
+ runs through a chain of increasingly fuzzy strategies and records which one
5
+ won. When it guesses wrong, ``lambda-watcher rename`` fixes it and can leave an
6
+ alias behind so the same filename maps correctly next time.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import re
13
+ from dataclasses import dataclass
14
+ from pathlib import Path
15
+
16
+ from .config import NamingConfig
17
+ from .db import Database
18
+ from .extract import peek_top_level
19
+ from .utils import LOG, short_hash, slugify
20
+
21
+ # Stems that carry no information about the function.
22
+ GENERIC_STEMS = {
23
+ "code", "function", "lambda", "lambda_function", "deployment", "package",
24
+ "archive", "download", "export", "backup", "source", "src", "app", "index",
25
+ "handler", "main", "bundle", "dist", "build", "output",
26
+ }
27
+
28
+ _HEX_ONLY = re.compile(r"^[0-9a-fA-F]{8,}$")
29
+ _BASE64ISH = re.compile(r"^[A-Za-z0-9+/=_-]{22,}$")
30
+
31
+
32
+ @dataclass
33
+ class Identification:
34
+ name: str
35
+ slug: str
36
+ confidence: str # high | medium | low
37
+ strategy: str
38
+ raw_stem: str
39
+
40
+ def as_dict(self) -> dict[str, str]:
41
+ return {
42
+ "name": self.name,
43
+ "slug": self.slug,
44
+ "confidence": self.confidence,
45
+ "strategy": self.strategy,
46
+ "raw_stem": self.raw_stem,
47
+ }
48
+
49
+
50
+ def clean_stem(stem: str, strip_patterns: list[str]) -> str:
51
+ """Strip browser suffixes, timestamps and hashes off a filename stem."""
52
+ cleaned = stem.strip()
53
+ # Apply each pattern repeatedly: "fn-2026-01-01-abc123 (1)" needs several passes.
54
+ for _ in range(4):
55
+ before = cleaned
56
+ for pattern in strip_patterns:
57
+ try:
58
+ cleaned = re.sub(pattern, "", cleaned).strip()
59
+ except re.error:
60
+ LOG.warning("invalid strip pattern %r in config, ignoring", pattern)
61
+ cleaned = cleaned.strip(" -_.")
62
+ if cleaned == before:
63
+ break
64
+ return cleaned
65
+
66
+
67
+ def _looks_meaningless(candidate: str) -> bool:
68
+ # Short names like "etl" or "fn" are unusual but real; only reject what
69
+ # carries no information at all.
70
+ if len(candidate) < 2:
71
+ return True
72
+ if candidate.lower() in GENERIC_STEMS:
73
+ return True
74
+ if _HEX_ONLY.match(candidate):
75
+ return True
76
+ if candidate.isdigit():
77
+ return True
78
+ # Long strings with no separators and no vowels are usually hashes.
79
+ if len(candidate) > 24 and _BASE64ISH.match(candidate) and not re.search(r"[-_]", candidate):
80
+ return True
81
+ return False
82
+
83
+
84
+ def _sidecar_name(zip_path: Path) -> str | None:
85
+ """Read a FunctionName out of a JSON file downloaded next to the zip.
86
+
87
+ ``aws lambda get-function > fn.json`` next to ``fn.zip`` is a common
88
+ workflow, and the JSON names the function exactly.
89
+ """
90
+ for candidate in (
91
+ zip_path.with_suffix(".json"),
92
+ zip_path.parent / f"{zip_path.stem}-configuration.json",
93
+ zip_path.parent / f"{zip_path.stem}.config.json",
94
+ ):
95
+ if not candidate.exists():
96
+ continue
97
+ try:
98
+ data = json.loads(candidate.read_text(encoding="utf-8", errors="replace"))
99
+ except (json.JSONDecodeError, OSError):
100
+ continue
101
+ for path in (("Configuration", "FunctionName"), ("FunctionName",), ("functionName",)):
102
+ node = data
103
+ for key in path:
104
+ if not isinstance(node, dict) or key not in node:
105
+ node = None
106
+ break
107
+ node = node[key]
108
+ if isinstance(node, str) and node:
109
+ return node
110
+ return None
111
+
112
+
113
+ def _zip_embedded_name(zip_path: Path) -> str | None:
114
+ """A single top-level directory in the archive often carries the name."""
115
+ tops = peek_top_level(zip_path)
116
+ if len(tops) == 1 and not tops[0].endswith((".py", ".js", ".json", ".mjs", ".cjs")):
117
+ candidate = tops[0]
118
+ if not _looks_meaningless(candidate):
119
+ return candidate
120
+ return None
121
+
122
+
123
+ def identify(
124
+ zip_path: Path,
125
+ naming: NamingConfig,
126
+ db: Database | None = None,
127
+ override: str | None = None,
128
+ ) -> Identification:
129
+ """Resolve the function name for ``zip_path``."""
130
+ filename = zip_path.name
131
+ stem = zip_path.stem
132
+
133
+ if override:
134
+ return Identification(override, slugify(override), "high", "explicit", stem)
135
+
136
+ # 1. Aliases recorded by a previous `rename --alias`.
137
+ if db is not None:
138
+ for alias in db.list_aliases():
139
+ pattern = alias["pattern"]
140
+ matched = False
141
+ if alias["is_regex"]:
142
+ try:
143
+ matched = re.search(pattern, filename) is not None
144
+ except re.error:
145
+ matched = False
146
+ else:
147
+ matched = pattern.lower() in filename.lower()
148
+ if matched:
149
+ name = alias["function_name"]
150
+ return Identification(name, slugify(name), "high", "alias", stem)
151
+
152
+ # 2. Explicit rules from the config file.
153
+ for rule in naming.rules:
154
+ pattern = rule.get("pattern")
155
+ target = rule.get("name")
156
+ if not pattern or not target:
157
+ continue
158
+ try:
159
+ match = re.search(pattern, filename)
160
+ except re.error:
161
+ LOG.warning("invalid naming rule pattern %r, ignoring", pattern)
162
+ continue
163
+ if match:
164
+ try:
165
+ name = match.expand(target) if "\\" in target else target
166
+ except re.error:
167
+ name = target
168
+ return Identification(name, slugify(name), "high", "config-rule", stem)
169
+
170
+ # 3. A JSON config export downloaded alongside the zip.
171
+ sidecar = _sidecar_name(zip_path)
172
+ if sidecar:
173
+ return Identification(sidecar, slugify(sidecar), "high", "sidecar-json", stem)
174
+
175
+ # 4. Cleaned-up filename, preferring an exact match on a known function.
176
+ cleaned = clean_stem(stem, naming.strip_patterns)
177
+ if db is not None and cleaned:
178
+ known = db.get_function_by_name(cleaned, naming.case_insensitive)
179
+ if known:
180
+ return Identification(
181
+ known["name"], known["slug"], "high", "known-function", stem
182
+ )
183
+ if db is not None:
184
+ known = db.get_function_by_name(stem, naming.case_insensitive)
185
+ if known:
186
+ return Identification(known["name"], known["slug"], "high", "known-function", stem)
187
+
188
+ if cleaned and not _looks_meaningless(cleaned):
189
+ return Identification(cleaned, slugify(cleaned), "medium", "filename", stem)
190
+
191
+ # 5. Look inside the archive.
192
+ if naming.infer_from_zip:
193
+ embedded = _zip_embedded_name(zip_path)
194
+ if embedded:
195
+ return Identification(embedded, slugify(embedded), "low", "zip-top-level", stem)
196
+
197
+ # 6. Give up, but stay stable: the same odd filename lands in the same place.
198
+ fallback = cleaned or stem or "unknown"
199
+ if _looks_meaningless(fallback):
200
+ fallback = f"unknown-{short_hash(slugify(stem) or 'x', 8)}"
201
+ return Identification(fallback, slugify(fallback), "low", "fallback", stem)
@@ -0,0 +1,480 @@
1
+ """The ingest pipeline: one downloaded archive in, one archived version out.
2
+
3
+ Steps, in order:
4
+
5
+ 1. hash the archive and check whether this exact download was already handled
6
+ 2. work out which Lambda function it belongs to
7
+ 3. extract it into a staging directory (safely)
8
+ 4. analyse the extracted tree
9
+ 5. compare the content hash against the latest version -> skip if unchanged
10
+ 6. move staging into its permanent version directory
11
+ 7. write ``manifest.json``, index everything in SQLite, mirror into git
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import shutil
17
+ import time
18
+ from dataclasses import dataclass
19
+ from datetime import datetime, timezone
20
+ from pathlib import Path
21
+
22
+ from .analysis import analyse
23
+ from .config import Config
24
+ from .db import Database
25
+ from .extract import ExtractError, extract_zip
26
+ from .gitmirror import GitUnavailable, commit_version, git_available
27
+ from .identify import Identification, identify
28
+ from .notify import notify
29
+ from .store import Store
30
+ from .utils import (
31
+ LOG,
32
+ human_size,
33
+ ref_from_dirname,
34
+ rmtree,
35
+ sha256_file,
36
+ short_hash,
37
+ utc_now_iso,
38
+ )
39
+
40
+
41
+ @dataclass
42
+ class IngestResult:
43
+ status: str # new | unchanged | duplicate-download | failed | skipped
44
+ source: Path
45
+ function_name: str | None = None
46
+ seq: int | None = None
47
+ version_dir: Path | None = None
48
+ tree_hash: str | None = None
49
+ identification: Identification | None = None
50
+ message: str = ""
51
+ changed_from: int | None = None
52
+ #: The comparison against the previous version, rendered as it was archived.
53
+ report_path: Path | None = None
54
+ #: How much changed, e.g. "2 added, 2 modified, 3 renamed, 52 vendored".
55
+ change_summary: str | None = None
56
+ #: What that means, e.g. "+24/-5 lines · new: 1 env var, 1 secret". May be empty.
57
+ change_impact: str | None = None
58
+
59
+ @property
60
+ def ok(self) -> bool:
61
+ return self.status in {"new", "unchanged", "duplicate-download"}
62
+
63
+
64
+ def wait_until_stable(
65
+ path: Path, stable_seconds: float = 2.0, max_wait: float = 900.0, poll: float = 0.5
66
+ ) -> bool:
67
+ """Block until a file stops growing, so we never read a partial download."""
68
+ deadline = time.monotonic() + max_wait
69
+ last_signature: tuple[int, float] | None = None
70
+ stable_since: float | None = None
71
+
72
+ while time.monotonic() < deadline:
73
+ try:
74
+ stat_result = path.stat()
75
+ except FileNotFoundError:
76
+ return False
77
+ except OSError:
78
+ time.sleep(poll)
79
+ continue
80
+
81
+ signature = (stat_result.st_size, stat_result.st_mtime)
82
+ now = time.monotonic()
83
+ if signature == last_signature:
84
+ if stable_since is None:
85
+ stable_since = now
86
+ elif now - stable_since >= stable_seconds:
87
+ # A final open() confirms nothing still holds it exclusively
88
+ # (matters on Windows, harmless elsewhere).
89
+ try:
90
+ with path.open("rb"):
91
+ pass
92
+ except OSError:
93
+ time.sleep(poll)
94
+ continue
95
+ return True
96
+ else:
97
+ last_signature = signature
98
+ stable_since = None
99
+ time.sleep(poll)
100
+
101
+ LOG.warning("gave up waiting for %s to finish downloading", path)
102
+ return False
103
+
104
+
105
+ def recently_written(mtime: float, max_age: float) -> bool:
106
+ """Is this file new enough to be a download that just landed?
107
+
108
+ A filesystem event does not mean a file was written. Windows raises
109
+ "modified" for attribute, security and last-access changes too, so an
110
+ antivirus scan, the search indexer or OneDrive rehydrating a file
111
+ re-announces zips that have sat in Downloads for weeks. Their mtime has not
112
+ moved, which is what tells the two apart.
113
+ """
114
+ if max_age <= 0:
115
+ return True
116
+ return time.time() - mtime <= max_age
117
+
118
+
119
+ class Ingestor:
120
+ """Runs the pipeline. Safe to call repeatedly from a single thread."""
121
+
122
+ def __init__(self, cfg: Config, db: Database, store: Store | None = None) -> None:
123
+ self.cfg = cfg
124
+ self.db = db
125
+ self.store = store or Store(cfg)
126
+
127
+ # -- public API ------------------------------------------------------
128
+ def is_candidate(self, path: Path) -> bool:
129
+ """Cheap filter applied before any I/O-heavy work."""
130
+ name = path.name
131
+ if name.startswith((".", "~$")):
132
+ return False
133
+ suffix = path.suffix.lower()
134
+ if suffix in {s.lower() for s in self.cfg.watch.partial_suffixes}:
135
+ return False
136
+ return suffix in {e.lower() for e in self.cfg.watch.extensions}
137
+
138
+ def ingest(
139
+ self,
140
+ zip_path: Path,
141
+ function_override: str | None = None,
142
+ force: bool = False,
143
+ label: str | None = None,
144
+ *,
145
+ just_downloaded: bool = True,
146
+ ) -> IngestResult:
147
+ """Archive one zip.
148
+
149
+ ``just_downloaded`` says this file arrived here rather than having been
150
+ found sitting in place, and only such a file may be cleared out of the
151
+ watched folder afterwards. A startup scan and a backfill pass ``False``:
152
+ neither can tell an overnight download from a zip something merely
153
+ touched.
154
+ """
155
+ zip_path = Path(zip_path)
156
+ now = utc_now_iso()
157
+
158
+ if not zip_path.exists():
159
+ return IngestResult("failed", zip_path, message="file disappeared before ingest")
160
+
161
+ try:
162
+ zip_sha = sha256_file(zip_path)
163
+ source_stat = zip_path.stat()
164
+ zip_size = source_stat.st_size
165
+ source_mtime = datetime.fromtimestamp(
166
+ source_stat.st_mtime, tz=timezone.utc
167
+ ).isoformat(timespec="seconds")
168
+ except OSError as exc:
169
+ return IngestResult("failed", zip_path, message=f"could not read file: {exc}")
170
+
171
+ # Deleting the download is the one step that destroys something outside
172
+ # the archive, so it takes more than a filesystem event: the file has to
173
+ # have arrived, and have been written recently enough to be that arrival.
174
+ may_discard = just_downloaded and recently_written(
175
+ source_stat.st_mtime, self.cfg.watch.arrival_max_age_seconds
176
+ )
177
+
178
+ # 1. Have we already handled this exact download?
179
+ already = self.db.seen_download(zip_sha)
180
+ self.db.mark_download_seen(zip_sha, now, zip_path.name)
181
+ if already and not force:
182
+ self.db.log_event("duplicate-download", now, source_path=str(zip_path),
183
+ detail={"zip_sha256": zip_sha})
184
+ LOG.info("skipping %s: identical download already archived", zip_path.name)
185
+ if may_discard:
186
+ self.store.discard_original(zip_path)
187
+ return IngestResult(
188
+ "duplicate-download", zip_path, message="this exact file was already archived"
189
+ )
190
+
191
+ # 2. Which function is it?
192
+ ident = identify(zip_path, self.cfg.naming, self.db, function_override)
193
+ LOG.info(
194
+ "%s -> function %r (%s, %s confidence)",
195
+ zip_path.name, ident.name, ident.strategy, ident.confidence,
196
+ )
197
+
198
+ # 3. Extract into staging.
199
+ staging = self.store.new_staging_dir(short_hash(zip_sha, 12))
200
+ code_staging = staging / "code"
201
+ try:
202
+ extraction = extract_zip(
203
+ zip_path,
204
+ code_staging,
205
+ max_uncompressed_bytes=self.cfg.store.max_uncompressed_mb * 1024 * 1024,
206
+ max_files=self.cfg.store.max_files,
207
+ strip_wrapper=self.cfg.store.strip_wrapper_dir,
208
+ )
209
+ except ExtractError as exc:
210
+ rmtree(staging)
211
+ self.store.quarantine(zip_path, str(exc))
212
+ self.db.log_event("failed", now, source_path=str(zip_path), detail=str(exc))
213
+ LOG.error("could not extract %s: %s", zip_path.name, exc)
214
+ return IngestResult("failed", zip_path, ident.name, identification=ident, message=str(exc))
215
+
216
+ # A source archive names its wrapper after the ref it was cut from, so
217
+ # the directory we just lifted away is the best label this version will
218
+ # ever get. An explicit --label always wins.
219
+ if label is None and extraction.wrapper_dir:
220
+ label = ref_from_dirname(extraction.wrapper_dir)
221
+ if label:
222
+ LOG.info("labelling this version %s (from %s/)", label, extraction.wrapper_dir)
223
+
224
+ # 4. Analyse.
225
+ try:
226
+ analysis = analyse(code_staging, self.cfg.analysis)
227
+ except Exception as exc: # noqa: BLE001 - analysis must never lose an archive
228
+ rmtree(staging)
229
+ self.db.log_event("failed", now, source_path=str(zip_path), detail=f"analysis: {exc}")
230
+ LOG.exception("analysis failed for %s", zip_path.name)
231
+ return IngestResult("failed", zip_path, ident.name, identification=ident,
232
+ message=f"analysis failed: {exc}")
233
+
234
+ tree_hash = analysis.inventory.tree_hash
235
+ function_id = self.db.upsert_function(ident.name, ident.slug, now)
236
+ function_row = self.db.get_function_by_name(ident.name)
237
+ slug = function_row["slug"] if function_row else ident.slug
238
+
239
+ # 5. Is this content already stored under this function?
240
+ existing = self.db.find_version_by_tree_hash(function_id, tree_hash)
241
+ if existing is not None and not force:
242
+ rmtree(staging)
243
+ self.db.log_event(
244
+ "unchanged", now, function_id=function_id, version_id=existing["id"],
245
+ source_path=str(zip_path), detail={"seq": existing["seq"]},
246
+ )
247
+ LOG.info(
248
+ "%s is byte-identical to %s v%04d - nothing new to archive",
249
+ zip_path.name, ident.name, existing["seq"],
250
+ )
251
+ if may_discard:
252
+ self.store.discard_original(zip_path)
253
+ return IngestResult(
254
+ "unchanged", zip_path, ident.name, int(existing["seq"]),
255
+ self.store.resolve_version_dir(existing["dir"]), tree_hash, ident,
256
+ message=f"identical to version {existing['seq']}",
257
+ )
258
+
259
+ previous = self.db.latest_version(function_id)
260
+ seq = self.db.next_seq(function_id)
261
+ paths = self.store.version_paths(slug, seq, tree_hash)
262
+
263
+ # 6. Promote staging to its permanent home.
264
+ paths.root.parent.mkdir(parents=True, exist_ok=True)
265
+ rmtree(paths.root)
266
+ try:
267
+ shutil.move(str(staging), str(paths.root))
268
+ except OSError as exc:
269
+ rmtree(staging)
270
+ self.db.log_event("failed", now, function_id=function_id, source_path=str(zip_path),
271
+ detail=f"store: {exc}")
272
+ return IngestResult("failed", zip_path, ident.name, identification=ident,
273
+ message=f"could not store version: {exc}")
274
+
275
+ kept_zip = self.store.keep_original(zip_path, paths)
276
+
277
+ # 7. Manifest + index.
278
+ manifest = analysis.to_manifest(
279
+ {
280
+ "function": {"name": ident.name, "slug": slug},
281
+ "version": {"seq": seq, "ingested_at": now, "label": label},
282
+ "source": {
283
+ "filename": zip_path.name,
284
+ "path": str(zip_path),
285
+ "mtime": source_mtime,
286
+ "zip_sha256": zip_sha,
287
+ "zip_size": zip_size,
288
+ "identification": ident.as_dict(),
289
+ "kept_at": self.store.relative(kept_zip) if kept_zip else None,
290
+ },
291
+ "archive": {
292
+ "file_count": extraction.file_count,
293
+ "dir_count": extraction.dir_count,
294
+ "compression_ratio": round(extraction.compression_ratio, 3),
295
+ "skipped_members": extraction.skipped[:50],
296
+ "wrapper_dir": extraction.wrapper_dir,
297
+ },
298
+ "previous_seq": int(previous["seq"]) if previous else None,
299
+ }
300
+ )
301
+ self.store.write_manifest(paths, manifest)
302
+
303
+ version_id = self._index_version(
304
+ function_id=function_id,
305
+ seq=seq,
306
+ tree_hash=tree_hash,
307
+ zip_sha=zip_sha,
308
+ zip_size=zip_size,
309
+ zip_path=zip_path,
310
+ source_mtime=source_mtime,
311
+ now=now,
312
+ version_dir=paths.root,
313
+ analysis=analysis,
314
+ label=label,
315
+ )
316
+
317
+ self.db.log_event(
318
+ "new-version", now, function_id=function_id, version_id=version_id,
319
+ source_path=str(zip_path),
320
+ detail={"seq": seq, "tree_hash": tree_hash, "identification": ident.as_dict()},
321
+ )
322
+
323
+ self._mirror_to_git(slug, paths.code, seq, ident.name, now, tree_hash, zip_path.name)
324
+ self._prune(slug, function_id)
325
+
326
+ report_path, change_summary, change_impact = self._render_report(
327
+ ident.name, slug, function_id, seq, previous
328
+ )
329
+
330
+ if self.cfg.notify.enabled:
331
+ previous_note = f" (was v{previous['seq']:04d})" if previous else ""
332
+ detail = f"{analysis.inventory.file_count} files · {human_size(analysis.inventory.total_size)}"
333
+ if self.cfg.notify.summarise_changes and change_summary:
334
+ detail = f"{change_summary} · {change_impact}" if change_impact else change_summary
335
+ notify(
336
+ f"Lambda archived: {ident.name}",
337
+ f"v{seq:04d}{previous_note} · {detail}",
338
+ enabled=True,
339
+ )
340
+
341
+ LOG.info(
342
+ "archived %s v%04d (%d files, %s)",
343
+ ident.name, seq, analysis.inventory.file_count,
344
+ human_size(analysis.inventory.total_size),
345
+ )
346
+ return IngestResult(
347
+ "new", zip_path, ident.name, seq, paths.root, tree_hash, ident,
348
+ report_path=report_path, change_summary=change_summary, change_impact=change_impact,
349
+ message="archived a new version",
350
+ changed_from=int(previous["seq"]) if previous else None,
351
+ )
352
+
353
+ # -- internals -------------------------------------------------------
354
+ def _index_version(self, **kw) -> int:
355
+ analysis = kw["analysis"]
356
+ with self.db.transaction():
357
+ version_id = self.db.insert_version(
358
+ {
359
+ "function_id": kw["function_id"],
360
+ "seq": kw["seq"],
361
+ "tree_hash": kw["tree_hash"],
362
+ "zip_sha256": kw["zip_sha"],
363
+ "zip_size": kw["zip_size"],
364
+ "source_name": kw["zip_path"].name,
365
+ "source_path": str(kw["zip_path"]),
366
+ "source_mtime": kw["source_mtime"],
367
+ "ingested_at": kw["now"],
368
+ "dir": self.store.relative(kw["version_dir"]),
369
+ "runtime": analysis.runtime.runtime,
370
+ "runtime_confidence": analysis.runtime.confidence,
371
+ "handler": analysis.primary_handler,
372
+ "file_count": analysis.inventory.file_count,
373
+ "total_size": analysis.inventory.total_size,
374
+ "code_file_count": analysis.inventory.code_file_count,
375
+ "code_size": analysis.inventory.code_size,
376
+ "code_lines": analysis.inventory.code_lines,
377
+ "label": kw["label"],
378
+ }
379
+ )
380
+ self.db.bulk_insert(
381
+ "files",
382
+ ["version_id", "path", "size", "sha256", "mode", "is_text", "is_vendor", "lang", "lines"],
383
+ [
384
+ (version_id, f.path, f.size, f.sha256, f.mode, int(f.is_text), int(f.is_vendor),
385
+ f.lang, f.lines)
386
+ for f in analysis.inventory.files
387
+ ],
388
+ )
389
+ self.db.bulk_insert(
390
+ "deps",
391
+ ["version_id", "manager", "name", "version", "source", "is_declared"],
392
+ [(version_id, d.manager, d.name, d.version, d.source, int(d.is_declared))
393
+ for d in analysis.dependencies],
394
+ )
395
+ self.db.bulk_insert(
396
+ "env_vars", ["version_id", "name", "path", "line"],
397
+ [(version_id, e.name, e.path, e.line) for e in analysis.env_vars if not e.is_reserved],
398
+ )
399
+ self.db.bulk_insert(
400
+ "services", ["version_id", "service", "path", "line"],
401
+ [(version_id, s.service, s.path, s.line) for s in analysis.services],
402
+ )
403
+ self.db.bulk_insert(
404
+ "findings", ["version_id", "kind", "severity", "path", "line", "detail", "is_vendor"],
405
+ [(version_id, f.kind, f.severity, f.path, f.line, f.detail, int(f.is_vendor))
406
+ for f in analysis.findings],
407
+ )
408
+ return version_id
409
+
410
+ def _render_report(
411
+ self, name: str, slug: str, function_id: int, seq: int, previous
412
+ ) -> tuple[Path | None, str | None, str | None]:
413
+ """Render the comparison against the previous version, as it is archived.
414
+
415
+ By the time a background watcher archives something, nobody is looking at
416
+ a terminal — so the answer to "what changed?" is written to disk now,
417
+ while the pipeline is already warm, rather than waiting for someone to
418
+ think of asking. ``latest.html`` is the same page under a name that never
419
+ goes stale, so it can be bookmarked.
420
+
421
+ Rendering is a convenience, never a reason to fail an ingest that has
422
+ already succeeded: every failure here is logged and swallowed.
423
+ """
424
+ if not self.cfg.report.auto_diff or previous is None:
425
+ return None, None, None
426
+ # Deferred: the renderer pulls in the whole presentation layer, and an
427
+ # ingest with nothing to compare against should not pay for the import.
428
+ from .diffing import diff_from_index
429
+ from .diffing.render_html import write_html
430
+
431
+ try:
432
+ current = self.db.get_version(function_id, seq)
433
+ if current is None:
434
+ return None, None, None
435
+ diff = diff_from_index(
436
+ self.db, self.store, self.cfg.diff, name, previous, current,
437
+ include_vendor=True if self.cfg.report.include_vendor else None,
438
+ )
439
+ target_dir = self.cfg.reports_dir / slug
440
+ target_dir.mkdir(parents=True, exist_ok=True)
441
+ target = target_dir / f"v{int(previous['seq']):04d}-v{seq:04d}.html"
442
+ write_html(diff, target)
443
+ write_html(diff, target_dir / "latest.html")
444
+ except Exception as exc: # noqa: BLE001 - never fail an ingest
445
+ LOG.warning("could not render the report for %s v%04d: %s", name, seq, exc)
446
+ return None, None, None
447
+ return target, diff.headline(), diff.impact_line()
448
+
449
+ def _mirror_to_git(
450
+ self, slug: str, code_dir: Path, seq: int, name: str, now: str, tree_hash: str, source: str
451
+ ) -> None:
452
+ if not self.cfg.git_mirror.enabled:
453
+ return
454
+ if not git_available():
455
+ LOG.debug("git not on PATH; skipping mirror for %s", name)
456
+ return
457
+ message = (
458
+ f"{name} v{seq:04d}\n\n"
459
+ f"source: {source}\n"
460
+ f"tree-hash: {tree_hash}\n"
461
+ f"ingested: {now}\n"
462
+ )
463
+ try:
464
+ commit_version(
465
+ self.store.repo_dir(slug), code_dir, self.cfg.git_mirror, seq, message, now,
466
+ vendor_globs=self.cfg.analysis.vendor_globs,
467
+ )
468
+ except (GitUnavailable, RuntimeError, OSError) as exc:
469
+ LOG.warning("git mirror failed for %s v%04d: %s", name, seq, exc)
470
+
471
+ def _prune(self, slug: str, function_id: int) -> None:
472
+ """Drop the oldest versions once a function exceeds the retention limit."""
473
+ keep = self.cfg.store.max_versions_per_function
474
+ if keep <= 0:
475
+ return
476
+ versions = self.db.list_versions(function_id) # newest first
477
+ for row in versions[keep:]:
478
+ rmtree(self.store.resolve_version_dir(row["dir"]))
479
+ self.db.delete_version(int(row["id"]))
480
+ LOG.info("pruned %s v%04d (retention limit %d)", slug, row["seq"], keep)