lambda-watcher 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lambda_watcher/__init__.py +4 -0
- lambda_watcher/__main__.py +4 -0
- lambda_watcher/analysis/__init__.py +115 -0
- lambda_watcher/analysis/deps.py +291 -0
- lambda_watcher/analysis/envvars.py +80 -0
- lambda_watcher/analysis/handler.py +111 -0
- lambda_watcher/analysis/inventory.py +118 -0
- lambda_watcher/analysis/runtime.py +117 -0
- lambda_watcher/analysis/secrets.py +178 -0
- lambda_watcher/analysis/services.py +76 -0
- lambda_watcher/cli.py +1406 -0
- lambda_watcher/config.py +324 -0
- lambda_watcher/db.py +466 -0
- lambda_watcher/diffing/__init__.py +14 -0
- lambda_watcher/diffing/build.py +51 -0
- lambda_watcher/diffing/compare.py +525 -0
- lambda_watcher/diffing/highlight.py +312 -0
- lambda_watcher/diffing/icons.py +132 -0
- lambda_watcher/diffing/intraline.py +162 -0
- lambda_watcher/diffing/render_html.py +697 -0
- lambda_watcher/diffing/render_text.py +198 -0
- lambda_watcher/extract.py +227 -0
- lambda_watcher/gitmirror.py +151 -0
- lambda_watcher/identify.py +201 -0
- lambda_watcher/ingest.py +480 -0
- lambda_watcher/notify.py +59 -0
- lambda_watcher/reindex.py +158 -0
- lambda_watcher/service.py +553 -0
- lambda_watcher/store.py +209 -0
- lambda_watcher/templates.py +124 -0
- lambda_watcher/utils.py +314 -0
- lambda_watcher/watcher.py +241 -0
- lambda_watcher-0.1.0.dist-info/METADATA +409 -0
- lambda_watcher-0.1.0.dist-info/RECORD +38 -0
- lambda_watcher-0.1.0.dist-info/WHEEL +5 -0
- lambda_watcher-0.1.0.dist-info/entry_points.txt +3 -0
- lambda_watcher-0.1.0.dist-info/licenses/LICENSE +201 -0
- lambda_watcher-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Work out which Lambda function a downloaded archive belongs to.
|
|
2
|
+
|
|
3
|
+
Downloads arrive with whatever name the browser gave them, so identification
|
|
4
|
+
runs through a chain of increasingly fuzzy strategies and records which one
|
|
5
|
+
won. When it guesses wrong, ``lambda-watcher rename`` fixes it and can leave an
|
|
6
|
+
alias behind so the same filename maps correctly next time.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from .config import NamingConfig
|
|
17
|
+
from .db import Database
|
|
18
|
+
from .extract import peek_top_level
|
|
19
|
+
from .utils import LOG, short_hash, slugify
|
|
20
|
+
|
|
21
|
+
# Stems that carry no information about the function.
|
|
22
|
+
GENERIC_STEMS = {
|
|
23
|
+
"code", "function", "lambda", "lambda_function", "deployment", "package",
|
|
24
|
+
"archive", "download", "export", "backup", "source", "src", "app", "index",
|
|
25
|
+
"handler", "main", "bundle", "dist", "build", "output",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
_HEX_ONLY = re.compile(r"^[0-9a-fA-F]{8,}$")
|
|
29
|
+
_BASE64ISH = re.compile(r"^[A-Za-z0-9+/=_-]{22,}$")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class Identification:
|
|
34
|
+
name: str
|
|
35
|
+
slug: str
|
|
36
|
+
confidence: str # high | medium | low
|
|
37
|
+
strategy: str
|
|
38
|
+
raw_stem: str
|
|
39
|
+
|
|
40
|
+
def as_dict(self) -> dict[str, str]:
|
|
41
|
+
return {
|
|
42
|
+
"name": self.name,
|
|
43
|
+
"slug": self.slug,
|
|
44
|
+
"confidence": self.confidence,
|
|
45
|
+
"strategy": self.strategy,
|
|
46
|
+
"raw_stem": self.raw_stem,
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def clean_stem(stem: str, strip_patterns: list[str]) -> str:
|
|
51
|
+
"""Strip browser suffixes, timestamps and hashes off a filename stem."""
|
|
52
|
+
cleaned = stem.strip()
|
|
53
|
+
# Apply each pattern repeatedly: "fn-2026-01-01-abc123 (1)" needs several passes.
|
|
54
|
+
for _ in range(4):
|
|
55
|
+
before = cleaned
|
|
56
|
+
for pattern in strip_patterns:
|
|
57
|
+
try:
|
|
58
|
+
cleaned = re.sub(pattern, "", cleaned).strip()
|
|
59
|
+
except re.error:
|
|
60
|
+
LOG.warning("invalid strip pattern %r in config, ignoring", pattern)
|
|
61
|
+
cleaned = cleaned.strip(" -_.")
|
|
62
|
+
if cleaned == before:
|
|
63
|
+
break
|
|
64
|
+
return cleaned
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _looks_meaningless(candidate: str) -> bool:
|
|
68
|
+
# Short names like "etl" or "fn" are unusual but real; only reject what
|
|
69
|
+
# carries no information at all.
|
|
70
|
+
if len(candidate) < 2:
|
|
71
|
+
return True
|
|
72
|
+
if candidate.lower() in GENERIC_STEMS:
|
|
73
|
+
return True
|
|
74
|
+
if _HEX_ONLY.match(candidate):
|
|
75
|
+
return True
|
|
76
|
+
if candidate.isdigit():
|
|
77
|
+
return True
|
|
78
|
+
# Long strings with no separators and no vowels are usually hashes.
|
|
79
|
+
if len(candidate) > 24 and _BASE64ISH.match(candidate) and not re.search(r"[-_]", candidate):
|
|
80
|
+
return True
|
|
81
|
+
return False
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _sidecar_name(zip_path: Path) -> str | None:
|
|
85
|
+
"""Read a FunctionName out of a JSON file downloaded next to the zip.
|
|
86
|
+
|
|
87
|
+
``aws lambda get-function > fn.json`` next to ``fn.zip`` is a common
|
|
88
|
+
workflow, and the JSON names the function exactly.
|
|
89
|
+
"""
|
|
90
|
+
for candidate in (
|
|
91
|
+
zip_path.with_suffix(".json"),
|
|
92
|
+
zip_path.parent / f"{zip_path.stem}-configuration.json",
|
|
93
|
+
zip_path.parent / f"{zip_path.stem}.config.json",
|
|
94
|
+
):
|
|
95
|
+
if not candidate.exists():
|
|
96
|
+
continue
|
|
97
|
+
try:
|
|
98
|
+
data = json.loads(candidate.read_text(encoding="utf-8", errors="replace"))
|
|
99
|
+
except (json.JSONDecodeError, OSError):
|
|
100
|
+
continue
|
|
101
|
+
for path in (("Configuration", "FunctionName"), ("FunctionName",), ("functionName",)):
|
|
102
|
+
node = data
|
|
103
|
+
for key in path:
|
|
104
|
+
if not isinstance(node, dict) or key not in node:
|
|
105
|
+
node = None
|
|
106
|
+
break
|
|
107
|
+
node = node[key]
|
|
108
|
+
if isinstance(node, str) and node:
|
|
109
|
+
return node
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _zip_embedded_name(zip_path: Path) -> str | None:
|
|
114
|
+
"""A single top-level directory in the archive often carries the name."""
|
|
115
|
+
tops = peek_top_level(zip_path)
|
|
116
|
+
if len(tops) == 1 and not tops[0].endswith((".py", ".js", ".json", ".mjs", ".cjs")):
|
|
117
|
+
candidate = tops[0]
|
|
118
|
+
if not _looks_meaningless(candidate):
|
|
119
|
+
return candidate
|
|
120
|
+
return None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def identify(
|
|
124
|
+
zip_path: Path,
|
|
125
|
+
naming: NamingConfig,
|
|
126
|
+
db: Database | None = None,
|
|
127
|
+
override: str | None = None,
|
|
128
|
+
) -> Identification:
|
|
129
|
+
"""Resolve the function name for ``zip_path``."""
|
|
130
|
+
filename = zip_path.name
|
|
131
|
+
stem = zip_path.stem
|
|
132
|
+
|
|
133
|
+
if override:
|
|
134
|
+
return Identification(override, slugify(override), "high", "explicit", stem)
|
|
135
|
+
|
|
136
|
+
# 1. Aliases recorded by a previous `rename --alias`.
|
|
137
|
+
if db is not None:
|
|
138
|
+
for alias in db.list_aliases():
|
|
139
|
+
pattern = alias["pattern"]
|
|
140
|
+
matched = False
|
|
141
|
+
if alias["is_regex"]:
|
|
142
|
+
try:
|
|
143
|
+
matched = re.search(pattern, filename) is not None
|
|
144
|
+
except re.error:
|
|
145
|
+
matched = False
|
|
146
|
+
else:
|
|
147
|
+
matched = pattern.lower() in filename.lower()
|
|
148
|
+
if matched:
|
|
149
|
+
name = alias["function_name"]
|
|
150
|
+
return Identification(name, slugify(name), "high", "alias", stem)
|
|
151
|
+
|
|
152
|
+
# 2. Explicit rules from the config file.
|
|
153
|
+
for rule in naming.rules:
|
|
154
|
+
pattern = rule.get("pattern")
|
|
155
|
+
target = rule.get("name")
|
|
156
|
+
if not pattern or not target:
|
|
157
|
+
continue
|
|
158
|
+
try:
|
|
159
|
+
match = re.search(pattern, filename)
|
|
160
|
+
except re.error:
|
|
161
|
+
LOG.warning("invalid naming rule pattern %r, ignoring", pattern)
|
|
162
|
+
continue
|
|
163
|
+
if match:
|
|
164
|
+
try:
|
|
165
|
+
name = match.expand(target) if "\\" in target else target
|
|
166
|
+
except re.error:
|
|
167
|
+
name = target
|
|
168
|
+
return Identification(name, slugify(name), "high", "config-rule", stem)
|
|
169
|
+
|
|
170
|
+
# 3. A JSON config export downloaded alongside the zip.
|
|
171
|
+
sidecar = _sidecar_name(zip_path)
|
|
172
|
+
if sidecar:
|
|
173
|
+
return Identification(sidecar, slugify(sidecar), "high", "sidecar-json", stem)
|
|
174
|
+
|
|
175
|
+
# 4. Cleaned-up filename, preferring an exact match on a known function.
|
|
176
|
+
cleaned = clean_stem(stem, naming.strip_patterns)
|
|
177
|
+
if db is not None and cleaned:
|
|
178
|
+
known = db.get_function_by_name(cleaned, naming.case_insensitive)
|
|
179
|
+
if known:
|
|
180
|
+
return Identification(
|
|
181
|
+
known["name"], known["slug"], "high", "known-function", stem
|
|
182
|
+
)
|
|
183
|
+
if db is not None:
|
|
184
|
+
known = db.get_function_by_name(stem, naming.case_insensitive)
|
|
185
|
+
if known:
|
|
186
|
+
return Identification(known["name"], known["slug"], "high", "known-function", stem)
|
|
187
|
+
|
|
188
|
+
if cleaned and not _looks_meaningless(cleaned):
|
|
189
|
+
return Identification(cleaned, slugify(cleaned), "medium", "filename", stem)
|
|
190
|
+
|
|
191
|
+
# 5. Look inside the archive.
|
|
192
|
+
if naming.infer_from_zip:
|
|
193
|
+
embedded = _zip_embedded_name(zip_path)
|
|
194
|
+
if embedded:
|
|
195
|
+
return Identification(embedded, slugify(embedded), "low", "zip-top-level", stem)
|
|
196
|
+
|
|
197
|
+
# 6. Give up, but stay stable: the same odd filename lands in the same place.
|
|
198
|
+
fallback = cleaned or stem or "unknown"
|
|
199
|
+
if _looks_meaningless(fallback):
|
|
200
|
+
fallback = f"unknown-{short_hash(slugify(stem) or 'x', 8)}"
|
|
201
|
+
return Identification(fallback, slugify(fallback), "low", "fallback", stem)
|
lambda_watcher/ingest.py
ADDED
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
"""The ingest pipeline: one downloaded archive in, one archived version out.
|
|
2
|
+
|
|
3
|
+
Steps, in order:
|
|
4
|
+
|
|
5
|
+
1. hash the archive and check whether this exact download was already handled
|
|
6
|
+
2. work out which Lambda function it belongs to
|
|
7
|
+
3. extract it into a staging directory (safely)
|
|
8
|
+
4. analyse the extracted tree
|
|
9
|
+
5. compare the content hash against the latest version -> skip if unchanged
|
|
10
|
+
6. move staging into its permanent version directory
|
|
11
|
+
7. write ``manifest.json``, index everything in SQLite, mirror into git
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import shutil
|
|
17
|
+
import time
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from datetime import datetime, timezone
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from .analysis import analyse
|
|
23
|
+
from .config import Config
|
|
24
|
+
from .db import Database
|
|
25
|
+
from .extract import ExtractError, extract_zip
|
|
26
|
+
from .gitmirror import GitUnavailable, commit_version, git_available
|
|
27
|
+
from .identify import Identification, identify
|
|
28
|
+
from .notify import notify
|
|
29
|
+
from .store import Store
|
|
30
|
+
from .utils import (
|
|
31
|
+
LOG,
|
|
32
|
+
human_size,
|
|
33
|
+
ref_from_dirname,
|
|
34
|
+
rmtree,
|
|
35
|
+
sha256_file,
|
|
36
|
+
short_hash,
|
|
37
|
+
utc_now_iso,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class IngestResult:
|
|
43
|
+
status: str # new | unchanged | duplicate-download | failed | skipped
|
|
44
|
+
source: Path
|
|
45
|
+
function_name: str | None = None
|
|
46
|
+
seq: int | None = None
|
|
47
|
+
version_dir: Path | None = None
|
|
48
|
+
tree_hash: str | None = None
|
|
49
|
+
identification: Identification | None = None
|
|
50
|
+
message: str = ""
|
|
51
|
+
changed_from: int | None = None
|
|
52
|
+
#: The comparison against the previous version, rendered as it was archived.
|
|
53
|
+
report_path: Path | None = None
|
|
54
|
+
#: How much changed, e.g. "2 added, 2 modified, 3 renamed, 52 vendored".
|
|
55
|
+
change_summary: str | None = None
|
|
56
|
+
#: What that means, e.g. "+24/-5 lines · new: 1 env var, 1 secret". May be empty.
|
|
57
|
+
change_impact: str | None = None
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def ok(self) -> bool:
|
|
61
|
+
return self.status in {"new", "unchanged", "duplicate-download"}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def wait_until_stable(
|
|
65
|
+
path: Path, stable_seconds: float = 2.0, max_wait: float = 900.0, poll: float = 0.5
|
|
66
|
+
) -> bool:
|
|
67
|
+
"""Block until a file stops growing, so we never read a partial download."""
|
|
68
|
+
deadline = time.monotonic() + max_wait
|
|
69
|
+
last_signature: tuple[int, float] | None = None
|
|
70
|
+
stable_since: float | None = None
|
|
71
|
+
|
|
72
|
+
while time.monotonic() < deadline:
|
|
73
|
+
try:
|
|
74
|
+
stat_result = path.stat()
|
|
75
|
+
except FileNotFoundError:
|
|
76
|
+
return False
|
|
77
|
+
except OSError:
|
|
78
|
+
time.sleep(poll)
|
|
79
|
+
continue
|
|
80
|
+
|
|
81
|
+
signature = (stat_result.st_size, stat_result.st_mtime)
|
|
82
|
+
now = time.monotonic()
|
|
83
|
+
if signature == last_signature:
|
|
84
|
+
if stable_since is None:
|
|
85
|
+
stable_since = now
|
|
86
|
+
elif now - stable_since >= stable_seconds:
|
|
87
|
+
# A final open() confirms nothing still holds it exclusively
|
|
88
|
+
# (matters on Windows, harmless elsewhere).
|
|
89
|
+
try:
|
|
90
|
+
with path.open("rb"):
|
|
91
|
+
pass
|
|
92
|
+
except OSError:
|
|
93
|
+
time.sleep(poll)
|
|
94
|
+
continue
|
|
95
|
+
return True
|
|
96
|
+
else:
|
|
97
|
+
last_signature = signature
|
|
98
|
+
stable_since = None
|
|
99
|
+
time.sleep(poll)
|
|
100
|
+
|
|
101
|
+
LOG.warning("gave up waiting for %s to finish downloading", path)
|
|
102
|
+
return False
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def recently_written(mtime: float, max_age: float) -> bool:
|
|
106
|
+
"""Is this file new enough to be a download that just landed?
|
|
107
|
+
|
|
108
|
+
A filesystem event does not mean a file was written. Windows raises
|
|
109
|
+
"modified" for attribute, security and last-access changes too, so an
|
|
110
|
+
antivirus scan, the search indexer or OneDrive rehydrating a file
|
|
111
|
+
re-announces zips that have sat in Downloads for weeks. Their mtime has not
|
|
112
|
+
moved, which is what tells the two apart.
|
|
113
|
+
"""
|
|
114
|
+
if max_age <= 0:
|
|
115
|
+
return True
|
|
116
|
+
return time.time() - mtime <= max_age
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class Ingestor:
|
|
120
|
+
"""Runs the pipeline. Safe to call repeatedly from a single thread."""
|
|
121
|
+
|
|
122
|
+
def __init__(self, cfg: Config, db: Database, store: Store | None = None) -> None:
|
|
123
|
+
self.cfg = cfg
|
|
124
|
+
self.db = db
|
|
125
|
+
self.store = store or Store(cfg)
|
|
126
|
+
|
|
127
|
+
# -- public API ------------------------------------------------------
|
|
128
|
+
def is_candidate(self, path: Path) -> bool:
|
|
129
|
+
"""Cheap filter applied before any I/O-heavy work."""
|
|
130
|
+
name = path.name
|
|
131
|
+
if name.startswith((".", "~$")):
|
|
132
|
+
return False
|
|
133
|
+
suffix = path.suffix.lower()
|
|
134
|
+
if suffix in {s.lower() for s in self.cfg.watch.partial_suffixes}:
|
|
135
|
+
return False
|
|
136
|
+
return suffix in {e.lower() for e in self.cfg.watch.extensions}
|
|
137
|
+
|
|
138
|
+
def ingest(
|
|
139
|
+
self,
|
|
140
|
+
zip_path: Path,
|
|
141
|
+
function_override: str | None = None,
|
|
142
|
+
force: bool = False,
|
|
143
|
+
label: str | None = None,
|
|
144
|
+
*,
|
|
145
|
+
just_downloaded: bool = True,
|
|
146
|
+
) -> IngestResult:
|
|
147
|
+
"""Archive one zip.
|
|
148
|
+
|
|
149
|
+
``just_downloaded`` says this file arrived here rather than having been
|
|
150
|
+
found sitting in place, and only such a file may be cleared out of the
|
|
151
|
+
watched folder afterwards. A startup scan and a backfill pass ``False``:
|
|
152
|
+
neither can tell an overnight download from a zip something merely
|
|
153
|
+
touched.
|
|
154
|
+
"""
|
|
155
|
+
zip_path = Path(zip_path)
|
|
156
|
+
now = utc_now_iso()
|
|
157
|
+
|
|
158
|
+
if not zip_path.exists():
|
|
159
|
+
return IngestResult("failed", zip_path, message="file disappeared before ingest")
|
|
160
|
+
|
|
161
|
+
try:
|
|
162
|
+
zip_sha = sha256_file(zip_path)
|
|
163
|
+
source_stat = zip_path.stat()
|
|
164
|
+
zip_size = source_stat.st_size
|
|
165
|
+
source_mtime = datetime.fromtimestamp(
|
|
166
|
+
source_stat.st_mtime, tz=timezone.utc
|
|
167
|
+
).isoformat(timespec="seconds")
|
|
168
|
+
except OSError as exc:
|
|
169
|
+
return IngestResult("failed", zip_path, message=f"could not read file: {exc}")
|
|
170
|
+
|
|
171
|
+
# Deleting the download is the one step that destroys something outside
|
|
172
|
+
# the archive, so it takes more than a filesystem event: the file has to
|
|
173
|
+
# have arrived, and have been written recently enough to be that arrival.
|
|
174
|
+
may_discard = just_downloaded and recently_written(
|
|
175
|
+
source_stat.st_mtime, self.cfg.watch.arrival_max_age_seconds
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
# 1. Have we already handled this exact download?
|
|
179
|
+
already = self.db.seen_download(zip_sha)
|
|
180
|
+
self.db.mark_download_seen(zip_sha, now, zip_path.name)
|
|
181
|
+
if already and not force:
|
|
182
|
+
self.db.log_event("duplicate-download", now, source_path=str(zip_path),
|
|
183
|
+
detail={"zip_sha256": zip_sha})
|
|
184
|
+
LOG.info("skipping %s: identical download already archived", zip_path.name)
|
|
185
|
+
if may_discard:
|
|
186
|
+
self.store.discard_original(zip_path)
|
|
187
|
+
return IngestResult(
|
|
188
|
+
"duplicate-download", zip_path, message="this exact file was already archived"
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
# 2. Which function is it?
|
|
192
|
+
ident = identify(zip_path, self.cfg.naming, self.db, function_override)
|
|
193
|
+
LOG.info(
|
|
194
|
+
"%s -> function %r (%s, %s confidence)",
|
|
195
|
+
zip_path.name, ident.name, ident.strategy, ident.confidence,
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
# 3. Extract into staging.
|
|
199
|
+
staging = self.store.new_staging_dir(short_hash(zip_sha, 12))
|
|
200
|
+
code_staging = staging / "code"
|
|
201
|
+
try:
|
|
202
|
+
extraction = extract_zip(
|
|
203
|
+
zip_path,
|
|
204
|
+
code_staging,
|
|
205
|
+
max_uncompressed_bytes=self.cfg.store.max_uncompressed_mb * 1024 * 1024,
|
|
206
|
+
max_files=self.cfg.store.max_files,
|
|
207
|
+
strip_wrapper=self.cfg.store.strip_wrapper_dir,
|
|
208
|
+
)
|
|
209
|
+
except ExtractError as exc:
|
|
210
|
+
rmtree(staging)
|
|
211
|
+
self.store.quarantine(zip_path, str(exc))
|
|
212
|
+
self.db.log_event("failed", now, source_path=str(zip_path), detail=str(exc))
|
|
213
|
+
LOG.error("could not extract %s: %s", zip_path.name, exc)
|
|
214
|
+
return IngestResult("failed", zip_path, ident.name, identification=ident, message=str(exc))
|
|
215
|
+
|
|
216
|
+
# A source archive names its wrapper after the ref it was cut from, so
|
|
217
|
+
# the directory we just lifted away is the best label this version will
|
|
218
|
+
# ever get. An explicit --label always wins.
|
|
219
|
+
if label is None and extraction.wrapper_dir:
|
|
220
|
+
label = ref_from_dirname(extraction.wrapper_dir)
|
|
221
|
+
if label:
|
|
222
|
+
LOG.info("labelling this version %s (from %s/)", label, extraction.wrapper_dir)
|
|
223
|
+
|
|
224
|
+
# 4. Analyse.
|
|
225
|
+
try:
|
|
226
|
+
analysis = analyse(code_staging, self.cfg.analysis)
|
|
227
|
+
except Exception as exc: # noqa: BLE001 - analysis must never lose an archive
|
|
228
|
+
rmtree(staging)
|
|
229
|
+
self.db.log_event("failed", now, source_path=str(zip_path), detail=f"analysis: {exc}")
|
|
230
|
+
LOG.exception("analysis failed for %s", zip_path.name)
|
|
231
|
+
return IngestResult("failed", zip_path, ident.name, identification=ident,
|
|
232
|
+
message=f"analysis failed: {exc}")
|
|
233
|
+
|
|
234
|
+
tree_hash = analysis.inventory.tree_hash
|
|
235
|
+
function_id = self.db.upsert_function(ident.name, ident.slug, now)
|
|
236
|
+
function_row = self.db.get_function_by_name(ident.name)
|
|
237
|
+
slug = function_row["slug"] if function_row else ident.slug
|
|
238
|
+
|
|
239
|
+
# 5. Is this content already stored under this function?
|
|
240
|
+
existing = self.db.find_version_by_tree_hash(function_id, tree_hash)
|
|
241
|
+
if existing is not None and not force:
|
|
242
|
+
rmtree(staging)
|
|
243
|
+
self.db.log_event(
|
|
244
|
+
"unchanged", now, function_id=function_id, version_id=existing["id"],
|
|
245
|
+
source_path=str(zip_path), detail={"seq": existing["seq"]},
|
|
246
|
+
)
|
|
247
|
+
LOG.info(
|
|
248
|
+
"%s is byte-identical to %s v%04d - nothing new to archive",
|
|
249
|
+
zip_path.name, ident.name, existing["seq"],
|
|
250
|
+
)
|
|
251
|
+
if may_discard:
|
|
252
|
+
self.store.discard_original(zip_path)
|
|
253
|
+
return IngestResult(
|
|
254
|
+
"unchanged", zip_path, ident.name, int(existing["seq"]),
|
|
255
|
+
self.store.resolve_version_dir(existing["dir"]), tree_hash, ident,
|
|
256
|
+
message=f"identical to version {existing['seq']}",
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
previous = self.db.latest_version(function_id)
|
|
260
|
+
seq = self.db.next_seq(function_id)
|
|
261
|
+
paths = self.store.version_paths(slug, seq, tree_hash)
|
|
262
|
+
|
|
263
|
+
# 6. Promote staging to its permanent home.
|
|
264
|
+
paths.root.parent.mkdir(parents=True, exist_ok=True)
|
|
265
|
+
rmtree(paths.root)
|
|
266
|
+
try:
|
|
267
|
+
shutil.move(str(staging), str(paths.root))
|
|
268
|
+
except OSError as exc:
|
|
269
|
+
rmtree(staging)
|
|
270
|
+
self.db.log_event("failed", now, function_id=function_id, source_path=str(zip_path),
|
|
271
|
+
detail=f"store: {exc}")
|
|
272
|
+
return IngestResult("failed", zip_path, ident.name, identification=ident,
|
|
273
|
+
message=f"could not store version: {exc}")
|
|
274
|
+
|
|
275
|
+
kept_zip = self.store.keep_original(zip_path, paths)
|
|
276
|
+
|
|
277
|
+
# 7. Manifest + index.
|
|
278
|
+
manifest = analysis.to_manifest(
|
|
279
|
+
{
|
|
280
|
+
"function": {"name": ident.name, "slug": slug},
|
|
281
|
+
"version": {"seq": seq, "ingested_at": now, "label": label},
|
|
282
|
+
"source": {
|
|
283
|
+
"filename": zip_path.name,
|
|
284
|
+
"path": str(zip_path),
|
|
285
|
+
"mtime": source_mtime,
|
|
286
|
+
"zip_sha256": zip_sha,
|
|
287
|
+
"zip_size": zip_size,
|
|
288
|
+
"identification": ident.as_dict(),
|
|
289
|
+
"kept_at": self.store.relative(kept_zip) if kept_zip else None,
|
|
290
|
+
},
|
|
291
|
+
"archive": {
|
|
292
|
+
"file_count": extraction.file_count,
|
|
293
|
+
"dir_count": extraction.dir_count,
|
|
294
|
+
"compression_ratio": round(extraction.compression_ratio, 3),
|
|
295
|
+
"skipped_members": extraction.skipped[:50],
|
|
296
|
+
"wrapper_dir": extraction.wrapper_dir,
|
|
297
|
+
},
|
|
298
|
+
"previous_seq": int(previous["seq"]) if previous else None,
|
|
299
|
+
}
|
|
300
|
+
)
|
|
301
|
+
self.store.write_manifest(paths, manifest)
|
|
302
|
+
|
|
303
|
+
version_id = self._index_version(
|
|
304
|
+
function_id=function_id,
|
|
305
|
+
seq=seq,
|
|
306
|
+
tree_hash=tree_hash,
|
|
307
|
+
zip_sha=zip_sha,
|
|
308
|
+
zip_size=zip_size,
|
|
309
|
+
zip_path=zip_path,
|
|
310
|
+
source_mtime=source_mtime,
|
|
311
|
+
now=now,
|
|
312
|
+
version_dir=paths.root,
|
|
313
|
+
analysis=analysis,
|
|
314
|
+
label=label,
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
self.db.log_event(
|
|
318
|
+
"new-version", now, function_id=function_id, version_id=version_id,
|
|
319
|
+
source_path=str(zip_path),
|
|
320
|
+
detail={"seq": seq, "tree_hash": tree_hash, "identification": ident.as_dict()},
|
|
321
|
+
)
|
|
322
|
+
|
|
323
|
+
self._mirror_to_git(slug, paths.code, seq, ident.name, now, tree_hash, zip_path.name)
|
|
324
|
+
self._prune(slug, function_id)
|
|
325
|
+
|
|
326
|
+
report_path, change_summary, change_impact = self._render_report(
|
|
327
|
+
ident.name, slug, function_id, seq, previous
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
if self.cfg.notify.enabled:
|
|
331
|
+
previous_note = f" (was v{previous['seq']:04d})" if previous else ""
|
|
332
|
+
detail = f"{analysis.inventory.file_count} files · {human_size(analysis.inventory.total_size)}"
|
|
333
|
+
if self.cfg.notify.summarise_changes and change_summary:
|
|
334
|
+
detail = f"{change_summary} · {change_impact}" if change_impact else change_summary
|
|
335
|
+
notify(
|
|
336
|
+
f"Lambda archived: {ident.name}",
|
|
337
|
+
f"v{seq:04d}{previous_note} · {detail}",
|
|
338
|
+
enabled=True,
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
LOG.info(
|
|
342
|
+
"archived %s v%04d (%d files, %s)",
|
|
343
|
+
ident.name, seq, analysis.inventory.file_count,
|
|
344
|
+
human_size(analysis.inventory.total_size),
|
|
345
|
+
)
|
|
346
|
+
return IngestResult(
|
|
347
|
+
"new", zip_path, ident.name, seq, paths.root, tree_hash, ident,
|
|
348
|
+
report_path=report_path, change_summary=change_summary, change_impact=change_impact,
|
|
349
|
+
message="archived a new version",
|
|
350
|
+
changed_from=int(previous["seq"]) if previous else None,
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
# -- internals -------------------------------------------------------
|
|
354
|
+
def _index_version(self, **kw) -> int:
|
|
355
|
+
analysis = kw["analysis"]
|
|
356
|
+
with self.db.transaction():
|
|
357
|
+
version_id = self.db.insert_version(
|
|
358
|
+
{
|
|
359
|
+
"function_id": kw["function_id"],
|
|
360
|
+
"seq": kw["seq"],
|
|
361
|
+
"tree_hash": kw["tree_hash"],
|
|
362
|
+
"zip_sha256": kw["zip_sha"],
|
|
363
|
+
"zip_size": kw["zip_size"],
|
|
364
|
+
"source_name": kw["zip_path"].name,
|
|
365
|
+
"source_path": str(kw["zip_path"]),
|
|
366
|
+
"source_mtime": kw["source_mtime"],
|
|
367
|
+
"ingested_at": kw["now"],
|
|
368
|
+
"dir": self.store.relative(kw["version_dir"]),
|
|
369
|
+
"runtime": analysis.runtime.runtime,
|
|
370
|
+
"runtime_confidence": analysis.runtime.confidence,
|
|
371
|
+
"handler": analysis.primary_handler,
|
|
372
|
+
"file_count": analysis.inventory.file_count,
|
|
373
|
+
"total_size": analysis.inventory.total_size,
|
|
374
|
+
"code_file_count": analysis.inventory.code_file_count,
|
|
375
|
+
"code_size": analysis.inventory.code_size,
|
|
376
|
+
"code_lines": analysis.inventory.code_lines,
|
|
377
|
+
"label": kw["label"],
|
|
378
|
+
}
|
|
379
|
+
)
|
|
380
|
+
self.db.bulk_insert(
|
|
381
|
+
"files",
|
|
382
|
+
["version_id", "path", "size", "sha256", "mode", "is_text", "is_vendor", "lang", "lines"],
|
|
383
|
+
[
|
|
384
|
+
(version_id, f.path, f.size, f.sha256, f.mode, int(f.is_text), int(f.is_vendor),
|
|
385
|
+
f.lang, f.lines)
|
|
386
|
+
for f in analysis.inventory.files
|
|
387
|
+
],
|
|
388
|
+
)
|
|
389
|
+
self.db.bulk_insert(
|
|
390
|
+
"deps",
|
|
391
|
+
["version_id", "manager", "name", "version", "source", "is_declared"],
|
|
392
|
+
[(version_id, d.manager, d.name, d.version, d.source, int(d.is_declared))
|
|
393
|
+
for d in analysis.dependencies],
|
|
394
|
+
)
|
|
395
|
+
self.db.bulk_insert(
|
|
396
|
+
"env_vars", ["version_id", "name", "path", "line"],
|
|
397
|
+
[(version_id, e.name, e.path, e.line) for e in analysis.env_vars if not e.is_reserved],
|
|
398
|
+
)
|
|
399
|
+
self.db.bulk_insert(
|
|
400
|
+
"services", ["version_id", "service", "path", "line"],
|
|
401
|
+
[(version_id, s.service, s.path, s.line) for s in analysis.services],
|
|
402
|
+
)
|
|
403
|
+
self.db.bulk_insert(
|
|
404
|
+
"findings", ["version_id", "kind", "severity", "path", "line", "detail", "is_vendor"],
|
|
405
|
+
[(version_id, f.kind, f.severity, f.path, f.line, f.detail, int(f.is_vendor))
|
|
406
|
+
for f in analysis.findings],
|
|
407
|
+
)
|
|
408
|
+
return version_id
|
|
409
|
+
|
|
410
|
+
def _render_report(
|
|
411
|
+
self, name: str, slug: str, function_id: int, seq: int, previous
|
|
412
|
+
) -> tuple[Path | None, str | None, str | None]:
|
|
413
|
+
"""Render the comparison against the previous version, as it is archived.
|
|
414
|
+
|
|
415
|
+
By the time a background watcher archives something, nobody is looking at
|
|
416
|
+
a terminal — so the answer to "what changed?" is written to disk now,
|
|
417
|
+
while the pipeline is already warm, rather than waiting for someone to
|
|
418
|
+
think of asking. ``latest.html`` is the same page under a name that never
|
|
419
|
+
goes stale, so it can be bookmarked.
|
|
420
|
+
|
|
421
|
+
Rendering is a convenience, never a reason to fail an ingest that has
|
|
422
|
+
already succeeded: every failure here is logged and swallowed.
|
|
423
|
+
"""
|
|
424
|
+
if not self.cfg.report.auto_diff or previous is None:
|
|
425
|
+
return None, None, None
|
|
426
|
+
# Deferred: the renderer pulls in the whole presentation layer, and an
|
|
427
|
+
# ingest with nothing to compare against should not pay for the import.
|
|
428
|
+
from .diffing import diff_from_index
|
|
429
|
+
from .diffing.render_html import write_html
|
|
430
|
+
|
|
431
|
+
try:
|
|
432
|
+
current = self.db.get_version(function_id, seq)
|
|
433
|
+
if current is None:
|
|
434
|
+
return None, None, None
|
|
435
|
+
diff = diff_from_index(
|
|
436
|
+
self.db, self.store, self.cfg.diff, name, previous, current,
|
|
437
|
+
include_vendor=True if self.cfg.report.include_vendor else None,
|
|
438
|
+
)
|
|
439
|
+
target_dir = self.cfg.reports_dir / slug
|
|
440
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
441
|
+
target = target_dir / f"v{int(previous['seq']):04d}-v{seq:04d}.html"
|
|
442
|
+
write_html(diff, target)
|
|
443
|
+
write_html(diff, target_dir / "latest.html")
|
|
444
|
+
except Exception as exc: # noqa: BLE001 - never fail an ingest
|
|
445
|
+
LOG.warning("could not render the report for %s v%04d: %s", name, seq, exc)
|
|
446
|
+
return None, None, None
|
|
447
|
+
return target, diff.headline(), diff.impact_line()
|
|
448
|
+
|
|
449
|
+
def _mirror_to_git(
|
|
450
|
+
self, slug: str, code_dir: Path, seq: int, name: str, now: str, tree_hash: str, source: str
|
|
451
|
+
) -> None:
|
|
452
|
+
if not self.cfg.git_mirror.enabled:
|
|
453
|
+
return
|
|
454
|
+
if not git_available():
|
|
455
|
+
LOG.debug("git not on PATH; skipping mirror for %s", name)
|
|
456
|
+
return
|
|
457
|
+
message = (
|
|
458
|
+
f"{name} v{seq:04d}\n\n"
|
|
459
|
+
f"source: {source}\n"
|
|
460
|
+
f"tree-hash: {tree_hash}\n"
|
|
461
|
+
f"ingested: {now}\n"
|
|
462
|
+
)
|
|
463
|
+
try:
|
|
464
|
+
commit_version(
|
|
465
|
+
self.store.repo_dir(slug), code_dir, self.cfg.git_mirror, seq, message, now,
|
|
466
|
+
vendor_globs=self.cfg.analysis.vendor_globs,
|
|
467
|
+
)
|
|
468
|
+
except (GitUnavailable, RuntimeError, OSError) as exc:
|
|
469
|
+
LOG.warning("git mirror failed for %s v%04d: %s", name, seq, exc)
|
|
470
|
+
|
|
471
|
+
def _prune(self, slug: str, function_id: int) -> None:
|
|
472
|
+
"""Drop the oldest versions once a function exceeds the retention limit."""
|
|
473
|
+
keep = self.cfg.store.max_versions_per_function
|
|
474
|
+
if keep <= 0:
|
|
475
|
+
return
|
|
476
|
+
versions = self.db.list_versions(function_id) # newest first
|
|
477
|
+
for row in versions[keep:]:
|
|
478
|
+
rmtree(self.store.resolve_version_dir(row["dir"]))
|
|
479
|
+
self.db.delete_version(int(row["id"]))
|
|
480
|
+
LOG.info("pruned %s v%04d (retention limit %d)", slug, row["seq"], keep)
|