sourcecode 1.31.17__py3-none-any.whl → 1.31.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sourcecode/__init__.py +1 -1
- sourcecode/cache.py +470 -0
- sourcecode/cli.py +17 -13
- {sourcecode-1.31.17.dist-info → sourcecode-1.31.18.dist-info}/METADATA +3 -3
- {sourcecode-1.31.17.dist-info → sourcecode-1.31.18.dist-info}/RECORD +8 -7
- {sourcecode-1.31.17.dist-info → sourcecode-1.31.18.dist-info}/WHEEL +0 -0
- {sourcecode-1.31.17.dist-info → sourcecode-1.31.18.dist-info}/entry_points.txt +0 -0
- {sourcecode-1.31.17.dist-info → sourcecode-1.31.18.dist-info}/licenses/LICENSE +0 -0
sourcecode/__init__.py
CHANGED
sourcecode/cache.py
ADDED
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Snapshot cache manager for sourcecode — v2.
|
|
3
|
+
|
|
4
|
+
Cache layout
|
|
5
|
+
------------
|
|
6
|
+
~/.sourcecode/cache/<repo_id>/
|
|
7
|
+
snapshot-<git_sha>-<flags_hash>.json.gz ← versioned envelope
|
|
8
|
+
cas/
|
|
9
|
+
<blob_hash16>.gz ← content-addressed blobs
|
|
10
|
+
|
|
11
|
+
Schema
|
|
12
|
+
------
|
|
13
|
+
Every snapshot file is a gzip-compressed JSON *envelope*:
|
|
14
|
+
|
|
15
|
+
{
|
|
16
|
+
"sv": "2", // schema version — bump to invalidate all
|
|
17
|
+
"key": "abc1234-aabbccdd", // cache key (git_sha + flags_hash)
|
|
18
|
+
"ts": "2026-05-24T22:00:00Z", // write timestamp (ISO-8601 UTC)
|
|
19
|
+
"fmt": "json", // output format: "json" | "yaml"
|
|
20
|
+
"layers": {"heuristic": "...", ...}, // analyzer fingerprints at write time
|
|
21
|
+
// ── content (one of two forms) ──────────────────────────────────────
|
|
22
|
+
"snap": {...}, // inline fields (small) — JSON mode
|
|
23
|
+
"cas": {"file_paths": "<h16>",…} // large fields deduped into CAS store
|
|
24
|
+
// — OR —
|
|
25
|
+
"raw": "<content string>" // YAML or unparseable JSON stored as-is
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
Content-addressed store (CAS)
|
|
29
|
+
-----------------------------
|
|
30
|
+
Large top-level JSON fields (> _CAS_THRESHOLD bytes) are extracted into the
|
|
31
|
+
``cas/`` directory as individual gzip-compressed blobs identified by a 16-char
|
|
32
|
+
SHA-256 hash of their uncompressed bytes. Two snapshots that share an
|
|
33
|
+
identical ``file_paths`` array reference the *same* blob — zero duplication.
|
|
34
|
+
|
|
35
|
+
Eviction / GC
|
|
36
|
+
-------------
|
|
37
|
+
After each write, ``_gc()`` keeps snapshots from the last
|
|
38
|
+
``SOURCECODE_CACHE_KEEP_COMMITS`` distinct git commits (default 5, override via
|
|
39
|
+
env var). A CAS sweep runs concurrently: blobs unreferenced by any surviving
|
|
40
|
+
snapshot are deleted.
|
|
41
|
+
|
|
42
|
+
Backward compatibility
|
|
43
|
+
----------------------
|
|
44
|
+
v1 files (raw gzip'd content, no envelope) are detected by the absence of an
|
|
45
|
+
``sv`` key in the decompressed JSON, and served transparently. Legacy files
|
|
46
|
+
in ``<repo>/.sourcecode-cache/`` are also checked as a final fallback.
|
|
47
|
+
|
|
48
|
+
Env vars
|
|
49
|
+
--------
|
|
50
|
+
SOURCECODE_CACHE_DIR Override global cache base (default: ~/.sourcecode/cache)
|
|
51
|
+
SOURCECODE_CACHE_KEEP_COMMITS How many git commits to retain (default: 5; 0 = unlimited)
|
|
52
|
+
"""
|
|
53
|
+
from __future__ import annotations
|
|
54
|
+
|
|
55
|
+
import gzip
|
|
56
|
+
import hashlib
|
|
57
|
+
import json
|
|
58
|
+
import os
|
|
59
|
+
import re
|
|
60
|
+
from datetime import datetime, timezone
|
|
61
|
+
from pathlib import Path
|
|
62
|
+
from typing import Any, Optional
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# ---------------------------------------------------------------------------
|
|
66
|
+
# Version / constants
|
|
67
|
+
# ---------------------------------------------------------------------------
|
|
68
|
+
|
|
69
|
+
#: Bump this string to invalidate *all* existing cached snapshots.
|
|
70
|
+
SCHEMA_VERSION: str = "2"
|
|
71
|
+
|
|
72
|
+
#: Fields eligible for CAS deduplication (applied to top-level JSON dict keys).
|
|
73
|
+
_CAS_FIELDS: frozenset[str] = frozenset([
|
|
74
|
+
"file_paths",
|
|
75
|
+
"entry_points",
|
|
76
|
+
"docs",
|
|
77
|
+
"dependencies",
|
|
78
|
+
"graph",
|
|
79
|
+
"semantic_calls",
|
|
80
|
+
"semantic_symbols",
|
|
81
|
+
"architecture",
|
|
82
|
+
"metrics",
|
|
83
|
+
"git_history",
|
|
84
|
+
"env_map",
|
|
85
|
+
"code_notes",
|
|
86
|
+
])
|
|
87
|
+
|
|
88
|
+
#: Serialised size threshold (bytes) above which a field is moved to CAS.
|
|
89
|
+
_CAS_THRESHOLD: int = 4096
|
|
90
|
+
|
|
91
|
+
_DEFAULT_KEEP_COMMITS: int = 5
|
|
92
|
+
|
|
93
|
+
# Matches "snapshot-<hex_commit>-<hex_flags>.json.gz"
|
|
94
|
+
_SNAPSHOT_RE = re.compile(r"^snapshot-([0-9a-f]+)-[0-9a-f]+\.json\.gz$")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
# ---------------------------------------------------------------------------
|
|
98
|
+
# Public API — location helpers
|
|
99
|
+
# ---------------------------------------------------------------------------
|
|
100
|
+
|
|
101
|
+
def repo_id(repo_root: Path) -> str:
|
|
102
|
+
"""Stable 16-char hex identifier derived from the canonical repo path."""
|
|
103
|
+
return hashlib.sha256(str(repo_root.resolve()).encode()).hexdigest()[:16]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def cache_dir(repo_root: Path) -> Path:
|
|
107
|
+
"""
|
|
108
|
+
Return the per-repo cache directory (``~/.sourcecode/cache/<repo_id>/``).
|
|
109
|
+
|
|
110
|
+
Override the base via ``SOURCECODE_CACHE_DIR``.
|
|
111
|
+
"""
|
|
112
|
+
env_base = os.environ.get("SOURCECODE_CACHE_DIR", "")
|
|
113
|
+
base: Path = Path(env_base) if env_base else Path.home() / ".sourcecode" / "cache"
|
|
114
|
+
return base / repo_id(repo_root)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# ---------------------------------------------------------------------------
|
|
118
|
+
# Public API — read / write
|
|
119
|
+
# ---------------------------------------------------------------------------
|
|
120
|
+
|
|
121
|
+
def read(repo_root: Path, cache_key: str) -> Optional[str]:
|
|
122
|
+
"""
|
|
123
|
+
Return the cached snapshot string for *cache_key*, or ``None`` on miss.
|
|
124
|
+
|
|
125
|
+
Lookup order:
|
|
126
|
+
1. ``<cache_dir>/snapshot-<cache_key>.json.gz`` — v2 envelope (new)
|
|
127
|
+
2. ``<repo_root>/.sourcecode-cache/snapshot-<cache_key>.json`` — legacy
|
|
128
|
+
"""
|
|
129
|
+
cache_d = cache_dir(repo_root)
|
|
130
|
+
|
|
131
|
+
# ── 1. Global location (.json.gz, v2 envelope or v1 raw) ───────────────
|
|
132
|
+
gz_path = cache_d / f"snapshot-{cache_key}.json.gz"
|
|
133
|
+
if gz_path.exists():
|
|
134
|
+
try:
|
|
135
|
+
result = _parse_envelope(gz_path.read_bytes(), cache_d)
|
|
136
|
+
if result is not None:
|
|
137
|
+
return result
|
|
138
|
+
except Exception:
|
|
139
|
+
pass
|
|
140
|
+
_safe_unlink(gz_path) # corrupted or version mismatch — evict
|
|
141
|
+
return None
|
|
142
|
+
|
|
143
|
+
# ── 2. Legacy location (<repo>/.sourcecode-cache/*.json) ───────────────
|
|
144
|
+
legacy = repo_root / ".sourcecode-cache" / f"snapshot-{cache_key}.json"
|
|
145
|
+
if legacy.exists():
|
|
146
|
+
try:
|
|
147
|
+
return legacy.read_text(encoding="utf-8")
|
|
148
|
+
except Exception:
|
|
149
|
+
return None
|
|
150
|
+
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def write(
|
|
155
|
+
repo_root: Path,
|
|
156
|
+
cache_key: str,
|
|
157
|
+
content: str,
|
|
158
|
+
*,
|
|
159
|
+
fmt: str = "json",
|
|
160
|
+
layers: Optional[dict[str, str]] = None,
|
|
161
|
+
) -> None:
|
|
162
|
+
"""
|
|
163
|
+
Persist *content* as a versioned, optionally CAS-deduped snapshot.
|
|
164
|
+
|
|
165
|
+
Parameters
|
|
166
|
+
----------
|
|
167
|
+
repo_root : Path
|
|
168
|
+
Root directory of the analysed repository.
|
|
169
|
+
cache_key : str
|
|
170
|
+
``"{git_sha}-{flags_hash}"`` identifying this analysis.
|
|
171
|
+
content : str
|
|
172
|
+
Final rendered output (JSON or YAML string).
|
|
173
|
+
fmt : str
|
|
174
|
+
``"json"`` or ``"yaml"`` — determines whether CAS extraction applies.
|
|
175
|
+
layers : dict[str, str], optional
|
|
176
|
+
Analyzer fingerprints (from ``_compute_analyzer_fingerprints()``).
|
|
177
|
+
Stored in the envelope for future layer-aware reuse.
|
|
178
|
+
|
|
179
|
+
Writes are always best-effort: any failure is silently swallowed.
|
|
180
|
+
"""
|
|
181
|
+
cache_d = cache_dir(repo_root)
|
|
182
|
+
dest = cache_d / f"snapshot-{cache_key}.json.gz"
|
|
183
|
+
try:
|
|
184
|
+
cache_d.mkdir(parents=True, exist_ok=True)
|
|
185
|
+
payload = _build_envelope(cache_key, content, fmt, layers or {}, cache_d)
|
|
186
|
+
dest.write_bytes(payload)
|
|
187
|
+
except Exception:
|
|
188
|
+
return # non-fatal
|
|
189
|
+
|
|
190
|
+
_gc(cache_d)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
# ---------------------------------------------------------------------------
|
|
194
|
+
# Envelope (de)serialisation
|
|
195
|
+
# ---------------------------------------------------------------------------
|
|
196
|
+
|
|
197
|
+
def _now_iso() -> str:
|
|
198
|
+
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _build_envelope(
|
|
202
|
+
cache_key: str,
|
|
203
|
+
content: str,
|
|
204
|
+
fmt: str,
|
|
205
|
+
layers: dict[str, str],
|
|
206
|
+
cache_d: Path,
|
|
207
|
+
) -> bytes:
|
|
208
|
+
"""Build a versioned envelope and return gzip-compressed bytes."""
|
|
209
|
+
envelope: dict[str, Any] = {
|
|
210
|
+
"sv": SCHEMA_VERSION,
|
|
211
|
+
"key": cache_key,
|
|
212
|
+
"ts": _now_iso(),
|
|
213
|
+
"fmt": fmt,
|
|
214
|
+
"layers": layers,
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
if fmt == "json":
|
|
218
|
+
# Try to parse and extract large fields into CAS
|
|
219
|
+
try:
|
|
220
|
+
snap_dict = json.loads(content)
|
|
221
|
+
if isinstance(snap_dict, dict):
|
|
222
|
+
inline, cas_refs = _cas_extract(snap_dict, cache_d)
|
|
223
|
+
envelope["snap"] = inline
|
|
224
|
+
if cas_refs:
|
|
225
|
+
envelope["cas"] = cas_refs
|
|
226
|
+
else:
|
|
227
|
+
# JSON array or primitive — store as-is
|
|
228
|
+
envelope["raw"] = content
|
|
229
|
+
except Exception:
|
|
230
|
+
envelope["raw"] = content
|
|
231
|
+
else:
|
|
232
|
+
# YAML or unknown format — store raw string
|
|
233
|
+
envelope["raw"] = content
|
|
234
|
+
|
|
235
|
+
return gzip.compress(
|
|
236
|
+
json.dumps(envelope, ensure_ascii=False).encode("utf-8"),
|
|
237
|
+
compresslevel=6,
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _parse_envelope(data: bytes, cache_d: Path) -> Optional[str]:
|
|
242
|
+
"""
|
|
243
|
+
Decompress *data*, parse envelope, resolve CAS refs, return content string.
|
|
244
|
+
|
|
245
|
+
Returns ``None`` on schema version mismatch, CAS miss, or parse failure.
|
|
246
|
+
v1 files (no envelope wrapper) are detected and served transparently.
|
|
247
|
+
"""
|
|
248
|
+
try:
|
|
249
|
+
raw_bytes = gzip.decompress(data)
|
|
250
|
+
except Exception:
|
|
251
|
+
return None
|
|
252
|
+
|
|
253
|
+
# ── v1 detection ────────────────────────────────────────────────────────
|
|
254
|
+
# v1 stored the content string directly (gzip'd UTF-8), not an envelope.
|
|
255
|
+
# Heuristic: if decompressed bytes are not a JSON object with an "sv" key,
|
|
256
|
+
# treat as v1 and return the raw bytes as the content string.
|
|
257
|
+
try:
|
|
258
|
+
envelope = json.loads(raw_bytes.decode("utf-8"))
|
|
259
|
+
except Exception:
|
|
260
|
+
# Not JSON at all (e.g. YAML v1) — return as-is
|
|
261
|
+
try:
|
|
262
|
+
return raw_bytes.decode("utf-8")
|
|
263
|
+
except Exception:
|
|
264
|
+
return None
|
|
265
|
+
|
|
266
|
+
if not isinstance(envelope, dict) or envelope.get("sv") != SCHEMA_VERSION:
|
|
267
|
+
# dict without "sv" → v1 JSON snapshot; non-matching sv → old envelope
|
|
268
|
+
# Serve v1 transparently; reject mismatched schema versions as a miss.
|
|
269
|
+
if isinstance(envelope, dict) and "sv" in envelope:
|
|
270
|
+
return None # schema version mismatch
|
|
271
|
+
# No "sv" at all → v1 format, raw content
|
|
272
|
+
return raw_bytes.decode("utf-8")
|
|
273
|
+
|
|
274
|
+
# ── v2 envelope ─────────────────────────────────────────────────────────
|
|
275
|
+
if "raw" in envelope:
|
|
276
|
+
return envelope["raw"]
|
|
277
|
+
|
|
278
|
+
if "snap" in envelope:
|
|
279
|
+
inline: dict[str, Any] = envelope["snap"]
|
|
280
|
+
cas_refs: dict[str, str] = envelope.get("cas", {})
|
|
281
|
+
if cas_refs:
|
|
282
|
+
restored = _cas_restore(inline, cas_refs, cache_d)
|
|
283
|
+
if restored is None:
|
|
284
|
+
return None # CAS miss (blob evicted or corrupted)
|
|
285
|
+
else:
|
|
286
|
+
restored = dict(inline)
|
|
287
|
+
# Re-serialise with the same parameters used by the pipeline.
|
|
288
|
+
# json.loads → json.dumps round-trips correctly: Python 3.7+ preserves
|
|
289
|
+
# dict insertion order and the pipeline uses indent=2, ensure_ascii=False.
|
|
290
|
+
return json.dumps(restored, indent=2, ensure_ascii=False)
|
|
291
|
+
|
|
292
|
+
return None # malformed envelope
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
# ---------------------------------------------------------------------------
|
|
296
|
+
# CAS store
|
|
297
|
+
# ---------------------------------------------------------------------------
|
|
298
|
+
|
|
299
|
+
def _cas_dir(cache_d: Path) -> Path:
|
|
300
|
+
return cache_d / "cas"
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _cas_path(cache_d: Path, blob_hash: str) -> Path:
|
|
304
|
+
return _cas_dir(cache_d) / f"{blob_hash}.gz"
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _cas_store_blob(cache_d: Path, serialised: str) -> str:
|
|
308
|
+
"""
|
|
309
|
+
Store *serialised* (a JSON string) in the CAS. Idempotent.
|
|
310
|
+
|
|
311
|
+
Returns the 16-char SHA-256 hex hash that identifies the blob.
|
|
312
|
+
"""
|
|
313
|
+
raw = serialised.encode("utf-8")
|
|
314
|
+
blob_hash = hashlib.sha256(raw).hexdigest()[:16]
|
|
315
|
+
path = _cas_path(cache_d, blob_hash)
|
|
316
|
+
if not path.exists():
|
|
317
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
318
|
+
path.write_bytes(gzip.compress(raw, compresslevel=6))
|
|
319
|
+
return blob_hash
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _cas_load_blob(cache_d: Path, blob_hash: str) -> Optional[str]:
|
|
323
|
+
"""Return the stored JSON string for *blob_hash*, or ``None`` if absent."""
|
|
324
|
+
path = _cas_path(cache_d, blob_hash)
|
|
325
|
+
if not path.exists():
|
|
326
|
+
return None
|
|
327
|
+
try:
|
|
328
|
+
return gzip.decompress(path.read_bytes()).decode("utf-8")
|
|
329
|
+
except Exception:
|
|
330
|
+
return None
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _cas_extract(
|
|
334
|
+
snap_dict: dict[str, Any],
|
|
335
|
+
cache_d: Path,
|
|
336
|
+
) -> tuple[dict[str, Any], dict[str, str]]:
|
|
337
|
+
"""
|
|
338
|
+
Walk *snap_dict* top-level fields. Fields that:
|
|
339
|
+
- are in ``_CAS_FIELDS``
|
|
340
|
+
- serialise to more than ``_CAS_THRESHOLD`` bytes
|
|
341
|
+
|
|
342
|
+
… are stored as CAS blobs and replaced with their hash in the returned
|
|
343
|
+
``cas_refs`` mapping. Other fields remain inline.
|
|
344
|
+
"""
|
|
345
|
+
inline: dict[str, Any] = {}
|
|
346
|
+
cas_refs: dict[str, str] = {}
|
|
347
|
+
|
|
348
|
+
for key, value in snap_dict.items():
|
|
349
|
+
if key in _CAS_FIELDS and value is not None:
|
|
350
|
+
serialised = json.dumps(value, ensure_ascii=False)
|
|
351
|
+
if len(serialised.encode("utf-8")) > _CAS_THRESHOLD:
|
|
352
|
+
blob_hash = _cas_store_blob(cache_d, serialised)
|
|
353
|
+
cas_refs[key] = blob_hash
|
|
354
|
+
continue
|
|
355
|
+
inline[key] = value
|
|
356
|
+
|
|
357
|
+
return inline, cas_refs
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _cas_restore(
|
|
361
|
+
inline: dict[str, Any],
|
|
362
|
+
cas_refs: dict[str, str],
|
|
363
|
+
cache_d: Path,
|
|
364
|
+
) -> Optional[dict[str, Any]]:
|
|
365
|
+
"""
|
|
366
|
+
Reconstruct a full snapshot dict by loading CAS blobs for *cas_refs*.
|
|
367
|
+
|
|
368
|
+
Returns ``None`` if any blob is missing (treat as cache miss).
|
|
369
|
+
"""
|
|
370
|
+
result: dict[str, Any] = dict(inline)
|
|
371
|
+
for field, blob_hash in cas_refs.items():
|
|
372
|
+
blob_str = _cas_load_blob(cache_d, blob_hash)
|
|
373
|
+
if blob_str is None:
|
|
374
|
+
return None # blob evicted or corrupted → full miss
|
|
375
|
+
try:
|
|
376
|
+
result[field] = json.loads(blob_str)
|
|
377
|
+
except Exception:
|
|
378
|
+
return None
|
|
379
|
+
return result
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
# ---------------------------------------------------------------------------
|
|
383
|
+
# Eviction / GC
|
|
384
|
+
# ---------------------------------------------------------------------------
|
|
385
|
+
|
|
386
|
+
def _gc(cache_d: Path) -> None:
|
|
387
|
+
"""
|
|
388
|
+
Evict old snapshots and sweep orphaned CAS blobs.
|
|
389
|
+
|
|
390
|
+
Keeps snapshots from the last ``SOURCECODE_CACHE_KEEP_COMMITS`` distinct
|
|
391
|
+
git commits (determined by mtime of files in each commit group).
|
|
392
|
+
"""
|
|
393
|
+
keep = int(os.environ.get("SOURCECODE_CACHE_KEEP_COMMITS", _DEFAULT_KEEP_COMMITS))
|
|
394
|
+
|
|
395
|
+
try:
|
|
396
|
+
all_snapshots = list(cache_d.glob("snapshot-*.json.gz"))
|
|
397
|
+
if not all_snapshots:
|
|
398
|
+
return
|
|
399
|
+
|
|
400
|
+
# Group snapshot files by commit SHA
|
|
401
|
+
groups: dict[str, list[Path]] = {}
|
|
402
|
+
for f in all_snapshots:
|
|
403
|
+
m = _SNAPSHOT_RE.match(f.name)
|
|
404
|
+
if m:
|
|
405
|
+
groups.setdefault(m.group(1), []).append(f)
|
|
406
|
+
|
|
407
|
+
surviving: list[Path]
|
|
408
|
+
|
|
409
|
+
if keep <= 0 or len(groups) <= keep:
|
|
410
|
+
# No eviction needed — but still sweep CAS
|
|
411
|
+
surviving = all_snapshots
|
|
412
|
+
else:
|
|
413
|
+
def _newest_mtime(commit: str) -> float:
|
|
414
|
+
return max(p.stat().st_mtime for p in groups[commit])
|
|
415
|
+
|
|
416
|
+
sorted_commits = sorted(groups, key=_newest_mtime, reverse=True)
|
|
417
|
+
surviving = []
|
|
418
|
+
for i, commit in enumerate(sorted_commits):
|
|
419
|
+
if i < keep:
|
|
420
|
+
surviving.extend(groups[commit])
|
|
421
|
+
else:
|
|
422
|
+
for f in groups[commit]:
|
|
423
|
+
_safe_unlink(f)
|
|
424
|
+
|
|
425
|
+
_gc_cas(cache_d, surviving)
|
|
426
|
+
|
|
427
|
+
except Exception:
|
|
428
|
+
pass # GC failure is non-fatal
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _gc_cas(cache_d: Path, surviving_snapshots: list[Path]) -> None:
|
|
432
|
+
"""
|
|
433
|
+
Delete CAS blobs not referenced by any snapshot in *surviving_snapshots*.
|
|
434
|
+
|
|
435
|
+
Walks each snapshot's ``cas`` dict to collect live hashes; deletes the rest.
|
|
436
|
+
"""
|
|
437
|
+
cas_d = _cas_dir(cache_d)
|
|
438
|
+
if not cas_d.exists():
|
|
439
|
+
return
|
|
440
|
+
|
|
441
|
+
try:
|
|
442
|
+
# Collect all hashes referenced by surviving snapshots
|
|
443
|
+
referenced: set[str] = set()
|
|
444
|
+
for snap_path in surviving_snapshots:
|
|
445
|
+
try:
|
|
446
|
+
raw = gzip.decompress(snap_path.read_bytes())
|
|
447
|
+
env = json.loads(raw.decode("utf-8"))
|
|
448
|
+
if isinstance(env, dict) and "cas" in env:
|
|
449
|
+
referenced.update(env["cas"].values())
|
|
450
|
+
except Exception:
|
|
451
|
+
pass # unreadable snapshot — conservatively keep its blobs unknown
|
|
452
|
+
|
|
453
|
+
# Delete blobs not referenced by any surviving snapshot
|
|
454
|
+
for blob in cas_d.glob("*.gz"):
|
|
455
|
+
if blob.stem not in referenced:
|
|
456
|
+
_safe_unlink(blob)
|
|
457
|
+
|
|
458
|
+
except Exception:
|
|
459
|
+
pass # CAS sweep failure is non-fatal
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
# ---------------------------------------------------------------------------
|
|
463
|
+
# Utilities
|
|
464
|
+
# ---------------------------------------------------------------------------
|
|
465
|
+
|
|
466
|
+
def _safe_unlink(path: Path) -> None:
|
|
467
|
+
try:
|
|
468
|
+
path.unlink(missing_ok=True)
|
|
469
|
+
except Exception:
|
|
470
|
+
pass
|
sourcecode/cli.py
CHANGED
|
@@ -876,14 +876,16 @@ def main(
|
|
|
876
876
|
architecture = True # agents need full architectural signal (M4)
|
|
877
877
|
graph_modules = True # IC-003: import graph needed for architecture confidence
|
|
878
878
|
|
|
879
|
-
# ── GAP-9: Cache check — serve from
|
|
879
|
+
# ── GAP-9: Cache check — serve from global cache when git SHA unchanged ──
|
|
880
|
+
# Cache is stored in ~/.sourcecode/cache/<repo_id>/ (outside the repo).
|
|
881
|
+
# Snapshots are gzip-compressed (.json.gz) — ~85 % smaller than plain JSON.
|
|
882
|
+
# Eviction keeps the last SOURCECODE_CACHE_KEEP_COMMITS commits (default 5).
|
|
880
883
|
import hashlib as _hashlib
|
|
881
884
|
import subprocess as _sub
|
|
882
|
-
|
|
885
|
+
from sourcecode import cache as _cache_mod
|
|
883
886
|
_cache_hit_content: Optional[str] = None
|
|
884
887
|
_git_sha = ""
|
|
885
888
|
_cache_key = ""
|
|
886
|
-
_cache_file: Optional[Path] = None
|
|
887
889
|
if not no_cache:
|
|
888
890
|
try:
|
|
889
891
|
_sha_r = _sub.run(
|
|
@@ -921,13 +923,10 @@ def main(
|
|
|
921
923
|
)
|
|
922
924
|
_flags_h = _hashlib.md5(_flags_str.encode()).hexdigest()[:8]
|
|
923
925
|
_cache_key = f"{_git_sha}-{_flags_h}"
|
|
924
|
-
|
|
925
|
-
if _cache_file.exists():
|
|
926
|
-
_cache_hit_content = _cache_file.read_text(encoding="utf-8")
|
|
926
|
+
_cache_hit_content = _cache_mod.read(target, _cache_key)
|
|
927
927
|
except Exception:
|
|
928
928
|
_git_sha = ""
|
|
929
929
|
_cache_key = ""
|
|
930
|
-
_cache_file = None
|
|
931
930
|
|
|
932
931
|
if _cache_hit_content is not None:
|
|
933
932
|
from sourcecode.serializer import write_output
|
|
@@ -1762,12 +1761,17 @@ def main(
|
|
|
1762
1761
|
write_output(content, output=output)
|
|
1763
1762
|
|
|
1764
1763
|
# GAP-9: Persist to cache for future identical runs (git SHA unchanged)
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1764
|
+
# Writes versioned envelope to ~/.sourcecode/cache/<repo_id>/<key>.json.gz.
|
|
1765
|
+
# Large JSON fields are extracted into shared CAS blobs (deduplication).
|
|
1766
|
+
# GC runs inline after each write (keep last N commits + CAS sweep).
|
|
1767
|
+
if not no_cache and _cache_key and not _pipeline_error:
|
|
1768
|
+
_cache_mod.write(
|
|
1769
|
+
target,
|
|
1770
|
+
_cache_key,
|
|
1771
|
+
content,
|
|
1772
|
+
fmt=format,
|
|
1773
|
+
layers=_compute_analyzer_fingerprints(),
|
|
1774
|
+
)
|
|
1771
1775
|
|
|
1772
1776
|
if _pipeline_error:
|
|
1773
1777
|
raise typer.Exit(code=2)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sourcecode
|
|
3
|
-
Version: 1.31.
|
|
3
|
+
Version: 1.31.18
|
|
4
4
|
Summary: Deterministic codebase context for AI coding agents
|
|
5
5
|
License: Apache License
|
|
6
6
|
Version 2.0, January 2004
|
|
@@ -225,7 +225,7 @@ Description-Content-Type: text/markdown
|
|
|
225
225
|
|
|
226
226
|
**AI-ready change intelligence for Java/Spring enterprise monoliths.**
|
|
227
227
|
|
|
228
|
-

|
|
229
229
|

|
|
230
230
|
|
|
231
231
|
---
|
|
@@ -263,7 +263,7 @@ pipx install sourcecode
|
|
|
263
263
|
|
|
264
264
|
```bash
|
|
265
265
|
sourcecode version
|
|
266
|
-
# sourcecode 1.31.
|
|
266
|
+
# sourcecode 1.31.18
|
|
267
267
|
```
|
|
268
268
|
|
|
269
269
|
---
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
sourcecode/__init__.py,sha256=
|
|
1
|
+
sourcecode/__init__.py,sha256=RKBkTCXd0nPibD6uZj_CLNSWfxJYQOS-gsplP4C8K_g,104
|
|
2
2
|
sourcecode/adaptive_scanner.py,sha256=XffluXKzJUXrMtjEiAOnSNPZnztdIcts17T9ouHeID0,10521
|
|
3
3
|
sourcecode/architecture_analyzer.py,sha256=4R13Yb02OrPeB4IH3z6V_g7HWhmGcRHbI8CobCVnRrc,39111
|
|
4
4
|
sourcecode/architecture_summary.py,sha256=z34_6v7cSwy98cof2UVciGho7SCrZ93tiqMmq5WNzRQ,20405
|
|
5
5
|
sourcecode/ast_extractor.py,sha256=XgrZg2DcWcUm9r87cRG3KGO7IK2TIL_N-CvhSbUmmh4,49901
|
|
6
|
+
sourcecode/cache.py,sha256=HDkUZqXOovBc1PjTg-JpOQlyKhUMmEhiG789R7L4Wms,16348
|
|
6
7
|
sourcecode/canonical_ir.py,sha256=NZu0XICv__hkQGKzW2LNQLRqb1L28K2p_WQCQKS5Zlk,23141
|
|
7
8
|
sourcecode/classifier.py,sha256=yWeq6agTjkFa3zuNa-gdVIHtjoBoPoVlJnX-b7tdVJs,7851
|
|
8
|
-
sourcecode/cli.py,sha256=
|
|
9
|
+
sourcecode/cli.py,sha256=zBJZqoOntf3m4UWqvixrNdSDdytevuYJF4rDvxXTM8k,139621
|
|
9
10
|
sourcecode/code_notes_analyzer.py,sha256=EJemNCNc9Dn-1RZYu-aNbK0ELzmsyC4s6FdHi3XyNEI,9392
|
|
10
11
|
sourcecode/confidence_analyzer.py,sha256=_jckZSxksV-OU38vbkxfVNBnWCtlCq8Vwfg23x1uspA,19054
|
|
11
12
|
sourcecode/context_scorer.py,sha256=QpChSpsmaAYz91rXA4Ue5xzQmNz_ZboZN09YOHScq1U,14679
|
|
@@ -75,8 +76,8 @@ sourcecode/telemetry/consent.py,sha256=wLMvGNJeSSyZoNkQXpoUioY6mMv4Qdvuw7S9jAEWn
|
|
|
75
76
|
sourcecode/telemetry/events.py,sha256=oEvvulfsv5GIDWG2174gSS6tNB95w38AIYiYeifGKlE,2294
|
|
76
77
|
sourcecode/telemetry/filters.py,sha256=Asa71oRl7q3Wt_FMwuufIZJFzSYdgRNKS8LHCIyFeYE,4805
|
|
77
78
|
sourcecode/telemetry/transport.py,sha256=KJeIPCPWMdmbCP3ySGs2iUlia34U6vWne2dZsUezesw,1560
|
|
78
|
-
sourcecode-1.31.
|
|
79
|
-
sourcecode-1.31.
|
|
80
|
-
sourcecode-1.31.
|
|
81
|
-
sourcecode-1.31.
|
|
82
|
-
sourcecode-1.31.
|
|
79
|
+
sourcecode-1.31.18.dist-info/METADATA,sha256=paObgQ32RFOKlwHD7oyNK6tRtbEBRStsmeXXSg4RaPw,31103
|
|
80
|
+
sourcecode-1.31.18.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
81
|
+
sourcecode-1.31.18.dist-info/entry_points.txt,sha256=ex3F9rmbXeyDIoFQHtkEqTsKSaJow8F0LrVu8XfIktQ,57
|
|
82
|
+
sourcecode-1.31.18.dist-info/licenses/LICENSE,sha256=7DdHrU9Z_3e7dSvq4ISijZNjnuHo5NIHNiHDouMQ9JU,10491
|
|
83
|
+
sourcecode-1.31.18.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|