bruriah 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bruriah/__init__.py +30 -0
- bruriah/approvals.py +174 -0
- bruriah/audit.py +93 -0
- bruriah/cache.py +320 -0
- bruriah/candidates.py +236 -0
- bruriah/classify.py +179 -0
- bruriah/cli.py +685 -0
- bruriah/clients.py +319 -0
- bruriah/context.py +331 -0
- bruriah/contracts.py +202 -0
- bruriah/corpus.py +191 -0
- bruriah/data/practices-pack.json +1 -0
- bruriah/data/practices-pack.manifest.json +8 -0
- bruriah/data/programming-policy.json +72 -0
- bruriah/data/programming-policy.manifest.json +8 -0
- bruriah/data/project-memory-policy.json +76 -0
- bruriah/data/project-memory-policy.manifest.json +8 -0
- bruriah/data/research-policy.json +1 -0
- bruriah/data/research-policy.manifest.json +8 -0
- bruriah/data/skills/falsifiability-probe/SKILL.md +50 -0
- bruriah/data/skills/find-the-time-bomb/SKILL.md +46 -0
- bruriah/data/skills/make-it-inexpressible/SKILL.md +41 -0
- bruriah/data/skills/preserve-behaviour-when-refactoring/SKILL.md +39 -0
- bruriah/data/skills/undiscoverable-is-unbuilt/SKILL.md +39 -0
- bruriah/data/skills/verify-before-asserting/SKILL.md +39 -0
- bruriah/data/trust-roots.json +3 -0
- bruriah/dispatch.py +137 -0
- bruriah/evaluation.py +757 -0
- bruriah/evidence.py +342 -0
- bruriah/fetch.py +388 -0
- bruriah/gitcorpus.py +79 -0
- bruriah/index.py +595 -0
- bruriah/language.py +86 -0
- bruriah/lookup.py +212 -0
- bruriah/mcp_server.py +175 -0
- bruriah/models.py +33 -0
- bruriah/packs.py +251 -0
- bruriah/platform.py +287 -0
- bruriah/pointer.py +158 -0
- bruriah/registries.py +32 -0
- bruriah/research.py +325 -0
- bruriah/retrieval.py +387 -0
- bruriah/route.py +162 -0
- bruriah/service.py +595 -0
- bruriah/signing.py +145 -0
- bruriah/skills.py +305 -0
- bruriah/skillset.py +624 -0
- bruriah-0.1.0.dist-info/METADATA +315 -0
- bruriah-0.1.0.dist-info/RECORD +53 -0
- bruriah-0.1.0.dist-info/WHEEL +4 -0
- bruriah-0.1.0.dist-info/entry_points.txt +3 -0
- bruriah-0.1.0.dist-info/licenses/LICENSE +201 -0
- bruriah-0.1.0.dist-info/licenses/NOTICE +21 -0
bruriah/__init__.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Bruriah: a read-only, two-tool MCP knowledge router over your own corpus."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import importlib.util
|
|
5
|
+
|
|
6
|
+
__version__ = "0.1.0"
|
|
7
|
+
|
|
8
|
+
# Bruriah needs POSIX file primitives and there is no partial mode without them. Activation swaps a
|
|
9
|
+
# pointer under `flock` with `O_NOFOLLOW` and re-confirms the file's identity afterwards, which is
|
|
10
|
+
# what makes promoting a snapshot atomic and unspoofable; Windows has different primitives with
|
|
11
|
+
# different semantics, and a half-ported version of that guarantee is worse than none, because it
|
|
12
|
+
# would fail silently instead of loudly.
|
|
13
|
+
#
|
|
14
|
+
# The wheel is `py3-none-any`, so pip installs it happily on Windows and the first command then
|
|
15
|
+
# died with a bare `ModuleNotFoundError: No module named 'fcntl'` raised several frames deep. That
|
|
16
|
+
# reads as a broken package rather than an unsupported platform. Detecting the capability rather
|
|
17
|
+
# than checking `sys.platform` keeps this honest about what is actually required.
|
|
18
|
+
if importlib.util.find_spec("fcntl") is None: # pragma: no cover -- POSIX-only by construction
|
|
19
|
+
raise ImportError(
|
|
20
|
+
"Bruriah requires macOS or Linux: it depends on POSIX file locking (fcntl), which this "
|
|
21
|
+
"platform does not provide.\n"
|
|
22
|
+
"\n"
|
|
23
|
+
"On Windows, run it under WSL -- that is a real Linux and needs no changes:\n"
|
|
24
|
+
" wsl --install (once, then reopen your terminal)\n"
|
|
25
|
+
" pip install bruriah (inside WSL)\n"
|
|
26
|
+
"\n"
|
|
27
|
+
"Native Windows support would mean reimplementing the atomic pointer swap that makes "
|
|
28
|
+
"activation safe, and that is deliberately not done half-way. If you want it, open an "
|
|
29
|
+
"issue saying so -- knowing someone is waiting is what would make it worth doing."
|
|
30
|
+
)
|
bruriah/approvals.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import tempfile
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import date
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from .candidates import AnalysisReport, CandidateError, analyze_candidate
|
|
12
|
+
|
|
13
|
+
# Human approval: the second of the two STRONG axes in this project's trust model, the other being
|
|
14
|
+
# the permission envelope. A signature says who published a skill; only this says a person read it.
|
|
15
|
+
#
|
|
16
|
+
# Two properties this module exists to hold, both enforced rather than documented:
|
|
17
|
+
#
|
|
18
|
+
# 1. APPROVAL IS BOUND TO A DIGEST, never to a name or a version. `load_approvals` returns
|
|
19
|
+
# `skill_id -> body_digest`, which `skillset.validate_skillset` re-checks at every activation.
|
|
20
|
+
# Editing an approved body does not merely invalidate a record filed away somewhere; it makes the
|
|
21
|
+
# whole skill set fail to activate.
|
|
22
|
+
#
|
|
23
|
+
# 2. EVERY ADVISORY MUST BE ACKNOWLEDGED INDIVIDUALLY. Not a count, not a blanket flag, not "I read
|
|
24
|
+
# them all" -- each finding by its own identifier. A single "yes" over a list is the interface
|
|
25
|
+
# equivalent of a checkbox nobody reads, and it is exactly the affordance that turns a review gate
|
|
26
|
+
# into a formality.
|
|
27
|
+
#
|
|
28
|
+
# Approval RE-RUNS the analysis from the candidate file rather than accepting a report from the
|
|
29
|
+
# caller. A report passed in is a report that can be fabricated, and the one place that must not be
|
|
30
|
+
# possible is the gate that grants trust.
|
|
31
|
+
|
|
32
|
+
_APPROVALS_DIRNAME = "approvals"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class ApprovalError(ValueError):
|
|
36
|
+
def __init__(self, code: str):
|
|
37
|
+
self.code = code
|
|
38
|
+
super().__init__(code)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class ApprovalRecord:
|
|
43
|
+
"""One approved skill. Deliberately records what was acknowledged, not a conclusion about it:
|
|
44
|
+
there is no `safe` here either, for the same reason `AnalysisReport` has none."""
|
|
45
|
+
|
|
46
|
+
skill_id: str
|
|
47
|
+
body_digest: str
|
|
48
|
+
candidate_digest: str
|
|
49
|
+
acknowledged: tuple[str, ...]
|
|
50
|
+
approved_on: date
|
|
51
|
+
|
|
52
|
+
def as_json(self) -> dict[str, object]:
|
|
53
|
+
return {
|
|
54
|
+
"skill_id": self.skill_id, "body_digest": self.body_digest,
|
|
55
|
+
"candidate_digest": self.candidate_digest, "acknowledged": list(self.acknowledged),
|
|
56
|
+
"approved_on": self.approved_on.isoformat(),
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def approve_candidate(
|
|
61
|
+
candidate: Path, data_dir: Path, *, acknowledge: Iterable[str], today: date,
|
|
62
|
+
) -> tuple[ApprovalRecord, ...]:
|
|
63
|
+
"""Approve every skill in a candidate, binding each to its current body digest.
|
|
64
|
+
|
|
65
|
+
Re-runs static analysis from the file itself. Refuses unless every advisory it produces is
|
|
66
|
+
acknowledged by identifier, and refuses just as firmly if acknowledgments are offered for
|
|
67
|
+
findings that do not exist -- a stale acknowledgment list means the reviewer approved a
|
|
68
|
+
different candidate than the one in front of them, which is precisely the case where waving it
|
|
69
|
+
through is worst.
|
|
70
|
+
|
|
71
|
+
`today` is injected, never read from the clock, so an approval record is reproducible."""
|
|
72
|
+
try:
|
|
73
|
+
report = analyze_candidate(candidate)
|
|
74
|
+
except CandidateError as error:
|
|
75
|
+
raise ApprovalError(error.code) from error
|
|
76
|
+
|
|
77
|
+
required = {item.identifier for item in report.advisories}
|
|
78
|
+
offered = set(acknowledge)
|
|
79
|
+
if required - offered:
|
|
80
|
+
raise ApprovalError("unacknowledged_advisories")
|
|
81
|
+
if offered - required:
|
|
82
|
+
raise ApprovalError("unknown_acknowledgment")
|
|
83
|
+
|
|
84
|
+
records = tuple(
|
|
85
|
+
ApprovalRecord(
|
|
86
|
+
skill_id=skill_id,
|
|
87
|
+
body_digest=digest,
|
|
88
|
+
candidate_digest=report.digest,
|
|
89
|
+
acknowledged=tuple(sorted(identifier for identifier in required
|
|
90
|
+
if identifier.startswith(f"{skill_id}:"))),
|
|
91
|
+
approved_on=today,
|
|
92
|
+
)
|
|
93
|
+
for skill_id, digest in _body_digests(candidate).items()
|
|
94
|
+
)
|
|
95
|
+
directory = data_dir / "skills" / _APPROVALS_DIRNAME
|
|
96
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
97
|
+
for record in records:
|
|
98
|
+
_write_private(directory / f"{record.skill_id}.json", record)
|
|
99
|
+
return records
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def load_approvals(data_dir: Path) -> dict[str, str]:
|
|
103
|
+
"""Return `skill_id -> body_digest` for `skillset.validate_skillset`.
|
|
104
|
+
|
|
105
|
+
An unreadable or malformed record is a hard failure, never a skipped entry. Silently dropping one
|
|
106
|
+
would turn a corrupted approval into an unapproved skill, and an unapproved skill fails
|
|
107
|
+
activation with a message about the wrong thing entirely."""
|
|
108
|
+
directory = data_dir / "skills" / _APPROVALS_DIRNAME
|
|
109
|
+
if not directory.is_dir():
|
|
110
|
+
return {}
|
|
111
|
+
approvals: dict[str, str] = {}
|
|
112
|
+
for path in sorted(directory.glob("*.json")):
|
|
113
|
+
record = read_approval(path)
|
|
114
|
+
approvals[record.skill_id] = record.body_digest
|
|
115
|
+
return approvals
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def read_approval(path: Path) -> ApprovalRecord:
|
|
119
|
+
try:
|
|
120
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
121
|
+
return ApprovalRecord(
|
|
122
|
+
skill_id=data["skill_id"], body_digest=data["body_digest"],
|
|
123
|
+
candidate_digest=data["candidate_digest"],
|
|
124
|
+
acknowledged=tuple(data["acknowledged"]),
|
|
125
|
+
approved_on=date.fromisoformat(data["approved_on"]),
|
|
126
|
+
)
|
|
127
|
+
except (OSError, ValueError, KeyError, TypeError) as error:
|
|
128
|
+
raise ApprovalError("malformed_approval") from error
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def revoke_approval(data_dir: Path, skill_id: str) -> bool:
|
|
132
|
+
"""Withdraw an approval. Returns whether one existed. Revocation is a first-class operation: a
|
|
133
|
+
reviewer who changes their mind must not have to hand-delete a file to act on it."""
|
|
134
|
+
path = data_dir / "skills" / _APPROVALS_DIRNAME / f"{skill_id}.json"
|
|
135
|
+
if not path.is_file():
|
|
136
|
+
return False
|
|
137
|
+
path.unlink()
|
|
138
|
+
return True
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _body_digests(candidate: Path) -> dict[str, str]:
|
|
142
|
+
from .candidates import _parse, _read_candidate # local: internal reuse, not a public seam
|
|
143
|
+
|
|
144
|
+
pack = _parse(_read_candidate(candidate), candidate.suffix.lower())
|
|
145
|
+
return {skill.skill_id: skill.body_digest for skill in pack.skills}
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _write_private(destination: Path, record: ApprovalRecord) -> None:
|
|
149
|
+
payload = (json.dumps(record.as_json(), indent=2, sort_keys=True) + "\n").encode("utf-8")
|
|
150
|
+
handle = tempfile.NamedTemporaryFile(
|
|
151
|
+
prefix=f".{destination.name}.", suffix=".tmp", dir=destination.parent, delete=False
|
|
152
|
+
)
|
|
153
|
+
temporary = Path(handle.name)
|
|
154
|
+
try:
|
|
155
|
+
handle.write(payload)
|
|
156
|
+
handle.flush()
|
|
157
|
+
os.fsync(handle.fileno())
|
|
158
|
+
handle.close()
|
|
159
|
+
os.chmod(temporary, 0o600)
|
|
160
|
+
os.replace(temporary, destination)
|
|
161
|
+
except BaseException:
|
|
162
|
+
handle.close()
|
|
163
|
+
temporary.unlink(missing_ok=True)
|
|
164
|
+
raise
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
__all__ = [
|
|
168
|
+
"ApprovalError",
|
|
169
|
+
"ApprovalRecord",
|
|
170
|
+
"approve_candidate",
|
|
171
|
+
"load_approvals",
|
|
172
|
+
"read_approval",
|
|
173
|
+
"revoke_approval",
|
|
174
|
+
]
|
bruriah/audit.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# Slice 9B (unit 1 of 2): the content-free redacted audit trail for `research.py`. Append-only
|
|
2
|
+
# JSON Lines; every record carries ONLY identifiers, a closed destination classification,
|
|
3
|
+
# decisions, counts, and timings -- NEVER the task/query text, the destination path or query
|
|
4
|
+
# string (either can encode a secret token or the caller's private task), response body, request
|
|
5
|
+
# or response headers, or any secret. design.md "Network": "Audit records IDs, destination
|
|
6
|
+
# class, decisions, counts and timings -- never query/body/secret content." Only the destination
|
|
7
|
+
# HOST is recorded, never the full URL.
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from datetime import datetime
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Literal
|
|
16
|
+
|
|
17
|
+
AuditDecision = Literal[
|
|
18
|
+
"disabled", "not_warranted", "cached", "fetched", "refused", "degraded", "error",
|
|
19
|
+
]
|
|
20
|
+
DestinationClass = Literal["public_https", "none"]
|
|
21
|
+
_RECORD_FIELDS = (
|
|
22
|
+
"request_id", "destination_host", "destination_class", "decision", "code",
|
|
23
|
+
"bytes_transferred", "elapsed_ms", "timestamp",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class AuditRecord:
|
|
29
|
+
request_id: str
|
|
30
|
+
destination_host: str | None
|
|
31
|
+
destination_class: DestinationClass
|
|
32
|
+
decision: AuditDecision
|
|
33
|
+
code: str
|
|
34
|
+
bytes_transferred: int
|
|
35
|
+
elapsed_ms: int
|
|
36
|
+
timestamp: datetime
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _encode(record: AuditRecord) -> str:
|
|
40
|
+
return json.dumps(
|
|
41
|
+
{
|
|
42
|
+
"request_id": record.request_id,
|
|
43
|
+
"destination_host": record.destination_host,
|
|
44
|
+
"destination_class": record.destination_class,
|
|
45
|
+
"decision": record.decision,
|
|
46
|
+
"code": record.code,
|
|
47
|
+
"bytes_transferred": record.bytes_transferred,
|
|
48
|
+
"elapsed_ms": record.elapsed_ms,
|
|
49
|
+
"timestamp": record.timestamp.isoformat(),
|
|
50
|
+
},
|
|
51
|
+
sort_keys=True,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def append_audit(audit_path: Path, record: AuditRecord) -> None:
|
|
56
|
+
"""Append one redacted line; never truncates or reorders prior entries -- an audit trail, not
|
|
57
|
+
a cache. The directory/file are private (0700/0600) on POSIX after every write."""
|
|
58
|
+
audit_path.parent.mkdir(parents=True, exist_ok=True)
|
|
59
|
+
with audit_path.open("a", encoding="utf-8") as handle:
|
|
60
|
+
handle.write(_encode(record) + "\n")
|
|
61
|
+
if os.name == "posix":
|
|
62
|
+
os.chmod(audit_path.parent, 0o700)
|
|
63
|
+
os.chmod(audit_path, 0o600)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def read_audit_records(audit_path: Path) -> list[AuditRecord]:
|
|
67
|
+
"""Diagnostic/test helper: parse every line back into an `AuditRecord`. Not used by
|
|
68
|
+
`research.py` itself -- production audit access is append-only, write-side only."""
|
|
69
|
+
if not audit_path.is_file():
|
|
70
|
+
return []
|
|
71
|
+
records: list[AuditRecord] = []
|
|
72
|
+
for line in audit_path.read_text(encoding="utf-8").splitlines():
|
|
73
|
+
if not line:
|
|
74
|
+
continue
|
|
75
|
+
payload = json.loads(line)
|
|
76
|
+
if not isinstance(payload, dict) or set(payload) != set(_RECORD_FIELDS):
|
|
77
|
+
raise ValueError("corrupt_audit_record")
|
|
78
|
+
records.append(
|
|
79
|
+
AuditRecord(
|
|
80
|
+
request_id=payload["request_id"],
|
|
81
|
+
destination_host=payload["destination_host"],
|
|
82
|
+
destination_class=payload["destination_class"],
|
|
83
|
+
decision=payload["decision"],
|
|
84
|
+
code=payload["code"],
|
|
85
|
+
bytes_transferred=payload["bytes_transferred"],
|
|
86
|
+
elapsed_ms=payload["elapsed_ms"],
|
|
87
|
+
timestamp=datetime.fromisoformat(payload["timestamp"]),
|
|
88
|
+
)
|
|
89
|
+
)
|
|
90
|
+
return records
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
__all__ = ["AuditDecision", "AuditRecord", "DestinationClass", "append_audit", "read_audit_records"]
|
bruriah/cache.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
# Slice 9B (unit 1 of 2): the private atomic evidence cache. Stores ONLY the permitted minimum
|
|
2
|
+
# for captured live evidence -- design.md "Network": "Atomic private cache stores permitted
|
|
3
|
+
# minimum excerpts plus digest, redirect chain, policy/pack versions, license and TTL; prohibited
|
|
4
|
+
# bodies are discarded." and the lifecycle spec's "External Evidence and Cache Lifecycle" /
|
|
5
|
+
# "Cache expiry or prohibited reuse": prohibited/unknown reuse MUST store only permitted
|
|
6
|
+
# metadata, citation, and minimum necessary excerpt; expired content MUST NOT be presented as
|
|
7
|
+
# current.
|
|
8
|
+
#
|
|
9
|
+
# `evidence: EvidenceRecord` is stored WHOLE -- it is already "permitted metadata" by contract
|
|
10
|
+
# (digest, locator, citation_locator, redirect_chain, license/reuse state, pack_version -- no
|
|
11
|
+
# body-text field exists on it at all), so persisting it fully is safe regardless of reuse state.
|
|
12
|
+
# `excerpt` is the ONLY body-derived text ever written to disk; `excerpt_only=True` (the default
|
|
13
|
+
# for every live fetch in this change, since `fetch.py` has no pack context and always reports
|
|
14
|
+
# license=reuse="unknown") means `excerpt` is a small, bounded citation excerpt, never the full
|
|
15
|
+
# body. `policy_version` records the research-policy pack version this cache decision was made
|
|
16
|
+
# under -- distinct from `evidence.pack_version`, which belongs to the evidence's own source pack.
|
|
17
|
+
#
|
|
18
|
+
# Cache key: sha256 of the CANONICAL request URL (deterministic and content-addressed -- never a
|
|
19
|
+
# random/UUID/timestamp name, matching the rest of the codebase's "no Date.now/random for
|
|
20
|
+
# identity" convention). Write path: write-temp-then-rename (`os.replace`, atomic on the same
|
|
21
|
+
# filesystem) with 0600 permissions applied BEFORE the rename makes the file visible under its
|
|
22
|
+
# final name, so no reader can ever observe a partially-written or over-permissioned file.
|
|
23
|
+
# `now` is injectable everywhere TTL is evaluated, so expiry is deterministic under test, never a
|
|
24
|
+
# wall-clock race.
|
|
25
|
+
#
|
|
26
|
+
# Slice 12D: deletion controls (closes 9B WARNING 2 -- "no cache deletion/retention"; the
|
|
27
|
+
# lifecycle spec's "External Evidence and Cache Lifecycle" requires the cache be "governed by TTL
|
|
28
|
+
# and deletion controls", not just TTL staleness detection). Two pieces, both typed-total and both
|
|
29
|
+
# free of wall-clock reads:
|
|
30
|
+
# - `prune_expired` is the explicit deletion control: it removes an entry once it has genuinely
|
|
31
|
+
# expired (`now > expires_at`, the same condition `read_cache` already treats as a miss) OR
|
|
32
|
+
# its `expires_at` exceeds what any legitimate write under the given `ttl` could have produced
|
|
33
|
+
# (`expires_at - now > ttl`) -- a tamper/corruption ceiling, since a freshly built live entry
|
|
34
|
+
# can never claim more than `ttl` of remaining life. Corrupt/unreadable entries are always
|
|
35
|
+
# removed (nothing about them is auditable or bound-checkable). `doctor` (cli.py) never calls
|
|
36
|
+
# this -- it only calls the read-only `cache_stats` below, preserving "doctor is read-only".
|
|
37
|
+
# - `write_cache_atomic` self-bounds by sweeping for already-expired entries once after every
|
|
38
|
+
# successful write, deriving `now` ENTIRELY from the entry just written
|
|
39
|
+
# (`entry.evidence.retrieved_at` stands in for "now"). This needs zero new parameters and zero
|
|
40
|
+
# wall-clock, so the frozen `research.py` callsite (`write_cache_atomic(deps.cache_dir,
|
|
41
|
+
# canonical, entry)`) gets self-bounding for free. Only the plain-expiry direction
|
|
42
|
+
# (`retrieved_at > other.expires_at`) is used here -- NOT the tamper ceiling -- because
|
|
43
|
+
# `retrieved_at` is always a real PAST timestamp of one fetch among possibly concurrent or
|
|
44
|
+
# out-of-order writes; wall-clock time only advances, so if this write's own retrieved_at
|
|
45
|
+
# already exceeds another entry's `expires_at`, the true current time exceeds it too, and the
|
|
46
|
+
# removal can never be a false positive. The ceiling check requires an authoritative, operator-
|
|
47
|
+
# supplied `now`/`ttl` pair (an out-of-order write's own `ttl` says nothing about whether
|
|
48
|
+
# ANOTHER entry's expiry is plausible), so it is reserved for the explicit `prune_expired` call
|
|
49
|
+
# below. If `retrieved_at` is absent (the field is optional on `EvidenceRecord`) or not
|
|
50
|
+
# comparable, self-pruning is skipped for that write -- best-effort, never a hard failure of
|
|
51
|
+
# the write itself.
|
|
52
|
+
from __future__ import annotations
|
|
53
|
+
|
|
54
|
+
import hashlib
|
|
55
|
+
import json
|
|
56
|
+
import os
|
|
57
|
+
import tempfile
|
|
58
|
+
from collections.abc import Callable
|
|
59
|
+
from dataclasses import dataclass
|
|
60
|
+
from datetime import datetime, timedelta
|
|
61
|
+
from pathlib import Path
|
|
62
|
+
from typing import Literal
|
|
63
|
+
|
|
64
|
+
from pydantic import ValidationError
|
|
65
|
+
|
|
66
|
+
from .contracts import EvidenceRecord
|
|
67
|
+
|
|
68
|
+
ReuseState = Literal["permitted", "restricted", "prohibited", "unknown"]
|
|
69
|
+
_ENTRY_FIELDS = frozenset({"evidence", "expires_at", "excerpt", "excerpt_only", "policy_version"})
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class CacheError(ValueError):
|
|
73
|
+
def __init__(self, code: str):
|
|
74
|
+
self.code = code
|
|
75
|
+
super().__init__(code)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class CacheEntry:
|
|
80
|
+
"""One cached research result. See module docstring for the excerpt-only/reuse contract."""
|
|
81
|
+
|
|
82
|
+
evidence: EvidenceRecord
|
|
83
|
+
expires_at: datetime
|
|
84
|
+
excerpt: str
|
|
85
|
+
excerpt_only: bool
|
|
86
|
+
policy_version: str
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass(frozen=True)
|
|
90
|
+
class CacheLookup:
|
|
91
|
+
hit: bool
|
|
92
|
+
expired: bool
|
|
93
|
+
entry: CacheEntry | None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True)
|
|
97
|
+
class PruneSummary:
|
|
98
|
+
"""Result of one `prune_expired` sweep. `corrupt` is a subset of `removed` (every corrupt
|
|
99
|
+
entry found is also deleted); `bytes_reclaimed` counts freed disk space across all removals."""
|
|
100
|
+
|
|
101
|
+
scanned: int
|
|
102
|
+
removed: int
|
|
103
|
+
corrupt: int
|
|
104
|
+
bytes_reclaimed: int
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass(frozen=True)
|
|
108
|
+
class CacheStats:
|
|
109
|
+
"""Read-only visibility for `doctor` -- see `cache_stats`. Never deletes anything."""
|
|
110
|
+
|
|
111
|
+
entries: int
|
|
112
|
+
expired: int
|
|
113
|
+
total_bytes: int
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def cache_key(canonical_url: str) -> str:
|
|
117
|
+
"""Deterministic, content-addressed cache filename stem -- the SAME canonical URL always
|
|
118
|
+
yields the SAME key, so a repeated request for the original URL hits cache even if the
|
|
119
|
+
fetched content ultimately resolved through redirects to a different final locator."""
|
|
120
|
+
return hashlib.sha256(canonical_url.encode("utf-8")).hexdigest()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _cache_path(cache_dir: Path, canonical_url: str) -> Path:
|
|
124
|
+
return cache_dir / f"{cache_key(canonical_url)}.json"
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _encode(entry: CacheEntry) -> str:
|
|
128
|
+
payload = {
|
|
129
|
+
"evidence": json.loads(entry.evidence.model_dump_json()),
|
|
130
|
+
"expires_at": entry.expires_at.isoformat(),
|
|
131
|
+
"excerpt": entry.excerpt,
|
|
132
|
+
"excerpt_only": entry.excerpt_only,
|
|
133
|
+
"policy_version": entry.policy_version,
|
|
134
|
+
}
|
|
135
|
+
return json.dumps(payload, sort_keys=True)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _decode(raw: str) -> CacheEntry:
|
|
139
|
+
try:
|
|
140
|
+
payload = json.loads(raw)
|
|
141
|
+
except json.JSONDecodeError as error:
|
|
142
|
+
raise CacheError("corrupt_cache_entry") from error
|
|
143
|
+
if not isinstance(payload, dict) or set(payload) != _ENTRY_FIELDS:
|
|
144
|
+
raise CacheError("corrupt_cache_entry")
|
|
145
|
+
try:
|
|
146
|
+
# `model_validate` (Python-object mode) rejects an ISO datetime STRING under this
|
|
147
|
+
# project's `strict=True` `ClosedModel` config; re-serializing the nested dict and using
|
|
148
|
+
# `model_validate_json` (JSON mode, where a datetime is necessarily a string) is the same
|
|
149
|
+
# pattern `packs.py` uses for its own nested pydantic parsing.
|
|
150
|
+
evidence = EvidenceRecord.model_validate_json(json.dumps(payload["evidence"]))
|
|
151
|
+
return CacheEntry(
|
|
152
|
+
evidence=evidence,
|
|
153
|
+
expires_at=datetime.fromisoformat(payload["expires_at"]),
|
|
154
|
+
excerpt=payload["excerpt"],
|
|
155
|
+
excerpt_only=payload["excerpt_only"],
|
|
156
|
+
policy_version=payload["policy_version"],
|
|
157
|
+
)
|
|
158
|
+
except (KeyError, TypeError, ValueError, ValidationError) as error:
|
|
159
|
+
raise CacheError("corrupt_cache_entry") from error
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def write_cache_atomic(cache_dir: Path, canonical_url: str, entry: CacheEntry) -> Path:
|
|
163
|
+
"""Write-temp-then-rename: a temp file created in the SAME directory (so `os.replace` is an
|
|
164
|
+
atomic rename on one filesystem, never a cross-device copy) is written, chmod'd 0600, then
|
|
165
|
+
atomically renamed onto the final content-addressed path. A reader can only ever observe the
|
|
166
|
+
prior complete entry or the new complete entry -- never a partial write. After the write lands,
|
|
167
|
+
an opportunistic self-prune sweep runs (see `_self_prune_after_write`) so the cache is
|
|
168
|
+
self-bounding without any new parameter or wall-clock read."""
|
|
169
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
170
|
+
if os.name == "posix":
|
|
171
|
+
os.chmod(cache_dir, 0o700)
|
|
172
|
+
final_path = _cache_path(cache_dir, canonical_url)
|
|
173
|
+
fd, tmp_name = tempfile.mkstemp(dir=cache_dir, prefix=".tmp-cache-")
|
|
174
|
+
try:
|
|
175
|
+
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
176
|
+
handle.write(_encode(entry))
|
|
177
|
+
os.chmod(tmp_name, 0o600)
|
|
178
|
+
os.replace(tmp_name, final_path)
|
|
179
|
+
except BaseException:
|
|
180
|
+
Path(tmp_name).unlink(missing_ok=True)
|
|
181
|
+
raise
|
|
182
|
+
_self_prune_after_write(cache_dir, entry)
|
|
183
|
+
return final_path
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _unlink_quietly(path: Path) -> bool:
|
|
187
|
+
"""Delete-if-possible, never raise: a concurrent deletion or transient OS error during a
|
|
188
|
+
best-effort sweep must not abort the sweep or, for `prune_expired`, escape untyped."""
|
|
189
|
+
try:
|
|
190
|
+
path.unlink()
|
|
191
|
+
return True
|
|
192
|
+
except OSError:
|
|
193
|
+
return False
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _sweep_prunable(cache_dir: Path, is_prunable: Callable[[CacheEntry], bool]) -> PruneSummary:
|
|
197
|
+
"""Shared typed-total core for both deletion paths: scan every `*.json` entry once, always
|
|
198
|
+
remove a corrupt/unreadable one, and remove a decodable one iff `is_prunable(entry)`. No
|
|
199
|
+
exception escapes a single bad or racing file."""
|
|
200
|
+
if not cache_dir.is_dir():
|
|
201
|
+
return PruneSummary(scanned=0, removed=0, corrupt=0, bytes_reclaimed=0)
|
|
202
|
+
scanned = removed = corrupt = bytes_reclaimed = 0
|
|
203
|
+
for path in sorted(cache_dir.glob("*.json")):
|
|
204
|
+
scanned += 1
|
|
205
|
+
try:
|
|
206
|
+
size = path.stat().st_size
|
|
207
|
+
except OSError:
|
|
208
|
+
continue
|
|
209
|
+
try:
|
|
210
|
+
entry = _decode(path.read_text(encoding="utf-8"))
|
|
211
|
+
except (CacheError, OSError):
|
|
212
|
+
corrupt += 1
|
|
213
|
+
if _unlink_quietly(path):
|
|
214
|
+
removed += 1
|
|
215
|
+
bytes_reclaimed += size
|
|
216
|
+
continue
|
|
217
|
+
if is_prunable(entry):
|
|
218
|
+
if _unlink_quietly(path):
|
|
219
|
+
removed += 1
|
|
220
|
+
bytes_reclaimed += size
|
|
221
|
+
return PruneSummary(scanned=scanned, removed=removed, corrupt=corrupt, bytes_reclaimed=bytes_reclaimed)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _self_prune_after_write(cache_dir: Path, entry: CacheEntry) -> PruneSummary | None:
|
|
225
|
+
"""See module docstring "Slice 12D" for the monotonic-safety rationale. Returns `None` (skips
|
|
226
|
+
pruning for this write) when the entry carries no usable timestamp; this is a best-effort
|
|
227
|
+
maintenance step, never a reason to fail the write that just succeeded."""
|
|
228
|
+
retrieved_at = entry.evidence.retrieved_at
|
|
229
|
+
if retrieved_at is None:
|
|
230
|
+
return None
|
|
231
|
+
try:
|
|
232
|
+
return _sweep_prunable(cache_dir, lambda other: retrieved_at > other.expires_at)
|
|
233
|
+
except TypeError:
|
|
234
|
+
return None
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def prune_expired(cache_dir: Path, *, now: datetime, ttl: timedelta) -> PruneSummary:
|
|
238
|
+
"""Explicit deletion control -- see module docstring "Slice 12D". An entry is removed once it
|
|
239
|
+
has genuinely expired (`now > expires_at`, the same condition `read_cache` already treats as a
|
|
240
|
+
miss) or its `expires_at` is implausibly far in the future for the given `ttl`
|
|
241
|
+
(`expires_at - now > ttl`, a tamper/corruption ceiling that a legitimately built live entry can
|
|
242
|
+
never trip, since `build_cache_entry` never grants more than `ttl` of remaining life). Corrupt
|
|
243
|
+
or unreadable entries are always removed. Typed-total throughout."""
|
|
244
|
+
return _sweep_prunable(cache_dir, lambda entry: now > entry.expires_at or entry.expires_at - now > ttl)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def cache_stats(cache_dir: Path, *, now: datetime) -> CacheStats:
|
|
248
|
+
"""Read-only visibility for `doctor` (design.md: "`doctor` is read-only"). Scans without ever
|
|
249
|
+
deleting, decoding into pruning decisions, or otherwise modifying a file -- `doctor` MUST NOT
|
|
250
|
+
delete anything; a separate explicit action would be required to expose `prune_expired` via the
|
|
251
|
+
CLI, and none is added here. A corrupt entry still occupies disk space, so it is counted in
|
|
252
|
+
`entries`/`total_bytes` even though it can never be validated as expired or live."""
|
|
253
|
+
if not cache_dir.is_dir():
|
|
254
|
+
return CacheStats(entries=0, expired=0, total_bytes=0)
|
|
255
|
+
entries = expired = total_bytes = 0
|
|
256
|
+
for path in sorted(cache_dir.glob("*.json")):
|
|
257
|
+
try:
|
|
258
|
+
size = path.stat().st_size
|
|
259
|
+
except OSError:
|
|
260
|
+
continue
|
|
261
|
+
entries += 1
|
|
262
|
+
total_bytes += size
|
|
263
|
+
try:
|
|
264
|
+
entry = _decode(path.read_text(encoding="utf-8"))
|
|
265
|
+
except CacheError:
|
|
266
|
+
continue
|
|
267
|
+
if now > entry.expires_at:
|
|
268
|
+
expired += 1
|
|
269
|
+
return CacheStats(entries=entries, expired=expired, total_bytes=total_bytes)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def read_cache(cache_dir: Path, canonical_url: str, *, now: datetime) -> CacheLookup:
|
|
273
|
+
"""Read the cache entry for `canonical_url`. A missing file, a corrupt file, and an expired
|
|
274
|
+
entry are ALL a miss for serving purposes (`hit=False`) -- expired content is never returned
|
|
275
|
+
as current. `expired=True` distinguishes "was cached but is stale" from "never cached" for
|
|
276
|
+
diagnostics without ever handing the caller stale content."""
|
|
277
|
+
path = _cache_path(cache_dir, canonical_url)
|
|
278
|
+
if not path.is_file():
|
|
279
|
+
return CacheLookup(hit=False, expired=False, entry=None)
|
|
280
|
+
try:
|
|
281
|
+
entry = _decode(path.read_text(encoding="utf-8"))
|
|
282
|
+
except CacheError:
|
|
283
|
+
return CacheLookup(hit=False, expired=False, entry=None)
|
|
284
|
+
if now > entry.expires_at:
|
|
285
|
+
return CacheLookup(hit=False, expired=True, entry=None)
|
|
286
|
+
return CacheLookup(hit=True, expired=False, entry=entry)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def build_cache_entry(
|
|
290
|
+
evidence: EvidenceRecord,
|
|
291
|
+
*,
|
|
292
|
+
retrieved_at: datetime,
|
|
293
|
+
ttl: timedelta,
|
|
294
|
+
body: bytes,
|
|
295
|
+
max_excerpt_chars: int,
|
|
296
|
+
policy_version: str,
|
|
297
|
+
) -> CacheEntry:
|
|
298
|
+
"""Build the entry that will actually be persisted. Only `evidence.reuse == "permitted"` may
|
|
299
|
+
store a body-derived excerpt up to `max_excerpt_chars`; every other state (`restricted`,
|
|
300
|
+
`prohibited`, or `unknown` -- the DEFAULT, since `fetch.py` has no pack context and always
|
|
301
|
+
reports `license=reuse="unknown"`) is capped to a small, bounded citation excerpt only, never
|
|
302
|
+
treated as complete content -- "Cache expiry or prohibited reuse": "retained material is
|
|
303
|
+
limited to permitted fields."""
|
|
304
|
+
text = body.decode("utf-8", errors="replace")
|
|
305
|
+
excerpt_only = evidence.reuse != "permitted"
|
|
306
|
+
limit = min(max_excerpt_chars, 280) if excerpt_only else max_excerpt_chars
|
|
307
|
+
return CacheEntry(
|
|
308
|
+
evidence=evidence,
|
|
309
|
+
expires_at=retrieved_at + ttl,
|
|
310
|
+
excerpt=text[:limit],
|
|
311
|
+
excerpt_only=excerpt_only,
|
|
312
|
+
policy_version=policy_version,
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
__all__ = [
|
|
317
|
+
"CacheEntry", "CacheError", "CacheLookup", "CacheStats", "PruneSummary", "ReuseState",
|
|
318
|
+
"build_cache_entry", "cache_key", "cache_stats", "prune_expired", "read_cache",
|
|
319
|
+
"write_cache_atomic",
|
|
320
|
+
]
|