toolgraph 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toolgraph/__init__.py +3 -0
- toolgraph/artifact_safety.py +97 -0
- toolgraph/artifacts.py +122 -0
- toolgraph/cli.py +951 -0
- toolgraph/config.py +106 -0
- toolgraph/control_plan.py +494 -0
- toolgraph/crawler/__init__.py +0 -0
- toolgraph/crawler/client.py +120 -0
- toolgraph/crawler/crawl.py +103 -0
- toolgraph/crawler/transports.py +60 -0
- toolgraph/graph/__init__.py +0 -0
- toolgraph/graph/driver.py +275 -0
- toolgraph/graph/loader.py +280 -0
- toolgraph/graph/queries.py +984 -0
- toolgraph/graph/schema.py +166 -0
- toolgraph/graph/selector.py +423 -0
- toolgraph/graph/store.py +57 -0
- toolgraph/manifest/__init__.py +0 -0
- toolgraph/manifest/ingest.py +542 -0
- toolgraph/manifest/parser.py +26 -0
- toolgraph/models.py +192 -0
- toolgraph/policy_bundle.py +212 -0
- toolgraph/policy_review.py +97 -0
- toolgraph/preflight.py +77 -0
- toolgraph/quickstart/governance.yaml +28 -0
- toolgraph/quickstart/policy_gateway_server.py +31 -0
- toolgraph/quickstart/servers.yaml +5 -0
- toolgraph/redaction.py +82 -0
- toolgraph/review_candidates.py +579 -0
- toolgraph/server/__init__.py +0 -0
- toolgraph/server/app.py +306 -0
- toolgraph/uris.py +64 -0
- toolgraph/yaml_loader.py +68 -0
- toolgraph-0.0.1.dist-info/METADATA +674 -0
- toolgraph-0.0.1.dist-info/RECORD +38 -0
- toolgraph-0.0.1.dist-info/WHEEL +4 -0
- toolgraph-0.0.1.dist-info/entry_points.txt +2 -0
- toolgraph-0.0.1.dist-info/licenses/LICENSE +191 -0
toolgraph/redaction.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Stable, credential-free labels for persisted endpoints and diagnostics."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pathlib import Path, PureWindowsPath
|
|
6
|
+
import re
|
|
7
|
+
import shlex
|
|
8
|
+
from urllib.parse import urlsplit
|
|
9
|
+
|
|
10
|
+
from toolgraph.models import ServerSpec
|
|
11
|
+
|
|
12
|
+
_URI = re.compile(r"[a-z][a-z0-9+.-]*://[^\s'\"<>]+", re.IGNORECASE)
|
|
13
|
+
_AUTHORIZATION = re.compile(
|
|
14
|
+
r"(?i)\bauthorization['\"]?\s*[:=]\s*['\"]?"
|
|
15
|
+
r"(?:(?:bearer|basic|token)\s+)?[^\s,'\"}\]]+"
|
|
16
|
+
)
|
|
17
|
+
_CREDENTIAL = re.compile(
|
|
18
|
+
r"(?i)(api[-_]?key|access[-_]?token|token|password|secret)"
|
|
19
|
+
r"(['\"]?\s*[:=]\s*['\"]?)([^\s,}\]]+)"
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _url_origin(value: str, fallback: str) -> str:
|
|
24
|
+
try:
|
|
25
|
+
parsed = urlsplit(value)
|
|
26
|
+
hostname = parsed.hostname
|
|
27
|
+
if parsed.scheme.lower() not in {"http", "https"} or not hostname:
|
|
28
|
+
return fallback
|
|
29
|
+
host = f"[{hostname}]" if ":" in hostname else hostname
|
|
30
|
+
port = f":{parsed.port}" if parsed.port is not None else ""
|
|
31
|
+
return f"{parsed.scheme.lower()}://{host}{port}"
|
|
32
|
+
except ValueError:
|
|
33
|
+
return fallback
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _diagnostic_uri_origin(value: str) -> str:
|
|
37
|
+
try:
|
|
38
|
+
parsed = urlsplit(value)
|
|
39
|
+
hostname = parsed.hostname
|
|
40
|
+
if not parsed.scheme or not hostname:
|
|
41
|
+
return "<redacted-uri>"
|
|
42
|
+
host = f"[{hostname}]" if ":" in hostname else hostname
|
|
43
|
+
port = f":{parsed.port}" if parsed.port is not None else ""
|
|
44
|
+
return f"{parsed.scheme.lower()}://{host}{port}"
|
|
45
|
+
except ValueError:
|
|
46
|
+
return "<redacted-uri>"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _executable_name(value: str | None) -> str:
|
|
50
|
+
if not value:
|
|
51
|
+
return "unknown"
|
|
52
|
+
try:
|
|
53
|
+
token = shlex.split(value)[0]
|
|
54
|
+
except (ValueError, IndexError):
|
|
55
|
+
return "unknown"
|
|
56
|
+
posix_name = Path(token).name
|
|
57
|
+
return PureWindowsPath(posix_name).name or "unknown"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def endpoint_label(spec: ServerSpec) -> str:
|
|
61
|
+
"""Return an endpoint label that never contains arguments or URL secrets."""
|
|
62
|
+
if spec.transport == "stdio":
|
|
63
|
+
return f"stdio:{_executable_name(spec.command)}"
|
|
64
|
+
return _url_origin(spec.url or "", f"{spec.transport}:<redacted>")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def persisted_endpoint(transport: str, value: str | None) -> str:
|
|
68
|
+
"""Enforce the persistence invariant even for programmatic crawl results."""
|
|
69
|
+
if transport == "stdio":
|
|
70
|
+
raw = value or "unknown"
|
|
71
|
+
if raw.startswith("stdio:"):
|
|
72
|
+
raw = raw.removeprefix("stdio:")
|
|
73
|
+
return f"stdio:{_executable_name(raw)}"
|
|
74
|
+
return _url_origin(value or "", f"{transport}:<redacted>")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def redact_text(value: object) -> str:
|
|
78
|
+
"""Scrub URLs and common credential-shaped values from an error message."""
|
|
79
|
+
text = str(value)
|
|
80
|
+
text = _URI.sub(lambda match: _diagnostic_uri_origin(match.group(0)), text)
|
|
81
|
+
text = _AUTHORIZATION.sub("Authorization=<redacted>", text)
|
|
82
|
+
return _CREDENTIAL.sub(lambda match: f"{match.group(1)}=<redacted>", text)
|
|
@@ -0,0 +1,579 @@
|
|
|
1
|
+
"""Human-review-only intake for Tracegraph review-candidate artifacts.
|
|
2
|
+
|
|
3
|
+
The Tracegraph report is immutable evidence. Toolgraph projects only the five
|
|
4
|
+
body-free v1 fields and stores operator dispositions in a separate, append-only
|
|
5
|
+
sidecar. This module deliberately has no graph, manifest, or selector imports.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from datetime import datetime, timezone
|
|
12
|
+
import functools
|
|
13
|
+
import hashlib
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
import re
|
|
18
|
+
import tempfile
|
|
19
|
+
from typing import Iterator, Literal
|
|
20
|
+
import uuid
|
|
21
|
+
import warnings
|
|
22
|
+
|
|
23
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
|
|
24
|
+
|
|
25
|
+
from toolgraph.artifacts import fsync_parent_dir
|
|
26
|
+
|
|
27
|
+
if os.name == "nt": # pragma: no cover - exercised on Windows CI/users
|
|
28
|
+
import msvcrt
|
|
29
|
+
else: # pragma: no branch - one platform branch per interpreter
|
|
30
|
+
import fcntl
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
TRACEGRAPH_SCHEMA_VERSION = 1
|
|
34
|
+
TRACEGRAPH_KIND = "tracegraph.review-candidates"
|
|
35
|
+
ANNOTATION_SCHEMA_VERSION = 1
|
|
36
|
+
ANNOTATION_KIND = "toolgraph.review-annotations"
|
|
37
|
+
LIST_KIND = "toolgraph.review-candidate-list"
|
|
38
|
+
|
|
39
|
+
Disposition = Literal["open", "accepted", "dismissed"]
|
|
40
|
+
DISPOSITIONS = frozenset({"open", "accepted", "dismissed"})
|
|
41
|
+
|
|
42
|
+
# SyncMill PR #57 made this exact-tuple UUIDv5 the ecosystem review-item
|
|
43
|
+
# identity. Reusing it lets operators correlate a Toolgraph disposition with
|
|
44
|
+
# the corresponding SyncMill human-required board item without sharing state.
|
|
45
|
+
_ITEM_NAMESPACE = uuid.uuid5(
|
|
46
|
+
uuid.NAMESPACE_URL, "https://syncmill.dev/review-items/tracegraph"
|
|
47
|
+
)
|
|
48
|
+
_CREDENTIAL = re.compile(
|
|
49
|
+
r"://[^/\s]+:[^/\s]+@|\b(?:password|token|api[_-]?key|credential)\s*=",
|
|
50
|
+
re.IGNORECASE,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class ReviewCandidateError(ValueError):
|
|
55
|
+
"""A review report or annotation sidecar violated the G3 contract."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ReviewCandidateDurabilityWarning(RuntimeWarning):
|
|
59
|
+
"""The sidecar was replaced, but crash durability could not be confirmed."""
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _normalized_reviewer(value: str) -> str:
|
|
63
|
+
reviewer = value.strip()
|
|
64
|
+
if (
|
|
65
|
+
not reviewer
|
|
66
|
+
or len(reviewer) > 256
|
|
67
|
+
or "\n" in reviewer
|
|
68
|
+
or "\r" in reviewer
|
|
69
|
+
or "\0" in reviewer
|
|
70
|
+
):
|
|
71
|
+
raise ValueError(
|
|
72
|
+
"reviewer must be a non-empty single line of at most 256 characters"
|
|
73
|
+
)
|
|
74
|
+
return reviewer
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _normalized_note(value: str | None) -> str | None:
|
|
78
|
+
if value is None:
|
|
79
|
+
return None
|
|
80
|
+
note = value.strip()
|
|
81
|
+
if (
|
|
82
|
+
not note
|
|
83
|
+
or len(note) > 500
|
|
84
|
+
or "\n" in note
|
|
85
|
+
or "\r" in note
|
|
86
|
+
or "\0" in note
|
|
87
|
+
):
|
|
88
|
+
raise ValueError(
|
|
89
|
+
"note must be a non-empty single line of at most 500 characters"
|
|
90
|
+
)
|
|
91
|
+
if _CREDENTIAL.search(note):
|
|
92
|
+
raise ValueError("note must not contain credentials")
|
|
93
|
+
return note
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class ReviewCandidate(BaseModel):
|
|
97
|
+
"""The body-free fields Toolgraph consumes from one Tracegraph finding."""
|
|
98
|
+
|
|
99
|
+
model_config = ConfigDict(extra="ignore", strict=True, frozen=True)
|
|
100
|
+
|
|
101
|
+
run_id: str = Field(min_length=1, max_length=256, pattern=r"^\S+$")
|
|
102
|
+
pattern_id: str = Field(pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
|
103
|
+
pattern_version: int = Field(ge=1)
|
|
104
|
+
tool_key: str = Field(min_length=4, max_length=512, pattern=r"^\S+::\S+$")
|
|
105
|
+
artifact_digest: str = Field(pattern=r"^sha256:[0-9a-f]{64}$")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class AnnotationCandidate(ReviewCandidate):
|
|
109
|
+
"""Closed snapshot shape used inside Toolgraph-owned sidecars."""
|
|
110
|
+
|
|
111
|
+
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class TracegraphReviewReport(BaseModel):
|
|
115
|
+
"""Supported producer envelope; additive v1 fields are discarded."""
|
|
116
|
+
|
|
117
|
+
model_config = ConfigDict(extra="ignore", strict=True)
|
|
118
|
+
|
|
119
|
+
schema_version: Literal[1]
|
|
120
|
+
kind: Literal["tracegraph.review-candidates"]
|
|
121
|
+
candidates: list[ReviewCandidate]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class AnnotationSource(BaseModel):
|
|
125
|
+
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
|
|
126
|
+
|
|
127
|
+
schema_version: Literal[1]
|
|
128
|
+
kind: Literal["tracegraph.review-candidates"]
|
|
129
|
+
source_report_digest: str = Field(pattern=r"^sha256:[0-9a-f]{64}$")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class AnnotationEvent(BaseModel):
|
|
133
|
+
model_config = ConfigDict(extra="forbid", strict=True, frozen=True)
|
|
134
|
+
|
|
135
|
+
sequence: int = Field(ge=1)
|
|
136
|
+
candidate_id: str
|
|
137
|
+
candidate: AnnotationCandidate
|
|
138
|
+
from_disposition: Disposition
|
|
139
|
+
disposition: Disposition
|
|
140
|
+
reviewer: str = Field(min_length=1, max_length=256)
|
|
141
|
+
recorded_at: str
|
|
142
|
+
note: str | None = Field(default=None, max_length=500)
|
|
143
|
+
|
|
144
|
+
@field_validator("reviewer")
|
|
145
|
+
@classmethod
|
|
146
|
+
def _reviewer_is_safe_metadata(cls, value: str) -> str:
|
|
147
|
+
normalized = _normalized_reviewer(value)
|
|
148
|
+
if normalized != value:
|
|
149
|
+
raise ValueError("must not contain leading or trailing whitespace")
|
|
150
|
+
return value
|
|
151
|
+
|
|
152
|
+
@field_validator("note")
|
|
153
|
+
@classmethod
|
|
154
|
+
def _note_is_safe_metadata(cls, value: str | None) -> str | None:
|
|
155
|
+
normalized = _normalized_note(value)
|
|
156
|
+
if normalized != value:
|
|
157
|
+
raise ValueError("must not contain leading or trailing whitespace")
|
|
158
|
+
return value
|
|
159
|
+
|
|
160
|
+
@field_validator("candidate_id")
|
|
161
|
+
@classmethod
|
|
162
|
+
def _candidate_id_is_uuid(cls, value: str) -> str:
|
|
163
|
+
try:
|
|
164
|
+
parsed = uuid.UUID(value)
|
|
165
|
+
except ValueError as exc:
|
|
166
|
+
raise ValueError("must be a UUID") from exc
|
|
167
|
+
if str(parsed) != value:
|
|
168
|
+
raise ValueError("must use canonical lowercase UUID form")
|
|
169
|
+
return value
|
|
170
|
+
|
|
171
|
+
@field_validator("recorded_at")
|
|
172
|
+
@classmethod
|
|
173
|
+
def _recorded_at_is_utc(cls, value: str) -> str:
|
|
174
|
+
try:
|
|
175
|
+
parsed = datetime.fromisoformat(value)
|
|
176
|
+
except ValueError as exc:
|
|
177
|
+
raise ValueError("must be an ISO-8601 timestamp") from exc
|
|
178
|
+
if parsed.tzinfo is None or parsed.utcoffset() is None:
|
|
179
|
+
raise ValueError("must include a timezone")
|
|
180
|
+
if parsed.utcoffset().total_seconds() != 0:
|
|
181
|
+
raise ValueError("must be UTC")
|
|
182
|
+
return value
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
class AnnotationReport(BaseModel):
|
|
186
|
+
"""Closed v1 sidecar; event history is append-only and state-chained."""
|
|
187
|
+
|
|
188
|
+
model_config = ConfigDict(extra="forbid", strict=True)
|
|
189
|
+
|
|
190
|
+
schema_version: Literal[1]
|
|
191
|
+
kind: Literal["toolgraph.review-annotations"]
|
|
192
|
+
source: AnnotationSource
|
|
193
|
+
events: list[AnnotationEvent]
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class LoadedReviewReport(BaseModel):
|
|
197
|
+
"""Validated projection plus the exact source-byte binding."""
|
|
198
|
+
|
|
199
|
+
model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
|
|
200
|
+
|
|
201
|
+
source_report_digest: str
|
|
202
|
+
candidates: tuple[ReviewCandidate, ...]
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _safe_validation_error(kind: str, exc: ValidationError) -> ReviewCandidateError:
|
|
206
|
+
first = exc.errors(include_input=False)[0]
|
|
207
|
+
location = ".".join(str(part) for part in first.get("loc", ())) or kind
|
|
208
|
+
return ReviewCandidateError(f"invalid {kind} at {location}: {first['msg']}")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _sha256(raw: bytes) -> str:
|
|
212
|
+
return f"sha256:{hashlib.sha256(raw).hexdigest()}"
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
# Reports and sidecars are operator-provided files; nothing else in this
|
|
216
|
+
# module bounds them, so a runaway file would otherwise be read whole.
|
|
217
|
+
MAX_REPORT_BYTES = 50_000_000
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _read_capped(path: Path, kind: str) -> bytes:
|
|
221
|
+
# Bounded read from one open descriptor: a stat-then-read pair is a
|
|
222
|
+
# TOCTOU — the file can be replaced or grown between the two calls.
|
|
223
|
+
try:
|
|
224
|
+
with path.open("rb") as handle:
|
|
225
|
+
raw = handle.read(MAX_REPORT_BYTES + 1)
|
|
226
|
+
except OSError as exc:
|
|
227
|
+
detail = exc.strerror or type(exc).__name__
|
|
228
|
+
raise ReviewCandidateError(f"cannot read {kind}: {detail}") from exc
|
|
229
|
+
if len(raw) > MAX_REPORT_BYTES:
|
|
230
|
+
raise ReviewCandidateError(
|
|
231
|
+
f"{kind} exceeds the {MAX_REPORT_BYTES}-byte input limit"
|
|
232
|
+
)
|
|
233
|
+
return raw
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _candidate_tuple(candidate: ReviewCandidate) -> tuple[str, str, int, str, str]:
|
|
237
|
+
return (
|
|
238
|
+
candidate.run_id,
|
|
239
|
+
candidate.pattern_id,
|
|
240
|
+
candidate.pattern_version,
|
|
241
|
+
candidate.tool_key,
|
|
242
|
+
candidate.artifact_digest,
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
@functools.lru_cache(maxsize=4096)
|
|
247
|
+
def _candidate_id_for(identity_tuple: tuple[str, str, int, str, str]) -> str:
|
|
248
|
+
identity = json.dumps(
|
|
249
|
+
[TRACEGRAPH_SCHEMA_VERSION, TRACEGRAPH_KIND, *identity_tuple],
|
|
250
|
+
ensure_ascii=False,
|
|
251
|
+
separators=(",", ":"),
|
|
252
|
+
)
|
|
253
|
+
return str(uuid.uuid5(_ITEM_NAMESPACE, identity))
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def candidate_id(candidate: ReviewCandidate) -> str:
|
|
257
|
+
"""Return the UUIDv5 used by the merged SyncMill board consumer.
|
|
258
|
+
|
|
259
|
+
Cached on the identity tuple: one review pass recomputes the same ids
|
|
260
|
+
in load, event validation, and payload rendering.
|
|
261
|
+
"""
|
|
262
|
+
return _candidate_id_for(_candidate_tuple(candidate))
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def load_report(path: str | Path) -> LoadedReviewReport:
|
|
266
|
+
"""Load, project, sort, and bind one complete Tracegraph report."""
|
|
267
|
+
raw = _read_capped(Path(path), "review-candidate report")
|
|
268
|
+
try:
|
|
269
|
+
report = TracegraphReviewReport.model_validate_json(raw)
|
|
270
|
+
except ValidationError as exc:
|
|
271
|
+
raise _safe_validation_error("review-candidate report", exc) from exc
|
|
272
|
+
|
|
273
|
+
candidates = sorted(report.candidates, key=_candidate_tuple)
|
|
274
|
+
ids = [candidate_id(candidate) for candidate in candidates]
|
|
275
|
+
if len(ids) != len(set(ids)):
|
|
276
|
+
raise ReviewCandidateError("review-candidate report contains a duplicate candidate")
|
|
277
|
+
return LoadedReviewReport(
|
|
278
|
+
source_report_digest=_sha256(raw),
|
|
279
|
+
candidates=tuple(candidates),
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def default_annotations_path(report_path: str | Path) -> Path:
|
|
284
|
+
return Path(f"{Path(report_path)}.toolgraph-review.json")
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _same_path(left: Path, right: Path) -> bool:
|
|
288
|
+
try:
|
|
289
|
+
if left.exists() and right.exists():
|
|
290
|
+
return os.path.samefile(left, right)
|
|
291
|
+
except OSError:
|
|
292
|
+
pass
|
|
293
|
+
return left.resolve() == right.resolve()
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _new_annotations(report: LoadedReviewReport) -> AnnotationReport:
|
|
297
|
+
return AnnotationReport(
|
|
298
|
+
schema_version=ANNOTATION_SCHEMA_VERSION,
|
|
299
|
+
kind=ANNOTATION_KIND,
|
|
300
|
+
source=AnnotationSource(
|
|
301
|
+
schema_version=TRACEGRAPH_SCHEMA_VERSION,
|
|
302
|
+
kind=TRACEGRAPH_KIND,
|
|
303
|
+
source_report_digest=report.source_report_digest,
|
|
304
|
+
),
|
|
305
|
+
events=[],
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _load_annotations(path: Path, report: LoadedReviewReport) -> AnnotationReport:
|
|
310
|
+
if not path.exists():
|
|
311
|
+
return _new_annotations(report)
|
|
312
|
+
raw = _read_capped(path, "review annotations")
|
|
313
|
+
try:
|
|
314
|
+
annotations = AnnotationReport.model_validate_json(raw)
|
|
315
|
+
except ValidationError as exc:
|
|
316
|
+
raise _safe_validation_error("review annotations", exc) from exc
|
|
317
|
+
if annotations.source.source_report_digest != report.source_report_digest:
|
|
318
|
+
raise ReviewCandidateError("review annotations are bound to a different source report")
|
|
319
|
+
return annotations
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _validate_events(
|
|
323
|
+
annotations: AnnotationReport,
|
|
324
|
+
report: LoadedReviewReport,
|
|
325
|
+
) -> tuple[dict[str, Disposition], dict[str, list[AnnotationEvent]]]:
|
|
326
|
+
candidates = {candidate_id(candidate): candidate for candidate in report.candidates}
|
|
327
|
+
states: dict[str, Disposition] = {item_id: "open" for item_id in candidates}
|
|
328
|
+
histories: dict[str, list[AnnotationEvent]] = {item_id: [] for item_id in candidates}
|
|
329
|
+
# Dump each source candidate once, not once per referencing event.
|
|
330
|
+
source_dumps = {
|
|
331
|
+
item_id: candidate.model_dump() for item_id, candidate in candidates.items()
|
|
332
|
+
}
|
|
333
|
+
for expected_sequence, event in enumerate(annotations.events, start=1):
|
|
334
|
+
if event.sequence != expected_sequence:
|
|
335
|
+
raise ReviewCandidateError("review annotation event sequence is not contiguous — the sidecar no longer satisfies its append-only contract; restore it from a backup, or delete the sidecar file to reset review history")
|
|
336
|
+
source_candidate = candidates.get(event.candidate_id)
|
|
337
|
+
if source_candidate is None:
|
|
338
|
+
raise ReviewCandidateError("review annotation references an unknown candidate")
|
|
339
|
+
if (
|
|
340
|
+
event.candidate.model_dump() != source_dumps[event.candidate_id]
|
|
341
|
+
or candidate_id(event.candidate) != event.candidate_id
|
|
342
|
+
):
|
|
343
|
+
raise ReviewCandidateError("review annotation candidate snapshot does not match its id — the sidecar no longer satisfies its append-only contract; restore it from a backup, or delete the sidecar file to reset review history")
|
|
344
|
+
if event.from_disposition != states[event.candidate_id]:
|
|
345
|
+
raise ReviewCandidateError("review annotation disposition chain is invalid — the sidecar no longer satisfies its append-only contract; restore it from a backup, or delete the sidecar file to reset review history")
|
|
346
|
+
states[event.candidate_id] = event.disposition
|
|
347
|
+
histories[event.candidate_id].append(event)
|
|
348
|
+
return states, histories
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _history_payload(event: AnnotationEvent) -> dict:
|
|
352
|
+
return {
|
|
353
|
+
"sequence": event.sequence,
|
|
354
|
+
"from_disposition": event.from_disposition,
|
|
355
|
+
"disposition": event.disposition,
|
|
356
|
+
"reviewer": event.reviewer,
|
|
357
|
+
"recorded_at": event.recorded_at,
|
|
358
|
+
"note": event.note,
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _candidate_payload(
|
|
363
|
+
candidate: ReviewCandidate,
|
|
364
|
+
disposition: Disposition,
|
|
365
|
+
history: list[AnnotationEvent],
|
|
366
|
+
) -> dict:
|
|
367
|
+
return {
|
|
368
|
+
"candidate_id": candidate_id(candidate),
|
|
369
|
+
**candidate.model_dump(mode="json"),
|
|
370
|
+
"disposition": disposition,
|
|
371
|
+
"history": [_history_payload(event) for event in history],
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def list_candidates(
|
|
376
|
+
report_path: str | Path,
|
|
377
|
+
*,
|
|
378
|
+
annotations_path: str | Path | None = None,
|
|
379
|
+
status: str | None = None,
|
|
380
|
+
) -> dict:
|
|
381
|
+
"""Return a deterministic merged view without creating or changing files."""
|
|
382
|
+
if status is not None and status not in DISPOSITIONS:
|
|
383
|
+
raise ReviewCandidateError(f"unknown review disposition {status!r}")
|
|
384
|
+
report_file = Path(report_path)
|
|
385
|
+
annotation_file = (
|
|
386
|
+
Path(annotations_path)
|
|
387
|
+
if annotations_path is not None
|
|
388
|
+
else default_annotations_path(report_file)
|
|
389
|
+
)
|
|
390
|
+
if _same_path(report_file, annotation_file):
|
|
391
|
+
raise ReviewCandidateError("review annotations must not overwrite the source report")
|
|
392
|
+
annotation_file = annotation_file.resolve()
|
|
393
|
+
report = load_report(report_file)
|
|
394
|
+
annotations = _load_annotations(annotation_file, report)
|
|
395
|
+
states, histories = _validate_events(annotations, report)
|
|
396
|
+
candidates = []
|
|
397
|
+
for candidate in report.candidates:
|
|
398
|
+
item_id = candidate_id(candidate)
|
|
399
|
+
if status is None or states[item_id] == status:
|
|
400
|
+
candidates.append(
|
|
401
|
+
_candidate_payload(candidate, states[item_id], histories[item_id])
|
|
402
|
+
)
|
|
403
|
+
return {
|
|
404
|
+
"schema_version": ANNOTATION_SCHEMA_VERSION,
|
|
405
|
+
"kind": LIST_KIND,
|
|
406
|
+
"source_report_digest": report.source_report_digest,
|
|
407
|
+
"candidates": candidates,
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def normalize_reviewer(value: str) -> str:
|
|
412
|
+
try:
|
|
413
|
+
return _normalized_reviewer(value)
|
|
414
|
+
except ValueError as exc:
|
|
415
|
+
raise ReviewCandidateError(str(exc)) from exc
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def normalize_note(value: str | None) -> str | None:
|
|
419
|
+
try:
|
|
420
|
+
return _normalized_note(value)
|
|
421
|
+
except ValueError as exc:
|
|
422
|
+
raise ReviewCandidateError(str(exc)) from exc
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
@contextmanager
|
|
426
|
+
def _writer_lock(path: Path) -> Iterator[None]:
|
|
427
|
+
"""Serialize local writers; the OS releases this lock after a crash."""
|
|
428
|
+
# The .lock file is deliberately never removed: unlinking a file another
|
|
429
|
+
# process is about to flock() reintroduces the race this lock exists to
|
|
430
|
+
# close. One empty file per sidecar is the accepted cost.
|
|
431
|
+
# Resolve aliases before deriving the lock name. Otherwise two processes
|
|
432
|
+
# can address one sidecar through real/symlink paths and take distinct locks.
|
|
433
|
+
lock_path = Path(f"{path.resolve()}.lock")
|
|
434
|
+
try:
|
|
435
|
+
handle = lock_path.open("a+b")
|
|
436
|
+
except OSError as exc:
|
|
437
|
+
detail = exc.strerror or type(exc).__name__
|
|
438
|
+
raise ReviewCandidateError(f"cannot lock review annotations: {detail}") from exc
|
|
439
|
+
with handle:
|
|
440
|
+
try:
|
|
441
|
+
if os.name == "nt": # pragma: no cover - platform-specific
|
|
442
|
+
handle.seek(0)
|
|
443
|
+
if not handle.read(1):
|
|
444
|
+
handle.write(b"\0")
|
|
445
|
+
handle.flush()
|
|
446
|
+
handle.seek(0)
|
|
447
|
+
msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
|
|
448
|
+
else:
|
|
449
|
+
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
|
450
|
+
except OSError as exc:
|
|
451
|
+
detail = exc.strerror or type(exc).__name__
|
|
452
|
+
raise ReviewCandidateError(f"cannot lock review annotations: {detail}") from exc
|
|
453
|
+
try:
|
|
454
|
+
yield
|
|
455
|
+
finally:
|
|
456
|
+
if os.name == "nt": # pragma: no cover - platform-specific
|
|
457
|
+
handle.seek(0)
|
|
458
|
+
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
|
459
|
+
else:
|
|
460
|
+
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _save_atomic(annotations: AnnotationReport, path: Path) -> None:
|
|
464
|
+
payload = json.dumps(
|
|
465
|
+
annotations.model_dump(mode="json"), indent=2, sort_keys=True
|
|
466
|
+
) + "\n"
|
|
467
|
+
temporary: str | None = None
|
|
468
|
+
try:
|
|
469
|
+
with tempfile.NamedTemporaryFile(
|
|
470
|
+
mode="w",
|
|
471
|
+
encoding="utf-8",
|
|
472
|
+
newline="\n",
|
|
473
|
+
dir=path.parent,
|
|
474
|
+
prefix=f".{path.name}.",
|
|
475
|
+
suffix=".tmp",
|
|
476
|
+
delete=False,
|
|
477
|
+
) as handle:
|
|
478
|
+
temporary = handle.name
|
|
479
|
+
handle.write(payload)
|
|
480
|
+
handle.flush()
|
|
481
|
+
os.fsync(handle.fileno())
|
|
482
|
+
os.replace(temporary, path)
|
|
483
|
+
try:
|
|
484
|
+
fsync_parent_dir(path)
|
|
485
|
+
except OSError:
|
|
486
|
+
# The event is already atomically visible. Reporting total failure
|
|
487
|
+
# would invite a retry that appends a duplicate event; surface the
|
|
488
|
+
# narrower durability uncertainty while returning persisted state.
|
|
489
|
+
warnings.warn(
|
|
490
|
+
"review annotations were replaced, but parent-directory fsync "
|
|
491
|
+
"failed; the event is persisted but crash durability is uncertain",
|
|
492
|
+
ReviewCandidateDurabilityWarning,
|
|
493
|
+
stacklevel=2,
|
|
494
|
+
)
|
|
495
|
+
except BaseException:
|
|
496
|
+
if temporary is not None:
|
|
497
|
+
Path(temporary).unlink(missing_ok=True)
|
|
498
|
+
raise
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def annotate_candidate(
|
|
502
|
+
report_path: str | Path,
|
|
503
|
+
item_id: str,
|
|
504
|
+
*,
|
|
505
|
+
disposition: str,
|
|
506
|
+
reviewer: str,
|
|
507
|
+
note: str | None = None,
|
|
508
|
+
annotations_path: str | Path | None = None,
|
|
509
|
+
recorded_at: datetime | None = None,
|
|
510
|
+
) -> dict:
|
|
511
|
+
"""Append one human disposition and return the candidate's updated view."""
|
|
512
|
+
if disposition not in DISPOSITIONS:
|
|
513
|
+
raise ReviewCandidateError(f"unknown review disposition {disposition!r}")
|
|
514
|
+
reviewer = normalize_reviewer(reviewer)
|
|
515
|
+
note = normalize_note(note)
|
|
516
|
+
report_file = Path(report_path)
|
|
517
|
+
annotation_file = (
|
|
518
|
+
Path(annotations_path)
|
|
519
|
+
if annotations_path is not None
|
|
520
|
+
else default_annotations_path(report_file)
|
|
521
|
+
)
|
|
522
|
+
if _same_path(report_file, annotation_file):
|
|
523
|
+
raise ReviewCandidateError("review annotations must not overwrite the source report")
|
|
524
|
+
# From this point forward, load, lock, temp-file placement, and replacement
|
|
525
|
+
# must all address one canonical target. Replacing a symlink path would
|
|
526
|
+
# otherwise split the alias from the original sidecar after the first write.
|
|
527
|
+
annotation_file = annotation_file.resolve()
|
|
528
|
+
try:
|
|
529
|
+
annotation_file.parent.mkdir(parents=True, exist_ok=True)
|
|
530
|
+
except OSError as exc:
|
|
531
|
+
detail = exc.strerror or type(exc).__name__
|
|
532
|
+
raise ReviewCandidateError(
|
|
533
|
+
f"cannot create review annotation directory: {detail}"
|
|
534
|
+
) from exc
|
|
535
|
+
|
|
536
|
+
with _writer_lock(annotation_file):
|
|
537
|
+
report = load_report(report_file)
|
|
538
|
+
annotations = _load_annotations(annotation_file, report)
|
|
539
|
+
states, histories = _validate_events(annotations, report)
|
|
540
|
+
candidates = {candidate_id(candidate): candidate for candidate in report.candidates}
|
|
541
|
+
candidate = candidates.get(item_id)
|
|
542
|
+
if candidate is None:
|
|
543
|
+
raise ReviewCandidateError("candidate id is not present in the source report")
|
|
544
|
+
|
|
545
|
+
timestamp = recorded_at or datetime.now(timezone.utc)
|
|
546
|
+
if timestamp.tzinfo is None or timestamp.utcoffset() is None:
|
|
547
|
+
raise ReviewCandidateError("recorded_at must include a timezone")
|
|
548
|
+
timestamp = timestamp.astimezone(timezone.utc)
|
|
549
|
+
event = AnnotationEvent(
|
|
550
|
+
sequence=len(annotations.events) + 1,
|
|
551
|
+
candidate_id=item_id,
|
|
552
|
+
candidate=candidate.model_dump(mode="json"),
|
|
553
|
+
from_disposition=states[item_id],
|
|
554
|
+
disposition=disposition,
|
|
555
|
+
reviewer=reviewer,
|
|
556
|
+
recorded_at=timestamp.isoformat(),
|
|
557
|
+
note=note,
|
|
558
|
+
)
|
|
559
|
+
updated = AnnotationReport(
|
|
560
|
+
schema_version=ANNOTATION_SCHEMA_VERSION,
|
|
561
|
+
kind=ANNOTATION_KIND,
|
|
562
|
+
source=annotations.source,
|
|
563
|
+
events=[*annotations.events, event],
|
|
564
|
+
)
|
|
565
|
+
|
|
566
|
+
# The source is immutable evidence. Catch a concurrent rewrite before
|
|
567
|
+
# publishing a sidecar that claims to describe different bytes. A
|
|
568
|
+
# digest comparison over the raw bytes is sufficient — any content
|
|
569
|
+
# change alters the digest — and skips re-validating the whole report.
|
|
570
|
+
current_raw = _read_capped(report_file, "review-candidate report")
|
|
571
|
+
if _sha256(current_raw) != report.source_report_digest:
|
|
572
|
+
raise ReviewCandidateError("source report changed while annotation was being written")
|
|
573
|
+
try:
|
|
574
|
+
_save_atomic(updated, annotation_file)
|
|
575
|
+
except OSError as exc:
|
|
576
|
+
detail = exc.strerror or type(exc).__name__
|
|
577
|
+
raise ReviewCandidateError(f"cannot write review annotations: {detail}") from exc
|
|
578
|
+
|
|
579
|
+
return _candidate_payload(candidate, disposition, [*histories[item_id], event])
|
|
File without changes
|