sourcebound 1.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clean_docs/__init__.py +26 -0
- clean_docs/__main__.py +3 -0
- clean_docs/accessibility.py +182 -0
- clean_docs/adapters/__init__.py +1 -0
- clean_docs/adapters/event_capture.py +56 -0
- clean_docs/adapters/mdx_dependencies.json +714 -0
- clean_docs/adapters/mdx_parser.mjs +29992 -0
- clean_docs/applicability.py +330 -0
- clean_docs/audit.py +1120 -0
- clean_docs/bootstrap.py +507 -0
- clean_docs/capabilities.py +200 -0
- clean_docs/changed.py +452 -0
- clean_docs/claims.py +840 -0
- clean_docs/cli.py +1612 -0
- clean_docs/context.py +307 -0
- clean_docs/corpus.py +377 -0
- clean_docs/demo.py +369 -0
- clean_docs/doctor.py +184 -0
- clean_docs/emit/__init__.py +4 -0
- clean_docs/emit/llms_txt.py +102 -0
- clean_docs/emit/stepwise.py +168 -0
- clean_docs/engine.py +324 -0
- clean_docs/errors.py +20 -0
- clean_docs/evaluation.py +867 -0
- clean_docs/execution.py +138 -0
- clean_docs/explain.py +123 -0
- clean_docs/extractors/__init__.py +19 -0
- clean_docs/extractors/command.py +51 -0
- clean_docs/extractors/inventory.py +176 -0
- clean_docs/extractors/json_pointer.py +84 -0
- clean_docs/extractors/python_literal.py +104 -0
- clean_docs/extractors/static.py +111 -0
- clean_docs/feedback.py +1390 -0
- clean_docs/impact.py +1624 -0
- clean_docs/improvements.py +1178 -0
- clean_docs/inventory.py +474 -0
- clean_docs/isolation.py +157 -0
- clean_docs/manifest.py +898 -0
- clean_docs/mdx.py +272 -0
- clean_docs/migration.py +121 -0
- clean_docs/models.py +194 -0
- clean_docs/outcomes.py +296 -0
- clean_docs/performance.py +123 -0
- clean_docs/phrasing.py +448 -0
- clean_docs/plugins.py +249 -0
- clean_docs/policy.py +536 -0
- clean_docs/projections.py +255 -0
- clean_docs/regions.py +75 -0
- clean_docs/release.py +232 -0
- clean_docs/renderers.py +57 -0
- clean_docs/residue.py +311 -0
- clean_docs/review_contracts.py +862 -0
- clean_docs/review_ledger.py +297 -0
- clean_docs/review_limits.py +9 -0
- clean_docs/sensitivity.py +602 -0
- clean_docs/snapshot.py +212 -0
- clean_docs/standard.py +281 -0
- clean_docs/standards/default.json +309 -0
- clean_docs/standards/exemplars.md +87 -0
- clean_docs/standards/v0-migrations.json +26 -0
- clean_docs/symbols.py +47 -0
- clean_docs/templates.py +96 -0
- clean_docs/verdict.py +1397 -0
- clean_docs/visuals.py +346 -0
- clean_docs/write_gate.py +170 -0
- sourcebound-1.2.1.dist-info/LICENSE +21 -0
- sourcebound-1.2.1.dist-info/METADATA +109 -0
- sourcebound-1.2.1.dist-info/RECORD +71 -0
- sourcebound-1.2.1.dist-info/WHEEL +5 -0
- sourcebound-1.2.1.dist-info/entry_points.txt +2 -0
- sourcebound-1.2.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,862 @@
|
|
|
1
|
+
"""Evaluate review-only source-to-document contracts at immutable Git refs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import ast
|
|
6
|
+
import copy
|
|
7
|
+
import hashlib
|
|
8
|
+
import html
|
|
9
|
+
import json
|
|
10
|
+
import math
|
|
11
|
+
import re
|
|
12
|
+
import subprocess
|
|
13
|
+
from dataclasses import asdict, dataclass, field
|
|
14
|
+
from datetime import date, datetime, time
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
try:
|
|
18
|
+
import tomllib # type: ignore[import-not-found]
|
|
19
|
+
except ModuleNotFoundError:
|
|
20
|
+
import tomli as tomllib # type: ignore[import-not-found, no-redef]
|
|
21
|
+
|
|
22
|
+
import yaml
|
|
23
|
+
|
|
24
|
+
from clean_docs.errors import ExtractionError
|
|
25
|
+
from clean_docs.mdx import MdxDocument, MdxParserError, parse_mdx_documents
|
|
26
|
+
from clean_docs.models import ReviewContract, ReviewLocator
|
|
27
|
+
from clean_docs.review_limits import (
|
|
28
|
+
MAX_REVIEW_FILE_BYTES,
|
|
29
|
+
MAX_REVIEW_STRUCTURED_NODES,
|
|
30
|
+
MAX_REVIEW_TOTAL_BYTES,
|
|
31
|
+
)
|
|
32
|
+
from clean_docs.snapshot import RepositorySnapshot
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
_MARKDOWN_MARKUP = re.compile(r"[`*_~]")
|
|
36
|
+
_HTML_TAG = re.compile(r"<[^>]*>")
|
|
37
|
+
_HTML_COMMENT = re.compile(br"<!--.*?-->", re.DOTALL)
|
|
38
|
+
_POINTER_ESCAPE = re.compile(r"~(?:0|1)")
|
|
39
|
+
_CHANGED_STATES = frozenset({"added", "changed", "removed"})
|
|
40
|
+
_SUBSTANTIVE_TARGET_STATES = frozenset({"added", "changed"})
|
|
41
|
+
_MASKED_MARKDOWN_NODE_TYPES = frozenset(
|
|
42
|
+
{
|
|
43
|
+
"code",
|
|
44
|
+
"yaml",
|
|
45
|
+
"mdxjsEsm",
|
|
46
|
+
"mdxFlowExpression",
|
|
47
|
+
"mdxTextExpression",
|
|
48
|
+
}
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class _LocatorMissing(Exception):
|
|
53
|
+
pass
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class _LocatorUnresolved(Exception):
|
|
57
|
+
pass
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class ReviewLocatorEvidence:
|
|
62
|
+
id: str
|
|
63
|
+
path: str
|
|
64
|
+
extractor: str
|
|
65
|
+
locator: str
|
|
66
|
+
base_digest: str | None
|
|
67
|
+
head_digest: str | None
|
|
68
|
+
state: str
|
|
69
|
+
|
|
70
|
+
def as_dict(self) -> dict[str, object]:
|
|
71
|
+
return asdict(self)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True)
|
|
75
|
+
class ReviewContractResult:
|
|
76
|
+
contract_id: str
|
|
77
|
+
mode: str
|
|
78
|
+
state: str
|
|
79
|
+
sources: tuple[ReviewLocatorEvidence, ...]
|
|
80
|
+
targets: tuple[ReviewLocatorEvidence, ...]
|
|
81
|
+
semantic_correctness_checked: bool = False
|
|
82
|
+
|
|
83
|
+
def as_dict(self) -> dict[str, object]:
|
|
84
|
+
return {
|
|
85
|
+
"id": self.contract_id,
|
|
86
|
+
"mode": self.mode,
|
|
87
|
+
"state": self.state,
|
|
88
|
+
"sources": [item.as_dict() for item in self.sources],
|
|
89
|
+
"targets": [item.as_dict() for item in self.targets],
|
|
90
|
+
"semantic_correctness_checked": self.semantic_correctness_checked,
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _sha256(value: str) -> str:
|
|
95
|
+
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _validate_path(path: Path) -> None:
|
|
99
|
+
if path.is_absolute() or ".." in path.parts:
|
|
100
|
+
raise _LocatorUnresolved
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _path_size(root: Path, ref: str, path: Path) -> int | None:
|
|
104
|
+
try:
|
|
105
|
+
process = subprocess.run(
|
|
106
|
+
["git", "-C", str(root), "cat-file", "-s", f"{ref}:{path.as_posix()}"],
|
|
107
|
+
text=True,
|
|
108
|
+
capture_output=True,
|
|
109
|
+
timeout=30,
|
|
110
|
+
check=False,
|
|
111
|
+
)
|
|
112
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
113
|
+
raise _LocatorUnresolved from exc
|
|
114
|
+
if process.returncode != 0:
|
|
115
|
+
return None
|
|
116
|
+
try:
|
|
117
|
+
size = int(process.stdout.strip())
|
|
118
|
+
except ValueError as exc:
|
|
119
|
+
raise _LocatorUnresolved from exc
|
|
120
|
+
if size < 0:
|
|
121
|
+
raise _LocatorUnresolved
|
|
122
|
+
return size
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@dataclass(frozen=True)
|
|
126
|
+
class _TextSnapshot:
|
|
127
|
+
resolution: str
|
|
128
|
+
text: str | None = None
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
_ContentKey = tuple[str, str]
|
|
132
|
+
_DigestKey = tuple[str, str, str, str]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@dataclass
|
|
136
|
+
class _ReviewCache:
|
|
137
|
+
total_bytes: int = 0
|
|
138
|
+
texts: dict[_ContentKey, _TextSnapshot] = field(default_factory=dict)
|
|
139
|
+
python_trees: dict[_ContentKey, ast.Module | None] = field(default_factory=dict)
|
|
140
|
+
structured_values: dict[_ContentKey, object] = field(default_factory=dict)
|
|
141
|
+
structured_failures: set[_ContentKey] = field(default_factory=set)
|
|
142
|
+
digests: dict[_DigestKey, tuple[str, str | None]] = field(
|
|
143
|
+
default_factory=dict
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
@staticmethod
|
|
147
|
+
def key(snapshot: RepositorySnapshot, path: Path) -> _ContentKey:
|
|
148
|
+
return (snapshot.label, path.as_posix())
|
|
149
|
+
|
|
150
|
+
def text(self, snapshot: RepositorySnapshot, path: Path) -> str:
|
|
151
|
+
_validate_path(path)
|
|
152
|
+
key = self.key(snapshot, path)
|
|
153
|
+
cached = self.texts.get(key)
|
|
154
|
+
if cached is None:
|
|
155
|
+
size = _path_size(snapshot.root, snapshot.label, path)
|
|
156
|
+
if size is None:
|
|
157
|
+
cached = _TextSnapshot("missing")
|
|
158
|
+
elif (
|
|
159
|
+
size > MAX_REVIEW_FILE_BYTES
|
|
160
|
+
or self.total_bytes + size > MAX_REVIEW_TOTAL_BYTES
|
|
161
|
+
):
|
|
162
|
+
cached = _TextSnapshot("unresolved")
|
|
163
|
+
else:
|
|
164
|
+
try:
|
|
165
|
+
text = snapshot.read_text(path)
|
|
166
|
+
except (ExtractionError, UnicodeDecodeError):
|
|
167
|
+
cached = _TextSnapshot("unresolved")
|
|
168
|
+
else:
|
|
169
|
+
if len(text.encode("utf-8")) != size:
|
|
170
|
+
cached = _TextSnapshot("unresolved")
|
|
171
|
+
else:
|
|
172
|
+
self.total_bytes += size
|
|
173
|
+
cached = _TextSnapshot("resolved", text)
|
|
174
|
+
self.texts[key] = cached
|
|
175
|
+
if cached.resolution == "missing":
|
|
176
|
+
raise _LocatorMissing
|
|
177
|
+
if cached.resolution != "resolved" or cached.text is None:
|
|
178
|
+
raise _LocatorUnresolved
|
|
179
|
+
return cached.text
|
|
180
|
+
|
|
181
|
+
def python_tree(
|
|
182
|
+
self,
|
|
183
|
+
snapshot: RepositorySnapshot,
|
|
184
|
+
path: Path,
|
|
185
|
+
text: str,
|
|
186
|
+
) -> ast.Module:
|
|
187
|
+
key = self.key(snapshot, path)
|
|
188
|
+
if key not in self.python_trees:
|
|
189
|
+
try:
|
|
190
|
+
self.python_trees[key] = ast.parse(text)
|
|
191
|
+
except SyntaxError:
|
|
192
|
+
self.python_trees[key] = None
|
|
193
|
+
tree = self.python_trees[key]
|
|
194
|
+
if tree is None:
|
|
195
|
+
raise _LocatorUnresolved
|
|
196
|
+
return tree
|
|
197
|
+
|
|
198
|
+
def structured_value(
|
|
199
|
+
self,
|
|
200
|
+
snapshot: RepositorySnapshot,
|
|
201
|
+
path: Path,
|
|
202
|
+
text: str,
|
|
203
|
+
) -> object:
|
|
204
|
+
key = self.key(snapshot, path)
|
|
205
|
+
if key not in self.structured_values and key not in self.structured_failures:
|
|
206
|
+
try:
|
|
207
|
+
self.structured_values[key] = _load_structured(path, text)
|
|
208
|
+
except _LocatorUnresolved:
|
|
209
|
+
self.structured_failures.add(key)
|
|
210
|
+
if key in self.structured_failures:
|
|
211
|
+
raise _LocatorUnresolved
|
|
212
|
+
return self.structured_values[key]
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _assignment_names(node: ast.AST) -> set[str]:
|
|
216
|
+
targets: list[ast.expr] = []
|
|
217
|
+
if isinstance(node, ast.Assign):
|
|
218
|
+
targets.extend(node.targets)
|
|
219
|
+
elif isinstance(node, ast.AnnAssign):
|
|
220
|
+
targets.append(node.target)
|
|
221
|
+
|
|
222
|
+
names: set[str] = set()
|
|
223
|
+
|
|
224
|
+
def collect(target: ast.expr) -> None:
|
|
225
|
+
if isinstance(target, ast.Name):
|
|
226
|
+
names.add(target.id)
|
|
227
|
+
elif isinstance(target, (ast.Tuple, ast.List)):
|
|
228
|
+
for item in target.elts:
|
|
229
|
+
collect(item)
|
|
230
|
+
|
|
231
|
+
for target in targets:
|
|
232
|
+
collect(target)
|
|
233
|
+
return names
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _scope_body(node: ast.AST) -> list[ast.stmt] | None:
|
|
237
|
+
if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
238
|
+
return node.body
|
|
239
|
+
return None
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _find_python_symbol(tree: ast.Module, locator: str) -> ast.AST:
|
|
243
|
+
current: ast.AST = tree
|
|
244
|
+
for part in locator.split("."):
|
|
245
|
+
body = _scope_body(current)
|
|
246
|
+
if body is None:
|
|
247
|
+
raise _LocatorMissing
|
|
248
|
+
matches = [
|
|
249
|
+
node
|
|
250
|
+
for node in body
|
|
251
|
+
if (
|
|
252
|
+
isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef))
|
|
253
|
+
and node.name == part
|
|
254
|
+
)
|
|
255
|
+
or part in _assignment_names(node)
|
|
256
|
+
]
|
|
257
|
+
if not matches:
|
|
258
|
+
raise _LocatorMissing
|
|
259
|
+
if len(matches) != 1:
|
|
260
|
+
raise _LocatorUnresolved
|
|
261
|
+
current = matches[0]
|
|
262
|
+
return current
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _is_docstring_statement(node: ast.stmt) -> bool:
|
|
266
|
+
return (
|
|
267
|
+
isinstance(node, ast.Expr)
|
|
268
|
+
and isinstance(node.value, ast.Constant)
|
|
269
|
+
and isinstance(node.value.value, str)
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
class _DocstringStripper(ast.NodeTransformer):
|
|
274
|
+
@staticmethod
|
|
275
|
+
def _strip(node: ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef) -> ast.AST:
|
|
276
|
+
if node.body and _is_docstring_statement(node.body[0]):
|
|
277
|
+
node.body = node.body[1:]
|
|
278
|
+
return node
|
|
279
|
+
|
|
280
|
+
def visit_ClassDef(self, node: ast.ClassDef) -> ast.AST:
|
|
281
|
+
self.generic_visit(node)
|
|
282
|
+
return self._strip(node)
|
|
283
|
+
|
|
284
|
+
def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST:
|
|
285
|
+
self.generic_visit(node)
|
|
286
|
+
return self._strip(node)
|
|
287
|
+
|
|
288
|
+
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> ast.AST:
|
|
289
|
+
self.generic_visit(node)
|
|
290
|
+
return self._strip(node)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _normalized_ast(node: ast.AST) -> str:
|
|
294
|
+
normalized = _DocstringStripper().visit(copy.deepcopy(node))
|
|
295
|
+
assert normalized is not None
|
|
296
|
+
return ast.dump(normalized, annotate_fields=True, include_attributes=False)
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _python_symbol_digest(tree: ast.Module, locator: str) -> str:
|
|
300
|
+
node = _find_python_symbol(tree, locator)
|
|
301
|
+
return _sha256(_normalized_ast(node))
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _markdown_slug(title: str) -> str:
|
|
305
|
+
plain = html.unescape(_HTML_TAG.sub("", title))
|
|
306
|
+
plain = _MARKDOWN_MARKUP.sub("", plain).casefold()
|
|
307
|
+
plain = re.sub(r"[^\w\s-]", "", plain)
|
|
308
|
+
return re.sub(r"[\s-]+", "-", plain).strip("-")
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
@dataclass(frozen=True)
|
|
312
|
+
class _Heading:
|
|
313
|
+
start_byte: int
|
|
314
|
+
end_byte: int
|
|
315
|
+
level: int
|
|
316
|
+
slug: str
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
@dataclass(frozen=True)
|
|
320
|
+
class _MarkdownSnapshot:
|
|
321
|
+
resolution: str
|
|
322
|
+
text: str | None = None
|
|
323
|
+
document: MdxDocument | None = None
|
|
324
|
+
pre_masked_ranges: tuple[tuple[int, int], ...] = ()
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
_MarkdownKey = tuple[str, str]
|
|
328
|
+
_MarkdownSnapshots = dict[_MarkdownKey, _MarkdownSnapshot]
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _html_block(node_type: str, name: str | None) -> bool:
|
|
332
|
+
return (
|
|
333
|
+
node_type == "mdxJsxFlowElement"
|
|
334
|
+
and name is not None
|
|
335
|
+
and name[:1].islower()
|
|
336
|
+
)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _masked_ranges(
|
|
340
|
+
document: MdxDocument,
|
|
341
|
+
pre_masked_ranges: tuple[tuple[int, int], ...] = (),
|
|
342
|
+
) -> tuple[tuple[int, int], ...]:
|
|
343
|
+
return pre_masked_ranges + tuple(
|
|
344
|
+
(node.start_byte, node.end_byte)
|
|
345
|
+
for node in document.nodes
|
|
346
|
+
if (
|
|
347
|
+
node.type in _MASKED_MARKDOWN_NODE_TYPES
|
|
348
|
+
or _html_block(node.type, node.name)
|
|
349
|
+
)
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _inside_range(
|
|
354
|
+
start: int,
|
|
355
|
+
end: int,
|
|
356
|
+
ranges: tuple[tuple[int, int], ...],
|
|
357
|
+
) -> bool:
|
|
358
|
+
return any(
|
|
359
|
+
range_start <= start and end <= range_end
|
|
360
|
+
for range_start, range_end in ranges
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _markdown_headings(
|
|
365
|
+
document: MdxDocument,
|
|
366
|
+
pre_masked_ranges: tuple[tuple[int, int], ...],
|
|
367
|
+
) -> tuple[_Heading, ...]:
|
|
368
|
+
masked_ranges = _masked_ranges(document, pre_masked_ranges)
|
|
369
|
+
headings: list[_Heading] = []
|
|
370
|
+
for node in document.nodes:
|
|
371
|
+
if node.type != "heading" or _inside_range(
|
|
372
|
+
node.start_byte,
|
|
373
|
+
node.end_byte,
|
|
374
|
+
masked_ranges,
|
|
375
|
+
):
|
|
376
|
+
continue
|
|
377
|
+
if node.depth is None or node.text is None:
|
|
378
|
+
raise _LocatorUnresolved
|
|
379
|
+
headings.append(
|
|
380
|
+
_Heading(
|
|
381
|
+
node.start_byte,
|
|
382
|
+
node.end_byte,
|
|
383
|
+
node.depth,
|
|
384
|
+
_markdown_slug(node.text),
|
|
385
|
+
)
|
|
386
|
+
)
|
|
387
|
+
return tuple(sorted(headings, key=lambda item: item.start_byte))
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _visible_markdown(
|
|
391
|
+
text: str,
|
|
392
|
+
document: MdxDocument,
|
|
393
|
+
pre_masked_ranges: tuple[tuple[int, int], ...],
|
|
394
|
+
) -> bytes:
|
|
395
|
+
encoded = bytearray(text.encode("utf-8"))
|
|
396
|
+
for start, end in _masked_ranges(document, pre_masked_ranges):
|
|
397
|
+
if start < 0 or end < start or end > len(encoded):
|
|
398
|
+
raise _LocatorUnresolved
|
|
399
|
+
for index in range(start, end):
|
|
400
|
+
if encoded[index] not in {10, 13}:
|
|
401
|
+
encoded[index] = 32
|
|
402
|
+
return bytes(encoded)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _markdown_section_digest(
|
|
406
|
+
text: str,
|
|
407
|
+
document: MdxDocument,
|
|
408
|
+
pre_masked_ranges: tuple[tuple[int, int], ...],
|
|
409
|
+
locator: str,
|
|
410
|
+
) -> str:
|
|
411
|
+
requested = locator.removeprefix("#")
|
|
412
|
+
headings = _markdown_headings(document, pre_masked_ranges)
|
|
413
|
+
matches = [heading for heading in headings if heading.slug == requested]
|
|
414
|
+
if not matches:
|
|
415
|
+
raise _LocatorMissing
|
|
416
|
+
if len(matches) != 1:
|
|
417
|
+
raise _LocatorUnresolved
|
|
418
|
+
selected = matches[0]
|
|
419
|
+
end = len(text.encode("utf-8"))
|
|
420
|
+
for heading in headings:
|
|
421
|
+
if (
|
|
422
|
+
heading.start_byte > selected.start_byte
|
|
423
|
+
and heading.level <= selected.level
|
|
424
|
+
):
|
|
425
|
+
end = heading.start_byte
|
|
426
|
+
break
|
|
427
|
+
visible = _visible_markdown(text, document, pre_masked_ranges)
|
|
428
|
+
try:
|
|
429
|
+
section = visible[selected.end_byte:end].decode("utf-8")
|
|
430
|
+
except UnicodeDecodeError as exc:
|
|
431
|
+
raise _LocatorUnresolved from exc
|
|
432
|
+
tokens = re.findall(r"\S+", section)
|
|
433
|
+
return _sha256(" ".join(tokens))
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _load_structured(path: Path, text: str) -> object:
|
|
437
|
+
try:
|
|
438
|
+
if path.suffix.lower() == ".json":
|
|
439
|
+
return json.loads(text)
|
|
440
|
+
if path.suffix.lower() in {".yaml", ".yml"}:
|
|
441
|
+
return yaml.safe_load(text)
|
|
442
|
+
if path.suffix.lower() == ".toml":
|
|
443
|
+
return tomllib.loads(text)
|
|
444
|
+
except (json.JSONDecodeError, tomllib.TOMLDecodeError, yaml.YAMLError) as exc:
|
|
445
|
+
raise _LocatorUnresolved from exc
|
|
446
|
+
raise _LocatorUnresolved
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _decode_pointer(pointer: str) -> list[str]:
|
|
450
|
+
if not pointer.startswith("/"):
|
|
451
|
+
raise _LocatorUnresolved
|
|
452
|
+
tokens = pointer[1:].split("/")
|
|
453
|
+
decoded = []
|
|
454
|
+
for token in tokens:
|
|
455
|
+
if "~" in _POINTER_ESCAPE.sub("", token):
|
|
456
|
+
raise _LocatorUnresolved
|
|
457
|
+
decoded.append(token.replace("~1", "/").replace("~0", "~"))
|
|
458
|
+
return decoded
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _resolve_pointer(value: object, pointer: str) -> object:
|
|
462
|
+
current = value
|
|
463
|
+
for token in _decode_pointer(pointer):
|
|
464
|
+
if isinstance(current, dict):
|
|
465
|
+
if token not in current:
|
|
466
|
+
raise _LocatorMissing
|
|
467
|
+
current = current[token]
|
|
468
|
+
continue
|
|
469
|
+
if isinstance(current, list):
|
|
470
|
+
if not token.isdigit() or (len(token) > 1 and token.startswith("0")):
|
|
471
|
+
raise _LocatorMissing
|
|
472
|
+
index = int(token)
|
|
473
|
+
if index >= len(current):
|
|
474
|
+
raise _LocatorMissing
|
|
475
|
+
current = current[index]
|
|
476
|
+
continue
|
|
477
|
+
raise _LocatorMissing
|
|
478
|
+
return current
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
@dataclass
|
|
482
|
+
class _StructuredNodeBudget:
|
|
483
|
+
remaining: int = MAX_REVIEW_STRUCTURED_NODES
|
|
484
|
+
|
|
485
|
+
def consume(self) -> None:
|
|
486
|
+
self.remaining -= 1
|
|
487
|
+
if self.remaining < 0:
|
|
488
|
+
raise _LocatorUnresolved
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _canonical_value(
|
|
492
|
+
value: object,
|
|
493
|
+
*,
|
|
494
|
+
budget: _StructuredNodeBudget,
|
|
495
|
+
ancestors: frozenset[int] = frozenset(),
|
|
496
|
+
) -> object:
|
|
497
|
+
budget.consume()
|
|
498
|
+
if value is None or isinstance(value, (str, bool, int)):
|
|
499
|
+
return value
|
|
500
|
+
if isinstance(value, float):
|
|
501
|
+
if not math.isfinite(value):
|
|
502
|
+
raise _LocatorUnresolved
|
|
503
|
+
return value
|
|
504
|
+
if isinstance(value, (date, datetime, time)):
|
|
505
|
+
return {"type": type(value).__name__, "value": value.isoformat()}
|
|
506
|
+
if isinstance(value, list):
|
|
507
|
+
identity = id(value)
|
|
508
|
+
if identity in ancestors:
|
|
509
|
+
raise _LocatorUnresolved
|
|
510
|
+
nested = ancestors | {identity}
|
|
511
|
+
return [
|
|
512
|
+
_canonical_value(item, budget=budget, ancestors=nested)
|
|
513
|
+
for item in value
|
|
514
|
+
]
|
|
515
|
+
if isinstance(value, dict):
|
|
516
|
+
if not all(isinstance(key, str) for key in value):
|
|
517
|
+
raise _LocatorUnresolved
|
|
518
|
+
identity = id(value)
|
|
519
|
+
if identity in ancestors:
|
|
520
|
+
raise _LocatorUnresolved
|
|
521
|
+
nested = ancestors | {identity}
|
|
522
|
+
return {
|
|
523
|
+
key: _canonical_value(
|
|
524
|
+
value[key],
|
|
525
|
+
budget=budget,
|
|
526
|
+
ancestors=nested,
|
|
527
|
+
)
|
|
528
|
+
for key in sorted(value)
|
|
529
|
+
}
|
|
530
|
+
raise _LocatorUnresolved
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _structured_data_digest(value: object, locator: str) -> str:
|
|
534
|
+
selected = _resolve_pointer(value, locator)
|
|
535
|
+
encoded = json.dumps(
|
|
536
|
+
_canonical_value(
|
|
537
|
+
selected,
|
|
538
|
+
budget=_StructuredNodeBudget(),
|
|
539
|
+
),
|
|
540
|
+
sort_keys=True,
|
|
541
|
+
separators=(",", ":"),
|
|
542
|
+
ensure_ascii=False,
|
|
543
|
+
allow_nan=False,
|
|
544
|
+
)
|
|
545
|
+
return _sha256(encoded)
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _markdown_key(
|
|
549
|
+
snapshot: RepositorySnapshot,
|
|
550
|
+
path: Path,
|
|
551
|
+
) -> _MarkdownKey:
|
|
552
|
+
return (snapshot.label, path.as_posix())
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def _mask_html_comments(
|
|
556
|
+
text: str,
|
|
557
|
+
) -> tuple[str, tuple[tuple[int, int], ...]]:
|
|
558
|
+
encoded = bytearray(text.encode("utf-8"))
|
|
559
|
+
ranges = tuple(
|
|
560
|
+
(match.start(), match.end())
|
|
561
|
+
for match in _HTML_COMMENT.finditer(encoded)
|
|
562
|
+
)
|
|
563
|
+
for start, end in ranges:
|
|
564
|
+
for index in range(start, end):
|
|
565
|
+
if encoded[index] not in {10, 13}:
|
|
566
|
+
encoded[index] = 32
|
|
567
|
+
return encoded.decode("utf-8"), ranges
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _prepare_markdown_snapshots(
|
|
571
|
+
contracts: tuple[ReviewContract, ...],
|
|
572
|
+
base_snapshot: RepositorySnapshot,
|
|
573
|
+
head_snapshot: RepositorySnapshot,
|
|
574
|
+
cache: _ReviewCache,
|
|
575
|
+
) -> _MarkdownSnapshots:
|
|
576
|
+
paths = {
|
|
577
|
+
locator.path
|
|
578
|
+
for contract in contracts
|
|
579
|
+
for locator in contract.sources + contract.targets
|
|
580
|
+
if locator.extractor == "markdown-section"
|
|
581
|
+
}
|
|
582
|
+
snapshots = (base_snapshot, head_snapshot)
|
|
583
|
+
prepared: _MarkdownSnapshots = {}
|
|
584
|
+
documents: dict[str, str] = {}
|
|
585
|
+
identifiers: dict[str, _MarkdownKey] = {}
|
|
586
|
+
source_texts: dict[str, str] = {}
|
|
587
|
+
pre_masked_ranges: dict[str, tuple[tuple[int, int], ...]] = {}
|
|
588
|
+
|
|
589
|
+
for snapshot in snapshots:
|
|
590
|
+
for path in sorted(paths):
|
|
591
|
+
key = _markdown_key(snapshot, path)
|
|
592
|
+
if key in prepared or key in identifiers.values():
|
|
593
|
+
continue
|
|
594
|
+
try:
|
|
595
|
+
text = cache.text(snapshot, path)
|
|
596
|
+
except _LocatorMissing:
|
|
597
|
+
prepared[key] = _MarkdownSnapshot("missing")
|
|
598
|
+
continue
|
|
599
|
+
except _LocatorUnresolved:
|
|
600
|
+
prepared[key] = _MarkdownSnapshot("unresolved")
|
|
601
|
+
continue
|
|
602
|
+
identifier = f"review-document-{len(documents)}"
|
|
603
|
+
parse_text, ranges = _mask_html_comments(text)
|
|
604
|
+
documents[identifier] = parse_text
|
|
605
|
+
source_texts[identifier] = text
|
|
606
|
+
pre_masked_ranges[identifier] = ranges
|
|
607
|
+
identifiers[identifier] = key
|
|
608
|
+
|
|
609
|
+
if not documents:
|
|
610
|
+
return prepared
|
|
611
|
+
try:
|
|
612
|
+
parsed, errors = parse_mdx_documents(documents)
|
|
613
|
+
except MdxParserError:
|
|
614
|
+
for identifier, key in identifiers.items():
|
|
615
|
+
prepared[key] = _MarkdownSnapshot(
|
|
616
|
+
"unresolved",
|
|
617
|
+
text=source_texts[identifier],
|
|
618
|
+
pre_masked_ranges=pre_masked_ranges[identifier],
|
|
619
|
+
)
|
|
620
|
+
return prepared
|
|
621
|
+
|
|
622
|
+
for identifier, key in identifiers.items():
|
|
623
|
+
text = source_texts[identifier]
|
|
624
|
+
if identifier in errors:
|
|
625
|
+
prepared[key] = _MarkdownSnapshot(
|
|
626
|
+
"unresolved",
|
|
627
|
+
text=text,
|
|
628
|
+
pre_masked_ranges=pre_masked_ranges[identifier],
|
|
629
|
+
)
|
|
630
|
+
else:
|
|
631
|
+
prepared[key] = _MarkdownSnapshot(
|
|
632
|
+
"resolved",
|
|
633
|
+
text=text,
|
|
634
|
+
document=parsed[identifier],
|
|
635
|
+
pre_masked_ranges=pre_masked_ranges[identifier],
|
|
636
|
+
)
|
|
637
|
+
return prepared
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def _digest_at(
|
|
641
|
+
snapshot: RepositorySnapshot,
|
|
642
|
+
locator: ReviewLocator,
|
|
643
|
+
markdown_snapshots: _MarkdownSnapshots,
|
|
644
|
+
cache: _ReviewCache,
|
|
645
|
+
) -> tuple[str, str | None]:
|
|
646
|
+
key = (
|
|
647
|
+
snapshot.label,
|
|
648
|
+
locator.path.as_posix(),
|
|
649
|
+
locator.extractor,
|
|
650
|
+
locator.locator,
|
|
651
|
+
)
|
|
652
|
+
cached = cache.digests.get(key)
|
|
653
|
+
if cached is not None:
|
|
654
|
+
return cached
|
|
655
|
+
result: tuple[str, str | None]
|
|
656
|
+
try:
|
|
657
|
+
if locator.extractor == "markdown-section":
|
|
658
|
+
markdown = markdown_snapshots.get(
|
|
659
|
+
_markdown_key(snapshot, locator.path)
|
|
660
|
+
)
|
|
661
|
+
if markdown is None or markdown.resolution == "unresolved":
|
|
662
|
+
raise _LocatorUnresolved
|
|
663
|
+
if markdown.resolution == "missing":
|
|
664
|
+
raise _LocatorMissing
|
|
665
|
+
if markdown.text is None or markdown.document is None:
|
|
666
|
+
raise _LocatorUnresolved
|
|
667
|
+
digest = _markdown_section_digest(
|
|
668
|
+
markdown.text,
|
|
669
|
+
markdown.document,
|
|
670
|
+
markdown.pre_masked_ranges,
|
|
671
|
+
locator.locator,
|
|
672
|
+
)
|
|
673
|
+
else:
|
|
674
|
+
text = cache.text(snapshot, locator.path)
|
|
675
|
+
if locator.extractor == "python-symbol":
|
|
676
|
+
digest = _python_symbol_digest(
|
|
677
|
+
cache.python_tree(snapshot, locator.path, text),
|
|
678
|
+
locator.locator,
|
|
679
|
+
)
|
|
680
|
+
elif locator.extractor == "structured-data":
|
|
681
|
+
digest = _structured_data_digest(
|
|
682
|
+
cache.structured_value(snapshot, locator.path, text),
|
|
683
|
+
locator.locator,
|
|
684
|
+
)
|
|
685
|
+
elif locator.extractor != "markdown-section":
|
|
686
|
+
raise _LocatorUnresolved
|
|
687
|
+
except _LocatorMissing:
|
|
688
|
+
result = ("missing", None)
|
|
689
|
+
except _LocatorUnresolved:
|
|
690
|
+
result = ("unresolved", None)
|
|
691
|
+
else:
|
|
692
|
+
result = ("resolved", digest)
|
|
693
|
+
cache.digests[key] = result
|
|
694
|
+
return result
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def _locator_state(
|
|
698
|
+
base_resolution: str,
|
|
699
|
+
base_digest: str | None,
|
|
700
|
+
head_resolution: str,
|
|
701
|
+
head_digest: str | None,
|
|
702
|
+
) -> str:
|
|
703
|
+
if "unresolved" in {base_resolution, head_resolution}:
|
|
704
|
+
return "unknown"
|
|
705
|
+
if base_resolution == "missing" and head_resolution == "missing":
|
|
706
|
+
return "unknown"
|
|
707
|
+
if base_resolution == "missing":
|
|
708
|
+
return "added"
|
|
709
|
+
if head_resolution == "missing":
|
|
710
|
+
return "removed"
|
|
711
|
+
return "unchanged" if base_digest == head_digest else "changed"
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
def _evaluate_locator(
|
|
715
|
+
locator: ReviewLocator,
|
|
716
|
+
base_snapshot: RepositorySnapshot,
|
|
717
|
+
head_snapshot: RepositorySnapshot,
|
|
718
|
+
markdown_snapshots: _MarkdownSnapshots,
|
|
719
|
+
cache: _ReviewCache,
|
|
720
|
+
) -> ReviewLocatorEvidence:
|
|
721
|
+
base_resolution, base_digest = _digest_at(
|
|
722
|
+
base_snapshot,
|
|
723
|
+
locator,
|
|
724
|
+
markdown_snapshots,
|
|
725
|
+
cache,
|
|
726
|
+
)
|
|
727
|
+
head_resolution, head_digest = _digest_at(
|
|
728
|
+
head_snapshot,
|
|
729
|
+
locator,
|
|
730
|
+
markdown_snapshots,
|
|
731
|
+
cache,
|
|
732
|
+
)
|
|
733
|
+
return ReviewLocatorEvidence(
|
|
734
|
+
id=locator.id,
|
|
735
|
+
path=locator.path.as_posix(),
|
|
736
|
+
extractor=locator.extractor,
|
|
737
|
+
locator=locator.locator,
|
|
738
|
+
base_digest=base_digest,
|
|
739
|
+
head_digest=head_digest,
|
|
740
|
+
state=_locator_state(
|
|
741
|
+
base_resolution,
|
|
742
|
+
base_digest,
|
|
743
|
+
head_resolution,
|
|
744
|
+
head_digest,
|
|
745
|
+
),
|
|
746
|
+
)
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
def _evaluate_review_contract_snapshots(
|
|
750
|
+
contract: ReviewContract,
|
|
751
|
+
base_snapshot: RepositorySnapshot,
|
|
752
|
+
head_snapshot: RepositorySnapshot,
|
|
753
|
+
markdown_snapshots: _MarkdownSnapshots,
|
|
754
|
+
cache: _ReviewCache,
|
|
755
|
+
) -> ReviewContractResult:
|
|
756
|
+
sources = tuple(
|
|
757
|
+
_evaluate_locator(
|
|
758
|
+
locator,
|
|
759
|
+
base_snapshot,
|
|
760
|
+
head_snapshot,
|
|
761
|
+
markdown_snapshots,
|
|
762
|
+
cache,
|
|
763
|
+
)
|
|
764
|
+
for locator in contract.sources
|
|
765
|
+
)
|
|
766
|
+
targets = tuple(
|
|
767
|
+
_evaluate_locator(
|
|
768
|
+
locator,
|
|
769
|
+
base_snapshot,
|
|
770
|
+
head_snapshot,
|
|
771
|
+
markdown_snapshots,
|
|
772
|
+
cache,
|
|
773
|
+
)
|
|
774
|
+
for locator in contract.targets
|
|
775
|
+
)
|
|
776
|
+
|
|
777
|
+
if (
|
|
778
|
+
not sources
|
|
779
|
+
or not targets
|
|
780
|
+
or any(item.state == "unknown" for item in sources + targets)
|
|
781
|
+
or any(item.head_digest is None for item in targets)
|
|
782
|
+
):
|
|
783
|
+
state = "unknown"
|
|
784
|
+
elif not any(item.state in _CHANGED_STATES for item in sources):
|
|
785
|
+
state = "unaffected"
|
|
786
|
+
elif all(item.state in _SUBSTANTIVE_TARGET_STATES for item in targets):
|
|
787
|
+
state = "cochanged"
|
|
788
|
+
else:
|
|
789
|
+
state = "review-recommended"
|
|
790
|
+
|
|
791
|
+
return ReviewContractResult(
|
|
792
|
+
contract_id=contract.id,
|
|
793
|
+
mode=contract.mode,
|
|
794
|
+
state=state,
|
|
795
|
+
sources=sources,
|
|
796
|
+
targets=targets,
|
|
797
|
+
)
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def _snapshots(
|
|
801
|
+
root: Path,
|
|
802
|
+
base: str,
|
|
803
|
+
head: str,
|
|
804
|
+
) -> tuple[RepositorySnapshot, RepositorySnapshot]:
|
|
805
|
+
resolved_root = root.resolve()
|
|
806
|
+
base_ref = RepositorySnapshot(resolved_root, base).label
|
|
807
|
+
head_ref = RepositorySnapshot(resolved_root, head).label
|
|
808
|
+
return (
|
|
809
|
+
RepositorySnapshot(resolved_root, base_ref),
|
|
810
|
+
RepositorySnapshot(resolved_root, head_ref),
|
|
811
|
+
)
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
def evaluate_review_contract(
|
|
815
|
+
root: Path,
|
|
816
|
+
contract: ReviewContract,
|
|
817
|
+
*,
|
|
818
|
+
base: str,
|
|
819
|
+
head: str,
|
|
820
|
+
) -> ReviewContractResult:
|
|
821
|
+
base_snapshot, head_snapshot = _snapshots(root, base, head)
|
|
822
|
+
cache = _ReviewCache()
|
|
823
|
+
markdown_snapshots = _prepare_markdown_snapshots(
|
|
824
|
+
(contract,),
|
|
825
|
+
base_snapshot,
|
|
826
|
+
head_snapshot,
|
|
827
|
+
cache,
|
|
828
|
+
)
|
|
829
|
+
return _evaluate_review_contract_snapshots(
|
|
830
|
+
contract,
|
|
831
|
+
base_snapshot,
|
|
832
|
+
head_snapshot,
|
|
833
|
+
markdown_snapshots,
|
|
834
|
+
cache,
|
|
835
|
+
)
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
def evaluate_review_contracts(
|
|
839
|
+
root: Path,
|
|
840
|
+
contracts: tuple[ReviewContract, ...],
|
|
841
|
+
*,
|
|
842
|
+
base: str,
|
|
843
|
+
head: str,
|
|
844
|
+
) -> tuple[ReviewContractResult, ...]:
|
|
845
|
+
base_snapshot, head_snapshot = _snapshots(root, base, head)
|
|
846
|
+
cache = _ReviewCache()
|
|
847
|
+
markdown_snapshots = _prepare_markdown_snapshots(
|
|
848
|
+
contracts,
|
|
849
|
+
base_snapshot,
|
|
850
|
+
head_snapshot,
|
|
851
|
+
cache,
|
|
852
|
+
)
|
|
853
|
+
return tuple(
|
|
854
|
+
_evaluate_review_contract_snapshots(
|
|
855
|
+
contract,
|
|
856
|
+
base_snapshot,
|
|
857
|
+
head_snapshot,
|
|
858
|
+
markdown_snapshots,
|
|
859
|
+
cache,
|
|
860
|
+
)
|
|
861
|
+
for contract in contracts
|
|
862
|
+
)
|