docs-drift 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- docs_drift/__init__.py +7 -0
- docs_drift/analysis.py +52 -0
- docs_drift/baseline.py +70 -0
- docs_drift/cli.py +871 -0
- docs_drift/collaboration.py +282 -0
- docs_drift/config.py +113 -0
- docs_drift/docs/__init__.py +1 -0
- docs_drift/docs/defaults.py +250 -0
- docs_drift/docs/discover.py +172 -0
- docs_drift/docs/extract_blocks.py +596 -0
- docs_drift/docs/extract_inline.py +561 -0
- docs_drift/docs/pipeline.py +234 -0
- docs_drift/docs/preprocess.py +71 -0
- docs_drift/docs/sections.py +247 -0
- docs_drift/errors.py +119 -0
- docs_drift/fix/__init__.py +1 -0
- docs_drift/fix/edits.py +124 -0
- docs_drift/fix/model.py +139 -0
- docs_drift/fix/rules.py +232 -0
- docs_drift/git_safety.py +35 -0
- docs_drift/graph/queries/evidence_path.cypher +6 -0
- docs_drift/history.py +412 -0
- docs_drift/history_cache.py +62 -0
- docs_drift/ids.py +50 -0
- docs_drift/index/__init__.py +211 -0
- docs_drift/index/base.py +239 -0
- docs_drift/index/config.py +185 -0
- docs_drift/index/js_cli.py +211 -0
- docs_drift/index/js_routes.py +68 -0
- docs_drift/index/js_symbols.py +219 -0
- docs_drift/index/python_cli.py +370 -0
- docs_drift/index/python_routes.py +102 -0
- docs_drift/index/python_source.py +20 -0
- docs_drift/index/python_symbols.py +335 -0
- docs_drift/index/queries/javascript.scm +4 -0
- docs_drift/index/queries/typescript.scm +4 -0
- docs_drift/index/routes_common.py +16 -0
- docs_drift/index/tasks.py +131 -0
- docs_drift/index/ts_parser.py +20 -0
- docs_drift/limits.py +18 -0
- docs_drift/llm/__init__.py +7 -0
- docs_drift/llm/candidates.py +81 -0
- docs_drift/llm/client.py +185 -0
- docs_drift/llm/pricing.py +54 -0
- docs_drift/llm/privacy.py +63 -0
- docs_drift/llm/prompts/rewrite_v1.md +9 -0
- docs_drift/llm/prompts/verify_v1.md +12 -0
- docs_drift/llm/prompts.py +28 -0
- docs_drift/llm/retrieve.py +60 -0
- docs_drift/llm/types.py +68 -0
- docs_drift/llm/verify.py +179 -0
- docs_drift/lsp.py +199 -0
- docs_drift/match/__init__.py +5 -0
- docs_drift/match/commands.py +201 -0
- docs_drift/match/defaults.py +79 -0
- docs_drift/match/engine.py +319 -0
- docs_drift/match/rename.py +69 -0
- docs_drift/models.py +249 -0
- docs_drift/ownership.py +70 -0
- docs_drift/policy.py +81 -0
- docs_drift/portfolio.py +147 -0
- docs_drift/py.typed +0 -0
- docs_drift/report/__init__.py +38 -0
- docs_drift/report/common.py +31 -0
- docs_drift/report/github.py +22 -0
- docs_drift/report/html.py +106 -0
- docs_drift/report/json.py +17 -0
- docs_drift/report/junit.py +30 -0
- docs_drift/report/markdown.py +58 -0
- docs_drift/report/partials/body.html +19 -0
- docs_drift/report/partials/report.css +12 -0
- docs_drift/report/partials/script.js +95 -0
- docs_drift/report/portfolio.html +14 -0
- docs_drift/report/report.html +8 -0
- docs_drift/report/sarif.py +68 -0
- docs_drift/report/terminal.py +90 -0
- docs_drift/report/vendor/CYTOSCAPE-LICENSE +19 -0
- docs_drift/report/vendor/cytoscape.min.js +32 -0
- docs_drift/report/workflow.html +12 -0
- docs_drift/review.py +113 -0
- docs_drift/safe_files.py +19 -0
- docs_drift/scanning.py +202 -0
- docs_drift/schemas/SARIF-LICENSE.md +7 -0
- docs_drift/schemas/SARIF-NOTICES.txt +3 -0
- docs_drift/schemas/sarif-schema-2.1.0.json +3389 -0
- docs_drift/schemas/scan-result.v1.json +1024 -0
- docs_drift/setup.py +130 -0
- docs_drift/snapshots.py +100 -0
- docs_drift/support.py +31 -0
- docs_drift/watch.py +157 -0
- docs_drift/web/__init__.py +1 -0
- docs_drift/web/__main__.py +23 -0
- docs_drift/web/app.py +289 -0
- docs_drift/web/cleanup.py +23 -0
- docs_drift/web/examples/clean-repo.json +646 -0
- docs_drift/web/examples/fastapi-svc.json +723 -0
- docs_drift/web/examples/files-to-prompt.json +3366 -0
- docs_drift/web/examples/judge-fixture.json +226 -0
- docs_drift/web/examples/manifest.json +89 -0
- docs_drift/web/examples/node-tool.json +823 -0
- docs_drift/web/examples/py-cli.json +1267 -0
- docs_drift/web/examples/rich-cli-reviewed.json +7855 -0
- docs_drift/web/examples/rich-cli.json +8010 -0
- docs_drift/web/examples/verification.json +16 -0
- docs_drift/web/git.py +274 -0
- docs_drift/web/jobs.py +352 -0
- docs_drift/web/settings.py +50 -0
- docs_drift/web/static/demo.css +37 -0
- docs_drift/web/static/demo.js +184 -0
- docs_drift/web/static/evidence/correction-verification.json +28 -0
- docs_drift/web/static/evidence/flow-highlights-2026-10-22.png +0 -0
- docs_drift/web/static/evidence/fresh-scan.json +7855 -0
- docs_drift/web/static/evidence/review.json +44 -0
- docs_drift/web/static/evidence/rich-cli-correction.patch +20 -0
- docs_drift/web/static/evidence/runtime.json +26 -0
- docs_drift/web/static/evidence/saved-scan.json +8010 -0
- docs_drift/web/static/evidence/usage-card-supplied.png +0 -0
- docs_drift/web/templates/base.html +34 -0
- docs_drift/web/templates/case.html +13 -0
- docs_drift/web/templates/landing.html +90 -0
- docs_drift/web/templates/scan.html +2 -0
- docs_drift/web/templates/voice.html +1 -0
- docs_drift/web/validation.py +47 -0
- docs_drift/web/worker.py +152 -0
- docs_drift/workflow_cli.py +96 -0
- docs_drift/workspaces.py +128 -0
- docs_drift-0.1.0.dist-info/METADATA +331 -0
- docs_drift-0.1.0.dist-info/RECORD +131 -0
- docs_drift-0.1.0.dist-info/WHEEL +4 -0
- docs_drift-0.1.0.dist-info/entry_points.txt +2 -0
- docs_drift-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Run the documentation-only extraction pipeline, isolating every input file."""
|
|
2
|
+
|
|
3
|
+
import fnmatch
|
|
4
|
+
import hashlib
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import subprocess
|
|
8
|
+
import time
|
|
9
|
+
import warnings
|
|
10
|
+
from collections.abc import Iterator
|
|
11
|
+
from contextlib import contextmanager
|
|
12
|
+
from datetime import UTC, datetime
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from docs_drift.config import Config
|
|
16
|
+
from docs_drift.docs.defaults import extract_defaults
|
|
17
|
+
from docs_drift.docs.discover import DiscoveryWarning, discover_docs, read_doc
|
|
18
|
+
from docs_drift.docs.extract_blocks import extract_block_claims, project_command_names
|
|
19
|
+
from docs_drift.docs.extract_inline import extract_claims
|
|
20
|
+
from docs_drift.docs.sections import ParsedDoc, is_historical_span, parse_document
|
|
21
|
+
from docs_drift.models import Claim, ClaimLocation, ScanResult, SuppressedClaim
|
|
22
|
+
|
|
23
|
+
MIN_EXTRACTION_CONFIDENCE = 0.3
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@contextmanager
|
|
27
|
+
def _timed(timings: dict[str, float], stage: str) -> Iterator[None]:
|
|
28
|
+
start = time.perf_counter()
|
|
29
|
+
try:
|
|
30
|
+
yield
|
|
31
|
+
finally:
|
|
32
|
+
timings[stage] = timings.get(stage, 0.0) + time.perf_counter() - start
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _head(root: Path) -> str:
|
|
36
|
+
env = {key: value for key, value in os.environ.items() if not key.upper().startswith("GIT_")}
|
|
37
|
+
env["GIT_OPTIONAL_LOCKS"] = "0"
|
|
38
|
+
try:
|
|
39
|
+
value = subprocess.run(
|
|
40
|
+
["git", "-c", "core.fsmonitor=false", "-C", str(root), "rev-parse", "HEAD"],
|
|
41
|
+
capture_output=True,
|
|
42
|
+
text=True,
|
|
43
|
+
check=True,
|
|
44
|
+
timeout=3,
|
|
45
|
+
env=env,
|
|
46
|
+
).stdout.strip()
|
|
47
|
+
except (OSError, subprocess.SubprocessError):
|
|
48
|
+
value = ""
|
|
49
|
+
return value if re.fullmatch(r"(?:[0-9a-f]{40}|[0-9a-f]{64})", value) else "0" * 40
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def snapshot_time(root: Path) -> datetime:
|
|
53
|
+
"""Use the commit timestamp for reproducible snapshots; unversioned input uses the epoch."""
|
|
54
|
+
try:
|
|
55
|
+
value = subprocess.run(
|
|
56
|
+
["git", "-c", "core.fsmonitor=false", "-C", str(root), "show", "-s", "--format=%ct", "HEAD"],
|
|
57
|
+
capture_output=True,
|
|
58
|
+
text=True,
|
|
59
|
+
timeout=3,
|
|
60
|
+
check=True,
|
|
61
|
+
)
|
|
62
|
+
return datetime.fromtimestamp(int(value.stdout.strip()), UTC)
|
|
63
|
+
except (OSError, ValueError, subprocess.SubprocessError):
|
|
64
|
+
return datetime(1970, 1, 1, tzinfo=UTC)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _suppression(parsed: ParsedDoc, claim: Claim) -> str | None: # noqa: PLR0911, PLR0912
|
|
68
|
+
if "docs-drift-ignore" not in parsed.text:
|
|
69
|
+
return None
|
|
70
|
+
lines = parsed.lines
|
|
71
|
+
if any("docs-drift-ignore-file" in line for line in lines[: min(len(lines), 100)]):
|
|
72
|
+
return "docs-drift-ignore-file"
|
|
73
|
+
if claim.line > 1 and "docs-drift-ignore-next-line" in lines[claim.line - 2]:
|
|
74
|
+
return "docs-drift-ignore-next-line"
|
|
75
|
+
if claim.line > 1 and "docs-drift-ignore" in lines[claim.line - 2]:
|
|
76
|
+
return "docs-drift-ignore"
|
|
77
|
+
# A paragraph/table/fence directive suppresses the following block through its closing boundary.
|
|
78
|
+
for index, line in enumerate(lines[: claim.line - 1]):
|
|
79
|
+
if "docs-drift-ignore" not in line or "next-line" in line or "file" in line:
|
|
80
|
+
continue
|
|
81
|
+
next_nonblank = next((n for n in range(index + 1, len(lines)) if lines[n].strip()), None)
|
|
82
|
+
if next_nonblank is None or claim.line - 1 < next_nonblank:
|
|
83
|
+
continue
|
|
84
|
+
opening = lines[next_nonblank].lstrip()
|
|
85
|
+
if opening.startswith("```") or opening.startswith("~~~"):
|
|
86
|
+
marker = opening[:3]
|
|
87
|
+
close = next(
|
|
88
|
+
(n for n in range(next_nonblank + 1, len(lines)) if lines[n].lstrip().startswith(marker)), len(lines)
|
|
89
|
+
)
|
|
90
|
+
if next_nonblank <= claim.line - 1 <= close:
|
|
91
|
+
return "docs-drift-ignore"
|
|
92
|
+
if opening.startswith("|"):
|
|
93
|
+
end = next_nonblank
|
|
94
|
+
while end + 1 < len(lines) and lines[end + 1].strip() and lines[end + 1].lstrip().startswith("|"):
|
|
95
|
+
end += 1
|
|
96
|
+
if next_nonblank <= claim.line - 1 <= end:
|
|
97
|
+
return "docs-drift-ignore"
|
|
98
|
+
if not opening.startswith(("#", "|")):
|
|
99
|
+
end = next_nonblank
|
|
100
|
+
while (
|
|
101
|
+
end + 1 < len(lines)
|
|
102
|
+
and lines[end + 1].strip()
|
|
103
|
+
and not lines[end + 1].lstrip().startswith(("#", "```", "~~~", "|"))
|
|
104
|
+
):
|
|
105
|
+
end += 1
|
|
106
|
+
if next_nonblank <= claim.line - 1 <= end:
|
|
107
|
+
return "docs-drift-ignore"
|
|
108
|
+
return None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _ignored(claim: Claim, patterns: list[str]) -> bool:
|
|
112
|
+
if claim.claim_id in patterns:
|
|
113
|
+
return True
|
|
114
|
+
return any(
|
|
115
|
+
":" in pattern
|
|
116
|
+
and pattern.split(":", 1)[0] == claim.kind.value
|
|
117
|
+
and fnmatch.fnmatchcase(claim.value, pattern.split(":", 1)[1])
|
|
118
|
+
for pattern in patterns
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _merge(parsed: ParsedDoc, claims: list[Claim]) -> list[Claim]:
|
|
123
|
+
groups: dict[tuple[str, str, str | None, str], Claim] = {}
|
|
124
|
+
for claim in claims:
|
|
125
|
+
section = parsed.line_to_section.get(claim.line)
|
|
126
|
+
if section is None:
|
|
127
|
+
continue
|
|
128
|
+
key = (claim.kind.value, claim.value, claim.qualifier, section.section_id)
|
|
129
|
+
existing = groups.get(key)
|
|
130
|
+
location = ClaimLocation(
|
|
131
|
+
line=claim.line, column=claim.column, raw_text=claim.raw_text, extraction=claim.extraction
|
|
132
|
+
)
|
|
133
|
+
group_key = hashlib.sha256("\0".join(part or "" for part in key[:3]).encode()).hexdigest()
|
|
134
|
+
if existing is None:
|
|
135
|
+
claim.locations = [*claim.locations, location]
|
|
136
|
+
claim.extraction_sources = list(dict.fromkeys([*claim.extraction_sources, claim.extraction]))
|
|
137
|
+
claim.group_key = group_key
|
|
138
|
+
groups[key] = claim
|
|
139
|
+
else:
|
|
140
|
+
existing.locations.append(location)
|
|
141
|
+
existing.extraction_sources = list(dict.fromkeys([*existing.extraction_sources, claim.extraction]))
|
|
142
|
+
if claim.extraction_confidence > existing.extraction_confidence:
|
|
143
|
+
existing.extraction_confidence = claim.extraction_confidence
|
|
144
|
+
existing.expected_default = claim.expected_default or existing.expected_default
|
|
145
|
+
return list(groups.values())
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def scan_claims(
|
|
149
|
+
repo: Path,
|
|
150
|
+
config: Config,
|
|
151
|
+
*,
|
|
152
|
+
doc_paths: set[str] | None = None,
|
|
153
|
+
contents: dict[str, str] | None = None,
|
|
154
|
+
source_repo: Path | None = None,
|
|
155
|
+
) -> ScanResult:
|
|
156
|
+
"""Extract claims without importing or running any code in the scanned repository."""
|
|
157
|
+
root = repo.resolve()
|
|
158
|
+
timings: dict[str, float] = {}
|
|
159
|
+
docs = []
|
|
160
|
+
sections = []
|
|
161
|
+
claims: list[Claim] = []
|
|
162
|
+
suppressed: list[SuppressedClaim] = []
|
|
163
|
+
warnings_out: list[str] = []
|
|
164
|
+
historical = 0
|
|
165
|
+
with warnings.catch_warnings(record=True) as caught:
|
|
166
|
+
warnings.simplefilter("always", DiscoveryWarning)
|
|
167
|
+
with _timed(timings, "discover"):
|
|
168
|
+
docs = discover_docs(root, config)
|
|
169
|
+
if doc_paths is not None:
|
|
170
|
+
docs = [doc for doc in docs if doc.path in doc_paths]
|
|
171
|
+
if contents:
|
|
172
|
+
from docs_drift.models import DocFile, DocFormat # noqa: PLC0415 - optional dependency or shared pipeline import.
|
|
173
|
+
|
|
174
|
+
for path, text in contents.items():
|
|
175
|
+
docs = [doc for doc in docs if doc.path != path]
|
|
176
|
+
docs.append(
|
|
177
|
+
DocFile(
|
|
178
|
+
path=path,
|
|
179
|
+
content_hash=hashlib.sha256(text.encode()).hexdigest(),
|
|
180
|
+
format={".rst": DocFormat.rst, ".mdx": DocFormat.mdx}.get(
|
|
181
|
+
Path(path).suffix, DocFormat.markdown
|
|
182
|
+
),
|
|
183
|
+
)
|
|
184
|
+
)
|
|
185
|
+
warnings_out.extend(str(item.message) for item in caught)
|
|
186
|
+
command_context = project_command_names(source_repo or root)
|
|
187
|
+
for doc in docs:
|
|
188
|
+
try:
|
|
189
|
+
with _timed(timings, "parse"):
|
|
190
|
+
parsed = parse_document(
|
|
191
|
+
doc, contents[doc.path] if contents and doc.path in contents else read_doc(root, doc)
|
|
192
|
+
)
|
|
193
|
+
sections.extend(parsed.sections)
|
|
194
|
+
with _timed(timings, "inline"):
|
|
195
|
+
inline = extract_claims(parsed)
|
|
196
|
+
with _timed(timings, "blocks"):
|
|
197
|
+
blocks = extract_block_claims(parsed, source_repo or root, command_context=command_context)
|
|
198
|
+
historical += inline.historical_skipped + blocks.historical_skipped
|
|
199
|
+
source = [claim for claim in [*inline.claims, *blocks.claims] if claim.kind.value != "default_value"]
|
|
200
|
+
if doc.format.value == "rst":
|
|
201
|
+
from docs_drift.docs.preprocess import rst_claims # noqa: PLC0415 - optional dependency or shared pipeline import.
|
|
202
|
+
|
|
203
|
+
source.extend(rst_claims(parsed))
|
|
204
|
+
with _timed(timings, "defaults"):
|
|
205
|
+
defaults = extract_defaults(parsed, source)
|
|
206
|
+
with _timed(timings, "merge_filter"):
|
|
207
|
+
merged = _merge(parsed, [*source, *defaults.claims])
|
|
208
|
+
for claim in merged:
|
|
209
|
+
section = parsed.line_to_section.get(claim.line)
|
|
210
|
+
reason = _suppression(parsed, claim)
|
|
211
|
+
if reason:
|
|
212
|
+
suppressed.append(SuppressedClaim(claim=claim, reason=reason))
|
|
213
|
+
elif section is not None and is_historical_span(section, claim.line):
|
|
214
|
+
historical += 1
|
|
215
|
+
elif claim.extraction_confidence >= MIN_EXTRACTION_CONFIDENCE and not _ignored(
|
|
216
|
+
claim, config.ignore
|
|
217
|
+
):
|
|
218
|
+
claims.append(claim)
|
|
219
|
+
warnings_out.extend(defaults.debug_notes)
|
|
220
|
+
except Exception as exc: # A malformed document must not prevent useful scans of its neighbors.
|
|
221
|
+
warnings_out.append(f"Could not process {doc.path}: {type(exc).__name__}: {exc}")
|
|
222
|
+
return ScanResult(
|
|
223
|
+
repo_path=root,
|
|
224
|
+
head_sha=_head(root),
|
|
225
|
+
scan_time=snapshot_time(source_repo or root),
|
|
226
|
+
config_used=config,
|
|
227
|
+
docs=docs,
|
|
228
|
+
sections=sections,
|
|
229
|
+
claims=claims,
|
|
230
|
+
timings=timings,
|
|
231
|
+
warnings=warnings_out,
|
|
232
|
+
suppressed=suppressed,
|
|
233
|
+
historical_ignored=historical,
|
|
234
|
+
)
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Line-preserving MDX stripping and explicit reStructuredText claims."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from docs_drift.docs.sections import ParsedDoc
|
|
6
|
+
from docs_drift.ids import make_claim_id
|
|
7
|
+
from docs_drift.models import Claim, ClaimKind, Extraction
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def strip_mdx(text: str) -> str:
|
|
11
|
+
lines, fence, importing = [], None, False
|
|
12
|
+
for line in text.splitlines(keepends=True):
|
|
13
|
+
marker = re.match(r"\s*(```+|~~~+)", line)
|
|
14
|
+
if marker:
|
|
15
|
+
fence = None if fence else marker[1][0]
|
|
16
|
+
if fence or marker:
|
|
17
|
+
lines.append(line)
|
|
18
|
+
continue
|
|
19
|
+
if re.match(r"^\s*(import|export)\b", line) or importing:
|
|
20
|
+
importing = not bool(re.search(r"(?:from\s+['\"].*['\"]|[;}])\s*;?\s*$", line))
|
|
21
|
+
lines.append("\n" if line.endswith("\n") else "")
|
|
22
|
+
else:
|
|
23
|
+
# Strip wrappers, retaining child code fences and inline Markdown.
|
|
24
|
+
lines.append(re.sub(r"</?[A-Z][\w.]*(?:\s[^>]*?)?/?>", lambda m: " " * len(m[0]), line))
|
|
25
|
+
return "".join(lines)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def rst_claims(parsed: ParsedDoc) -> list[Claim]:
|
|
29
|
+
claims = []
|
|
30
|
+
counts: dict[tuple[str, str, str], int] = {}
|
|
31
|
+
for line_no, line in enumerate(parsed.text.splitlines(), 1):
|
|
32
|
+
for match in re.finditer(r":(option|envvar):`([^`]+)`|``([^`]+)``", line):
|
|
33
|
+
value = match[2] or match[3]
|
|
34
|
+
kind = ClaimKind.env_var if match[1] == "envvar" else ClaimKind.cli_flag if value.startswith("-") else None
|
|
35
|
+
if kind is None:
|
|
36
|
+
from docs_drift.docs.extract_inline import ( # noqa: PLC0415 - shared lexical rules.
|
|
37
|
+
_classify,
|
|
38
|
+
_explicit_api_reference,
|
|
39
|
+
path_context,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
# Share explicit API disambiguation without broadening unrelated
|
|
43
|
+
# RST literals into Markdown-only configuration/prose heuristics.
|
|
44
|
+
classified = _classify(value)
|
|
45
|
+
if _explicit_api_reference(value, line):
|
|
46
|
+
kind = ClaimKind.symbol
|
|
47
|
+
elif classified:
|
|
48
|
+
kind = classified[0]
|
|
49
|
+
value = classified[1]
|
|
50
|
+
section = parsed.line_to_section.get(line_no)
|
|
51
|
+
if kind is None or section is None:
|
|
52
|
+
continue
|
|
53
|
+
key = (section.section_id, kind.value, value)
|
|
54
|
+
occurrence = counts.get(key, 0)
|
|
55
|
+
counts[key] = occurrence + 1
|
|
56
|
+
claims.append(
|
|
57
|
+
Claim(
|
|
58
|
+
claim_id=make_claim_id(parsed.doc.path, section.heading_path, kind.value, value, occurrence),
|
|
59
|
+
kind=kind,
|
|
60
|
+
raw_text=line,
|
|
61
|
+
value=value,
|
|
62
|
+
qualifier=path_context(line, value) if kind == ClaimKind.file_path else None,
|
|
63
|
+
doc_path=parsed.doc.path,
|
|
64
|
+
line=line_no,
|
|
65
|
+
column=match.start(2 if match[2] else 3) + 1,
|
|
66
|
+
section_id=section.section_id,
|
|
67
|
+
extraction=Extraction.inline_code,
|
|
68
|
+
extraction_confidence=0.99 if match[1] else 0.9,
|
|
69
|
+
)
|
|
70
|
+
)
|
|
71
|
+
return claims
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Parse documentation locations and historical context without executing embedded content."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from collections import Counter
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from functools import cached_property
|
|
7
|
+
from pathlib import PurePosixPath
|
|
8
|
+
|
|
9
|
+
from markdown_it import MarkdownIt
|
|
10
|
+
from markdown_it.token import Token
|
|
11
|
+
|
|
12
|
+
from docs_drift.ids import make_section_id
|
|
13
|
+
from docs_drift.limits import MAX_DOC_CHARS, MAX_LINE_CHARS, MAX_MARKUP_OPENERS
|
|
14
|
+
from docs_drift.models import DocFile, DocFormat, Section, SectionContext
|
|
15
|
+
|
|
16
|
+
MIN_ADORNMENT_LENGTH = 3
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class ParsedDoc:
|
|
21
|
+
doc: DocFile
|
|
22
|
+
sections: list[Section]
|
|
23
|
+
tokens: list[Token]
|
|
24
|
+
line_to_section: dict[int, Section]
|
|
25
|
+
text: str
|
|
26
|
+
|
|
27
|
+
@cached_property
|
|
28
|
+
def lines(self) -> list[str]:
|
|
29
|
+
return self.text.splitlines()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _historical_heading(heading: str) -> bool:
|
|
33
|
+
words = (
|
|
34
|
+
r"\b(?:change[- ]?log|release notes|history|migrat\w*|upgrad\w*|deprecat\w*|legacy|"
|
|
35
|
+
r"breaking changes|what['\u2019]s new)\b"
|
|
36
|
+
)
|
|
37
|
+
versions = r"\bv\d+(?:\.\d+)*(?:\.x)?\b|\b\d+(?:\.\d+)*\.x\b|\b\d+\.\d+\.\d+\b"
|
|
38
|
+
illustrative = r"\b(?:synthetic|illustrative)\s+(?:examples?|fixtures?)\b"
|
|
39
|
+
return re.search(f"{words}|{versions}|{illustrative}", heading, re.IGNORECASE) is not None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _historical_file(path: str) -> bool:
|
|
43
|
+
return (
|
|
44
|
+
re.match(
|
|
45
|
+
r"^(?:change[-_ ]?log|history|releases?|upgrading|migrations?)(?:[._ -]|$)",
|
|
46
|
+
PurePosixPath(path).stem,
|
|
47
|
+
re.IGNORECASE,
|
|
48
|
+
)
|
|
49
|
+
is not None
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _historical_paragraph(text: str) -> bool:
|
|
54
|
+
return (
|
|
55
|
+
re.search(
|
|
56
|
+
r"\b(?:previously|formerly|was\s+renamed\s+to|no\s+longer|deprecated\s+in|removed\s+in|used\s+to\s+be)\b",
|
|
57
|
+
text,
|
|
58
|
+
re.IGNORECASE,
|
|
59
|
+
)
|
|
60
|
+
is not None
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _skip_front_matter(text: str) -> tuple[str, int]:
|
|
65
|
+
lines = text.splitlines(keepends=True)
|
|
66
|
+
if lines and lines[0].rstrip().removeprefix("\ufeff") == "---":
|
|
67
|
+
for index, line in enumerate(lines[1:], 1):
|
|
68
|
+
if line.rstrip() == "---":
|
|
69
|
+
return "\n" * (index + 1) + "".join(lines[index + 1 :]), index + 1
|
|
70
|
+
return text, 0
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _inline_text(token: Token, *, include_code: bool = True) -> str:
|
|
74
|
+
if token.children is None:
|
|
75
|
+
return token.content
|
|
76
|
+
return "".join(
|
|
77
|
+
child.content
|
|
78
|
+
if child.type in {"text", "image"} or (include_code and child.type == "code_inline")
|
|
79
|
+
else (" " if child.type in {"softbreak", "hardbreak", "code_inline"} else "")
|
|
80
|
+
for child in token.children
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass(frozen=True)
|
|
85
|
+
class _Heading:
|
|
86
|
+
text: str
|
|
87
|
+
level: int
|
|
88
|
+
line: int
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _headings(tokens: list[Token]) -> list[_Heading]:
|
|
92
|
+
return [
|
|
93
|
+
_Heading(_inline_text(tokens[index + 1]), int(token.tag[1:]), token.map[0] + 1)
|
|
94
|
+
for index, token in enumerate(tokens)
|
|
95
|
+
if token.type == "heading_open" and token.map is not None
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _sections(doc: DocFile, text: str, tokens: list[Token], skipped: int) -> list[Section]:
|
|
100
|
+
headings = _headings(tokens)
|
|
101
|
+
lines = text.splitlines()
|
|
102
|
+
first_heading = headings[0].line - 1 if headings else len(lines)
|
|
103
|
+
if any(line.strip() for line in lines[skipped:first_heading]):
|
|
104
|
+
headings.insert(0, _Heading("(top)", 0, skipped + 1))
|
|
105
|
+
stack: list[_Heading] = []
|
|
106
|
+
occurrences: Counter[tuple[str, ...]] = Counter()
|
|
107
|
+
sections = []
|
|
108
|
+
for index, heading in enumerate(headings):
|
|
109
|
+
while stack and stack[-1].level >= heading.level:
|
|
110
|
+
stack.pop()
|
|
111
|
+
trail = [parent.text for parent in stack if parent.level] + ([heading.text] if heading.level else [])
|
|
112
|
+
occurrence = occurrences[tuple(trail)]
|
|
113
|
+
occurrences[tuple(trail)] += 1
|
|
114
|
+
context = (
|
|
115
|
+
SectionContext.historical
|
|
116
|
+
if _historical_file(doc.path) or any(map(_historical_heading, trail))
|
|
117
|
+
else SectionContext.current
|
|
118
|
+
)
|
|
119
|
+
sections.append(
|
|
120
|
+
Section(
|
|
121
|
+
section_id=make_section_id(doc.path, trail, occurrence),
|
|
122
|
+
doc_path=doc.path,
|
|
123
|
+
heading=heading.text,
|
|
124
|
+
heading_level=heading.level,
|
|
125
|
+
heading_path=trail,
|
|
126
|
+
start_line=heading.line,
|
|
127
|
+
end_line=headings[index + 1].line - 1 if index + 1 < len(headings) else len(lines),
|
|
128
|
+
context=context,
|
|
129
|
+
)
|
|
130
|
+
)
|
|
131
|
+
stack.append(heading)
|
|
132
|
+
return sections
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _adornment(line: str) -> str | None:
|
|
136
|
+
text = line.strip()
|
|
137
|
+
return text[0] if len(text) >= MIN_ADORNMENT_LENGTH and text[0] in '=-~^"+#*' and len(set(text)) == 1 else None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _rst_heading(lines: list[str], index: int) -> tuple[str, str, int] | None:
|
|
141
|
+
if index + 2 < len(lines) and _adornment(lines[index]) and _adornment(lines[index]) == _adornment(lines[index + 2]):
|
|
142
|
+
return lines[index + 1].strip(), "over:" + lines[index].strip()[0], 3
|
|
143
|
+
if (
|
|
144
|
+
index + 1 < len(lines)
|
|
145
|
+
and lines[index].strip()
|
|
146
|
+
and _adornment(lines[index + 1])
|
|
147
|
+
and len(lines[index + 1].strip()) >= len(lines[index].strip())
|
|
148
|
+
):
|
|
149
|
+
return lines[index].strip(), "under:" + lines[index + 1].strip()[0], 2
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _simple_tokens(text: str, *, rst: bool) -> list[Token]:
|
|
154
|
+
lines = text.splitlines()
|
|
155
|
+
tokens: list[Token] = []
|
|
156
|
+
levels: dict[str, int] = {}
|
|
157
|
+
index = 0
|
|
158
|
+
while index < len(lines):
|
|
159
|
+
directive = re.match(r"\s*\.\.\s+code-block::\s*(\S*)", lines[index]) if rst else None
|
|
160
|
+
if directive:
|
|
161
|
+
start = index
|
|
162
|
+
index += 1
|
|
163
|
+
while index < len(lines) and (not lines[index].strip() or lines[index].lstrip().startswith(":")):
|
|
164
|
+
index += 1
|
|
165
|
+
content_start = index
|
|
166
|
+
while index < len(lines) and (not lines[index].strip() or lines[index].startswith((" ", "\t"))):
|
|
167
|
+
index += 1
|
|
168
|
+
import textwrap # noqa: PLC0415 - optional dependency or shared pipeline import.
|
|
169
|
+
|
|
170
|
+
token = Token(
|
|
171
|
+
"fence",
|
|
172
|
+
"code",
|
|
173
|
+
0,
|
|
174
|
+
map=[content_start - 1, index],
|
|
175
|
+
info=directive[1],
|
|
176
|
+
content=textwrap.dedent("\n".join(lines[content_start:index])) + "\n",
|
|
177
|
+
)
|
|
178
|
+
tokens.append(token)
|
|
179
|
+
continue
|
|
180
|
+
heading = _rst_heading(lines, index) if rst else None
|
|
181
|
+
if heading is not None:
|
|
182
|
+
title, adornment, count = heading
|
|
183
|
+
level = levels.setdefault(adornment, len(levels) + 1)
|
|
184
|
+
tokens.extend(
|
|
185
|
+
[
|
|
186
|
+
Token("heading_open", f"h{level}", 1, map=[index, index + count]),
|
|
187
|
+
Token("inline", "", 0, content=title, map=[index, index + count]),
|
|
188
|
+
Token("heading_close", f"h{level}", -1),
|
|
189
|
+
]
|
|
190
|
+
)
|
|
191
|
+
index += count
|
|
192
|
+
elif lines[index].strip():
|
|
193
|
+
start = index
|
|
194
|
+
index += 1
|
|
195
|
+
while index < len(lines) and lines[index].strip() and not (rst and _rst_heading(lines, index)):
|
|
196
|
+
index += 1
|
|
197
|
+
tokens.extend(
|
|
198
|
+
[
|
|
199
|
+
Token("paragraph_open", "p", 1, map=[start, index]),
|
|
200
|
+
Token("inline", "", 0, content="\n".join(lines[start:index]), map=[start, index]),
|
|
201
|
+
Token("paragraph_close", "p", -1),
|
|
202
|
+
]
|
|
203
|
+
)
|
|
204
|
+
else:
|
|
205
|
+
index += 1
|
|
206
|
+
return tokens
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _mark_paragraphs(tokens: list[Token], mapping: dict[int, Section]) -> None:
|
|
210
|
+
for index, token in enumerate(tokens):
|
|
211
|
+
if token.type != "paragraph_open" or token.map is None:
|
|
212
|
+
continue
|
|
213
|
+
if _historical_paragraph(_inline_text(tokens[index + 1], include_code=False)):
|
|
214
|
+
first, end = token.map[0] + 1, token.map[1]
|
|
215
|
+
section = mapping.get(first)
|
|
216
|
+
if section is not None:
|
|
217
|
+
section.historical_spans.append((first, end))
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def is_historical_span(section: Section, line: int) -> bool:
|
|
221
|
+
"""Paragraph markers suppress their full paragraph, never the rest of a current section."""
|
|
222
|
+
return section.start_line <= line <= section.end_line and (
|
|
223
|
+
section.context == SectionContext.historical
|
|
224
|
+
or any(start <= line <= end for start, end in section.historical_spans)
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def parse_document(doc: DocFile, text: str) -> ParsedDoc:
|
|
229
|
+
"""Partition physical lines at every heading; map lines to their most specific section."""
|
|
230
|
+
if len(text) > MAX_DOC_CHARS or any(len(line) > MAX_LINE_CHARS for line in text.splitlines()):
|
|
231
|
+
raise ValueError("Document exceeds the parser size or 8,192-character line limit; split generated content.")
|
|
232
|
+
if doc.format in {DocFormat.markdown, DocFormat.mdx}:
|
|
233
|
+
if any(line.count("[") + line.count("(") > MAX_MARKUP_OPENERS for line in text.splitlines()):
|
|
234
|
+
raise ValueError("Document exceeds the Markdown parser delimiter limit; split densely nested markup.")
|
|
235
|
+
if doc.format == DocFormat.mdx:
|
|
236
|
+
from docs_drift.docs.preprocess import strip_mdx # noqa: PLC0415 - optional dependency or shared pipeline import.
|
|
237
|
+
|
|
238
|
+
text = strip_mdx(text)
|
|
239
|
+
source, skipped = _skip_front_matter(text)
|
|
240
|
+
tokens = MarkdownIt("commonmark", {"html": False, "maxNesting": 20}).enable("table").parse(source)
|
|
241
|
+
else:
|
|
242
|
+
source, skipped = text, 0
|
|
243
|
+
tokens = _simple_tokens(source, rst=doc.format == DocFormat.rst)
|
|
244
|
+
sections = _sections(doc, source, tokens, skipped)
|
|
245
|
+
mapping = {line: section for section in sections for line in range(section.start_line, section.end_line + 1)}
|
|
246
|
+
_mark_paragraphs(tokens, mapping)
|
|
247
|
+
return ParsedDoc(doc, sections, tokens, mapping, text)
|
docs_drift/errors.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Friendly CLI boundary, with local debug logs for unexpected failures."""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
import traceback
|
|
5
|
+
from contextlib import nullcontext, redirect_stdout
|
|
6
|
+
from io import StringIO
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from typer import _click as click
|
|
11
|
+
from typer._click import Context
|
|
12
|
+
from typer.core import TyperCommand, TyperGroup
|
|
13
|
+
from typer.exceptions import Abort, Exit
|
|
14
|
+
|
|
15
|
+
from docs_drift.llm.privacy import mask_secrets
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class DocsDriftError(Exception):
|
|
19
|
+
def __init__(self, message: str, hint: str | None = None):
|
|
20
|
+
super().__init__(message)
|
|
21
|
+
self.hint = hint
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
COMMON_VALUES = {"--repo", "--config", "--format", "--output", "--output-json", "--output-sarif"}
|
|
25
|
+
COMMON_FLAGS = {"--verbose", "-v", "--quiet", "-q", "--no-color", "--debug", "--show-authors"}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Command(TyperCommand):
|
|
29
|
+
def get_help(self, ctx: Context) -> str:
|
|
30
|
+
path = " ".join(ctx.command_path.split()[1:])
|
|
31
|
+
argument = " <id>" if self.name in {"explain", "ignore"} else " --yes" if self.name == "init" else ""
|
|
32
|
+
self.epilog = self.epilog or f"Example: docs-drift {path}{argument} --repo ."
|
|
33
|
+
return (
|
|
34
|
+
super().get_help(ctx)
|
|
35
|
+
+ "\nCommon options: --repo PATH, --format FORMAT, --output PATH, -q/--quiet, -v/--verbose,"
|
|
36
|
+
" --no-color, --debug.\n"
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Group(TyperGroup):
|
|
41
|
+
def get_help(self, ctx: Context) -> str:
|
|
42
|
+
path = " ".join(ctx.command_path.split()[1:])
|
|
43
|
+
leaf = "update" if path.endswith("baseline") else "clear" if path.endswith("cache") else "scan"
|
|
44
|
+
self.epilog = self.epilog or f"Example: docs-drift {path} {leaf} --repo ."
|
|
45
|
+
return super().get_help(ctx)
|
|
46
|
+
|
|
47
|
+
def parse_args(self, ctx: Context, args: list[str]) -> list[str]:
|
|
48
|
+
if ctx.parent is not None:
|
|
49
|
+
return super().parse_args(ctx, args)
|
|
50
|
+
# Global options have identical meaning before and after any subcommand.
|
|
51
|
+
common, rest = [], []
|
|
52
|
+
i = 0
|
|
53
|
+
while i < len(args):
|
|
54
|
+
token = args[i]
|
|
55
|
+
key = token.split("=", 1)[0]
|
|
56
|
+
if token == "--":
|
|
57
|
+
rest.extend(args[i:])
|
|
58
|
+
break
|
|
59
|
+
if key in COMMON_VALUES or key in COMMON_FLAGS:
|
|
60
|
+
common.append(token)
|
|
61
|
+
if key in COMMON_VALUES and "=" not in token and i + 1 < len(args):
|
|
62
|
+
i += 1
|
|
63
|
+
common.append(args[i])
|
|
64
|
+
else:
|
|
65
|
+
rest.append(token)
|
|
66
|
+
i += 1
|
|
67
|
+
ctx.meta["help_only"] = "--help" in rest
|
|
68
|
+
return super().parse_args(ctx, common + rest)
|
|
69
|
+
|
|
70
|
+
def invoke(self, ctx: Context) -> Any:
|
|
71
|
+
try:
|
|
72
|
+
destination = ctx.params.get("output")
|
|
73
|
+
quiet = ctx.params.get("quiet", False)
|
|
74
|
+
capture = ctx.parent is None and ctx._protected_args[:1] != ["lsp"] and bool(destination or quiet)
|
|
75
|
+
buffer = StringIO()
|
|
76
|
+
with redirect_stdout(buffer) if capture else nullcontext():
|
|
77
|
+
value = super().invoke(ctx)
|
|
78
|
+
# Scan renderers write destinations directly. Other commands share
|
|
79
|
+
# this boundary so --output and --quiet have the same meaning.
|
|
80
|
+
if capture and destination and buffer.getvalue():
|
|
81
|
+
target = Path(destination)
|
|
82
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
83
|
+
target.write_text(buffer.getvalue(), encoding="utf-8", newline="\n")
|
|
84
|
+
return value
|
|
85
|
+
except Exit:
|
|
86
|
+
if capture and buffer.getvalue():
|
|
87
|
+
# A failed --quiet check must still explain its findings.
|
|
88
|
+
sys.stdout.write(buffer.getvalue())
|
|
89
|
+
raise
|
|
90
|
+
except (click.ClickException, Abort):
|
|
91
|
+
raise
|
|
92
|
+
except Exception as exc:
|
|
93
|
+
if ctx.parent is not None:
|
|
94
|
+
raise
|
|
95
|
+
debug = ctx.params.get("debug", False)
|
|
96
|
+
root = Path(ctx.params.get("repo", ".")).resolve()
|
|
97
|
+
if isinstance(exc, DocsDriftError):
|
|
98
|
+
message = str(exc) + (f"\nHint: {exc.hint}" if exc.hint else "")
|
|
99
|
+
elif isinstance(exc, (OSError, ValueError)):
|
|
100
|
+
friendly = DocsDriftError(
|
|
101
|
+
str(exc), "Check the selected paths, file permissions and configuration, then retry."
|
|
102
|
+
)
|
|
103
|
+
message = f"{friendly}\nHint: {friendly.hint}"
|
|
104
|
+
else:
|
|
105
|
+
log = root / ".docs-drift" / "last-error.log"
|
|
106
|
+
try:
|
|
107
|
+
log.parent.mkdir(parents=True, exist_ok=True)
|
|
108
|
+
log.write_text(mask_secrets(traceback.format_exc()), encoding="utf-8")
|
|
109
|
+
location = str(log)
|
|
110
|
+
except OSError:
|
|
111
|
+
location = "unavailable (repository is not writable)"
|
|
112
|
+
message = (
|
|
113
|
+
f"The scan could not finish ({type(exc).__name__}).\n"
|
|
114
|
+
f"Debug log file: last-error.log\nLocation: {location}\n"
|
|
115
|
+
"Retry with --debug; report this at https://github.com/ayushagarwalhere/docs-drift/issues/new"
|
|
116
|
+
)
|
|
117
|
+
if debug:
|
|
118
|
+
traceback.print_exc(file=sys.stderr)
|
|
119
|
+
raise click.ClickException(message) from None
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Reviewable edits; generation never writes documentation or executes examples."""
|