whykit 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- whykit/__init__.py +21 -0
- whykit/adopt.py +776 -0
- whykit/backlinks.py +106 -0
- whykit/check.py +320 -0
- whykit/ci_formats.py +283 -0
- whykit/cli.py +1719 -0
- whykit/completion.py +193 -0
- whykit/config.py +233 -0
- whykit/console.py +217 -0
- whykit/context.py +204 -0
- whykit/contract.py +210 -0
- whykit/contract_schemas/backlinks-report.schema.json +61 -0
- whykit/contract_schemas/context-bundle.schema.json +27 -0
- whykit/contract_schemas/context-pack.schema.json +29 -0
- whykit/contract_schemas/impact-report.schema.json +122 -0
- whykit/contract_schemas/query-result.schema.json +105 -0
- whykit/contract_schemas/status-report.schema.json +176 -0
- whykit/contract_schemas/trace-report.schema.json +211 -0
- whykit/diff.py +823 -0
- whykit/evidence.py +178 -0
- whykit/explorer_index.py +554 -0
- whykit/graph.py +318 -0
- whykit/immutability.py +624 -0
- whykit/impact.py +272 -0
- whykit/init_layout.py +205 -0
- whykit/io.py +563 -0
- whykit/lint.py +2190 -0
- whykit/lsp.py +1241 -0
- whykit/markdown_code.py +107 -0
- whykit/mcp_server.py +2289 -0
- whykit/messages.py +46 -0
- whykit/pack.py +294 -0
- whykit/parse_cache.py +752 -0
- whykit/placeholders.py +84 -0
- whykit/promote.py +651 -0
- whykit/py.typed +0 -0
- whykit/query.py +174 -0
- whykit/review.py +527 -0
- whykit/rule_policy.py +1063 -0
- whykit/rules.py +274 -0
- whykit/scaffold.py +784 -0
- whykit/sensitivity.py +95 -0
- whykit/snapshot.py +319 -0
- whykit/status.py +167 -0
- whykit/tables.py +120 -0
- whykit/template/00-context/README.md +17 -0
- whykit/template/00-context/company.md +52 -0
- whykit/template/00-context/evidence-register.md +34 -0
- whykit/template/00-context/goals.md +38 -0
- whykit/template/00-context/review-log.md +22 -0
- whykit/template/00-context/terminology.md +34 -0
- whykit/template/01-strategy/README.md +7 -0
- whykit/template/02-discoverability/README.md +10 -0
- whykit/template/03-website/README.md +7 -0
- whykit/template/04-automation/README.md +10 -0
- whykit/template/05-operations/README.md +19 -0
- whykit/template/06-decisions/README.md +11 -0
- whykit/template/06-decisions/decision-log.md +38 -0
- whykit/template/07-research/README.md +10 -0
- whykit/template/AGENTS.md +163 -0
- whykit/template/Home.md +57 -0
- whykit/template/INTEROP.md +117 -0
- whykit/template/OBSIDIAN.md +92 -0
- whykit/template/README.md +51 -0
- whykit/template/assets/README.md +6 -0
- whykit/template/reports/README.md +7 -0
- whykit/template/templates/README.md +7 -0
- whykit/template/templates/automation-spec-template.md +70 -0
- whykit/template/templates/decision-record-template.md +65 -0
- whykit/template/templates/document-template.md +55 -0
- whykit/template/templates/experiment-template.md +62 -0
- whykit/template/templates/meeting-note-template.md +47 -0
- whykit/template/templates/page-brief-template.md +63 -0
- whykit/template/templates/research-note-template.md +53 -0
- whykit/template/templates/source-ingestion-template.md +43 -0
- whykit/template/whykit.toml +39 -0
- whykit/trace.py +272 -0
- whykit/vault_index.py +190 -0
- whykit-0.3.1.dist-info/METADATA +162 -0
- whykit-0.3.1.dist-info/RECORD +84 -0
- whykit-0.3.1.dist-info/WHEEL +4 -0
- whykit-0.3.1.dist-info/entry_points.txt +3 -0
- whykit-0.3.1.dist-info/licenses/LICENSE +201 -0
- whykit-0.3.1.dist-info/licenses/NOTICE +13 -0
whykit/lint.py
ADDED
|
@@ -0,0 +1,2190 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""WhyKit vault linter.
|
|
3
|
+
|
|
4
|
+
The linter checks *shape*, not truth. It is intentionally deterministic and has
|
|
5
|
+
no runtime dependency beyond Python 3.11+. WhyKit parses the documented YAML
|
|
6
|
+
subset itself so installing an unrelated optional package cannot change lint
|
|
7
|
+
behavior.
|
|
8
|
+
|
|
9
|
+
Examples:
|
|
10
|
+
whykit lint # finds the vault root by walking up
|
|
11
|
+
whykit lint --strict
|
|
12
|
+
whykit lint examples/northline # a vault directory is used as the root
|
|
13
|
+
whykit lint --json --quiet
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import contextlib
|
|
19
|
+
import contextvars
|
|
20
|
+
import datetime as dt
|
|
21
|
+
import functools
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import stat
|
|
26
|
+
import unicodedata
|
|
27
|
+
from urllib.parse import unquote
|
|
28
|
+
from dataclasses import asdict, dataclass, field
|
|
29
|
+
from pathlib import Path, PurePath
|
|
30
|
+
from collections.abc import Callable, Iterator
|
|
31
|
+
from typing import Iterable
|
|
32
|
+
|
|
33
|
+
from .io import _lock_owner, consistent_read
|
|
34
|
+
from .tables import split_table_row as _split_table_row, review_table_header
|
|
35
|
+
from .config import CONFIG_FILE, ConfigError, load_config
|
|
36
|
+
from .markdown_code import mask_code as _mask_code
|
|
37
|
+
from .rule_policy import EMPTY_POLICY, Override, check_custom_rules, policy_from_config, secret_scan_skipped
|
|
38
|
+
from .console import emit_machine, one_line
|
|
39
|
+
from .placeholders import (
|
|
40
|
+
PROSE_SECTIONS as PLACEHOLDER_PROSE_SECTIONS,
|
|
41
|
+
SCAFFOLD_EVIDENCE_TODO,
|
|
42
|
+
normalize as normalize_placeholder,
|
|
43
|
+
paragraphs as placeholder_paragraphs,
|
|
44
|
+
split_sections as split_placeholder_sections,
|
|
45
|
+
template_section_prompts,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
VAULT_MARKERS = ("Home.md", "00-context")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class VaultPathError(ValueError):
|
|
52
|
+
"""Raised when a requested lint path escapes the selected vault root."""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# ``Path.resolve()`` walks every path component with ``lstat``. One command
|
|
56
|
+
# resolves the same few thousand paths hundreds of thousands of times (link
|
|
57
|
+
# targets, containment checks, relative paths), which dominated run time on
|
|
58
|
+
# large vaults. Inside a ``path_cache()`` scope each distinct path is resolved
|
|
59
|
+
# once. The scope is request-sized on purpose: a long-lived process such as the
|
|
60
|
+
# MCP server must not see a stale answer after the vault changes on disk.
|
|
61
|
+
class _RequestCache:
|
|
62
|
+
__slots__ = ("realpaths", "relative", "within", "registers", "owner", "active")
|
|
63
|
+
|
|
64
|
+
def __init__(self) -> None:
|
|
65
|
+
self.owner = _lock_owner()
|
|
66
|
+
self.active = True
|
|
67
|
+
# Keyed by the path string, not the Path: Windows paths compare
|
|
68
|
+
# case-insensitively, and a cache must never merge two spellings.
|
|
69
|
+
self.realpaths: dict[str, Path] = {}
|
|
70
|
+
self.relative: dict[tuple[str, str], str] = {}
|
|
71
|
+
self.within: dict[tuple[str, str], bool] = {}
|
|
72
|
+
# Parsed evidence registers keyed by (path, mtime_ns, size), so a write
|
|
73
|
+
# inside the scope is still seen.
|
|
74
|
+
self.registers: dict[tuple[Path, int, int], tuple] = {}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
_REQUEST: contextvars.ContextVar[_RequestCache | None] = contextvars.ContextVar(
|
|
78
|
+
"whykit_request_cache", default=None,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _request_cache() -> _RequestCache | None:
|
|
83
|
+
cache = _REQUEST.get()
|
|
84
|
+
return cache if cache is not None and cache.active and cache.owner == _lock_owner() else None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@contextlib.contextmanager
|
|
88
|
+
def path_cache(*, fresh: bool = False):
|
|
89
|
+
"""Memoise ``Path.resolve()`` (and register parsing) for one read-only request.
|
|
90
|
+
|
|
91
|
+
Nested reads reuse the outer cache; protocol requests can force a fresh one.
|
|
92
|
+
"""
|
|
93
|
+
if not fresh and _request_cache() is not None:
|
|
94
|
+
yield
|
|
95
|
+
return
|
|
96
|
+
cache = _RequestCache()
|
|
97
|
+
token = _REQUEST.set(cache)
|
|
98
|
+
try:
|
|
99
|
+
yield
|
|
100
|
+
finally:
|
|
101
|
+
cache.active = False
|
|
102
|
+
_REQUEST.reset(token)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
_POSIX = os.name == "posix"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _real(path: Path) -> Path:
|
|
109
|
+
cache = _request_cache()
|
|
110
|
+
if cache is None:
|
|
111
|
+
return path.resolve()
|
|
112
|
+
key = os.fspath(path)
|
|
113
|
+
hit = cache.realpaths.get(key)
|
|
114
|
+
if hit is None:
|
|
115
|
+
hit = _real_uncached(path)
|
|
116
|
+
if type(hit) is type(path) and os.fspath(hit) == key:
|
|
117
|
+
# Already real (the common case under a resolved vault root): keep
|
|
118
|
+
# the caller's object instead of an equal copy of every path.
|
|
119
|
+
hit = path
|
|
120
|
+
cache.realpaths[key] = hit
|
|
121
|
+
return hit
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# Characters the Win32 namespace rejects in a file name before any file system
|
|
125
|
+
# is asked. NUL is impossible on every platform.
|
|
126
|
+
_WINDOWS_FORBIDDEN = frozenset('<>:"|?*') | frozenset(chr(code) for code in range(32))
|
|
127
|
+
# Device names: on Windows ``notes/CON.md`` or ``nul`` opens a device, not a file.
|
|
128
|
+
_WINDOWS_DEVICES = frozenset(
|
|
129
|
+
{"con", "prn", "aux", "nul"}
|
|
130
|
+
| {f"{kind}{digit}" for kind in ("com", "lpt") for digit in "123456789\u00b9\u00b2\u00b3"}
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _impossible_name(name: str, windows: bool = not _POSIX) -> bool:
|
|
135
|
+
"""Whether no file or directory can be called *name* on this platform.
|
|
136
|
+
|
|
137
|
+
A pure string test, so a link target such as ``a<b`` or one holding a NUL
|
|
138
|
+
is answered without a system call (which on Windows can raise, or be slow).
|
|
139
|
+
"""
|
|
140
|
+
if "\x00" in name:
|
|
141
|
+
return True
|
|
142
|
+
if not windows:
|
|
143
|
+
return False
|
|
144
|
+
if not _WINDOWS_FORBIDDEN.isdisjoint(name):
|
|
145
|
+
return True
|
|
146
|
+
return name.split(".", 1)[0].rstrip(" ").casefold() in _WINDOWS_DEVICES
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _first_impossible_part(path: PurePath, windows: bool = not _POSIX) -> int | None:
|
|
150
|
+
"""Index into ``path.parts`` of the first component that cannot exist."""
|
|
151
|
+
parts = path.parts
|
|
152
|
+
start = 1 if (path.drive or path.root) else 0 # the anchor (``C:\``) is not a name
|
|
153
|
+
for index in range(start, len(parts)):
|
|
154
|
+
if _impossible_name(parts[index], windows):
|
|
155
|
+
return index
|
|
156
|
+
return None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _foreign_anchor(root: PurePath, path: PurePath) -> bool:
|
|
160
|
+
"""True when *path* is absolute on another drive, share or root than *root*.
|
|
161
|
+
|
|
162
|
+
``\\\\host\\share\\x`` in a note is a UNC path on Windows: resolving
|
|
163
|
+
it asks the network for ``host``, which takes seconds and can offer the
|
|
164
|
+
user's network credentials to whoever answers. Such a path can never be inside
|
|
165
|
+
the vault, so containment is decided from the strings alone.
|
|
166
|
+
"""
|
|
167
|
+
if not (path.anchor and root.anchor):
|
|
168
|
+
return False
|
|
169
|
+
return path.anchor.casefold() != root.anchor.casefold()
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _real_uncached(path: Path) -> Path:
|
|
173
|
+
# On POSIX, realpath(dir/name) is realpath(dir)/name whenever `name` is a
|
|
174
|
+
# plain component that is not itself a symlink. Reusing the cached parent
|
|
175
|
+
# turns one lstat per path component into one lstat per path. Anything
|
|
176
|
+
# else (Windows, relative paths, `.`/`..`, symlinks) takes the full route.
|
|
177
|
+
if _POSIX and path.is_absolute():
|
|
178
|
+
name = path.name
|
|
179
|
+
parent = path.parent
|
|
180
|
+
if name not in ("", ".", "..") and parent != path and not os.path.islink(path):
|
|
181
|
+
return _real(parent) / name
|
|
182
|
+
if not _POSIX and path.is_absolute():
|
|
183
|
+
# Windows normalises `..` lexically, so after normpath nothing at or
|
|
184
|
+
# below a component that cannot exist touches the file system: resolve
|
|
185
|
+
# the part above it and keep the rest as written.
|
|
186
|
+
normal = type(path)(os.path.normpath(path))
|
|
187
|
+
index = _first_impossible_part(normal)
|
|
188
|
+
if index is not None:
|
|
189
|
+
parts = normal.parts
|
|
190
|
+
return _real(type(path)(*parts[:index])).joinpath(*parts[index:])
|
|
191
|
+
return path.resolve()
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _relative_text(path: Path, root: Path) -> str | None:
|
|
195
|
+
"""``path.relative_to(root).as_posix()``, or ``None`` when it raises.
|
|
196
|
+
|
|
197
|
+
For two real (absolute, normalised) POSIX paths this is a prefix test on
|
|
198
|
+
the strings. ``relative_to`` gets the same answer by comparing every
|
|
199
|
+
ancestor of *path* with *root*, and caches a list of parts on both paths
|
|
200
|
+
while doing it, which at vault scale is both slow and tens of megabytes.
|
|
201
|
+
"""
|
|
202
|
+
if not _POSIX:
|
|
203
|
+
try:
|
|
204
|
+
return path.relative_to(root).as_posix()
|
|
205
|
+
except ValueError:
|
|
206
|
+
return None
|
|
207
|
+
inner, outer = os.fspath(path), os.fspath(root)
|
|
208
|
+
if inner == outer:
|
|
209
|
+
return "."
|
|
210
|
+
if not (inner.startswith("/") and outer.startswith("/")):
|
|
211
|
+
try:
|
|
212
|
+
return path.relative_to(root).as_posix()
|
|
213
|
+
except ValueError:
|
|
214
|
+
return None
|
|
215
|
+
prefix = outer if outer.endswith("/") else outer + "/"
|
|
216
|
+
return inner[len(prefix):] if inner.startswith(prefix) else None
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _within(root: Path, path: Path) -> bool:
|
|
220
|
+
cache = _request_cache()
|
|
221
|
+
key = (os.fspath(root), os.fspath(path))
|
|
222
|
+
if cache is not None:
|
|
223
|
+
hit = cache.within.get(key)
|
|
224
|
+
if hit is not None:
|
|
225
|
+
return hit
|
|
226
|
+
if _foreign_anchor(root, path):
|
|
227
|
+
inside = False
|
|
228
|
+
else:
|
|
229
|
+
try:
|
|
230
|
+
inside = _relative_text(_real(path), _real(root)) is not None
|
|
231
|
+
except (OSError, ValueError):
|
|
232
|
+
# A name the operating system rejects (an embedded NUL) is a link
|
|
233
|
+
# that does not resolve, never a crash.
|
|
234
|
+
inside = False
|
|
235
|
+
if cache is not None:
|
|
236
|
+
cache.within[key] = inside
|
|
237
|
+
return inside
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def path_exists(path: Path) -> bool:
|
|
241
|
+
"""``path.exists()`` that answers False instead of raising.
|
|
242
|
+
|
|
243
|
+
A link target comes from the note, so it can name a path the operating
|
|
244
|
+
system rejects outright (a component longer than the filesystem allows,
|
|
245
|
+
an embedded NUL). That is a link that does not resolve, not a crash.
|
|
246
|
+
"""
|
|
247
|
+
if _first_impossible_part(path) is not None:
|
|
248
|
+
return False
|
|
249
|
+
try:
|
|
250
|
+
return path.exists()
|
|
251
|
+
except (OSError, ValueError):
|
|
252
|
+
return False
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def path_is_file(path: Path) -> bool:
|
|
256
|
+
"""``path.is_file()`` with the same tolerance as :func:`path_exists`."""
|
|
257
|
+
if _first_impossible_part(path) is not None:
|
|
258
|
+
return False
|
|
259
|
+
try:
|
|
260
|
+
return path.is_file()
|
|
261
|
+
except (OSError, ValueError):
|
|
262
|
+
return False
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
MARKDOWN_SUFFIX = ".md"
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def is_markdown_name(name: str) -> bool:
|
|
269
|
+
"""Whether a file name is Markdown: ``.md`` in any letter case.
|
|
270
|
+
|
|
271
|
+
One rule for every command. ``notes/old.MD`` is the same kind of file as
|
|
272
|
+
``notes/old.md`` on every filesystem, so lint, adopt, the index and the
|
|
273
|
+
snapshot must not see it on one platform and miss it on another.
|
|
274
|
+
"""
|
|
275
|
+
return name.lower().endswith(MARKDOWN_SUFFIX)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def strip_markdown_suffix(value: str) -> str:
|
|
279
|
+
"""``value`` without a trailing ``.md`` in any letter case (a node ID from a path)."""
|
|
280
|
+
return value[: -len(MARKDOWN_SUFFIX)] if is_markdown_name(value) else value
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def iter_markdown(root: Path) -> Iterator[Path]:
|
|
284
|
+
"""Every path under *root* whose name is Markdown, in any letter case.
|
|
285
|
+
|
|
286
|
+
``Path.rglob("*.md")`` matches case-sensitively on POSIX, so it would skip
|
|
287
|
+
``old.MD`` on Linux and macOS alike. Callers still check ``is_file`` and
|
|
288
|
+
confinement, exactly as they did for the glob.
|
|
289
|
+
"""
|
|
290
|
+
for path in root.rglob("*"):
|
|
291
|
+
if is_markdown_name(path.name):
|
|
292
|
+
yield path
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def is_vault_root(path: Path) -> bool:
|
|
296
|
+
"""A directory is a vault root when it carries the map of content and the context dir."""
|
|
297
|
+
return (path / "Home.md").is_file() and (path / "00-context").is_dir()
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def find_vault_root(start: Path | None = None) -> Path | None:
|
|
301
|
+
"""Walk up from `start` looking for a vault root, so the CLI works from any subdirectory."""
|
|
302
|
+
current = (start or Path.cwd()).resolve()
|
|
303
|
+
for candidate in (current, *current.parents):
|
|
304
|
+
if is_vault_root(candidate):
|
|
305
|
+
return candidate
|
|
306
|
+
return None
|
|
307
|
+
|
|
308
|
+
REQUIRED_KEYS = (
|
|
309
|
+
"title",
|
|
310
|
+
"type",
|
|
311
|
+
"status",
|
|
312
|
+
"owner",
|
|
313
|
+
"created",
|
|
314
|
+
"last_updated",
|
|
315
|
+
"source_of_truth",
|
|
316
|
+
"sensitivity",
|
|
317
|
+
)
|
|
318
|
+
OPTIONAL_KEYS = (
|
|
319
|
+
"aliases",
|
|
320
|
+
"source_ids",
|
|
321
|
+
"tags",
|
|
322
|
+
"reviewers",
|
|
323
|
+
"review_by",
|
|
324
|
+
"decision_id",
|
|
325
|
+
"supersedes",
|
|
326
|
+
"superseded_by",
|
|
327
|
+
"delivery_status",
|
|
328
|
+
"sent_at",
|
|
329
|
+
"provenance",
|
|
330
|
+
"workstream",
|
|
331
|
+
)
|
|
332
|
+
ALLOWED_STATUS = {"template", "draft", "in_review", "approved", "superseded", "archived"}
|
|
333
|
+
# Records whose reasoning is closed: kept as written, never brought up to date.
|
|
334
|
+
HISTORICAL_STATUSES = frozenset({"superseded", "archived"})
|
|
335
|
+
ALLOWED_TYPE = {
|
|
336
|
+
"strategy",
|
|
337
|
+
"research",
|
|
338
|
+
"framework",
|
|
339
|
+
"specification",
|
|
340
|
+
"decision",
|
|
341
|
+
"map-of-content",
|
|
342
|
+
"guide",
|
|
343
|
+
"reference",
|
|
344
|
+
}
|
|
345
|
+
ALLOWED_SENSITIVITY = {"public", "internal", "confidential", "restricted"}
|
|
346
|
+
ALLOWED_DELIVERY_STATUS = {"draft", "ready_to_send", "sent"}
|
|
347
|
+
NO_FRONT_MATTER_REQUIRED = {"AGENTS.md", "SECURITY.md", "CONTRIBUTING.md", "CHANGELOG.md"}
|
|
348
|
+
PLACEHOLDER_DATE = "YYYY-MM-DD"
|
|
349
|
+
PLACEHOLDER_OWNER = "TODO"
|
|
350
|
+
DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
351
|
+
DATED_FILENAME_RE = re.compile(r"-(\d{4}-\d{2}-\d{2})(?:-[A-Za-z]{2,4})?$")
|
|
352
|
+
# Inside a Markdown table Obsidian writes the alias pipe as `\|`; the optional
|
|
353
|
+
# backslash keeps it out of the target so `[[a/b\|alias]]` resolves as `a/b`.
|
|
354
|
+
# No part of a wikilink may hold `[` or a line break (Obsidian forbids both in
|
|
355
|
+
# link targets). Besides matching what Obsidian reads, that keeps every scan
|
|
356
|
+
# short: an unclosed `[[` stops at the next `[` or line end instead of
|
|
357
|
+
# reading to the end of the file once per opener, which made a note of
|
|
358
|
+
# repeated `[` characters quadratic.
|
|
359
|
+
WIKILINK_RE = re.compile(r"\[\[([^\[\]|#\n]+?)\\?(?:#[^\[\]|\n]+)?(?:\|[^\[\]\n]*)?\]\]")
|
|
360
|
+
# Link text may not hold `[` and a destination may not hold `[` or a line
|
|
361
|
+
# break, for the same reason: each scan then ends at the next opener.
|
|
362
|
+
MARKDOWN_LINK_RE = re.compile(r"(!?)\[[^\[\]]*\]\(([^)\[\n]+)\)")
|
|
363
|
+
# ASCII digits only: `\d` also matches fullwidth and other Unicode digits, so
|
|
364
|
+
# `E-001` would be a second ID that looks exactly like `E-001`.
|
|
365
|
+
EVIDENCE_ID_RE = re.compile(r"\bE-[0-9]{3,}\b")
|
|
366
|
+
DECISION_ID_RE = re.compile(r"\bD-[0-9]{3,}\b")
|
|
367
|
+
DECISION_FILE_RE = re.compile(r"^d-([0-9]{3,})-")
|
|
368
|
+
TEXT_SECRET_EXTENSIONS = {
|
|
369
|
+
".md", ".txt", ".csv", ".tsv", ".json", ".jsonl", ".yaml", ".yml",
|
|
370
|
+
".xml", ".html", ".log", ".env", ".toml", ".ini", ".conf",
|
|
371
|
+
".py", ".js", ".mjs", ".cjs", ".ts", ".tsx", ".sh",
|
|
372
|
+
}
|
|
373
|
+
SECRET_PATTERNS = (
|
|
374
|
+
("private key block", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")),
|
|
375
|
+
("OpenAI-style key", re.compile(r"\bsk-[A-Za-z0-9_-]{24,}\b")),
|
|
376
|
+
("AWS access key id", re.compile(r"\bAKIA[0-9A-Z]{16}\b")),
|
|
377
|
+
("GitHub token", re.compile(r"\bgh[pousr]_[A-Za-z0-9]{30,}\b")),
|
|
378
|
+
("Slack token", re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{16,}\b")),
|
|
379
|
+
("credential assignment", re.compile(
|
|
380
|
+
r"(?i)\b(api[_-]?key|access[_-]?token|client[_-]?secret|password)[\"']?\s*[:=]\s*"
|
|
381
|
+
r"[\"']?[A-Za-z0-9_\-/+=.]{20,}"
|
|
382
|
+
)),
|
|
383
|
+
)
|
|
384
|
+
_CREDENTIAL_WORDS = ("key", "token", "secret", "password")
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _may_hold_credential(text: str) -> bool:
|
|
388
|
+
# Every credential-assignment match contains one of these words, compared
|
|
389
|
+
# case-insensitively. For ASCII text, `re.IGNORECASE` and `str.lower()`
|
|
390
|
+
# agree letter for letter. Outside ASCII they do not (the regex also
|
|
391
|
+
# matches the Kelvin sign as `k` and the long s as `s`), so such text
|
|
392
|
+
# always gets the full scan.
|
|
393
|
+
if not text.isascii():
|
|
394
|
+
return True
|
|
395
|
+
lowered = text.lower()
|
|
396
|
+
return any(word in lowered for word in _CREDENTIAL_WORDS)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
# A necessary condition for each same-index SECRET_PATTERNS entry: if it is
|
|
400
|
+
# false for a text, the pattern cannot match anywhere in it. The patterns start
|
|
401
|
+
# with ``\b``, so the regex engine tries them at every offset; the prefilters
|
|
402
|
+
# start with a literal that it finds with a fast substring search. Skipping a
|
|
403
|
+
# pattern only when its prefilter fails leaves the findings unchanged.
|
|
404
|
+
# ``tests/test_performance.py`` checks each prefilter against its pattern.
|
|
405
|
+
SECRET_PREFILTERS: tuple[Callable[[str], object], ...] = (
|
|
406
|
+
re.compile(r"-----BEGIN ").search,
|
|
407
|
+
re.compile(r"sk-").search,
|
|
408
|
+
re.compile(r"AKIA").search,
|
|
409
|
+
re.compile(r"gh[pousr]_").search,
|
|
410
|
+
re.compile(r"xox[baprs]-").search,
|
|
411
|
+
_may_hold_credential,
|
|
412
|
+
)
|
|
413
|
+
CONTENT_SKIP_DIRS = {
|
|
414
|
+
".git", ".obsidian", ".import-staging", "node_modules", "__pycache__",
|
|
415
|
+
"apps", "examples", "tests", ".github", "schemas",
|
|
416
|
+
}
|
|
417
|
+
ATTACHMENT_SKIP_DIRS = {".git", ".obsidian", ".import-staging", "node_modules", "__pycache__", ".whykit"}
|
|
418
|
+
SECRET_SKIP_DIRS = {
|
|
419
|
+
".git", ".obsidian", ".import-staging", "node_modules", "__pycache__", "tests",
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
@dataclass
|
|
424
|
+
class Finding:
|
|
425
|
+
path: str
|
|
426
|
+
line: int | None
|
|
427
|
+
level: str
|
|
428
|
+
code: str
|
|
429
|
+
message: str
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
@dataclass
|
|
433
|
+
class Note:
|
|
434
|
+
path: Path
|
|
435
|
+
text: str
|
|
436
|
+
front: dict = field(default_factory=dict)
|
|
437
|
+
has_front: bool = False
|
|
438
|
+
front_error: str | None = None
|
|
439
|
+
body_offset: int = 0 # line count, not a character index
|
|
440
|
+
|
|
441
|
+
@functools.cached_property
|
|
442
|
+
def masked(self) -> str:
|
|
443
|
+
"""The text with code spans and fences blanked; offsets match ``text``."""
|
|
444
|
+
return _mask_code(self.text)
|
|
445
|
+
|
|
446
|
+
@functools.cached_property
|
|
447
|
+
def cited_evidence(self) -> tuple[str, ...]:
|
|
448
|
+
"""Sorted E-NNN IDs this note cites, in front matter or prose.
|
|
449
|
+
|
|
450
|
+
An ID inside inline code or a fenced code block is an example of the
|
|
451
|
+
syntax, not a citation, so graph, impact, context, query and trace all
|
|
452
|
+
ignore it; lint's fact-callout check reads the same masked text.
|
|
453
|
+
"""
|
|
454
|
+
return tuple(sorted(set(EVIDENCE_ID_RE.findall(self.masked))))
|
|
455
|
+
|
|
456
|
+
# The views below are pure functions of ``text``. Each is computed at most
|
|
457
|
+
# once per note, and the persistent parse cache (``whykit.parse_cache``)
|
|
458
|
+
# stores them so an unchanged note is not read again on the next run.
|
|
459
|
+
# NOTE_FACTS lists them; a new one must be added there too.
|
|
460
|
+
|
|
461
|
+
@functools.cached_property
|
|
462
|
+
def wikilink_hits(self) -> tuple[tuple[str, int, bool], ...]:
|
|
463
|
+
"""``(target, line, is_embed)`` for every wikilink outside code.
|
|
464
|
+
|
|
465
|
+
*target* is stripped but otherwise unfiltered: callers decide whether
|
|
466
|
+
an empty or ``http`` target counts.
|
|
467
|
+
"""
|
|
468
|
+
text = self.text
|
|
469
|
+
lines = _LineNumbers(text)
|
|
470
|
+
hits = []
|
|
471
|
+
for match in WIKILINK_RE.finditer(self.masked):
|
|
472
|
+
start = match.start()
|
|
473
|
+
# A vault links the same few targets from many notes; share them.
|
|
474
|
+
hits.append((_shared_target(match.group(1).strip()), lines.at(start), start > 0 and text[start - 1] == "!"))
|
|
475
|
+
return tuple(hits)
|
|
476
|
+
|
|
477
|
+
@functools.cached_property
|
|
478
|
+
def markdown_link_hits(self) -> tuple[tuple[str, str, int], ...]:
|
|
479
|
+
"""``(kind, target, line)`` for each local Markdown link or image outside code."""
|
|
480
|
+
lines = _LineNumbers(self.text)
|
|
481
|
+
hits = []
|
|
482
|
+
for match in MARKDOWN_LINK_RE.finditer(self.masked):
|
|
483
|
+
kind = "image" if match.group(1) else "link"
|
|
484
|
+
raw = match.group(2).strip()
|
|
485
|
+
# Markdown permits <path with spaces>; optional titles are intentionally
|
|
486
|
+
# ignored here rather than pretending to implement a full CommonMark parser.
|
|
487
|
+
if raw.startswith("<") and ">" in raw:
|
|
488
|
+
target = raw[1:raw.index(">")].strip()
|
|
489
|
+
else:
|
|
490
|
+
target = raw.split(None, 1)[0].strip() if raw else ""
|
|
491
|
+
if not target or target.startswith("#") or URL_SCHEME_RE.match(target):
|
|
492
|
+
continue
|
|
493
|
+
target = unquote(target.split("#", 1)[0].split("?", 1)[0])
|
|
494
|
+
if not target:
|
|
495
|
+
continue
|
|
496
|
+
hits.append((kind, target, lines.at(match.start())))
|
|
497
|
+
return tuple(hits)
|
|
498
|
+
|
|
499
|
+
@functools.cached_property
|
|
500
|
+
def fact_callouts(self) -> tuple[tuple[int, tuple[str, ...]], ...]:
|
|
501
|
+
"""``(line, cited IDs in first-seen order)`` for each ``[!fact]`` callout."""
|
|
502
|
+
lines = self.masked.splitlines()
|
|
503
|
+
callouts = []
|
|
504
|
+
i = 0
|
|
505
|
+
while i < len(lines):
|
|
506
|
+
if FACT_CALLOUT_RE.match(lines[i]):
|
|
507
|
+
start = i
|
|
508
|
+
block = [lines[i]]
|
|
509
|
+
i += 1
|
|
510
|
+
while i < len(lines) and lines[i].startswith(">"):
|
|
511
|
+
block.append(lines[i])
|
|
512
|
+
i += 1
|
|
513
|
+
callouts.append((start + 1, tuple(dict.fromkeys(EVIDENCE_ID_RE.findall("\n".join(block))))))
|
|
514
|
+
continue
|
|
515
|
+
i += 1
|
|
516
|
+
return tuple(callouts)
|
|
517
|
+
|
|
518
|
+
@functools.cached_property
|
|
519
|
+
def section_decision_id(self) -> str | None:
|
|
520
|
+
"""The ID under a ``## Decision ID`` heading, if the note has one."""
|
|
521
|
+
m = DECISION_ID_SECTION_RE.search(self.text)
|
|
522
|
+
return m.group(1) if m else None
|
|
523
|
+
|
|
524
|
+
@functools.cached_property
|
|
525
|
+
def secret_hits(self) -> tuple[tuple[int, str], ...]:
|
|
526
|
+
"""``(line, label)`` for each credential-shaped match, in report order."""
|
|
527
|
+
text = self.text
|
|
528
|
+
hits = []
|
|
529
|
+
for (label, pattern), prefilter in zip(SECRET_PATTERNS, SECRET_PREFILTERS, strict=True):
|
|
530
|
+
if not prefilter(text):
|
|
531
|
+
continue
|
|
532
|
+
numbers = _LineNumbers(text)
|
|
533
|
+
for match in pattern.finditer(text):
|
|
534
|
+
hits.append((numbers.at(match.start()), label))
|
|
535
|
+
return tuple(hits)
|
|
536
|
+
|
|
537
|
+
@functools.cached_property
|
|
538
|
+
def decision_placeholders(self) -> tuple[tuple[int, str], ...]:
|
|
539
|
+
"""``(line, section)`` for each section still holding template placeholders."""
|
|
540
|
+
return tuple(_decision_placeholders(self))
|
|
541
|
+
|
|
542
|
+
@property
|
|
543
|
+
def body(self) -> str:
|
|
544
|
+
"""The note text after the front matter block (the whole text when there is none)."""
|
|
545
|
+
if not self.has_front:
|
|
546
|
+
return self.text
|
|
547
|
+
# Same as "\n".join(text.split("\n")[body_offset:]) without building
|
|
548
|
+
# a list of every line: skip past the first body_offset newlines.
|
|
549
|
+
text = self.text
|
|
550
|
+
at = 0
|
|
551
|
+
for _ in range(self.body_offset):
|
|
552
|
+
at = text.find("\n", at) + 1
|
|
553
|
+
if at == 0:
|
|
554
|
+
return ""
|
|
555
|
+
return text[at:]
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
def _strip_yaml_comment(value: str) -> str:
|
|
559
|
+
quote: str | None = None
|
|
560
|
+
escaped = False
|
|
561
|
+
for idx, char in enumerate(value):
|
|
562
|
+
if escaped:
|
|
563
|
+
escaped = False
|
|
564
|
+
continue
|
|
565
|
+
if char == "\\" and quote == '"':
|
|
566
|
+
escaped = True
|
|
567
|
+
continue
|
|
568
|
+
if char in {"'", '"'}:
|
|
569
|
+
if quote is None:
|
|
570
|
+
quote = char
|
|
571
|
+
elif quote == char:
|
|
572
|
+
quote = None
|
|
573
|
+
continue
|
|
574
|
+
if char == "#" and quote is None and (idx == 0 or value[idx - 1].isspace()):
|
|
575
|
+
return value[:idx].rstrip()
|
|
576
|
+
return value.strip()
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _split_inline_list(inner: str) -> list[str]:
|
|
580
|
+
parts: list[str] = []
|
|
581
|
+
buf: list[str] = []
|
|
582
|
+
quote: str | None = None
|
|
583
|
+
escaped = False
|
|
584
|
+
for char in inner:
|
|
585
|
+
if escaped:
|
|
586
|
+
buf.append(char)
|
|
587
|
+
escaped = False
|
|
588
|
+
continue
|
|
589
|
+
if char == "\\" and quote == '"':
|
|
590
|
+
buf.append(char)
|
|
591
|
+
escaped = True
|
|
592
|
+
continue
|
|
593
|
+
if char in {"'", '"'}:
|
|
594
|
+
if quote is None:
|
|
595
|
+
quote = char
|
|
596
|
+
elif quote == char:
|
|
597
|
+
quote = None
|
|
598
|
+
buf.append(char)
|
|
599
|
+
continue
|
|
600
|
+
if char == "," and quote is None:
|
|
601
|
+
parts.append("".join(buf).strip())
|
|
602
|
+
buf = []
|
|
603
|
+
continue
|
|
604
|
+
buf.append(char)
|
|
605
|
+
if quote is not None:
|
|
606
|
+
raise ValueError("unterminated quote in inline list")
|
|
607
|
+
if buf or inner.strip():
|
|
608
|
+
parts.append("".join(buf).strip())
|
|
609
|
+
return parts
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
# Front matter repeats the same keys and many of the same short values
|
|
613
|
+
# (status, type, owner, tags) in every note. Sharing one string object per
|
|
614
|
+
# spelling saves tens of megabytes at 20,000 notes. The pool is bounded so a
|
|
615
|
+
# long-lived process reading many vaults cannot grow it without limit.
|
|
616
|
+
_SHARED: dict[str, str] = {}
|
|
617
|
+
_SHARED_MAX_ENTRIES = 8192
|
|
618
|
+
_SHARED_MAX_CHARS = 64
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def _shared(value: str) -> str:
|
|
622
|
+
if len(value) > _SHARED_MAX_CHARS:
|
|
623
|
+
return value
|
|
624
|
+
hit = _SHARED.get(value)
|
|
625
|
+
if hit is not None:
|
|
626
|
+
return hit
|
|
627
|
+
if len(_SHARED) < _SHARED_MAX_ENTRIES:
|
|
628
|
+
_SHARED[value] = value
|
|
629
|
+
return value
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
# Link targets get a pool of their own: a large vault has more distinct
|
|
633
|
+
# targets than the front-matter pool holds, and filling that pool with them
|
|
634
|
+
# would stop front-matter values from being shared.
|
|
635
|
+
_SHARED_TARGETS: dict[str, str] = {}
|
|
636
|
+
_SHARED_TARGETS_MAX_ENTRIES = 65536
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _shared_target(value: str) -> str:
|
|
640
|
+
if len(value) > _SHARED_MAX_CHARS:
|
|
641
|
+
return value
|
|
642
|
+
hit = _SHARED_TARGETS.get(value)
|
|
643
|
+
if hit is not None:
|
|
644
|
+
return hit
|
|
645
|
+
if len(_SHARED_TARGETS) < _SHARED_TARGETS_MAX_ENTRIES:
|
|
646
|
+
_SHARED_TARGETS[value] = value
|
|
647
|
+
return value
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
def _yaml_scalar(value: str) -> object:
|
|
651
|
+
result = _yaml_scalar_raw(value)
|
|
652
|
+
return _shared(result) if type(result) is str else result
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def _yaml_scalar_raw(value: str) -> object:
|
|
656
|
+
import json as _json
|
|
657
|
+
|
|
658
|
+
value = _strip_yaml_comment(value).strip()
|
|
659
|
+
if value == "[]":
|
|
660
|
+
return []
|
|
661
|
+
if value.startswith("[") and value.endswith("]"):
|
|
662
|
+
inner = value[1:-1].strip()
|
|
663
|
+
items = _split_inline_list(inner) if inner else []
|
|
664
|
+
# One level only, like block lists. Recursing let a line of brackets
|
|
665
|
+
# cost quadratic time and then a RecursionError.
|
|
666
|
+
for item in items:
|
|
667
|
+
if item.startswith("[") and item.endswith("]"):
|
|
668
|
+
raise ValueError("nested lists are not supported in front matter")
|
|
669
|
+
return [_yaml_scalar(item) for item in items]
|
|
670
|
+
if value.lower() in ("true", "false"):
|
|
671
|
+
return value.lower() == "true"
|
|
672
|
+
if value in ("null", "~"):
|
|
673
|
+
return None
|
|
674
|
+
if len(value) >= 2 and value[0] == value[-1] == '"':
|
|
675
|
+
try:
|
|
676
|
+
return _json.loads(value)
|
|
677
|
+
except _json.JSONDecodeError as exc:
|
|
678
|
+
raise ValueError(f"invalid double-quoted scalar: {value!r}") from exc
|
|
679
|
+
if len(value) >= 2 and value[0] == value[-1] == "'":
|
|
680
|
+
return value[1:-1].replace("''", "'")
|
|
681
|
+
return value
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
BLOCK_SCALAR_RE = re.compile(r"^[|>][+-]?[1-9]?[+-]?$")
|
|
685
|
+
FRONT_CLOSE_RE = re.compile(r"^---$", re.MULTILINE)
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def _front_value(value: str, key: str) -> object:
|
|
689
|
+
if BLOCK_SCALAR_RE.match(_strip_yaml_comment(value)):
|
|
690
|
+
raise ValueError(
|
|
691
|
+
f"{key!r} uses a block scalar ({value.strip()}); WhyKit reads single-line values only"
|
|
692
|
+
)
|
|
693
|
+
return _yaml_scalar(value)
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
def _is_sequence_item(content: str) -> bool:
|
|
697
|
+
return content.startswith("- ")
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
def _parse_front_matter(raw: str) -> dict:
|
|
701
|
+
"""Parse the deliberately small YAML subset supported by WhyKit.
|
|
702
|
+
|
|
703
|
+
Supported: top-level scalars/lists, block lists, and one nested mapping
|
|
704
|
+
level (used by `provenance`), including block lists inside that mapping.
|
|
705
|
+
Block-list items may sit at any consistent indentation, including the
|
|
706
|
+
column-0 form (`tags:` followed by `- a`) that many editors write.
|
|
707
|
+
Rich YAML features such as anchors, block scalars and arbitrary nesting are
|
|
708
|
+
rejected instead of behaving differently depending on installed packages.
|
|
709
|
+
"""
|
|
710
|
+
out: dict = {}
|
|
711
|
+
current_top: str | None = None
|
|
712
|
+
child_indent: int | None = None
|
|
713
|
+
current_nested: str | None = None
|
|
714
|
+
nested_indent: int | None = None
|
|
715
|
+
nested_item_indent: int | None = None
|
|
716
|
+
|
|
717
|
+
for raw_line in raw.splitlines():
|
|
718
|
+
if not raw_line.strip() or raw_line.lstrip().startswith("#"):
|
|
719
|
+
continue
|
|
720
|
+
if "\t" in raw_line[: len(raw_line) - len(raw_line.lstrip())]:
|
|
721
|
+
raise ValueError("tabs are not supported for YAML indentation")
|
|
722
|
+
indent = len(raw_line) - len(raw_line.lstrip(" "))
|
|
723
|
+
content = raw_line.strip()
|
|
724
|
+
|
|
725
|
+
if indent == 0 and not _is_sequence_item(content):
|
|
726
|
+
if ":" not in content:
|
|
727
|
+
raise ValueError(f"cannot read line: {raw_line!r}")
|
|
728
|
+
key, _, value = content.partition(":")
|
|
729
|
+
key = _shared(key.strip())
|
|
730
|
+
if not key:
|
|
731
|
+
raise ValueError("front matter key cannot be empty")
|
|
732
|
+
if key in out:
|
|
733
|
+
raise ValueError(f"duplicate front matter key: {key!r}")
|
|
734
|
+
value = value.strip()
|
|
735
|
+
current_top = key
|
|
736
|
+
child_indent = None
|
|
737
|
+
current_nested = None
|
|
738
|
+
nested_indent = None
|
|
739
|
+
out[key] = None if value == "" else _front_value(value, key)
|
|
740
|
+
continue
|
|
741
|
+
|
|
742
|
+
if current_top is None:
|
|
743
|
+
raise ValueError(f"cannot read line: {raw_line!r}")
|
|
744
|
+
parent = out.get(current_top)
|
|
745
|
+
|
|
746
|
+
# A list item under the open nested key: deeper than the key, or at the
|
|
747
|
+
# same column (YAML's compact form) once the key holds a list.
|
|
748
|
+
if (
|
|
749
|
+
current_nested is not None
|
|
750
|
+
and nested_indent is not None
|
|
751
|
+
and _is_sequence_item(content)
|
|
752
|
+
and (indent > nested_indent or (indent == nested_indent and isinstance(parent, dict) and isinstance(parent.get(current_nested), list)))
|
|
753
|
+
):
|
|
754
|
+
if not isinstance(parent, dict):
|
|
755
|
+
raise ValueError(f"cannot nest list under {current_top!r}")
|
|
756
|
+
nested = parent.get(current_nested)
|
|
757
|
+
if not isinstance(nested, list):
|
|
758
|
+
raise ValueError(f"{current_top}.{current_nested} is not a list")
|
|
759
|
+
if nested_item_indent is None:
|
|
760
|
+
nested_item_indent = indent
|
|
761
|
+
elif indent != nested_item_indent:
|
|
762
|
+
raise ValueError(f"inconsistent list indentation under {current_top}.{current_nested}")
|
|
763
|
+
nested.append(_yaml_scalar(content[2:].strip()))
|
|
764
|
+
continue
|
|
765
|
+
|
|
766
|
+
if child_indent is None:
|
|
767
|
+
child_indent = indent
|
|
768
|
+
if indent != child_indent:
|
|
769
|
+
raise ValueError(f"unsupported YAML indentation/structure: {raw_line!r}")
|
|
770
|
+
|
|
771
|
+
if _is_sequence_item(content):
|
|
772
|
+
if parent is None:
|
|
773
|
+
parent = []
|
|
774
|
+
out[current_top] = parent
|
|
775
|
+
if not isinstance(parent, list):
|
|
776
|
+
raise ValueError(f"{current_top!r} mixes mapping/scalar and list values")
|
|
777
|
+
current_nested = None
|
|
778
|
+
nested_indent = None
|
|
779
|
+
parent.append(_yaml_scalar(content[2:].strip()))
|
|
780
|
+
continue
|
|
781
|
+
if indent == 0 or ":" not in content:
|
|
782
|
+
raise ValueError(f"cannot read nested line: {raw_line!r}")
|
|
783
|
+
if parent == []:
|
|
784
|
+
raise ValueError(f"{current_top!r} mixes list and mapping values")
|
|
785
|
+
if parent is None:
|
|
786
|
+
parent = {}
|
|
787
|
+
out[current_top] = parent
|
|
788
|
+
if not isinstance(parent, dict):
|
|
789
|
+
raise ValueError(f"cannot nest under scalar key {current_top!r}")
|
|
790
|
+
nested_key, _, value = content.partition(":")
|
|
791
|
+
nested_key = _shared(nested_key.strip())
|
|
792
|
+
if not nested_key:
|
|
793
|
+
raise ValueError("front matter nested key cannot be empty")
|
|
794
|
+
if nested_key in parent:
|
|
795
|
+
raise ValueError(f"duplicate front matter key: {current_top}.{nested_key!r}")
|
|
796
|
+
value = value.strip()
|
|
797
|
+
current_nested = nested_key
|
|
798
|
+
nested_indent = indent
|
|
799
|
+
nested_item_indent = None
|
|
800
|
+
parent[nested_key] = [] if value == "" else _front_value(value, f"{current_top}.{nested_key}")
|
|
801
|
+
return out
|
|
802
|
+
|
|
803
|
+
|
|
804
|
+
def read_vault_text(path: Path) -> tuple[str, str | None]:
|
|
805
|
+
"""Read a vault file for checking: (text, problem).
|
|
806
|
+
|
|
807
|
+
A file that is not valid UTF-8 is read with U+FFFD for the bad bytes and
|
|
808
|
+
the problem names the first one, so one damaged or hostile file becomes a
|
|
809
|
+
finding instead of stopping every command that reads the vault.
|
|
810
|
+
"""
|
|
811
|
+
data = path.read_bytes()
|
|
812
|
+
problem: str | None = None
|
|
813
|
+
try:
|
|
814
|
+
text = data.decode("utf-8")
|
|
815
|
+
except UnicodeDecodeError as exc:
|
|
816
|
+
text = data.decode("utf-8", "replace")
|
|
817
|
+
problem = f"file is not valid UTF-8 (byte {exc.start}); WhyKit read it with the bad bytes replaced"
|
|
818
|
+
# The same newline handling as Path.read_text().
|
|
819
|
+
return text.replace("\r\n", "\n").replace("\r", "\n"), problem
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
def read_vault_lines(path: Path) -> list[str]:
|
|
823
|
+
"""The lines of a vault file, with any bytes that are not UTF-8 replaced."""
|
|
824
|
+
return read_vault_text(path)[0].splitlines()
|
|
825
|
+
|
|
826
|
+
|
|
827
|
+
def _normalise(text: str) -> str:
|
|
828
|
+
"""Decoded file text as every parser sees it.
|
|
829
|
+
|
|
830
|
+
Path.read_text() uses universal newlines; callers with decoded bytes
|
|
831
|
+
(notably adopt's hash-preserving scan) must see the same parser input.
|
|
832
|
+
Editors on Windows commonly save UTF-8 with a byte-order mark; it is not
|
|
833
|
+
content, and leaving it in place hides the front matter fence.
|
|
834
|
+
"""
|
|
835
|
+
return text.replace("\r\n", "\n").replace("\r", "\n").removeprefix("\ufeff")
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
def load_note(path: Path, *, text: str | None = None) -> Note:
|
|
839
|
+
encoding_problem = None
|
|
840
|
+
if text is None:
|
|
841
|
+
text, encoding_problem = read_vault_text(path)
|
|
842
|
+
text = text.removeprefix("\ufeff")
|
|
843
|
+
else:
|
|
844
|
+
text = _normalise(text)
|
|
845
|
+
note = Note(path=path, text=text)
|
|
846
|
+
if not (text.startswith("---\n") or text == "---"):
|
|
847
|
+
note.front_error = encoding_problem
|
|
848
|
+
return note
|
|
849
|
+
# The closing fence may be the very next line (empty front matter) or the
|
|
850
|
+
# last line of a file with no trailing newline.
|
|
851
|
+
close = FRONT_CLOSE_RE.search(text, 4)
|
|
852
|
+
if close is None:
|
|
853
|
+
note.front_error = encoding_problem or "front matter opened with --- but never closed"
|
|
854
|
+
return note
|
|
855
|
+
note.has_front = True
|
|
856
|
+
note.body_offset = text[: close.end()].count("\n") + 1
|
|
857
|
+
try:
|
|
858
|
+
note.front = _parse_front_matter(text[4:close.start()])
|
|
859
|
+
except Exception as exc: # noqa: BLE001
|
|
860
|
+
note.front_error = f"front matter is not valid YAML: {exc}"
|
|
861
|
+
if encoding_problem is not None:
|
|
862
|
+
note.front_error = encoding_problem
|
|
863
|
+
return note
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
def rel(root: Path, path: Path) -> str:
|
|
867
|
+
# Resolve both sides so macOS /var vs /private/var (and similar aliasing)
|
|
868
|
+
# does not break relative paths or silently fall back to absolutes.
|
|
869
|
+
cache = _request_cache()
|
|
870
|
+
key = (os.fspath(root), os.fspath(path))
|
|
871
|
+
if cache is not None:
|
|
872
|
+
hit = cache.relative.get(key)
|
|
873
|
+
if hit is not None:
|
|
874
|
+
return hit
|
|
875
|
+
resolved = _real(path)
|
|
876
|
+
out = _relative_text(resolved, _real(root))
|
|
877
|
+
if out is None:
|
|
878
|
+
out = resolved.as_posix()
|
|
879
|
+
if cache is not None:
|
|
880
|
+
cache.relative[key] = out
|
|
881
|
+
return out
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
def add(findings: list[Finding], root: Path, path: Path, line: int | None, level: str, code: str, message: str) -> None:
|
|
885
|
+
findings.append(Finding(rel(root, path), line, level, code, message))
|
|
886
|
+
|
|
887
|
+
|
|
888
|
+
def _as_list(value: object) -> list[str]:
|
|
889
|
+
if value is None or value == "":
|
|
890
|
+
return []
|
|
891
|
+
if isinstance(value, list):
|
|
892
|
+
return [str(v).strip() for v in value if str(v).strip()]
|
|
893
|
+
return [str(value).strip()]
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def _parse_date(value: object) -> dt.date | None:
|
|
897
|
+
text = str(value)
|
|
898
|
+
if not DATE_RE.match(text):
|
|
899
|
+
return None
|
|
900
|
+
try:
|
|
901
|
+
return dt.date.fromisoformat(text)
|
|
902
|
+
except ValueError:
|
|
903
|
+
return None
|
|
904
|
+
|
|
905
|
+
|
|
906
|
+
def check_front_matter(root: Path, note: Note, findings: list[Finding], today: dt.date) -> None:
|
|
907
|
+
r = Path(rel(root, note.path))
|
|
908
|
+
exempt = r.name in NO_FRONT_MATTER_REQUIRED or r.name == "README.md"
|
|
909
|
+
if note.front_error:
|
|
910
|
+
add(findings, root, note.path, 1, "error", "frontmatter.invalid", note.front_error)
|
|
911
|
+
return
|
|
912
|
+
if not note.has_front:
|
|
913
|
+
if not exempt:
|
|
914
|
+
add(findings, root, note.path, 1, "error", "frontmatter.missing", "no YAML front matter — see OBSIDIAN.md")
|
|
915
|
+
return
|
|
916
|
+
|
|
917
|
+
for key in REQUIRED_KEYS:
|
|
918
|
+
if key not in note.front:
|
|
919
|
+
add(findings, root, note.path, 1, "error", "frontmatter.required", f"front matter is missing `{key}`")
|
|
920
|
+
elif note.front[key] is None or (isinstance(note.front[key], (str, list)) and not note.front[key]):
|
|
921
|
+
# `owner:` with nothing after it parses, so the key is "present"; every
|
|
922
|
+
# value check below would then skip it silently.
|
|
923
|
+
add(findings, root, note.path, 1, "warning", "frontmatter.empty", f"`{key}` is present but empty")
|
|
924
|
+
|
|
925
|
+
source_of_truth = note.front.get("source_of_truth")
|
|
926
|
+
if source_of_truth is not None and not isinstance(source_of_truth, bool):
|
|
927
|
+
add(
|
|
928
|
+
findings, root, note.path, 1, "warning", "source_of_truth.invalid",
|
|
929
|
+
f"source_of_truth `{source_of_truth}` is not a boolean; use true or false "
|
|
930
|
+
"(the canonical-document checks do not apply to this note)",
|
|
931
|
+
)
|
|
932
|
+
|
|
933
|
+
status = note.front.get("status")
|
|
934
|
+
if status is not None and (not isinstance(status, str) or status not in ALLOWED_STATUS):
|
|
935
|
+
add(findings, root, note.path, 1, "error", "status.invalid", f"status `{status}` is not allowed")
|
|
936
|
+
doc_type = note.front.get("type")
|
|
937
|
+
if doc_type is not None and (not isinstance(doc_type, str) or doc_type not in ALLOWED_TYPE):
|
|
938
|
+
add(findings, root, note.path, 1, "error", "type.invalid", f"type `{doc_type}` is not allowed")
|
|
939
|
+
sensitivity = note.front.get("sensitivity")
|
|
940
|
+
if sensitivity is not None and (not isinstance(sensitivity, str) or sensitivity not in ALLOWED_SENSITIVITY):
|
|
941
|
+
add(findings, root, note.path, 1, "error", "sensitivity.invalid", f"sensitivity `{sensitivity}` is not allowed")
|
|
942
|
+
|
|
943
|
+
if note.front.get("source_of_truth") is True and status != "approved":
|
|
944
|
+
add(
|
|
945
|
+
findings, root, note.path, 1, "error", "canonical.unapproved",
|
|
946
|
+
"source_of_truth=true requires status=approved; drafts cannot be canonical",
|
|
947
|
+
)
|
|
948
|
+
|
|
949
|
+
is_template = status == "template"
|
|
950
|
+
parsed_dates: dict[str, dt.date] = {}
|
|
951
|
+
for key in ("created", "last_updated"):
|
|
952
|
+
value = note.front.get(key)
|
|
953
|
+
if value is None:
|
|
954
|
+
continue
|
|
955
|
+
text = str(value)
|
|
956
|
+
if text == PLACEHOLDER_DATE:
|
|
957
|
+
if not is_template:
|
|
958
|
+
add(findings, root, note.path, 1, "error", "date.placeholder", f"`{key}` still holds {PLACEHOLDER_DATE}")
|
|
959
|
+
continue
|
|
960
|
+
parsed = _parse_date(text)
|
|
961
|
+
if parsed is None:
|
|
962
|
+
add(findings, root, note.path, 1, "error", "date.invalid", f"`{key}` is not a real ISO date: {text}")
|
|
963
|
+
else:
|
|
964
|
+
parsed_dates[key] = parsed
|
|
965
|
+
if parsed_dates.get("last_updated") and parsed_dates.get("created") and parsed_dates["last_updated"] < parsed_dates["created"]:
|
|
966
|
+
add(findings, root, note.path, 1, "error", "date.order", "last_updated is before created")
|
|
967
|
+
|
|
968
|
+
review_by = note.front.get("review_by")
|
|
969
|
+
if review_by and str(review_by) != PLACEHOLDER_DATE:
|
|
970
|
+
parsed_review = _parse_date(review_by)
|
|
971
|
+
if parsed_review is None:
|
|
972
|
+
add(findings, root, note.path, 1, "error", "review_by.invalid", f"review_by is not a real ISO date: {review_by}")
|
|
973
|
+
elif parsed_review < today and status == "approved":
|
|
974
|
+
add(findings, root, note.path, 1, "warning", "review_by.overdue", f"approved document review was due {parsed_review.isoformat()}")
|
|
975
|
+
|
|
976
|
+
owner = str(note.front.get("owner", ""))
|
|
977
|
+
if note.front.get("source_of_truth") is True and owner in ("", PLACEHOLDER_OWNER):
|
|
978
|
+
add(findings, root, note.path, 1, "warning", "canonical.owner", "canonical document has no real owner")
|
|
979
|
+
|
|
980
|
+
delivery_status = note.front.get("delivery_status")
|
|
981
|
+
if delivery_status and (not isinstance(delivery_status, str) or delivery_status not in ALLOWED_DELIVERY_STATUS):
|
|
982
|
+
add(findings, root, note.path, 1, "error", "delivery_status.invalid", f"delivery_status `{delivery_status}` is not allowed")
|
|
983
|
+
if delivery_status == "sent" and not note.front.get("sent_at"):
|
|
984
|
+
add(findings, root, note.path, 1, "error", "delivery_status.sent_at", "delivery_status=sent requires sent_at")
|
|
985
|
+
|
|
986
|
+
# Point-in-time reports must encode their date in the filename.
|
|
987
|
+
if "reports" in r.parts and r.name != "README.md" and not DATED_FILENAME_RE.search(note.path.stem):
|
|
988
|
+
add(findings, root, note.path, 1, "error", "report.undated", "report filename must end in YYYY-MM-DD")
|
|
989
|
+
|
|
990
|
+
m = DATED_FILENAME_RE.search(note.path.stem)
|
|
991
|
+
if m and parsed_dates.get("created"):
|
|
992
|
+
try:
|
|
993
|
+
filename_date = dt.date.fromisoformat(m.group(1))
|
|
994
|
+
except ValueError:
|
|
995
|
+
filename_date = None
|
|
996
|
+
if filename_date and filename_date != parsed_dates["created"]:
|
|
997
|
+
add(findings, root, note.path, 1, "warning", "date.filename_mismatch", f"filename says {filename_date} but created says {parsed_dates['created']}")
|
|
998
|
+
|
|
999
|
+
|
|
1000
|
+
class _LineNumbers:
|
|
1001
|
+
"""1-based line of a character offset, counted incrementally.
|
|
1002
|
+
|
|
1003
|
+
Same answer as ``text[:offset].count("\\n") + 1``, without rescanning the
|
|
1004
|
+
text from the start for every match of a left-to-right ``finditer``.
|
|
1005
|
+
"""
|
|
1006
|
+
|
|
1007
|
+
__slots__ = ("text", "offset", "line")
|
|
1008
|
+
|
|
1009
|
+
def __init__(self, text: str) -> None:
|
|
1010
|
+
self.text = text
|
|
1011
|
+
self.offset = 0
|
|
1012
|
+
self.line = 1
|
|
1013
|
+
|
|
1014
|
+
def at(self, offset: int) -> int:
|
|
1015
|
+
if offset < self.offset:
|
|
1016
|
+
self.offset, self.line = 0, 1
|
|
1017
|
+
self.line += self.text.count("\n", self.offset, offset)
|
|
1018
|
+
self.offset = offset
|
|
1019
|
+
return self.line
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
URL_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:")
|
|
1023
|
+
FACT_CALLOUT_RE = re.compile(r"^> \[!fact\]", re.I)
|
|
1024
|
+
DECISION_ID_SECTION_RE = re.compile(r"^## Decision ID\s*$\n+\s*(D-[0-9]{3,})\s*$", re.MULTILINE)
|
|
1025
|
+
def _link_key(value: str) -> str:
|
|
1026
|
+
# Filenames written on macOS are often NFD while typed link text is NFC; on
|
|
1027
|
+
# a Linux CI checkout the two spellings are different byte strings.
|
|
1028
|
+
return unicodedata.normalize("NFC", value).casefold()
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
def _relative_key(root: Path, path: Path) -> str | None:
|
|
1032
|
+
return _relative_key_of(os.fspath(root), os.fspath(path))
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
@functools.lru_cache(maxsize=65536)
|
|
1036
|
+
def _relative_key_of(root: str, path: str) -> str | None:
|
|
1037
|
+
# A pure string function (no filesystem access), so a process-wide cache
|
|
1038
|
+
# cannot go stale; link resolution asks for the same few paths repeatedly.
|
|
1039
|
+
if _POSIX and path.startswith("/") and root.startswith("/"):
|
|
1040
|
+
# Both are spellings of already-normalised absolute paths (``os.fspath``
|
|
1041
|
+
# of a ``Path``), so ``relative_to`` reduces to a prefix test on the
|
|
1042
|
+
# strings, without building and comparing a list of parts.
|
|
1043
|
+
prefix = root if root.endswith("/") else root + "/"
|
|
1044
|
+
if path == root:
|
|
1045
|
+
return _link_key(".")
|
|
1046
|
+
return _link_key(path[len(prefix):]) if path.startswith(prefix) else None
|
|
1047
|
+
try:
|
|
1048
|
+
return _link_key(PurePath(path).relative_to(root).as_posix())
|
|
1049
|
+
except ValueError:
|
|
1050
|
+
return None
|
|
1051
|
+
|
|
1052
|
+
|
|
1053
|
+
def _build_index(notes: list[Note]) -> dict[str, set[Path]]:
|
|
1054
|
+
index: dict[str, set[Path]] = {}
|
|
1055
|
+
for note in notes:
|
|
1056
|
+
for key in {note.path.stem, *[a for a in _as_list(note.front.get("aliases")) if a]}:
|
|
1057
|
+
index.setdefault(_link_key(key), set()).add(note.path)
|
|
1058
|
+
return index
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
Resolver = Callable[[str], "tuple[Path | None, bool]"]
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
@functools.lru_cache(maxsize=65536)
|
|
1065
|
+
def _leaves_vault(normalized: str) -> bool:
|
|
1066
|
+
"""True when a link target has a `..` component (pure string function)."""
|
|
1067
|
+
return ".." in Path(normalized).parts
|
|
1068
|
+
|
|
1069
|
+
|
|
1070
|
+
def _resolve(root: Path, target: str, index: dict[str, set[Path]]) -> tuple[Path | None, bool]:
|
|
1071
|
+
normalized = target.strip().replace("\\", "/").lstrip("/")
|
|
1072
|
+
if _leaves_vault(normalized):
|
|
1073
|
+
return None, False
|
|
1074
|
+
relative_target = normalized if is_markdown_name(normalized) else normalized + ".md"
|
|
1075
|
+
stem = _link_key(strip_markdown_suffix(normalized.rstrip("/").split("/")[-1]))
|
|
1076
|
+
hits = {path for path in index.get(stem, set()) if _within(root, path)}
|
|
1077
|
+
# Match the typed path against real note paths case- and NFC-insensitively,
|
|
1078
|
+
# as Obsidian does. `exists()` alone answers differently per filesystem:
|
|
1079
|
+
# on macOS and Windows it accepts `[[NOTES/readme]]` but hands back the
|
|
1080
|
+
# typed spelling, which is not a node ID; on Linux it misses entirely.
|
|
1081
|
+
wanted = _link_key(relative_target)
|
|
1082
|
+
spelled = [path for path in hits if _relative_key(root, path) == wanted]
|
|
1083
|
+
if len(spelled) == 1:
|
|
1084
|
+
return spelled[0], False
|
|
1085
|
+
candidate = root / relative_target
|
|
1086
|
+
# Containment first: it rejects another drive or a UNC share from the
|
|
1087
|
+
# strings alone, before anything asks the file system about the path.
|
|
1088
|
+
if _within(root, candidate) and path_exists(candidate):
|
|
1089
|
+
return candidate, False
|
|
1090
|
+
if len(hits) == 1:
|
|
1091
|
+
return next(iter(hits)), False
|
|
1092
|
+
if len(hits) > 1:
|
|
1093
|
+
return None, True
|
|
1094
|
+
return None, False
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
class AttachmentIndex:
|
|
1098
|
+
"""Non-Markdown files a wikilink may target (`![[diagram.png]]`, `[[brief.pdf]]`).
|
|
1099
|
+
|
|
1100
|
+
Built lazily: most vaults never link an attachment, and walking the tree is
|
|
1101
|
+
the expensive part.
|
|
1102
|
+
"""
|
|
1103
|
+
|
|
1104
|
+
def __init__(self, root: Path) -> None:
|
|
1105
|
+
self.root = root
|
|
1106
|
+
self._by_name: dict[str, list[Path]] | None = None
|
|
1107
|
+
|
|
1108
|
+
def _names(self) -> dict[str, list[Path]]:
|
|
1109
|
+
if self._by_name is None:
|
|
1110
|
+
self._by_name = {}
|
|
1111
|
+
# rglob yields root-prefixed paths, so slicing the parts is the
|
|
1112
|
+
# same as relative_to() without re-parsing every path.
|
|
1113
|
+
depth = len(self.root.parts)
|
|
1114
|
+
for path in self.root.rglob("*"):
|
|
1115
|
+
if is_markdown_name(path.name) or not path.is_file() or not _within(self.root, path):
|
|
1116
|
+
continue
|
|
1117
|
+
if any(part in ATTACHMENT_SKIP_DIRS for part in path.parts[depth:-1]):
|
|
1118
|
+
continue
|
|
1119
|
+
self._by_name.setdefault(_link_key(path.name), []).append(path)
|
|
1120
|
+
return self._by_name
|
|
1121
|
+
|
|
1122
|
+
def resolves(self, target: str, note_dir: Path) -> bool:
|
|
1123
|
+
normalized = target.strip().replace("\\", "/")
|
|
1124
|
+
suffix = Path(normalized).suffix.lower()
|
|
1125
|
+
if not suffix or suffix == ".md":
|
|
1126
|
+
return False
|
|
1127
|
+
for candidate in (self.root / normalized.lstrip("/"), note_dir / normalized):
|
|
1128
|
+
if _within(self.root, candidate) and path_is_file(candidate):
|
|
1129
|
+
return True
|
|
1130
|
+
# Obsidian resolves attachments by file name anywhere in the vault and
|
|
1131
|
+
# picks the nearest copy, so several matches are not an ambiguity here.
|
|
1132
|
+
return bool(self._names().get(_link_key(normalized.rstrip("/").split("/")[-1])))
|
|
1133
|
+
|
|
1134
|
+
|
|
1135
|
+
def check_wikilinks(
|
|
1136
|
+
root: Path,
|
|
1137
|
+
note: Note,
|
|
1138
|
+
index: dict[str, set[Path]],
|
|
1139
|
+
findings: list[Finding],
|
|
1140
|
+
attachments: AttachmentIndex | None = None,
|
|
1141
|
+
resolve: Resolver | None = None,
|
|
1142
|
+
) -> None:
|
|
1143
|
+
for target, line, is_embed in note.wikilink_hits:
|
|
1144
|
+
if not target or target.startswith(("http://", "https://")):
|
|
1145
|
+
continue
|
|
1146
|
+
if _leaves_vault(target.replace("\\", "/")):
|
|
1147
|
+
add(findings, root, note.path, line, "warning", "wikilink.outside", f"wikilink tries to leave the vault: [[{target}]]")
|
|
1148
|
+
continue
|
|
1149
|
+
resolved, ambiguous = resolve(target) if resolve else _resolve(root, target, index)
|
|
1150
|
+
if resolved is None and not ambiguous:
|
|
1151
|
+
if attachments is None:
|
|
1152
|
+
attachments = AttachmentIndex(root)
|
|
1153
|
+
if attachments.resolves(target, note.path.parent):
|
|
1154
|
+
continue
|
|
1155
|
+
if ambiguous:
|
|
1156
|
+
add(findings, root, note.path, line, "error", "wikilink.ambiguous", f"wikilink is ambiguous: [[{target}]] — use a path")
|
|
1157
|
+
elif resolved is None:
|
|
1158
|
+
if is_embed:
|
|
1159
|
+
add(
|
|
1160
|
+
findings, root, note.path, line, "error", "embed.missing",
|
|
1161
|
+
f"embed does not resolve: ![[{target}]]",
|
|
1162
|
+
)
|
|
1163
|
+
else:
|
|
1164
|
+
add(findings, root, note.path, line, "error", "wikilink.missing", f"wikilink does not resolve: [[{target}]]")
|
|
1165
|
+
|
|
1166
|
+
|
|
1167
|
+
def check_markdown_links(
|
|
1168
|
+
root: Path,
|
|
1169
|
+
note: Note,
|
|
1170
|
+
findings: list[Finding],
|
|
1171
|
+
exists: Callable[[Path], bool] | None = None,
|
|
1172
|
+
) -> None:
|
|
1173
|
+
"""Flag local Markdown links that leave the vault or point nowhere.
|
|
1174
|
+
|
|
1175
|
+
*exists* overrides the on-disk check, so a confined view of the vault can
|
|
1176
|
+
treat notes it does not contain exactly like files that do not exist.
|
|
1177
|
+
"""
|
|
1178
|
+
for kind, target, line in note.markdown_link_hits:
|
|
1179
|
+
candidate = (root / target.lstrip("/")) if target.startswith("/") else (note.path.parent / target)
|
|
1180
|
+
if not _within(root, candidate):
|
|
1181
|
+
add(findings, root, note.path, line, "warning", "markdown_link.outside", f"Markdown {kind} leaves the vault: ({target})")
|
|
1182
|
+
elif not (exists(candidate) if exists else path_exists(candidate)):
|
|
1183
|
+
add(findings, root, note.path, line, "warning", "markdown_link.missing", f"local Markdown {kind} does not exist: ({target})")
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
@dataclass
|
|
1187
|
+
class _EvidenceView:
|
|
1188
|
+
root: Path
|
|
1189
|
+
rows: tuple | Callable[[], tuple]
|
|
1190
|
+
request: _RequestCache | None
|
|
1191
|
+
active: bool = True
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
_EVIDENCE_VIEW: contextvars.ContextVar[_EvidenceView | None] = contextvars.ContextVar("whykit_evidence_view", default=None)
|
|
1195
|
+
|
|
1196
|
+
|
|
1197
|
+
@contextlib.contextmanager
|
|
1198
|
+
def evidence_view(root: Path, rows: tuple | Callable[[], tuple]):
|
|
1199
|
+
"""Use one captured, filtered register throughout a read-only request."""
|
|
1200
|
+
with path_cache():
|
|
1201
|
+
view = _EvidenceView(_real(root), rows, _request_cache())
|
|
1202
|
+
token = _EVIDENCE_VIEW.set(view)
|
|
1203
|
+
try:
|
|
1204
|
+
yield
|
|
1205
|
+
finally:
|
|
1206
|
+
view.active = False
|
|
1207
|
+
_EVIDENCE_VIEW.reset(token)
|
|
1208
|
+
|
|
1209
|
+
|
|
1210
|
+
def evidence_register(root: Path) -> tuple[dict[str, dict[str, str]], dict[str, dict[str, str]], list[tuple[str, int]]]:
|
|
1211
|
+
view = _EVIDENCE_VIEW.get()
|
|
1212
|
+
cache = _request_cache()
|
|
1213
|
+
if view is not None and view.active and cache is not None and view.request is cache and view.root == _real(root):
|
|
1214
|
+
rows = view.rows() if callable(view.rows) else view.rows
|
|
1215
|
+
active, retired, occurrences = rows
|
|
1216
|
+
return ({key: dict(row) for key, row in active.items()}, {key: dict(row) for key, row in retired.items()}, list(occurrences))
|
|
1217
|
+
path = root / "00-context" / "evidence-register.md"
|
|
1218
|
+
if not path.exists() or not _within(root, path):
|
|
1219
|
+
return {}, {}, []
|
|
1220
|
+
if cache is None:
|
|
1221
|
+
return _parse_evidence_register(path)
|
|
1222
|
+
try:
|
|
1223
|
+
stat = path.stat()
|
|
1224
|
+
except OSError:
|
|
1225
|
+
return _parse_evidence_register(path)
|
|
1226
|
+
key = (_real(path), stat.st_mtime_ns, stat.st_size)
|
|
1227
|
+
parsed = cache.registers.get(key)
|
|
1228
|
+
if parsed is None:
|
|
1229
|
+
parsed = cache.registers[key] = _parse_evidence_register(path)
|
|
1230
|
+
active, retired, occurrences = parsed
|
|
1231
|
+
# Callers own what they get back; never hand out the cached containers.
|
|
1232
|
+
return (
|
|
1233
|
+
{key: dict(row) for key, row in active.items()},
|
|
1234
|
+
{key: dict(row) for key, row in retired.items()},
|
|
1235
|
+
list(occurrences),
|
|
1236
|
+
)
|
|
1237
|
+
|
|
1238
|
+
|
|
1239
|
+
def _parse_evidence_register(path: Path) -> tuple[dict[str, dict[str, str]], dict[str, dict[str, str]], list[tuple[str, int]]]:
|
|
1240
|
+
return _parse_evidence_register_text(read_vault_text(path)[0])
|
|
1241
|
+
|
|
1242
|
+
|
|
1243
|
+
def _parse_evidence_register_text(text: str) -> tuple[dict[str, dict[str, str]], dict[str, dict[str, str]], list[tuple[str, int]]]:
|
|
1244
|
+
"""Parse register Markdown; also used for an unsaved editor buffer."""
|
|
1245
|
+
from .tables import EVIDENCE_COLUMNS
|
|
1246
|
+
from .markdown_code import mask_code
|
|
1247
|
+
|
|
1248
|
+
active: dict[str, dict[str, str]] = {}
|
|
1249
|
+
retired: dict[str, dict[str, str]] = {}
|
|
1250
|
+
occurrences: list[tuple[str, int]] = []
|
|
1251
|
+
mode = "active"
|
|
1252
|
+
sensitivity_column: int | None = -1
|
|
1253
|
+
lines = text.splitlines()
|
|
1254
|
+
masked = mask_code(text).splitlines()
|
|
1255
|
+
uncommented = mask_code(text, keep_code=True).splitlines()
|
|
1256
|
+
for lineno, (line, visible) in enumerate(zip(lines, masked, strict=True), 1):
|
|
1257
|
+
if visible.strip().lower().startswith("## retired sources"):
|
|
1258
|
+
mode = "retired"
|
|
1259
|
+
sensitivity_column = -1
|
|
1260
|
+
continue
|
|
1261
|
+
cells = _split_table_row(line) if visible.lstrip().startswith("|") else []
|
|
1262
|
+
if cells and cells[0].casefold() == "id":
|
|
1263
|
+
header = tuple(cell.casefold() for cell in cells)
|
|
1264
|
+
expected = EVIDENCE_COLUMNS[mode]
|
|
1265
|
+
sensitivity_column = len(expected) if header == (*expected, "sensitivity") else None if header == expected else -1
|
|
1266
|
+
separator = _split_table_row(lines[lineno]) if lineno < len(lines) else []
|
|
1267
|
+
if (cells != _split_table_row(visible) or len(separator) != len(cells)
|
|
1268
|
+
or not all(re.fullmatch(r":?-{3,}:?", cell) for cell in separator)
|
|
1269
|
+
or separator != _split_table_row(masked[lineno])):
|
|
1270
|
+
sensitivity_column = -1
|
|
1271
|
+
continue
|
|
1272
|
+
if not cells or not EVIDENCE_ID_RE.fullmatch(cells[0]):
|
|
1273
|
+
continue
|
|
1274
|
+
eid = cells[0]
|
|
1275
|
+
if mode == "active" and len(cells) >= 7:
|
|
1276
|
+
# A placeholder ID does not become evidence until source + location exist.
|
|
1277
|
+
if not cells[1] or not cells[5]:
|
|
1278
|
+
continue
|
|
1279
|
+
occurrences.append((eid, lineno))
|
|
1280
|
+
active[eid] = {
|
|
1281
|
+
"source": cells[1], "type": cells[2], "date": cells[3],
|
|
1282
|
+
"accessed": cells[4], "location": cells[5], "claims": cells[6],
|
|
1283
|
+
"line": str(lineno),
|
|
1284
|
+
}
|
|
1285
|
+
elif mode == "retired" and len(cells) >= 5:
|
|
1286
|
+
if not cells[1]:
|
|
1287
|
+
continue
|
|
1288
|
+
occurrences.append((eid, lineno))
|
|
1289
|
+
retired[eid] = {
|
|
1290
|
+
"source": cells[1], "retired_on": cells[2], "why": cells[3],
|
|
1291
|
+
"replaced_by": cells[4], "line": str(lineno),
|
|
1292
|
+
}
|
|
1293
|
+
else:
|
|
1294
|
+
continue
|
|
1295
|
+
row = active[eid] if mode == "active" else retired[eid]
|
|
1296
|
+
expected_length = 7 if mode == "active" else 5
|
|
1297
|
+
if (len(_split_table_row(visible)) != len(cells) or _split_table_row(uncommented[lineno - 1]) != cells):
|
|
1298
|
+
row["sensitivity"] = ""
|
|
1299
|
+
continue
|
|
1300
|
+
if sensitivity_column is not None:
|
|
1301
|
+
row["sensitivity"] = cells[sensitivity_column] if sensitivity_column >= 0 and len(cells) == expected_length + 1 else ""
|
|
1302
|
+
elif len(cells) != expected_length:
|
|
1303
|
+
row["sensitivity"] = ""
|
|
1304
|
+
return active, retired, occurrences
|
|
1305
|
+
|
|
1306
|
+
|
|
1307
|
+
def evidence_rows(root: Path) -> dict[str, dict[str, str]]:
|
|
1308
|
+
return evidence_register(root)[0]
|
|
1309
|
+
|
|
1310
|
+
|
|
1311
|
+
def check_evidence_register(root: Path, findings: list[Finding], today: dt.date, config: dict | None = None) -> None:
|
|
1312
|
+
from .sensitivity import SENSITIVITY_LEVEL
|
|
1313
|
+
|
|
1314
|
+
path = root / "00-context" / "evidence-register.md"
|
|
1315
|
+
if not path.exists():
|
|
1316
|
+
return
|
|
1317
|
+
active, retired, occurrences = evidence_register(root)
|
|
1318
|
+
try:
|
|
1319
|
+
if config is None:
|
|
1320
|
+
config, _ = load_config(root)
|
|
1321
|
+
access_age_policy = config["evidence_access_age_days"]
|
|
1322
|
+
except ConfigError:
|
|
1323
|
+
# check_config reports the malformed policy once, without extra findings.
|
|
1324
|
+
access_age_policy = {}
|
|
1325
|
+
seen: dict[str, int] = {}
|
|
1326
|
+
for eid, line in occurrences:
|
|
1327
|
+
if eid in seen:
|
|
1328
|
+
add(findings, root, path, line, "error", "evidence.duplicate", f"{eid} is already registered on line {seen[eid]}; IDs are never reused")
|
|
1329
|
+
else:
|
|
1330
|
+
seen[eid] = line
|
|
1331
|
+
for eid, row in active.items():
|
|
1332
|
+
if "sensitivity" in row and row["sensitivity"] not in SENSITIVITY_LEVEL:
|
|
1333
|
+
add(findings, root, path, int(row["line"]), "warning", "evidence.sensitivity", f"{eid} has an empty or invalid Sensitivity cell; filtered readers withhold it")
|
|
1334
|
+
for key in ("date", "accessed"):
|
|
1335
|
+
value = row.get(key, "")
|
|
1336
|
+
if value and _parse_date(value) is None:
|
|
1337
|
+
add(findings, root, path, int(row["line"]), "error", "evidence.date", f"{eid} has invalid {key} date `{value}`")
|
|
1338
|
+
max_age = access_age_policy.get(row["type"])
|
|
1339
|
+
if max_age is not None:
|
|
1340
|
+
accessed = row["accessed"]
|
|
1341
|
+
if not accessed:
|
|
1342
|
+
add(findings, root, path, int(row["line"]), "warning", "evidence.access_missing", f"{eid} has no Accessed date for its {row['type']} access-age policy")
|
|
1343
|
+
elif (access_date := _parse_date(accessed)) is not None:
|
|
1344
|
+
if access_date > today:
|
|
1345
|
+
add(findings, root, path, int(row["line"]), "warning", "evidence.access_future", f"{eid} has Accessed date {accessed} after the lint date {today.isoformat()}")
|
|
1346
|
+
elif (today - access_date).days > max_age:
|
|
1347
|
+
add(findings, root, path, int(row["line"]), "warning", "evidence.access_stale", f"{eid} was last accessed {accessed}, more than {max_age} days ago")
|
|
1348
|
+
known = set(active) | set(retired)
|
|
1349
|
+
for eid, row in retired.items():
|
|
1350
|
+
if "sensitivity" in row and row["sensitivity"] not in SENSITIVITY_LEVEL:
|
|
1351
|
+
add(findings, root, path, int(row["line"]), "warning", "evidence.sensitivity", f"{eid} has an empty or invalid Sensitivity cell; filtered readers withhold it")
|
|
1352
|
+
value = row.get("retired_on", "")
|
|
1353
|
+
if value and _parse_date(value) is None:
|
|
1354
|
+
add(findings, root, path, int(row["line"]), "error", "evidence.retired_date", f"{eid} has invalid retired-on date `{value}`")
|
|
1355
|
+
replaced = row.get("replaced_by", "").strip()
|
|
1356
|
+
# `whykit evidence retire` writes an em dash when nothing replaced the source.
|
|
1357
|
+
if replaced in {"\u2014", "-"}:
|
|
1358
|
+
replaced = ""
|
|
1359
|
+
if replaced:
|
|
1360
|
+
if not EVIDENCE_ID_RE.fullmatch(replaced):
|
|
1361
|
+
add(findings, root, path, int(row["line"]), "error", "evidence.replaced_by_format", f"{eid} has invalid replacement ID `{replaced}`")
|
|
1362
|
+
elif replaced not in known:
|
|
1363
|
+
add(findings, root, path, int(row["line"]), "error", "evidence.replaced_by_missing", f"{eid} says it was replaced by {replaced}, but that ID is not registered")
|
|
1364
|
+
|
|
1365
|
+
|
|
1366
|
+
def check_evidence_ids(root: Path, notes: list[Note], findings: list[Finding]) -> None:
|
|
1367
|
+
active, retired, _ = evidence_register(root)
|
|
1368
|
+
register = root / "00-context" / "evidence-register.md"
|
|
1369
|
+
for note in notes:
|
|
1370
|
+
if note.path == register:
|
|
1371
|
+
continue
|
|
1372
|
+
# A superseded or archived record is history: it may not be edited, and
|
|
1373
|
+
# citing what was believed at the time is exactly what it should do.
|
|
1374
|
+
# Warning on it would leave a strict gate red after the normal
|
|
1375
|
+
# supersede-then-retire workflow, with no allowed edit that clears it.
|
|
1376
|
+
historical = str(note.front.get("status") or "").strip() in HISTORICAL_STATUSES
|
|
1377
|
+
for sid in _as_list(note.front.get("source_ids")):
|
|
1378
|
+
if not EVIDENCE_ID_RE.fullmatch(sid):
|
|
1379
|
+
add(findings, root, note.path, 1, "error", "evidence.id_format", f"source_ids contains invalid evidence ID `{sid}`")
|
|
1380
|
+
elif sid in active:
|
|
1381
|
+
continue
|
|
1382
|
+
elif sid in retired:
|
|
1383
|
+
if historical:
|
|
1384
|
+
continue
|
|
1385
|
+
add(findings, root, note.path, 1, "warning", "evidence.retired", f"source_ids cites retired evidence {sid}; preserve historical records but review current claims")
|
|
1386
|
+
else:
|
|
1387
|
+
add(findings, root, note.path, 1, "error", "evidence.missing", f"source_ids cites {sid}, but the register has no populated active or retired row for it")
|
|
1388
|
+
|
|
1389
|
+
def _decision_own_id(note: Note) -> str | None:
|
|
1390
|
+
explicit = str(note.front.get("decision_id", "")).strip()
|
|
1391
|
+
if DECISION_ID_RE.fullmatch(explicit):
|
|
1392
|
+
return explicit
|
|
1393
|
+
for alias in _as_list(note.front.get("aliases")):
|
|
1394
|
+
if DECISION_ID_RE.fullmatch(alias):
|
|
1395
|
+
return alias
|
|
1396
|
+
title = str(note.front.get("title", ""))
|
|
1397
|
+
m = DECISION_ID_RE.search(title)
|
|
1398
|
+
if m:
|
|
1399
|
+
return m.group(0)
|
|
1400
|
+
return note.section_decision_id
|
|
1401
|
+
|
|
1402
|
+
|
|
1403
|
+
def check_decision_ids(root: Path, notes: list[Note], findings: list[Finding]) -> None:
|
|
1404
|
+
seen: dict[str, Path] = {}
|
|
1405
|
+
for note in notes:
|
|
1406
|
+
r = Path(rel(root, note.path))
|
|
1407
|
+
if "06-decisions" not in r.parts or r.name in ("decision-log.md", "README.md"):
|
|
1408
|
+
continue
|
|
1409
|
+
if not DECISION_FILE_RE.match(r.name):
|
|
1410
|
+
continue
|
|
1411
|
+
did = _decision_own_id(note)
|
|
1412
|
+
if did is None:
|
|
1413
|
+
add(findings, root, note.path, 1, "error", "decision.id_missing", "decision record has no own D-NNN identifier; add decision_id to front matter")
|
|
1414
|
+
continue
|
|
1415
|
+
expected = "D-" + DECISION_FILE_RE.match(r.name).group(1) # type: ignore[union-attr]
|
|
1416
|
+
if did != expected:
|
|
1417
|
+
add(findings, root, note.path, 1, "error", "decision.id_filename", f"decision_id {did} does not match filename ({expected})")
|
|
1418
|
+
if did in seen and seen[did] != note.path:
|
|
1419
|
+
add(findings, root, note.path, 1, "error", "decision.duplicate", f"{did} is also used by {rel(root, seen[did])} — IDs are never reused")
|
|
1420
|
+
else:
|
|
1421
|
+
seen[did] = note.path
|
|
1422
|
+
|
|
1423
|
+
|
|
1424
|
+
def decision_log_rows(root: Path, *, text: str | None = None) -> list[dict[str, str | int]]:
|
|
1425
|
+
path = root / "06-decisions" / "decision-log.md"
|
|
1426
|
+
if text is None and not path.exists():
|
|
1427
|
+
return []
|
|
1428
|
+
rows: list[dict[str, str | int]] = []
|
|
1429
|
+
for lineno, line in enumerate(text.splitlines() if text is not None else read_vault_lines(path), 1):
|
|
1430
|
+
if not line.lstrip().startswith("|"):
|
|
1431
|
+
continue
|
|
1432
|
+
cells = _split_table_row(line)
|
|
1433
|
+
if len(cells) < 6 or not DECISION_ID_RE.fullmatch(cells[0]):
|
|
1434
|
+
continue
|
|
1435
|
+
# Empty starter rows are placeholders, not decisions.
|
|
1436
|
+
if not cells[1] or not cells[5]:
|
|
1437
|
+
continue
|
|
1438
|
+
target_match = WIKILINK_RE.search(cells[5])
|
|
1439
|
+
rows.append({
|
|
1440
|
+
"id": cells[0], "title": cells[1], "date": cells[2], "owner": cells[3],
|
|
1441
|
+
"status": cells[4], "record": target_match.group(1).strip() if target_match else "",
|
|
1442
|
+
"line": lineno,
|
|
1443
|
+
})
|
|
1444
|
+
return rows
|
|
1445
|
+
|
|
1446
|
+
|
|
1447
|
+
def check_decision_log(root: Path, notes: list[Note], findings: list[Finding], resolve: Resolver | None = None) -> None:
|
|
1448
|
+
path = root / "06-decisions" / "decision-log.md"
|
|
1449
|
+
if not path.exists():
|
|
1450
|
+
return
|
|
1451
|
+
record_notes: dict[str, Note] = {}
|
|
1452
|
+
for note in notes:
|
|
1453
|
+
r = Path(rel(root, note.path))
|
|
1454
|
+
if "06-decisions" not in r.parts or r.name in ("decision-log.md", "README.md") or not DECISION_FILE_RE.match(r.name):
|
|
1455
|
+
continue
|
|
1456
|
+
did = _decision_own_id(note)
|
|
1457
|
+
if did:
|
|
1458
|
+
record_notes[did] = note
|
|
1459
|
+
|
|
1460
|
+
rows = decision_log_rows(root)
|
|
1461
|
+
log_seen: dict[str, int] = {}
|
|
1462
|
+
row_ids: set[str] = set()
|
|
1463
|
+
status_map = {"draft": "proposed", "in_review": "proposed", "approved": "accepted", "superseded": "superseded", "archived": "archived"}
|
|
1464
|
+
index = _build_index(notes) if resolve is None else {}
|
|
1465
|
+
# First note wins for a resolved path, exactly like the linear scan it replaces.
|
|
1466
|
+
notes_by_real: dict[Path, Note] = {}
|
|
1467
|
+
for note in notes:
|
|
1468
|
+
notes_by_real.setdefault(_real(note.path), note)
|
|
1469
|
+
for row in rows:
|
|
1470
|
+
did = str(row["id"])
|
|
1471
|
+
line = int(row["line"])
|
|
1472
|
+
if did in log_seen:
|
|
1473
|
+
add(findings, root, path, line, "error", "decision_log.duplicate", f"{did} appears more than once in the decision log")
|
|
1474
|
+
log_seen[did] = line
|
|
1475
|
+
row_ids.add(did)
|
|
1476
|
+
target = str(row["record"])
|
|
1477
|
+
if not target:
|
|
1478
|
+
add(findings, root, path, line, "error", "decision_log.record", f"{did} has no wikilinked decision record")
|
|
1479
|
+
continue
|
|
1480
|
+
resolved, ambiguous = _resolve(root, target, index) if resolve is None else resolve(target)
|
|
1481
|
+
if ambiguous:
|
|
1482
|
+
add(findings, root, path, line, "error", "decision_log.ambiguous", f"{did} record link [[{target}]] is ambiguous")
|
|
1483
|
+
continue
|
|
1484
|
+
if resolved is None:
|
|
1485
|
+
add(findings, root, path, line, "error", "decision_log.missing_record", f"{did} points to missing decision record [[{target}]]")
|
|
1486
|
+
continue
|
|
1487
|
+
target_note = notes_by_real.get(_real(resolved))
|
|
1488
|
+
target_id = _decision_own_id(target_note) if target_note else None
|
|
1489
|
+
if target_id != did:
|
|
1490
|
+
add(findings, root, path, line, "error", "decision_log.id_mismatch", f"log row {did} points to a record whose decision_id is {target_id or 'missing'}")
|
|
1491
|
+
continue
|
|
1492
|
+
expected = status_map.get(str(target_note.front.get("status", ""))) if target_note else None
|
|
1493
|
+
if expected and str(row["status"]) != expected:
|
|
1494
|
+
add(findings, root, path, line, "error", "decision_log.status_mismatch", f"{did} log status `{row['status']}` does not match record status `{target_note.front.get('status')}` (expected `{expected}`)")
|
|
1495
|
+
created = str(target_note.front.get("created", "")) if target_note else ""
|
|
1496
|
+
if created and str(row["date"]) and created != str(row["date"]):
|
|
1497
|
+
add(findings, root, path, line, "warning", "decision_log.date_mismatch", f"{did} log date {row['date']} differs from record created date {created}")
|
|
1498
|
+
|
|
1499
|
+
for did, note in record_notes.items():
|
|
1500
|
+
if did not in row_ids:
|
|
1501
|
+
add(findings, root, note.path, 1, "error", "decision_log.unindexed", f"{did} has a record but no populated row in decision-log.md")
|
|
1502
|
+
|
|
1503
|
+
for did, note in record_notes.items():
|
|
1504
|
+
supersedes = str(note.front.get("supersedes", "")).strip()
|
|
1505
|
+
if supersedes:
|
|
1506
|
+
if not DECISION_ID_RE.fullmatch(supersedes):
|
|
1507
|
+
add(findings, root, note.path, 1, "error", "decision.supersedes_format", f"supersedes `{supersedes}` is not a D-NNN ID")
|
|
1508
|
+
elif supersedes not in record_notes:
|
|
1509
|
+
add(findings, root, note.path, 1, "error", "decision.supersedes_missing", f"{did} supersedes {supersedes}, but that decision record does not exist")
|
|
1510
|
+
elif note.front.get("status") == "approved" and record_notes[supersedes].front.get("status") != "superseded":
|
|
1511
|
+
add(findings, root, note.path, 1, "warning", "decision.supersedes_status", f"{did} is approved and supersedes {supersedes}, but the older record is not marked superseded")
|
|
1512
|
+
|
|
1513
|
+
replacements: dict[str, list[str]] = {}
|
|
1514
|
+
historical_replacements: dict[str, list[str]] = {}
|
|
1515
|
+
supersedes_edges: dict[str, str] = {}
|
|
1516
|
+
for replacement_id, replacement_note in record_notes.items():
|
|
1517
|
+
predecessor = str(replacement_note.front.get("supersedes", "")).strip()
|
|
1518
|
+
if DECISION_ID_RE.fullmatch(predecessor) and predecessor in record_notes:
|
|
1519
|
+
supersedes_edges[replacement_id] = predecessor
|
|
1520
|
+
if DECISION_ID_RE.fullmatch(predecessor) and replacement_note.front.get("status") == "approved":
|
|
1521
|
+
replacements.setdefault(predecessor, []).append(replacement_id)
|
|
1522
|
+
elif DECISION_ID_RE.fullmatch(predecessor) and replacement_note.front.get("status") == "superseded":
|
|
1523
|
+
# A successor that was itself later superseded still replaces its
|
|
1524
|
+
# predecessor; the chain's own tail is checked on that record.
|
|
1525
|
+
historical_replacements.setdefault(predecessor, []).append(replacement_id)
|
|
1526
|
+
|
|
1527
|
+
for predecessor, approved_replacements in replacements.items():
|
|
1528
|
+
if len(approved_replacements) > 1:
|
|
1529
|
+
joined = ", ".join(sorted(approved_replacements))
|
|
1530
|
+
add(findings, root, record_notes[predecessor].path, 1, "error", "decision.supersession_fork", f"{predecessor} has multiple approved replacements: {joined}")
|
|
1531
|
+
|
|
1532
|
+
reported_cycles: set[tuple[str, ...]] = set()
|
|
1533
|
+
for start in sorted(supersedes_edges):
|
|
1534
|
+
order: list[str] = []
|
|
1535
|
+
positions: dict[str, int] = {}
|
|
1536
|
+
current = start
|
|
1537
|
+
while current in supersedes_edges:
|
|
1538
|
+
if current in positions:
|
|
1539
|
+
cycle = order[positions[current]:]
|
|
1540
|
+
key = tuple(sorted(cycle))
|
|
1541
|
+
if key not in reported_cycles:
|
|
1542
|
+
reported_cycles.add(key)
|
|
1543
|
+
rendered = " -> ".join([*cycle, cycle[0]])
|
|
1544
|
+
add(findings, root, record_notes[current].path, 1, "error", "decision.supersession_cycle", f"decision supersession contains a cycle: {rendered}")
|
|
1545
|
+
break
|
|
1546
|
+
positions[current] = len(order)
|
|
1547
|
+
order.append(current)
|
|
1548
|
+
current = supersedes_edges[current]
|
|
1549
|
+
|
|
1550
|
+
for did, note in record_notes.items():
|
|
1551
|
+
approved_replacements = sorted(replacements.get(did, []) + historical_replacements.get(did, []))
|
|
1552
|
+
declared = str(note.front.get("superseded_by", "")).strip()
|
|
1553
|
+
if note.front.get("status") == "superseded" and not approved_replacements:
|
|
1554
|
+
add(findings, root, note.path, 1, "warning", "decision.superseded_by_missing", f"{did} is superseded but no approved newer record declares `supersedes: {did}`")
|
|
1555
|
+
if declared:
|
|
1556
|
+
if not DECISION_ID_RE.fullmatch(declared):
|
|
1557
|
+
add(findings, root, note.path, 1, "warning", "decision.superseded_by_format", f"superseded_by `{declared}` is not a D-NNN ID")
|
|
1558
|
+
elif declared not in approved_replacements:
|
|
1559
|
+
expected = ", ".join(approved_replacements) if approved_replacements else "no approved replacement"
|
|
1560
|
+
add(findings, root, note.path, 1, "warning", "decision.superseded_by_mismatch", f"{did} declares superseded_by {declared}, but the reverse supersedes edge resolves to {expected}")
|
|
1561
|
+
|
|
1562
|
+
|
|
1563
|
+
def check_fact_evidence(
|
|
1564
|
+
root: Path,
|
|
1565
|
+
note: Note,
|
|
1566
|
+
findings: list[Finding],
|
|
1567
|
+
known_evidence: set[str] | None = None,
|
|
1568
|
+
) -> None:
|
|
1569
|
+
"""Fact callouts must cite evidence, and the evidence they cite must exist.
|
|
1570
|
+
|
|
1571
|
+
`known_evidence` is every active or retired register ID; None skips the
|
|
1572
|
+
existence check (callers without a register view).
|
|
1573
|
+
"""
|
|
1574
|
+
for line, cited in note.fact_callouts:
|
|
1575
|
+
if not cited:
|
|
1576
|
+
add(findings, root, note.path, line, "warning", "fact.inline_evidence", "verified fact callout has no inline E-NNN citation")
|
|
1577
|
+
elif known_evidence is not None:
|
|
1578
|
+
for eid in cited:
|
|
1579
|
+
if eid not in known_evidence:
|
|
1580
|
+
add(
|
|
1581
|
+
findings, root, note.path, line, "warning", "fact.evidence_missing",
|
|
1582
|
+
f"fact callout cites {eid}, but the evidence register has no populated row for it",
|
|
1583
|
+
)
|
|
1584
|
+
|
|
1585
|
+
|
|
1586
|
+
AGENTS_TODO_RE = re.compile(r"(?m)^\s*(?:[-*]\s*|\d+[.)]\s*|#{1,6}\s*)?TODO\b[ :\u2014-]")
|
|
1587
|
+
|
|
1588
|
+
|
|
1589
|
+
def check_path_collisions(root: Path, notes: list[Note], findings: list[Finding]) -> None:
|
|
1590
|
+
"""Paths that only a case-sensitive, normalization-sensitive filesystem keeps apart.
|
|
1591
|
+
|
|
1592
|
+
A Linux checkout can hold `notes/Plan.md` and `notes/plan.md` (or an NFC and
|
|
1593
|
+
an NFD spelling of `café.md`); Git on macOS and Windows checks out only one.
|
|
1594
|
+
"""
|
|
1595
|
+
groups: dict[str, list[Path]] = {}
|
|
1596
|
+
for note in notes:
|
|
1597
|
+
key = _relative_key(root, note.path)
|
|
1598
|
+
if key is not None:
|
|
1599
|
+
groups.setdefault(key, []).append(note.path)
|
|
1600
|
+
for paths in groups.values():
|
|
1601
|
+
if len(paths) < 2:
|
|
1602
|
+
continue
|
|
1603
|
+
ordered = sorted(paths, key=lambda path: path.relative_to(root).as_posix())
|
|
1604
|
+
first = ordered[0].relative_to(root).as_posix()
|
|
1605
|
+
for path in ordered[1:]:
|
|
1606
|
+
add(findings, root, path, None, "warning", "path.case_collision",
|
|
1607
|
+
f"path differs from {first} only by case or Unicode normalization; macOS and Windows checkouts keep one of them")
|
|
1608
|
+
|
|
1609
|
+
|
|
1610
|
+
def check_agents_configured(root: Path, findings: list[Finding]) -> None:
|
|
1611
|
+
"""The agent contract ships with blanks on purpose. Unfilled, it is worse than absent.
|
|
1612
|
+
|
|
1613
|
+
A vault that never answers "which language", "commit or PR", "what may an agent
|
|
1614
|
+
touch" hands those decisions back to the agent, which will invent an answer.
|
|
1615
|
+
"""
|
|
1616
|
+
path = root / "AGENTS.md"
|
|
1617
|
+
if not path.exists():
|
|
1618
|
+
add(findings, root, root / "AGENTS.md", None, "warning", "agents.absent",
|
|
1619
|
+
"no AGENTS.md — agents working here have no written contract")
|
|
1620
|
+
return
|
|
1621
|
+
text = read_vault_text(path)[0]
|
|
1622
|
+
numbers = _LineNumbers(text)
|
|
1623
|
+
for match in AGENTS_TODO_RE.finditer(_mask_code(text)):
|
|
1624
|
+
line = numbers.at(match.start())
|
|
1625
|
+
add(findings, root, path, line, "warning", "agents.unconfigured",
|
|
1626
|
+
"AGENTS.md still has an unanswered TODO — adopters must fill these before trusting the contract")
|
|
1627
|
+
|
|
1628
|
+
|
|
1629
|
+
def check_decision_review(root: Path, notes: list[Note], findings: list[Finding]) -> None:
|
|
1630
|
+
"""An accepted decision with no review date is how a vault quietly goes stale.
|
|
1631
|
+
|
|
1632
|
+
The record stays valid-looking forever while the world moves; nobody is ever
|
|
1633
|
+
prompted to ask whether it still holds. Shape cannot verify truth, but it can
|
|
1634
|
+
insist that somebody named a date on which truth gets re-checked.
|
|
1635
|
+
"""
|
|
1636
|
+
for note in notes:
|
|
1637
|
+
r = Path(rel(root, note.path))
|
|
1638
|
+
if "06-decisions" not in r.parts or not DECISION_FILE_RE.match(r.name):
|
|
1639
|
+
continue
|
|
1640
|
+
if note.front.get("status") != "approved":
|
|
1641
|
+
continue
|
|
1642
|
+
provenance = note.front.get("provenance")
|
|
1643
|
+
if isinstance(provenance, dict) and "human_reviewed" in provenance and provenance["human_reviewed"] is not True:
|
|
1644
|
+
add(findings, root, note.path, 1, "warning", "decision.unreviewed",
|
|
1645
|
+
"approved decision declares it was not human-reviewed — keep it in review until an authorized person checks the record")
|
|
1646
|
+
review_by = str(note.front.get("review_by", "")).strip()
|
|
1647
|
+
if not review_by or review_by == PLACEHOLDER_DATE:
|
|
1648
|
+
add(findings, root, note.path, 1, "warning", "decision.review_missing",
|
|
1649
|
+
"approved decision has no `review_by` date — nothing will ever prompt a re-check")
|
|
1650
|
+
|
|
1651
|
+
|
|
1652
|
+
# Statuses whose decision records must not keep template prompts. Superseded
|
|
1653
|
+
# and archived records are history and may not be edited, so they are skipped;
|
|
1654
|
+
# templates are meant to hold prompts. Every level is a warning for now (new
|
|
1655
|
+
# rules land as warnings); `--strict` and the `ci` profile still fail on it.
|
|
1656
|
+
PLACEHOLDER_LEVELS = {"approved": "warning", "in_review": "warning", "draft": "warning"}
|
|
1657
|
+
_BARE_LINES = frozenset({"", "-", "*", "+"})
|
|
1658
|
+
|
|
1659
|
+
|
|
1660
|
+
def _decision_placeholders(note: Note) -> list[tuple[int, str]]:
|
|
1661
|
+
"""``(line, section)`` for each section still holding template placeholders.
|
|
1662
|
+
|
|
1663
|
+
Headings and placeholder prose are read from the code-masked text, so a
|
|
1664
|
+
record that quotes the template inside a code block is not flagged. Whether
|
|
1665
|
+
a section is empty is judged on the raw text, so a section holding only a
|
|
1666
|
+
code block still counts as written.
|
|
1667
|
+
"""
|
|
1668
|
+
masked = note.masked.split("\n")
|
|
1669
|
+
raw = note.text.split("\n")
|
|
1670
|
+
prompts = template_section_prompts()
|
|
1671
|
+
found: list[tuple[int, str]] = []
|
|
1672
|
+
for heading, start, end in split_placeholder_sections(masked):
|
|
1673
|
+
body = masked[start:end]
|
|
1674
|
+
known = prompts.get(heading, frozenset())
|
|
1675
|
+
if heading in PLACEHOLDER_PROSE_SECTIONS:
|
|
1676
|
+
# A prompt left on its own line above real prose is still a leftover.
|
|
1677
|
+
hits = [start + index + 1 for index, text in placeholder_paragraphs(body) if text in known]
|
|
1678
|
+
hits += [start + index + 1 for index, line in enumerate(body) if normalize_placeholder(line) in known]
|
|
1679
|
+
if hits:
|
|
1680
|
+
found.append((min(hits), heading))
|
|
1681
|
+
elif all(line.strip() in _BARE_LINES for line in raw[start:end]):
|
|
1682
|
+
found.append((start, heading))
|
|
1683
|
+
elif heading == "Evidence":
|
|
1684
|
+
hits = [start + index + 1 for index, line in enumerate(body) if line.strip() == SCAFFOLD_EVIDENCE_TODO]
|
|
1685
|
+
if hits:
|
|
1686
|
+
found.append((hits[0], heading))
|
|
1687
|
+
elif heading == "Alternatives considered":
|
|
1688
|
+
rows = [line.strip() for line in body if line.strip().startswith("|")]
|
|
1689
|
+
filled = [row for row in rows[2:] if any(cell.strip() for cell in row.strip("|").split("|"))]
|
|
1690
|
+
prose = [text for _, text in placeholder_paragraphs(body) if text not in known]
|
|
1691
|
+
if rows and not filled and not prose:
|
|
1692
|
+
found.append((start, heading))
|
|
1693
|
+
elif heading == "Consequences":
|
|
1694
|
+
if all(line.strip() in _BARE_LINES or line.lstrip().startswith("#") for line in raw[start:end]):
|
|
1695
|
+
found.append((start, heading))
|
|
1696
|
+
return found
|
|
1697
|
+
|
|
1698
|
+
|
|
1699
|
+
def check_decision_placeholders(root: Path, notes: list[Note], findings: list[Finding]) -> None:
|
|
1700
|
+
"""Template prose left in a decision record that is meant to be read.
|
|
1701
|
+
|
|
1702
|
+
Shape alone cannot tell whether a rationale is good, but it can tell that
|
|
1703
|
+
the record still says "State the choice in one sentence." The prompts are
|
|
1704
|
+
the ones WhyKit itself writes (see ``whykit.placeholders``), so this rule
|
|
1705
|
+
follows any change to the scaffold or the shipped template.
|
|
1706
|
+
"""
|
|
1707
|
+
for note in notes:
|
|
1708
|
+
r = Path(rel(root, note.path))
|
|
1709
|
+
if "06-decisions" not in r.parts or not DECISION_FILE_RE.match(r.name):
|
|
1710
|
+
continue
|
|
1711
|
+
status = str(note.front.get("status") or "").strip()
|
|
1712
|
+
level = PLACEHOLDER_LEVELS.get(status)
|
|
1713
|
+
if level is None:
|
|
1714
|
+
continue
|
|
1715
|
+
leftovers = note.decision_placeholders
|
|
1716
|
+
if not leftovers:
|
|
1717
|
+
continue
|
|
1718
|
+
sections = ", ".join(f"`## {heading}`" for _, heading in leftovers)
|
|
1719
|
+
add(
|
|
1720
|
+
findings, root, note.path, leftovers[0][0], level, "decision.placeholder",
|
|
1721
|
+
f"{status} decision still has template text or nothing in {sections} — write the real content",
|
|
1722
|
+
)
|
|
1723
|
+
|
|
1724
|
+
|
|
1725
|
+
REVIEW_OUTCOMES = {"confirmed", "update-required", "supersede-required", "archived", "approved"}
|
|
1726
|
+
|
|
1727
|
+
def check_config(root: Path, findings: list[Finding]) -> None:
|
|
1728
|
+
path = root / CONFIG_FILE
|
|
1729
|
+
if not path.exists():
|
|
1730
|
+
return
|
|
1731
|
+
try:
|
|
1732
|
+
load_config(root)
|
|
1733
|
+
except Exception as exc: # noqa: BLE001
|
|
1734
|
+
add(findings, root, path, 1, "error", "config.invalid", str(exc))
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
def check_review_log(root: Path, notes: list[Note], findings: list[Finding], resolve: Resolver | None = None) -> None:
|
|
1738
|
+
path = root / "00-context" / "review-log.md"
|
|
1739
|
+
if not path.exists():
|
|
1740
|
+
return
|
|
1741
|
+
real = _real(path)
|
|
1742
|
+
note = next((item for item in notes if _real(item.path) == real), None) or load_note(path)
|
|
1743
|
+
lines = note.text.splitlines()
|
|
1744
|
+
header_idx = review_table_header(lines)
|
|
1745
|
+
if header_idx is None or header_idx + 1 >= len(lines):
|
|
1746
|
+
add(findings, root, path, None, "error", "review_log.table", "review log table is missing or malformed")
|
|
1747
|
+
return
|
|
1748
|
+
if resolve is None:
|
|
1749
|
+
resolve = functools.partial(_resolve, root, index=_build_index(notes))
|
|
1750
|
+
i = header_idx + 2
|
|
1751
|
+
while i < len(lines) and lines[i].lstrip().startswith("|"):
|
|
1752
|
+
cells = _split_table_row(lines[i])
|
|
1753
|
+
line_no = i + 1
|
|
1754
|
+
if len(cells) < 7:
|
|
1755
|
+
add(findings, root, path, line_no, "error", "review_log.row", "review log row must have seven columns")
|
|
1756
|
+
i += 1
|
|
1757
|
+
continue
|
|
1758
|
+
date, target_cell, reviewer, outcome, _previous, next_review, _note = cells[:7]
|
|
1759
|
+
if _parse_date(date) is None:
|
|
1760
|
+
add(findings, root, path, line_no, "error", "review_log.date", f"invalid review date `{date}`")
|
|
1761
|
+
if not reviewer.strip() or reviewer.strip() == PLACEHOLDER_OWNER:
|
|
1762
|
+
add(findings, root, path, line_no, "error", "review_log.reviewer", "review event must name a real reviewer")
|
|
1763
|
+
if outcome not in REVIEW_OUTCOMES:
|
|
1764
|
+
add(findings, root, path, line_no, "error", "review_log.outcome", f"unsupported review outcome `{outcome}`")
|
|
1765
|
+
if next_review not in {"", "—", "-"} and _parse_date(next_review) is None:
|
|
1766
|
+
add(findings, root, path, line_no, "error", "review_log.next_review", f"invalid next-review date `{next_review}`")
|
|
1767
|
+
match = WIKILINK_RE.search(target_cell)
|
|
1768
|
+
if not match:
|
|
1769
|
+
add(findings, root, path, line_no, "error", "review_log.target", "review event target must be a wikilink")
|
|
1770
|
+
else:
|
|
1771
|
+
resolved, ambiguous = resolve(match.group(1).strip())
|
|
1772
|
+
if resolved is None or ambiguous:
|
|
1773
|
+
add(findings, root, path, line_no, "error", "review_log.target", "review event target is missing or ambiguous")
|
|
1774
|
+
i += 1
|
|
1775
|
+
|
|
1776
|
+
def check_orphans(root: Path, notes: list[Note], findings: list[Finding], resolve: Resolver | None = None) -> None:
|
|
1777
|
+
exempt_names = {"README.md", "AGENTS.md", "OBSIDIAN.md", "INTEROP.md", "Home.md", "SECURITY.md", "CONTRIBUTING.md", "CHANGELOG.md"}
|
|
1778
|
+
linked: set[Path] = set()
|
|
1779
|
+
if resolve is None:
|
|
1780
|
+
resolve = functools.partial(_resolve, root, index=_build_index(notes))
|
|
1781
|
+
for note in notes:
|
|
1782
|
+
for target, _line, _embed in note.wikilink_hits:
|
|
1783
|
+
resolved, ambiguous = resolve(target)
|
|
1784
|
+
if resolved and not ambiguous:
|
|
1785
|
+
linked.add(_real(resolved))
|
|
1786
|
+
for note in notes:
|
|
1787
|
+
r = Path(rel(root, note.path))
|
|
1788
|
+
if r.name in exempt_names or "templates" in r.parts:
|
|
1789
|
+
continue
|
|
1790
|
+
if _real(note.path) not in linked:
|
|
1791
|
+
add(findings, root, note.path, None, "warning", "note.orphan", "nothing links to this note — add it to Home.md or a workstream map")
|
|
1792
|
+
|
|
1793
|
+
|
|
1794
|
+
def check_hub_links(root: Path, notes: list[Note], findings: list[Finding]) -> None:
|
|
1795
|
+
"""Require Home.md to wikilink into each top-level working directory."""
|
|
1796
|
+
home = next((note for note in notes if rel(root, note.path) == "Home.md"), None)
|
|
1797
|
+
if home is None:
|
|
1798
|
+
return
|
|
1799
|
+
linked_prefixes: set[str] = set()
|
|
1800
|
+
for match in WIKILINK_RE.finditer(_mask_code(home.text)):
|
|
1801
|
+
target = match.group(1).strip().replace("\\", "/")
|
|
1802
|
+
if not target:
|
|
1803
|
+
continue
|
|
1804
|
+
linked_prefixes.add(target.split("/", 1)[0])
|
|
1805
|
+
linked_prefixes.add(target)
|
|
1806
|
+
skip = {
|
|
1807
|
+
"assets", "templates", "reports", "schemas", "tests", "apps", "examples",
|
|
1808
|
+
".git", ".obsidian", ".import-staging", ".whykit", "node_modules",
|
|
1809
|
+
}
|
|
1810
|
+
for child in sorted(root.iterdir()):
|
|
1811
|
+
if not child.is_dir() or child.name in skip or child.name.startswith("."):
|
|
1812
|
+
continue
|
|
1813
|
+
# Foundations and decisions are linked by path in the default Home maps.
|
|
1814
|
+
marker = child.name
|
|
1815
|
+
hit = any(
|
|
1816
|
+
prefix == marker or prefix.startswith(marker + "/")
|
|
1817
|
+
for prefix in linked_prefixes
|
|
1818
|
+
)
|
|
1819
|
+
if not hit:
|
|
1820
|
+
add(
|
|
1821
|
+
findings,
|
|
1822
|
+
root,
|
|
1823
|
+
home.path,
|
|
1824
|
+
None,
|
|
1825
|
+
"warning",
|
|
1826
|
+
"hub.unlinked_workstream",
|
|
1827
|
+
f"Home.md does not link into `{marker}/` — add a map link or remove the empty folder",
|
|
1828
|
+
)
|
|
1829
|
+
|
|
1830
|
+
|
|
1831
|
+
def _text_files_for_secret_scan(root: Path) -> Iterable[tuple[Path, int]]:
|
|
1832
|
+
"""Yield ``(path, size)`` for every regular text file the secret scan covers.
|
|
1833
|
+
|
|
1834
|
+
The name filters run before the ``stat``: they are pure string tests, and
|
|
1835
|
+
most files in a large vault pass them, but no file is stat-ed twice.
|
|
1836
|
+
"""
|
|
1837
|
+
depth = len(root.parts)
|
|
1838
|
+
for p in root.rglob("*"):
|
|
1839
|
+
if any(part in SECRET_SKIP_DIRS for part in p.parts[depth:]):
|
|
1840
|
+
continue
|
|
1841
|
+
if not (p.name.startswith(".env") or p.suffix.lower() in TEXT_SECRET_EXTENSIONS):
|
|
1842
|
+
continue
|
|
1843
|
+
try:
|
|
1844
|
+
info = p.stat() # follows symlinks, like Path.is_file()
|
|
1845
|
+
except (OSError, ValueError):
|
|
1846
|
+
continue
|
|
1847
|
+
if stat.S_ISREG(info.st_mode) and _within(root, p):
|
|
1848
|
+
yield p, info.st_size
|
|
1849
|
+
|
|
1850
|
+
|
|
1851
|
+
def check_secrets(
|
|
1852
|
+
root: Path,
|
|
1853
|
+
findings: list[Finding],
|
|
1854
|
+
loaded: dict[Path, str] | None = None,
|
|
1855
|
+
*,
|
|
1856
|
+
notes: dict[Path, Note] | None = None,
|
|
1857
|
+
) -> None:
|
|
1858
|
+
"""Scan every text file under *root* for credential-shaped strings.
|
|
1859
|
+
|
|
1860
|
+
*loaded* maps a note's path to the text the vault index already read, so a
|
|
1861
|
+
Markdown note is not read from disk a second time. The index stripped a
|
|
1862
|
+
leading byte-order mark; no pattern can match one or depends on it, so the
|
|
1863
|
+
findings (including line numbers) are the same as for the raw file.
|
|
1864
|
+
*notes* maps a path to its parsed note instead, whose ``secret_hits`` are
|
|
1865
|
+
the same matches computed from the same text.
|
|
1866
|
+
"""
|
|
1867
|
+
max_bytes = 5_000_000
|
|
1868
|
+
for path, size in _text_files_for_secret_scan(root):
|
|
1869
|
+
try:
|
|
1870
|
+
if size > max_bytes:
|
|
1871
|
+
add(
|
|
1872
|
+
findings,
|
|
1873
|
+
root,
|
|
1874
|
+
path,
|
|
1875
|
+
None,
|
|
1876
|
+
"error",
|
|
1877
|
+
"secret.scan_skipped_large_file",
|
|
1878
|
+
(
|
|
1879
|
+
f"secret scan skipped {size} byte file; "
|
|
1880
|
+
"large files must be scanned externally or excluded explicitly"
|
|
1881
|
+
),
|
|
1882
|
+
)
|
|
1883
|
+
continue
|
|
1884
|
+
note = notes.get(path) if notes else None
|
|
1885
|
+
if note is not None:
|
|
1886
|
+
for line, label in note.secret_hits:
|
|
1887
|
+
add(findings, root, path, line, "error", "secret.detected", f"looks like a {label} — keep locations, never secret values")
|
|
1888
|
+
continue
|
|
1889
|
+
text = loaded.get(path) if loaded else None
|
|
1890
|
+
if text is None:
|
|
1891
|
+
text = path.read_text(encoding="utf-8")
|
|
1892
|
+
except UnicodeDecodeError:
|
|
1893
|
+
add(
|
|
1894
|
+
findings,
|
|
1895
|
+
root,
|
|
1896
|
+
path,
|
|
1897
|
+
None,
|
|
1898
|
+
"error",
|
|
1899
|
+
"secret.scan_non_utf8",
|
|
1900
|
+
"secret scan cannot verify this non-UTF-8 file",
|
|
1901
|
+
)
|
|
1902
|
+
continue
|
|
1903
|
+
except OSError as exc:
|
|
1904
|
+
add(
|
|
1905
|
+
findings,
|
|
1906
|
+
root,
|
|
1907
|
+
path,
|
|
1908
|
+
None,
|
|
1909
|
+
"error",
|
|
1910
|
+
"secret.scan_unreadable",
|
|
1911
|
+
f"secret scan could not read file: {exc}",
|
|
1912
|
+
)
|
|
1913
|
+
continue
|
|
1914
|
+
for (label, pattern), prefilter in zip(SECRET_PATTERNS, SECRET_PREFILTERS, strict=True):
|
|
1915
|
+
if not prefilter(text):
|
|
1916
|
+
continue
|
|
1917
|
+
numbers = _LineNumbers(text)
|
|
1918
|
+
for match in pattern.finditer(text):
|
|
1919
|
+
line = numbers.at(match.start())
|
|
1920
|
+
add(findings, root, path, line, "error", "secret.detected", f"looks like a {label} — keep locations, never secret values")
|
|
1921
|
+
|
|
1922
|
+
|
|
1923
|
+
def collect_markdown(root: Path, paths: list[str]) -> list[Path]:
|
|
1924
|
+
root = root.resolve()
|
|
1925
|
+
if paths:
|
|
1926
|
+
out: list[Path] = []
|
|
1927
|
+
for raw in paths:
|
|
1928
|
+
p = Path(raw).expanduser()
|
|
1929
|
+
if not p.is_absolute():
|
|
1930
|
+
p = root / p
|
|
1931
|
+
p = p.resolve()
|
|
1932
|
+
if not _within(root, p):
|
|
1933
|
+
raise VaultPathError(f"path escapes vault root: {raw}")
|
|
1934
|
+
if p.is_dir():
|
|
1935
|
+
out.extend(sorted(
|
|
1936
|
+
child for child in iter_markdown(p)
|
|
1937
|
+
if child.is_file() and _within(root, child)
|
|
1938
|
+
))
|
|
1939
|
+
elif is_markdown_name(p.name) and p.is_file():
|
|
1940
|
+
out.append(p)
|
|
1941
|
+
elif not p.exists():
|
|
1942
|
+
# A typo in a CI path list must not turn into "0 files — clean".
|
|
1943
|
+
raise VaultPathError(f"path not found: {raw}")
|
|
1944
|
+
else:
|
|
1945
|
+
raise VaultPathError(f"not a Markdown file or directory: {raw}")
|
|
1946
|
+
return out
|
|
1947
|
+
depth = len(root.parts)
|
|
1948
|
+
return sorted(
|
|
1949
|
+
(
|
|
1950
|
+
p for p in iter_markdown(root)
|
|
1951
|
+
if p.is_file()
|
|
1952
|
+
and _within(root, p)
|
|
1953
|
+
and not any(part in CONTENT_SKIP_DIRS for part in p.parts[depth:])
|
|
1954
|
+
),
|
|
1955
|
+
key=_path_order,
|
|
1956
|
+
)
|
|
1957
|
+
|
|
1958
|
+
|
|
1959
|
+
def _path_order(path: Path) -> list[str]:
|
|
1960
|
+
"""Sort key giving the same order as comparing the paths themselves.
|
|
1961
|
+
|
|
1962
|
+
``PurePath.__lt__`` compares the case-normalised string split on the
|
|
1963
|
+
separator, and caches that list on each path for the life of the object.
|
|
1964
|
+
The paths of a vault index live as long as the index, so sorting them
|
|
1965
|
+
directly would keep a second copy of every path's parts.
|
|
1966
|
+
"""
|
|
1967
|
+
return os.path.normcase(os.fspath(path)).split(os.sep)
|
|
1968
|
+
|
|
1969
|
+
|
|
1970
|
+
@consistent_read
|
|
1971
|
+
@path_cache()
|
|
1972
|
+
def lint(
|
|
1973
|
+
root: Path,
|
|
1974
|
+
paths: list[str] | None = None,
|
|
1975
|
+
*,
|
|
1976
|
+
orphans: bool = True,
|
|
1977
|
+
secrets: bool = True,
|
|
1978
|
+
hub_links: bool | None = None,
|
|
1979
|
+
today: dt.date | None = None,
|
|
1980
|
+
vault: object | None = None,
|
|
1981
|
+
overrides: list[Override] | None = None,
|
|
1982
|
+
config: dict | None = None,
|
|
1983
|
+
) -> tuple[list[Path], list[Finding]]:
|
|
1984
|
+
"""Lint *root* (or the scoped *paths* inside it).
|
|
1985
|
+
|
|
1986
|
+
The vault's team policy from ``whykit.toml`` is part of the result: its
|
|
1987
|
+
custom rules run with the built-in ones and its overrides are applied last.
|
|
1988
|
+
Pass a list as *overrides* to receive the override entries in effect, each
|
|
1989
|
+
counting what it changed and holding the findings it suppressed.
|
|
1990
|
+
"""
|
|
1991
|
+
root = root.resolve()
|
|
1992
|
+
from .vault_index import VaultIndex
|
|
1993
|
+
|
|
1994
|
+
index_model = vault if isinstance(vault, VaultIndex) else VaultIndex.load(root)
|
|
1995
|
+
all_notes = index_model.notes
|
|
1996
|
+
all_files = [note.path for note in all_notes]
|
|
1997
|
+
if paths:
|
|
1998
|
+
files = collect_markdown(root, paths)
|
|
1999
|
+
selected = {_real(path) for path in files}
|
|
2000
|
+
notes = [note for note in all_notes if _real(note.path) in selected]
|
|
2001
|
+
else:
|
|
2002
|
+
files = all_files
|
|
2003
|
+
notes = all_notes
|
|
2004
|
+
index = index_model.link_index
|
|
2005
|
+
findings: list[Finding] = []
|
|
2006
|
+
today = today or dt.date.today()
|
|
2007
|
+
attachments = AttachmentIndex(root)
|
|
2008
|
+
active_evidence, retired_evidence, _ = evidence_register(root)
|
|
2009
|
+
known_evidence = set(active_evidence) | set(retired_evidence)
|
|
2010
|
+
link_exists: Callable[[Path], bool] | None = None
|
|
2011
|
+
if getattr(index_model, "confined", False):
|
|
2012
|
+
def _confined_exists(candidate: Path) -> bool:
|
|
2013
|
+
# A confined index hides some notes; a link to one must look the
|
|
2014
|
+
# same as a link to a file that does not exist.
|
|
2015
|
+
if candidate.suffix.lower() == ".md":
|
|
2016
|
+
return index_model.note_for(candidate) is not None
|
|
2017
|
+
return path_exists(candidate)
|
|
2018
|
+
|
|
2019
|
+
link_exists = _confined_exists
|
|
2020
|
+
|
|
2021
|
+
for note in notes:
|
|
2022
|
+
check_front_matter(root, note, findings, today)
|
|
2023
|
+
# The index memoises link resolution: a vault repeats the same targets
|
|
2024
|
+
# many times, and each uncached lookup costs several filesystem calls.
|
|
2025
|
+
check_wikilinks(root, note, index, findings, attachments, index_model.resolve_link)
|
|
2026
|
+
check_markdown_links(root, note, findings, link_exists)
|
|
2027
|
+
check_fact_evidence(root, note, findings, known_evidence)
|
|
2028
|
+
check_evidence_register(root, findings, today, config)
|
|
2029
|
+
check_evidence_ids(root, notes, findings)
|
|
2030
|
+
check_decision_ids(root, notes, findings)
|
|
2031
|
+
check_decision_log(root, all_notes, findings, index_model.resolve_link)
|
|
2032
|
+
check_decision_review(root, notes, findings)
|
|
2033
|
+
check_decision_placeholders(root, notes, findings)
|
|
2034
|
+
if config is None:
|
|
2035
|
+
try:
|
|
2036
|
+
config = load_config(root)[0]
|
|
2037
|
+
except Exception: # noqa: BLE001 — reported as config.invalid by check_config
|
|
2038
|
+
config = None
|
|
2039
|
+
policy = policy_from_config(config) if config is not None else EMPTY_POLICY
|
|
2040
|
+
if policy.custom:
|
|
2041
|
+
check_custom_rules(policy, notes, findings, today, lambda path: rel(root, path))
|
|
2042
|
+
if not paths:
|
|
2043
|
+
check_config(root, findings)
|
|
2044
|
+
check_path_collisions(root, all_notes, findings)
|
|
2045
|
+
check_review_log(root, all_notes, findings, index_model.resolve_link)
|
|
2046
|
+
check_agents_configured(root, findings)
|
|
2047
|
+
if orphans and not paths:
|
|
2048
|
+
check_orphans(root, notes, findings, index_model.resolve_link)
|
|
2049
|
+
if hub_links is None and not paths:
|
|
2050
|
+
hub_links = bool(config.get("defaults", {}).get("require_hub_links")) if config is not None else False
|
|
2051
|
+
if hub_links and not paths:
|
|
2052
|
+
check_hub_links(root, all_notes, findings)
|
|
2053
|
+
if secrets and not paths:
|
|
2054
|
+
check_secrets(root, findings, notes={note.path: note for note in all_notes})
|
|
2055
|
+
findings = policy.apply_overrides(findings)
|
|
2056
|
+
if overrides is not None:
|
|
2057
|
+
overrides.extend(policy.overrides)
|
|
2058
|
+
return files, findings
|
|
2059
|
+
|
|
2060
|
+
|
|
2061
|
+
def resolve_root(explicit: str | None, paths: list[str], cwd: Path | None = None) -> tuple[Path, list[str], str | None]:
|
|
2062
|
+
"""Work out which vault is being linted, and say so when it was not obvious.
|
|
2063
|
+
|
|
2064
|
+
Three cases, in order:
|
|
2065
|
+
1. --root wins, always.
|
|
2066
|
+
2. A single positional argument that is itself a vault root becomes the root.
|
|
2067
|
+
Without this, `whykit lint examples/northline` lints another vault's files
|
|
2068
|
+
against *this* directory's evidence register and reports every wikilink as
|
|
2069
|
+
broken — a first run that looks like the tool is broken.
|
|
2070
|
+
3. Otherwise walk up from the working directory.
|
|
2071
|
+
"""
|
|
2072
|
+
cwd = (cwd or Path.cwd()).resolve()
|
|
2073
|
+
if explicit:
|
|
2074
|
+
return Path(explicit).expanduser().resolve(), paths, None
|
|
2075
|
+
|
|
2076
|
+
if len(paths) == 1:
|
|
2077
|
+
candidate = (cwd / paths[0]).resolve()
|
|
2078
|
+
if candidate.is_dir() and is_vault_root(candidate):
|
|
2079
|
+
return candidate, [], f"using {paths[0]} as the vault root"
|
|
2080
|
+
|
|
2081
|
+
found = find_vault_root(cwd)
|
|
2082
|
+
if found:
|
|
2083
|
+
note = None if found == cwd else f"vault root: {found}"
|
|
2084
|
+
return found, paths, note
|
|
2085
|
+
return cwd, paths, None
|
|
2086
|
+
|
|
2087
|
+
|
|
2088
|
+
LINT_FORMATS = ("text", "json", "sarif", "github")
|
|
2089
|
+
|
|
2090
|
+
|
|
2091
|
+
def main(argv: list[str] | None = None) -> int:
|
|
2092
|
+
parser = argparse.ArgumentParser(prog="whykit lint", description="Check a WhyKit vault.")
|
|
2093
|
+
parser.add_argument("paths", nargs="*", help="Markdown files/directories to check")
|
|
2094
|
+
parser.add_argument("--root", help="vault root (default: nearest vault at or above the working directory)")
|
|
2095
|
+
parser.add_argument("--strict", action="store_true", help="warnings count as failures")
|
|
2096
|
+
parser.add_argument("--quiet", action="store_true", help="only print summary")
|
|
2097
|
+
parser.add_argument("--json", action="store_true", help="emit machine-readable JSON")
|
|
2098
|
+
parser.add_argument("--no-orphans", action="store_true", help="skip orphan-note warnings")
|
|
2099
|
+
parser.add_argument("--no-secrets", action="store_true", help="skip secret scan")
|
|
2100
|
+
parser.add_argument("--today", help="evaluate review dates as of this ISO date instead of today")
|
|
2101
|
+
parser.add_argument("--format", choices=LINT_FORMATS, default=None, help="output format (default: text)")
|
|
2102
|
+
args = parser.parse_args(argv)
|
|
2103
|
+
from .contract import emit_error, vault_not_found
|
|
2104
|
+
|
|
2105
|
+
if args.json and args.format not in (None, "json"):
|
|
2106
|
+
return emit_error(
|
|
2107
|
+
"usage",
|
|
2108
|
+
f"--json conflicts with --format {args.format}\nhint: pass one of them to `whykit lint`",
|
|
2109
|
+
json_mode=True,
|
|
2110
|
+
)
|
|
2111
|
+
fmt = "json" if args.json else (args.format or "text")
|
|
2112
|
+
# Only the JSON format has an error object; a SARIF or annotation consumer
|
|
2113
|
+
# gets the human error on stderr and an empty stdout.
|
|
2114
|
+
json_mode = fmt == "json"
|
|
2115
|
+
|
|
2116
|
+
as_of = None
|
|
2117
|
+
if args.today:
|
|
2118
|
+
as_of = _parse_date(args.today)
|
|
2119
|
+
if as_of is None:
|
|
2120
|
+
return emit_error("invalid_argument", f"--today is not a real ISO date: {args.today}", json_mode=json_mode)
|
|
2121
|
+
|
|
2122
|
+
root, paths, note = resolve_root(args.root, list(args.paths))
|
|
2123
|
+
if not is_vault_root(root):
|
|
2124
|
+
return vault_not_found(args.root, json_mode=json_mode)
|
|
2125
|
+
|
|
2126
|
+
try:
|
|
2127
|
+
applied: list[Override] = []
|
|
2128
|
+
files, findings = lint(
|
|
2129
|
+
root, paths, orphans=not args.no_orphans, secrets=not args.no_secrets, today=as_of, overrides=applied,
|
|
2130
|
+
)
|
|
2131
|
+
if args.no_secrets:
|
|
2132
|
+
applied.append(secret_scan_skipped("--no-secrets"))
|
|
2133
|
+
except VaultPathError as exc:
|
|
2134
|
+
return emit_error(
|
|
2135
|
+
"invalid_argument",
|
|
2136
|
+
str(exc),
|
|
2137
|
+
hint=f"hint: paths are relative to the vault root ({root}); lint checks .md files and directories",
|
|
2138
|
+
json_mode=json_mode,
|
|
2139
|
+
)
|
|
2140
|
+
errors = [f for f in findings if f.level == "error"]
|
|
2141
|
+
warnings = [f for f in findings if f.level == "warning"]
|
|
2142
|
+
failed = bool(errors or (warnings and args.strict))
|
|
2143
|
+
|
|
2144
|
+
security_overrides = [entry for entry in applied if entry.security]
|
|
2145
|
+
if fmt == "sarif":
|
|
2146
|
+
from .ci_formats import to_sarif
|
|
2147
|
+
emit_machine(json.dumps(
|
|
2148
|
+
to_sarif([asdict(f) for f in findings], root, files=len(files), overrides=applied),
|
|
2149
|
+
ensure_ascii=False, indent=2,
|
|
2150
|
+
))
|
|
2151
|
+
elif fmt == "github":
|
|
2152
|
+
from .ci_formats import github_annotations, override_notices, workflow_command
|
|
2153
|
+
lines = github_annotations([asdict(f) for f in findings], root)
|
|
2154
|
+
lines.extend(override_notices(security_overrides))
|
|
2155
|
+
summary = f"{len(files)} files — {len(errors)} error(s), {len(warnings)} warning(s)"
|
|
2156
|
+
lines.append(f"whykit lint: {summary}")
|
|
2157
|
+
if failed and not errors:
|
|
2158
|
+
lines.append(workflow_command("error", f"whykit lint --strict: {len(warnings)} warning(s) fail this gate", title="WhyKit"))
|
|
2159
|
+
emit_machine("\n".join(lines))
|
|
2160
|
+
elif fmt == "json":
|
|
2161
|
+
payload: dict = {
|
|
2162
|
+
"contract_version": 1,
|
|
2163
|
+
"root": str(root),
|
|
2164
|
+
"files": len(files),
|
|
2165
|
+
"errors": len(errors),
|
|
2166
|
+
"warnings": len(warnings),
|
|
2167
|
+
"findings": [asdict(f) for f in findings],
|
|
2168
|
+
}
|
|
2169
|
+
if applied:
|
|
2170
|
+
payload["overrides"] = [entry.report() for entry in applied]
|
|
2171
|
+
emit_machine(json.dumps(payload, ensure_ascii=False, indent=2))
|
|
2172
|
+
else:
|
|
2173
|
+
if note and not args.quiet:
|
|
2174
|
+
print(note)
|
|
2175
|
+
if not args.quiet:
|
|
2176
|
+
by_path: dict[str, list[Finding]] = {}
|
|
2177
|
+
for f in findings:
|
|
2178
|
+
by_path.setdefault(f.path, []).append(f)
|
|
2179
|
+
for path in sorted(by_path):
|
|
2180
|
+
print(f"\n{one_line(path)}")
|
|
2181
|
+
for f in sorted(by_path[path], key=lambda x: (x.line or 0, x.code)):
|
|
2182
|
+
where = f"line {f.line}" if f.line else ""
|
|
2183
|
+
print(f" {f.level:<7} {where:<10} [{f.code}] {one_line(f.message)}")
|
|
2184
|
+
print(f"\n{len(files)} files — {len(errors)} error(s), {len(warnings)} warning(s)" if findings else f"\n{len(files)} files — clean")
|
|
2185
|
+
# Printed even with --quiet: a security rule turned down by policy is
|
|
2186
|
+
# part of what this result means, not detail.
|
|
2187
|
+
for entry in security_overrides:
|
|
2188
|
+
print(f"policy: {entry.describe()}")
|
|
2189
|
+
|
|
2190
|
+
return 1 if failed else 0
|