project-spec 0.0.0-stage → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +323 -2
- package/bin/project-spec.mjs +279 -0
- package/package.json +40 -4
- package/skills/project-knowledge/LICENSE +21 -0
- package/skills/project-knowledge/SKILL.md +139 -0
- package/skills/project-knowledge/agents/openai.yaml +4 -0
- package/skills/project-knowledge/assets/viewer.html +3615 -0
- package/skills/project-knowledge/references/audit.md +52 -0
- package/skills/project-knowledge/references/format.md +215 -0
- package/skills/project-knowledge/references/gaps.md +159 -0
- package/skills/project-knowledge/references/investigate.md +33 -0
- package/skills/project-knowledge/references/merge.md +70 -0
- package/skills/project-knowledge/references/migrate.md +56 -0
- package/skills/project-knowledge/references/sync.md +102 -0
- package/skills/project-knowledge/scripts/knowledge.py +929 -0
- package/skills/project-knowledge/scripts/maintain.py +545 -0
- package/skills/project-knowledge/scripts/merge.py +182 -0
- package/skills/project-knowledge/scripts/test_knowledge.py +1121 -0
- package/skills/project-knowledge/scripts/test_maintain.py +864 -0
- package/skills/project-knowledge/scripts/test_merge.py +165 -0
- package/skills/spec-change/LICENSE +21 -0
- package/skills/spec-change/SKILL.md +63 -0
- package/skills/spec-change/agents/openai.yaml +4 -0
- package/skills/spec-change/references/gaps.md +159 -0
- package/skills/spec-change/references/plan.md +40 -0
|
@@ -0,0 +1,929 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Validate and build a project-knowledge vault using only the standard library."""
|
|
3
|
+
import argparse
|
|
4
|
+
import datetime as dt
|
|
5
|
+
import hashlib
|
|
6
|
+
import html
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import posixpath
|
|
10
|
+
import re
|
|
11
|
+
import sys
|
|
12
|
+
import subprocess
|
|
13
|
+
import tempfile
|
|
14
|
+
import unicodedata
|
|
15
|
+
from urllib.parse import unquote, urlsplit
|
|
16
|
+
from pathlib import Path, PurePosixPath
|
|
17
|
+
|
|
18
|
+
TYPES = {'index', 'area', 'note', 'gap', 'plan', 'run'}
|
|
19
|
+
SPEC_STATUSES = {'current', 'partial', 'stale', 'superseded'}
|
|
20
|
+
NAV_STATUSES = SPEC_STATUSES | {'planned'}
|
|
21
|
+
KINDS = ('product-gap', 'verification-gap', 'product-plan', 'conflict', 'knowledge-gap')
|
|
22
|
+
RECHECKS = ('not-checked', 'confirmed', 'not-reproduced', 'outdated')
|
|
23
|
+
DECISIONS = ('open', 'accepted', 'rejected', 'deferred', 'done', 'duplicate')
|
|
24
|
+
DECIDED = ('accepted', 'rejected', 'deferred')
|
|
25
|
+
STATE_ORDER = ('needs-recheck', 'needs-decision', 'accepted', 'deferred', 'rejected', 'done', 'duplicate')
|
|
26
|
+
GAP_ONLY = ('kind', 'recheck', 'rechecked_by', 'rechecked_on', 'decision', 'decided_by', 'decided_on', 'duplicate_of')
|
|
27
|
+
GAP_SECTIONS = ('Current behavior', 'Gap', 'Evidence', 'Recheck', 'Discussion', 'Decision')
|
|
28
|
+
LAYERS = {'area': 'spec', 'note': 'spec', 'gap': 'gap', 'plan': 'gap', 'index': 'nav', 'run': 'nav'}
|
|
29
|
+
UNCOUNTED = ' (uncommitted change, not counted)' # suffix of a freshness warning that --ignore-dirty does not count
|
|
30
|
+
CONTROL = re.compile('[\x00-\x1f\x7f\x85\u2028\u2029]')
|
|
31
|
+
SCHEME = re.compile(r'^[A-Za-z][A-Za-z0-9+.-]*:')
|
|
32
|
+
MD_LINK = re.compile(r'\]\(\s*(<[^>]*>|[^\s)]*)')
|
|
33
|
+
# The delimiter row of a GFM table: | --- | :-: |, with or without the outer pipes, at least one pipe.
|
|
34
|
+
TABLE_RULE = re.compile(r'^ {0,3}\|?[ \t]*:?-+:?[ \t]*(\|[ \t]*:?-+:?[ \t]*)+\|?[ \t]*$|^ {0,3}\|[ \t]*:?-+:?[ \t]*\|?[ \t]*$')
|
|
35
|
+
# A numbered business rule: the first cell of a table row, e.g. | BR-PROFILE-01 | ... |
|
|
36
|
+
RULE_ROW = re.compile(r'^ {0,3}\|[ \t]*(BR-[A-Z0-9]+(?:-[A-Z0-9]+)*-\d{2,4})[ \t]*\|')
|
|
37
|
+
MD_REF = re.compile(r'^ {0,3}\[[^\]\n]+\]:\s*(<[^>]*>|\S+)', re.M)
|
|
38
|
+
GAP_PAGE_ONLY = ('priority', 'confidence')
|
|
39
|
+
ARRAYS = {'areas', 'sources', 'tags', 'aliases', 'task_ids'}
|
|
40
|
+
STRINGS = {'id', 'title', 'type', 'status', 'updated', 'summary', 'verification', 'priority', 'confidence', 'gap',
|
|
41
|
+
*GAP_ONLY}
|
|
42
|
+
REQUIRED = {'id', 'title', 'type', 'areas', 'sources', 'updated'}
|
|
43
|
+
LINK = re.compile(r'(!?)\[\[([^\[\]\n]+)\]\]') # no brackets inside, so a run of '[[' cannot backtrack
|
|
44
|
+
FENCE = re.compile(r'^ {0,3}(`{3,}|~{3,})(.*)$')
|
|
45
|
+
HEADING = re.compile(r'^ {0,3}(#{1,6})(?:[ \t]+(.*?))?[ \t]*$')
|
|
46
|
+
CREDENTIAL_NAMES = {'.env', '.aws', '.ssh', '.gnupg', 'credentials', 'secrets', '.npmrc', '.netrc', '.htpasswd',
|
|
47
|
+
'id_rsa', 'id_ed25519', 'id_ecdsa', 'id_dsa', '.git', '.git-credentials', 'credentials.json',
|
|
48
|
+
'.envrc', '.pgpass', '.pypirc', '_netrc', '.vault-token', '.terraformrc',
|
|
49
|
+
'application_default_credentials.json'}
|
|
50
|
+
CREDENTIAL_SUFFIXES = ('.pem', '.key', '.p12', '.pfx', '.tfstate', '.tfstate.backup', '.jks', '.ppk', '.p8', '.tfvars', '.gpg')
|
|
51
|
+
CREDENTIAL_PAIRS = {('.kube', 'config'), ('.docker', 'config.json')}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def safe_path(root, value):
|
|
55
|
+
p = PurePosixPath(value)
|
|
56
|
+
if not value or p.is_absolute() or '..' in p.parts or '\\' in value:
|
|
57
|
+
raise ValueError('path must be relative and stay inside its root')
|
|
58
|
+
candidate = root.joinpath(*p.parts)
|
|
59
|
+
if not candidate.resolve().is_relative_to(root):
|
|
60
|
+
raise ValueError('path resolves outside its root')
|
|
61
|
+
return candidate
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def output_path(root, value):
|
|
65
|
+
path = safe_path(root, value)
|
|
66
|
+
current = root
|
|
67
|
+
for part in PurePosixPath(value).parts:
|
|
68
|
+
current = current / part
|
|
69
|
+
if current.is_symlink():
|
|
70
|
+
raise ValueError(f'output path contains symlink: {value}')
|
|
71
|
+
return path
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def fold(part):
|
|
75
|
+
"""Compatibility-fold a path part so look-alike characters (e.g. U+017F) cannot dodge name checks."""
|
|
76
|
+
return unicodedata.normalize('NFKC', part).casefold()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def credential_path(value):
|
|
80
|
+
parts = [fold(p) for p in PurePosixPath(value).parts]
|
|
81
|
+
if any(pair in CREDENTIAL_PAIRS for pair in zip(parts, parts[1:])):
|
|
82
|
+
return True
|
|
83
|
+
return any(p in CREDENTIAL_NAMES or p.startswith('.env.') or p.endswith(CREDENTIAL_SUFFIXES) or
|
|
84
|
+
(p.startswith('service-account') and p.endswith('.json')) or
|
|
85
|
+
(p.startswith('client_secret') and p.endswith('.json')) or p.startswith('secrets.') for p in parts)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def lines_with_code(body, state=None):
|
|
89
|
+
"""Yield (line, in_code) following CommonMark fenced code rules.
|
|
90
|
+
|
|
91
|
+
A backtick opener cannot have a backtick in its info string, and a closer
|
|
92
|
+
cannot have an info string at all.
|
|
93
|
+
"""
|
|
94
|
+
fence = None
|
|
95
|
+
for line in re.split(r'\r\n|\n|\r', body):
|
|
96
|
+
match = FENCE.match(line)
|
|
97
|
+
if fence is None:
|
|
98
|
+
if match and not (match.group(1)[0] == '`' and '`' in match.group(2)):
|
|
99
|
+
fence = (match.group(1)[0], len(match.group(1)))
|
|
100
|
+
yield line, True
|
|
101
|
+
else:
|
|
102
|
+
yield line, False
|
|
103
|
+
else:
|
|
104
|
+
yield line, True
|
|
105
|
+
if match and match.group(1)[0] == fence[0] and len(match.group(1)) >= fence[1] and not match.group(2).strip():
|
|
106
|
+
fence = None
|
|
107
|
+
if state is not None:
|
|
108
|
+
state['fence'] = fence # still open when the text ended
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def prose(body):
|
|
112
|
+
return '\n'.join(re.sub(r'(`+).*?\1', '', line) for line, in_code in lines_with_code(body) if not in_code)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def heading_of(line):
|
|
116
|
+
"""Return (level, text) for an ATX heading line, else None."""
|
|
117
|
+
match = HEADING.match(line)
|
|
118
|
+
if not match:
|
|
119
|
+
return None
|
|
120
|
+
return len(match.group(1)), re.sub(r'(?:^|[ \t]+)#+$', '', match.group(2) or '').strip()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def sections(body):
|
|
124
|
+
"""Return [(name or None, text)] for each level-1/2 heading outside code, in order."""
|
|
125
|
+
found, current = [], None
|
|
126
|
+
for line, in_code in lines_with_code(body):
|
|
127
|
+
head = None if in_code else heading_of(line)
|
|
128
|
+
if head and head[0] <= 2:
|
|
129
|
+
current = (head[1] if head[0] == 2 else None, [])
|
|
130
|
+
found.append(current)
|
|
131
|
+
elif current is not None:
|
|
132
|
+
current[1].append(line)
|
|
133
|
+
return [(name, '\n'.join(lines)) for name, lines in found]
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def atomic_write(path, text):
|
|
137
|
+
"""Write text next to its destination and replace it, so readers never see a partial file."""
|
|
138
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
139
|
+
mask = os.umask(0)
|
|
140
|
+
os.umask(mask)
|
|
141
|
+
mode = path.stat().st_mode & 0o777 if path.is_file() else 0o666 & ~mask
|
|
142
|
+
fd, name = tempfile.mkstemp(dir=path.parent, prefix='.knowledge-', suffix='.tmp')
|
|
143
|
+
try:
|
|
144
|
+
with os.fdopen(fd, 'wb') as handle:
|
|
145
|
+
handle.write(text.encode('utf-8'))
|
|
146
|
+
os.chmod(name, mode)
|
|
147
|
+
os.replace(name, path)
|
|
148
|
+
finally:
|
|
149
|
+
if os.path.exists(name):
|
|
150
|
+
os.unlink(name)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def check_date(key, value):
|
|
154
|
+
if not re.fullmatch(r'\d{4}-\d{2}-\d{2}', value):
|
|
155
|
+
raise ValueError(f'{key} must be YYYY-MM-DD')
|
|
156
|
+
try:
|
|
157
|
+
dt.date.fromisoformat(value)
|
|
158
|
+
except ValueError:
|
|
159
|
+
raise ValueError(f'{key} is not a real date')
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def gap_state(meta):
|
|
163
|
+
"""Derived, never stored: needs-recheck, needs-decision, or the decision itself."""
|
|
164
|
+
if meta['decision'] != 'open':
|
|
165
|
+
return meta['decision']
|
|
166
|
+
return 'needs-recheck' if meta['recheck'] in {'not-checked', 'outdated'} else 'needs-decision'
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def validate_meta(meta):
|
|
170
|
+
"""Raise ValueError with an author-facing message for the first frontmatter problem."""
|
|
171
|
+
missing = REQUIRED - meta.keys()
|
|
172
|
+
if missing:
|
|
173
|
+
raise ValueError('missing properties: ' + ', '.join(sorted(missing)))
|
|
174
|
+
for key in STRINGS & meta.keys():
|
|
175
|
+
if not isinstance(meta[key], str):
|
|
176
|
+
raise ValueError(f'{key} must be a string')
|
|
177
|
+
for key in ARRAYS & meta.keys():
|
|
178
|
+
if not isinstance(meta[key], list) or any(not isinstance(x, str) for x in meta[key]):
|
|
179
|
+
raise ValueError(f'{key} must be a string array')
|
|
180
|
+
for key in (STRINGS | ARRAYS) & meta.keys():
|
|
181
|
+
for value in (meta[key] if isinstance(meta[key], list) else [meta[key]]):
|
|
182
|
+
if CONTROL.search(value):
|
|
183
|
+
raise ValueError(f'{key} must not contain control characters (line breaks, tabs, U+0085, U+2028, U+2029)')
|
|
184
|
+
if not re.fullmatch(r'[a-z0-9-]+', meta['id']):
|
|
185
|
+
raise ValueError('invalid id')
|
|
186
|
+
kind = meta['type']
|
|
187
|
+
if not meta['title'].strip() or kind not in TYPES:
|
|
188
|
+
raise ValueError('invalid title or type; type must be one of ' + ', '.join(sorted(TYPES)))
|
|
189
|
+
check_date('updated', meta['updated'])
|
|
190
|
+
if kind in {'note', 'gap', 'plan'} and not meta['areas']:
|
|
191
|
+
raise ValueError(f'{kind} requires an area')
|
|
192
|
+
if kind == 'gap':
|
|
193
|
+
if 'status' in meta:
|
|
194
|
+
raise ValueError('a gap must not have status; gaps use recheck and decision instead')
|
|
195
|
+
validate_gap(meta)
|
|
196
|
+
else:
|
|
197
|
+
for key in GAP_ONLY:
|
|
198
|
+
if key in meta:
|
|
199
|
+
raise ValueError(f'{key} is only valid on gap pages')
|
|
200
|
+
if kind != 'plan':
|
|
201
|
+
for key in GAP_PAGE_ONLY:
|
|
202
|
+
if key in meta:
|
|
203
|
+
raise ValueError(f'{key} is only valid on gap and plan pages')
|
|
204
|
+
if kind in {'area', 'note'}:
|
|
205
|
+
for key in ('title', 'summary'):
|
|
206
|
+
if re.search(r'\[\[|\[[^\]\n]*\]\([^)\n]*\)', meta.get(key, '')):
|
|
207
|
+
raise ValueError(f'{key} of a spec page must not contain links; describe the page in plain text')
|
|
208
|
+
if 'status' not in meta:
|
|
209
|
+
raise ValueError('missing properties: status')
|
|
210
|
+
allowed = SPEC_STATUSES if kind in {'area', 'note'} else NAV_STATUSES
|
|
211
|
+
if meta['status'] not in allowed:
|
|
212
|
+
hint = ''
|
|
213
|
+
if meta['status'] in {'unverified', 'conflict'}:
|
|
214
|
+
hint = f"; '{meta['status']}' is no longer a status: record it as a gap under gaps/"
|
|
215
|
+
raise ValueError(f"status must be one of {', '.join(sorted(allowed))}{hint}")
|
|
216
|
+
if kind == 'note' and meta['status'] == 'current' and not meta['sources']:
|
|
217
|
+
raise ValueError('current note requires sources')
|
|
218
|
+
if kind == 'plan':
|
|
219
|
+
if not meta.get('gap', '').strip():
|
|
220
|
+
raise ValueError('plan requires gap: the id of the accepted gap it implements')
|
|
221
|
+
elif 'gap' in meta:
|
|
222
|
+
raise ValueError('gap is only valid on plan pages')
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def validate_gap(meta):
|
|
226
|
+
for key, values in (('kind', KINDS), ('recheck', RECHECKS), ('decision', DECISIONS)):
|
|
227
|
+
if key not in meta:
|
|
228
|
+
raise ValueError(f'gap requires {key} (one of {", ".join(values)})')
|
|
229
|
+
if meta[key] not in values:
|
|
230
|
+
raise ValueError(f'{key} must be one of {", ".join(values)}')
|
|
231
|
+
if meta['recheck'] == 'not-checked':
|
|
232
|
+
for key in ('rechecked_by', 'rechecked_on'):
|
|
233
|
+
if key in meta:
|
|
234
|
+
raise ValueError(f"{key} must be absent while recheck is 'not-checked'")
|
|
235
|
+
else:
|
|
236
|
+
for key in ('rechecked_by', 'rechecked_on'):
|
|
237
|
+
if key not in meta:
|
|
238
|
+
raise ValueError(f"{key} is required when recheck is '{meta['recheck']}'")
|
|
239
|
+
if not meta['rechecked_by'].strip():
|
|
240
|
+
raise ValueError('rechecked_by must name a person, or be "agent"')
|
|
241
|
+
check_date('rechecked_on', meta['rechecked_on'])
|
|
242
|
+
if meta['decision'] == 'open':
|
|
243
|
+
for key in ('decided_by', 'decided_on'):
|
|
244
|
+
if key in meta:
|
|
245
|
+
raise ValueError(f"{key} must be absent while decision is 'open'")
|
|
246
|
+
elif meta['decision'] in DECIDED:
|
|
247
|
+
for key in ('decided_by', 'decided_on'):
|
|
248
|
+
if key not in meta:
|
|
249
|
+
raise ValueError(f"{key} is required when decision is '{meta['decision']}'")
|
|
250
|
+
if 'decided_by' in meta and (not meta['decided_by'].strip() or meta['decided_by'].strip().lower() == 'agent'):
|
|
251
|
+
raise ValueError("decided_by must be a person's name: an agent may not decide, only record what a person decided")
|
|
252
|
+
if 'decided_on' in meta:
|
|
253
|
+
check_date('decided_on', meta['decided_on'])
|
|
254
|
+
if meta['decision'] == 'duplicate':
|
|
255
|
+
if not meta.get('duplicate_of', '').strip():
|
|
256
|
+
raise ValueError("duplicate_of is required when decision is 'duplicate': give the id of the other gap")
|
|
257
|
+
elif 'duplicate_of' in meta:
|
|
258
|
+
raise ValueError("duplicate_of is only valid when decision is 'duplicate'")
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def has_conflict_markers(content):
|
|
262
|
+
# Git emits labeled openers; keep candidates of different widths independent.
|
|
263
|
+
active = {}
|
|
264
|
+
for line in content.splitlines():
|
|
265
|
+
opener = re.fullmatch(r'(<+)[ \t]+\S.*', line)
|
|
266
|
+
if opener:
|
|
267
|
+
active.setdefault(len(opener.group(1)), set()).add(1)
|
|
268
|
+
for width, states in active.items():
|
|
269
|
+
if 1 in states and re.fullmatch(r'\|{' + str(width) + r'}(?:[ \t].*)?', line):
|
|
270
|
+
states.add(2)
|
|
271
|
+
if (1 in states or 2 in states) and line == '=' * width:
|
|
272
|
+
states.add(3)
|
|
273
|
+
if 3 in states and re.fullmatch(r'>{' + str(width) + r'}(?:[ \t].*)?', line):
|
|
274
|
+
return True
|
|
275
|
+
return False
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def tolerates_missing_source(meta, ids):
|
|
279
|
+
"""Missing sources only warn on stale/superseded pages, closed gaps and plans of done gaps."""
|
|
280
|
+
if meta.get('status') in {'stale', 'superseded'}:
|
|
281
|
+
return True
|
|
282
|
+
if meta['type'] == 'gap':
|
|
283
|
+
return meta['decision'] in {'done', 'rejected', 'duplicate'}
|
|
284
|
+
if meta['type'] == 'plan':
|
|
285
|
+
return ids.get(meta['gap'], {}).get('meta', {}).get('decision') == 'done'
|
|
286
|
+
return False
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def markdown_target(rel, raw):
|
|
290
|
+
"""Vault-relative path a Markdown link points at, or None for URLs, anchors and absolute paths."""
|
|
291
|
+
target = unquote(re.split(r'[?#]', raw[1:-1] if raw.startswith('<') else raw, maxsplit=1)[0])
|
|
292
|
+
if not target or target.startswith('/') or SCHEME.match(target):
|
|
293
|
+
return None
|
|
294
|
+
return posixpath.normpath(posixpath.join(posixpath.dirname(rel), target))
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def attachment_exists(vault, target, cache):
|
|
298
|
+
"""Embedded non-Markdown files (images, PDFs) resolve by path or bare name anywhere in the vault."""
|
|
299
|
+
if not cache:
|
|
300
|
+
cache['names'], cache['paths'] = set(), set()
|
|
301
|
+
for file in vault.rglob('*'):
|
|
302
|
+
parts = file.relative_to(vault).parts
|
|
303
|
+
if file.is_file() and not any(part in {'.obsidian', '.trash', '.knowledge'} for part in parts):
|
|
304
|
+
cache['names'].add(parts[-1].casefold())
|
|
305
|
+
cache['paths'].add('/'.join(parts).casefold())
|
|
306
|
+
wanted = target.casefold()
|
|
307
|
+
return wanted in cache['paths'] or ('/' not in wanted and wanted in cache['names'])
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def line_ending_variant(data, old):
|
|
311
|
+
"""True when data matches the baseline digest after converting all line endings to LF or CRLF."""
|
|
312
|
+
lf = data.replace(b'\r\n', b'\n')
|
|
313
|
+
return old in {hashlib.sha256(lf).hexdigest(), hashlib.sha256(lf.replace(b'\n', b'\r\n')).hexdigest()}
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def head_blob(repo, source):
|
|
317
|
+
"""Bytes of the source as committed at HEAD, or None when HEAD has no such file (untracked, new or no commits)."""
|
|
318
|
+
result = subprocess.run(['git', '-C', str(repo), 'cat-file', 'blob', f'HEAD:./{posixpath.normpath(source)}'], capture_output=True)
|
|
319
|
+
return None if result.returncode else result.stdout
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def uncommitted_paths(repo):
|
|
323
|
+
"""Repo-relative paths Git reports as modified, staged or untracked (one status call), or None outside a work tree."""
|
|
324
|
+
probe = subprocess.run(['git', '-C', str(repo), 'rev-parse', '--is-inside-work-tree', '--show-prefix'], capture_output=True)
|
|
325
|
+
lines = probe.stdout.decode(errors='replace').split('\n')
|
|
326
|
+
if probe.returncode or lines[0] != 'true':
|
|
327
|
+
return None
|
|
328
|
+
prefix = lines[1] if len(lines) > 1 else ''
|
|
329
|
+
status = subprocess.run(['git', '-C', str(repo), 'status', '--porcelain=v1', '-z', '--untracked-files=all'], capture_output=True)
|
|
330
|
+
if status.returncode:
|
|
331
|
+
return None
|
|
332
|
+
raw, found, i = status.stdout.split(b'\0'), set(), 0
|
|
333
|
+
while i < len(raw) and raw[i]:
|
|
334
|
+
item = os.fsdecode(raw[i])
|
|
335
|
+
i += 1
|
|
336
|
+
names = [item[3:]]
|
|
337
|
+
if 'R' in item[:2] or 'C' in item[:2]:
|
|
338
|
+
names.append(os.fsdecode(raw[i]))
|
|
339
|
+
i += 1
|
|
340
|
+
found.update(name[len(prefix):] for name in names if name.startswith(prefix))
|
|
341
|
+
return found
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def error_page(message):
|
|
345
|
+
"""The vault-relative page an error message is about, or None for vault-wide errors."""
|
|
346
|
+
head = message.split(': ', 1)[0]
|
|
347
|
+
return head if head.endswith('.md') else None
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def load_vault(repo, vault, freshness=True, dirty=None, report=None):
|
|
351
|
+
"""dirty: repo-relative paths with uncommitted changes. Their freshness warning is marked, not counted, only when
|
|
352
|
+
the accepted fingerprint equals the file's HEAD blob (up to LF/CRLF), so the sole drift is uncommitted.
|
|
353
|
+
|
|
354
|
+
report, when given, receives 'failed': {page path: parsed id or None} for pages that failed validation.
|
|
355
|
+
"""
|
|
356
|
+
errors, warnings, notes, failed, failed_kind, cascade = [], [], [], {}, {}, []
|
|
357
|
+
baseline_file = vault / '.knowledge/sources.json'
|
|
358
|
+
baseline = {}
|
|
359
|
+
if baseline_file.exists():
|
|
360
|
+
try:
|
|
361
|
+
safe_path(vault, '.knowledge/sources.json')
|
|
362
|
+
baseline = json.loads(baseline_file.read_text(encoding='utf-8'))
|
|
363
|
+
if not isinstance(baseline, dict) or any(not isinstance(v, dict) for v in baseline.values()):
|
|
364
|
+
raise ValueError('expected object of note source maps')
|
|
365
|
+
except (OSError, ValueError) as exc:
|
|
366
|
+
errors.append(f'Invalid source snapshot: {exc}')
|
|
367
|
+
baseline = {}
|
|
368
|
+
for file in sorted(vault.rglob('*.md')):
|
|
369
|
+
relative = file.relative_to(vault)
|
|
370
|
+
if any(part in {'.obsidian', '.trash', '.knowledge'} for part in relative.parts):
|
|
371
|
+
continue
|
|
372
|
+
rel = relative.as_posix()
|
|
373
|
+
meta = None
|
|
374
|
+
try:
|
|
375
|
+
safe_path(vault, rel)
|
|
376
|
+
content = file.read_text(encoding='utf-8').removeprefix('\ufeff')
|
|
377
|
+
if has_conflict_markers(content):
|
|
378
|
+
raise ValueError('unresolved Git conflict markers')
|
|
379
|
+
match = re.match(r'\A---\s*\n(.*?)\n---\s*(?:\n|$)', content, re.S)
|
|
380
|
+
if not match:
|
|
381
|
+
raise ValueError('missing frontmatter')
|
|
382
|
+
meta = {}
|
|
383
|
+
for line in match.group(1).splitlines():
|
|
384
|
+
if not line.strip() or line.lstrip().startswith('#'):
|
|
385
|
+
continue
|
|
386
|
+
key, sep, value = line.partition(':')
|
|
387
|
+
if not sep or not re.fullmatch(r'[a-z_]+', key) or key in meta:
|
|
388
|
+
raise ValueError('invalid or duplicate property')
|
|
389
|
+
meta[key] = json.loads(value.strip())
|
|
390
|
+
validate_meta(meta)
|
|
391
|
+
notes.append({'meta': meta, 'path': rel, 'body': content[match.end():]})
|
|
392
|
+
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
|
393
|
+
errors.append(f'{rel}: {exc}')
|
|
394
|
+
# The page still exists as a link target; only its own errors are real.
|
|
395
|
+
ident = meta.get('id') if isinstance(meta, dict) else None
|
|
396
|
+
failed[rel] = ident if isinstance(ident, str) and re.fullmatch(r'[a-z0-9-]+', ident) else None
|
|
397
|
+
kind = meta.get('type') if isinstance(meta, dict) else None
|
|
398
|
+
failed_kind[rel] = kind if kind in TYPES else None
|
|
399
|
+
failed_ids = {i for i in failed.values() if i}
|
|
400
|
+
failed_id_kinds = {}
|
|
401
|
+
for rel, ident in failed.items():
|
|
402
|
+
if ident:
|
|
403
|
+
failed_id_kinds.setdefault(ident, set()).add(failed_kind[rel])
|
|
404
|
+
failed_stems = {rel[:-3]: failed_kind[rel] for rel in failed}
|
|
405
|
+
failed_names = {}
|
|
406
|
+
for stem, kind in failed_stems.items():
|
|
407
|
+
failed_names.setdefault(PurePosixPath(stem).name, []).append(kind)
|
|
408
|
+
ids, paths, basenames = {}, {}, {}
|
|
409
|
+
for note in notes:
|
|
410
|
+
meta, rel = note['meta'], note['path']
|
|
411
|
+
if meta['id'] in ids:
|
|
412
|
+
errors.append(f"{rel}: duplicate id {meta['id']}")
|
|
413
|
+
ids[meta['id']] = note
|
|
414
|
+
stem = rel[:-3]
|
|
415
|
+
paths[stem] = note
|
|
416
|
+
basenames.setdefault(PurePosixPath(stem).name, []).append(note)
|
|
417
|
+
area_ids = {n['meta']['id'] for n in notes if n['meta']['type'] == 'area'}
|
|
418
|
+
edges, attachments, rule_pages = set(), {}, {}
|
|
419
|
+
for note in notes:
|
|
420
|
+
meta, rel = note['meta'], note['path']
|
|
421
|
+
note['stale'] = False
|
|
422
|
+
if meta['type'] == 'gap':
|
|
423
|
+
note['state'] = gap_state(meta)
|
|
424
|
+
if meta['type'] == 'gap' and meta.get('duplicate_of'):
|
|
425
|
+
other = ids.get(meta['duplicate_of'])
|
|
426
|
+
if other is None and meta['duplicate_of'] in failed_ids and failed_id_kinds[meta['duplicate_of']] & {None, 'gap'}:
|
|
427
|
+
cascade.append(f"{rel}: duplicate_of {meta['duplicate_of']} names a page that failed validation")
|
|
428
|
+
elif other is None or other['meta']['type'] != 'gap':
|
|
429
|
+
errors.append(f"{rel}: duplicate_of {meta['duplicate_of']} is not the id of an existing gap")
|
|
430
|
+
elif other is note:
|
|
431
|
+
errors.append(f'{rel}: duplicate_of must name a different gap')
|
|
432
|
+
if meta['type'] == 'plan':
|
|
433
|
+
other = ids.get(meta['gap'])
|
|
434
|
+
if other is None and meta['gap'] in failed_ids and failed_id_kinds[meta['gap']] & {None, 'gap'}:
|
|
435
|
+
cascade.append(f"{rel}: gap {meta['gap']} names a page that failed validation")
|
|
436
|
+
elif other is None or other['meta']['type'] != 'gap':
|
|
437
|
+
errors.append(f"{rel}: gap {meta['gap']} is not the id of an existing gap")
|
|
438
|
+
else:
|
|
439
|
+
edges.add((meta['id'], other['meta']['id']))
|
|
440
|
+
if other is not None and other['meta']['type'] == 'gap' and other['meta']['decision'] not in {'accepted', 'done'}:
|
|
441
|
+
errors.append(f"{rel}: gap {meta['gap']} has decision '{other['meta']['decision']}'; a plan needs an accepted or done gap")
|
|
442
|
+
if meta['type'] == 'gap':
|
|
443
|
+
found = [name for name, _ in sections(note['body']) if name is not None]
|
|
444
|
+
for name in GAP_SECTIONS:
|
|
445
|
+
if found.count(name) != 1:
|
|
446
|
+
errors.append(f"{rel}: gap needs exactly one '## {name}' section (found {found.count(name)})")
|
|
447
|
+
texts = {name: text for name, text in sections(note['body']) if name in GAP_SECTIONS}
|
|
448
|
+
if meta['decision'] != 'open' and found.count('Decision') == 1 and not texts['Decision'].strip():
|
|
449
|
+
errors.append(f"{rel}: the Decision section must say who decided and why once decision is '{meta['decision']}'")
|
|
450
|
+
if meta['recheck'] != 'not-checked' and found.count('Recheck') == 1 and not texts['Recheck'].strip():
|
|
451
|
+
errors.append(f"{rel}: the Recheck section must say what was checked once recheck is '{meta['recheck']}'")
|
|
452
|
+
for area in meta['areas']:
|
|
453
|
+
if area not in ids and area in failed_ids and failed_id_kinds[area] & {None, 'area'}:
|
|
454
|
+
# Only the "missing" error is hidden; a failed page known to be another type still reports it.
|
|
455
|
+
cascade.append(f'{rel}: area {area} names a page that failed validation')
|
|
456
|
+
continue
|
|
457
|
+
if area not in area_ids:
|
|
458
|
+
errors.append(f'{rel}: unknown area {area}')
|
|
459
|
+
elif area != meta['id']:
|
|
460
|
+
edges.add((area, meta['id']))
|
|
461
|
+
text = prose(note['body'])
|
|
462
|
+
run_links = 0
|
|
463
|
+
for embed, link in LINK.findall(text):
|
|
464
|
+
# Inside Markdown tables Obsidian requires the alias separator to be written as \|.
|
|
465
|
+
target = re.split(r'\\?\|', link, maxsplit=1)[0].split('#', 1)[0].split('^', 1)[0].strip()
|
|
466
|
+
if not target:
|
|
467
|
+
continue
|
|
468
|
+
if target.endswith('.md'):
|
|
469
|
+
target = target[:-3]
|
|
470
|
+
choices = basenames.get(target, []) if '/' not in target else ([paths[target]] if target in paths else [])
|
|
471
|
+
broken_kinds = failed_names.get(target, []) if '/' not in target else ([failed_stems[target]] if target in failed_stems else [])
|
|
472
|
+
broken = len(broken_kinds)
|
|
473
|
+
if broken == 1 and not choices:
|
|
474
|
+
# The one candidate exists but failed validation; its own errors are the root cause of "missing".
|
|
475
|
+
if meta['type'] in {'area', 'note'} and broken_kinds[0] in {'gap', 'plan'}:
|
|
476
|
+
errors.append(f"{rel}: spec pages must not link to the gap layer ({'embed ' if embed else ''}{target}); link from the gap to this page instead")
|
|
477
|
+
else:
|
|
478
|
+
cascade.append(f'{rel}: link {target} names a page that failed validation')
|
|
479
|
+
if meta['type'] == 'run':
|
|
480
|
+
run_links += 1
|
|
481
|
+
continue
|
|
482
|
+
if embed and not choices and PurePosixPath(target).suffix:
|
|
483
|
+
# Not a note: Obsidian embeds images and PDFs by file name.
|
|
484
|
+
if not attachment_exists(vault, target, attachments):
|
|
485
|
+
errors.append(f'{rel}: missing embed {target}')
|
|
486
|
+
continue
|
|
487
|
+
if len(choices) + broken != 1:
|
|
488
|
+
errors.append(f"{rel}: {'ambiguous' if len(choices) + broken > 1 else 'missing'} {'embed' if embed else 'wikilink'} {target}")
|
|
489
|
+
elif meta['type'] in {'area', 'note'} and choices[0]['meta']['type'] in {'gap', 'plan'}:
|
|
490
|
+
errors.append(f"{rel}: spec pages must not link to the gap layer ({'embed ' if embed else ''}{target}); link from the gap to this page instead")
|
|
491
|
+
elif meta['type'] == 'run':
|
|
492
|
+
run_links += 1
|
|
493
|
+
elif meta['type'] == 'area' and choices[0]['meta']['type'] == 'note' and re.search(r'[#^]', re.split(r'\\?\|', link, maxsplit=1)[0]):
|
|
494
|
+
errors.append(f"{rel}: an area page links whole notes, not a part of one ({link.split('|')[0].replace(chr(92), '')}); "
|
|
495
|
+
"if that part needs its own link, make it its own note and list it here")
|
|
496
|
+
else:
|
|
497
|
+
edges.add((meta['id'], choices[0]['meta']['id']))
|
|
498
|
+
if meta['type'] == 'area':
|
|
499
|
+
# An area page is a map of its notes. A table on it is behavior written in the wrong place.
|
|
500
|
+
tables = sum(1 for line in text.split('\n') if TABLE_RULE.match(line))
|
|
501
|
+
if tables:
|
|
502
|
+
errors.append(f"{rel}: an area page is a map and holds no table ({tables} found); a table describes behavior: "
|
|
503
|
+
"write a note for each thing it describes and list the notes on the area page")
|
|
504
|
+
if meta['type'] == 'note':
|
|
505
|
+
# People quote a rule by its number in tickets, tests and chats, so a number means one rule for good.
|
|
506
|
+
seen = set()
|
|
507
|
+
for line in text.split('\n'):
|
|
508
|
+
rule = RULE_ROW.match(line)
|
|
509
|
+
if not rule:
|
|
510
|
+
continue
|
|
511
|
+
ident = rule.group(1)
|
|
512
|
+
first = rule_pages.setdefault(ident, rel)
|
|
513
|
+
if first != rel or ident in seen:
|
|
514
|
+
errors.append(f"{rel}: rule {ident} is numbered twice ({'on this page' if first == rel else 'also on ' + first}); "
|
|
515
|
+
"a rule keeps its number and a number is never used again, so give the newer rule the next free one")
|
|
516
|
+
seen.add(ident)
|
|
517
|
+
if meta['type'] in {'area', 'note'}:
|
|
518
|
+
gap_files = {n['path'].casefold() for n in notes if n['meta']['type'] in {'gap', 'plan'}} | {'gaps.md'}
|
|
519
|
+
gap_files |= {f.casefold() for f, kind in failed_kind.items() if kind in {'gap', 'plan'}}
|
|
520
|
+
unknown_files = {f.casefold() for f, kind in failed_kind.items() if kind is None}
|
|
521
|
+
for raw in MD_LINK.findall(text) + MD_REF.findall(text):
|
|
522
|
+
resolved = markdown_target(rel, raw)
|
|
523
|
+
if resolved is not None and resolved.casefold() in gap_files:
|
|
524
|
+
errors.append(f"{rel}: spec pages must not link to the gap layer ({raw.strip('<>')}); link from the gap to this page instead")
|
|
525
|
+
elif resolved is not None and resolved.casefold() in unknown_files:
|
|
526
|
+
cascade.append(f'{rel}: link {raw.strip("<>")} names a page that failed validation')
|
|
527
|
+
if meta['type'] == 'run':
|
|
528
|
+
# A run touches many pages; as links they would turn it into a hub that glues every cluster together.
|
|
529
|
+
page_files = {n['path'].casefold() for n in notes} | {f.casefold() for f in failed}
|
|
530
|
+
run_links += sum(1 for raw in MD_LINK.findall(text) + MD_REF.findall(text)
|
|
531
|
+
if (markdown_target(rel, raw) or '').casefold() in page_files)
|
|
532
|
+
if run_links:
|
|
533
|
+
errors.append(f"{rel}: run pages are history, not part of the map, and must not link to vault pages ({run_links} link{'s' if run_links != 1 else ''}); name a page by its path in code formatting, e.g. `notes/example`")
|
|
534
|
+
closed = tolerates_missing_source(meta, ids)
|
|
535
|
+
note['changed'] = False
|
|
536
|
+
for source in meta['sources']:
|
|
537
|
+
try:
|
|
538
|
+
if credential_path(source):
|
|
539
|
+
raise ValueError('credential source is forbidden')
|
|
540
|
+
path = safe_path(repo, source)
|
|
541
|
+
if credential_path(path.resolve().relative_to(repo).as_posix()):
|
|
542
|
+
raise ValueError('credential source is forbidden')
|
|
543
|
+
if not path.is_file():
|
|
544
|
+
if closed:
|
|
545
|
+
warnings.append(f'{rel}: missing source {source}')
|
|
546
|
+
note['stale'] = True
|
|
547
|
+
continue
|
|
548
|
+
raise ValueError('source is missing or not a file')
|
|
549
|
+
if freshness:
|
|
550
|
+
old = baseline.get(meta['id'], {}).get(source)
|
|
551
|
+
data = path.read_bytes()
|
|
552
|
+
if old != hashlib.sha256(data).hexdigest():
|
|
553
|
+
mark = ''
|
|
554
|
+
if old and dirty is not None and posixpath.normpath(source) in dirty:
|
|
555
|
+
# Forgive only pure uncommitted drift: the accepted bytes are what HEAD holds (up to line endings).
|
|
556
|
+
blob = head_blob(repo, source)
|
|
557
|
+
if blob is not None and (hashlib.sha256(blob).hexdigest() == old or line_ending_variant(blob, old)):
|
|
558
|
+
mark = UNCOUNTED
|
|
559
|
+
if old and line_ending_variant(data, old):
|
|
560
|
+
warnings.append(f'{rel}: line endings differ in source {source}{mark}')
|
|
561
|
+
else:
|
|
562
|
+
note['changed'] = note['changed'] or bool(old)
|
|
563
|
+
warnings.append(f"{rel}: {'changed' if old else 'unbaselined'} source {source}{mark}")
|
|
564
|
+
note['stale'] = True
|
|
565
|
+
except (OSError, ValueError) as exc:
|
|
566
|
+
errors.append(f'{rel}: {source}: {exc}')
|
|
567
|
+
if report is not None:
|
|
568
|
+
report['failed'] = failed
|
|
569
|
+
report['cascade'] = cascade
|
|
570
|
+
return notes, sorted(edges), errors, warnings, baseline
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def write_json(path, value):
|
|
574
|
+
atomic_write(path, json.dumps(value, ensure_ascii=False, indent=2) + '\n')
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def recheck_marker(note):
|
|
578
|
+
"""Automatic marker: a recorded recheck exists but a cited source's bytes changed since its fingerprint."""
|
|
579
|
+
return note['meta']['recheck'] in {'confirmed', 'not-reproduced'} and bool(note.get('changed'))
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
FILE_LIMIT = 300_000 # characters of one cited file sent to the viewer
|
|
583
|
+
FILES_LIMIT = 4_000_000 # characters of all cited files together
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def cited_files(repo, vault, notes):
|
|
587
|
+
"""Text of the repository files the pages cite, so the viewer can show them instead of sending the reader to a raw file.
|
|
588
|
+
|
|
589
|
+
Keys are repository-relative paths. Only files inside the repository and outside the vault are taken, never
|
|
590
|
+
credential paths, binaries or symlinks that leave the repository. Returns (files, skipped paths).
|
|
591
|
+
"""
|
|
592
|
+
wanted = set()
|
|
593
|
+
for note in notes:
|
|
594
|
+
for source in note['meta']['sources']:
|
|
595
|
+
wanted.add(posixpath.normpath(source))
|
|
596
|
+
text = prose(note['body'])
|
|
597
|
+
for raw in MD_LINK.findall(text) + MD_REF.findall(text):
|
|
598
|
+
target = markdown_target(note['path'], raw)
|
|
599
|
+
if target is not None:
|
|
600
|
+
wanted.add(Path(os.path.relpath(os.path.normpath(vault / target), repo)).as_posix())
|
|
601
|
+
files, skipped, total = {}, [], 0
|
|
602
|
+
for rel in sorted(wanted):
|
|
603
|
+
if rel.startswith('../') or rel in ('..', '.') or credential_path(rel):
|
|
604
|
+
continue
|
|
605
|
+
file = repo / rel
|
|
606
|
+
try:
|
|
607
|
+
resolved = file.resolve()
|
|
608
|
+
if not file.is_file() or not resolved.is_relative_to(repo) or resolved.is_relative_to(vault):
|
|
609
|
+
continue
|
|
610
|
+
data = file.read_bytes()
|
|
611
|
+
if b'\0' in data:
|
|
612
|
+
continue
|
|
613
|
+
text = data.decode('utf-8')
|
|
614
|
+
except (OSError, UnicodeDecodeError, ValueError):
|
|
615
|
+
continue
|
|
616
|
+
truncated = len(text) > FILE_LIMIT
|
|
617
|
+
if truncated:
|
|
618
|
+
text = text[:FILE_LIMIT].rsplit('\n', 1)[0]
|
|
619
|
+
if total + len(text) > FILES_LIMIT:
|
|
620
|
+
skipped.append(rel)
|
|
621
|
+
continue
|
|
622
|
+
total += len(text)
|
|
623
|
+
files[rel] = {'text': text, 'truncated': truncated}
|
|
624
|
+
return files, skipped
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def node_of(note):
|
|
628
|
+
m = note['meta']
|
|
629
|
+
gap = m['type'] == 'gap'
|
|
630
|
+
return {k: m[k] for k in ('id', 'title', 'type', 'areas', 'sources', 'updated')} | {
|
|
631
|
+
'status': '' if gap else m['status'],
|
|
632
|
+
'tags': m.get('tags', []), 'path': note['path'], 'body': note['body'],
|
|
633
|
+
'summary': m.get('summary', ''), 'stale': note['stale'],
|
|
634
|
+
'kind': m.get('kind', ''), 'recheck': m.get('recheck', ''), 'recheckedBy': m.get('rechecked_by', ''),
|
|
635
|
+
'recheckedOn': m.get('rechecked_on', ''), 'decision': m.get('decision', ''), 'decidedBy': m.get('decided_by', ''),
|
|
636
|
+
'decidedOn': m.get('decided_on', ''), 'priority': m.get('priority', ''), 'confidence': m.get('confidence', ''),
|
|
637
|
+
'duplicateOf': m.get('duplicate_of', ''), 'recheckOutdated': gap and (m['recheck'] == 'outdated' or recheck_marker(note)), 'state': note.get('state', ''), 'gap': m.get('gap', ''),
|
|
638
|
+
'layer': LAYERS[m['type']]}
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
def graph_of(notes, edges, warnings):
|
|
642
|
+
nodes = [node_of(note) for note in notes]
|
|
643
|
+
title = next((n['title'] for n in nodes if n['path'] == '00-index.md'), 'Project knowledge')
|
|
644
|
+
return {'title': title, 'generatedAt': dt.datetime.now(dt.timezone.utc).isoformat(), 'nodes': nodes,
|
|
645
|
+
'edges': [{'source': a, 'target': b} for a, b in edges], 'warnings': warnings}
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def viewer_html(payload):
|
|
649
|
+
template = (Path(__file__).resolve().parent.parent / 'assets/viewer.html').read_text(encoding='utf-8')
|
|
650
|
+
if template.count('__KNOWLEDGE_DATA__') != 1:
|
|
651
|
+
raise ValueError('viewer template must contain exactly one placeholder')
|
|
652
|
+
serialized = json.dumps(payload, ensure_ascii=False).replace('&', '\\u0026').replace('<', '\\u003c').replace('>', '\\u003e').replace('\u2028', '\\u2028').replace('\u2029', '\\u2029')
|
|
653
|
+
return template.replace('__KNOWLEDGE_DATA__', serialized)
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def find_vault(repo):
|
|
657
|
+
"""The vault of a repository when there is exactly one in the usual place."""
|
|
658
|
+
found = sorted(p.parent for p in repo.glob('knowledge/*/00-index.md'))
|
|
659
|
+
if (repo / 'knowledge/00-index.md').is_file():
|
|
660
|
+
found.append(repo / 'knowledge')
|
|
661
|
+
if len(found) != 1:
|
|
662
|
+
raise ValueError(('no vault found under knowledge/' if not found else
|
|
663
|
+
'several vaults found (' + ', '.join(p.relative_to(repo).as_posix() for p in found) + ')') + '; pass --vault')
|
|
664
|
+
return found[0]
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
HISTORY_COMMITS = 30 # commits shown under Recent changes
|
|
668
|
+
HISTORY_PAGES = 80 # pages listed for one commit
|
|
669
|
+
_history_cache = {}
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
def page_pathspec(repo, vault):
|
|
673
|
+
"""Git pathspec for the vault's own pages (no generated files), and the prefix to strip from the paths Git prints."""
|
|
674
|
+
rel = Path(os.path.relpath(vault, repo)).as_posix()
|
|
675
|
+
if rel == '..' or rel.startswith('../'):
|
|
676
|
+
return None, None
|
|
677
|
+
prefix = '' if rel == '.' else rel + '/'
|
|
678
|
+
return [f':(glob){prefix}**/*.md', f':(exclude,glob){prefix}AS-BUILT.md', f':(exclude,glob){prefix}GAPS.md',
|
|
679
|
+
f':(exclude,glob){prefix}.knowledge/**'], prefix
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def front_value(line):
|
|
683
|
+
"""The string a `key: "value"` frontmatter line holds, as the vault writes it (a JSON string)."""
|
|
684
|
+
raw = line.split(':', 1)[1].strip()
|
|
685
|
+
try:
|
|
686
|
+
value = json.loads(raw)
|
|
687
|
+
except ValueError:
|
|
688
|
+
value = raw
|
|
689
|
+
return value if isinstance(value, str) else ''
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def vault_history(repo, vault):
|
|
693
|
+
"""Recent commits that touched vault pages, newest first: sha, date, subject and what happened to each page.
|
|
694
|
+
|
|
695
|
+
A page entry carries `was` and `now` when its summary (or, failing that, its title) changed in that commit.
|
|
696
|
+
Empty outside a Git work tree. Cached per HEAD, because every page load asks for it.
|
|
697
|
+
"""
|
|
698
|
+
spec, prefix = page_pathspec(repo, vault)
|
|
699
|
+
if spec is None:
|
|
700
|
+
return []
|
|
701
|
+
head = subprocess.run(['git', '-C', str(repo), 'rev-parse', 'HEAD'], capture_output=True)
|
|
702
|
+
if head.returncode:
|
|
703
|
+
return []
|
|
704
|
+
key = (str(repo), str(vault), head.stdout.strip())
|
|
705
|
+
if key in _history_cache:
|
|
706
|
+
return _history_cache[key]
|
|
707
|
+
# Git prints paths from the top of the work tree, which may be above the repository folder given here.
|
|
708
|
+
top = subprocess.run(['git', '-C', str(repo), 'rev-parse', '--show-prefix'], capture_output=True).stdout.decode(errors='replace').strip()
|
|
709
|
+
prefix = top + prefix
|
|
710
|
+
# A marker nobody could have written into a commit message or a page beforehand separates the commits.
|
|
711
|
+
marker = '\x01' + os.urandom(12).hex() + '\x01'
|
|
712
|
+
try:
|
|
713
|
+
log = subprocess.run(['git', '-C', str(repo), '-c', 'core.quotepath=off', 'log', f'-n{HISTORY_COMMITS}', '--date=short', '--no-color',
|
|
714
|
+
'--no-renames', '-p', '-U0', f'--format={marker}%H%x02%ad%x02%s', '--', *spec],
|
|
715
|
+
capture_output=True, timeout=20)
|
|
716
|
+
except subprocess.TimeoutExpired:
|
|
717
|
+
return []
|
|
718
|
+
if log.returncode:
|
|
719
|
+
return []
|
|
720
|
+
commits = []
|
|
721
|
+
for chunk in log.stdout.decode('utf-8', errors='replace').split(marker)[1:]:
|
|
722
|
+
header, _, body = chunk.partition('\n')
|
|
723
|
+
sha, date, subject = (header.split('\x02', 2) + ['', ''])[:3]
|
|
724
|
+
subject = CONTROL.sub('', subject)
|
|
725
|
+
pages, page = [], None
|
|
726
|
+
for line in body.split('\n'):
|
|
727
|
+
if line.startswith('diff --git '):
|
|
728
|
+
name = line.rsplit(' b/', 1)[-1]
|
|
729
|
+
page = {'path': name[len(prefix):], 'status': 'changed', 'fields': {}} if name.startswith(prefix) else None
|
|
730
|
+
if page:
|
|
731
|
+
pages.append(page)
|
|
732
|
+
elif page is None:
|
|
733
|
+
continue
|
|
734
|
+
elif line.startswith('new file mode'):
|
|
735
|
+
page['status'] = 'added'
|
|
736
|
+
elif line.startswith('deleted file mode'):
|
|
737
|
+
page['status'] = 'removed'
|
|
738
|
+
elif line[:1] in '+-' and line[1:].startswith(('summary:', 'title:')):
|
|
739
|
+
field = line[1:].split(':', 1)[0]
|
|
740
|
+
page['fields'].setdefault(field, {})['now' if line[0] == '+' else 'was'] = front_value(line[1:])
|
|
741
|
+
for page in pages:
|
|
742
|
+
fields = page.pop('fields')
|
|
743
|
+
change = next((fields[f] for f in ('summary', 'title') if fields.get(f, {}).get('was') != fields.get(f, {}).get('now')), {})
|
|
744
|
+
if page['status'] == 'changed' and change.get('was') and change.get('now'):
|
|
745
|
+
page['was'], page['now'] = change['was'], change['now']
|
|
746
|
+
if pages:
|
|
747
|
+
commits.append({'sha': sha, 'date': date, 'subject': subject, 'pages': pages[:HISTORY_PAGES], 'more': max(0, len(pages) - HISTORY_PAGES)})
|
|
748
|
+
_history_cache.clear()
|
|
749
|
+
_history_cache[key] = commits
|
|
750
|
+
return commits
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
def vault_pending(repo, vault):
|
|
754
|
+
"""Vault pages with uncommitted changes: [{path, status}], empty outside Git."""
|
|
755
|
+
spec, prefix = page_pathspec(repo, vault)
|
|
756
|
+
if spec is None:
|
|
757
|
+
return []
|
|
758
|
+
status = subprocess.run(['git', '-C', str(repo), '-c', 'core.quotepath=off', 'status', '--porcelain=v1', '-z', '--untracked-files=all', '--no-renames', '--', *spec],
|
|
759
|
+
capture_output=True)
|
|
760
|
+
if status.returncode:
|
|
761
|
+
return []
|
|
762
|
+
top = subprocess.run(['git', '-C', str(repo), 'rev-parse', '--show-prefix'], capture_output=True).stdout.decode(errors='replace').strip()
|
|
763
|
+
pending = []
|
|
764
|
+
for item in status.stdout.split(b'\0'):
|
|
765
|
+
text = os.fsdecode(item)
|
|
766
|
+
name = text[3:]
|
|
767
|
+
name = name[len(top):] if name.startswith(top) else name
|
|
768
|
+
if len(text) > 3 and name.startswith(prefix):
|
|
769
|
+
code = text[:2]
|
|
770
|
+
pending.append({'path': name[len(prefix):],
|
|
771
|
+
'status': 'added' if '?' in code or 'A' in code else 'removed' if 'D' in code else 'changed'})
|
|
772
|
+
return sorted(pending, key=lambda p: p['path'])
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
def live_page(repo, vault):
|
|
776
|
+
"""The viewer for the vault as it is on disk right now, or a page listing what stops it from loading."""
|
|
777
|
+
try:
|
|
778
|
+
notes, edges, errors, warnings, _ = load_vault(repo, vault)
|
|
779
|
+
except (OSError, ValueError) as exc:
|
|
780
|
+
notes, errors = [], [str(exc)]
|
|
781
|
+
if errors:
|
|
782
|
+
items = ''.join(f'<li>{html.escape(message)}</li>' for message in errors)
|
|
783
|
+
return ('<!doctype html><meta charset="utf-8"><title>Spec cannot be shown</title>'
|
|
784
|
+
'<body style="font:15px/1.5 system-ui,sans-serif;max-width:860px;margin:48px auto;padding:0 20px">'
|
|
785
|
+
f'<h1>The spec has {len(errors)} error{"s" if len(errors) != 1 else ""}</h1>'
|
|
786
|
+
'<p>Fix the pages below (the <code>check</code> command prints the same list), then reload.</p>'
|
|
787
|
+
f'<ul>{items}</ul></body>')
|
|
788
|
+
files, _ = cited_files(repo, vault, notes)
|
|
789
|
+
return viewer_html(graph_of(notes, edges, warnings) | {
|
|
790
|
+
'live': True, 'vaultPath': Path(os.path.relpath(vault, repo)).as_posix(), 'files': files,
|
|
791
|
+
'history': vault_history(repo, vault), 'pending': vault_pending(repo, vault)})
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
def make_server(repo, vault, port):
|
|
795
|
+
"""A read-only viewer on this machine only. Each page load reads the vault and the cited files again."""
|
|
796
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
797
|
+
|
|
798
|
+
class Handler(BaseHTTPRequestHandler):
|
|
799
|
+
def do_GET(self):
|
|
800
|
+
# The page carries source code, so answer only when addressed as this machine (no DNS rebinding).
|
|
801
|
+
host = (self.headers.get('Host') or '').rsplit(':', 1)[0].strip('[]')
|
|
802
|
+
if host not in ('127.0.0.1', 'localhost', '::1'):
|
|
803
|
+
return self.send_error(403)
|
|
804
|
+
if urlsplit(self.path).path not in ('/', '/index.html'):
|
|
805
|
+
return self.send_error(404)
|
|
806
|
+
body = live_page(repo, vault).encode('utf-8')
|
|
807
|
+
self.send_response(200)
|
|
808
|
+
self.send_header('Content-Type', 'text/html; charset=utf-8')
|
|
809
|
+
self.send_header('Content-Length', str(len(body)))
|
|
810
|
+
self.send_header('Cache-Control', 'no-store')
|
|
811
|
+
self.send_header('Content-Security-Policy', "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'")
|
|
812
|
+
self.end_headers()
|
|
813
|
+
self.wfile.write(body)
|
|
814
|
+
|
|
815
|
+
def log_message(self, *args):
|
|
816
|
+
pass
|
|
817
|
+
|
|
818
|
+
return ThreadingHTTPServer(('127.0.0.1', port), Handler)
|
|
819
|
+
|
|
820
|
+
|
|
821
|
+
def main(argv=None):
|
|
822
|
+
parser = argparse.ArgumentParser()
|
|
823
|
+
parser.add_argument('command', choices=['check', 'snapshot', 'serve'])
|
|
824
|
+
parser.add_argument('--repo', required=True, type=Path)
|
|
825
|
+
parser.add_argument('--vault', type=Path, help='default: the single vault under knowledge/ in the repository')
|
|
826
|
+
parser.add_argument('--strict', action='store_true', help='fail check when freshness warnings exist')
|
|
827
|
+
parser.add_argument('--port', type=int, default=8757, help='serve: port on 127.0.0.1 (another free port is used if it is taken)')
|
|
828
|
+
parser.add_argument('--open', action='store_true', help='serve: open the page in the default browser')
|
|
829
|
+
parser.add_argument('--ignore-dirty', action='store_true',
|
|
830
|
+
help='check: do not count freshness warnings for sources with uncommitted changes (needs a Git work tree)')
|
|
831
|
+
group = parser.add_mutually_exclusive_group()
|
|
832
|
+
group.add_argument('--all', action='store_true')
|
|
833
|
+
group.add_argument('--notes', nargs='+')
|
|
834
|
+
args = parser.parse_args(argv)
|
|
835
|
+
if args.strict and args.command != 'check':
|
|
836
|
+
parser.error('--strict is only for check')
|
|
837
|
+
if args.ignore_dirty and args.command != 'check':
|
|
838
|
+
parser.error('--ignore-dirty is only for check')
|
|
839
|
+
if args.open and args.command != 'serve':
|
|
840
|
+
parser.error('--open is only for serve')
|
|
841
|
+
if args.command == 'snapshot' and not (args.all or args.notes):
|
|
842
|
+
parser.error('snapshot requires --all or --notes')
|
|
843
|
+
if args.command != 'snapshot' and (args.all or args.notes):
|
|
844
|
+
parser.error('--all and --notes are only for snapshot')
|
|
845
|
+
repo = args.repo.resolve()
|
|
846
|
+
if not repo.is_dir():
|
|
847
|
+
parser.error('repo must be an existing directory')
|
|
848
|
+
vault = args.vault.resolve() if args.vault else find_vault(repo)
|
|
849
|
+
if not vault.is_dir():
|
|
850
|
+
parser.error('repo and vault must be existing directories')
|
|
851
|
+
if args.command == 'serve':
|
|
852
|
+
try:
|
|
853
|
+
server = make_server(repo, vault, args.port)
|
|
854
|
+
except OSError: # the usual port is taken
|
|
855
|
+
server = make_server(repo, vault, 0)
|
|
856
|
+
url = f'http://127.0.0.1:{server.server_address[1]}/'
|
|
857
|
+
print(f'serve: {url} shows {vault} as it is on disk; reload the page after a change, Ctrl+C to stop', flush=True)
|
|
858
|
+
if args.open:
|
|
859
|
+
import webbrowser
|
|
860
|
+
webbrowser.open(url)
|
|
861
|
+
try:
|
|
862
|
+
server.serve_forever()
|
|
863
|
+
except KeyboardInterrupt:
|
|
864
|
+
pass
|
|
865
|
+
finally:
|
|
866
|
+
server.server_close()
|
|
867
|
+
return 0
|
|
868
|
+
dirty = None
|
|
869
|
+
if args.ignore_dirty:
|
|
870
|
+
dirty = uncommitted_paths(repo)
|
|
871
|
+
if dirty is None:
|
|
872
|
+
parser.error('--ignore-dirty needs --repo to be a Git work tree')
|
|
873
|
+
report = {}
|
|
874
|
+
notes, edges, errors, warnings, baseline = load_vault(repo, vault, args.command != 'snapshot', dirty, report)
|
|
875
|
+
ignored = []
|
|
876
|
+
if args.command == 'snapshot' and args.notes:
|
|
877
|
+
# Only the selected pages must be valid; errors elsewhere are counted, not blocking.
|
|
878
|
+
wanted = set(args.notes)
|
|
879
|
+
by_id = {n['meta']['id']: n['path'] for n in notes}
|
|
880
|
+
by_id.update({ident: page for page, ident in report['failed'].items() if ident})
|
|
881
|
+
selected = {by_id[i] for i in wanted if i in by_id}
|
|
882
|
+
blocking = []
|
|
883
|
+
for message in errors:
|
|
884
|
+
page = error_page(message)
|
|
885
|
+
if page is None or page in selected or any(f'duplicate id {i}' in message for i in wanted):
|
|
886
|
+
blocking.append(message)
|
|
887
|
+
else:
|
|
888
|
+
ignored.append(message)
|
|
889
|
+
# A selected page that leans on a page which failed validation cannot be accepted yet.
|
|
890
|
+
blocking += [m for m in report['cascade'] if error_page(m) in selected]
|
|
891
|
+
errors = blocking
|
|
892
|
+
missing = wanted - by_id.keys()
|
|
893
|
+
if missing:
|
|
894
|
+
errors.append('Unknown note IDs: ' + ', '.join(sorted(missing)))
|
|
895
|
+
for message in errors:
|
|
896
|
+
print('ERROR:', message, file=sys.stderr)
|
|
897
|
+
for message in warnings:
|
|
898
|
+
print('WARNING:', message, file=sys.stderr)
|
|
899
|
+
counted = [w for w in warnings if not w.endswith(UNCOUNTED)]
|
|
900
|
+
if errors:
|
|
901
|
+
pages = len({error_page(m) for m in errors if error_page(m)})
|
|
902
|
+
print(f'{args.command}: failed, {len(errors)} errors on {pages} pages', file=sys.stderr)
|
|
903
|
+
if errors or (args.strict and counted):
|
|
904
|
+
return 1
|
|
905
|
+
if ignored:
|
|
906
|
+
print(f'{args.command}: {len(ignored)} errors on {len({error_page(m) for m in ignored if error_page(m)})} other pages do not block this snapshot (run check to see them)', file=sys.stderr)
|
|
907
|
+
if args.command == 'snapshot':
|
|
908
|
+
for note in notes:
|
|
909
|
+
meta = note['meta']
|
|
910
|
+
if args.all or meta['id'] in args.notes:
|
|
911
|
+
hashes = {}
|
|
912
|
+
for source in meta['sources']:
|
|
913
|
+
path = safe_path(repo, source)
|
|
914
|
+
if path.is_file():
|
|
915
|
+
hashes[source] = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
916
|
+
baseline[meta['id']] = hashes
|
|
917
|
+
dest = output_path(vault, '.knowledge/sources.json')
|
|
918
|
+
write_json(dest, dict(sorted(baseline.items())))
|
|
919
|
+
uncounted = len(warnings) - len(counted)
|
|
920
|
+
print(f'{args.command}: {len(notes)} notes, {len(edges)} links, {len(warnings)} warnings' + (f' ({uncounted} uncommitted, not counted)' if uncounted else ''))
|
|
921
|
+
return 0
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
if __name__ == '__main__':
|
|
925
|
+
try:
|
|
926
|
+
sys.exit(main())
|
|
927
|
+
except (OSError, ValueError) as exc:
|
|
928
|
+
print(f'ERROR: {exc}', file=sys.stderr)
|
|
929
|
+
sys.exit(1)
|