projectkeeper 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +56 -0
- package/LICENSE +21 -0
- package/README.md +283 -2
- package/README.zh-CN.md +285 -0
- package/THIRD_PARTY_NOTICES.md +53 -0
- package/bin/pk.mjs +6 -0
- package/bin/postinstall.mjs +5 -0
- package/bin/strip-types.mjs +28 -0
- package/package.json +68 -4
- package/patches/README.md +111 -0
- package/patches/pi-ai-0.85.0.ts +122 -0
- package/patches/pi-ai-0.87.1.ts +122 -0
- package/scripts/apply-pi-streaming-patch.ts +86 -0
- package/skills/clerk/coverage/SKILL.md +35 -0
- package/skills/clerk/cross-check/SKILL.md +118 -0
- package/skills/clerk/deepen-dispatch/SKILL.md +69 -0
- package/skills/clerk/lane-follow-up/SKILL.md +94 -0
- package/skills/clerk/lane-plan/SKILL.md +107 -0
- package/skills/clerk/lane-slot/SKILL.md +113 -0
- package/skills/clerk/lane-topic/SKILL.md +116 -0
- package/skills/clerk/orientation/SKILL.md +160 -0
- package/skills/clerk/reconcile/SKILL.md +77 -0
- package/skills/clerk/skeleton-dispatch/SKILL.md +43 -0
- package/skills/clerk/spot-check/SKILL.md +52 -0
- package/skills/clerk/synthesis/SKILL.md +65 -0
- package/skills/table-follow-up/SKILL.md +42 -0
- package/src/cli.ts +319 -0
- package/src/codemap/README.md +13 -0
- package/src/codemap/engines.ts +102 -0
- package/src/codemap/facts.ts +504 -0
- package/src/context/agent-entry.ts +232 -0
- package/src/context/ask.ts +97 -0
- package/src/context/assemble.ts +1058 -0
- package/src/context/code-source.ts +103 -0
- package/src/context/k-briefs.ts +114 -0
- package/src/context/lineage.ts +100 -0
- package/src/context/object-brief.ts +300 -0
- package/src/context/owner-words.ts +191 -0
- package/src/context/pack-fixture.ts +342 -0
- package/src/context/project-rules.ts +160 -0
- package/src/intake/intake.ts +392 -0
- package/src/intake/material-rules.ts +166 -0
- package/src/intake/removal.ts +316 -0
- package/src/intake/skipped.ts +72 -0
- package/src/keeper/adjustment.ts +565 -0
- package/src/keeper/bounds/boundary.ts +509 -0
- package/src/keeper/bounds/command.ts +1665 -0
- package/src/keeper/bounds/paths.ts +164 -0
- package/src/keeper/bounds/reads.ts +718 -0
- package/src/keeper/bounds/shell-write-guard.ts +521 -0
- package/src/keeper/capabilities.ts +136 -0
- package/src/keeper/clerk-steps.ts +151 -0
- package/src/keeper/clerk-tools.ts +1424 -0
- package/src/keeper/conversation.ts +253 -0
- package/src/keeper/evidence.ts +662 -0
- package/src/keeper/fake-provider.ts +132 -0
- package/src/keeper/held.ts +15 -0
- package/src/keeper/history-tools.ts +163 -0
- package/src/keeper/jobs/answering.ts +53 -0
- package/src/keeper/key-check.ts +105 -0
- package/src/keeper/ledger-tools.ts +295 -0
- package/src/keeper/link-check.ts +185 -0
- package/src/keeper/merge.ts +260 -0
- package/src/keeper/note-origin.ts +56 -0
- package/src/keeper/numbers-check.ts +102 -0
- package/src/keeper/open-in-pi.ts +57 -0
- package/src/keeper/organize/absence.ts +266 -0
- package/src/keeper/organize/carried-on.ts +214 -0
- package/src/keeper/organize/clerk-coverage.ts +293 -0
- package/src/keeper/organize/clerk-prompts.ts +132 -0
- package/src/keeper/organize/clerk.ts +1395 -0
- package/src/keeper/organize/coverage-gate.ts +19 -0
- package/src/keeper/organize/coverage-tools.ts +765 -0
- package/src/keeper/organize/entries.ts +153 -0
- package/src/keeper/organize/entry-gate.ts +289 -0
- package/src/keeper/organize/fill-tools.ts +718 -0
- package/src/keeper/organize/follow-up.ts +280 -0
- package/src/keeper/organize/generation-check.ts +504 -0
- package/src/keeper/organize/graph.ts +230 -0
- package/src/keeper/organize/lane-tools.ts +289 -0
- package/src/keeper/organize/looked-tools.ts +54 -0
- package/src/keeper/organize/materials.ts +243 -0
- package/src/keeper/organize/owner-lines.ts +493 -0
- package/src/keeper/organize/place-tools.ts +216 -0
- package/src/keeper/organize/placement-inference.ts +1045 -0
- package/src/keeper/organize/placement-time.ts +320 -0
- package/src/keeper/organize/placing.ts +821 -0
- package/src/keeper/organize/positioning.ts +33 -0
- package/src/keeper/organize/readiness.ts +47 -0
- package/src/keeper/organize/reading.ts +401 -0
- package/src/keeper/organize/round-blocks.ts +65 -0
- package/src/keeper/organize/round-news.ts +164 -0
- package/src/keeper/organize/round-open.ts +586 -0
- package/src/keeper/organize/schedule-state.ts +39 -0
- package/src/keeper/organize/schedule.ts +137 -0
- package/src/keeper/organize/service.ts +409 -0
- package/src/keeper/organize/skills.ts +116 -0
- package/src/keeper/organize/slots.ts +85 -0
- package/src/keeper/organize/stage-gate.ts +175 -0
- package/src/keeper/organize/stage-tools.ts +471 -0
- package/src/keeper/organize/standing-notes.ts +141 -0
- package/src/keeper/organize/table-follow-up.ts +330 -0
- package/src/keeper/organize/takeover-clerk.ts +832 -0
- package/src/keeper/organize/takeover-state.ts +103 -0
- package/src/keeper/organize/takeover.ts +216 -0
- package/src/keeper/organize/update-pending.ts +535 -0
- package/src/keeper/organize/workbench-placement.ts +92 -0
- package/src/keeper/own-keys.ts +114 -0
- package/src/keeper/owner-text.ts +165 -0
- package/src/keeper/project-folder.ts +334 -0
- package/src/keeper/prompts.ts +101 -0
- package/src/keeper/references.ts +177 -0
- package/src/keeper/relook/compose.ts +120 -0
- package/src/keeper/relook/prompts.ts +36 -0
- package/src/keeper/roles.ts +259 -0
- package/src/keeper/round-tools.ts +227 -0
- package/src/keeper/route.ts +148 -0
- package/src/keeper/rules.ts +97 -0
- package/src/keeper/runtime.ts +1858 -0
- package/src/keeper/scope-tools.ts +194 -0
- package/src/keeper/sendback-text.ts +83 -0
- package/src/keeper/step-timing.ts +207 -0
- package/src/keeper/testing/clerk-script.ts +212 -0
- package/src/keeper/testing/streaming-arguments.ts +57 -0
- package/src/keeper/tools.ts +2340 -0
- package/src/keeper/utterance-tools.ts +64 -0
- package/src/keeper/verbatim.ts +397 -0
- package/src/ledger/adapters.ts +186 -0
- package/src/ledger/arrangements.ts +257 -0
- package/src/ledger/blame.ts +58 -0
- package/src/ledger/code-engine-child.ts +69 -0
- package/src/ledger/code-engine.ts +548 -0
- package/src/ledger/code.ts +607 -0
- package/src/ledger/docs.ts +294 -0
- package/src/ledger/git-read.ts +321 -0
- package/src/ledger/index.ts +1850 -0
- package/src/ledger/item-range.ts +16 -0
- package/src/ledger/lines.ts +328 -0
- package/src/ledger/numbering.ts +269 -0
- package/src/ledger/ranges.ts +74 -0
- package/src/ledger/rebuild-worker.ts +15 -0
- package/src/ledger/rebuild.ts +387 -0
- package/src/ledger/repo-scan.ts +181 -0
- package/src/ledger/schema.ts +450 -0
- package/src/ledger/sessions.ts +134 -0
- package/src/ledger/text-scan.ts +466 -0
- package/src/ledger/time.ts +185 -0
- package/src/ledger/ts-symbols.ts +180 -0
- package/src/ledger/views.ts +377 -0
- package/src/mcp/server.ts +127 -0
- package/src/mcp/tools.ts +187 -0
- package/src/model/ids.ts +23 -0
- package/src/model/k-types.ts +912 -0
- package/src/model/time.ts +85 -0
- package/src/model/types.ts +1382 -0
- package/src/model/views-k.ts +707 -0
- package/src/model/vocab.ts +269 -0
- package/src/process/analysis.ts +79 -0
- package/src/process/breakpoint-candidates.ts +383 -0
- package/src/process/breakpoints.ts +746 -0
- package/src/process/claim-check.ts +295 -0
- package/src/process/delivery-links.ts +52 -0
- package/src/process/facts.ts +487 -0
- package/src/process/generations.ts +138 -0
- package/src/process/index.ts +243 -0
- package/src/process/integration.ts +128 -0
- package/src/process/placement.ts +107 -0
- package/src/process/plan.ts +276 -0
- package/src/process/sendbacks.ts +244 -0
- package/src/process/text.ts +185 -0
- package/src/process/units.ts +543 -0
- package/src/process/unnumbered.ts +431 -0
- package/src/process/work.ts +884 -0
- package/src/scope/candidates.ts +86 -0
- package/src/scope/decisions.ts +229 -0
- package/src/scope/discover.ts +741 -0
- package/src/scope/ignore.ts +126 -0
- package/src/scope/ignored-place.ts +93 -0
- package/src/scope/skip.ts +161 -0
- package/src/scope/toolchain.ts +180 -0
- package/src/scope/worktree.ts +187 -0
- package/src/server/api.ts +549 -0
- package/src/server/app.ts +595 -0
- package/src/server/drafts-api.ts +170 -0
- package/src/server/folders-api.ts +243 -0
- package/src/server/graph-view.ts +639 -0
- package/src/server/http.ts +224 -0
- package/src/server/k-agent-api.ts +56 -0
- package/src/server/k-rounds-d99.ts +91 -0
- package/src/server/k-views.ts +405 -0
- package/src/server/keeper-page.ts +446 -0
- package/src/server/keys-api.ts +239 -0
- package/src/server/project-folder-api.ts +104 -0
- package/src/server/vendor.ts +14 -0
- package/src/server/workbench-content.ts +561 -0
- package/src/sources/anchor.ts +129 -0
- package/src/sources/files.ts +267 -0
- package/src/sources/gitobs.ts +41 -0
- package/src/sources/history.ts +231 -0
- package/src/sources/scan-worker.ts +10 -0
- package/src/sources/sessions/locate.ts +292 -0
- package/src/sources/sessions/read.ts +346 -0
- package/src/sources/sessions/scope.ts +42 -0
- package/src/sources/sessions/utterances.ts +223 -0
- package/src/sources/watch.ts +215 -0
- package/src/store/json-file.ts +50 -0
- package/src/store/paths.ts +39 -0
- package/src/store/project-store.ts +295 -0
- package/src/store/versions.ts +172 -0
- package/src/store/workspace.ts +216 -0
- package/src/table/check.ts +333 -0
- package/src/table/follow-up-tools.ts +30 -0
- package/src/table/follow-up.ts +344 -0
- package/src/table/import.ts +610 -0
- package/src/table/project-home.ts +61 -0
- package/src/table/sessions.ts +168 -0
- package/src/table/takeover.ts +483 -0
- package/src/util/git-check.ts +29 -0
- package/src/util/git.ts +411 -0
- package/src/util/own-ts.ts +15 -0
- package/src/util/paths.ts +225 -0
- package/template/FILLING-IN.md +148 -0
- package/template/examples/papertrail.json +747 -0
- package/template/projectkeeper.schema.json +1001 -0
- package/ui/app.js +1135 -0
- package/ui/failures.js +82 -0
- package/ui/flyout.js +120 -0
- package/ui/folder-chooser.js +264 -0
- package/ui/glossary.js +156 -0
- package/ui/graph-fit.js +157 -0
- package/ui/graph-material.js +61 -0
- package/ui/graph-palette.js +109 -0
- package/ui/graph-tools.js +62 -0
- package/ui/graph.js +1712 -0
- package/ui/index.html +31 -0
- package/ui/k/code-view.js +230 -0
- package/ui/k/draft-view.js +93 -0
- package/ui/k/k-views.css +224 -0
- package/ui/k/round-tree.js +450 -0
- package/ui/k/scope-k.js +159 -0
- package/ui/k-data.js +118 -0
- package/ui/k-fold.js +106 -0
- package/ui/k-process.css +302 -0
- package/ui/k-process.js +934 -0
- package/ui/keeper-activity.js +175 -0
- package/ui/keeper-view.js +801 -0
- package/ui/keeper.css +101 -0
- package/ui/key-order.js +62 -0
- package/ui/list-fold.js +67 -0
- package/ui/markdown.d.ts +21 -0
- package/ui/markdown.js +472 -0
- package/ui/placement.d.ts +92 -0
- package/ui/placement.js +445 -0
- package/ui/popover-place.d.ts +16 -0
- package/ui/popover-place.js +89 -0
- package/ui/popover.js +198 -0
- package/ui/project-folder.js +37 -0
- package/ui/shell-state.js +40 -0
- package/ui/strip-rows.js +171 -0
- package/ui/styles.css +492 -0
- package/ui/theme-pick.js +36 -0
- package/ui/theme.js +74 -0
- package/ui/themes/LICENSE.md +13 -0
- package/ui/themes/README.md +44 -0
- package/ui/themes/TOKENS.md +291 -0
- package/ui/themes/a1/board.css +1557 -0
- package/ui/themes/a1/pk.css +392 -0
- package/ui/themes/b1/board.css +1163 -0
- package/ui/themes/b1/pk.css +218 -0
- package/ui/themes/d2/board.css +518 -0
- package/ui/themes/d2/pk.css +243 -0
- package/ui/themes/guard.css +73 -0
- package/ui/themes/i1/board.css +776 -0
- package/ui/themes/i1/pk.css +223 -0
- package/ui/themes/manifest.json +13 -0
- package/ui/views.js +2796 -0
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Source identity and credential redaction shared by every reader (files, sessions, git).
|
|
3
|
+
*/
|
|
4
|
+
import type { FileAnchor, SessionAnchor, Source, SourceAnchor } from '../model/types.ts';
|
|
5
|
+
import { fingerprint, stableId } from '../model/ids.ts';
|
|
6
|
+
import { pathKey } from '../util/paths.ts';
|
|
7
|
+
|
|
8
|
+
export function fileSourceId(path: string, headingPath: readonly string[]): string {
|
|
9
|
+
return stableId('src', 'file', pathKey(path), headingPath.join(' / '));
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function sessionSourceId(host: string, sessionId: string, messageStart: number): string {
|
|
13
|
+
return stableId('src', 'session', host, sessionId, messageStart);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function commitSourceId(repo: string, commit: string): string {
|
|
17
|
+
return stableId('src', 'commit', pathKey(repo), commit);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** A file as it was at a commit (§1.2 `History only`): one source per repository, commit and path. */
|
|
21
|
+
export function revisionSourceId(repo: string, commit: string, path: string): string {
|
|
22
|
+
return stableId('src', 'revision', pathKey(repo), commit, path.split('\\').join('/'));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function anchorId(anchor: SourceAnchor): string {
|
|
26
|
+
switch (anchor.kind) {
|
|
27
|
+
case 'file': return fileSourceId(anchor.path, anchor.headingPath);
|
|
28
|
+
case 'session': return sessionSourceId(anchor.host, anchor.sessionId, anchor.messageStart);
|
|
29
|
+
case 'commit': return commitSourceId(anchor.repo, anchor.commit);
|
|
30
|
+
case 'command': return stableId('src', 'command', anchor.cwd, anchor.command, anchor.ranAt);
|
|
31
|
+
case 'status': return stableId('src', 'status', anchor.description, anchor.at);
|
|
32
|
+
case 'revision': return revisionSourceId(anchor.repo, anchor.commit, anchor.path);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Human-readable locator, e.g. `design/DECISIONS.md › 一 · 已定 › D14 (L102–L107)`. */
|
|
37
|
+
export function anchorLabel(anchor: SourceAnchor): string {
|
|
38
|
+
switch (anchor.kind) {
|
|
39
|
+
case 'file': {
|
|
40
|
+
const heading = anchor.headingPath.length ? ` › ${anchor.headingPath.join(' › ')}` : '';
|
|
41
|
+
return `${anchor.path}${heading} (L${anchor.lineStart}–L${anchor.lineEnd})`;
|
|
42
|
+
}
|
|
43
|
+
case 'session': return `${anchor.host} session ${anchor.sessionId.slice(0, 8)} · messages ${anchor.messageStart}–${anchor.messageEnd}${anchor.at ? ` · ${anchor.at}` : ''}`;
|
|
44
|
+
case 'commit': return `${anchor.repo} @ ${anchor.commit.slice(0, 10)}`;
|
|
45
|
+
case 'command': return `${anchor.command} (in ${anchor.cwd}, ${anchor.ranAt})`;
|
|
46
|
+
case 'status': return `${anchor.description} (${anchor.at})`;
|
|
47
|
+
case 'revision': return `${anchor.path} @ ${anchor.commit.slice(0, 10)} in ${anchor.repo} (version history)`;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const CREDENTIAL_PATTERNS: readonly RegExp[] = [
|
|
52
|
+
/\b(sk-[A-Za-z0-9_-]{16,})/g, // OpenAI-style keys
|
|
53
|
+
/\b(sk-ant-[A-Za-z0-9_-]{16,})/g,
|
|
54
|
+
/\b(ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,})/g, // GitHub tokens
|
|
55
|
+
/\b(AKIA[0-9A-Z]{16})\b/g, // AWS access key id
|
|
56
|
+
/\b(xox[baprs]-[A-Za-z0-9-]{10,})/g, // Slack
|
|
57
|
+
/\b(AIza[0-9A-Za-z_-]{30,})/g, // Google API key
|
|
58
|
+
/((?:api[_-]?key|apikey|secret|token|password|passwd|authorization)\s*[:=]\s*["']?)([A-Za-z0-9_\-./+=]{12,})/gi,
|
|
59
|
+
/(Bearer\s+)([A-Za-z0-9_\-.=]{20,})/g,
|
|
60
|
+
/\b(eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})/g, // JWT
|
|
61
|
+
];
|
|
62
|
+
|
|
63
|
+
export const REDACTED = '[credential redacted]';
|
|
64
|
+
|
|
65
|
+
/** Replace credential values with a marker; the fact that a credential is there stays visible (§3.1). */
|
|
66
|
+
export function redactCredentials(text: string): { text: string; found: boolean } {
|
|
67
|
+
let found = false;
|
|
68
|
+
let out = text;
|
|
69
|
+
for (const pattern of CREDENTIAL_PATTERNS) {
|
|
70
|
+
out = out.replace(pattern, (...args: unknown[]) => {
|
|
71
|
+
found = true;
|
|
72
|
+
// The replacer gets the match, the capture groups, then the offset and the whole input: only the groups count.
|
|
73
|
+
// Counting the input as a group kept every token-shaped credential (sk-…, ghp_…, AKIA…) in the text.
|
|
74
|
+
const groups = args.slice(1, args.findIndex((a, i) => i > 0 && typeof a === 'number'));
|
|
75
|
+
const captured = groups.filter((g): g is string => typeof g === 'string');
|
|
76
|
+
if (captured.length >= 2) return `${captured[0]}${REDACTED}`;
|
|
77
|
+
return REDACTED;
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
return { text: out, found };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function makeFileSource(input: {
|
|
84
|
+
projectId: string; path: string; headingPath: readonly string[]; lineStart: number; lineEnd: number;
|
|
85
|
+
excerpt: string; fileText: string; scopeItemId: string; title?: string; ids?: readonly string[];
|
|
86
|
+
commit?: string | null; readAt?: string; tableRow?: number;
|
|
87
|
+
}): Source {
|
|
88
|
+
const anchor: FileAnchor = {
|
|
89
|
+
kind: 'file', path: input.path, headingPath: input.headingPath, lineStart: input.lineStart,
|
|
90
|
+
lineEnd: input.lineEnd, ...(input.tableRow === undefined ? {} : { tableRow: input.tableRow }),
|
|
91
|
+
};
|
|
92
|
+
const redacted = redactCredentials(input.excerpt);
|
|
93
|
+
return {
|
|
94
|
+
id: fileSourceId(input.path, input.headingPath), projectId: input.projectId,
|
|
95
|
+
title: input.title ?? (input.headingPath.length ? input.headingPath[input.headingPath.length - 1]! : input.path.split(/[\\/]/).pop()!),
|
|
96
|
+
anchor, ids: input.ids ?? [],
|
|
97
|
+
version: { fingerprint: fingerprint(input.fileText), readAt: input.readAt ?? new Date().toISOString(), commit: input.commit ?? null },
|
|
98
|
+
excerpt: redacted.text, usedAs: null, usedAsBy: null, availability: null, movedTo: null,
|
|
99
|
+
scopeItemId: input.scopeItemId, hasCredential: redacted.found, bytes: Buffer.byteLength(input.excerpt, 'utf8'),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function makeSessionSource(input: {
|
|
104
|
+
projectId: string; host: SessionAnchor['host']; sessionId: string; file: string; cwd: string | null;
|
|
105
|
+
messageStart: number; messageEnd: number; at: string | null; excerpt: string; title: string;
|
|
106
|
+
scopeItemId: string; ids?: readonly string[]; readAt?: string;
|
|
107
|
+
}): Source {
|
|
108
|
+
const anchor: SessionAnchor = {
|
|
109
|
+
kind: 'session', host: input.host, sessionId: input.sessionId, file: input.file, cwd: input.cwd,
|
|
110
|
+
messageStart: input.messageStart, messageEnd: input.messageEnd, at: input.at,
|
|
111
|
+
};
|
|
112
|
+
const redacted = redactCredentials(input.excerpt);
|
|
113
|
+
return {
|
|
114
|
+
id: sessionSourceId(input.host, input.sessionId, input.messageStart), projectId: input.projectId,
|
|
115
|
+
title: input.title, anchor, ids: input.ids ?? [],
|
|
116
|
+
version: { fingerprint: fingerprint(input.excerpt), readAt: input.readAt ?? new Date().toISOString(), commit: null },
|
|
117
|
+
excerpt: redacted.text, usedAs: null, usedAsBy: null, availability: null, movedTo: null,
|
|
118
|
+
scopeItemId: input.scopeItemId, hasCredential: redacted.found, bytes: Buffer.byteLength(input.excerpt, 'utf8'),
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Numbers the material itself uses: DEC-020, TASK-7.2, CKC-01, REQ-3, #18, D14, PA-6 … */
|
|
123
|
+
export function extractIds(text: string): string[] {
|
|
124
|
+
const out = new Set<string>();
|
|
125
|
+
for (const m of text.matchAll(/\b([A-Z]{2,6}-\d{1,3}(?:\.\d{1,3})?)\b/g)) out.add(m[1]!);
|
|
126
|
+
for (const m of text.matchAll(/(?<![\w-])(#\d{1,5})\b/g)) out.add(m[1]!);
|
|
127
|
+
for (const m of text.matchAll(/(?<![A-Za-z])(D\d{1,2}|PA-\d{1,2}|U\d{1,2}|S\d{1,2}|G\d)(?![\w.])/g)) out.add(m[1]!);
|
|
128
|
+
return [...out].sort();
|
|
129
|
+
}
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read project files into sources (Spec §1.2). Markdown is split into sections by heading,
|
|
3
|
+
* each section a source with its heading path and line range; other text files become one
|
|
4
|
+
* source each with a head excerpt. Binary and oversized files are skipped and reported.
|
|
5
|
+
* Nothing is written to the project.
|
|
6
|
+
*
|
|
7
|
+
* What is read from where follows the scope (Spec §1.1; ../scope/skip.ts): the project's ignore rules leave content out
|
|
8
|
+
* (git decides), generated output is not read, third-party material gives only its documents (as `Reference only`),
|
|
9
|
+
* and a registered worktree gives only what it adds to the trunk (E60).
|
|
10
|
+
*/
|
|
11
|
+
import { readFileSync, readdirSync, statSync, type Dirent } from 'node:fs';
|
|
12
|
+
import { dirname, extname, join } from 'node:path';
|
|
13
|
+
import { Worker } from 'node:worker_threads';
|
|
14
|
+
import type { ScopeItem, Source } from '../model/types.ts';
|
|
15
|
+
import type { UsedAs } from '../model/vocab.ts';
|
|
16
|
+
import { fingerprint } from '../model/ids.ts';
|
|
17
|
+
import { isWithin, nameForm, normalizePath, pathKey, relativeDisplay } from '../util/paths.ts';
|
|
18
|
+
import { extractIds, makeFileSource } from './anchor.ts';
|
|
19
|
+
import { isDocumentPath, isReadRoot, isSkippedName, overridesIgnoreRules, readingOf, treatmentOf } from '../scope/skip.ts';
|
|
20
|
+
import { ignoredPredicate } from '../scope/ignore.ts';
|
|
21
|
+
import { measureWorktree } from '../scope/worktree.ts';
|
|
22
|
+
import { OWN_TS_WORKER_OPTIONS } from '../util/own-ts.ts';
|
|
23
|
+
|
|
24
|
+
export interface FileEntry {
|
|
25
|
+
readonly path: string;
|
|
26
|
+
readonly scopeItemId: string;
|
|
27
|
+
readonly bytes: number;
|
|
28
|
+
readonly mtimeMs: number;
|
|
29
|
+
/** The `Used as` the file's location sets (§1.1: the documents third-party material brings are `Reference only`). */
|
|
30
|
+
readonly usedAs?: { readonly value: UsedAs; readonly by: 'keeper' | 'owner'; readonly scopeItemId: string } | null;
|
|
31
|
+
}
|
|
32
|
+
export interface ScanResult {
|
|
33
|
+
readonly files: readonly FileEntry[];
|
|
34
|
+
readonly sources: readonly Source[];
|
|
35
|
+
readonly skipped: readonly { readonly path: string; readonly reason: string }[];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const TEXT_EXT = new Set(['.md', '.markdown', '.txt', '.json', '.jsonl', '.yaml', '.yml', '.toml', '.ts', '.tsx', '.js', '.mjs', '.cjs', '.py', '.rs', '.go', '.java', '.kt', '.cs', '.c', '.cc', '.cpp', '.h', '.hpp', '.css', '.html', '.sh', '.ps1', '.bat', '.sql', '.xml', '.ini', '.cfg', '.env.example', '.gitignore', '.gitattributes']);
|
|
39
|
+
const MAX_FILE_BYTES = 2_000_000;
|
|
40
|
+
const MAX_SECTION_CHARS = 14_000;
|
|
41
|
+
const CODE_HEAD_LINES = 120;
|
|
42
|
+
|
|
43
|
+
export function isTextFile(path: string): boolean {
|
|
44
|
+
const ext = extname(path).toLowerCase();
|
|
45
|
+
if (TEXT_EXT.has(ext)) return true;
|
|
46
|
+
const base = path.split(/[\\/]/).pop() ?? '';
|
|
47
|
+
return ['README', 'LICENSE', 'AGENTS.md', 'CLAUDE.md', 'Makefile'].includes(base);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function looksBinary(buf: Buffer): boolean {
|
|
51
|
+
const n = Math.min(buf.length, 4000);
|
|
52
|
+
for (let i = 0; i < n; i += 1) if (buf[i] === 0) return true;
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface MarkdownSection {
|
|
57
|
+
readonly headingPath: readonly string[];
|
|
58
|
+
readonly lineStart: number;
|
|
59
|
+
readonly lineEnd: number;
|
|
60
|
+
readonly text: string;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Split markdown into sections by heading; text before the first heading is its own section. */
|
|
64
|
+
export function splitMarkdown(text: string): MarkdownSection[] {
|
|
65
|
+
const lines = text.split(/\r?\n/);
|
|
66
|
+
const sections: MarkdownSection[] = [];
|
|
67
|
+
const stack: { level: number; title: string }[] = [];
|
|
68
|
+
let start = 0;
|
|
69
|
+
let inFence = false;
|
|
70
|
+
const flush = (end: number) => {
|
|
71
|
+
if (end < start) return;
|
|
72
|
+
const body = lines.slice(start, end + 1).join('\n');
|
|
73
|
+
if (body.trim().length === 0) return;
|
|
74
|
+
sections.push({ headingPath: stack.map((s) => s.title), lineStart: start + 1, lineEnd: end + 1, text: body });
|
|
75
|
+
};
|
|
76
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
77
|
+
const line = lines[i]!;
|
|
78
|
+
if (/^\s*(```|~~~)/.test(line)) inFence = !inFence;
|
|
79
|
+
if (inFence) continue;
|
|
80
|
+
const m = /^(#{1,6})\s+(.*)$/.exec(line);
|
|
81
|
+
if (!m) continue;
|
|
82
|
+
flush(i - 1);
|
|
83
|
+
const level = m[1]!.length;
|
|
84
|
+
while (stack.length && stack[stack.length - 1]!.level >= level) stack.pop();
|
|
85
|
+
stack.push({ level, title: m[2]!.trim().replace(/\s+#+\s*$/, '') });
|
|
86
|
+
start = i;
|
|
87
|
+
}
|
|
88
|
+
flush(lines.length - 1);
|
|
89
|
+
return sections;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Cut a long section into parts at paragraph boundaries so excerpts stay readable. */
|
|
93
|
+
function splitLong(section: MarkdownSection): MarkdownSection[] {
|
|
94
|
+
if (section.text.length <= MAX_SECTION_CHARS) return [section];
|
|
95
|
+
const parts: MarkdownSection[] = [];
|
|
96
|
+
const lines = section.text.split('\n');
|
|
97
|
+
let buf: string[] = [];
|
|
98
|
+
let bufStart = section.lineStart;
|
|
99
|
+
let part = 1;
|
|
100
|
+
const flush = (endLine: number) => {
|
|
101
|
+
if (buf.length === 0) return;
|
|
102
|
+
parts.push({ headingPath: [...section.headingPath, `part ${part}`], lineStart: bufStart, lineEnd: endLine, text: buf.join('\n') });
|
|
103
|
+
part += 1;
|
|
104
|
+
buf = [];
|
|
105
|
+
bufStart = endLine + 1;
|
|
106
|
+
};
|
|
107
|
+
let size = 0;
|
|
108
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
109
|
+
const line = lines[i]!;
|
|
110
|
+
if (size + line.length > MAX_SECTION_CHARS && buf.length > 0 && line.trim() === '') flush(section.lineStart + i - 1);
|
|
111
|
+
buf.push(line);
|
|
112
|
+
size = buf.reduce((n, l) => n + l.length + 1, 0);
|
|
113
|
+
}
|
|
114
|
+
flush(section.lineEnd);
|
|
115
|
+
return parts;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* The files of one scope item's own region: what the one skip list leaves out is not entered, links are not followed,
|
|
120
|
+
* and a location that is a scope item of its own (`separate`) is left to that item. `ignored` is the project's ignore
|
|
121
|
+
* rules for this root (relative paths), `keep` the ignored files the owner chose to include anyway.
|
|
122
|
+
*/
|
|
123
|
+
function walkRegion(root: string, scopeItemId: string, opts: {
|
|
124
|
+
readonly separate?: ReadonlySet<string>; readonly ignored?: ((rel: string) => boolean) | null; readonly keep?: ReadonlySet<string>;
|
|
125
|
+
readonly documentsOnly?: boolean; readonly usedAs?: FileEntry['usedAs'];
|
|
126
|
+
} = {}): FileEntry[] {
|
|
127
|
+
const out: FileEntry[] = [];
|
|
128
|
+
const walk = (dir: string, depth: number) => {
|
|
129
|
+
if (depth > 12) return;
|
|
130
|
+
let entries: Dirent[] = [];
|
|
131
|
+
try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
|
132
|
+
for (const e of entries) {
|
|
133
|
+
if (e.isSymbolicLink()) continue;
|
|
134
|
+
const name = nameForm(e.name); // as git names it, where the listing gives another Unicode form (macOS)
|
|
135
|
+
const full = join(dir, name);
|
|
136
|
+
if (opts.separate?.has(pathKey(full))) continue;
|
|
137
|
+
if (opts.ignored && opts.ignored(relativeDisplay(root, full)) && !opts.keep?.has(pathKey(full))) continue;
|
|
138
|
+
if (e.isDirectory()) { if (!isSkippedName(name)) walk(full, depth + 1); continue; }
|
|
139
|
+
if (!e.isFile()) continue;
|
|
140
|
+
if (opts.documentsOnly && !isDocumentPath(name)) continue;
|
|
141
|
+
let st;
|
|
142
|
+
try { st = statSync(full); } catch { continue; }
|
|
143
|
+
out.push({ path: normalizePath(full), scopeItemId, bytes: st.size, mtimeMs: st.mtimeMs, usedAs: opts.usedAs ?? null });
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
walk(root, 0);
|
|
147
|
+
return out;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** Every file under `root` except what the skip list leaves out and the `excluded` locations. */
|
|
151
|
+
export function listFiles(root: string, scopeItemId: string, excluded: readonly string[]): FileEntry[] {
|
|
152
|
+
return walkRegion(root, scopeItemId).filter((f) => !excluded.some((e) => isWithin(e, f.path)));
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Read one file into sources. `scope`, when given, sets the `Used as` the file's location implies (third-party
|
|
157
|
+
* documents are `Reference only`) for an entry that does not carry it already.
|
|
158
|
+
*/
|
|
159
|
+
export function readFileSources(projectId: string, entry: FileEntry, commit: string | null = null, scope: readonly ScopeItem[] | null = null): { sources: Source[]; skipped: string | null } {
|
|
160
|
+
if (entry.bytes > MAX_FILE_BYTES) return { sources: [], skipped: `larger than ${MAX_FILE_BYTES} bytes` };
|
|
161
|
+
if (!isTextFile(entry.path)) return { sources: [], skipped: 'not a text file type' };
|
|
162
|
+
let buf: Buffer;
|
|
163
|
+
try { buf = readFileSync(entry.path); } catch (e) { return { sources: [], skipped: `unreadable: ${(e as Error).message}` }; }
|
|
164
|
+
if (looksBinary(buf)) return { sources: [], skipped: 'binary content' };
|
|
165
|
+
const text = buf.toString('utf8');
|
|
166
|
+
const readAt = new Date().toISOString();
|
|
167
|
+
const ext = extname(entry.path).toLowerCase();
|
|
168
|
+
const sources: Source[] = [];
|
|
169
|
+
const usedAs = entry.usedAs ?? (scope ? readingOf(scope, entry.path).usedAs : null);
|
|
170
|
+
const judged = (s: Source): Source => (usedAs ? { ...s, usedAs: usedAs.value, usedAsBy: usedAs.by, usedAsByScopeItemId: usedAs.scopeItemId } : s);
|
|
171
|
+
if (ext === '.md' || ext === '.markdown') {
|
|
172
|
+
for (const section of splitMarkdown(text).flatMap(splitLong)) {
|
|
173
|
+
sources.push(judged(makeFileSource({
|
|
174
|
+
projectId, path: entry.path, headingPath: section.headingPath, lineStart: section.lineStart, lineEnd: section.lineEnd,
|
|
175
|
+
excerpt: section.text, fileText: text, scopeItemId: entry.scopeItemId, ids: extractIds(section.text), commit, readAt,
|
|
176
|
+
title: section.headingPath.length ? section.headingPath[section.headingPath.length - 1]! : entry.path.split(/[\\/]/).pop()!,
|
|
177
|
+
})));
|
|
178
|
+
}
|
|
179
|
+
return { sources, skipped: null };
|
|
180
|
+
}
|
|
181
|
+
const lines = text.split(/\r?\n/);
|
|
182
|
+
const head = lines.slice(0, CODE_HEAD_LINES).join('\n');
|
|
183
|
+
sources.push(judged(makeFileSource({
|
|
184
|
+
projectId, path: entry.path, headingPath: [], lineStart: 1, lineEnd: Math.min(lines.length, CODE_HEAD_LINES),
|
|
185
|
+
excerpt: lines.length > CODE_HEAD_LINES ? `${head}\n…(${lines.length} lines in total; the Keeper reads the rest on demand)` : head,
|
|
186
|
+
fileText: text, scopeItemId: entry.scopeItemId, ids: extractIds(head), commit, readAt,
|
|
187
|
+
})));
|
|
188
|
+
return { sources, skipped: null };
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** Full scan of every included directory in scope. */
|
|
192
|
+
/** The same scan on a worker thread: a large project must not freeze the workbench while it is read (CKC-09 AC-1). */
|
|
193
|
+
export function scanFilesAsync(projectId: string, scope: readonly ScopeItem[]): Promise<ScanResult> {
|
|
194
|
+
return new Promise((resolve, reject) => {
|
|
195
|
+
const worker = new Worker(new URL('./scan-worker.ts', import.meta.url), { workerData: { projectId, scope }, ...OWN_TS_WORKER_OPTIONS });
|
|
196
|
+
let settled = false;
|
|
197
|
+
worker.once('message', (result: ScanResult) => { settled = true; resolve(result); void worker.terminate(); });
|
|
198
|
+
worker.once('error', (error) => { settled = true; reject(error); });
|
|
199
|
+
worker.once('exit', (code) => { if (!settled) reject(new Error(`file scan worker exited with code ${code}`)); });
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** The files a registered worktree adds to the trunk (E60), measured now from git's records. */
|
|
204
|
+
function worktreeFiles(item: ScopeItem): FileEntry[] {
|
|
205
|
+
if (!item.worktreeOf) return [];
|
|
206
|
+
const measured = measureWorktree(item.path, item.worktreeOf);
|
|
207
|
+
const out: FileEntry[] = [];
|
|
208
|
+
for (const t of measured.taken) {
|
|
209
|
+
if (t.deleted) continue;
|
|
210
|
+
const full = join(item.path, t.path);
|
|
211
|
+
try {
|
|
212
|
+
const st = statSync(full);
|
|
213
|
+
if (st.isFile()) out.push({ path: normalizePath(full), scopeItemId: item.id, bytes: st.size, mtimeMs: st.mtimeMs, usedAs: null });
|
|
214
|
+
} catch { /* gone since it was measured */ }
|
|
215
|
+
}
|
|
216
|
+
return out;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Every scope item read in its own region, as its relation says (../scope/skip.ts `treatmentOf`): a location that is
|
|
221
|
+
* an item of its own is read — or not — by that item, so nothing is read twice and nothing left out comes in through
|
|
222
|
+
* a directory above it.
|
|
223
|
+
*/
|
|
224
|
+
export function scanFiles(projectId: string, scope: readonly ScopeItem[]): ScanResult {
|
|
225
|
+
const locations = scope.filter((i) => i.category !== 'Session source');
|
|
226
|
+
// Locations read (or left out) by an item of their own. Ignored files the owner took in are read where they lie.
|
|
227
|
+
const itemPaths = new Set(locations.filter((i) => !(i.ignoredBy?.paths && i.relation !== 'Excluded')).map((i) => pathKey(i.path)));
|
|
228
|
+
// Ignored files the owner chose to include anyway (an answer to the scope question, or a correction).
|
|
229
|
+
// Files ignored one by one are listed per directory, so each lives in the directory of its item's path.
|
|
230
|
+
const keep = new Set(locations.filter((i) => i.relation !== 'Excluded' && i.ignoredBy?.paths).flatMap((i) => (i.ignoredBy!.paths ?? []).map((p) => pathKey(join(dirname(i.path), p.split('/').pop()!)))));
|
|
231
|
+
const files: FileEntry[] = [];
|
|
232
|
+
const sources: Source[] = [];
|
|
233
|
+
const skipped: { path: string; reason: string }[] = [];
|
|
234
|
+
for (const root of locations.filter((i) => isReadRoot(scope, i))) {
|
|
235
|
+
const treatment = treatmentOf(root);
|
|
236
|
+
let entries: FileEntry[];
|
|
237
|
+
if (root.category === 'Worktree' && (treatment === 'changes' || treatment === 'history')) {
|
|
238
|
+
entries = worktreeFiles(root);
|
|
239
|
+
} else {
|
|
240
|
+
try { if (!statSync(root.path).isDirectory()) continue; } catch { continue; }
|
|
241
|
+
const separate = new Set([...itemPaths].filter((p) => p !== pathKey(root.path)));
|
|
242
|
+
const documents = treatment === 'documents';
|
|
243
|
+
entries = walkRegion(root.path, root.id, {
|
|
244
|
+
separate,
|
|
245
|
+
ignored: overridesIgnoreRules(scope, root) ? null : ignoredPredicate(root.path),
|
|
246
|
+
keep,
|
|
247
|
+
documentsOnly: documents,
|
|
248
|
+
usedAs: documents ? { value: 'Reference only', by: root.classification?.by === 'owner' ? 'owner' : 'keeper', scopeItemId: root.id } : null,
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
for (const entry of entries) {
|
|
252
|
+
files.push(entry);
|
|
253
|
+
const read = readFileSources(projectId, entry);
|
|
254
|
+
if (read.skipped) { if (isTextFile(entry.path) || entry.bytes > MAX_FILE_BYTES) skipped.push({ path: entry.path, reason: read.skipped }); continue; }
|
|
255
|
+
sources.push(...read.sources);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
return { files, sources, skipped };
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
export function fileFingerprint(path: string): string | null {
|
|
262
|
+
try { return fingerprint(readFileSync(path)); } catch { return null; }
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
export function displayPath(root: string, path: string): string {
|
|
266
|
+
return relativeDisplay(root, path);
|
|
267
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Observe repositories in scope: recent commits become commit sources; the current branch,
|
|
3
|
+
* head and uncommitted changes become one status source per repository. Read-only.
|
|
4
|
+
*
|
|
5
|
+
* When a commit was made goes on its anchor, where the program reads it (how recent a repository's commits are); the
|
|
6
|
+
* excerpt is the commit as people read it, never parsed back into facts.
|
|
7
|
+
*/
|
|
8
|
+
import type { ScopeItem, Source } from '../model/types.ts';
|
|
9
|
+
import { fingerprint, stableId } from '../model/ids.ts';
|
|
10
|
+
import { gitRecentCommits, gitStatus } from '../util/git.ts';
|
|
11
|
+
import { pathKey } from '../util/paths.ts';
|
|
12
|
+
import { commitSourceId, extractIds, redactCredentials } from './anchor.ts';
|
|
13
|
+
|
|
14
|
+
/** Commit sources of a repository: every commit reachable from HEAD (since `since`), unless `limit` asks for fewer (D77). */
|
|
15
|
+
export function commitSources(projectId: string, repo: ScopeItem, since: string | null, limit: number | null = null, notFrom: string | null = null): Source[] {
|
|
16
|
+
const readAt = new Date().toISOString();
|
|
17
|
+
return gitRecentCommits(repo.path, since, limit, notFrom).map((c) => {
|
|
18
|
+
const excerpt = `${c.subject}\n\nauthor: ${c.author}\nat: ${c.at}\nfiles:\n${c.files.map((f) => ` ${f}`).join('\n')}`;
|
|
19
|
+
const red = redactCredentials(excerpt);
|
|
20
|
+
return {
|
|
21
|
+
id: commitSourceId(repo.path, c.hash), projectId, title: `${c.hash.slice(0, 8)} ${c.subject.slice(0, 80)}`,
|
|
22
|
+
anchor: { kind: 'commit', repo: repo.path, commit: c.hash, ...(c.at ? { at: c.at } : {}) }, ids: extractIds(c.subject),
|
|
23
|
+
version: { fingerprint: fingerprint(excerpt), readAt, commit: c.hash }, excerpt: red.text, usedAs: null, usedAsBy: null,
|
|
24
|
+
availability: null, movedTo: null, scopeItemId: repo.id, hasCredential: red.found, bytes: Buffer.byteLength(excerpt),
|
|
25
|
+
};
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function statusSource(projectId: string, repo: ScopeItem): Source | null {
|
|
30
|
+
const st = gitStatus(repo.path);
|
|
31
|
+
if (!st) return null;
|
|
32
|
+
const readAt = new Date().toISOString();
|
|
33
|
+
const description = `git status of ${repo.path}`;
|
|
34
|
+
const excerpt = `branch: ${st.branch ?? '(detached)'}\nhead: ${st.head ?? '(none)'}\nuncommitted: ${st.dirty.length}\n${st.dirty.slice(0, 80).map((l) => ` ${l}`).join('\n')}${st.dirty.length > 80 ? `\n …${st.dirty.length - 80} more` : ''}`;
|
|
35
|
+
return {
|
|
36
|
+
id: stableId('src', 'status', pathKey(repo.path)), projectId, title: `Repository status · ${repo.path.split(/[\\/]/).pop()}`,
|
|
37
|
+
anchor: { kind: 'status', description, at: readAt }, ids: [],
|
|
38
|
+
version: { fingerprint: fingerprint(excerpt), readAt, commit: st.head }, excerpt, usedAs: 'Status', usedAsBy: null,
|
|
39
|
+
availability: null, movedTo: null, scopeItemId: repo.id, hasCredential: false, bytes: Buffer.byteLength(excerpt),
|
|
40
|
+
};
|
|
41
|
+
}
|