skillwiki 0.10.21 → 0.10.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-6ZDKTNLA.js +294 -0
- package/dist/{chunk-65Q5UGND.js → chunk-BASWDOQB.js} +2 -2
- package/dist/{chunk-Y6KRDGI2.js → chunk-C2DKFJFA.js} +6 -1
- package/dist/chunk-EQPU2BPM.js +468 -0
- package/dist/{chunk-GNS2ZV5P.js → chunk-PQG26AGJ.js} +1 -1
- package/dist/{chunk-I5JD3BQZ.js → chunk-QBZYEEBD.js} +13 -296
- package/dist/chunk-QNTBNNEL.js +616 -0
- package/dist/{chunk-UNPZDCWN.js → chunk-SYKSL3JQ.js} +860 -700
- package/dist/cli.js +2334 -273
- package/dist/{index-projection-HAXDEM2F.js → index-projection-FRWTZB5Y.js} +3 -2
- package/dist/{managed-write-preflight-CTXX2MHQ.js → managed-write-preflight-4LPVMP42.js} +3 -3
- package/dist/skillwiki-mcp.js +6 -4
- package/dist/sources-L2SQV63E.js +10 -0
- package/package.json +1 -1
- package/skills/.claude-plugin/plugin.json +1 -1
- package/skills/.codex-plugin/plugin.json +1 -1
- package/skills/README.md +13 -2
- package/skills/agents/wiki-add-task.md +2 -1
- package/skills/agents/wiki-archive.md +8 -9
- package/skills/agents/wiki-audit.md +2 -1
- package/skills/agents/wiki-lint.md +2 -1
- package/skills/agents/wiki-query.md +3 -1
- package/skills/agents/wiki-reingest.md +9 -8
- package/skills/package.json +1 -1
- package/skills/proj-decide/SKILL.md +35 -5
- package/skills/skills/proj-decide/SKILL.md +35 -5
- package/skills/skills/using-skillwiki/SKILL.md +19 -13
- package/skills/skills/wiki-add-task/SKILL.md +4 -3
- package/skills/skills/wiki-archive/SKILL.md +23 -18
- package/skills/skills/wiki-audit/SKILL.md +3 -1
- package/skills/skills/wiki-init/SKILL.md +12 -1
- package/skills/skills/wiki-lint/SKILL.md +2 -1
- package/skills/skills/wiki-query/SKILL.md +7 -0
- package/skills/skills/wiki-reingest/SKILL.md +12 -8
- package/skills/skills/wiki-remove/SKILL.md +22 -5
- package/skills/skills/wiki-sync/SKILL.md +39 -54
- package/skills/using-skillwiki/SKILL.md +19 -13
- package/skills/wiki-add-task/SKILL.md +4 -3
- package/skills/wiki-archive/SKILL.md +23 -18
- package/skills/wiki-audit/SKILL.md +3 -1
- package/skills/wiki-init/SKILL.md +12 -1
- package/skills/wiki-lint/SKILL.md +2 -1
- package/skills/wiki-query/SKILL.md +7 -0
- package/skills/wiki-reingest/SKILL.md +12 -8
- package/skills/wiki-remove/SKILL.md +22 -5
- package/skills/wiki-sync/SKILL.md +39 -54
- package/templates/SCHEMA.md +8 -5
- package/templates/web-clipper/llm-wiki-clippings.json +56 -0
- package/templates/web-clipper/readme.txt +16 -0
|
@@ -0,0 +1,468 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
buildSourceReferenceIndex,
|
|
4
|
+
buildSourceRelocationProjection,
|
|
5
|
+
classifyRawPath,
|
|
6
|
+
operationId,
|
|
7
|
+
projectSourceRelocations,
|
|
8
|
+
readLogEvents,
|
|
9
|
+
resolveExistingRegularFileInsideVault,
|
|
10
|
+
writeLogEvent
|
|
11
|
+
} from "./chunk-QNTBNNEL.js";
|
|
12
|
+
import {
|
|
13
|
+
extractFrontmatter,
|
|
14
|
+
readPage,
|
|
15
|
+
scanSensitiveContent,
|
|
16
|
+
scanVault,
|
|
17
|
+
splitFrontmatter
|
|
18
|
+
} from "./chunk-6ZDKTNLA.js";
|
|
19
|
+
import {
|
|
20
|
+
ExitCode,
|
|
21
|
+
RawSourceSchema,
|
|
22
|
+
err,
|
|
23
|
+
ok
|
|
24
|
+
} from "./chunk-C2DKFJFA.js";
|
|
25
|
+
|
|
26
|
+
// src/utils/source-lifecycle.ts
|
|
27
|
+
import { basename } from "path";
|
|
28
|
+
|
|
29
|
+
// src/utils/source-dispositions.ts
|
|
30
|
+
import { createHash as createHash2 } from "crypto";
|
|
31
|
+
import { readFile } from "fs/promises";
|
|
32
|
+
|
|
33
|
+
// src/utils/source-action-approval.ts
|
|
34
|
+
import { createHash } from "crypto";
|
|
35
|
+
var VERSION = "swsrc1";
|
|
36
|
+
function sha256(value) {
|
|
37
|
+
return createHash("sha256").update(value, "utf8").digest("hex");
|
|
38
|
+
}
|
|
39
|
+
function encodeSourceActionApproval(payload) {
|
|
40
|
+
const middle = Buffer.from(JSON.stringify(payload), "utf8").toString("base64url");
|
|
41
|
+
return `${VERSION}.${middle}.${sha256(middle)}`;
|
|
42
|
+
}
|
|
43
|
+
function decodeSourceActionApproval(token) {
|
|
44
|
+
const parts = token.split(".");
|
|
45
|
+
if (parts.length !== 3 || parts[0] !== VERSION || !/^[0-9a-f]{64}$/.test(parts[2] ?? "")) {
|
|
46
|
+
return err("APPROVAL_INVALID", { message: "invalid source-action approval token" });
|
|
47
|
+
}
|
|
48
|
+
if (sha256(parts[1]) !== parts[2]) return err("APPROVAL_INVALID", { message: "source-action approval checksum mismatch" });
|
|
49
|
+
try {
|
|
50
|
+
const parsed = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8"));
|
|
51
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return err("APPROVAL_INVALID", { message: "approval payload must be an object" });
|
|
52
|
+
return ok(parsed);
|
|
53
|
+
} catch {
|
|
54
|
+
return err("APPROVAL_INVALID", { message: "source-action approval payload is invalid" });
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// src/utils/source-dispositions.ts
|
|
59
|
+
var STATUSES = /* @__PURE__ */ new Set(["reviewed-no-op", "deferred", "duplicate", "out-of-scope", "superseded", "reopened"]);
|
|
60
|
+
function bodySha256(text) {
|
|
61
|
+
const split = splitFrontmatter(text);
|
|
62
|
+
const body = split.ok ? split.data.body : text;
|
|
63
|
+
return createHash2("sha256").update(Buffer.from(body, "utf8")).digest("hex");
|
|
64
|
+
}
|
|
65
|
+
function completeSha256(text) {
|
|
66
|
+
return createHash2("sha256").update(Buffer.from(text, "utf8")).digest("hex");
|
|
67
|
+
}
|
|
68
|
+
function exactActiveSource(path) {
|
|
69
|
+
const parsed = classifyRawPath(path);
|
|
70
|
+
if (!parsed.ok) return parsed;
|
|
71
|
+
if (parsed.data.storage !== "active" || !["articles", "papers"].includes(parsed.data.category)) {
|
|
72
|
+
return err("SOURCE_DISPOSITION_TARGET_INVALID", { path, message: "dispositions require an exact active raw article or paper" });
|
|
73
|
+
}
|
|
74
|
+
return ok(true);
|
|
75
|
+
}
|
|
76
|
+
function validDate(value) {
|
|
77
|
+
return /^\d{4}-\d{2}-\d{2}$/.test(value) && (/* @__PURE__ */ new Date(`${value}T00:00:00Z`)).toISOString().slice(0, 10) === value;
|
|
78
|
+
}
|
|
79
|
+
function parseSourceDispositionEvent(event) {
|
|
80
|
+
if (event.kind !== "source-disposition") return ok(null);
|
|
81
|
+
const metadata = event.metadata;
|
|
82
|
+
if (!STATUSES.has(metadata.status) || typeof metadata.raw_path !== "string" || typeof metadata.complete_sha256 !== "string" && typeof metadata.body_sha256 !== "string" || typeof metadata.complete_sha256 === "string" && !/^[0-9a-f]{64}$/.test(metadata.complete_sha256) || typeof metadata.body_sha256 === "string" && !/^[0-9a-f]{64}$/.test(metadata.body_sha256) || typeof metadata.reason !== "string") {
|
|
83
|
+
return err("SOURCE_DISPOSITION_INVALID", { operation_id: event.operation_id });
|
|
84
|
+
}
|
|
85
|
+
return ok({
|
|
86
|
+
operation_id: event.operation_id,
|
|
87
|
+
occurred_at: event.occurred_at,
|
|
88
|
+
raw_path: metadata.raw_path,
|
|
89
|
+
...typeof metadata.complete_sha256 === "string" ? { complete_sha256: metadata.complete_sha256 } : {},
|
|
90
|
+
...typeof metadata.body_sha256 === "string" ? { body_sha256: metadata.body_sha256 } : {},
|
|
91
|
+
status: metadata.status,
|
|
92
|
+
reason: metadata.reason,
|
|
93
|
+
...typeof metadata.review_after === "string" ? { review_after: metadata.review_after } : {},
|
|
94
|
+
...typeof metadata.duplicate_of === "string" ? { duplicate_of: metadata.duplicate_of } : {}
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
function projectSourceDispositions(events) {
|
|
98
|
+
const dispositions = [];
|
|
99
|
+
for (const event of events) {
|
|
100
|
+
const parsed = parseSourceDispositionEvent(event);
|
|
101
|
+
if (!parsed.ok) return parsed;
|
|
102
|
+
if (parsed.data) dispositions.push(parsed.data);
|
|
103
|
+
}
|
|
104
|
+
return ok(dispositions.sort((a, b) => a.occurred_at.localeCompare(b.occurred_at) || a.operation_id.localeCompare(b.operation_id)));
|
|
105
|
+
}
|
|
106
|
+
async function readSourceDispositions(vault) {
|
|
107
|
+
const events = await readLogEvents(vault);
|
|
108
|
+
if (!events.ok) return events;
|
|
109
|
+
return projectSourceDispositions(events.data);
|
|
110
|
+
}
|
|
111
|
+
function effectiveSourceDisposition(input) {
|
|
112
|
+
const candidates = input.dispositions.filter((event) => event.raw_path === input.rawPath);
|
|
113
|
+
const latest = candidates.at(-1);
|
|
114
|
+
if (!latest) return { lifecycle: "pending", identityMismatch: false };
|
|
115
|
+
const identityMatches = latest.complete_sha256 ? latest.complete_sha256 === input.completeSha256 : latest.body_sha256 === input.bodySha256;
|
|
116
|
+
if (!identityMatches) return { lifecycle: "pending", identityMismatch: true, disposition: latest };
|
|
117
|
+
if (latest.status === "reopened") return { lifecycle: "pending", identityMismatch: false, disposition: latest };
|
|
118
|
+
if (latest.status === "deferred") {
|
|
119
|
+
return latest.review_after && latest.review_after > input.today ? { lifecycle: "deferred", identityMismatch: false, disposition: latest } : { lifecycle: "pending", identityMismatch: false, disposition: latest };
|
|
120
|
+
}
|
|
121
|
+
return { lifecycle: latest.status, identityMismatch: false, disposition: latest };
|
|
122
|
+
}
|
|
123
|
+
function latestDuplicateTarget(dispositions, rawPath) {
|
|
124
|
+
const latest = dispositions.filter((event) => event.raw_path === rawPath).at(-1);
|
|
125
|
+
return latest?.status === "duplicate" ? latest.duplicate_of : void 0;
|
|
126
|
+
}
|
|
127
|
+
async function planSourceDisposition(input) {
|
|
128
|
+
const target = exactActiveSource(input.rawPath);
|
|
129
|
+
if (!target.ok) return target;
|
|
130
|
+
if (!STATUSES.has(input.status)) return err("SOURCE_DISPOSITION_INVALID", { field: "status" });
|
|
131
|
+
const reason = input.reason.normalize("NFC").trim();
|
|
132
|
+
if (!reason) return err("SOURCE_DISPOSITION_INVALID", { field: "reason", message: "reason is required" });
|
|
133
|
+
const sensitive = scanSensitiveContent(reason);
|
|
134
|
+
if (sensitive.length > 0) return err("SENSITIVE_CONTENT_DETECTED", { findings: sensitive });
|
|
135
|
+
const today = input.today ?? (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
136
|
+
if (input.status === "deferred" && (!input.reviewAfter || !validDate(input.reviewAfter) || input.reviewAfter <= today)) {
|
|
137
|
+
return err("SOURCE_DISPOSITION_INVALID", { field: "review_after", message: "deferred requires a future YYYY-MM-DD" });
|
|
138
|
+
}
|
|
139
|
+
if (input.status === "duplicate") {
|
|
140
|
+
if (!input.duplicateOf || input.duplicateOf === input.rawPath) return err("SOURCE_DISPOSITION_INVALID", { field: "duplicate_of" });
|
|
141
|
+
const duplicateTarget = exactActiveSource(input.duplicateOf);
|
|
142
|
+
if (!duplicateTarget.ok) return duplicateTarget;
|
|
143
|
+
const duplicateResolved = await resolveExistingRegularFileInsideVault(input.vault, input.duplicateOf);
|
|
144
|
+
if (!duplicateResolved.ok) return err("SOURCE_DISPOSITION_INVALID", { field: "duplicate_of", message: "canonical duplicate target not found", cause: duplicateResolved });
|
|
145
|
+
}
|
|
146
|
+
const sourceResolved = await resolveExistingRegularFileInsideVault(input.vault, input.rawPath);
|
|
147
|
+
if (!sourceResolved.ok) return err("FILE_NOT_FOUND", { path: input.rawPath, cause: sourceResolved });
|
|
148
|
+
const text = await readFile(sourceResolved.data, "utf8");
|
|
149
|
+
const bodyHash = bodySha256(text);
|
|
150
|
+
const completeHash = completeSha256(text);
|
|
151
|
+
const existing = await readSourceDispositions(input.vault);
|
|
152
|
+
if (!existing.ok) return existing;
|
|
153
|
+
if (input.status === "duplicate" && input.duplicateOf) {
|
|
154
|
+
const seen = /* @__PURE__ */ new Set([input.rawPath]);
|
|
155
|
+
let current = input.duplicateOf;
|
|
156
|
+
while (current) {
|
|
157
|
+
if (seen.has(current)) {
|
|
158
|
+
return err("SOURCE_DISPOSITION_INVALID", {
|
|
159
|
+
field: "duplicate_of",
|
|
160
|
+
message: "duplicate dispositions may not create a direct or transitive cycle"
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
seen.add(current);
|
|
164
|
+
current = latestDuplicateTarget(existing.data, current);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
if (input.status === "reopened") {
|
|
168
|
+
const effective = effectiveSourceDisposition({ dispositions: existing.data, rawPath: input.rawPath, bodySha256: bodyHash, completeSha256: completeHash, today });
|
|
169
|
+
if (!effective.disposition || effective.lifecycle === "pending") return err("SOURCE_DISPOSITION_INVALID", { field: "status", message: "reopened requires an effective prior disposition" });
|
|
170
|
+
}
|
|
171
|
+
const occurredAt = input.now ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
172
|
+
const operation_id = operationId("source-disposition", [input.rawPath, completeHash, input.status, reason, input.reviewAfter ?? "", input.duplicateOf ?? ""]);
|
|
173
|
+
const base = {
|
|
174
|
+
operation_id,
|
|
175
|
+
occurred_at: occurredAt,
|
|
176
|
+
raw_path: input.rawPath,
|
|
177
|
+
complete_sha256: completeHash,
|
|
178
|
+
status: input.status,
|
|
179
|
+
reason,
|
|
180
|
+
...input.reviewAfter ? { review_after: input.reviewAfter } : {},
|
|
181
|
+
...input.duplicateOf ? { duplicate_of: input.duplicateOf } : {}
|
|
182
|
+
};
|
|
183
|
+
return ok({ ...base, approval_token: encodeSourceActionApproval({ contract: "source-disposition/v1", ...base }), write: false });
|
|
184
|
+
}
|
|
185
|
+
async function applySourceDisposition(input) {
|
|
186
|
+
const decoded = decodeSourceActionApproval(input.approve);
|
|
187
|
+
if (!decoded.ok) return decoded;
|
|
188
|
+
const occurredAt = typeof decoded.data.occurred_at === "string" ? decoded.data.occurred_at : void 0;
|
|
189
|
+
const planned = await planSourceDisposition({ ...input, now: occurredAt });
|
|
190
|
+
if (!planned.ok) return planned;
|
|
191
|
+
if (planned.data.approval_token !== input.approve) return err("APPROVAL_INVALID", { message: "source disposition approval does not match live state" });
|
|
192
|
+
const event = await writeLogEvent(input.vault, {
|
|
193
|
+
schema: "skillwiki-log-event/v1",
|
|
194
|
+
operation_id: planned.data.operation_id,
|
|
195
|
+
occurred_at: planned.data.occurred_at,
|
|
196
|
+
host_id: input.hostId ?? "local",
|
|
197
|
+
actor: input.actor ?? "skillwiki-cli",
|
|
198
|
+
kind: "source-disposition",
|
|
199
|
+
target: input.rawPath,
|
|
200
|
+
note: input.reason,
|
|
201
|
+
metadata: {
|
|
202
|
+
status: input.status,
|
|
203
|
+
raw_path: input.rawPath,
|
|
204
|
+
complete_sha256: planned.data.complete_sha256,
|
|
205
|
+
reason: planned.data.reason,
|
|
206
|
+
...input.reviewAfter ? { review_after: input.reviewAfter } : {},
|
|
207
|
+
...input.duplicateOf ? { duplicate_of: input.duplicateOf } : {}
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
if (!event.ok) return event;
|
|
211
|
+
return ok({ event_path: event.data.path, operation_id: planned.data.operation_id, created: event.data.created });
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// src/utils/source-lifecycle.ts
|
|
215
|
+
function portableDate(value) {
|
|
216
|
+
if (typeof value !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(value)) return null;
|
|
217
|
+
const parsed = /* @__PURE__ */ new Date(`${value}T00:00:00Z`);
|
|
218
|
+
return !Number.isNaN(parsed.getTime()) && parsed.toISOString().slice(0, 10) === value ? value : null;
|
|
219
|
+
}
|
|
220
|
+
function filenameDate(path) {
|
|
221
|
+
const match = basename(path).match(/^(\d{4}-\d{2}-\d{2})(?:-|$)/);
|
|
222
|
+
return match ? portableDate(match[1]) : null;
|
|
223
|
+
}
|
|
224
|
+
function captureDate(path, fm) {
|
|
225
|
+
const ingested = portableDate(fm.ingested);
|
|
226
|
+
if (ingested) return { captured: ingested, source: "ingested" };
|
|
227
|
+
const created = portableDate(fm.created);
|
|
228
|
+
if (created) return { captured: created, source: "created" };
|
|
229
|
+
const fromName = filenameDate(path);
|
|
230
|
+
if (fromName) return { captured: fromName, source: "filename" };
|
|
231
|
+
return { captured: null, source: "unavailable" };
|
|
232
|
+
}
|
|
233
|
+
function ageBucket(captured, today) {
|
|
234
|
+
if (!captured) return "unknown";
|
|
235
|
+
const days = Math.floor((Date.parse(`${today}T00:00:00Z`) - Date.parse(`${captured}T00:00:00Z`)) / 864e5);
|
|
236
|
+
if (!Number.isFinite(days)) return "unknown";
|
|
237
|
+
if (days <= 7) return "fresh";
|
|
238
|
+
if (days <= 30) return "aging";
|
|
239
|
+
if (days <= 90) return "stale";
|
|
240
|
+
return "old";
|
|
241
|
+
}
|
|
242
|
+
function storageStatus(path) {
|
|
243
|
+
if (path.startsWith("raw/archived/")) return "archived";
|
|
244
|
+
if (path.startsWith("raw/duplicates/")) return "duplicate";
|
|
245
|
+
if (path.startsWith("_archive/raw/")) return "legacy-archived";
|
|
246
|
+
return "active";
|
|
247
|
+
}
|
|
248
|
+
function supportedSource(path) {
|
|
249
|
+
return /^raw\/(articles|papers)\/.+\.md$/.test(path) || /^raw\/(archived|duplicates)\/(articles|papers)\/.+\.md$/.test(path) || /^_archive\/raw\/(articles|papers)\/.+\.md$/.test(path);
|
|
250
|
+
}
|
|
251
|
+
function channel(fm, schemaStatus) {
|
|
252
|
+
if (fm.ingested_by === "wiki-ingest") return "wiki-ingest";
|
|
253
|
+
if (fm.ingested_by === "proj-work") return "proj-work";
|
|
254
|
+
if (fm.ingested_by === "manual") return "manual";
|
|
255
|
+
if (schemaStatus === "legacy" && typeof fm.source === "string") return "web-clipper-legacy";
|
|
256
|
+
return "unknown";
|
|
257
|
+
}
|
|
258
|
+
function titleFor(path, fm) {
|
|
259
|
+
if (typeof fm.title === "string" && fm.title.trim()) return fm.title.trim();
|
|
260
|
+
return basename(path, ".md");
|
|
261
|
+
}
|
|
262
|
+
function classifySchema(fmResult) {
|
|
263
|
+
if (!fmResult.ok) {
|
|
264
|
+
return { fm: {}, status: "invalid", issues: ["frontmatter is invalid or incomplete"] };
|
|
265
|
+
}
|
|
266
|
+
const fm = fmResult.data;
|
|
267
|
+
const parsed = RawSourceSchema.safeParse(fm);
|
|
268
|
+
if (parsed.success) return { fm, status: "valid", issues: [] };
|
|
269
|
+
const issues = [];
|
|
270
|
+
if (typeof fm.source === "string" && typeof fm.source_url !== "string") {
|
|
271
|
+
issues.push("source_url missing; legacy source property found");
|
|
272
|
+
}
|
|
273
|
+
if (!portableDate(fm.ingested) && portableDate(fm.created)) issues.push("ingested missing");
|
|
274
|
+
if (Object.keys(fm).length === 0) issues.push("frontmatter missing");
|
|
275
|
+
for (const issue of parsed.error.issues.slice(0, 8)) {
|
|
276
|
+
const message = `${issue.path.join(".") || "frontmatter"}: ${issue.message}`;
|
|
277
|
+
if (!issues.includes(message)) issues.push(message);
|
|
278
|
+
}
|
|
279
|
+
const recognizedLegacy = typeof fm.source === "string" || portableDate(fm.created) !== null;
|
|
280
|
+
return { fm, status: recognizedLegacy ? "legacy" : "invalid", issues };
|
|
281
|
+
}
|
|
282
|
+
async function inventorySources(input) {
|
|
283
|
+
const scan = await scanVault(input.vault);
|
|
284
|
+
if (!scan.ok) return { exitCode: 9, error: scan };
|
|
285
|
+
const today = input.today ?? (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
286
|
+
const legacyPages = scan.data.allMarkdown.filter((page) => page.relPath.startsWith("_archive/raw/"));
|
|
287
|
+
const pages = [...scan.data.raw, ...legacyPages].filter((page) => supportedSource(page.relPath));
|
|
288
|
+
const availablePaths = pages.map((page) => page.relPath);
|
|
289
|
+
const typedPaths = new Set(scan.data.typedKnowledge.map((page) => page.relPath));
|
|
290
|
+
const otherPages = scan.data.allMarkdown.filter(
|
|
291
|
+
(page) => !typedPaths.has(page.relPath) && !page.relPath.startsWith("raw/") && !page.relPath.startsWith("_archive/")
|
|
292
|
+
);
|
|
293
|
+
const diagnostics = [];
|
|
294
|
+
const logEvents = await readLogEvents(input.vault);
|
|
295
|
+
const relocationsResult = logEvents.ok ? projectSourceRelocations(logEvents.data) : logEvents;
|
|
296
|
+
const relocationProjection = buildSourceRelocationProjection(relocationsResult.ok ? relocationsResult.data : []);
|
|
297
|
+
const historicalByCurrent = /* @__PURE__ */ new Map();
|
|
298
|
+
for (const [historical, current] of relocationProjection) {
|
|
299
|
+
const paths = historicalByCurrent.get(current) ?? [];
|
|
300
|
+
paths.push(historical);
|
|
301
|
+
historicalByCurrent.set(current, paths);
|
|
302
|
+
}
|
|
303
|
+
for (const paths of historicalByCurrent.values()) paths.sort();
|
|
304
|
+
const refs = await buildSourceReferenceIndex({
|
|
305
|
+
typedPages: scan.data.typedKnowledge,
|
|
306
|
+
otherPages,
|
|
307
|
+
availableRawPaths: availablePaths,
|
|
308
|
+
relocationProjection
|
|
309
|
+
});
|
|
310
|
+
const dispositionsResult = logEvents.ok ? projectSourceDispositions(logEvents.data) : logEvents;
|
|
311
|
+
const dispositions = dispositionsResult.ok ? dispositionsResult.data : [];
|
|
312
|
+
const latestDispositionByPath = /* @__PURE__ */ new Map();
|
|
313
|
+
for (const disposition of dispositions) latestDispositionByPath.set(disposition.raw_path, disposition);
|
|
314
|
+
const dispositionReadError = dispositionsResult.ok ? null : dispositionsResult;
|
|
315
|
+
const items = [];
|
|
316
|
+
if (!relocationsResult.ok) {
|
|
317
|
+
diagnostics.push({ raw_path: "meta/log-events", code: "source_relocation_invalid", message: JSON.stringify(relocationsResult) });
|
|
318
|
+
}
|
|
319
|
+
for (const unresolved of refs.unresolved) {
|
|
320
|
+
diagnostics.push({
|
|
321
|
+
raw_path: unresolved.sourcePath,
|
|
322
|
+
code: "source_reference_unresolved",
|
|
323
|
+
message: `${unresolved.kind} reference does not resolve: ${unresolved.target}`
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
for (const page of pages) {
|
|
327
|
+
let text;
|
|
328
|
+
try {
|
|
329
|
+
text = await readPage(page);
|
|
330
|
+
} catch (error) {
|
|
331
|
+
diagnostics.push({ raw_path: page.relPath, code: "source_unreadable", message: String(error) });
|
|
332
|
+
continue;
|
|
333
|
+
}
|
|
334
|
+
const classification = classifySchema(extractFrontmatter(text));
|
|
335
|
+
const date = captureDate(page.relPath, classification.fm);
|
|
336
|
+
const referencedBy = refs.integratedBy.get(page.relPath) ?? [];
|
|
337
|
+
const elsewhere = refs.referencedElsewhereBy.get(page.relPath) ?? [];
|
|
338
|
+
const projectedDisposition = effectiveSourceDisposition({
|
|
339
|
+
dispositions: latestDispositionByPath.has(page.relPath) ? [latestDispositionByPath.get(page.relPath)] : [],
|
|
340
|
+
rawPath: page.relPath,
|
|
341
|
+
bodySha256: bodySha256(text),
|
|
342
|
+
completeSha256: completeSha256(text),
|
|
343
|
+
today
|
|
344
|
+
});
|
|
345
|
+
const sourceUrl = typeof classification.fm.source_url === "string" ? classification.fm.source_url : typeof classification.fm.source === "string" ? classification.fm.source : null;
|
|
346
|
+
if (classification.status === "invalid") {
|
|
347
|
+
diagnostics.push({ raw_path: page.relPath, code: "source_schema_invalid", message: classification.issues.join("; ") });
|
|
348
|
+
}
|
|
349
|
+
const storage = storageStatus(page.relPath);
|
|
350
|
+
items.push({
|
|
351
|
+
raw_path: page.relPath,
|
|
352
|
+
current_raw_path: page.relPath,
|
|
353
|
+
historical_raw_paths: historicalByCurrent.get(page.relPath) ?? [],
|
|
354
|
+
storage_status: storage,
|
|
355
|
+
title: titleFor(page.relPath, classification.fm),
|
|
356
|
+
source_url: sourceUrl,
|
|
357
|
+
captured: date.captured,
|
|
358
|
+
date_source: date.source,
|
|
359
|
+
age_bucket: ageBucket(date.captured, today),
|
|
360
|
+
capture_channel: channel(classification.fm, classification.status),
|
|
361
|
+
lifecycle_status: referencedBy.length > 0 ? "integrated" : storage === "duplicate" ? "duplicate" : projectedDisposition.lifecycle,
|
|
362
|
+
schema_status: classification.status,
|
|
363
|
+
schema_issues: classification.issues,
|
|
364
|
+
reference_count: referencedBy.length,
|
|
365
|
+
referenced_by: referencedBy,
|
|
366
|
+
referenced_elsewhere: elsewhere,
|
|
367
|
+
...projectedDisposition.disposition ? { effective_disposition: projectedDisposition.disposition } : {},
|
|
368
|
+
...projectedDisposition.identityMismatch ? { disposition_identity_mismatch: true } : {}
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
if (dispositionReadError) {
|
|
372
|
+
diagnostics.push({ raw_path: "meta/log-events", code: "source_disposition_invalid", message: JSON.stringify(dispositionReadError) });
|
|
373
|
+
}
|
|
374
|
+
items.sort((a, b) => (b.captured ?? "").localeCompare(a.captured ?? "") || a.raw_path.localeCompare(b.raw_path));
|
|
375
|
+
return { exitCode: 0, output: { items, diagnostics } };
|
|
376
|
+
}
|
|
377
|
+
function sourceMatches(item, text) {
|
|
378
|
+
const terms = text.toLocaleLowerCase().split(/\s+/).filter(Boolean);
|
|
379
|
+
const haystack = `${item.title}
|
|
380
|
+
${item.source_url ?? ""}
|
|
381
|
+
${item.raw_path}`.toLocaleLowerCase();
|
|
382
|
+
return terms.every((term) => haystack.includes(term));
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
// src/commands/sources.ts
|
|
386
|
+
function validDate2(value) {
|
|
387
|
+
return /^\d{4}-\d{2}-\d{2}$/.test(value) && (/* @__PURE__ */ new Date(`${value}T00:00:00Z`)).toISOString().slice(0, 10) === value;
|
|
388
|
+
}
|
|
389
|
+
function validate(input) {
|
|
390
|
+
if (input.since && !validDate2(input.since)) return err("SOURCES_DATE_INVALID", { field: "since", value: input.since });
|
|
391
|
+
if (input.olderThan !== void 0 && (!Number.isInteger(input.olderThan) || input.olderThan < 0)) {
|
|
392
|
+
return err("SOURCES_AGE_INVALID", { field: "older-than", value: input.olderThan });
|
|
393
|
+
}
|
|
394
|
+
if (input.limit !== void 0 && (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > 1e4)) {
|
|
395
|
+
return err("SOURCES_LIMIT_INVALID", { value: input.limit });
|
|
396
|
+
}
|
|
397
|
+
if (input.scope && !["articles", "papers", "all"].includes(input.scope)) {
|
|
398
|
+
return err("SOURCES_SCOPE_INVALID", { value: input.scope });
|
|
399
|
+
}
|
|
400
|
+
if (input.sort && !["newest", "oldest"].includes(input.sort)) {
|
|
401
|
+
return err("SOURCES_SORT_INVALID", { value: input.sort });
|
|
402
|
+
}
|
|
403
|
+
return ok(true);
|
|
404
|
+
}
|
|
405
|
+
function channelMatches(item, selected) {
|
|
406
|
+
if (!selected) return true;
|
|
407
|
+
const channel2 = item.capture_channel;
|
|
408
|
+
if (selected === "manual") return channel2 === "manual" || channel2 === "web-clipper-legacy";
|
|
409
|
+
return channel2 === selected;
|
|
410
|
+
}
|
|
411
|
+
function summary(items) {
|
|
412
|
+
return {
|
|
413
|
+
total: items.length,
|
|
414
|
+
pending: items.filter((item) => item.lifecycle_status === "pending").length,
|
|
415
|
+
integrated: items.filter((item) => item.lifecycle_status === "integrated").length,
|
|
416
|
+
valid: items.filter((item) => item.schema_status === "valid").length,
|
|
417
|
+
legacy: items.filter((item) => item.schema_status === "legacy").length,
|
|
418
|
+
invalid: items.filter((item) => item.schema_status === "invalid").length,
|
|
419
|
+
fresh: items.filter((item) => item.age_bucket === "fresh").length,
|
|
420
|
+
aging: items.filter((item) => item.age_bucket === "aging").length,
|
|
421
|
+
stale: items.filter((item) => item.age_bucket === "stale").length,
|
|
422
|
+
old: items.filter((item) => item.age_bucket === "old").length,
|
|
423
|
+
unknown_age: items.filter((item) => item.age_bucket === "unknown").length
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
async function runSourcesPending(input) {
|
|
427
|
+
const validated = validate(input);
|
|
428
|
+
if (!validated.ok) return { exitCode: ExitCode.USAGE, result: validated };
|
|
429
|
+
const inventory = await inventorySources({ vault: input.vault, today: input.today });
|
|
430
|
+
if (!inventory.output) {
|
|
431
|
+
return { exitCode: inventory.exitCode, result: inventory.error };
|
|
432
|
+
}
|
|
433
|
+
const today = input.today ?? (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
434
|
+
const cutoff = input.olderThan === void 0 ? null : new Date(Date.parse(`${today}T00:00:00Z`) - input.olderThan * 864e5).toISOString().slice(0, 10);
|
|
435
|
+
let items = inventory.output.items.filter((item) => {
|
|
436
|
+
if (!input.includeIntegrated && item.lifecycle_status !== "pending") return false;
|
|
437
|
+
if (!input.includeArchived && item.storage_status === "archived") return false;
|
|
438
|
+
if (!input.includeDuplicates && item.storage_status === "duplicate") return false;
|
|
439
|
+
if (!input.includeLegacyArchived && item.storage_status === "legacy-archived") return false;
|
|
440
|
+
if (input.scope && input.scope !== "all" && !item.raw_path.startsWith(`raw/${input.scope}/`) && !item.raw_path.includes(`/${input.scope}/`)) return false;
|
|
441
|
+
if (input.since && (!item.captured || item.captured < input.since)) return false;
|
|
442
|
+
if (cutoff && (!item.captured || item.captured > cutoff)) return false;
|
|
443
|
+
if (input.match && !sourceMatches(item, input.match)) return false;
|
|
444
|
+
if (!channelMatches(item, input.ingestedBy)) return false;
|
|
445
|
+
return true;
|
|
446
|
+
});
|
|
447
|
+
if ((input.sort ?? "newest") === "oldest") {
|
|
448
|
+
items = items.sort((a, b) => (a.captured ?? "9999-99-99").localeCompare(b.captured ?? "9999-99-99") || a.raw_path.localeCompare(b.raw_path));
|
|
449
|
+
}
|
|
450
|
+
const counts = summary(items);
|
|
451
|
+
const unbounded = input.all === true;
|
|
452
|
+
items = items.slice(0, unbounded ? void 0 : input.limit ?? 50);
|
|
453
|
+
const visibleRawPaths = new Set(items.map((item) => item.raw_path));
|
|
454
|
+
const diagnostics = inventory.output.diagnostics.filter(
|
|
455
|
+
(diagnostic) => visibleRawPaths.has(diagnostic.raw_path) || diagnostic.raw_path === "meta/log-events"
|
|
456
|
+
);
|
|
457
|
+
const humanHint = items.length === 0 ? "no pending sources" : items.map((item) => `${item.captured ?? "unknown-date"} ${item.schema_status} ${item.raw_path} \u2014 ${item.title}`).join("\n");
|
|
458
|
+
return { exitCode: ExitCode.OK, result: ok({ items, summary: counts, diagnostics, humanHint }) };
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
export {
|
|
462
|
+
encodeSourceActionApproval,
|
|
463
|
+
decodeSourceActionApproval,
|
|
464
|
+
planSourceDisposition,
|
|
465
|
+
applySourceDisposition,
|
|
466
|
+
inventorySources,
|
|
467
|
+
runSourcesPending
|
|
468
|
+
};
|