skillwiki 0.9.42 → 0.9.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -663,6 +663,24 @@ import { readFile as readFile2, readdir, stat } from "fs/promises";
|
|
|
663
663
|
import { join as join2, relative as relative2, sep as sep2 } from "path";
|
|
664
664
|
var TYPED_DIRS = ["entities", "concepts", "comparisons", "queries", "meta"];
|
|
665
665
|
var SKIP_DIRS = /* @__PURE__ */ new Set([".git", "node_modules"]);
|
|
666
|
+
var DEFAULT_IO_CONCURRENCY = 12;
|
|
667
|
+
function vaultIoConcurrency() {
|
|
668
|
+
const raw = Number.parseInt(process.env.SKILLWIKI_VAULT_IO_CONCURRENCY ?? "", 10);
|
|
669
|
+
return Number.isFinite(raw) && raw > 0 ? Math.min(raw, 64) : DEFAULT_IO_CONCURRENCY;
|
|
670
|
+
}
|
|
671
|
+
async function mapWithConcurrency(items, limit, mapper) {
|
|
672
|
+
const out = new Array(items.length);
|
|
673
|
+
let next = 0;
|
|
674
|
+
const workers = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
|
|
675
|
+
for (; ; ) {
|
|
676
|
+
const index = next++;
|
|
677
|
+
if (index >= items.length) return;
|
|
678
|
+
out[index] = await mapper(items[index], index);
|
|
679
|
+
}
|
|
680
|
+
});
|
|
681
|
+
await Promise.all(workers);
|
|
682
|
+
return out;
|
|
683
|
+
}
|
|
666
684
|
async function scanVault(root) {
|
|
667
685
|
try {
|
|
668
686
|
await stat(join2(root, "SCHEMA.md"));
|
|
@@ -683,18 +701,29 @@ async function scanVault(root) {
|
|
|
683
701
|
async function walk(dir) {
|
|
684
702
|
const entries = await readdir(dir, { withFileTypes: true });
|
|
685
703
|
const out = [];
|
|
704
|
+
const subdirs = [];
|
|
686
705
|
for (const e of entries) {
|
|
687
706
|
const p = join2(dir, e.name);
|
|
688
707
|
if (e.isDirectory()) {
|
|
689
708
|
if (SKIP_DIRS.has(e.name)) continue;
|
|
690
|
-
|
|
709
|
+
subdirs.push(p);
|
|
691
710
|
} else if (e.isFile() && e.name.endsWith(".md")) out.push(p);
|
|
692
711
|
}
|
|
712
|
+
const nested = await mapWithConcurrency(subdirs, Math.min(8, vaultIoConcurrency()), walk);
|
|
713
|
+
for (const files of nested) out.push(...files);
|
|
693
714
|
return out;
|
|
694
715
|
}
|
|
695
716
|
async function readPage(p) {
|
|
696
717
|
return readFile2(p.absPath, "utf8");
|
|
697
718
|
}
|
|
719
|
+
async function readPageCached(p, cache) {
|
|
720
|
+
if (!cache) return readPage(p);
|
|
721
|
+
const existing = cache.get(p.absPath);
|
|
722
|
+
if (existing) return existing;
|
|
723
|
+
const pending = readPage(p);
|
|
724
|
+
cache.set(p.absPath, pending);
|
|
725
|
+
return pending;
|
|
726
|
+
}
|
|
698
727
|
|
|
699
728
|
// src/parsers/wikilinks.ts
|
|
700
729
|
var FENCE = /```[\s\S]*?```|`[^`\n]*`/g;
|
|
@@ -715,20 +744,20 @@ function extractBodyWikilinks(body) {
|
|
|
715
744
|
}
|
|
716
745
|
|
|
717
746
|
// src/utils/community.ts
|
|
718
|
-
async function buildWikilinkAdjacency(typedKnowledge) {
|
|
747
|
+
async function buildWikilinkAdjacency(typedKnowledge, pageTextCache) {
|
|
719
748
|
const adjacency = {};
|
|
720
749
|
const slugToPath = {};
|
|
721
750
|
for (const p of typedKnowledge) {
|
|
722
751
|
const slug = p.relPath.replace(/\.md$/, "").split("/").pop();
|
|
723
752
|
slugToPath[slug] = p.relPath;
|
|
724
753
|
}
|
|
725
|
-
|
|
726
|
-
const text = await
|
|
754
|
+
await mapWithConcurrency(typedKnowledge, vaultIoConcurrency(), async (p) => {
|
|
755
|
+
const text = await readPageCached(p, pageTextCache);
|
|
727
756
|
const split = splitFrontmatter(text);
|
|
728
757
|
const body = split.ok ? split.data.body : text;
|
|
729
758
|
const links = extractBodyWikilinks(body);
|
|
730
759
|
adjacency[p.relPath] = links.map((slug) => slugToPath[slug.split("/").pop()]).filter((x) => Boolean(x));
|
|
731
|
-
}
|
|
760
|
+
});
|
|
732
761
|
return adjacency;
|
|
733
762
|
}
|
|
734
763
|
function toUndirectedWeighted(adj) {
|
|
@@ -1123,7 +1152,7 @@ async function runOrphans(input) {
|
|
|
1123
1152
|
}
|
|
1124
1153
|
vault = r.data.path;
|
|
1125
1154
|
}
|
|
1126
|
-
const scan = await scanVault(vault);
|
|
1155
|
+
const scan = input.scan ? ok(input.scan) : await scanVault(vault);
|
|
1127
1156
|
if (!scan.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scan };
|
|
1128
1157
|
const slugToPath = {};
|
|
1129
1158
|
for (const p of scan.data.typedKnowledge) {
|
|
@@ -1134,8 +1163,8 @@ async function runOrphans(input) {
|
|
|
1134
1163
|
}
|
|
1135
1164
|
const adj = {};
|
|
1136
1165
|
for (const p of scan.data.typedKnowledge) adj[p.relPath] = /* @__PURE__ */ new Set();
|
|
1137
|
-
|
|
1138
|
-
const text = await
|
|
1166
|
+
await mapWithConcurrency(scan.data.typedKnowledge, vaultIoConcurrency(), async (p) => {
|
|
1167
|
+
const text = await readPageCached(p, input.pageTextCache);
|
|
1139
1168
|
const split = splitFrontmatter(text);
|
|
1140
1169
|
const body = split.ok ? split.data.body : text;
|
|
1141
1170
|
for (const slug of extractBodyWikilinks(body)) {
|
|
@@ -1145,7 +1174,7 @@ async function runOrphans(input) {
|
|
|
1145
1174
|
adj[tgt].add(p.relPath);
|
|
1146
1175
|
}
|
|
1147
1176
|
}
|
|
1148
|
-
}
|
|
1177
|
+
});
|
|
1149
1178
|
const orphans = Object.keys(adj).filter((k) => adj[k].size === 0);
|
|
1150
1179
|
const componentOf = {};
|
|
1151
1180
|
let cid = 0;
|
|
@@ -1459,24 +1488,27 @@ function buildSlugMap(pages) {
|
|
|
1459
1488
|
|
|
1460
1489
|
// src/commands/links.ts
|
|
1461
1490
|
async function runLinks(input) {
|
|
1462
|
-
const
|
|
1463
|
-
if (!
|
|
1464
|
-
const
|
|
1491
|
+
const scanResult = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
1492
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
1493
|
+
const scan = scanResult.data;
|
|
1494
|
+
const allPages = [...scan.typedKnowledge, ...scan.raw, ...scan.workItems, ...scan.compound];
|
|
1465
1495
|
const slugs = buildSlugMap(allPages);
|
|
1466
|
-
const
|
|
1467
|
-
|
|
1468
|
-
const text = await readPage(p);
|
|
1496
|
+
const perPage = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (p) => {
|
|
1497
|
+
const text = await readPageCached(p, input.pageTextCache);
|
|
1469
1498
|
const split = splitFrontmatter(text);
|
|
1470
1499
|
const body = split.ok ? split.data.body : text;
|
|
1471
1500
|
const lines = body.split("\n");
|
|
1501
|
+
const broken2 = [];
|
|
1472
1502
|
for (const slug of extractBodyWikilinks(body)) {
|
|
1473
1503
|
const tail = slug.split("/").pop().replace(/\.md$/, "");
|
|
1474
1504
|
if (!slugs.has(tail.toLowerCase())) {
|
|
1475
1505
|
const line = lines.findIndex((l) => l.includes(`[[${slug}`));
|
|
1476
|
-
|
|
1506
|
+
broken2.push({ page: p.relPath, slug, line: line >= 0 ? line + 1 : 0 });
|
|
1477
1507
|
}
|
|
1478
1508
|
}
|
|
1479
|
-
|
|
1509
|
+
return broken2;
|
|
1510
|
+
});
|
|
1511
|
+
const broken = perPage.flat();
|
|
1480
1512
|
if (broken.length > 0) {
|
|
1481
1513
|
return { exitCode: ExitCode.BROKEN_WIKILINKS, result: ok({ broken, humanHint: `broken: ${broken.length}
|
|
1482
1514
|
${broken.map((b) => ` ${b.page}:[[${b.slug}]] (line ${b.line})`).join("\n")}` }) };
|
|
@@ -1515,24 +1547,33 @@ function extractTaxonomy(schemaText) {
|
|
|
1515
1547
|
|
|
1516
1548
|
// src/commands/tag-audit.ts
|
|
1517
1549
|
async function runTagAudit(input) {
|
|
1518
|
-
const
|
|
1519
|
-
if (!
|
|
1550
|
+
const scanResult = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
1551
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
1552
|
+
const scan = scanResult.data;
|
|
1520
1553
|
const schemaText = await readFile5(join6(input.vault, "SCHEMA.md"), "utf8");
|
|
1521
1554
|
const tax = extractTaxonomy(schemaText);
|
|
1522
1555
|
if (!tax.ok) return { exitCode: ExitCode.INVALID_FRONTMATTER, result: tax };
|
|
1523
1556
|
const allowed = new Set(tax.data);
|
|
1524
1557
|
const violations = [];
|
|
1525
|
-
|
|
1526
|
-
const text = await
|
|
1558
|
+
const perPage = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (p) => {
|
|
1559
|
+
const text = await readPageCached(p, input.pageTextCache);
|
|
1527
1560
|
const fm = extractFrontmatter(text);
|
|
1528
|
-
if (!fm.ok) return
|
|
1561
|
+
if (!fm.ok) return fm;
|
|
1562
|
+
const pageViolations = [];
|
|
1529
1563
|
const tags = fm.data.tags;
|
|
1530
|
-
if (!Array.isArray(tags))
|
|
1564
|
+
if (!Array.isArray(tags)) return pageViolations;
|
|
1531
1565
|
for (const t of tags) {
|
|
1532
1566
|
if (typeof t === "string" && !allowed.has(t)) {
|
|
1533
|
-
|
|
1567
|
+
pageViolations.push({ page: p.relPath, tag: t });
|
|
1534
1568
|
}
|
|
1535
1569
|
}
|
|
1570
|
+
return pageViolations;
|
|
1571
|
+
});
|
|
1572
|
+
for (const result of perPage) {
|
|
1573
|
+
if (!Array.isArray(result)) {
|
|
1574
|
+
return { exitCode: ExitCode.INVALID_FRONTMATTER, result };
|
|
1575
|
+
}
|
|
1576
|
+
violations.push(...result);
|
|
1536
1577
|
}
|
|
1537
1578
|
if (violations.length > 0) {
|
|
1538
1579
|
return { exitCode: ExitCode.TAG_NOT_IN_TAXONOMY, result: ok({ violations, taxonomy: tax.data, humanHint: violations.map((v) => `${v.page}: "${v.tag}" not in taxonomy`).join("\n") }) };
|
|
@@ -1544,7 +1585,7 @@ async function runTagAudit(input) {
|
|
|
1544
1585
|
import { readFile as readFile6 } from "fs/promises";
|
|
1545
1586
|
import { join as join7 } from "path";
|
|
1546
1587
|
async function runIndexCheck(input) {
|
|
1547
|
-
const scan = await scanVault(input.vault);
|
|
1588
|
+
const scan = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
1548
1589
|
if (!scan.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scan };
|
|
1549
1590
|
let indexText = "";
|
|
1550
1591
|
try {
|
|
@@ -1671,8 +1712,9 @@ function daysSince(isoDate2) {
|
|
|
1671
1712
|
return Math.floor((Date.now() - Date.parse(isoDate2)) / 864e5);
|
|
1672
1713
|
}
|
|
1673
1714
|
async function runStale(input) {
|
|
1674
|
-
const
|
|
1675
|
-
if (!
|
|
1715
|
+
const scanResult = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
1716
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
1717
|
+
const scan = scanResult.data;
|
|
1676
1718
|
const staleTranscripts = [];
|
|
1677
1719
|
const incompleteWorkItems = [];
|
|
1678
1720
|
const archived = [];
|
|
@@ -1734,16 +1776,16 @@ async function runStale(input) {
|
|
|
1734
1776
|
const TERMINAL_STATUSES = /* @__PURE__ */ new Set(["completed", "abandoned", "done", "invalid"]);
|
|
1735
1777
|
const KIND_FROM_FILENAME = /^(?:\d{4}-\d{2}-\d{2})-(task|bug|idea|note|observation)-.+\.md$/;
|
|
1736
1778
|
const LOOP_CYCLE_PATTERN = /loop-cycle-/;
|
|
1737
|
-
let transcripts = scan.
|
|
1779
|
+
let transcripts = scan.raw.filter((p) => p.relPath.startsWith("raw/transcripts/") && p.relPath.endsWith(".md"));
|
|
1738
1780
|
const claimedPaths = /* @__PURE__ */ new Set();
|
|
1739
1781
|
const transcriptMeta = /* @__PURE__ */ new Map();
|
|
1740
|
-
|
|
1782
|
+
const transcriptEntries = await mapWithConcurrency(transcripts, vaultIoConcurrency(), async (t) => {
|
|
1741
1783
|
try {
|
|
1742
|
-
const content = await
|
|
1784
|
+
const content = await readPageCached(t, input.pageTextCache);
|
|
1743
1785
|
const fm = extractFrontmatter(content);
|
|
1744
1786
|
let kind = fm.ok && typeof fm.data.kind === "string" ? fm.data.kind : "";
|
|
1745
1787
|
let project = fm.ok && typeof fm.data.project === "string" ? fm.data.project : "";
|
|
1746
|
-
if (input.project && !project.includes(input.project))
|
|
1788
|
+
if (input.project && !project.includes(input.project)) return null;
|
|
1747
1789
|
let inferred = false;
|
|
1748
1790
|
if (input.forceScan && !kind) {
|
|
1749
1791
|
const basename3 = t.relPath.split("/").pop();
|
|
@@ -1769,9 +1811,13 @@ async function runStale(input) {
|
|
|
1769
1811
|
}
|
|
1770
1812
|
}
|
|
1771
1813
|
}
|
|
1772
|
-
|
|
1814
|
+
return [t.relPath, { kind, project, slug: extractSlug(project), inferred }];
|
|
1773
1815
|
} catch {
|
|
1816
|
+
return null;
|
|
1774
1817
|
}
|
|
1818
|
+
});
|
|
1819
|
+
for (const entry of transcriptEntries) {
|
|
1820
|
+
if (entry) transcriptMeta.set(entry[0], entry[1]);
|
|
1775
1821
|
}
|
|
1776
1822
|
for (const t of transcripts) {
|
|
1777
1823
|
const datePrefix = t.relPath.split("/").pop().slice(0, 10);
|
|
@@ -1806,7 +1852,7 @@ async function runStale(input) {
|
|
|
1806
1852
|
}
|
|
1807
1853
|
}
|
|
1808
1854
|
}
|
|
1809
|
-
|
|
1855
|
+
await mapWithConcurrency([...workDirs.keys()], vaultIoConcurrency(), async (relDir) => {
|
|
1810
1856
|
const specPath = join9(input.vault, relDir, "spec.md");
|
|
1811
1857
|
try {
|
|
1812
1858
|
const specContent = await readFile7(specPath, "utf8");
|
|
@@ -1817,7 +1863,7 @@ async function runStale(input) {
|
|
|
1817
1863
|
}
|
|
1818
1864
|
} catch {
|
|
1819
1865
|
}
|
|
1820
|
-
}
|
|
1866
|
+
});
|
|
1821
1867
|
const unclaimedTranscripts = [];
|
|
1822
1868
|
const CLAIMABLE_KINDS = /* @__PURE__ */ new Set(["task", "bug"]);
|
|
1823
1869
|
for (const t of transcripts) {
|
|
@@ -1852,42 +1898,45 @@ async function runStale(input) {
|
|
|
1852
1898
|
}
|
|
1853
1899
|
}
|
|
1854
1900
|
const stale = [];
|
|
1855
|
-
|
|
1901
|
+
const stalePageResults = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (page) => {
|
|
1856
1902
|
try {
|
|
1857
|
-
const text = await
|
|
1903
|
+
const text = await readPageCached(page, input.pageTextCache);
|
|
1858
1904
|
const fm = extractFrontmatter(text);
|
|
1859
1905
|
if (fm.ok && typeof fm.data.updated === "string") {
|
|
1860
1906
|
if (input.project) {
|
|
1861
1907
|
const pp = fm.data.provenance_projects;
|
|
1862
1908
|
const linked = Array.isArray(pp) && pp.some((p) => String(p).includes(input.project));
|
|
1863
|
-
if (!linked)
|
|
1909
|
+
if (!linked) return null;
|
|
1864
1910
|
}
|
|
1865
1911
|
const age = daysSince(fm.data.updated);
|
|
1866
1912
|
const threshold = typeof fm.data.stale_ttl === "number" && fm.data.stale_ttl > 0 ? fm.data.stale_ttl : input.days;
|
|
1867
1913
|
if (age >= threshold) {
|
|
1868
|
-
|
|
1914
|
+
return { page: page.relPath, reason: `updated ${age} days ago (threshold: ${threshold})` };
|
|
1869
1915
|
}
|
|
1870
1916
|
}
|
|
1871
1917
|
} catch {
|
|
1872
1918
|
}
|
|
1873
|
-
|
|
1919
|
+
return null;
|
|
1920
|
+
});
|
|
1921
|
+
stale.push(...stalePageResults.filter((item) => item !== null));
|
|
1874
1922
|
const staleSections = [];
|
|
1875
|
-
|
|
1923
|
+
const staleSectionResults = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (page) => {
|
|
1924
|
+
const pageSections = [];
|
|
1876
1925
|
try {
|
|
1877
|
-
const text = await
|
|
1926
|
+
const text = await readPageCached(page, input.pageTextCache);
|
|
1878
1927
|
const projectFilter = input.project;
|
|
1879
1928
|
if (projectFilter) {
|
|
1880
1929
|
const fm = extractFrontmatter(text);
|
|
1881
1930
|
if (fm.ok) {
|
|
1882
1931
|
const pp = fm.data.provenance_projects;
|
|
1883
1932
|
const linked = Array.isArray(pp) && pp.some((p) => String(p).includes(projectFilter));
|
|
1884
|
-
if (!linked)
|
|
1933
|
+
if (!linked) return pageSections;
|
|
1885
1934
|
}
|
|
1886
1935
|
}
|
|
1887
1936
|
const annotations = parseExpiryAnnotations(text, page.relPath);
|
|
1888
1937
|
for (const ann of annotations) {
|
|
1889
1938
|
if (daysSince(ann.expires) >= 0) {
|
|
1890
|
-
|
|
1939
|
+
pageSections.push({
|
|
1891
1940
|
page: ann.page,
|
|
1892
1941
|
heading: ann.heading,
|
|
1893
1942
|
line: ann.line,
|
|
@@ -1900,14 +1949,16 @@ async function runStale(input) {
|
|
|
1900
1949
|
}
|
|
1901
1950
|
} catch {
|
|
1902
1951
|
}
|
|
1903
|
-
|
|
1952
|
+
return pageSections;
|
|
1953
|
+
});
|
|
1954
|
+
staleSections.push(...staleSectionResults.flat());
|
|
1904
1955
|
const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
1905
1956
|
if (input.archive) {
|
|
1906
1957
|
const archiveDir = join9(input.vault, "_archive", today);
|
|
1907
1958
|
await mkdir3(archiveDir, { recursive: true });
|
|
1908
1959
|
const citedRawPaths = /* @__PURE__ */ new Set();
|
|
1909
|
-
for (const page of scan.
|
|
1910
|
-
const text = await
|
|
1960
|
+
for (const page of scan.typedKnowledge) {
|
|
1961
|
+
const text = await readPageCached(page, input.pageTextCache).catch(() => "");
|
|
1911
1962
|
for (const line of text.split("\n")) {
|
|
1912
1963
|
for (const m of line.matchAll(/\^\[(raw\/[^\]]+)\]/g)) {
|
|
1913
1964
|
citedRawPaths.add(m[1]);
|
|
@@ -1982,16 +2033,16 @@ async function runStale(input) {
|
|
|
1982
2033
|
|
|
1983
2034
|
// src/commands/pagesize.ts
|
|
1984
2035
|
async function runPagesize(input) {
|
|
1985
|
-
const
|
|
1986
|
-
if (!
|
|
1987
|
-
const
|
|
1988
|
-
|
|
1989
|
-
const text = await readPage(p);
|
|
2036
|
+
const scanResult = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
2037
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
2038
|
+
const perPage = await mapWithConcurrency(scanResult.data.typedKnowledge, vaultIoConcurrency(), async (p) => {
|
|
2039
|
+
const text = await readPageCached(p, input.pageTextCache);
|
|
1990
2040
|
const split = splitFrontmatter(text);
|
|
1991
2041
|
const body = split.ok ? split.data.body : text;
|
|
1992
2042
|
const count = body.split("\n").length;
|
|
1993
|
-
|
|
1994
|
-
}
|
|
2043
|
+
return count > input.lines ? { page: p.relPath, lines: count } : null;
|
|
2044
|
+
});
|
|
2045
|
+
const oversized = perPage.filter((item) => item !== null);
|
|
1995
2046
|
if (oversized.length > 0) return { exitCode: ExitCode.PAGE_TOO_LARGE, result: ok({ oversized, humanHint: oversized.map((p) => `${p.page}: ${p.lines} lines`).join("\n") }) };
|
|
1996
2047
|
return { exitCode: ExitCode.OK, result: ok({ oversized, humanHint: "all pages within size limit" }) };
|
|
1997
2048
|
}
|
|
@@ -2055,7 +2106,7 @@ Chronological action log. Newest entries last. Skill writes append entries; lint
|
|
|
2055
2106
|
var DEFAULT_THRESHOLD = 200;
|
|
2056
2107
|
async function runTopicMapCheck(input) {
|
|
2057
2108
|
const threshold = input.threshold ?? DEFAULT_THRESHOLD;
|
|
2058
|
-
const scan = await scanVault(input.vault);
|
|
2109
|
+
const scan = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
2059
2110
|
if (!scan.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scan };
|
|
2060
2111
|
const page_count = scan.data.typedKnowledge.length;
|
|
2061
2112
|
const recommended = page_count >= threshold;
|
|
@@ -2152,23 +2203,28 @@ async function runDedup(input) {
|
|
|
2152
2203
|
if (input.remoteDelete && !input.apply && !input.manifestIn) {
|
|
2153
2204
|
return { exitCode: ExitCode.USAGE, result: err("USAGE", { message: "--remote-delete requires --apply or --manifest-in" }) };
|
|
2154
2205
|
}
|
|
2155
|
-
const
|
|
2156
|
-
if (!
|
|
2206
|
+
const scanResult = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
2207
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
2208
|
+
const scan = scanResult.data;
|
|
2157
2209
|
const manifestFromFile = input.manifestIn ? readManifest(input.manifestIn) : null;
|
|
2158
2210
|
if (manifestFromFile && !manifestFromFile.ok) {
|
|
2159
2211
|
return { exitCode: ExitCode.INVALID_FRONTMATTER, result: manifestFromFile };
|
|
2160
2212
|
}
|
|
2161
2213
|
const hashMap = /* @__PURE__ */ new Map();
|
|
2162
2214
|
let totalFiles = 0;
|
|
2163
|
-
|
|
2164
|
-
const fm = extractFrontmatter(await
|
|
2165
|
-
if (!fm.ok)
|
|
2215
|
+
const rawEntries = await mapWithConcurrency(scan.raw, vaultIoConcurrency(), async (raw) => {
|
|
2216
|
+
const fm = extractFrontmatter(await readPageCached(raw, input.pageTextCache));
|
|
2217
|
+
if (!fm.ok) return null;
|
|
2166
2218
|
const sha = typeof fm.data.sha256 === "string" ? fm.data.sha256 : null;
|
|
2167
|
-
if (!sha || sha.length !== 64)
|
|
2219
|
+
if (!sha || sha.length !== 64) return null;
|
|
2220
|
+
return { sha, relPath: raw.relPath };
|
|
2221
|
+
});
|
|
2222
|
+
for (const entry of rawEntries) {
|
|
2223
|
+
if (!entry) continue;
|
|
2168
2224
|
totalFiles++;
|
|
2169
|
-
const existing = hashMap.get(sha);
|
|
2170
|
-
if (existing) existing.push(
|
|
2171
|
-
else hashMap.set(sha, [
|
|
2225
|
+
const existing = hashMap.get(entry.sha);
|
|
2226
|
+
if (existing) existing.push(entry.relPath);
|
|
2227
|
+
else hashMap.set(entry.sha, [entry.relPath]);
|
|
2172
2228
|
}
|
|
2173
2229
|
const canonicalPolicy = input.canonicalPolicy ?? "stable-path";
|
|
2174
2230
|
const duplicates = [...hashMap.entries()].filter(([, files]) => files.length > 1).map(([sha256, files]) => ({ sha256, files: orderFiles(files, canonicalPolicy) }));
|
|
@@ -2205,7 +2261,7 @@ async function runDedup(input) {
|
|
|
2205
2261
|
replacements.set(duplicate, entry.canonical);
|
|
2206
2262
|
}
|
|
2207
2263
|
}
|
|
2208
|
-
for (const page of scan.
|
|
2264
|
+
for (const page of scan.allMarkdown.filter((p) => !p.relPath.startsWith("raw/"))) {
|
|
2209
2265
|
const text = readFileSync2(join12(input.vault, page.relPath), "utf-8");
|
|
2210
2266
|
let updated = text;
|
|
2211
2267
|
let changed = false;
|
|
@@ -2504,9 +2560,9 @@ import { join as join15, relative as relative3, sep as sep3 } from "path";
|
|
|
2504
2560
|
|
|
2505
2561
|
// src/commands/sparse-community.ts
|
|
2506
2562
|
async function runSparseCommunity(input) {
|
|
2507
|
-
const scan = await scanVault(input.vault);
|
|
2563
|
+
const scan = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
2508
2564
|
if (!scan.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scan };
|
|
2509
|
-
const adjacency = await buildWikilinkAdjacency(scan.data.typedKnowledge);
|
|
2565
|
+
const adjacency = await buildWikilinkAdjacency(scan.data.typedKnowledge, input.pageTextCache);
|
|
2510
2566
|
const communities = findSparseCommunities(adjacency, {
|
|
2511
2567
|
minSize: input.minSize,
|
|
2512
2568
|
maxCohesion: input.maxCohesion
|
|
@@ -2517,28 +2573,29 @@ async function runSparseCommunity(input) {
|
|
|
2517
2573
|
|
|
2518
2574
|
// src/commands/raw-body-dedup.ts
|
|
2519
2575
|
import { createHash as createHash3 } from "crypto";
|
|
2520
|
-
async function runRawBodyDedup(vault) {
|
|
2521
|
-
const
|
|
2522
|
-
if (!
|
|
2576
|
+
async function runRawBodyDedup(vault, scan, pageTextCache) {
|
|
2577
|
+
const scanResult = scan ? ok(scan) : await scanVault(vault);
|
|
2578
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
2523
2579
|
const bodyHashMap = /* @__PURE__ */ new Map();
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
const text = await readPage(raw);
|
|
2580
|
+
const rawEntries = await mapWithConcurrency(scanResult.data.raw, vaultIoConcurrency(), async (raw) => {
|
|
2581
|
+
const text = await readPageCached(raw, pageTextCache);
|
|
2527
2582
|
const split = splitFrontmatter(text);
|
|
2528
|
-
if (!split.ok)
|
|
2529
|
-
totalFiles++;
|
|
2583
|
+
if (!split.ok) return null;
|
|
2530
2584
|
const bodyHash = createHash3("sha256").update(split.data.body).digest("hex");
|
|
2531
2585
|
const fm = extractFrontmatter(text);
|
|
2532
2586
|
let fmSha256 = null;
|
|
2533
2587
|
if (fm.ok && typeof fm.data.sha256 === "string" && fm.data.sha256.length === 64) {
|
|
2534
2588
|
fmSha256 = fm.data.sha256;
|
|
2535
2589
|
}
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2590
|
+
return { bodyHash, relPath: raw.relPath, sha256: fmSha256 };
|
|
2591
|
+
});
|
|
2592
|
+
let totalFiles = 0;
|
|
2593
|
+
for (const entry of rawEntries) {
|
|
2594
|
+
if (!entry) continue;
|
|
2595
|
+
totalFiles++;
|
|
2596
|
+
const existing = bodyHashMap.get(entry.bodyHash);
|
|
2597
|
+
if (existing) existing.push({ relPath: entry.relPath, sha256: entry.sha256 });
|
|
2598
|
+
else bodyHashMap.set(entry.bodyHash, [{ relPath: entry.relPath, sha256: entry.sha256 }]);
|
|
2542
2599
|
}
|
|
2543
2600
|
const duplicates = [];
|
|
2544
2601
|
for (const [bodyHash, files] of bodyHashMap) {
|
|
@@ -2562,7 +2619,7 @@ import { dirname as dirname6, join as join14, posix, resolve as resolve4 } from
|
|
|
2562
2619
|
var MAX_PATH_LENGTH = 240;
|
|
2563
2620
|
var WINDOWS_ABSOLUTE_PATH_LIMIT = 259;
|
|
2564
2621
|
async function runPathTooLong(input) {
|
|
2565
|
-
const scan = await scanVault(input.vault);
|
|
2622
|
+
const scan = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
2566
2623
|
if (!scan.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scan };
|
|
2567
2624
|
const violations = findPathTooLongViolations(scan.data.allMarkdown, MAX_PATH_LENGTH);
|
|
2568
2625
|
if (violations.length > 0) {
|
|
@@ -3038,6 +3095,37 @@ function summarizeBucket(bucket, severity, vaultPath, examplesLimit) {
|
|
|
3038
3095
|
details_command: `skillwiki lint ${shellQuote(vaultPath)} --only ${bucket.kind}`
|
|
3039
3096
|
};
|
|
3040
3097
|
}
|
|
3098
|
+
function severityForBucket(kind) {
|
|
3099
|
+
if (ERROR_ORDER.includes(kind)) return "error";
|
|
3100
|
+
if (WARNING_ORDER.includes(kind)) return "warning";
|
|
3101
|
+
return "info";
|
|
3102
|
+
}
|
|
3103
|
+
function outputForOnlyBucket(input, match, fixed, unresolved) {
|
|
3104
|
+
const severity = severityForBucket(input.only);
|
|
3105
|
+
const filtered = severity === "error" ? { error: match, warning: [], info: [] } : severity === "warning" ? { error: [], warning: match, info: [] } : { error: [], warning: [], info: match };
|
|
3106
|
+
const summary = {
|
|
3107
|
+
errors: filtered.error.reduce((n, b) => n + b.items.length, 0),
|
|
3108
|
+
warnings: filtered.warning.reduce((n, b) => n + b.items.length, 0),
|
|
3109
|
+
info: filtered.info.reduce((n, b) => n + b.items.length, 0)
|
|
3110
|
+
};
|
|
3111
|
+
let exitCode = ExitCode.OK;
|
|
3112
|
+
if (summary.errors > 0) exitCode = ExitCode.LINT_HAS_ERRORS;
|
|
3113
|
+
else if (summary.warnings > 0 || summary.info > 0) exitCode = ExitCode.LINT_HAS_WARNINGS;
|
|
3114
|
+
const output = {
|
|
3115
|
+
vault: { path: input.vault, source: input.source ?? "resolved" },
|
|
3116
|
+
summary,
|
|
3117
|
+
by_severity: filtered,
|
|
3118
|
+
fixed,
|
|
3119
|
+
unresolved,
|
|
3120
|
+
humanHint: `--only ${input.only}
|
|
3121
|
+
${match.length === 0 ? "0 violations" : match.map((b) => ` ${b.kind}: ${b.items.length}`).join("\n")}`
|
|
3122
|
+
};
|
|
3123
|
+
if (input.fix) appendLintFixLastOp(input.vault, fixed);
|
|
3124
|
+
return {
|
|
3125
|
+
exitCode,
|
|
3126
|
+
result: ok(input.summary ? summarizeLintOutput(output, input.examplesLimit) : output)
|
|
3127
|
+
};
|
|
3128
|
+
}
|
|
3041
3129
|
function recomputeRawSha256IfPresent(content) {
|
|
3042
3130
|
const split = splitFrontmatter(content);
|
|
3043
3131
|
if (!split.ok) return content;
|
|
@@ -3119,7 +3207,7 @@ async function runCliRefsOnly(input) {
|
|
|
3119
3207
|
const cliRefFlags = [];
|
|
3120
3208
|
const cliSurface = buildCliSurface();
|
|
3121
3209
|
for (const page of pages.data) {
|
|
3122
|
-
const text = await
|
|
3210
|
+
const text = await readPageCached(page);
|
|
3123
3211
|
const violations = validateCliRefs(text, page.relPath, cliSurface);
|
|
3124
3212
|
for (const v of violations) {
|
|
3125
3213
|
cliRefFlags.push(`${v.page}: ${v.ref} (${v.reason})`);
|
|
@@ -3142,6 +3230,151 @@ ${cliRefFlags.length === 0 ? "0 violations" : ` cli_refs: ${cliRefFlags.length}
|
|
|
3142
3230
|
result: ok(input.summary ? summarizeLintOutput(output, input.examplesLimit) : output)
|
|
3143
3231
|
};
|
|
3144
3232
|
}
|
|
3233
|
+
async function collectFileSourceUrlFindings(scan, pageTextCache, options) {
|
|
3234
|
+
const fileSourceUrlFlags = /* @__PURE__ */ new Set();
|
|
3235
|
+
const fileSourceUrlFrontmatterFlags = /* @__PURE__ */ new Set();
|
|
3236
|
+
const rawIdentityConflicts = [];
|
|
3237
|
+
const rawPageBodyByPath = /* @__PURE__ */ new Map();
|
|
3238
|
+
const rawResults = await mapWithConcurrency(scan.raw, vaultIoConcurrency(), async (raw) => {
|
|
3239
|
+
const flags = [];
|
|
3240
|
+
const frontmatterFlags = [];
|
|
3241
|
+
const conflicts = [];
|
|
3242
|
+
let body;
|
|
3243
|
+
try {
|
|
3244
|
+
const text = await readPageCached(raw, pageTextCache);
|
|
3245
|
+
const split = splitFrontmatter(text);
|
|
3246
|
+
if (!split.ok) return { relPath: raw.relPath, body, flags, frontmatterFlags, conflicts };
|
|
3247
|
+
body = split.data.body;
|
|
3248
|
+
if (/^source_url:\s*file:\/\//m.test(split.data.rawFrontmatter)) {
|
|
3249
|
+
flags.push(raw.relPath);
|
|
3250
|
+
frontmatterFlags.push(raw.relPath);
|
|
3251
|
+
}
|
|
3252
|
+
if (options.includeRawIdentityConflicts) {
|
|
3253
|
+
const sourceUrl = split.data.rawFrontmatter.match(/^source_url:\s*(.+)$/m)?.[1]?.trim().replace(/^["']|["']$/g, "") ?? "";
|
|
3254
|
+
const assessment = assessSourceIdentity({
|
|
3255
|
+
rawPath: raw.relPath,
|
|
3256
|
+
sourceUrl,
|
|
3257
|
+
body: split.data.body
|
|
3258
|
+
});
|
|
3259
|
+
if (assessment.status === "conflict") {
|
|
3260
|
+
conflicts.push({
|
|
3261
|
+
file: raw.relPath,
|
|
3262
|
+
status: assessment.status,
|
|
3263
|
+
reasons: assessment.reasons,
|
|
3264
|
+
pathSignals: assessment.pathSignals,
|
|
3265
|
+
sourceSignals: assessment.sourceSignals,
|
|
3266
|
+
bodySignals: assessment.bodySignals
|
|
3267
|
+
});
|
|
3268
|
+
}
|
|
3269
|
+
}
|
|
3270
|
+
} catch {
|
|
3271
|
+
}
|
|
3272
|
+
return { relPath: raw.relPath, body, flags, frontmatterFlags, conflicts };
|
|
3273
|
+
});
|
|
3274
|
+
for (const result of rawResults) {
|
|
3275
|
+
if (result.body !== void 0) rawPageBodyByPath.set(result.relPath, result.body);
|
|
3276
|
+
for (const flag of result.flags) fileSourceUrlFlags.add(flag);
|
|
3277
|
+
for (const flag of result.frontmatterFlags) fileSourceUrlFrontmatterFlags.add(flag);
|
|
3278
|
+
rawIdentityConflicts.push(...result.conflicts);
|
|
3279
|
+
}
|
|
3280
|
+
const canonicalSourcePages = [
|
|
3281
|
+
...scan.raw,
|
|
3282
|
+
...scan.typedKnowledge,
|
|
3283
|
+
...scan.compound,
|
|
3284
|
+
...scan.workItems
|
|
3285
|
+
].filter(shouldCheckCanonicalLocalSourceAssertion);
|
|
3286
|
+
const canonicalFlags = await mapWithConcurrency(canonicalSourcePages, vaultIoConcurrency(), async (page) => {
|
|
3287
|
+
try {
|
|
3288
|
+
let body = rawPageBodyByPath.get(page.relPath);
|
|
3289
|
+
if (body === void 0) {
|
|
3290
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3291
|
+
const split = splitFrontmatter(text);
|
|
3292
|
+
if (!split.ok) return null;
|
|
3293
|
+
body = split.data.body;
|
|
3294
|
+
}
|
|
3295
|
+
return hasCanonicalLocalSourceAssertion(body) ? page.relPath : null;
|
|
3296
|
+
} catch {
|
|
3297
|
+
return null;
|
|
3298
|
+
}
|
|
3299
|
+
});
|
|
3300
|
+
for (const flag of canonicalFlags) {
|
|
3301
|
+
if (flag) fileSourceUrlFlags.add(flag);
|
|
3302
|
+
}
|
|
3303
|
+
return { fileSourceUrlFlags, fileSourceUrlFrontmatterFlags, rawIdentityConflicts };
|
|
3304
|
+
}
|
|
3305
|
+
async function applyFileSourceUrlFix(input, scan, fileSourceUrlFlags, fileSourceUrlFrontmatterFlags, fixed, unresolved) {
|
|
3306
|
+
if (!input.fix || fileSourceUrlFrontmatterFlags.size === 0) return fileSourceUrlFlags;
|
|
3307
|
+
const fileFixed = [];
|
|
3308
|
+
for (const relPath of fileSourceUrlFrontmatterFlags) {
|
|
3309
|
+
try {
|
|
3310
|
+
const absPath = `${input.vault}/${relPath}`;
|
|
3311
|
+
const raw = await readFile12(absPath, "utf8");
|
|
3312
|
+
const parts = raw.split("---", 3);
|
|
3313
|
+
if (parts.length < 3) {
|
|
3314
|
+
unresolved.push(relPath);
|
|
3315
|
+
continue;
|
|
3316
|
+
}
|
|
3317
|
+
const rawFm = parts[1];
|
|
3318
|
+
const rest = parts[2];
|
|
3319
|
+
const sourceMatch = rest.match(/^source:\s*"?(https?:\/\/[^\s\n"]+)"?\s*$/m);
|
|
3320
|
+
if (!sourceMatch) {
|
|
3321
|
+
unresolved.push(relPath);
|
|
3322
|
+
continue;
|
|
3323
|
+
}
|
|
3324
|
+
const realUrl = sourceMatch[1];
|
|
3325
|
+
const newRawFm = rawFm.replace(/^source_url:\s*file:\/\/[^\n]+/m, `source_url: ${realUrl}`);
|
|
3326
|
+
const newContent = `---${newRawFm}---${rest}`;
|
|
3327
|
+
const w = await safeWritePage(absPath, newContent);
|
|
3328
|
+
if (!w.ok) {
|
|
3329
|
+
unresolved.push(relPath);
|
|
3330
|
+
continue;
|
|
3331
|
+
}
|
|
3332
|
+
fileFixed.push(relPath);
|
|
3333
|
+
} catch {
|
|
3334
|
+
unresolved.push(relPath);
|
|
3335
|
+
}
|
|
3336
|
+
}
|
|
3337
|
+
fixed.push(...fileFixed);
|
|
3338
|
+
if (fileFixed.length === 0) return fileSourceUrlFlags;
|
|
3339
|
+
const remaining = new Set(fileSourceUrlFlags);
|
|
3340
|
+
for (const relPath of fileFixed) {
|
|
3341
|
+
try {
|
|
3342
|
+
const page = scan.allMarkdown.find((p) => p.relPath === relPath);
|
|
3343
|
+
if (!page) {
|
|
3344
|
+
remaining.delete(relPath);
|
|
3345
|
+
continue;
|
|
3346
|
+
}
|
|
3347
|
+
const text = await readPage(page);
|
|
3348
|
+
const split = splitFrontmatter(text);
|
|
3349
|
+
if (!split.ok) continue;
|
|
3350
|
+
const stillHasFileSourceUrl = /^source_url:\s*file:\/\//m.test(split.data.rawFrontmatter);
|
|
3351
|
+
const stillHasCanonicalBodyAssertion = shouldCheckCanonicalLocalSourceAssertion(page) && hasCanonicalLocalSourceAssertion(split.data.body);
|
|
3352
|
+
if (!stillHasFileSourceUrl && !stillHasCanonicalBodyAssertion) {
|
|
3353
|
+
remaining.delete(relPath);
|
|
3354
|
+
}
|
|
3355
|
+
} catch {
|
|
3356
|
+
}
|
|
3357
|
+
}
|
|
3358
|
+
return remaining;
|
|
3359
|
+
}
|
|
3360
|
+
async function runFileSourceUrlOnly(input) {
|
|
3361
|
+
const scanResult = await scanVault(input.vault);
|
|
3362
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
3363
|
+
const pageTextCache = /* @__PURE__ */ new Map();
|
|
3364
|
+
const fixed = [];
|
|
3365
|
+
const unresolved = [];
|
|
3366
|
+
const findings = await collectFileSourceUrlFindings(scanResult.data, pageTextCache, { includeRawIdentityConflicts: false });
|
|
3367
|
+
const remaining = await applyFileSourceUrlFix(
|
|
3368
|
+
input,
|
|
3369
|
+
scanResult.data,
|
|
3370
|
+
findings.fileSourceUrlFlags,
|
|
3371
|
+
findings.fileSourceUrlFrontmatterFlags,
|
|
3372
|
+
fixed,
|
|
3373
|
+
unresolved
|
|
3374
|
+
);
|
|
3375
|
+
const match = remaining.size > 0 ? [{ kind: "file_source_url", items: [...remaining] }] : [];
|
|
3376
|
+
return outputForOnlyBucket(input, match, fixed, unresolved);
|
|
3377
|
+
}
|
|
3145
3378
|
async function runLint(input) {
|
|
3146
3379
|
if (input.only && !KNOWN_BUCKETS.includes(input.only)) {
|
|
3147
3380
|
return {
|
|
@@ -3152,21 +3385,30 @@ async function runLint(input) {
|
|
|
3152
3385
|
if (input.only === "cli_refs") {
|
|
3153
3386
|
return runCliRefsOnly(input);
|
|
3154
3387
|
}
|
|
3388
|
+
if (input.only === "file_source_url") {
|
|
3389
|
+
return runFileSourceUrlOnly(input);
|
|
3390
|
+
}
|
|
3155
3391
|
const shouldFix = (bucket) => !!input.fix && (!input.only || input.only === bucket);
|
|
3156
3392
|
const buckets = {};
|
|
3157
3393
|
const fixed = [];
|
|
3158
3394
|
const unresolved = [];
|
|
3159
|
-
const
|
|
3395
|
+
const scanResult = await scanVault(input.vault);
|
|
3396
|
+
if (!scanResult.ok) {
|
|
3397
|
+
return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
3398
|
+
}
|
|
3399
|
+
const scan = scanResult.data;
|
|
3400
|
+
const pageTextCache = /* @__PURE__ */ new Map();
|
|
3401
|
+
const links = await runLinks({ vault: input.vault, scan, pageTextCache });
|
|
3160
3402
|
if (links.result.ok && links.result.data.broken.length > 0) buckets.broken_wikilinks = links.result.data.broken;
|
|
3161
3403
|
if (!links.result.ok && links.result.error === "INVALID_FRONTMATTER") {
|
|
3162
3404
|
buckets.invalid_frontmatter = [links.result.detail ?? {}];
|
|
3163
3405
|
}
|
|
3164
|
-
const tags = await runTagAudit({ vault: input.vault });
|
|
3406
|
+
const tags = await runTagAudit({ vault: input.vault, scan, pageTextCache });
|
|
3165
3407
|
if (tags.result.ok && tags.result.data.violations.length > 0) buckets.tag_not_in_taxonomy = tags.result.data.violations;
|
|
3166
3408
|
if (!tags.result.ok && tags.result.error === "INVALID_FRONTMATTER") {
|
|
3167
3409
|
buckets.invalid_frontmatter = [...buckets.invalid_frontmatter ?? [], tags.result.detail ?? {}];
|
|
3168
3410
|
}
|
|
3169
|
-
const idx = await runIndexCheck({ vault: input.vault });
|
|
3411
|
+
const idx = await runIndexCheck({ vault: input.vault, scan });
|
|
3170
3412
|
if (idx.result.ok && (idx.result.data.missing_from_index.length > 0 || idx.result.data.ghost_entries.length > 0)) {
|
|
3171
3413
|
buckets.index_incomplete = [{
|
|
3172
3414
|
missing_from_index: idx.result.data.missing_from_index,
|
|
@@ -3177,34 +3419,34 @@ async function runLint(input) {
|
|
|
3177
3419
|
if (linkFmt.result.ok && linkFmt.result.data.markdown_links.length > 0) {
|
|
3178
3420
|
buckets.index_link_format = linkFmt.result.data.markdown_links;
|
|
3179
3421
|
}
|
|
3180
|
-
const staleResult = await runStale({ vault: input.vault, days: input.days });
|
|
3422
|
+
const staleResult = await runStale({ vault: input.vault, days: input.days, scan, pageTextCache });
|
|
3181
3423
|
if (staleResult.result.ok) {
|
|
3182
3424
|
const st = staleResult.result.data;
|
|
3183
3425
|
const staleList = [...st.stale_transcripts.map((t) => t.path), ...(st.unclaimed_transcripts ?? []).map((t) => t.path), ...st.incomplete_work_items.map((w) => w.path), ...(st.done_work_items ?? []).map((w) => w.path)];
|
|
3184
3426
|
if (staleList.length > 0) buckets.stale_page = staleList;
|
|
3185
3427
|
}
|
|
3186
|
-
const pagesize = await runPagesize({ vault: input.vault, lines: input.lines });
|
|
3428
|
+
const pagesize = await runPagesize({ vault: input.vault, lines: input.lines, scan, pageTextCache });
|
|
3187
3429
|
if (pagesize.result.ok && pagesize.result.data.oversized.length > 0) buckets.page_too_large = pagesize.result.data.oversized;
|
|
3188
3430
|
const rotate = await runLogRotate({ vault: input.vault, threshold: input.logThreshold, apply: false });
|
|
3189
3431
|
if (rotate.result.ok && rotate.exitCode === ExitCode.LOG_ROTATE_NEEDED) {
|
|
3190
3432
|
buckets.log_rotate_needed = [{ entries: rotate.result.data.entries, threshold: rotate.result.data.threshold }];
|
|
3191
3433
|
}
|
|
3192
|
-
const orphans = await runOrphans({ vault: input.vault });
|
|
3434
|
+
const orphans = await runOrphans({ vault: input.vault, scan, pageTextCache });
|
|
3193
3435
|
if (orphans.result.ok) {
|
|
3194
3436
|
if (orphans.result.data.orphans.length > 0) buckets.orphans = orphans.result.data.orphans;
|
|
3195
3437
|
if (orphans.result.data.bridges.length > 0) buckets.bridges = orphans.result.data.bridges;
|
|
3196
3438
|
}
|
|
3197
|
-
const sparse = await runSparseCommunity({ vault: input.vault });
|
|
3439
|
+
const sparse = await runSparseCommunity({ vault: input.vault, scan, pageTextCache });
|
|
3198
3440
|
if (sparse.result.ok && sparse.result.data.communities.length > 0) {
|
|
3199
3441
|
buckets.sparse_community = sparse.result.data.communities;
|
|
3200
3442
|
}
|
|
3201
|
-
const topicMap = await runTopicMapCheck({ vault: input.vault });
|
|
3443
|
+
const topicMap = await runTopicMapCheck({ vault: input.vault, scan });
|
|
3202
3444
|
if (topicMap.result.ok && topicMap.result.data.recommended) {
|
|
3203
3445
|
buckets.topic_map_recommended = [{ page_count: topicMap.result.data.page_count, threshold: topicMap.result.data.threshold }];
|
|
3204
3446
|
}
|
|
3205
|
-
const dedup = await runDedup({ vault: input.vault });
|
|
3447
|
+
const dedup = await runDedup({ vault: input.vault, scan, pageTextCache });
|
|
3206
3448
|
if (dedup.result.ok && dedup.result.data.duplicates.length > 0) buckets.raw_dedup = dedup.result.data.duplicates;
|
|
3207
|
-
const bodyDedup = await runRawBodyDedup(input.vault);
|
|
3449
|
+
const bodyDedup = await runRawBodyDedup(input.vault, scan, pageTextCache);
|
|
3208
3450
|
if (bodyDedup.result.ok && bodyDedup.result.data.duplicates.length > 0) {
|
|
3209
3451
|
buckets.raw_body_duplicate = bodyDedup.result.data.duplicates.map((d) => ({
|
|
3210
3452
|
body_hash: d.bodyHash.slice(0, 12),
|
|
@@ -3213,40 +3455,35 @@ async function runLint(input) {
|
|
|
3213
3455
|
}
|
|
3214
3456
|
const compoundRefs = await validateCompoundReferences(input.vault);
|
|
3215
3457
|
if (compoundRefs.ok && compoundRefs.data.length > 0) buckets.compound_refs = compoundRefs.data;
|
|
3216
|
-
const pathCheck = await runPathTooLong({ vault: input.vault });
|
|
3458
|
+
const pathCheck = await runPathTooLong({ vault: input.vault, scan });
|
|
3217
3459
|
if (pathCheck.result.ok && pathCheck.result.data.violations.length > 0) buckets.path_too_long = pathCheck.result.data.violations;
|
|
3218
|
-
const
|
|
3219
|
-
const
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
try {
|
|
3225
|
-
const text = await readPage(page);
|
|
3226
|
-
const findings = scanSensitiveContent(text, { file: page.relPath });
|
|
3227
|
-
sensitiveFlags.push(...findings);
|
|
3228
|
-
} catch {
|
|
3229
|
-
}
|
|
3230
|
-
}
|
|
3231
|
-
if (sensitiveFlags.length > 0) buckets.sensitive_content = sensitiveFlags;
|
|
3232
|
-
const fmYamlInvalid = [];
|
|
3233
|
-
for (const page of scan.data.allMarkdown) {
|
|
3460
|
+
const allPages = [...scan.typedKnowledge, ...scan.raw, ...scan.workItems, ...scan.compound];
|
|
3461
|
+
const slugs = buildSlugMap(allPages);
|
|
3462
|
+
{
|
|
3463
|
+
const allPageResults = await mapWithConcurrency(scan.allMarkdown, vaultIoConcurrency(), async (page) => {
|
|
3464
|
+
const sensitiveFlags2 = [];
|
|
3465
|
+
let fmYamlInvalid2 = null;
|
|
3234
3466
|
try {
|
|
3235
|
-
const text = await
|
|
3467
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3468
|
+
sensitiveFlags2.push(...scanSensitiveContent(text, { file: page.relPath }));
|
|
3236
3469
|
const fm = extractFrontmatter(text);
|
|
3237
3470
|
if (!fm.ok && fm.error === "INVALID_FRONTMATTER") {
|
|
3238
3471
|
const detail = fm.detail;
|
|
3239
3472
|
const message = detail?.message ?? "invalid YAML";
|
|
3240
|
-
|
|
3473
|
+
fmYamlInvalid2 = { path: page.relPath, message };
|
|
3241
3474
|
}
|
|
3242
3475
|
} catch {
|
|
3243
3476
|
}
|
|
3244
|
-
|
|
3477
|
+
return { sensitiveFlags: sensitiveFlags2, fmYamlInvalid: fmYamlInvalid2 };
|
|
3478
|
+
});
|
|
3479
|
+
const sensitiveFlags = allPageResults.flatMap((result) => result.sensitiveFlags);
|
|
3480
|
+
if (sensitiveFlags.length > 0) buckets.sensitive_content = sensitiveFlags;
|
|
3481
|
+
const fmYamlInvalid = allPageResults.map((result) => result.fmYamlInvalid).filter((item) => item !== null);
|
|
3245
3482
|
if (fmYamlInvalid.length > 0) buckets.frontmatter_yaml_invalid = fmYamlInvalid;
|
|
3246
3483
|
const subDirDupes = [];
|
|
3247
3484
|
const flatStems = /* @__PURE__ */ new Map();
|
|
3248
3485
|
const deepFiles = [];
|
|
3249
|
-
for (const raw of scan.
|
|
3486
|
+
for (const raw of scan.raw) {
|
|
3250
3487
|
const parts = raw.relPath.split("/");
|
|
3251
3488
|
if (parts.length === 3) {
|
|
3252
3489
|
const stem = parts[2].replace(/\.md$/, "");
|
|
@@ -3265,55 +3502,10 @@ async function runLint(input) {
|
|
|
3265
3502
|
if (subDirDupes.length > 0) {
|
|
3266
3503
|
buckets.raw_subdirectory_duplicate = subDirDupes;
|
|
3267
3504
|
}
|
|
3268
|
-
const
|
|
3269
|
-
|
|
3270
|
-
const
|
|
3271
|
-
const
|
|
3272
|
-
for (const raw of scan.data.raw) {
|
|
3273
|
-
const text = await readPage(raw);
|
|
3274
|
-
const split = splitFrontmatter(text);
|
|
3275
|
-
if (!split.ok) continue;
|
|
3276
|
-
rawPageBodyByPath.set(raw.relPath, split.data.body);
|
|
3277
|
-
if (/^source_url:\s*file:\/\//m.test(split.data.rawFrontmatter)) {
|
|
3278
|
-
fileSourceUrlFlags.add(raw.relPath);
|
|
3279
|
-
fileSourceUrlFrontmatterFlags.add(raw.relPath);
|
|
3280
|
-
}
|
|
3281
|
-
const sourceUrl = split.data.rawFrontmatter.match(/^source_url:\s*(.+)$/m)?.[1]?.trim().replace(/^["']|["']$/g, "") ?? "";
|
|
3282
|
-
const assessment = assessSourceIdentity({
|
|
3283
|
-
rawPath: raw.relPath,
|
|
3284
|
-
sourceUrl,
|
|
3285
|
-
body: split.data.body
|
|
3286
|
-
});
|
|
3287
|
-
if (assessment.status === "conflict") {
|
|
3288
|
-
rawIdentityConflicts.push({
|
|
3289
|
-
file: raw.relPath,
|
|
3290
|
-
status: assessment.status,
|
|
3291
|
-
reasons: assessment.reasons,
|
|
3292
|
-
pathSignals: assessment.pathSignals,
|
|
3293
|
-
sourceSignals: assessment.sourceSignals,
|
|
3294
|
-
bodySignals: assessment.bodySignals
|
|
3295
|
-
});
|
|
3296
|
-
}
|
|
3297
|
-
}
|
|
3298
|
-
const canonicalSourcePages = [
|
|
3299
|
-
...scan.data.raw,
|
|
3300
|
-
...scan.data.typedKnowledge,
|
|
3301
|
-
...scan.data.compound,
|
|
3302
|
-
...scan.data.workItems
|
|
3303
|
-
];
|
|
3304
|
-
for (const page of canonicalSourcePages) {
|
|
3305
|
-
if (!shouldCheckCanonicalLocalSourceAssertion(page)) continue;
|
|
3306
|
-
let body = rawPageBodyByPath.get(page.relPath);
|
|
3307
|
-
if (body === void 0) {
|
|
3308
|
-
const text = await readPage(page);
|
|
3309
|
-
const split = splitFrontmatter(text);
|
|
3310
|
-
if (!split.ok) continue;
|
|
3311
|
-
body = split.data.body;
|
|
3312
|
-
}
|
|
3313
|
-
if (hasCanonicalLocalSourceAssertion(body)) {
|
|
3314
|
-
fileSourceUrlFlags.add(page.relPath);
|
|
3315
|
-
}
|
|
3316
|
-
}
|
|
3505
|
+
const fileSourceUrlFindings = await collectFileSourceUrlFindings(scan, pageTextCache, { includeRawIdentityConflicts: true });
|
|
3506
|
+
let fileSourceUrlFlags = fileSourceUrlFindings.fileSourceUrlFlags;
|
|
3507
|
+
const fileSourceUrlFrontmatterFlags = fileSourceUrlFindings.fileSourceUrlFrontmatterFlags;
|
|
3508
|
+
const rawIdentityConflicts = fileSourceUrlFindings.rawIdentityConflicts;
|
|
3317
3509
|
if (fileSourceUrlFlags.size > 0) buckets.file_source_url = [...fileSourceUrlFlags];
|
|
3318
3510
|
if (rawIdentityConflicts.length > 0) buckets.raw_source_identity_conflict = rawIdentityConflicts;
|
|
3319
3511
|
const legacyPages = [];
|
|
@@ -3326,57 +3518,85 @@ async function runLint(input) {
|
|
|
3326
3518
|
const brokenSourceFlags = /* @__PURE__ */ new Set();
|
|
3327
3519
|
const missingTldrFlags = [];
|
|
3328
3520
|
const missingDiagramFlags = [];
|
|
3329
|
-
|
|
3330
|
-
const
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
3344
|
-
|
|
3521
|
+
const typedPageResults = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (page) => {
|
|
3522
|
+
const result = {
|
|
3523
|
+
legacyPages: [],
|
|
3524
|
+
orphanedPages: [],
|
|
3525
|
+
structFlags: [],
|
|
3526
|
+
dupFrontmatter: [],
|
|
3527
|
+
noOverview: [],
|
|
3528
|
+
fmWikilinkFlags: [],
|
|
3529
|
+
wikilinkCitationFlags: [],
|
|
3530
|
+
brokenSourceFlags: [],
|
|
3531
|
+
missingTldrFlags: [],
|
|
3532
|
+
missingDiagramFlags: []
|
|
3533
|
+
};
|
|
3534
|
+
try {
|
|
3535
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3536
|
+
const split = splitFrontmatter(text);
|
|
3537
|
+
if (!split.ok) return result;
|
|
3538
|
+
const body = split.data.body;
|
|
3539
|
+
const rawFm = split.data.rawFrontmatter;
|
|
3540
|
+
if (hasDuplicateFrontmatter(body)) result.dupFrontmatter.push(page.relPath);
|
|
3541
|
+
if (isLegacyCitationStyle(body)) result.legacyPages.push(page.relPath);
|
|
3542
|
+
if (hasOrphanedCitations(body)) result.orphanedPages.push(page.relPath);
|
|
3543
|
+
if (hasWikilinkCitations(body)) result.wikilinkCitationFlags.push(page.relPath);
|
|
3544
|
+
const sourcesEntries = extractSourceEntries(rawFm);
|
|
3545
|
+
for (const entry of sourcesEntries) {
|
|
3546
|
+
const rawPath = normalizeRawSourceTarget(entry);
|
|
3547
|
+
if (!rawPath) continue;
|
|
3548
|
+
if (!rawSourceTargetExistsSync(input.vault, rawPath)) {
|
|
3549
|
+
result.brokenSourceFlags.push(`${page.relPath}: ${rawPath}`);
|
|
3550
|
+
}
|
|
3345
3551
|
}
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3552
|
+
for (const marker of extractCitationMarkers(body)) {
|
|
3553
|
+
if (!rawSourceTargetExistsSync(input.vault, marker.target)) {
|
|
3554
|
+
result.brokenSourceFlags.push(`${page.relPath}: ${marker.target}`);
|
|
3555
|
+
}
|
|
3350
3556
|
}
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3557
|
+
const fmLinks = rawFm.match(/\[\[([^\[\]|]+)(?:\|[^\[\]]*)?\]\]/g) ?? [];
|
|
3558
|
+
for (const link of fmLinks) {
|
|
3559
|
+
const target = link.replace(/^\[\[/, "").replace(/(?:\|[^\[\]]*)?\]\]$/, "").trim();
|
|
3560
|
+
const tail = target.split("/").pop().replace(/\.md$/, "");
|
|
3561
|
+
if (!slugs.has(tail.toLowerCase())) {
|
|
3562
|
+
result.fmWikilinkFlags.push(`${page.relPath}: [[${target}]] does not resolve`);
|
|
3563
|
+
}
|
|
3358
3564
|
}
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
|
|
3565
|
+
const bodyLines = body.split("\n").filter((l) => l.trim().length > 0).length;
|
|
3566
|
+
const hasOverview = /^## Overview/m.test(body);
|
|
3567
|
+
if (!hasOverview) result.noOverview.push(page.relPath);
|
|
3568
|
+
const bodyFirst15 = body.split("\n").slice(0, 15).join("\n");
|
|
3569
|
+
if (!/^>\s*\*\*TL;DR:?\*\*/m.test(bodyFirst15) && !/^##\s+TL;\s*DR/m.test(bodyFirst15)) result.missingTldrFlags.push(page.relPath);
|
|
3570
|
+
const fmData = extractFrontmatter(text);
|
|
3571
|
+
const pageTags = fmData.ok && Array.isArray(fmData.data.tags) ? fmData.data.tags : [];
|
|
3572
|
+
if (pageTags.includes("architecture") && !body.includes("```mermaid")) {
|
|
3573
|
+
result.missingDiagramFlags.push(page.relPath);
|
|
3574
|
+
}
|
|
3575
|
+
if (bodyLines < STRUCT_MIN_BODY_LINES) {
|
|
3576
|
+
const hasRelated = /^## (Related|Relationships)/m.test(body);
|
|
3577
|
+
const sectionCount = (body.match(/^## /gm) ?? []).length;
|
|
3578
|
+
if (!hasRelated || sectionCount < STRUCT_MIN_SECTIONS) {
|
|
3579
|
+
const reasons = [];
|
|
3580
|
+
if (!hasRelated) reasons.push("no Related or Relationships");
|
|
3581
|
+
if (sectionCount < STRUCT_MIN_SECTIONS) reasons.push(`only ${sectionCount} sections`);
|
|
3582
|
+
result.structFlags.push(`${page.relPath}: ${bodyLines} lines, ${reasons.join(", ")}`);
|
|
3583
|
+
}
|
|
3378
3584
|
}
|
|
3585
|
+
} catch {
|
|
3379
3586
|
}
|
|
3587
|
+
return result;
|
|
3588
|
+
});
|
|
3589
|
+
for (const result of typedPageResults) {
|
|
3590
|
+
legacyPages.push(...result.legacyPages);
|
|
3591
|
+
orphanedPages.push(...result.orphanedPages);
|
|
3592
|
+
structFlags.push(...result.structFlags);
|
|
3593
|
+
dupFrontmatter.push(...result.dupFrontmatter);
|
|
3594
|
+
noOverview.push(...result.noOverview);
|
|
3595
|
+
fmWikilinkFlags.push(...result.fmWikilinkFlags);
|
|
3596
|
+
wikilinkCitationFlags.push(...result.wikilinkCitationFlags);
|
|
3597
|
+
for (const flag of result.brokenSourceFlags) brokenSourceFlags.add(flag);
|
|
3598
|
+
missingTldrFlags.push(...result.missingTldrFlags);
|
|
3599
|
+
missingDiagramFlags.push(...result.missingDiagramFlags);
|
|
3380
3600
|
}
|
|
3381
3601
|
if (legacyPages.length > 0) buckets.legacy_citation_style = legacyPages;
|
|
3382
3602
|
if (orphanedPages.length > 0) buckets.orphaned_citations = orphanedPages;
|
|
@@ -3390,19 +3610,20 @@ async function runLint(input) {
|
|
|
3390
3610
|
if (missingDiagramFlags.length > 0) buckets.missing_diagram = missingDiagramFlags;
|
|
3391
3611
|
const workItemHealth = [];
|
|
3392
3612
|
const workItemDirs = /* @__PURE__ */ new Map();
|
|
3393
|
-
for (const page of scan.
|
|
3613
|
+
for (const page of scan.workItems) {
|
|
3394
3614
|
const dir = page.relPath.replace(/\/(spec|plan|log)\.md$/, "");
|
|
3395
3615
|
const pages = workItemDirs.get(dir) ?? [];
|
|
3396
3616
|
pages.push(page);
|
|
3397
3617
|
workItemDirs.set(dir, pages);
|
|
3398
3618
|
}
|
|
3399
|
-
|
|
3619
|
+
const workItemHealthResults = await mapWithConcurrency([...workItemDirs.entries()], vaultIoConcurrency(), async ([dir, pages]) => {
|
|
3620
|
+
const flags = [];
|
|
3400
3621
|
const specPage = pages.find((p) => p.relPath.endsWith("/spec.md"));
|
|
3401
3622
|
const hasPlan = pages.some((p) => p.relPath.endsWith("/plan.md"));
|
|
3402
3623
|
let specStatus;
|
|
3403
3624
|
let specStarted;
|
|
3404
3625
|
if (specPage) {
|
|
3405
|
-
const text = await
|
|
3626
|
+
const text = await readPageCached(specPage, pageTextCache);
|
|
3406
3627
|
const fm = extractFrontmatter(text);
|
|
3407
3628
|
if (fm.ok) {
|
|
3408
3629
|
specStatus = typeof fm.data.status === "string" ? fm.data.status : void 0;
|
|
@@ -3416,74 +3637,96 @@ async function runLint(input) {
|
|
|
3416
3637
|
if (dateMatch) {
|
|
3417
3638
|
const dirDate = Date.parse(dateMatch[1]);
|
|
3418
3639
|
if (!isNaN(dirDate) && Date.now() - dirDate > 24 * 60 * 60 * 1e3) {
|
|
3419
|
-
|
|
3640
|
+
flags.push(`${dir}/spec.md: has spec but no plan after 24h`);
|
|
3420
3641
|
}
|
|
3421
3642
|
}
|
|
3422
3643
|
}
|
|
3423
3644
|
if (specPage && specStatus === "in-progress" && !specStarted) {
|
|
3424
|
-
|
|
3645
|
+
flags.push(`${specPage.relPath}: in-progress without started date`);
|
|
3425
3646
|
}
|
|
3426
|
-
|
|
3647
|
+
return flags;
|
|
3648
|
+
});
|
|
3649
|
+
workItemHealth.push(...workItemHealthResults.flat());
|
|
3427
3650
|
if (workItemHealth.length > 0) buckets.work_item_health = workItemHealth;
|
|
3428
3651
|
const orphanedProjectPages = [];
|
|
3429
|
-
|
|
3430
|
-
|
|
3431
|
-
const
|
|
3432
|
-
if (
|
|
3433
|
-
const
|
|
3434
|
-
|
|
3435
|
-
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
const
|
|
3442
|
-
|
|
3443
|
-
|
|
3652
|
+
const knowledgeContentCache = /* @__PURE__ */ new Map();
|
|
3653
|
+
const readKnowledgeContent = (slug) => {
|
|
3654
|
+
const existing = knowledgeContentCache.get(slug);
|
|
3655
|
+
if (existing) return existing;
|
|
3656
|
+
const knowledgePath = join15(input.vault, "projects", slug, "knowledge.md");
|
|
3657
|
+
const pending = existsSync4(knowledgePath) ? readFile12(knowledgePath, "utf8").catch(() => null) : Promise.resolve(null);
|
|
3658
|
+
knowledgeContentCache.set(slug, pending);
|
|
3659
|
+
return pending;
|
|
3660
|
+
};
|
|
3661
|
+
const orphanedProjectPageResults = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (page) => {
|
|
3662
|
+
const flags = [];
|
|
3663
|
+
try {
|
|
3664
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3665
|
+
const fm = extractFrontmatter(text);
|
|
3666
|
+
if (!fm.ok) return flags;
|
|
3667
|
+
const pp = fm.data.provenance_projects;
|
|
3668
|
+
if (!Array.isArray(pp)) return flags;
|
|
3669
|
+
for (const entry of pp) {
|
|
3670
|
+
const slugMatch = String(entry).match(/\[\[([^\]]+)\]\]/);
|
|
3671
|
+
if (!slugMatch) continue;
|
|
3672
|
+
const slug = slugMatch[1];
|
|
3673
|
+
const knowledgeContent = await readKnowledgeContent(slug);
|
|
3674
|
+
if (knowledgeContent === null) continue;
|
|
3675
|
+
const pageRef = page.relPath.replace(/\.md$/, "");
|
|
3444
3676
|
if (!knowledgeContent.includes(`[[${pageRef}]]`)) {
|
|
3445
|
-
|
|
3677
|
+
flags.push(`${page.relPath}: not in projects/${slug}/knowledge.md`);
|
|
3446
3678
|
}
|
|
3447
|
-
} catch {
|
|
3448
3679
|
}
|
|
3680
|
+
} catch {
|
|
3449
3681
|
}
|
|
3450
|
-
|
|
3682
|
+
return flags;
|
|
3683
|
+
});
|
|
3684
|
+
orphanedProjectPages.push(...orphanedProjectPageResults.flat());
|
|
3451
3685
|
if (orphanedProjectPages.length > 0) buckets.orphaned_project_pages = orphanedProjectPages;
|
|
3452
3686
|
const cliRefFlags = [];
|
|
3453
3687
|
const cliSurface = buildCliSurface();
|
|
3454
|
-
const allScanPages = [...scan.
|
|
3455
|
-
|
|
3456
|
-
const
|
|
3457
|
-
|
|
3458
|
-
|
|
3459
|
-
|
|
3688
|
+
const allScanPages = [...scan.typedKnowledge];
|
|
3689
|
+
const cliRefResults = await mapWithConcurrency(allScanPages, vaultIoConcurrency(), async (page) => {
|
|
3690
|
+
const flags = [];
|
|
3691
|
+
try {
|
|
3692
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3693
|
+
const violations = validateCliRefs(text, page.relPath, cliSurface);
|
|
3694
|
+
for (const v of violations) {
|
|
3695
|
+
flags.push(`${v.page}: ${v.ref} (${v.reason})`);
|
|
3696
|
+
}
|
|
3697
|
+
} catch {
|
|
3460
3698
|
}
|
|
3461
|
-
|
|
3699
|
+
return flags;
|
|
3700
|
+
});
|
|
3701
|
+
cliRefFlags.push(...cliRefResults.flat());
|
|
3462
3702
|
if (cliRefFlags.length > 0) buckets.cli_refs = cliRefFlags;
|
|
3463
3703
|
const staleSectionFlags = [];
|
|
3464
3704
|
const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
3465
3705
|
const approachingThreshold = 7;
|
|
3466
|
-
|
|
3706
|
+
const staleSectionResults = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (page) => {
|
|
3707
|
+
const flags = [];
|
|
3467
3708
|
try {
|
|
3468
|
-
const text = await
|
|
3709
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3469
3710
|
const annotations = parseExpiryAnnotations(text, page.relPath);
|
|
3470
3711
|
for (const ann of annotations) {
|
|
3471
3712
|
if (ann.expires < today) {
|
|
3472
|
-
|
|
3713
|
+
flags.push(`${page.relPath}: section "${ann.heading}" expired on ${ann.expires}`);
|
|
3473
3714
|
} else {
|
|
3474
3715
|
const daysUntilExpiry = Math.floor((Date.parse(ann.expires) - Date.now()) / 864e5);
|
|
3475
3716
|
if (daysUntilExpiry <= approachingThreshold) {
|
|
3476
|
-
|
|
3717
|
+
flags.push(`${page.relPath}: section "${ann.heading}" expires in ${daysUntilExpiry} day(s) (${ann.expires})`);
|
|
3477
3718
|
}
|
|
3478
3719
|
}
|
|
3479
3720
|
}
|
|
3480
3721
|
} catch {
|
|
3481
3722
|
}
|
|
3482
|
-
|
|
3723
|
+
return flags;
|
|
3724
|
+
});
|
|
3725
|
+
staleSectionFlags.push(...staleSectionResults.flat());
|
|
3483
3726
|
if (staleSectionFlags.length > 0) buckets.stale_sections = staleSectionFlags;
|
|
3484
3727
|
if (shouldFix("sensitive_content") && buckets.sensitive_content) {
|
|
3485
3728
|
const sensitiveFixed = [];
|
|
3486
|
-
for (const page of scan.
|
|
3729
|
+
for (const page of scan.allMarkdown) {
|
|
3487
3730
|
try {
|
|
3488
3731
|
const raw = await readPage(page);
|
|
3489
3732
|
const redacted = redactSensitiveContent(raw, { file: page.relPath });
|
|
@@ -3501,7 +3744,7 @@ async function runLint(input) {
|
|
|
3501
3744
|
}
|
|
3502
3745
|
fixed.push(...sensitiveFixed);
|
|
3503
3746
|
const remainingSensitiveFlags = [];
|
|
3504
|
-
for (const page of scan.
|
|
3747
|
+
for (const page of scan.allMarkdown) {
|
|
3505
3748
|
try {
|
|
3506
3749
|
const text = await readPage(page);
|
|
3507
3750
|
remainingSensitiveFlags.push(...scanSensitiveContent(text, { file: page.relPath }));
|
|
@@ -3790,60 +4033,9 @@ ${newBody}`;
|
|
|
3790
4033
|
}
|
|
3791
4034
|
}
|
|
3792
4035
|
if (shouldFix("file_source_url") && fileSourceUrlFrontmatterFlags.size > 0) {
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
const absPath = `${input.vault}/${relPath}`;
|
|
3797
|
-
const raw = await readFile12(absPath, "utf8");
|
|
3798
|
-
const parts = raw.split("---", 3);
|
|
3799
|
-
if (parts.length < 3) {
|
|
3800
|
-
unresolved.push(relPath);
|
|
3801
|
-
continue;
|
|
3802
|
-
}
|
|
3803
|
-
const rawFm = parts[1];
|
|
3804
|
-
const rest = parts[2];
|
|
3805
|
-
const sourceMatch = rest.match(/^source:\s*"?(https?:\/\/[^\s\n"]+)"?\s*$/m);
|
|
3806
|
-
if (!sourceMatch) {
|
|
3807
|
-
unresolved.push(relPath);
|
|
3808
|
-
continue;
|
|
3809
|
-
}
|
|
3810
|
-
const realUrl = sourceMatch[1];
|
|
3811
|
-
const newRawFm = rawFm.replace(/^source_url:\s*file:\/\/[^\n]+/m, `source_url: ${realUrl}`);
|
|
3812
|
-
const newContent = `---${newRawFm}---${rest}`;
|
|
3813
|
-
const w = await safeWritePage(absPath, newContent);
|
|
3814
|
-
if (!w.ok) {
|
|
3815
|
-
unresolved.push(relPath);
|
|
3816
|
-
continue;
|
|
3817
|
-
}
|
|
3818
|
-
FILE_FIXED.push(relPath);
|
|
3819
|
-
} catch {
|
|
3820
|
-
unresolved.push(relPath);
|
|
3821
|
-
}
|
|
3822
|
-
}
|
|
3823
|
-
fixed.push(...FILE_FIXED);
|
|
3824
|
-
if (FILE_FIXED.length > 0) {
|
|
3825
|
-
const remaining = new Set(fileSourceUrlFlags);
|
|
3826
|
-
for (const relPath of FILE_FIXED) {
|
|
3827
|
-
try {
|
|
3828
|
-
const page = scan.data.allMarkdown.find((p) => p.relPath === relPath);
|
|
3829
|
-
if (!page) {
|
|
3830
|
-
remaining.delete(relPath);
|
|
3831
|
-
continue;
|
|
3832
|
-
}
|
|
3833
|
-
const text = await readPage(page);
|
|
3834
|
-
const split = splitFrontmatter(text);
|
|
3835
|
-
if (!split.ok) continue;
|
|
3836
|
-
const stillHasFileSourceUrl = /^source_url:\s*file:\/\//m.test(split.data.rawFrontmatter);
|
|
3837
|
-
const stillHasCanonicalBodyAssertion = shouldCheckCanonicalLocalSourceAssertion(page) && hasCanonicalLocalSourceAssertion(split.data.body);
|
|
3838
|
-
if (!stillHasFileSourceUrl && !stillHasCanonicalBodyAssertion) {
|
|
3839
|
-
remaining.delete(relPath);
|
|
3840
|
-
}
|
|
3841
|
-
} catch {
|
|
3842
|
-
}
|
|
3843
|
-
}
|
|
3844
|
-
if (remaining.size > 0) buckets.file_source_url = [...remaining];
|
|
3845
|
-
else delete buckets.file_source_url;
|
|
3846
|
-
}
|
|
4036
|
+
fileSourceUrlFlags = await applyFileSourceUrlFix(input, scan, fileSourceUrlFlags, fileSourceUrlFrontmatterFlags, fixed, unresolved);
|
|
4037
|
+
if (fileSourceUrlFlags.size > 0) buckets.file_source_url = [...fileSourceUrlFlags];
|
|
4038
|
+
else delete buckets.file_source_url;
|
|
3847
4039
|
}
|
|
3848
4040
|
const pathViolations = buckets.path_too_long;
|
|
3849
4041
|
if (shouldFix("path_too_long") && pathViolations && pathViolations.length > 0) {
|
|
@@ -3863,7 +4055,7 @@ ${newBody}`;
|
|
|
3863
4055
|
const invalidItems = buckets.frontmatter_yaml_invalid;
|
|
3864
4056
|
const remaining = [];
|
|
3865
4057
|
for (const item of invalidItems) {
|
|
3866
|
-
const page = scan.
|
|
4058
|
+
const page = scan.allMarkdown.find((p) => p.relPath === item.path);
|
|
3867
4059
|
if (!page) {
|
|
3868
4060
|
unresolved.push(item.path);
|
|
3869
4061
|
remaining.push(item);
|
|
@@ -3909,30 +4101,7 @@ ${split.data.body}`;
|
|
|
3909
4101
|
const infoOut = INFO_ORDER.flatMap((k) => buckets[k] ? [{ kind: k, items: buckets[k] }] : []);
|
|
3910
4102
|
if (input.only) {
|
|
3911
4103
|
const match = [...errorOut, ...warningOut, ...infoOut].filter((b) => b.kind === input.only);
|
|
3912
|
-
|
|
3913
|
-
const filtered = severity === "error" ? { error: match, warning: [], info: [] } : severity === "warning" ? { error: [], warning: match, info: [] } : { error: [], warning: [], info: match };
|
|
3914
|
-
const fSummary = {
|
|
3915
|
-
errors: filtered.error.reduce((n, b) => n + b.items.length, 0),
|
|
3916
|
-
warnings: filtered.warning.reduce((n, b) => n + b.items.length, 0),
|
|
3917
|
-
info: filtered.info.reduce((n, b) => n + b.items.length, 0)
|
|
3918
|
-
};
|
|
3919
|
-
let fExit = ExitCode.OK;
|
|
3920
|
-
if (fSummary.errors > 0) fExit = ExitCode.LINT_HAS_ERRORS;
|
|
3921
|
-
else if (fSummary.warnings > 0 || fSummary.info > 0) fExit = ExitCode.LINT_HAS_WARNINGS;
|
|
3922
|
-
const output2 = {
|
|
3923
|
-
vault: { path: input.vault, source: input.source ?? "resolved" },
|
|
3924
|
-
summary: fSummary,
|
|
3925
|
-
by_severity: filtered,
|
|
3926
|
-
fixed,
|
|
3927
|
-
unresolved,
|
|
3928
|
-
humanHint: `--only ${input.only}
|
|
3929
|
-
${match.length === 0 ? "0 violations" : match.map((b) => ` ${b.kind}: ${b.items.length}`).join("\n")}`
|
|
3930
|
-
};
|
|
3931
|
-
if (input.fix) appendLintFixLastOp(input.vault, fixed);
|
|
3932
|
-
return {
|
|
3933
|
-
exitCode: fExit,
|
|
3934
|
-
result: ok(input.summary ? summarizeLintOutput(output2, input.examplesLimit) : output2)
|
|
3935
|
-
};
|
|
4104
|
+
return outputForOnlyBucket(input, match, fixed, unresolved);
|
|
3936
4105
|
}
|
|
3937
4106
|
const summary = {
|
|
3938
4107
|
errors: errorOut.reduce((n, b) => n + b.items.length, 0),
|