skillwiki 0.9.43 → 0.9.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -663,6 +663,24 @@ import { readFile as readFile2, readdir, stat } from "fs/promises";
|
|
|
663
663
|
import { join as join2, relative as relative2, sep as sep2 } from "path";
|
|
664
664
|
var TYPED_DIRS = ["entities", "concepts", "comparisons", "queries", "meta"];
|
|
665
665
|
var SKIP_DIRS = /* @__PURE__ */ new Set([".git", "node_modules"]);
|
|
666
|
+
var DEFAULT_IO_CONCURRENCY = 12;
|
|
667
|
+
function vaultIoConcurrency() {
|
|
668
|
+
const raw = Number.parseInt(process.env.SKILLWIKI_VAULT_IO_CONCURRENCY ?? "", 10);
|
|
669
|
+
return Number.isFinite(raw) && raw > 0 ? Math.min(raw, 64) : DEFAULT_IO_CONCURRENCY;
|
|
670
|
+
}
|
|
671
|
+
async function mapWithConcurrency(items, limit, mapper) {
|
|
672
|
+
const out = new Array(items.length);
|
|
673
|
+
let next = 0;
|
|
674
|
+
const workers = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
|
|
675
|
+
for (; ; ) {
|
|
676
|
+
const index = next++;
|
|
677
|
+
if (index >= items.length) return;
|
|
678
|
+
out[index] = await mapper(items[index], index);
|
|
679
|
+
}
|
|
680
|
+
});
|
|
681
|
+
await Promise.all(workers);
|
|
682
|
+
return out;
|
|
683
|
+
}
|
|
666
684
|
async function scanVault(root) {
|
|
667
685
|
try {
|
|
668
686
|
await stat(join2(root, "SCHEMA.md"));
|
|
@@ -683,18 +701,29 @@ async function scanVault(root) {
|
|
|
683
701
|
async function walk(dir) {
|
|
684
702
|
const entries = await readdir(dir, { withFileTypes: true });
|
|
685
703
|
const out = [];
|
|
704
|
+
const subdirs = [];
|
|
686
705
|
for (const e of entries) {
|
|
687
706
|
const p = join2(dir, e.name);
|
|
688
707
|
if (e.isDirectory()) {
|
|
689
708
|
if (SKIP_DIRS.has(e.name)) continue;
|
|
690
|
-
|
|
709
|
+
subdirs.push(p);
|
|
691
710
|
} else if (e.isFile() && e.name.endsWith(".md")) out.push(p);
|
|
692
711
|
}
|
|
712
|
+
const nested = await mapWithConcurrency(subdirs, Math.min(8, vaultIoConcurrency()), walk);
|
|
713
|
+
for (const files of nested) out.push(...files);
|
|
693
714
|
return out;
|
|
694
715
|
}
|
|
695
716
|
async function readPage(p) {
|
|
696
717
|
return readFile2(p.absPath, "utf8");
|
|
697
718
|
}
|
|
719
|
+
async function readPageCached(p, cache) {
|
|
720
|
+
if (!cache) return readPage(p);
|
|
721
|
+
const existing = cache.get(p.absPath);
|
|
722
|
+
if (existing) return existing;
|
|
723
|
+
const pending = readPage(p);
|
|
724
|
+
cache.set(p.absPath, pending);
|
|
725
|
+
return pending;
|
|
726
|
+
}
|
|
698
727
|
|
|
699
728
|
// src/parsers/wikilinks.ts
|
|
700
729
|
var FENCE = /```[\s\S]*?```|`[^`\n]*`/g;
|
|
@@ -715,20 +744,20 @@ function extractBodyWikilinks(body) {
|
|
|
715
744
|
}
|
|
716
745
|
|
|
717
746
|
// src/utils/community.ts
|
|
718
|
-
async function buildWikilinkAdjacency(typedKnowledge) {
|
|
747
|
+
async function buildWikilinkAdjacency(typedKnowledge, pageTextCache) {
|
|
719
748
|
const adjacency = {};
|
|
720
749
|
const slugToPath = {};
|
|
721
750
|
for (const p of typedKnowledge) {
|
|
722
751
|
const slug = p.relPath.replace(/\.md$/, "").split("/").pop();
|
|
723
752
|
slugToPath[slug] = p.relPath;
|
|
724
753
|
}
|
|
725
|
-
|
|
726
|
-
const text = await
|
|
754
|
+
await mapWithConcurrency(typedKnowledge, vaultIoConcurrency(), async (p) => {
|
|
755
|
+
const text = await readPageCached(p, pageTextCache);
|
|
727
756
|
const split = splitFrontmatter(text);
|
|
728
757
|
const body = split.ok ? split.data.body : text;
|
|
729
758
|
const links = extractBodyWikilinks(body);
|
|
730
759
|
adjacency[p.relPath] = links.map((slug) => slugToPath[slug.split("/").pop()]).filter((x) => Boolean(x));
|
|
731
|
-
}
|
|
760
|
+
});
|
|
732
761
|
return adjacency;
|
|
733
762
|
}
|
|
734
763
|
function toUndirectedWeighted(adj) {
|
|
@@ -1123,7 +1152,7 @@ async function runOrphans(input) {
|
|
|
1123
1152
|
}
|
|
1124
1153
|
vault = r.data.path;
|
|
1125
1154
|
}
|
|
1126
|
-
const scan = await scanVault(vault);
|
|
1155
|
+
const scan = input.scan ? ok(input.scan) : await scanVault(vault);
|
|
1127
1156
|
if (!scan.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scan };
|
|
1128
1157
|
const slugToPath = {};
|
|
1129
1158
|
for (const p of scan.data.typedKnowledge) {
|
|
@@ -1134,8 +1163,8 @@ async function runOrphans(input) {
|
|
|
1134
1163
|
}
|
|
1135
1164
|
const adj = {};
|
|
1136
1165
|
for (const p of scan.data.typedKnowledge) adj[p.relPath] = /* @__PURE__ */ new Set();
|
|
1137
|
-
|
|
1138
|
-
const text = await
|
|
1166
|
+
await mapWithConcurrency(scan.data.typedKnowledge, vaultIoConcurrency(), async (p) => {
|
|
1167
|
+
const text = await readPageCached(p, input.pageTextCache);
|
|
1139
1168
|
const split = splitFrontmatter(text);
|
|
1140
1169
|
const body = split.ok ? split.data.body : text;
|
|
1141
1170
|
for (const slug of extractBodyWikilinks(body)) {
|
|
@@ -1145,7 +1174,7 @@ async function runOrphans(input) {
|
|
|
1145
1174
|
adj[tgt].add(p.relPath);
|
|
1146
1175
|
}
|
|
1147
1176
|
}
|
|
1148
|
-
}
|
|
1177
|
+
});
|
|
1149
1178
|
const orphans = Object.keys(adj).filter((k) => adj[k].size === 0);
|
|
1150
1179
|
const componentOf = {};
|
|
1151
1180
|
let cid = 0;
|
|
@@ -1459,24 +1488,27 @@ function buildSlugMap(pages) {
|
|
|
1459
1488
|
|
|
1460
1489
|
// src/commands/links.ts
|
|
1461
1490
|
async function runLinks(input) {
|
|
1462
|
-
const
|
|
1463
|
-
if (!
|
|
1464
|
-
const
|
|
1491
|
+
const scanResult = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
1492
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
1493
|
+
const scan = scanResult.data;
|
|
1494
|
+
const allPages = [...scan.typedKnowledge, ...scan.raw, ...scan.workItems, ...scan.compound];
|
|
1465
1495
|
const slugs = buildSlugMap(allPages);
|
|
1466
|
-
const
|
|
1467
|
-
|
|
1468
|
-
const text = await readPage(p);
|
|
1496
|
+
const perPage = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (p) => {
|
|
1497
|
+
const text = await readPageCached(p, input.pageTextCache);
|
|
1469
1498
|
const split = splitFrontmatter(text);
|
|
1470
1499
|
const body = split.ok ? split.data.body : text;
|
|
1471
1500
|
const lines = body.split("\n");
|
|
1501
|
+
const broken2 = [];
|
|
1472
1502
|
for (const slug of extractBodyWikilinks(body)) {
|
|
1473
1503
|
const tail = slug.split("/").pop().replace(/\.md$/, "");
|
|
1474
1504
|
if (!slugs.has(tail.toLowerCase())) {
|
|
1475
1505
|
const line = lines.findIndex((l) => l.includes(`[[${slug}`));
|
|
1476
|
-
|
|
1506
|
+
broken2.push({ page: p.relPath, slug, line: line >= 0 ? line + 1 : 0 });
|
|
1477
1507
|
}
|
|
1478
1508
|
}
|
|
1479
|
-
|
|
1509
|
+
return broken2;
|
|
1510
|
+
});
|
|
1511
|
+
const broken = perPage.flat();
|
|
1480
1512
|
if (broken.length > 0) {
|
|
1481
1513
|
return { exitCode: ExitCode.BROKEN_WIKILINKS, result: ok({ broken, humanHint: `broken: ${broken.length}
|
|
1482
1514
|
${broken.map((b) => ` ${b.page}:[[${b.slug}]] (line ${b.line})`).join("\n")}` }) };
|
|
@@ -1515,24 +1547,33 @@ function extractTaxonomy(schemaText) {
|
|
|
1515
1547
|
|
|
1516
1548
|
// src/commands/tag-audit.ts
|
|
1517
1549
|
async function runTagAudit(input) {
|
|
1518
|
-
const
|
|
1519
|
-
if (!
|
|
1550
|
+
const scanResult = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
1551
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
1552
|
+
const scan = scanResult.data;
|
|
1520
1553
|
const schemaText = await readFile5(join6(input.vault, "SCHEMA.md"), "utf8");
|
|
1521
1554
|
const tax = extractTaxonomy(schemaText);
|
|
1522
1555
|
if (!tax.ok) return { exitCode: ExitCode.INVALID_FRONTMATTER, result: tax };
|
|
1523
1556
|
const allowed = new Set(tax.data);
|
|
1524
1557
|
const violations = [];
|
|
1525
|
-
|
|
1526
|
-
const text = await
|
|
1558
|
+
const perPage = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (p) => {
|
|
1559
|
+
const text = await readPageCached(p, input.pageTextCache);
|
|
1527
1560
|
const fm = extractFrontmatter(text);
|
|
1528
|
-
if (!fm.ok) return
|
|
1561
|
+
if (!fm.ok) return fm;
|
|
1562
|
+
const pageViolations = [];
|
|
1529
1563
|
const tags = fm.data.tags;
|
|
1530
|
-
if (!Array.isArray(tags))
|
|
1564
|
+
if (!Array.isArray(tags)) return pageViolations;
|
|
1531
1565
|
for (const t of tags) {
|
|
1532
1566
|
if (typeof t === "string" && !allowed.has(t)) {
|
|
1533
|
-
|
|
1567
|
+
pageViolations.push({ page: p.relPath, tag: t });
|
|
1534
1568
|
}
|
|
1535
1569
|
}
|
|
1570
|
+
return pageViolations;
|
|
1571
|
+
});
|
|
1572
|
+
for (const result of perPage) {
|
|
1573
|
+
if (!Array.isArray(result)) {
|
|
1574
|
+
return { exitCode: ExitCode.INVALID_FRONTMATTER, result };
|
|
1575
|
+
}
|
|
1576
|
+
violations.push(...result);
|
|
1536
1577
|
}
|
|
1537
1578
|
if (violations.length > 0) {
|
|
1538
1579
|
return { exitCode: ExitCode.TAG_NOT_IN_TAXONOMY, result: ok({ violations, taxonomy: tax.data, humanHint: violations.map((v) => `${v.page}: "${v.tag}" not in taxonomy`).join("\n") }) };
|
|
@@ -1544,7 +1585,7 @@ async function runTagAudit(input) {
|
|
|
1544
1585
|
import { readFile as readFile6 } from "fs/promises";
|
|
1545
1586
|
import { join as join7 } from "path";
|
|
1546
1587
|
async function runIndexCheck(input) {
|
|
1547
|
-
const scan = await scanVault(input.vault);
|
|
1588
|
+
const scan = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
1548
1589
|
if (!scan.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scan };
|
|
1549
1590
|
let indexText = "";
|
|
1550
1591
|
try {
|
|
@@ -1671,8 +1712,9 @@ function daysSince(isoDate2) {
|
|
|
1671
1712
|
return Math.floor((Date.now() - Date.parse(isoDate2)) / 864e5);
|
|
1672
1713
|
}
|
|
1673
1714
|
async function runStale(input) {
|
|
1674
|
-
const
|
|
1675
|
-
if (!
|
|
1715
|
+
const scanResult = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
1716
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
1717
|
+
const scan = scanResult.data;
|
|
1676
1718
|
const staleTranscripts = [];
|
|
1677
1719
|
const incompleteWorkItems = [];
|
|
1678
1720
|
const archived = [];
|
|
@@ -1734,16 +1776,16 @@ async function runStale(input) {
|
|
|
1734
1776
|
const TERMINAL_STATUSES = /* @__PURE__ */ new Set(["completed", "abandoned", "done", "invalid"]);
|
|
1735
1777
|
const KIND_FROM_FILENAME = /^(?:\d{4}-\d{2}-\d{2})-(task|bug|idea|note|observation)-.+\.md$/;
|
|
1736
1778
|
const LOOP_CYCLE_PATTERN = /loop-cycle-/;
|
|
1737
|
-
let transcripts = scan.
|
|
1779
|
+
let transcripts = scan.raw.filter((p) => p.relPath.startsWith("raw/transcripts/") && p.relPath.endsWith(".md"));
|
|
1738
1780
|
const claimedPaths = /* @__PURE__ */ new Set();
|
|
1739
1781
|
const transcriptMeta = /* @__PURE__ */ new Map();
|
|
1740
|
-
|
|
1782
|
+
const transcriptEntries = await mapWithConcurrency(transcripts, vaultIoConcurrency(), async (t) => {
|
|
1741
1783
|
try {
|
|
1742
|
-
const content = await
|
|
1784
|
+
const content = await readPageCached(t, input.pageTextCache);
|
|
1743
1785
|
const fm = extractFrontmatter(content);
|
|
1744
1786
|
let kind = fm.ok && typeof fm.data.kind === "string" ? fm.data.kind : "";
|
|
1745
1787
|
let project = fm.ok && typeof fm.data.project === "string" ? fm.data.project : "";
|
|
1746
|
-
if (input.project && !project.includes(input.project))
|
|
1788
|
+
if (input.project && !project.includes(input.project)) return null;
|
|
1747
1789
|
let inferred = false;
|
|
1748
1790
|
if (input.forceScan && !kind) {
|
|
1749
1791
|
const basename3 = t.relPath.split("/").pop();
|
|
@@ -1769,9 +1811,13 @@ async function runStale(input) {
|
|
|
1769
1811
|
}
|
|
1770
1812
|
}
|
|
1771
1813
|
}
|
|
1772
|
-
|
|
1814
|
+
return [t.relPath, { kind, project, slug: extractSlug(project), inferred }];
|
|
1773
1815
|
} catch {
|
|
1816
|
+
return null;
|
|
1774
1817
|
}
|
|
1818
|
+
});
|
|
1819
|
+
for (const entry of transcriptEntries) {
|
|
1820
|
+
if (entry) transcriptMeta.set(entry[0], entry[1]);
|
|
1775
1821
|
}
|
|
1776
1822
|
for (const t of transcripts) {
|
|
1777
1823
|
const datePrefix = t.relPath.split("/").pop().slice(0, 10);
|
|
@@ -1806,7 +1852,7 @@ async function runStale(input) {
|
|
|
1806
1852
|
}
|
|
1807
1853
|
}
|
|
1808
1854
|
}
|
|
1809
|
-
|
|
1855
|
+
await mapWithConcurrency([...workDirs.keys()], vaultIoConcurrency(), async (relDir) => {
|
|
1810
1856
|
const specPath = join9(input.vault, relDir, "spec.md");
|
|
1811
1857
|
try {
|
|
1812
1858
|
const specContent = await readFile7(specPath, "utf8");
|
|
@@ -1817,7 +1863,7 @@ async function runStale(input) {
|
|
|
1817
1863
|
}
|
|
1818
1864
|
} catch {
|
|
1819
1865
|
}
|
|
1820
|
-
}
|
|
1866
|
+
});
|
|
1821
1867
|
const unclaimedTranscripts = [];
|
|
1822
1868
|
const CLAIMABLE_KINDS = /* @__PURE__ */ new Set(["task", "bug"]);
|
|
1823
1869
|
for (const t of transcripts) {
|
|
@@ -1852,42 +1898,45 @@ async function runStale(input) {
|
|
|
1852
1898
|
}
|
|
1853
1899
|
}
|
|
1854
1900
|
const stale = [];
|
|
1855
|
-
|
|
1901
|
+
const stalePageResults = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (page) => {
|
|
1856
1902
|
try {
|
|
1857
|
-
const text = await
|
|
1903
|
+
const text = await readPageCached(page, input.pageTextCache);
|
|
1858
1904
|
const fm = extractFrontmatter(text);
|
|
1859
1905
|
if (fm.ok && typeof fm.data.updated === "string") {
|
|
1860
1906
|
if (input.project) {
|
|
1861
1907
|
const pp = fm.data.provenance_projects;
|
|
1862
1908
|
const linked = Array.isArray(pp) && pp.some((p) => String(p).includes(input.project));
|
|
1863
|
-
if (!linked)
|
|
1909
|
+
if (!linked) return null;
|
|
1864
1910
|
}
|
|
1865
1911
|
const age = daysSince(fm.data.updated);
|
|
1866
1912
|
const threshold = typeof fm.data.stale_ttl === "number" && fm.data.stale_ttl > 0 ? fm.data.stale_ttl : input.days;
|
|
1867
1913
|
if (age >= threshold) {
|
|
1868
|
-
|
|
1914
|
+
return { page: page.relPath, reason: `updated ${age} days ago (threshold: ${threshold})` };
|
|
1869
1915
|
}
|
|
1870
1916
|
}
|
|
1871
1917
|
} catch {
|
|
1872
1918
|
}
|
|
1873
|
-
|
|
1919
|
+
return null;
|
|
1920
|
+
});
|
|
1921
|
+
stale.push(...stalePageResults.filter((item) => item !== null));
|
|
1874
1922
|
const staleSections = [];
|
|
1875
|
-
|
|
1923
|
+
const staleSectionResults = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (page) => {
|
|
1924
|
+
const pageSections = [];
|
|
1876
1925
|
try {
|
|
1877
|
-
const text = await
|
|
1926
|
+
const text = await readPageCached(page, input.pageTextCache);
|
|
1878
1927
|
const projectFilter = input.project;
|
|
1879
1928
|
if (projectFilter) {
|
|
1880
1929
|
const fm = extractFrontmatter(text);
|
|
1881
1930
|
if (fm.ok) {
|
|
1882
1931
|
const pp = fm.data.provenance_projects;
|
|
1883
1932
|
const linked = Array.isArray(pp) && pp.some((p) => String(p).includes(projectFilter));
|
|
1884
|
-
if (!linked)
|
|
1933
|
+
if (!linked) return pageSections;
|
|
1885
1934
|
}
|
|
1886
1935
|
}
|
|
1887
1936
|
const annotations = parseExpiryAnnotations(text, page.relPath);
|
|
1888
1937
|
for (const ann of annotations) {
|
|
1889
1938
|
if (daysSince(ann.expires) >= 0) {
|
|
1890
|
-
|
|
1939
|
+
pageSections.push({
|
|
1891
1940
|
page: ann.page,
|
|
1892
1941
|
heading: ann.heading,
|
|
1893
1942
|
line: ann.line,
|
|
@@ -1900,14 +1949,16 @@ async function runStale(input) {
|
|
|
1900
1949
|
}
|
|
1901
1950
|
} catch {
|
|
1902
1951
|
}
|
|
1903
|
-
|
|
1952
|
+
return pageSections;
|
|
1953
|
+
});
|
|
1954
|
+
staleSections.push(...staleSectionResults.flat());
|
|
1904
1955
|
const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
1905
1956
|
if (input.archive) {
|
|
1906
1957
|
const archiveDir = join9(input.vault, "_archive", today);
|
|
1907
1958
|
await mkdir3(archiveDir, { recursive: true });
|
|
1908
1959
|
const citedRawPaths = /* @__PURE__ */ new Set();
|
|
1909
|
-
for (const page of scan.
|
|
1910
|
-
const text = await
|
|
1960
|
+
for (const page of scan.typedKnowledge) {
|
|
1961
|
+
const text = await readPageCached(page, input.pageTextCache).catch(() => "");
|
|
1911
1962
|
for (const line of text.split("\n")) {
|
|
1912
1963
|
for (const m of line.matchAll(/\^\[(raw\/[^\]]+)\]/g)) {
|
|
1913
1964
|
citedRawPaths.add(m[1]);
|
|
@@ -1982,16 +2033,16 @@ async function runStale(input) {
|
|
|
1982
2033
|
|
|
1983
2034
|
// src/commands/pagesize.ts
|
|
1984
2035
|
async function runPagesize(input) {
|
|
1985
|
-
const
|
|
1986
|
-
if (!
|
|
1987
|
-
const
|
|
1988
|
-
|
|
1989
|
-
const text = await readPage(p);
|
|
2036
|
+
const scanResult = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
2037
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
2038
|
+
const perPage = await mapWithConcurrency(scanResult.data.typedKnowledge, vaultIoConcurrency(), async (p) => {
|
|
2039
|
+
const text = await readPageCached(p, input.pageTextCache);
|
|
1990
2040
|
const split = splitFrontmatter(text);
|
|
1991
2041
|
const body = split.ok ? split.data.body : text;
|
|
1992
2042
|
const count = body.split("\n").length;
|
|
1993
|
-
|
|
1994
|
-
}
|
|
2043
|
+
return count > input.lines ? { page: p.relPath, lines: count } : null;
|
|
2044
|
+
});
|
|
2045
|
+
const oversized = perPage.filter((item) => item !== null);
|
|
1995
2046
|
if (oversized.length > 0) return { exitCode: ExitCode.PAGE_TOO_LARGE, result: ok({ oversized, humanHint: oversized.map((p) => `${p.page}: ${p.lines} lines`).join("\n") }) };
|
|
1996
2047
|
return { exitCode: ExitCode.OK, result: ok({ oversized, humanHint: "all pages within size limit" }) };
|
|
1997
2048
|
}
|
|
@@ -2055,7 +2106,7 @@ Chronological action log. Newest entries last. Skill writes append entries; lint
|
|
|
2055
2106
|
var DEFAULT_THRESHOLD = 200;
|
|
2056
2107
|
async function runTopicMapCheck(input) {
|
|
2057
2108
|
const threshold = input.threshold ?? DEFAULT_THRESHOLD;
|
|
2058
|
-
const scan = await scanVault(input.vault);
|
|
2109
|
+
const scan = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
2059
2110
|
if (!scan.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scan };
|
|
2060
2111
|
const page_count = scan.data.typedKnowledge.length;
|
|
2061
2112
|
const recommended = page_count >= threshold;
|
|
@@ -2152,23 +2203,28 @@ async function runDedup(input) {
|
|
|
2152
2203
|
if (input.remoteDelete && !input.apply && !input.manifestIn) {
|
|
2153
2204
|
return { exitCode: ExitCode.USAGE, result: err("USAGE", { message: "--remote-delete requires --apply or --manifest-in" }) };
|
|
2154
2205
|
}
|
|
2155
|
-
const
|
|
2156
|
-
if (!
|
|
2206
|
+
const scanResult = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
2207
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
2208
|
+
const scan = scanResult.data;
|
|
2157
2209
|
const manifestFromFile = input.manifestIn ? readManifest(input.manifestIn) : null;
|
|
2158
2210
|
if (manifestFromFile && !manifestFromFile.ok) {
|
|
2159
2211
|
return { exitCode: ExitCode.INVALID_FRONTMATTER, result: manifestFromFile };
|
|
2160
2212
|
}
|
|
2161
2213
|
const hashMap = /* @__PURE__ */ new Map();
|
|
2162
2214
|
let totalFiles = 0;
|
|
2163
|
-
|
|
2164
|
-
const fm = extractFrontmatter(await
|
|
2165
|
-
if (!fm.ok)
|
|
2215
|
+
const rawEntries = await mapWithConcurrency(scan.raw, vaultIoConcurrency(), async (raw) => {
|
|
2216
|
+
const fm = extractFrontmatter(await readPageCached(raw, input.pageTextCache));
|
|
2217
|
+
if (!fm.ok) return null;
|
|
2166
2218
|
const sha = typeof fm.data.sha256 === "string" ? fm.data.sha256 : null;
|
|
2167
|
-
if (!sha || sha.length !== 64)
|
|
2219
|
+
if (!sha || sha.length !== 64) return null;
|
|
2220
|
+
return { sha, relPath: raw.relPath };
|
|
2221
|
+
});
|
|
2222
|
+
for (const entry of rawEntries) {
|
|
2223
|
+
if (!entry) continue;
|
|
2168
2224
|
totalFiles++;
|
|
2169
|
-
const existing = hashMap.get(sha);
|
|
2170
|
-
if (existing) existing.push(
|
|
2171
|
-
else hashMap.set(sha, [
|
|
2225
|
+
const existing = hashMap.get(entry.sha);
|
|
2226
|
+
if (existing) existing.push(entry.relPath);
|
|
2227
|
+
else hashMap.set(entry.sha, [entry.relPath]);
|
|
2172
2228
|
}
|
|
2173
2229
|
const canonicalPolicy = input.canonicalPolicy ?? "stable-path";
|
|
2174
2230
|
const duplicates = [...hashMap.entries()].filter(([, files]) => files.length > 1).map(([sha256, files]) => ({ sha256, files: orderFiles(files, canonicalPolicy) }));
|
|
@@ -2205,7 +2261,7 @@ async function runDedup(input) {
|
|
|
2205
2261
|
replacements.set(duplicate, entry.canonical);
|
|
2206
2262
|
}
|
|
2207
2263
|
}
|
|
2208
|
-
for (const page of scan.
|
|
2264
|
+
for (const page of scan.allMarkdown.filter((p) => !p.relPath.startsWith("raw/"))) {
|
|
2209
2265
|
const text = readFileSync2(join12(input.vault, page.relPath), "utf-8");
|
|
2210
2266
|
let updated = text;
|
|
2211
2267
|
let changed = false;
|
|
@@ -2504,9 +2560,9 @@ import { join as join15, relative as relative3, sep as sep3 } from "path";
|
|
|
2504
2560
|
|
|
2505
2561
|
// src/commands/sparse-community.ts
|
|
2506
2562
|
async function runSparseCommunity(input) {
|
|
2507
|
-
const scan = await scanVault(input.vault);
|
|
2563
|
+
const scan = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
2508
2564
|
if (!scan.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scan };
|
|
2509
|
-
const adjacency = await buildWikilinkAdjacency(scan.data.typedKnowledge);
|
|
2565
|
+
const adjacency = await buildWikilinkAdjacency(scan.data.typedKnowledge, input.pageTextCache);
|
|
2510
2566
|
const communities = findSparseCommunities(adjacency, {
|
|
2511
2567
|
minSize: input.minSize,
|
|
2512
2568
|
maxCohesion: input.maxCohesion
|
|
@@ -2517,28 +2573,29 @@ async function runSparseCommunity(input) {
|
|
|
2517
2573
|
|
|
2518
2574
|
// src/commands/raw-body-dedup.ts
|
|
2519
2575
|
import { createHash as createHash3 } from "crypto";
|
|
2520
|
-
async function runRawBodyDedup(vault) {
|
|
2521
|
-
const
|
|
2522
|
-
if (!
|
|
2576
|
+
async function runRawBodyDedup(vault, scan, pageTextCache) {
|
|
2577
|
+
const scanResult = scan ? ok(scan) : await scanVault(vault);
|
|
2578
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
2523
2579
|
const bodyHashMap = /* @__PURE__ */ new Map();
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
const text = await readPage(raw);
|
|
2580
|
+
const rawEntries = await mapWithConcurrency(scanResult.data.raw, vaultIoConcurrency(), async (raw) => {
|
|
2581
|
+
const text = await readPageCached(raw, pageTextCache);
|
|
2527
2582
|
const split = splitFrontmatter(text);
|
|
2528
|
-
if (!split.ok)
|
|
2529
|
-
totalFiles++;
|
|
2583
|
+
if (!split.ok) return null;
|
|
2530
2584
|
const bodyHash = createHash3("sha256").update(split.data.body).digest("hex");
|
|
2531
2585
|
const fm = extractFrontmatter(text);
|
|
2532
2586
|
let fmSha256 = null;
|
|
2533
2587
|
if (fm.ok && typeof fm.data.sha256 === "string" && fm.data.sha256.length === 64) {
|
|
2534
2588
|
fmSha256 = fm.data.sha256;
|
|
2535
2589
|
}
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2590
|
+
return { bodyHash, relPath: raw.relPath, sha256: fmSha256 };
|
|
2591
|
+
});
|
|
2592
|
+
let totalFiles = 0;
|
|
2593
|
+
for (const entry of rawEntries) {
|
|
2594
|
+
if (!entry) continue;
|
|
2595
|
+
totalFiles++;
|
|
2596
|
+
const existing = bodyHashMap.get(entry.bodyHash);
|
|
2597
|
+
if (existing) existing.push({ relPath: entry.relPath, sha256: entry.sha256 });
|
|
2598
|
+
else bodyHashMap.set(entry.bodyHash, [{ relPath: entry.relPath, sha256: entry.sha256 }]);
|
|
2542
2599
|
}
|
|
2543
2600
|
const duplicates = [];
|
|
2544
2601
|
for (const [bodyHash, files] of bodyHashMap) {
|
|
@@ -2562,7 +2619,7 @@ import { dirname as dirname6, join as join14, posix, resolve as resolve4 } from
|
|
|
2562
2619
|
var MAX_PATH_LENGTH = 240;
|
|
2563
2620
|
var WINDOWS_ABSOLUTE_PATH_LIMIT = 259;
|
|
2564
2621
|
async function runPathTooLong(input) {
|
|
2565
|
-
const scan = await scanVault(input.vault);
|
|
2622
|
+
const scan = input.scan ? ok(input.scan) : await scanVault(input.vault);
|
|
2566
2623
|
if (!scan.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scan };
|
|
2567
2624
|
const violations = findPathTooLongViolations(scan.data.allMarkdown, MAX_PATH_LENGTH);
|
|
2568
2625
|
if (violations.length > 0) {
|
|
@@ -3038,6 +3095,37 @@ function summarizeBucket(bucket, severity, vaultPath, examplesLimit) {
|
|
|
3038
3095
|
details_command: `skillwiki lint ${shellQuote(vaultPath)} --only ${bucket.kind}`
|
|
3039
3096
|
};
|
|
3040
3097
|
}
|
|
3098
|
+
function severityForBucket(kind) {
|
|
3099
|
+
if (ERROR_ORDER.includes(kind)) return "error";
|
|
3100
|
+
if (WARNING_ORDER.includes(kind)) return "warning";
|
|
3101
|
+
return "info";
|
|
3102
|
+
}
|
|
3103
|
+
function outputForOnlyBucket(input, match, fixed, unresolved) {
|
|
3104
|
+
const severity = severityForBucket(input.only);
|
|
3105
|
+
const filtered = severity === "error" ? { error: match, warning: [], info: [] } : severity === "warning" ? { error: [], warning: match, info: [] } : { error: [], warning: [], info: match };
|
|
3106
|
+
const summary = {
|
|
3107
|
+
errors: filtered.error.reduce((n, b) => n + b.items.length, 0),
|
|
3108
|
+
warnings: filtered.warning.reduce((n, b) => n + b.items.length, 0),
|
|
3109
|
+
info: filtered.info.reduce((n, b) => n + b.items.length, 0)
|
|
3110
|
+
};
|
|
3111
|
+
let exitCode = ExitCode.OK;
|
|
3112
|
+
if (summary.errors > 0) exitCode = ExitCode.LINT_HAS_ERRORS;
|
|
3113
|
+
else if (summary.warnings > 0 || summary.info > 0) exitCode = ExitCode.LINT_HAS_WARNINGS;
|
|
3114
|
+
const output = {
|
|
3115
|
+
vault: { path: input.vault, source: input.source ?? "resolved" },
|
|
3116
|
+
summary,
|
|
3117
|
+
by_severity: filtered,
|
|
3118
|
+
fixed,
|
|
3119
|
+
unresolved,
|
|
3120
|
+
humanHint: `--only ${input.only}
|
|
3121
|
+
${match.length === 0 ? "0 violations" : match.map((b) => ` ${b.kind}: ${b.items.length}`).join("\n")}`
|
|
3122
|
+
};
|
|
3123
|
+
if (input.fix) appendLintFixLastOp(input.vault, fixed);
|
|
3124
|
+
return {
|
|
3125
|
+
exitCode,
|
|
3126
|
+
result: ok(input.summary ? summarizeLintOutput(output, input.examplesLimit) : output)
|
|
3127
|
+
};
|
|
3128
|
+
}
|
|
3041
3129
|
function recomputeRawSha256IfPresent(content) {
|
|
3042
3130
|
const split = splitFrontmatter(content);
|
|
3043
3131
|
if (!split.ok) return content;
|
|
@@ -3119,7 +3207,7 @@ async function runCliRefsOnly(input) {
|
|
|
3119
3207
|
const cliRefFlags = [];
|
|
3120
3208
|
const cliSurface = buildCliSurface();
|
|
3121
3209
|
for (const page of pages.data) {
|
|
3122
|
-
const text = await
|
|
3210
|
+
const text = await readPageCached(page);
|
|
3123
3211
|
const violations = validateCliRefs(text, page.relPath, cliSurface);
|
|
3124
3212
|
for (const v of violations) {
|
|
3125
3213
|
cliRefFlags.push(`${v.page}: ${v.ref} (${v.reason})`);
|
|
@@ -3142,6 +3230,151 @@ ${cliRefFlags.length === 0 ? "0 violations" : ` cli_refs: ${cliRefFlags.length}
|
|
|
3142
3230
|
result: ok(input.summary ? summarizeLintOutput(output, input.examplesLimit) : output)
|
|
3143
3231
|
};
|
|
3144
3232
|
}
|
|
3233
|
+
async function collectFileSourceUrlFindings(scan, pageTextCache, options) {
|
|
3234
|
+
const fileSourceUrlFlags = /* @__PURE__ */ new Set();
|
|
3235
|
+
const fileSourceUrlFrontmatterFlags = /* @__PURE__ */ new Set();
|
|
3236
|
+
const rawIdentityConflicts = [];
|
|
3237
|
+
const rawPageBodyByPath = /* @__PURE__ */ new Map();
|
|
3238
|
+
const rawResults = await mapWithConcurrency(scan.raw, vaultIoConcurrency(), async (raw) => {
|
|
3239
|
+
const flags = [];
|
|
3240
|
+
const frontmatterFlags = [];
|
|
3241
|
+
const conflicts = [];
|
|
3242
|
+
let body;
|
|
3243
|
+
try {
|
|
3244
|
+
const text = await readPageCached(raw, pageTextCache);
|
|
3245
|
+
const split = splitFrontmatter(text);
|
|
3246
|
+
if (!split.ok) return { relPath: raw.relPath, body, flags, frontmatterFlags, conflicts };
|
|
3247
|
+
body = split.data.body;
|
|
3248
|
+
if (/^source_url:\s*file:\/\//m.test(split.data.rawFrontmatter)) {
|
|
3249
|
+
flags.push(raw.relPath);
|
|
3250
|
+
frontmatterFlags.push(raw.relPath);
|
|
3251
|
+
}
|
|
3252
|
+
if (options.includeRawIdentityConflicts) {
|
|
3253
|
+
const sourceUrl = split.data.rawFrontmatter.match(/^source_url:\s*(.+)$/m)?.[1]?.trim().replace(/^["']|["']$/g, "") ?? "";
|
|
3254
|
+
const assessment = assessSourceIdentity({
|
|
3255
|
+
rawPath: raw.relPath,
|
|
3256
|
+
sourceUrl,
|
|
3257
|
+
body: split.data.body
|
|
3258
|
+
});
|
|
3259
|
+
if (assessment.status === "conflict") {
|
|
3260
|
+
conflicts.push({
|
|
3261
|
+
file: raw.relPath,
|
|
3262
|
+
status: assessment.status,
|
|
3263
|
+
reasons: assessment.reasons,
|
|
3264
|
+
pathSignals: assessment.pathSignals,
|
|
3265
|
+
sourceSignals: assessment.sourceSignals,
|
|
3266
|
+
bodySignals: assessment.bodySignals
|
|
3267
|
+
});
|
|
3268
|
+
}
|
|
3269
|
+
}
|
|
3270
|
+
} catch {
|
|
3271
|
+
}
|
|
3272
|
+
return { relPath: raw.relPath, body, flags, frontmatterFlags, conflicts };
|
|
3273
|
+
});
|
|
3274
|
+
for (const result of rawResults) {
|
|
3275
|
+
if (result.body !== void 0) rawPageBodyByPath.set(result.relPath, result.body);
|
|
3276
|
+
for (const flag of result.flags) fileSourceUrlFlags.add(flag);
|
|
3277
|
+
for (const flag of result.frontmatterFlags) fileSourceUrlFrontmatterFlags.add(flag);
|
|
3278
|
+
rawIdentityConflicts.push(...result.conflicts);
|
|
3279
|
+
}
|
|
3280
|
+
const canonicalSourcePages = [
|
|
3281
|
+
...scan.raw,
|
|
3282
|
+
...scan.typedKnowledge,
|
|
3283
|
+
...scan.compound,
|
|
3284
|
+
...scan.workItems
|
|
3285
|
+
].filter(shouldCheckCanonicalLocalSourceAssertion);
|
|
3286
|
+
const canonicalFlags = await mapWithConcurrency(canonicalSourcePages, vaultIoConcurrency(), async (page) => {
|
|
3287
|
+
try {
|
|
3288
|
+
let body = rawPageBodyByPath.get(page.relPath);
|
|
3289
|
+
if (body === void 0) {
|
|
3290
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3291
|
+
const split = splitFrontmatter(text);
|
|
3292
|
+
if (!split.ok) return null;
|
|
3293
|
+
body = split.data.body;
|
|
3294
|
+
}
|
|
3295
|
+
return hasCanonicalLocalSourceAssertion(body) ? page.relPath : null;
|
|
3296
|
+
} catch {
|
|
3297
|
+
return null;
|
|
3298
|
+
}
|
|
3299
|
+
});
|
|
3300
|
+
for (const flag of canonicalFlags) {
|
|
3301
|
+
if (flag) fileSourceUrlFlags.add(flag);
|
|
3302
|
+
}
|
|
3303
|
+
return { fileSourceUrlFlags, fileSourceUrlFrontmatterFlags, rawIdentityConflicts };
|
|
3304
|
+
}
|
|
3305
|
+
async function applyFileSourceUrlFix(input, scan, fileSourceUrlFlags, fileSourceUrlFrontmatterFlags, fixed, unresolved) {
|
|
3306
|
+
if (!input.fix || fileSourceUrlFrontmatterFlags.size === 0) return fileSourceUrlFlags;
|
|
3307
|
+
const fileFixed = [];
|
|
3308
|
+
for (const relPath of fileSourceUrlFrontmatterFlags) {
|
|
3309
|
+
try {
|
|
3310
|
+
const absPath = `${input.vault}/${relPath}`;
|
|
3311
|
+
const raw = await readFile12(absPath, "utf8");
|
|
3312
|
+
const parts = raw.split("---", 3);
|
|
3313
|
+
if (parts.length < 3) {
|
|
3314
|
+
unresolved.push(relPath);
|
|
3315
|
+
continue;
|
|
3316
|
+
}
|
|
3317
|
+
const rawFm = parts[1];
|
|
3318
|
+
const rest = parts[2];
|
|
3319
|
+
const sourceMatch = rest.match(/^source:\s*"?(https?:\/\/[^\s\n"]+)"?\s*$/m);
|
|
3320
|
+
if (!sourceMatch) {
|
|
3321
|
+
unresolved.push(relPath);
|
|
3322
|
+
continue;
|
|
3323
|
+
}
|
|
3324
|
+
const realUrl = sourceMatch[1];
|
|
3325
|
+
const newRawFm = rawFm.replace(/^source_url:\s*file:\/\/[^\n]+/m, `source_url: ${realUrl}`);
|
|
3326
|
+
const newContent = `---${newRawFm}---${rest}`;
|
|
3327
|
+
const w = await safeWritePage(absPath, newContent);
|
|
3328
|
+
if (!w.ok) {
|
|
3329
|
+
unresolved.push(relPath);
|
|
3330
|
+
continue;
|
|
3331
|
+
}
|
|
3332
|
+
fileFixed.push(relPath);
|
|
3333
|
+
} catch {
|
|
3334
|
+
unresolved.push(relPath);
|
|
3335
|
+
}
|
|
3336
|
+
}
|
|
3337
|
+
fixed.push(...fileFixed);
|
|
3338
|
+
if (fileFixed.length === 0) return fileSourceUrlFlags;
|
|
3339
|
+
const remaining = new Set(fileSourceUrlFlags);
|
|
3340
|
+
for (const relPath of fileFixed) {
|
|
3341
|
+
try {
|
|
3342
|
+
const page = scan.allMarkdown.find((p) => p.relPath === relPath);
|
|
3343
|
+
if (!page) {
|
|
3344
|
+
remaining.delete(relPath);
|
|
3345
|
+
continue;
|
|
3346
|
+
}
|
|
3347
|
+
const text = await readPage(page);
|
|
3348
|
+
const split = splitFrontmatter(text);
|
|
3349
|
+
if (!split.ok) continue;
|
|
3350
|
+
const stillHasFileSourceUrl = /^source_url:\s*file:\/\//m.test(split.data.rawFrontmatter);
|
|
3351
|
+
const stillHasCanonicalBodyAssertion = shouldCheckCanonicalLocalSourceAssertion(page) && hasCanonicalLocalSourceAssertion(split.data.body);
|
|
3352
|
+
if (!stillHasFileSourceUrl && !stillHasCanonicalBodyAssertion) {
|
|
3353
|
+
remaining.delete(relPath);
|
|
3354
|
+
}
|
|
3355
|
+
} catch {
|
|
3356
|
+
}
|
|
3357
|
+
}
|
|
3358
|
+
return remaining;
|
|
3359
|
+
}
|
|
3360
|
+
async function runFileSourceUrlOnly(input) {
|
|
3361
|
+
const scanResult = await scanVault(input.vault);
|
|
3362
|
+
if (!scanResult.ok) return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
3363
|
+
const pageTextCache = /* @__PURE__ */ new Map();
|
|
3364
|
+
const fixed = [];
|
|
3365
|
+
const unresolved = [];
|
|
3366
|
+
const findings = await collectFileSourceUrlFindings(scanResult.data, pageTextCache, { includeRawIdentityConflicts: false });
|
|
3367
|
+
const remaining = await applyFileSourceUrlFix(
|
|
3368
|
+
input,
|
|
3369
|
+
scanResult.data,
|
|
3370
|
+
findings.fileSourceUrlFlags,
|
|
3371
|
+
findings.fileSourceUrlFrontmatterFlags,
|
|
3372
|
+
fixed,
|
|
3373
|
+
unresolved
|
|
3374
|
+
);
|
|
3375
|
+
const match = remaining.size > 0 ? [{ kind: "file_source_url", items: [...remaining] }] : [];
|
|
3376
|
+
return outputForOnlyBucket(input, match, fixed, unresolved);
|
|
3377
|
+
}
|
|
3145
3378
|
async function runLint(input) {
|
|
3146
3379
|
if (input.only && !KNOWN_BUCKETS.includes(input.only)) {
|
|
3147
3380
|
return {
|
|
@@ -3152,21 +3385,38 @@ async function runLint(input) {
|
|
|
3152
3385
|
if (input.only === "cli_refs") {
|
|
3153
3386
|
return runCliRefsOnly(input);
|
|
3154
3387
|
}
|
|
3388
|
+
if (input.only === "file_source_url") {
|
|
3389
|
+
return runFileSourceUrlOnly(input);
|
|
3390
|
+
}
|
|
3155
3391
|
const shouldFix = (bucket) => !!input.fix && (!input.only || input.only === bucket);
|
|
3156
3392
|
const buckets = {};
|
|
3157
3393
|
const fixed = [];
|
|
3158
3394
|
const unresolved = [];
|
|
3159
|
-
const
|
|
3395
|
+
const scanResult = await scanVault(input.vault);
|
|
3396
|
+
if (!scanResult.ok) {
|
|
3397
|
+
return { exitCode: ExitCode.VAULT_PATH_INVALID, result: scanResult };
|
|
3398
|
+
}
|
|
3399
|
+
const scan = scanResult.data;
|
|
3400
|
+
const pageTextCache = /* @__PURE__ */ new Map();
|
|
3401
|
+
if (!input.fix) {
|
|
3402
|
+
await mapWithConcurrency(scan.allMarkdown, vaultIoConcurrency(), async (page) => {
|
|
3403
|
+
try {
|
|
3404
|
+
await readPageCached(page, pageTextCache);
|
|
3405
|
+
} catch {
|
|
3406
|
+
}
|
|
3407
|
+
});
|
|
3408
|
+
}
|
|
3409
|
+
const links = await runLinks({ vault: input.vault, scan, pageTextCache });
|
|
3160
3410
|
if (links.result.ok && links.result.data.broken.length > 0) buckets.broken_wikilinks = links.result.data.broken;
|
|
3161
3411
|
if (!links.result.ok && links.result.error === "INVALID_FRONTMATTER") {
|
|
3162
3412
|
buckets.invalid_frontmatter = [links.result.detail ?? {}];
|
|
3163
3413
|
}
|
|
3164
|
-
const tags = await runTagAudit({ vault: input.vault });
|
|
3414
|
+
const tags = await runTagAudit({ vault: input.vault, scan, pageTextCache });
|
|
3165
3415
|
if (tags.result.ok && tags.result.data.violations.length > 0) buckets.tag_not_in_taxonomy = tags.result.data.violations;
|
|
3166
3416
|
if (!tags.result.ok && tags.result.error === "INVALID_FRONTMATTER") {
|
|
3167
3417
|
buckets.invalid_frontmatter = [...buckets.invalid_frontmatter ?? [], tags.result.detail ?? {}];
|
|
3168
3418
|
}
|
|
3169
|
-
const idx = await runIndexCheck({ vault: input.vault });
|
|
3419
|
+
const idx = await runIndexCheck({ vault: input.vault, scan });
|
|
3170
3420
|
if (idx.result.ok && (idx.result.data.missing_from_index.length > 0 || idx.result.data.ghost_entries.length > 0)) {
|
|
3171
3421
|
buckets.index_incomplete = [{
|
|
3172
3422
|
missing_from_index: idx.result.data.missing_from_index,
|
|
@@ -3177,34 +3427,34 @@ async function runLint(input) {
|
|
|
3177
3427
|
if (linkFmt.result.ok && linkFmt.result.data.markdown_links.length > 0) {
|
|
3178
3428
|
buckets.index_link_format = linkFmt.result.data.markdown_links;
|
|
3179
3429
|
}
|
|
3180
|
-
const staleResult = await runStale({ vault: input.vault, days: input.days });
|
|
3430
|
+
const staleResult = await runStale({ vault: input.vault, days: input.days, scan, pageTextCache });
|
|
3181
3431
|
if (staleResult.result.ok) {
|
|
3182
3432
|
const st = staleResult.result.data;
|
|
3183
3433
|
const staleList = [...st.stale_transcripts.map((t) => t.path), ...(st.unclaimed_transcripts ?? []).map((t) => t.path), ...st.incomplete_work_items.map((w) => w.path), ...(st.done_work_items ?? []).map((w) => w.path)];
|
|
3184
3434
|
if (staleList.length > 0) buckets.stale_page = staleList;
|
|
3185
3435
|
}
|
|
3186
|
-
const pagesize = await runPagesize({ vault: input.vault, lines: input.lines });
|
|
3436
|
+
const pagesize = await runPagesize({ vault: input.vault, lines: input.lines, scan, pageTextCache });
|
|
3187
3437
|
if (pagesize.result.ok && pagesize.result.data.oversized.length > 0) buckets.page_too_large = pagesize.result.data.oversized;
|
|
3188
3438
|
const rotate = await runLogRotate({ vault: input.vault, threshold: input.logThreshold, apply: false });
|
|
3189
3439
|
if (rotate.result.ok && rotate.exitCode === ExitCode.LOG_ROTATE_NEEDED) {
|
|
3190
3440
|
buckets.log_rotate_needed = [{ entries: rotate.result.data.entries, threshold: rotate.result.data.threshold }];
|
|
3191
3441
|
}
|
|
3192
|
-
const orphans = await runOrphans({ vault: input.vault });
|
|
3442
|
+
const orphans = await runOrphans({ vault: input.vault, scan, pageTextCache });
|
|
3193
3443
|
if (orphans.result.ok) {
|
|
3194
3444
|
if (orphans.result.data.orphans.length > 0) buckets.orphans = orphans.result.data.orphans;
|
|
3195
3445
|
if (orphans.result.data.bridges.length > 0) buckets.bridges = orphans.result.data.bridges;
|
|
3196
3446
|
}
|
|
3197
|
-
const sparse = await runSparseCommunity({ vault: input.vault });
|
|
3447
|
+
const sparse = await runSparseCommunity({ vault: input.vault, scan, pageTextCache });
|
|
3198
3448
|
if (sparse.result.ok && sparse.result.data.communities.length > 0) {
|
|
3199
3449
|
buckets.sparse_community = sparse.result.data.communities;
|
|
3200
3450
|
}
|
|
3201
|
-
const topicMap = await runTopicMapCheck({ vault: input.vault });
|
|
3451
|
+
const topicMap = await runTopicMapCheck({ vault: input.vault, scan });
|
|
3202
3452
|
if (topicMap.result.ok && topicMap.result.data.recommended) {
|
|
3203
3453
|
buckets.topic_map_recommended = [{ page_count: topicMap.result.data.page_count, threshold: topicMap.result.data.threshold }];
|
|
3204
3454
|
}
|
|
3205
|
-
const dedup = await runDedup({ vault: input.vault });
|
|
3455
|
+
const dedup = await runDedup({ vault: input.vault, scan, pageTextCache });
|
|
3206
3456
|
if (dedup.result.ok && dedup.result.data.duplicates.length > 0) buckets.raw_dedup = dedup.result.data.duplicates;
|
|
3207
|
-
const bodyDedup = await runRawBodyDedup(input.vault);
|
|
3457
|
+
const bodyDedup = await runRawBodyDedup(input.vault, scan, pageTextCache);
|
|
3208
3458
|
if (bodyDedup.result.ok && bodyDedup.result.data.duplicates.length > 0) {
|
|
3209
3459
|
buckets.raw_body_duplicate = bodyDedup.result.data.duplicates.map((d) => ({
|
|
3210
3460
|
body_hash: d.bodyHash.slice(0, 12),
|
|
@@ -3213,40 +3463,35 @@ async function runLint(input) {
|
|
|
3213
3463
|
}
|
|
3214
3464
|
const compoundRefs = await validateCompoundReferences(input.vault);
|
|
3215
3465
|
if (compoundRefs.ok && compoundRefs.data.length > 0) buckets.compound_refs = compoundRefs.data;
|
|
3216
|
-
const pathCheck = await runPathTooLong({ vault: input.vault });
|
|
3466
|
+
const pathCheck = await runPathTooLong({ vault: input.vault, scan });
|
|
3217
3467
|
if (pathCheck.result.ok && pathCheck.result.data.violations.length > 0) buckets.path_too_long = pathCheck.result.data.violations;
|
|
3218
|
-
const
|
|
3219
|
-
const
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
try {
|
|
3225
|
-
const text = await readPage(page);
|
|
3226
|
-
const findings = scanSensitiveContent(text, { file: page.relPath });
|
|
3227
|
-
sensitiveFlags.push(...findings);
|
|
3228
|
-
} catch {
|
|
3229
|
-
}
|
|
3230
|
-
}
|
|
3231
|
-
if (sensitiveFlags.length > 0) buckets.sensitive_content = sensitiveFlags;
|
|
3232
|
-
const fmYamlInvalid = [];
|
|
3233
|
-
for (const page of scan.data.allMarkdown) {
|
|
3468
|
+
const allPages = [...scan.typedKnowledge, ...scan.raw, ...scan.workItems, ...scan.compound];
|
|
3469
|
+
const slugs = buildSlugMap(allPages);
|
|
3470
|
+
{
|
|
3471
|
+
const allPageResults = await mapWithConcurrency(scan.allMarkdown, vaultIoConcurrency(), async (page) => {
|
|
3472
|
+
const sensitiveFlags2 = [];
|
|
3473
|
+
let fmYamlInvalid2 = null;
|
|
3234
3474
|
try {
|
|
3235
|
-
const text = await
|
|
3475
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3476
|
+
sensitiveFlags2.push(...scanSensitiveContent(text, { file: page.relPath }));
|
|
3236
3477
|
const fm = extractFrontmatter(text);
|
|
3237
3478
|
if (!fm.ok && fm.error === "INVALID_FRONTMATTER") {
|
|
3238
3479
|
const detail = fm.detail;
|
|
3239
3480
|
const message = detail?.message ?? "invalid YAML";
|
|
3240
|
-
|
|
3481
|
+
fmYamlInvalid2 = { path: page.relPath, message };
|
|
3241
3482
|
}
|
|
3242
3483
|
} catch {
|
|
3243
3484
|
}
|
|
3244
|
-
|
|
3485
|
+
return { sensitiveFlags: sensitiveFlags2, fmYamlInvalid: fmYamlInvalid2 };
|
|
3486
|
+
});
|
|
3487
|
+
const sensitiveFlags = allPageResults.flatMap((result) => result.sensitiveFlags);
|
|
3488
|
+
if (sensitiveFlags.length > 0) buckets.sensitive_content = sensitiveFlags;
|
|
3489
|
+
const fmYamlInvalid = allPageResults.map((result) => result.fmYamlInvalid).filter((item) => item !== null);
|
|
3245
3490
|
if (fmYamlInvalid.length > 0) buckets.frontmatter_yaml_invalid = fmYamlInvalid;
|
|
3246
3491
|
const subDirDupes = [];
|
|
3247
3492
|
const flatStems = /* @__PURE__ */ new Map();
|
|
3248
3493
|
const deepFiles = [];
|
|
3249
|
-
for (const raw of scan.
|
|
3494
|
+
for (const raw of scan.raw) {
|
|
3250
3495
|
const parts = raw.relPath.split("/");
|
|
3251
3496
|
if (parts.length === 3) {
|
|
3252
3497
|
const stem = parts[2].replace(/\.md$/, "");
|
|
@@ -3265,55 +3510,10 @@ async function runLint(input) {
|
|
|
3265
3510
|
if (subDirDupes.length > 0) {
|
|
3266
3511
|
buckets.raw_subdirectory_duplicate = subDirDupes;
|
|
3267
3512
|
}
|
|
3268
|
-
const
|
|
3269
|
-
|
|
3270
|
-
const
|
|
3271
|
-
const
|
|
3272
|
-
for (const raw of scan.data.raw) {
|
|
3273
|
-
const text = await readPage(raw);
|
|
3274
|
-
const split = splitFrontmatter(text);
|
|
3275
|
-
if (!split.ok) continue;
|
|
3276
|
-
rawPageBodyByPath.set(raw.relPath, split.data.body);
|
|
3277
|
-
if (/^source_url:\s*file:\/\//m.test(split.data.rawFrontmatter)) {
|
|
3278
|
-
fileSourceUrlFlags.add(raw.relPath);
|
|
3279
|
-
fileSourceUrlFrontmatterFlags.add(raw.relPath);
|
|
3280
|
-
}
|
|
3281
|
-
const sourceUrl = split.data.rawFrontmatter.match(/^source_url:\s*(.+)$/m)?.[1]?.trim().replace(/^["']|["']$/g, "") ?? "";
|
|
3282
|
-
const assessment = assessSourceIdentity({
|
|
3283
|
-
rawPath: raw.relPath,
|
|
3284
|
-
sourceUrl,
|
|
3285
|
-
body: split.data.body
|
|
3286
|
-
});
|
|
3287
|
-
if (assessment.status === "conflict") {
|
|
3288
|
-
rawIdentityConflicts.push({
|
|
3289
|
-
file: raw.relPath,
|
|
3290
|
-
status: assessment.status,
|
|
3291
|
-
reasons: assessment.reasons,
|
|
3292
|
-
pathSignals: assessment.pathSignals,
|
|
3293
|
-
sourceSignals: assessment.sourceSignals,
|
|
3294
|
-
bodySignals: assessment.bodySignals
|
|
3295
|
-
});
|
|
3296
|
-
}
|
|
3297
|
-
}
|
|
3298
|
-
const canonicalSourcePages = [
|
|
3299
|
-
...scan.data.raw,
|
|
3300
|
-
...scan.data.typedKnowledge,
|
|
3301
|
-
...scan.data.compound,
|
|
3302
|
-
...scan.data.workItems
|
|
3303
|
-
];
|
|
3304
|
-
for (const page of canonicalSourcePages) {
|
|
3305
|
-
if (!shouldCheckCanonicalLocalSourceAssertion(page)) continue;
|
|
3306
|
-
let body = rawPageBodyByPath.get(page.relPath);
|
|
3307
|
-
if (body === void 0) {
|
|
3308
|
-
const text = await readPage(page);
|
|
3309
|
-
const split = splitFrontmatter(text);
|
|
3310
|
-
if (!split.ok) continue;
|
|
3311
|
-
body = split.data.body;
|
|
3312
|
-
}
|
|
3313
|
-
if (hasCanonicalLocalSourceAssertion(body)) {
|
|
3314
|
-
fileSourceUrlFlags.add(page.relPath);
|
|
3315
|
-
}
|
|
3316
|
-
}
|
|
3513
|
+
const fileSourceUrlFindings = await collectFileSourceUrlFindings(scan, pageTextCache, { includeRawIdentityConflicts: true });
|
|
3514
|
+
let fileSourceUrlFlags = fileSourceUrlFindings.fileSourceUrlFlags;
|
|
3515
|
+
const fileSourceUrlFrontmatterFlags = fileSourceUrlFindings.fileSourceUrlFrontmatterFlags;
|
|
3516
|
+
const rawIdentityConflicts = fileSourceUrlFindings.rawIdentityConflicts;
|
|
3317
3517
|
if (fileSourceUrlFlags.size > 0) buckets.file_source_url = [...fileSourceUrlFlags];
|
|
3318
3518
|
if (rawIdentityConflicts.length > 0) buckets.raw_source_identity_conflict = rawIdentityConflicts;
|
|
3319
3519
|
const legacyPages = [];
|
|
@@ -3326,57 +3526,85 @@ async function runLint(input) {
|
|
|
3326
3526
|
const brokenSourceFlags = /* @__PURE__ */ new Set();
|
|
3327
3527
|
const missingTldrFlags = [];
|
|
3328
3528
|
const missingDiagramFlags = [];
|
|
3329
|
-
|
|
3330
|
-
const
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
3344
|
-
|
|
3529
|
+
const typedPageResults = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (page) => {
|
|
3530
|
+
const result = {
|
|
3531
|
+
legacyPages: [],
|
|
3532
|
+
orphanedPages: [],
|
|
3533
|
+
structFlags: [],
|
|
3534
|
+
dupFrontmatter: [],
|
|
3535
|
+
noOverview: [],
|
|
3536
|
+
fmWikilinkFlags: [],
|
|
3537
|
+
wikilinkCitationFlags: [],
|
|
3538
|
+
brokenSourceFlags: [],
|
|
3539
|
+
missingTldrFlags: [],
|
|
3540
|
+
missingDiagramFlags: []
|
|
3541
|
+
};
|
|
3542
|
+
try {
|
|
3543
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3544
|
+
const split = splitFrontmatter(text);
|
|
3545
|
+
if (!split.ok) return result;
|
|
3546
|
+
const body = split.data.body;
|
|
3547
|
+
const rawFm = split.data.rawFrontmatter;
|
|
3548
|
+
if (hasDuplicateFrontmatter(body)) result.dupFrontmatter.push(page.relPath);
|
|
3549
|
+
if (isLegacyCitationStyle(body)) result.legacyPages.push(page.relPath);
|
|
3550
|
+
if (hasOrphanedCitations(body)) result.orphanedPages.push(page.relPath);
|
|
3551
|
+
if (hasWikilinkCitations(body)) result.wikilinkCitationFlags.push(page.relPath);
|
|
3552
|
+
const sourcesEntries = extractSourceEntries(rawFm);
|
|
3553
|
+
for (const entry of sourcesEntries) {
|
|
3554
|
+
const rawPath = normalizeRawSourceTarget(entry);
|
|
3555
|
+
if (!rawPath) continue;
|
|
3556
|
+
if (!rawSourceTargetExistsSync(input.vault, rawPath)) {
|
|
3557
|
+
result.brokenSourceFlags.push(`${page.relPath}: ${rawPath}`);
|
|
3558
|
+
}
|
|
3345
3559
|
}
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3560
|
+
for (const marker of extractCitationMarkers(body)) {
|
|
3561
|
+
if (!rawSourceTargetExistsSync(input.vault, marker.target)) {
|
|
3562
|
+
result.brokenSourceFlags.push(`${page.relPath}: ${marker.target}`);
|
|
3563
|
+
}
|
|
3350
3564
|
}
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3565
|
+
const fmLinks = rawFm.match(/\[\[([^\[\]|]+)(?:\|[^\[\]]*)?\]\]/g) ?? [];
|
|
3566
|
+
for (const link of fmLinks) {
|
|
3567
|
+
const target = link.replace(/^\[\[/, "").replace(/(?:\|[^\[\]]*)?\]\]$/, "").trim();
|
|
3568
|
+
const tail = target.split("/").pop().replace(/\.md$/, "");
|
|
3569
|
+
if (!slugs.has(tail.toLowerCase())) {
|
|
3570
|
+
result.fmWikilinkFlags.push(`${page.relPath}: [[${target}]] does not resolve`);
|
|
3571
|
+
}
|
|
3358
3572
|
}
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
|
|
3373
|
-
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
|
|
3377
|
-
|
|
3573
|
+
const bodyLines = body.split("\n").filter((l) => l.trim().length > 0).length;
|
|
3574
|
+
const hasOverview = /^## Overview/m.test(body);
|
|
3575
|
+
if (!hasOverview) result.noOverview.push(page.relPath);
|
|
3576
|
+
const bodyFirst15 = body.split("\n").slice(0, 15).join("\n");
|
|
3577
|
+
if (!/^>\s*\*\*TL;DR:?\*\*/m.test(bodyFirst15) && !/^##\s+TL;\s*DR/m.test(bodyFirst15)) result.missingTldrFlags.push(page.relPath);
|
|
3578
|
+
const fmData = extractFrontmatter(text);
|
|
3579
|
+
const pageTags = fmData.ok && Array.isArray(fmData.data.tags) ? fmData.data.tags : [];
|
|
3580
|
+
if (pageTags.includes("architecture") && !body.includes("```mermaid")) {
|
|
3581
|
+
result.missingDiagramFlags.push(page.relPath);
|
|
3582
|
+
}
|
|
3583
|
+
if (bodyLines < STRUCT_MIN_BODY_LINES) {
|
|
3584
|
+
const hasRelated = /^## (Related|Relationships)/m.test(body);
|
|
3585
|
+
const sectionCount = (body.match(/^## /gm) ?? []).length;
|
|
3586
|
+
if (!hasRelated || sectionCount < STRUCT_MIN_SECTIONS) {
|
|
3587
|
+
const reasons = [];
|
|
3588
|
+
if (!hasRelated) reasons.push("no Related or Relationships");
|
|
3589
|
+
if (sectionCount < STRUCT_MIN_SECTIONS) reasons.push(`only ${sectionCount} sections`);
|
|
3590
|
+
result.structFlags.push(`${page.relPath}: ${bodyLines} lines, ${reasons.join(", ")}`);
|
|
3591
|
+
}
|
|
3378
3592
|
}
|
|
3593
|
+
} catch {
|
|
3379
3594
|
}
|
|
3595
|
+
return result;
|
|
3596
|
+
});
|
|
3597
|
+
for (const result of typedPageResults) {
|
|
3598
|
+
legacyPages.push(...result.legacyPages);
|
|
3599
|
+
orphanedPages.push(...result.orphanedPages);
|
|
3600
|
+
structFlags.push(...result.structFlags);
|
|
3601
|
+
dupFrontmatter.push(...result.dupFrontmatter);
|
|
3602
|
+
noOverview.push(...result.noOverview);
|
|
3603
|
+
fmWikilinkFlags.push(...result.fmWikilinkFlags);
|
|
3604
|
+
wikilinkCitationFlags.push(...result.wikilinkCitationFlags);
|
|
3605
|
+
for (const flag of result.brokenSourceFlags) brokenSourceFlags.add(flag);
|
|
3606
|
+
missingTldrFlags.push(...result.missingTldrFlags);
|
|
3607
|
+
missingDiagramFlags.push(...result.missingDiagramFlags);
|
|
3380
3608
|
}
|
|
3381
3609
|
if (legacyPages.length > 0) buckets.legacy_citation_style = legacyPages;
|
|
3382
3610
|
if (orphanedPages.length > 0) buckets.orphaned_citations = orphanedPages;
|
|
@@ -3390,19 +3618,20 @@ async function runLint(input) {
|
|
|
3390
3618
|
if (missingDiagramFlags.length > 0) buckets.missing_diagram = missingDiagramFlags;
|
|
3391
3619
|
const workItemHealth = [];
|
|
3392
3620
|
const workItemDirs = /* @__PURE__ */ new Map();
|
|
3393
|
-
for (const page of scan.
|
|
3621
|
+
for (const page of scan.workItems) {
|
|
3394
3622
|
const dir = page.relPath.replace(/\/(spec|plan|log)\.md$/, "");
|
|
3395
3623
|
const pages = workItemDirs.get(dir) ?? [];
|
|
3396
3624
|
pages.push(page);
|
|
3397
3625
|
workItemDirs.set(dir, pages);
|
|
3398
3626
|
}
|
|
3399
|
-
|
|
3627
|
+
const workItemHealthResults = await mapWithConcurrency([...workItemDirs.entries()], vaultIoConcurrency(), async ([dir, pages]) => {
|
|
3628
|
+
const flags = [];
|
|
3400
3629
|
const specPage = pages.find((p) => p.relPath.endsWith("/spec.md"));
|
|
3401
3630
|
const hasPlan = pages.some((p) => p.relPath.endsWith("/plan.md"));
|
|
3402
3631
|
let specStatus;
|
|
3403
3632
|
let specStarted;
|
|
3404
3633
|
if (specPage) {
|
|
3405
|
-
const text = await
|
|
3634
|
+
const text = await readPageCached(specPage, pageTextCache);
|
|
3406
3635
|
const fm = extractFrontmatter(text);
|
|
3407
3636
|
if (fm.ok) {
|
|
3408
3637
|
specStatus = typeof fm.data.status === "string" ? fm.data.status : void 0;
|
|
@@ -3416,74 +3645,96 @@ async function runLint(input) {
|
|
|
3416
3645
|
if (dateMatch) {
|
|
3417
3646
|
const dirDate = Date.parse(dateMatch[1]);
|
|
3418
3647
|
if (!isNaN(dirDate) && Date.now() - dirDate > 24 * 60 * 60 * 1e3) {
|
|
3419
|
-
|
|
3648
|
+
flags.push(`${dir}/spec.md: has spec but no plan after 24h`);
|
|
3420
3649
|
}
|
|
3421
3650
|
}
|
|
3422
3651
|
}
|
|
3423
3652
|
if (specPage && specStatus === "in-progress" && !specStarted) {
|
|
3424
|
-
|
|
3653
|
+
flags.push(`${specPage.relPath}: in-progress without started date`);
|
|
3425
3654
|
}
|
|
3426
|
-
|
|
3655
|
+
return flags;
|
|
3656
|
+
});
|
|
3657
|
+
workItemHealth.push(...workItemHealthResults.flat());
|
|
3427
3658
|
if (workItemHealth.length > 0) buckets.work_item_health = workItemHealth;
|
|
3428
3659
|
const orphanedProjectPages = [];
|
|
3429
|
-
|
|
3430
|
-
|
|
3431
|
-
const
|
|
3432
|
-
if (
|
|
3433
|
-
const
|
|
3434
|
-
|
|
3435
|
-
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
const
|
|
3442
|
-
|
|
3443
|
-
|
|
3660
|
+
const knowledgeContentCache = /* @__PURE__ */ new Map();
|
|
3661
|
+
const readKnowledgeContent = (slug) => {
|
|
3662
|
+
const existing = knowledgeContentCache.get(slug);
|
|
3663
|
+
if (existing) return existing;
|
|
3664
|
+
const knowledgePath = join15(input.vault, "projects", slug, "knowledge.md");
|
|
3665
|
+
const pending = existsSync4(knowledgePath) ? readFile12(knowledgePath, "utf8").catch(() => null) : Promise.resolve(null);
|
|
3666
|
+
knowledgeContentCache.set(slug, pending);
|
|
3667
|
+
return pending;
|
|
3668
|
+
};
|
|
3669
|
+
const orphanedProjectPageResults = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (page) => {
|
|
3670
|
+
const flags = [];
|
|
3671
|
+
try {
|
|
3672
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3673
|
+
const fm = extractFrontmatter(text);
|
|
3674
|
+
if (!fm.ok) return flags;
|
|
3675
|
+
const pp = fm.data.provenance_projects;
|
|
3676
|
+
if (!Array.isArray(pp)) return flags;
|
|
3677
|
+
for (const entry of pp) {
|
|
3678
|
+
const slugMatch = String(entry).match(/\[\[([^\]]+)\]\]/);
|
|
3679
|
+
if (!slugMatch) continue;
|
|
3680
|
+
const slug = slugMatch[1];
|
|
3681
|
+
const knowledgeContent = await readKnowledgeContent(slug);
|
|
3682
|
+
if (knowledgeContent === null) continue;
|
|
3683
|
+
const pageRef = page.relPath.replace(/\.md$/, "");
|
|
3444
3684
|
if (!knowledgeContent.includes(`[[${pageRef}]]`)) {
|
|
3445
|
-
|
|
3685
|
+
flags.push(`${page.relPath}: not in projects/${slug}/knowledge.md`);
|
|
3446
3686
|
}
|
|
3447
|
-
} catch {
|
|
3448
3687
|
}
|
|
3688
|
+
} catch {
|
|
3449
3689
|
}
|
|
3450
|
-
|
|
3690
|
+
return flags;
|
|
3691
|
+
});
|
|
3692
|
+
orphanedProjectPages.push(...orphanedProjectPageResults.flat());
|
|
3451
3693
|
if (orphanedProjectPages.length > 0) buckets.orphaned_project_pages = orphanedProjectPages;
|
|
3452
3694
|
const cliRefFlags = [];
|
|
3453
3695
|
const cliSurface = buildCliSurface();
|
|
3454
|
-
const allScanPages = [...scan.
|
|
3455
|
-
|
|
3456
|
-
const
|
|
3457
|
-
|
|
3458
|
-
|
|
3459
|
-
|
|
3696
|
+
const allScanPages = [...scan.typedKnowledge];
|
|
3697
|
+
const cliRefResults = await mapWithConcurrency(allScanPages, vaultIoConcurrency(), async (page) => {
|
|
3698
|
+
const flags = [];
|
|
3699
|
+
try {
|
|
3700
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3701
|
+
const violations = validateCliRefs(text, page.relPath, cliSurface);
|
|
3702
|
+
for (const v of violations) {
|
|
3703
|
+
flags.push(`${v.page}: ${v.ref} (${v.reason})`);
|
|
3704
|
+
}
|
|
3705
|
+
} catch {
|
|
3460
3706
|
}
|
|
3461
|
-
|
|
3707
|
+
return flags;
|
|
3708
|
+
});
|
|
3709
|
+
cliRefFlags.push(...cliRefResults.flat());
|
|
3462
3710
|
if (cliRefFlags.length > 0) buckets.cli_refs = cliRefFlags;
|
|
3463
3711
|
const staleSectionFlags = [];
|
|
3464
3712
|
const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
3465
3713
|
const approachingThreshold = 7;
|
|
3466
|
-
|
|
3714
|
+
const staleSectionResults = await mapWithConcurrency(scan.typedKnowledge, vaultIoConcurrency(), async (page) => {
|
|
3715
|
+
const flags = [];
|
|
3467
3716
|
try {
|
|
3468
|
-
const text = await
|
|
3717
|
+
const text = await readPageCached(page, pageTextCache);
|
|
3469
3718
|
const annotations = parseExpiryAnnotations(text, page.relPath);
|
|
3470
3719
|
for (const ann of annotations) {
|
|
3471
3720
|
if (ann.expires < today) {
|
|
3472
|
-
|
|
3721
|
+
flags.push(`${page.relPath}: section "${ann.heading}" expired on ${ann.expires}`);
|
|
3473
3722
|
} else {
|
|
3474
3723
|
const daysUntilExpiry = Math.floor((Date.parse(ann.expires) - Date.now()) / 864e5);
|
|
3475
3724
|
if (daysUntilExpiry <= approachingThreshold) {
|
|
3476
|
-
|
|
3725
|
+
flags.push(`${page.relPath}: section "${ann.heading}" expires in ${daysUntilExpiry} day(s) (${ann.expires})`);
|
|
3477
3726
|
}
|
|
3478
3727
|
}
|
|
3479
3728
|
}
|
|
3480
3729
|
} catch {
|
|
3481
3730
|
}
|
|
3482
|
-
|
|
3731
|
+
return flags;
|
|
3732
|
+
});
|
|
3733
|
+
staleSectionFlags.push(...staleSectionResults.flat());
|
|
3483
3734
|
if (staleSectionFlags.length > 0) buckets.stale_sections = staleSectionFlags;
|
|
3484
3735
|
if (shouldFix("sensitive_content") && buckets.sensitive_content) {
|
|
3485
3736
|
const sensitiveFixed = [];
|
|
3486
|
-
for (const page of scan.
|
|
3737
|
+
for (const page of scan.allMarkdown) {
|
|
3487
3738
|
try {
|
|
3488
3739
|
const raw = await readPage(page);
|
|
3489
3740
|
const redacted = redactSensitiveContent(raw, { file: page.relPath });
|
|
@@ -3501,7 +3752,7 @@ async function runLint(input) {
|
|
|
3501
3752
|
}
|
|
3502
3753
|
fixed.push(...sensitiveFixed);
|
|
3503
3754
|
const remainingSensitiveFlags = [];
|
|
3504
|
-
for (const page of scan.
|
|
3755
|
+
for (const page of scan.allMarkdown) {
|
|
3505
3756
|
try {
|
|
3506
3757
|
const text = await readPage(page);
|
|
3507
3758
|
remainingSensitiveFlags.push(...scanSensitiveContent(text, { file: page.relPath }));
|
|
@@ -3790,60 +4041,9 @@ ${newBody}`;
|
|
|
3790
4041
|
}
|
|
3791
4042
|
}
|
|
3792
4043
|
if (shouldFix("file_source_url") && fileSourceUrlFrontmatterFlags.size > 0) {
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
const absPath = `${input.vault}/${relPath}`;
|
|
3797
|
-
const raw = await readFile12(absPath, "utf8");
|
|
3798
|
-
const parts = raw.split("---", 3);
|
|
3799
|
-
if (parts.length < 3) {
|
|
3800
|
-
unresolved.push(relPath);
|
|
3801
|
-
continue;
|
|
3802
|
-
}
|
|
3803
|
-
const rawFm = parts[1];
|
|
3804
|
-
const rest = parts[2];
|
|
3805
|
-
const sourceMatch = rest.match(/^source:\s*"?(https?:\/\/[^\s\n"]+)"?\s*$/m);
|
|
3806
|
-
if (!sourceMatch) {
|
|
3807
|
-
unresolved.push(relPath);
|
|
3808
|
-
continue;
|
|
3809
|
-
}
|
|
3810
|
-
const realUrl = sourceMatch[1];
|
|
3811
|
-
const newRawFm = rawFm.replace(/^source_url:\s*file:\/\/[^\n]+/m, `source_url: ${realUrl}`);
|
|
3812
|
-
const newContent = `---${newRawFm}---${rest}`;
|
|
3813
|
-
const w = await safeWritePage(absPath, newContent);
|
|
3814
|
-
if (!w.ok) {
|
|
3815
|
-
unresolved.push(relPath);
|
|
3816
|
-
continue;
|
|
3817
|
-
}
|
|
3818
|
-
FILE_FIXED.push(relPath);
|
|
3819
|
-
} catch {
|
|
3820
|
-
unresolved.push(relPath);
|
|
3821
|
-
}
|
|
3822
|
-
}
|
|
3823
|
-
fixed.push(...FILE_FIXED);
|
|
3824
|
-
if (FILE_FIXED.length > 0) {
|
|
3825
|
-
const remaining = new Set(fileSourceUrlFlags);
|
|
3826
|
-
for (const relPath of FILE_FIXED) {
|
|
3827
|
-
try {
|
|
3828
|
-
const page = scan.data.allMarkdown.find((p) => p.relPath === relPath);
|
|
3829
|
-
if (!page) {
|
|
3830
|
-
remaining.delete(relPath);
|
|
3831
|
-
continue;
|
|
3832
|
-
}
|
|
3833
|
-
const text = await readPage(page);
|
|
3834
|
-
const split = splitFrontmatter(text);
|
|
3835
|
-
if (!split.ok) continue;
|
|
3836
|
-
const stillHasFileSourceUrl = /^source_url:\s*file:\/\//m.test(split.data.rawFrontmatter);
|
|
3837
|
-
const stillHasCanonicalBodyAssertion = shouldCheckCanonicalLocalSourceAssertion(page) && hasCanonicalLocalSourceAssertion(split.data.body);
|
|
3838
|
-
if (!stillHasFileSourceUrl && !stillHasCanonicalBodyAssertion) {
|
|
3839
|
-
remaining.delete(relPath);
|
|
3840
|
-
}
|
|
3841
|
-
} catch {
|
|
3842
|
-
}
|
|
3843
|
-
}
|
|
3844
|
-
if (remaining.size > 0) buckets.file_source_url = [...remaining];
|
|
3845
|
-
else delete buckets.file_source_url;
|
|
3846
|
-
}
|
|
4044
|
+
fileSourceUrlFlags = await applyFileSourceUrlFix(input, scan, fileSourceUrlFlags, fileSourceUrlFrontmatterFlags, fixed, unresolved);
|
|
4045
|
+
if (fileSourceUrlFlags.size > 0) buckets.file_source_url = [...fileSourceUrlFlags];
|
|
4046
|
+
else delete buckets.file_source_url;
|
|
3847
4047
|
}
|
|
3848
4048
|
const pathViolations = buckets.path_too_long;
|
|
3849
4049
|
if (shouldFix("path_too_long") && pathViolations && pathViolations.length > 0) {
|
|
@@ -3863,7 +4063,7 @@ ${newBody}`;
|
|
|
3863
4063
|
const invalidItems = buckets.frontmatter_yaml_invalid;
|
|
3864
4064
|
const remaining = [];
|
|
3865
4065
|
for (const item of invalidItems) {
|
|
3866
|
-
const page = scan.
|
|
4066
|
+
const page = scan.allMarkdown.find((p) => p.relPath === item.path);
|
|
3867
4067
|
if (!page) {
|
|
3868
4068
|
unresolved.push(item.path);
|
|
3869
4069
|
remaining.push(item);
|
|
@@ -3909,30 +4109,7 @@ ${split.data.body}`;
|
|
|
3909
4109
|
const infoOut = INFO_ORDER.flatMap((k) => buckets[k] ? [{ kind: k, items: buckets[k] }] : []);
|
|
3910
4110
|
if (input.only) {
|
|
3911
4111
|
const match = [...errorOut, ...warningOut, ...infoOut].filter((b) => b.kind === input.only);
|
|
3912
|
-
|
|
3913
|
-
const filtered = severity === "error" ? { error: match, warning: [], info: [] } : severity === "warning" ? { error: [], warning: match, info: [] } : { error: [], warning: [], info: match };
|
|
3914
|
-
const fSummary = {
|
|
3915
|
-
errors: filtered.error.reduce((n, b) => n + b.items.length, 0),
|
|
3916
|
-
warnings: filtered.warning.reduce((n, b) => n + b.items.length, 0),
|
|
3917
|
-
info: filtered.info.reduce((n, b) => n + b.items.length, 0)
|
|
3918
|
-
};
|
|
3919
|
-
let fExit = ExitCode.OK;
|
|
3920
|
-
if (fSummary.errors > 0) fExit = ExitCode.LINT_HAS_ERRORS;
|
|
3921
|
-
else if (fSummary.warnings > 0 || fSummary.info > 0) fExit = ExitCode.LINT_HAS_WARNINGS;
|
|
3922
|
-
const output2 = {
|
|
3923
|
-
vault: { path: input.vault, source: input.source ?? "resolved" },
|
|
3924
|
-
summary: fSummary,
|
|
3925
|
-
by_severity: filtered,
|
|
3926
|
-
fixed,
|
|
3927
|
-
unresolved,
|
|
3928
|
-
humanHint: `--only ${input.only}
|
|
3929
|
-
${match.length === 0 ? "0 violations" : match.map((b) => ` ${b.kind}: ${b.items.length}`).join("\n")}`
|
|
3930
|
-
};
|
|
3931
|
-
if (input.fix) appendLintFixLastOp(input.vault, fixed);
|
|
3932
|
-
return {
|
|
3933
|
-
exitCode: fExit,
|
|
3934
|
-
result: ok(input.summary ? summarizeLintOutput(output2, input.examplesLimit) : output2)
|
|
3935
|
-
};
|
|
4112
|
+
return outputForOnlyBucket(input, match, fixed, unresolved);
|
|
3936
4113
|
}
|
|
3937
4114
|
const summary = {
|
|
3938
4115
|
errors: errorOut.reduce((n, b) => n + b.items.length, 0),
|