sdl-mcp 0.11.12 → 0.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/cli/commands/serve.d.ts.map +1 -1
- package/dist/cli/commands/serve.js +233 -217
- package/dist/cli/commands/serve.js.map +1 -1
- package/dist/db/ladybug.d.ts +6 -0
- package/dist/db/ladybug.d.ts.map +1 -1
- package/dist/db/ladybug.js +32 -0
- package/dist/db/ladybug.js.map +1 -1
- package/dist/indexer/indexer-pass1.d.ts.map +1 -1
- package/dist/indexer/indexer-pass1.js +3 -1
- package/dist/indexer/indexer-pass1.js.map +1 -1
- package/dist/indexer/indexer.d.ts.map +1 -1
- package/dist/indexer/indexer.js +627 -599
- package/dist/indexer/indexer.js.map +1 -1
- package/dist/indexer/language-packs.js +16 -1
- package/dist/indexer/language-packs.js.map +1 -1
- package/dist/main.js +62 -52
- package/dist/main.js.map +1 -1
- package/dist/startup/process-handlers.d.ts.map +1 -1
- package/dist/startup/process-handlers.js +13 -3
- package/dist/startup/process-handlers.js.map +1 -1
- package/dist/util/logger.d.ts.map +1 -1
- package/dist/util/logger.js +3 -2
- package/dist/util/logger.js.map +1 -1
- package/dist/util/shutdown.d.ts +4 -0
- package/dist/util/shutdown.d.ts.map +1 -1
- package/dist/util/shutdown.js +26 -6
- package/dist/util/shutdown.js.map +1 -1
- package/dist/util/stdio-safety.d.ts +3 -0
- package/dist/util/stdio-safety.d.ts.map +1 -0
- package/dist/util/stdio-safety.js +20 -0
- package/dist/util/stdio-safety.js.map +1 -0
- package/package.json +5 -5
package/dist/indexer/indexer.js
CHANGED
|
@@ -11,7 +11,7 @@ import { finalizeDerivedState } from "./finalize-derived-state.js";
|
|
|
11
11
|
import { watchRepositoryWithIndexer } from "./watcher.js";
|
|
12
12
|
import { IndexingConfigSchema, } from "../config/types.js";
|
|
13
13
|
import { loadConfig } from "../config/loadConfig.js";
|
|
14
|
-
import { buildDeferredIndexes, closeLadybugDb, flushStaleFinalizers, getLadybugConn, getLadybugDbPath, initLadybugDb, preIndexCheckpoint, withWriteConn, } from "../db/ladybug.js";
|
|
14
|
+
import { buildDeferredIndexes, closeLadybugDb, ensureCriticalSymbolFtsIndex, flushStaleFinalizers, getLadybugConn, getLadybugDbPath, initLadybugDb, preIndexCheckpoint, withWriteConn, } from "../db/ladybug.js";
|
|
15
15
|
import { withPostIndexWriteSession } from "../db/write-session.js";
|
|
16
16
|
import * as ladybugDb from "../db/ladybug-queries.js";
|
|
17
17
|
import { derivedStateIsStale, getDerivedState, markDerivedStateDirty, recordDerivedStateError, } from "../db/ladybug-derived-state.js";
|
|
@@ -1461,6 +1461,11 @@ async function indexRepoImpl(repoId, mode, onProgress, signal, options, scipPreR
|
|
|
1461
1461
|
onProgress,
|
|
1462
1462
|
}));
|
|
1463
1463
|
recordFinalizeIndexingSubphaseTimings(finalizeResult.timings);
|
|
1464
|
+
await measurePhase("ensureCriticalSymbolFts", async () => {
|
|
1465
|
+
await ensureCriticalSymbolFtsIndex({
|
|
1466
|
+
recordTiming: recordIndexSubphaseTiming,
|
|
1467
|
+
});
|
|
1468
|
+
});
|
|
1464
1469
|
const freshConn = await getLadybugConn();
|
|
1465
1470
|
const derivedResult = params.skipDerivedStateReason
|
|
1466
1471
|
? await measurePhase("skipDerivedState", async () => {
|
|
@@ -1655,673 +1660,691 @@ async function indexRepoImpl(repoId, mode, onProgress, signal, options, scipPreR
|
|
|
1655
1660
|
providerFirstExecutor &&
|
|
1656
1661
|
providerFirstIncrementalBootstrapped) {
|
|
1657
1662
|
providerFirstTimingStartedAt = Date.now();
|
|
1658
|
-
|
|
1659
|
-
message: "scanning repository for provider-first coverage",
|
|
1660
|
-
});
|
|
1661
|
-
const providerCoverageScan = await measureProviderFirstPhase("coverageScan", "providerFirstCoverageScan", () => scanRepoForIndex({
|
|
1662
|
-
repoId,
|
|
1663
|
-
repoRoot: repoRow.rootPath,
|
|
1664
|
-
config,
|
|
1665
|
-
onProgress,
|
|
1666
|
-
deleteRemovedFiles: false,
|
|
1667
|
-
}));
|
|
1668
|
-
emitProviderFirstProgress(onProgress, "coverageScan", {
|
|
1669
|
-
stageCurrent: providerCoverageScan.files.length,
|
|
1670
|
-
stageTotal: providerCoverageScan.files.length,
|
|
1671
|
-
message: `scanned ${providerCoverageScan.files.length} file(s) for provider-first coverage`,
|
|
1672
|
-
});
|
|
1673
|
-
let providerFirstRemovedFilesDeleted = false;
|
|
1674
|
-
if (providerFirstIncrementalActive &&
|
|
1675
|
-
providerCoverageScan.removedFileIds.length > 0) {
|
|
1676
|
-
await measureProviderFirstPhase("deleteRemovedFiles", "providerFirstIncrementalDeleteRemovedFiles", () => withWriteConn(async (conn) => {
|
|
1677
|
-
await ladybugDb.withTransaction(conn, async (txConn) => {
|
|
1678
|
-
await ladybugDb.deleteFilesByIds(txConn, providerCoverageScan.removedFileIds);
|
|
1679
|
-
});
|
|
1680
|
-
}, postIndexSessionTimeoutMs));
|
|
1681
|
-
providerFirstRemovedFilesDeleted = true;
|
|
1682
|
-
}
|
|
1683
|
-
const providerChangedFiles = providerFirstIncrementalActive
|
|
1684
|
-
? providerFirstIncrementalChangedFiles(providerCoverageScan)
|
|
1685
|
-
: providerCoverageScan.files;
|
|
1686
|
-
const providerExecutionScan = providerFirstIncrementalActive
|
|
1687
|
-
? filterProviderFirstIncrementalScan(providerCoverageScan, providerChangedFiles)
|
|
1688
|
-
: providerCoverageScan;
|
|
1689
|
-
if (providerFirstIncrementalActive && providerChangedFiles.length === 0) {
|
|
1690
|
-
providerFirstScan = providerCoverageScan;
|
|
1663
|
+
try {
|
|
1691
1664
|
emitProviderFirstProgress(onProgress, "coverageScan", {
|
|
1692
|
-
|
|
1693
|
-
stageTotal: 0,
|
|
1694
|
-
message: "no changed files for provider-first incremental execution",
|
|
1665
|
+
message: "scanning repository for provider-first coverage",
|
|
1695
1666
|
});
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1667
|
+
const providerCoverageScan = await measureProviderFirstPhase("coverageScan", "providerFirstCoverageScan", () => scanRepoForIndex({
|
|
1668
|
+
repoId,
|
|
1669
|
+
repoRoot: repoRow.rootPath,
|
|
1670
|
+
config,
|
|
1671
|
+
onProgress,
|
|
1672
|
+
deleteRemovedFiles: false,
|
|
1673
|
+
}));
|
|
1674
|
+
emitProviderFirstProgress(onProgress, "coverageScan", {
|
|
1675
|
+
stageCurrent: providerCoverageScan.files.length,
|
|
1676
|
+
stageTotal: providerCoverageScan.files.length,
|
|
1677
|
+
message: `scanned ${providerCoverageScan.files.length} file(s) for provider-first coverage`,
|
|
1678
|
+
});
|
|
1679
|
+
let providerFirstRemovedFilesDeleted = false;
|
|
1680
|
+
if (providerFirstIncrementalActive &&
|
|
1681
|
+
providerCoverageScan.removedFileIds.length > 0) {
|
|
1682
|
+
await measureProviderFirstPhase("deleteRemovedFiles", "providerFirstIncrementalDeleteRemovedFiles", () => withWriteConn(async (conn) => {
|
|
1683
|
+
await ladybugDb.withTransaction(conn, async (txConn) => {
|
|
1684
|
+
await ladybugDb.deleteFilesByIds(txConn, providerCoverageScan.removedFileIds);
|
|
1685
|
+
});
|
|
1686
|
+
}, postIndexSessionTimeoutMs));
|
|
1687
|
+
providerFirstRemovedFilesDeleted = true;
|
|
1688
|
+
}
|
|
1689
|
+
const providerChangedFiles = providerFirstIncrementalActive
|
|
1690
|
+
? providerFirstIncrementalChangedFiles(providerCoverageScan)
|
|
1691
|
+
: providerCoverageScan.files;
|
|
1692
|
+
const providerExecutionScan = providerFirstIncrementalActive
|
|
1693
|
+
? filterProviderFirstIncrementalScan(providerCoverageScan, providerChangedFiles)
|
|
1694
|
+
: providerCoverageScan;
|
|
1695
|
+
if (providerFirstIncrementalActive && providerChangedFiles.length === 0) {
|
|
1696
|
+
providerFirstScan = providerCoverageScan;
|
|
1697
|
+
emitProviderFirstProgress(onProgress, "coverageScan", {
|
|
1698
|
+
stageCurrent: 0,
|
|
1699
|
+
stageTotal: 0,
|
|
1700
|
+
message: "no changed files for provider-first incremental execution",
|
|
1701
|
+
});
|
|
1702
|
+
}
|
|
1703
|
+
let providerResult;
|
|
1704
|
+
if (!providerFirstIncrementalActive || providerChangedFiles.length > 0) {
|
|
1702
1705
|
try {
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
appConfig,
|
|
1709
|
-
changedFiles: providerChangedFiles,
|
|
1710
|
-
signal,
|
|
1711
|
-
}));
|
|
1712
|
-
providerScipPreRefresh = scopedScip.diagnostics;
|
|
1713
|
-
activeScipPreRefresh = scopedScip.diagnostics;
|
|
1714
|
-
providerIncrementalTempPaths = scopedScip.tempPaths;
|
|
1715
|
-
}
|
|
1716
|
-
providerResult = await measureProviderFirstPhase("providerCollection", providerFirstLspExecutor(providerFirstExecutor)
|
|
1717
|
-
? providerFirstIncrementalActive
|
|
1718
|
-
? "providerFirstLspIncremental"
|
|
1719
|
-
: "providerFirstLspFull"
|
|
1720
|
-
: providerFirstIncrementalActive
|
|
1721
|
-
? "providerFirstScipIncremental"
|
|
1722
|
-
: "providerFirstScipFull", () => {
|
|
1723
|
-
if (providerFirstExecutor === "lspFull") {
|
|
1724
|
-
return executeProviderFirstLspFull({
|
|
1706
|
+
let providerScipPreRefresh = activeScipPreRefresh;
|
|
1707
|
+
let providerIncrementalTempPaths = [];
|
|
1708
|
+
try {
|
|
1709
|
+
if (providerFirstExecutor === "scipIncremental") {
|
|
1710
|
+
const scopedScip = await measureProviderFirstPhase("scipIncrementalGenerate", "providerFirstScipIncrementalGenerate", () => runProviderFirstIncrementalScipIo({
|
|
1725
1711
|
repoId,
|
|
1726
1712
|
repoRoot: repoRow.rootPath,
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1713
|
+
repoConfig: config,
|
|
1714
|
+
appConfig,
|
|
1715
|
+
changedFiles: providerChangedFiles,
|
|
1730
1716
|
signal,
|
|
1731
|
-
});
|
|
1717
|
+
}));
|
|
1718
|
+
providerScipPreRefresh = scopedScip.diagnostics;
|
|
1719
|
+
activeScipPreRefresh = scopedScip.diagnostics;
|
|
1720
|
+
providerIncrementalTempPaths = scopedScip.tempPaths;
|
|
1732
1721
|
}
|
|
1733
|
-
|
|
1734
|
-
|
|
1722
|
+
providerResult = await measureProviderFirstPhase("providerCollection", providerFirstLspExecutor(providerFirstExecutor)
|
|
1723
|
+
? providerFirstIncrementalActive
|
|
1724
|
+
? "providerFirstLspIncremental"
|
|
1725
|
+
: "providerFirstLspFull"
|
|
1726
|
+
: providerFirstIncrementalActive
|
|
1727
|
+
? "providerFirstScipIncremental"
|
|
1728
|
+
: "providerFirstScipFull", () => {
|
|
1729
|
+
if (providerFirstExecutor === "lspFull") {
|
|
1730
|
+
return executeProviderFirstLspFull({
|
|
1731
|
+
repoId,
|
|
1732
|
+
repoRoot: repoRow.rootPath,
|
|
1733
|
+
config: appConfig,
|
|
1734
|
+
scannedFiles: providerExecutionScan.files,
|
|
1735
|
+
recordPhaseTiming: recordProviderFirstPhaseTiming,
|
|
1736
|
+
signal,
|
|
1737
|
+
});
|
|
1738
|
+
}
|
|
1739
|
+
if (providerFirstExecutor === "lspIncremental") {
|
|
1740
|
+
return executeProviderFirstLspIncremental({
|
|
1741
|
+
repoId,
|
|
1742
|
+
repoRoot: repoRow.rootPath,
|
|
1743
|
+
config: appConfig,
|
|
1744
|
+
scannedFiles: providerExecutionScan.files,
|
|
1745
|
+
recordPhaseTiming: recordProviderFirstPhaseTiming,
|
|
1746
|
+
signal,
|
|
1747
|
+
});
|
|
1748
|
+
}
|
|
1749
|
+
const scipParams = {
|
|
1735
1750
|
repoId,
|
|
1736
1751
|
repoRoot: repoRow.rootPath,
|
|
1737
1752
|
config: appConfig,
|
|
1753
|
+
generatedIndexes: providerScipPreRefresh?.generatedIndexes,
|
|
1754
|
+
generatorFailures: providerScipPreRefresh?.failures,
|
|
1755
|
+
generatorCacheKey: providerScipPreRefresh?.cache?.key,
|
|
1756
|
+
scannedPaths: providerExecutionScan.files.map((file) => file.path),
|
|
1738
1757
|
scannedFiles: providerExecutionScan.files,
|
|
1739
1758
|
recordPhaseTiming: recordProviderFirstPhaseTiming,
|
|
1759
|
+
onProgress,
|
|
1740
1760
|
signal,
|
|
1741
|
-
}
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
scannedPaths: providerExecutionScan.files.map((file) => file.path),
|
|
1751
|
-
scannedFiles: providerExecutionScan.files,
|
|
1752
|
-
recordPhaseTiming: recordProviderFirstPhaseTiming,
|
|
1753
|
-
onProgress,
|
|
1754
|
-
signal,
|
|
1755
|
-
};
|
|
1756
|
-
return providerFirstExecutor === "scipIncremental"
|
|
1757
|
-
? executeProviderFirstScipIncremental(scipParams)
|
|
1758
|
-
: executeProviderFirstScipFull(scipParams);
|
|
1759
|
-
});
|
|
1760
|
-
}
|
|
1761
|
-
finally {
|
|
1762
|
-
await cleanupProviderFirstIncrementalTempPaths(providerIncrementalTempPaths);
|
|
1763
|
-
}
|
|
1764
|
-
}
|
|
1765
|
-
catch (err) {
|
|
1766
|
-
if (err instanceof ProviderFirstGraphValidationError) {
|
|
1767
|
-
throw err;
|
|
1761
|
+
};
|
|
1762
|
+
return providerFirstExecutor === "scipIncremental"
|
|
1763
|
+
? executeProviderFirstScipIncremental(scipParams)
|
|
1764
|
+
: executeProviderFirstScipFull(scipParams);
|
|
1765
|
+
});
|
|
1766
|
+
}
|
|
1767
|
+
finally {
|
|
1768
|
+
await cleanupProviderFirstIncrementalTempPaths(providerIncrementalTempPaths);
|
|
1769
|
+
}
|
|
1768
1770
|
}
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
const
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1771
|
+
catch (err) {
|
|
1772
|
+
if (err instanceof ProviderFirstGraphValidationError) {
|
|
1773
|
+
throw err;
|
|
1774
|
+
}
|
|
1775
|
+
const reason = err instanceof Error
|
|
1776
|
+
? err.message
|
|
1777
|
+
: `provider-first SCIP execution failed: ${String(err)}`;
|
|
1778
|
+
if (providerFirst.requestedMode === "auto") {
|
|
1779
|
+
const fallbackReason = formatProviderFirstAutoFallbackReason({
|
|
1780
|
+
reason,
|
|
1781
|
+
providerFirstIncrementalActive,
|
|
1782
|
+
});
|
|
1783
|
+
logger.warn("indexRepo: provider-first execution failed, using legacy fallback", {
|
|
1784
|
+
repoId,
|
|
1785
|
+
reason: fallbackReason,
|
|
1786
|
+
});
|
|
1787
|
+
providerFirstExecutionFallback = providerFirstFallbackSummary([
|
|
1788
|
+
fallbackReason,
|
|
1789
|
+
]);
|
|
1790
|
+
if (providerFirstIncrementalActive) {
|
|
1791
|
+
// Keep provider-first incremental failures scoped; otherwise the
|
|
1792
|
+
// legacy safety fallback degenerates into a full repo pass.
|
|
1793
|
+
providerFirstScan = providerExecutionScan;
|
|
1794
|
+
for (const file of providerExecutionScan.files) {
|
|
1795
|
+
providerFirstFallbackPaths.add(file.path);
|
|
1796
|
+
}
|
|
1797
|
+
providerFirstLegacyFallbackComplete = true;
|
|
1798
|
+
providerFirstLegacyFallbackFileCount = providerExecutionScan.files.length;
|
|
1799
|
+
providerFirstLegacyFallbackSamplePaths = providerExecutionScan.files
|
|
1800
|
+
.map((file) => file.path)
|
|
1801
|
+
.slice(0, 10);
|
|
1802
|
+
providerFirstLegacyFallbackStartedAt = Date.now();
|
|
1790
1803
|
}
|
|
1791
|
-
providerFirstLegacyFallbackComplete = true;
|
|
1792
|
-
providerFirstLegacyFallbackFileCount = providerExecutionScan.files.length;
|
|
1793
|
-
providerFirstLegacyFallbackSamplePaths = providerExecutionScan.files
|
|
1794
|
-
.map((file) => file.path)
|
|
1795
|
-
.slice(0, 10);
|
|
1796
|
-
providerFirstLegacyFallbackStartedAt = Date.now();
|
|
1797
1804
|
}
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1805
|
+
else {
|
|
1806
|
+
throw err;
|
|
1807
|
+
}
|
|
1801
1808
|
}
|
|
1802
1809
|
}
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1810
|
+
if (providerResult) {
|
|
1811
|
+
const executionFailureReasons = providerFirstFatalFailureReasons({
|
|
1812
|
+
failures: "failures" in providerResult ? providerResult.failures : [],
|
|
1813
|
+
providerRowsAvailable: providerResult.rows.files.length > 0,
|
|
1814
|
+
});
|
|
1815
|
+
if (executionFailureReasons.length > 0) {
|
|
1816
|
+
if (providerFirst.requestedMode === "auto") {
|
|
1817
|
+
logger.warn("indexRepo: provider-first execution had failures, using legacy fallback", {
|
|
1818
|
+
repoId,
|
|
1819
|
+
reasons: executionFailureReasons,
|
|
1820
|
+
});
|
|
1821
|
+
providerFirstExecutionFallback = providerFirstFallbackSummary(executionFailureReasons);
|
|
1822
|
+
}
|
|
1823
|
+
else {
|
|
1824
|
+
throw new Error(`Provider-first indexing cannot execute for ${repoId}: ${executionFailureReasons.join("; ")}`);
|
|
1825
|
+
}
|
|
1816
1826
|
}
|
|
1817
1827
|
else {
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1828
|
+
const providerScan = providerExecutionScan;
|
|
1829
|
+
providerFirstScan = providerScan;
|
|
1830
|
+
if (providerFactsShouldDropCoveragePayloads(providerResult.facts)) {
|
|
1831
|
+
clearProviderFactPayloadsForCoverageAnalysis(providerResult.facts);
|
|
1832
|
+
emitProviderFirstProgress(onProgress, "coverageAnalyze", {
|
|
1833
|
+
message: "large provider occurrence payloads released",
|
|
1834
|
+
});
|
|
1835
|
+
}
|
|
1836
|
+
const scanScopedProvider = filterProviderFirstDataToScannedScope({
|
|
1837
|
+
rows: providerResult.rows,
|
|
1838
|
+
facts: providerResult.facts,
|
|
1839
|
+
scannedPaths: providerScan.files.map((file) => file.path),
|
|
1840
|
+
});
|
|
1826
1841
|
emitProviderFirstProgress(onProgress, "coverageAnalyze", {
|
|
1827
|
-
message: "
|
|
1842
|
+
message: "provider rows filtered to scan scope",
|
|
1828
1843
|
});
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
const scanHasCppSemanticPaths = providerScan.files.some((file) => isCppSemanticScanPath(file.path));
|
|
1840
|
-
const semanticEligiblePaths = scanHasCppSemanticPaths
|
|
1841
|
-
? await resolveProviderFirstSemanticEligiblePaths({
|
|
1842
|
-
repoRoot: repoRow.rootPath,
|
|
1844
|
+
const scanScopedProviderPaths = scanScopedProvider.rows.files.map((file) => file.relPath);
|
|
1845
|
+
const scanHasCppSemanticPaths = providerScan.files.some((file) => isCppSemanticScanPath(file.path));
|
|
1846
|
+
const semanticEligiblePaths = scanHasCppSemanticPaths
|
|
1847
|
+
? await resolveProviderFirstSemanticEligiblePaths({
|
|
1848
|
+
repoRoot: repoRow.rootPath,
|
|
1849
|
+
scannedPaths: providerScan.files.map((file) => file.path),
|
|
1850
|
+
providerPaths: scanScopedProviderPaths,
|
|
1851
|
+
})
|
|
1852
|
+
: undefined;
|
|
1853
|
+
const coverageReport = analyzeProviderFirstCoverage({
|
|
1843
1854
|
scannedPaths: providerScan.files.map((file) => file.path),
|
|
1855
|
+
semanticEligiblePaths: semanticEligiblePaths && semanticEligiblePaths.size > 0
|
|
1856
|
+
? semanticEligiblePaths
|
|
1857
|
+
: undefined,
|
|
1844
1858
|
providerPaths: scanScopedProviderPaths,
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
semanticEligiblePaths: semanticEligiblePaths && semanticEligiblePaths.size > 0
|
|
1850
|
-
? semanticEligiblePaths
|
|
1851
|
-
: undefined,
|
|
1852
|
-
providerPaths: scanScopedProviderPaths,
|
|
1853
|
-
coverage: scanScopedProvider.facts.coverage,
|
|
1854
|
-
symbols: scanScopedProvider.facts.symbols,
|
|
1855
|
-
occurrences: scanScopedProvider.facts.occurrences,
|
|
1856
|
-
sourceLinesByPath: scanScopedProvider.facts.sourceLinesByPath,
|
|
1857
|
-
});
|
|
1858
|
-
emitProviderFirstProgress(onProgress, "coverageAnalyze", {
|
|
1859
|
-
message: "provider coverage analyzed",
|
|
1860
|
-
});
|
|
1861
|
-
if (coverageReport.fatalReasons.length > 0) {
|
|
1862
|
-
throw new Error(`Provider-first indexing cannot execute for ${repoId}: ${coverageReport.fatalReasons.join("; ")}`);
|
|
1863
|
-
}
|
|
1864
|
-
else {
|
|
1865
|
-
const legacyFallbackPlan = resolveProviderFirstLegacyFallbackPlan({
|
|
1866
|
-
fallbackFileCount: coverageReport.fallbackPaths.size,
|
|
1867
|
-
semanticEligibleFallbackFileCount: coverageReport.summary.semanticEligibilityGap?.totalFiles,
|
|
1868
|
-
maxLegacyFallbackFiles: providerFirstConfig.maxLegacyFallbackFiles,
|
|
1869
|
-
maxSemanticEligibleFallbackFiles: providerFirstConfig.maxSemanticEligibleFallbackFiles,
|
|
1859
|
+
coverage: scanScopedProvider.facts.coverage,
|
|
1860
|
+
symbols: scanScopedProvider.facts.symbols,
|
|
1861
|
+
occurrences: scanScopedProvider.facts.occurrences,
|
|
1862
|
+
sourceLinesByPath: scanScopedProvider.facts.sourceLinesByPath,
|
|
1870
1863
|
});
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
callProofSkipReason: callProofSkipDerivedStateReason(coverageReport.callProofIncompletePaths),
|
|
1874
|
-
skippedLegacyFallbackReason,
|
|
1864
|
+
emitProviderFirstProgress(onProgress, "coverageAnalyze", {
|
|
1865
|
+
message: "provider coverage analyzed",
|
|
1875
1866
|
});
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
providerFirstShadowStagingSkipReason =
|
|
1879
|
-
readinessGates.shadowStagingSkipReason;
|
|
1880
|
-
if (scopedSourceFileListActive) {
|
|
1881
|
-
providerFirstShadowStagingSkipReason =
|
|
1882
|
-
providerFirstShadowStagingSkipReason
|
|
1883
|
-
? `${providerFirstShadowStagingSkipReason}; ${scopedSourceFileListReason}`
|
|
1884
|
-
: scopedSourceFileListReason;
|
|
1885
|
-
}
|
|
1886
|
-
if (providerFirstShadowRepoScopeSkipReason) {
|
|
1887
|
-
// ponytail: revisit multi-repo shadow DBs when shadow builds can copy
|
|
1888
|
-
// a complete DB, including non-target repos, before activation.
|
|
1889
|
-
providerFirstShadowStagingSkipReason =
|
|
1890
|
-
providerFirstShadowStagingSkipReason
|
|
1891
|
-
? `${providerFirstShadowStagingSkipReason}; ${providerFirstShadowRepoScopeSkipReason}`
|
|
1892
|
-
: providerFirstShadowRepoScopeSkipReason;
|
|
1867
|
+
if (coverageReport.fatalReasons.length > 0) {
|
|
1868
|
+
throw new Error(`Provider-first indexing cannot execute for ${repoId}: ${coverageReport.fatalReasons.join("; ")}`);
|
|
1893
1869
|
}
|
|
1894
|
-
|
|
1895
|
-
const
|
|
1870
|
+
else {
|
|
1871
|
+
const legacyFallbackPlan = resolveProviderFirstLegacyFallbackPlan({
|
|
1872
|
+
fallbackFileCount: coverageReport.fallbackPaths.size,
|
|
1873
|
+
semanticEligibleFallbackFileCount: coverageReport.summary.semanticEligibilityGap?.totalFiles,
|
|
1874
|
+
maxLegacyFallbackFiles: providerFirstConfig.maxLegacyFallbackFiles,
|
|
1875
|
+
maxSemanticEligibleFallbackFiles: providerFirstConfig.maxSemanticEligibleFallbackFiles,
|
|
1876
|
+
});
|
|
1877
|
+
const skippedLegacyFallbackReason = skippedProviderFirstLegacyFallbackReason(legacyFallbackPlan);
|
|
1878
|
+
const readinessGates = resolveProviderFirstReadinessGates({
|
|
1879
|
+
callProofSkipReason: callProofSkipDerivedStateReason(coverageReport.callProofIncompletePaths),
|
|
1880
|
+
skippedLegacyFallbackReason,
|
|
1881
|
+
});
|
|
1882
|
+
providerFirstSkipDerivedStateReason =
|
|
1883
|
+
readinessGates.skipDerivedStateReason;
|
|
1896
1884
|
providerFirstShadowStagingSkipReason =
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
semanticEligiblePaths,
|
|
1904
|
-
parsedFiles: legacyFallbackPlan.parsedFiles,
|
|
1905
|
-
});
|
|
1906
|
-
const providerRowsExcludedByLegacyFallback = legacyFallbackPlan.runLegacyFallback
|
|
1907
|
-
? selectProviderRowsExcludedByLegacyFallback({
|
|
1908
|
-
fallbackPaths: legacyFallbackPaths,
|
|
1909
|
-
coverage: scanScopedProvider.facts.coverage,
|
|
1910
|
-
})
|
|
1911
|
-
: new Set();
|
|
1912
|
-
const materializedRows = filterProviderRowsForFallback(scanScopedProvider.rows, providerRowsExcludedByLegacyFallback);
|
|
1913
|
-
await measureProviderFirstPhase("persistProvenance", "providerFirstPersistProvenance", () => withWriteConn((conn) => persistProviderFirstProvenance(conn, scanScopedProvider.facts)));
|
|
1914
|
-
// The provider fact set can include every SCIP occurrence and source
|
|
1915
|
-
// line. Drop it before legacy fallback and versioning so large repos
|
|
1916
|
-
// do not carry decoded provider payloads into post-index finalization.
|
|
1917
|
-
clearProviderFactPayloadsForGc(providerResult.facts);
|
|
1918
|
-
if (scanScopedProvider.facts !== providerResult.facts) {
|
|
1919
|
-
clearProviderFactPayloadsForGc(scanScopedProvider.facts);
|
|
1920
|
-
}
|
|
1921
|
-
if (providerResult.rows !== materializedRows) {
|
|
1922
|
-
clearProviderGraphRowsForGc(providerResult.rows);
|
|
1923
|
-
}
|
|
1924
|
-
if (scanScopedProvider.rows !== materializedRows &&
|
|
1925
|
-
scanScopedProvider.rows !== providerResult.rows) {
|
|
1926
|
-
clearProviderGraphRowsForGc(scanScopedProvider.rows);
|
|
1927
|
-
}
|
|
1928
|
-
await measureProviderFirstPhase("postProviderGc", "providerFirstPostProviderGc", () => flushProviderFirstPayloadFinalizers());
|
|
1929
|
-
emitProviderFirstProgress(onProgress, "coverageAnalyze", {
|
|
1930
|
-
message: "provider payload finalizers flushed",
|
|
1931
|
-
});
|
|
1932
|
-
providerFirstProviderRows = materializedRows;
|
|
1933
|
-
providerFirstGenerationId = providerResult.generationId;
|
|
1934
|
-
applyScannedFileMetadataToProviderRows({
|
|
1935
|
-
rows: materializedRows,
|
|
1936
|
-
scannedFiles: providerScan.files,
|
|
1937
|
-
});
|
|
1938
|
-
let shadowBuild;
|
|
1939
|
-
if (!legacyFallbackPlan.runLegacyFallback) {
|
|
1940
|
-
if (providerFirstShadowStagingSkipReason) {
|
|
1941
|
-
const shadowStageTotal = providerFirstGraphRowTotal(materializedRows);
|
|
1942
|
-
emitProviderFirstProgress(onProgress, "shadowStage", {
|
|
1943
|
-
stageCurrent: 0,
|
|
1944
|
-
stageTotal: shadowStageTotal,
|
|
1945
|
-
message: `shadow staging skipped: ${providerFirstShadowStagingSkipReason}`,
|
|
1946
|
-
});
|
|
1947
|
-
shadowBuild = skippedProviderFirstShadowBuild({
|
|
1948
|
-
generationId: providerResult.generationId,
|
|
1949
|
-
activation: providerFirstConfig.activation,
|
|
1950
|
-
requestedFormat: providerFirstConfig.stagingFormat,
|
|
1951
|
-
rows: materializedRows,
|
|
1952
|
-
reason: providerFirstShadowStagingSkipReason,
|
|
1953
|
-
});
|
|
1885
|
+
readinessGates.shadowStagingSkipReason;
|
|
1886
|
+
if (scopedSourceFileListActive) {
|
|
1887
|
+
providerFirstShadowStagingSkipReason =
|
|
1888
|
+
providerFirstShadowStagingSkipReason
|
|
1889
|
+
? `${providerFirstShadowStagingSkipReason}; ${scopedSourceFileListReason}`
|
|
1890
|
+
: scopedSourceFileListReason;
|
|
1954
1891
|
}
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1892
|
+
if (providerFirstShadowRepoScopeSkipReason) {
|
|
1893
|
+
// ponytail: revisit multi-repo shadow DBs when shadow builds can copy
|
|
1894
|
+
// a complete DB, including non-target repos, before activation.
|
|
1895
|
+
providerFirstShadowStagingSkipReason =
|
|
1896
|
+
providerFirstShadowStagingSkipReason
|
|
1897
|
+
? `${providerFirstShadowStagingSkipReason}; ${providerFirstShadowRepoScopeSkipReason}`
|
|
1898
|
+
: providerFirstShadowRepoScopeSkipReason;
|
|
1899
|
+
}
|
|
1900
|
+
if (providerFirstIncrementalActive) {
|
|
1901
|
+
const incrementalShadowSkipReason = "shadow staging skipped because provider-first incremental materializes only changed files";
|
|
1902
|
+
providerFirstShadowStagingSkipReason =
|
|
1903
|
+
providerFirstShadowStagingSkipReason
|
|
1904
|
+
? `${providerFirstShadowStagingSkipReason}; ${incrementalShadowSkipReason}`
|
|
1905
|
+
: incrementalShadowSkipReason;
|
|
1906
|
+
}
|
|
1907
|
+
const legacyFallbackPaths = selectProviderFirstLegacyFallbackPaths({
|
|
1908
|
+
fallbackPaths: coverageReport.fallbackPaths,
|
|
1909
|
+
semanticEligiblePaths,
|
|
1910
|
+
parsedFiles: legacyFallbackPlan.parsedFiles,
|
|
1911
|
+
});
|
|
1912
|
+
const providerRowsExcludedByLegacyFallback = legacyFallbackPlan.runLegacyFallback
|
|
1913
|
+
? selectProviderRowsExcludedByLegacyFallback({
|
|
1914
|
+
fallbackPaths: legacyFallbackPaths,
|
|
1915
|
+
coverage: scanScopedProvider.facts.coverage,
|
|
1916
|
+
})
|
|
1917
|
+
: new Set();
|
|
1918
|
+
const materializedRows = filterProviderRowsForFallback(scanScopedProvider.rows, providerRowsExcludedByLegacyFallback);
|
|
1919
|
+
await measureProviderFirstPhase("persistProvenance", "providerFirstPersistProvenance", () => withWriteConn((conn) => persistProviderFirstProvenance(conn, scanScopedProvider.facts)));
|
|
1920
|
+
// The provider fact set can include every SCIP occurrence and source
|
|
1921
|
+
// line. Drop it before legacy fallback and versioning so large repos
|
|
1922
|
+
// do not carry decoded provider payloads into post-index finalization.
|
|
1923
|
+
clearProviderFactPayloadsForGc(providerResult.facts);
|
|
1924
|
+
if (scanScopedProvider.facts !== providerResult.facts) {
|
|
1925
|
+
clearProviderFactPayloadsForGc(scanScopedProvider.facts);
|
|
1926
|
+
}
|
|
1927
|
+
if (providerResult.rows !== materializedRows) {
|
|
1928
|
+
clearProviderGraphRowsForGc(providerResult.rows);
|
|
1929
|
+
}
|
|
1930
|
+
if (scanScopedProvider.rows !== materializedRows &&
|
|
1931
|
+
scanScopedProvider.rows !== providerResult.rows) {
|
|
1932
|
+
clearProviderGraphRowsForGc(scanScopedProvider.rows);
|
|
1933
|
+
}
|
|
1934
|
+
await measureProviderFirstPhase("postProviderGc", "providerFirstPostProviderGc", () => flushProviderFirstPayloadFinalizers());
|
|
1935
|
+
emitProviderFirstProgress(onProgress, "coverageAnalyze", {
|
|
1936
|
+
message: "provider payload finalizers flushed",
|
|
1937
|
+
});
|
|
1938
|
+
providerFirstProviderRows = materializedRows;
|
|
1939
|
+
providerFirstGenerationId = providerResult.generationId;
|
|
1940
|
+
applyScannedFileMetadataToProviderRows({
|
|
1941
|
+
rows: materializedRows,
|
|
1942
|
+
scannedFiles: providerScan.files,
|
|
1943
|
+
});
|
|
1944
|
+
let shadowBuild;
|
|
1945
|
+
if (!legacyFallbackPlan.runLegacyFallback) {
|
|
1946
|
+
if (providerFirstShadowStagingSkipReason) {
|
|
1947
|
+
const shadowStageTotal = providerFirstGraphRowTotal(materializedRows);
|
|
1948
|
+
emitProviderFirstProgress(onProgress, "shadowStage", {
|
|
1949
|
+
stageCurrent: 0,
|
|
1950
|
+
stageTotal: shadowStageTotal,
|
|
1951
|
+
message: `shadow staging skipped: ${providerFirstShadowStagingSkipReason}`,
|
|
1981
1952
|
});
|
|
1982
|
-
|
|
1983
|
-
|
|
1953
|
+
shadowBuild = skippedProviderFirstShadowBuild({
|
|
1954
|
+
generationId: providerResult.generationId,
|
|
1955
|
+
activation: providerFirstConfig.activation,
|
|
1956
|
+
requestedFormat: providerFirstConfig.stagingFormat,
|
|
1957
|
+
rows: materializedRows,
|
|
1958
|
+
reason: providerFirstShadowStagingSkipReason,
|
|
1959
|
+
});
|
|
1960
|
+
}
|
|
1961
|
+
else {
|
|
1962
|
+
const shadowStageTotal = providerFirstGraphRowTotal(materializedRows);
|
|
1984
1963
|
emitProviderFirstProgress(onProgress, "shadowStage", {
|
|
1985
|
-
stageCurrent:
|
|
1986
|
-
stageTotal:
|
|
1987
|
-
message: "provider rows
|
|
1964
|
+
stageCurrent: 0,
|
|
1965
|
+
stageTotal: shadowStageTotal,
|
|
1966
|
+
message: "staging provider rows for shadow bulk load",
|
|
1988
1967
|
});
|
|
1968
|
+
shadowBuild = await measureProviderFirstPhase("shadowStage", "providerFirstShadowStage", () => stageProviderFirstShadowBuild({
|
|
1969
|
+
repoId,
|
|
1970
|
+
generationId: providerResult.generationId,
|
|
1971
|
+
activation: providerFirstConfig.activation,
|
|
1972
|
+
requestedFormat: providerFirstConfig.stagingFormat,
|
|
1973
|
+
activeDbPath: getLadybugDbPath(),
|
|
1974
|
+
repoRoot: repoRow.rootPath,
|
|
1975
|
+
repoConfigJson: repoRow.configJson,
|
|
1976
|
+
rows: materializedRows,
|
|
1977
|
+
}));
|
|
1978
|
+
shadowBuild.activationResult =
|
|
1979
|
+
summarizeProviderFirstShadowActivationReadiness({
|
|
1980
|
+
shadowBuild,
|
|
1981
|
+
fallbackFiles: 0,
|
|
1982
|
+
graphDerivedStateReady: providerFirstSkipDerivedStateReason === undefined,
|
|
1983
|
+
shadowContainsFinalizedGraph: false,
|
|
1984
|
+
finalizedGraphReasons: providerFirstSkipDerivedStateReason
|
|
1985
|
+
? [providerFirstSkipDerivedStateReason]
|
|
1986
|
+
: undefined,
|
|
1987
|
+
});
|
|
1988
|
+
if (shadowBuild.status === "staged") {
|
|
1989
|
+
const stagedTotal = providerFirstShadowStageTotal(shadowBuild.counts);
|
|
1990
|
+
emitProviderFirstProgress(onProgress, "shadowStage", {
|
|
1991
|
+
stageCurrent: stagedTotal,
|
|
1992
|
+
stageTotal: stagedTotal,
|
|
1993
|
+
message: "provider rows staged for shadow bulk load",
|
|
1994
|
+
});
|
|
1995
|
+
}
|
|
1989
1996
|
}
|
|
1990
1997
|
}
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1998
|
+
const activeProviderInputHash = scopedSourceFileListActive || providerFirstIncrementalActive
|
|
1999
|
+
? null
|
|
2000
|
+
: "generatedIndexes" in providerResult
|
|
2001
|
+
? providerFirstActiveInputFingerprint(providerResult.generatedIndexes)
|
|
2002
|
+
: null;
|
|
2003
|
+
const activeProviderInputRecord = activeProviderInputHash
|
|
2004
|
+
? await ladybugDb.getScipIngestionRecord(conn, repoId, PROVIDER_FIRST_ACTIVE_INPUT_RECORD_PATH)
|
|
1996
2005
|
: null;
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
activeProviderInputMatches,
|
|
2019
|
-
existingProviderSymbolCount,
|
|
2020
|
-
});
|
|
2021
|
-
if (activeMaterializationPlan.reuseExistingProviderRows) {
|
|
2022
|
-
const deleteTotal = providerFirstMaterializePhaseTotal("deleteFileSymbols", materializedRows, activeMaterializationPlan);
|
|
2023
|
-
emitProviderFirstProgress(onProgress, "materialize.deleteFileSymbols", {
|
|
2024
|
-
stageCurrent: 0,
|
|
2025
|
-
stageTotal: deleteTotal,
|
|
2026
|
-
message: "provider active stale cleanup skipped for large symbol set",
|
|
2027
|
-
});
|
|
2028
|
-
const symbolTotal = providerFirstMaterializePhaseTotal("upsertSymbols", materializedRows, activeMaterializationPlan);
|
|
2029
|
-
emitProviderFirstProgress(onProgress, "materialize.upsertSymbols", {
|
|
2030
|
-
stageCurrent: symbolTotal,
|
|
2031
|
-
stageTotal: symbolTotal,
|
|
2032
|
-
message: "provider active rows reused for existing large symbol set",
|
|
2033
|
-
});
|
|
2034
|
-
}
|
|
2035
|
-
await measureProviderFirstPhase("materialize", "providerFirstMaterialize", () => {
|
|
2006
|
+
const activeProviderInputMatches = Boolean(activeProviderInputHash &&
|
|
2007
|
+
activeProviderInputRecord?.contentHash ===
|
|
2008
|
+
activeProviderInputHash &&
|
|
2009
|
+
activeProviderInputRecord.truncated !== true);
|
|
2010
|
+
const existingProviderSymbolCount = await countExistingScipProviderSymbols(conn, repoId);
|
|
2011
|
+
const activeMaterializationPlan = providerFirstIncrementalActive
|
|
2012
|
+
? {
|
|
2013
|
+
deleteExistingFileSymbols: true,
|
|
2014
|
+
useKnownFreshWriters: true,
|
|
2015
|
+
writeEdges: true,
|
|
2016
|
+
reuseExistingProviderRows: false,
|
|
2017
|
+
}
|
|
2018
|
+
: resolveProviderFirstActiveMaterializationPlan({
|
|
2019
|
+
existingProviderFileCount: countExistingProviderPrimaryFiles({
|
|
2020
|
+
providerFiles: materializedRows.files,
|
|
2021
|
+
existingByPath: providerScan.existingByPath,
|
|
2022
|
+
}),
|
|
2023
|
+
providerSymbolCount: materializedRows.symbols.length,
|
|
2024
|
+
activeProviderInputMatches,
|
|
2025
|
+
existingProviderSymbolCount,
|
|
2026
|
+
});
|
|
2036
2027
|
if (activeMaterializationPlan.reuseExistingProviderRows) {
|
|
2037
|
-
|
|
2028
|
+
const deleteTotal = providerFirstMaterializePhaseTotal("deleteFileSymbols", materializedRows, activeMaterializationPlan);
|
|
2029
|
+
emitProviderFirstProgress(onProgress, "materialize.deleteFileSymbols", {
|
|
2030
|
+
stageCurrent: 0,
|
|
2031
|
+
stageTotal: deleteTotal,
|
|
2032
|
+
message: "provider active stale cleanup skipped for large symbol set",
|
|
2033
|
+
});
|
|
2034
|
+
const symbolTotal = providerFirstMaterializePhaseTotal("upsertSymbols", materializedRows, activeMaterializationPlan);
|
|
2035
|
+
emitProviderFirstProgress(onProgress, "materialize.upsertSymbols", {
|
|
2036
|
+
stageCurrent: symbolTotal,
|
|
2037
|
+
stageTotal: symbolTotal,
|
|
2038
|
+
message: "provider active rows reused for existing large symbol set",
|
|
2039
|
+
});
|
|
2038
2040
|
}
|
|
2039
|
-
|
|
2040
|
-
if (
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
});
|
|
2060
|
-
const startedAt = Date.now();
|
|
2061
|
-
try {
|
|
2062
|
-
return await fn();
|
|
2063
|
-
}
|
|
2064
|
-
finally {
|
|
2065
|
-
recordProviderFirstPhaseTiming(`materialize.${phaseName}`, Date.now() - startedAt);
|
|
2041
|
+
await measureProviderFirstPhase("materialize", "providerFirstMaterialize", () => {
|
|
2042
|
+
if (activeMaterializationPlan.reuseExistingProviderRows) {
|
|
2043
|
+
return Promise.resolve();
|
|
2044
|
+
}
|
|
2045
|
+
return withWriteConn(async (conn) => {
|
|
2046
|
+
if (providerRowsHaveMaterialization(materializedRows)) {
|
|
2047
|
+
// materializeProviderFacts owns its transaction. Wrapping it
|
|
2048
|
+
// in another transaction makes large provider-first writes
|
|
2049
|
+
// crash in LadybugDB's native transaction handling.
|
|
2050
|
+
await materializeProviderFacts(conn, materializedRows, {
|
|
2051
|
+
replaceFileSymbols: true,
|
|
2052
|
+
deleteExistingFileSymbols: activeMaterializationPlan.deleteExistingFileSymbols,
|
|
2053
|
+
useKnownFreshWriters: activeMaterializationPlan.useKnownFreshWriters,
|
|
2054
|
+
writeEdges: activeMaterializationPlan.writeEdges,
|
|
2055
|
+
pruneExternalSymbols: !scopedSourceFileListActive &&
|
|
2056
|
+
!providerFirstIncrementalActive,
|
|
2057
|
+
symbolFtsIndexName: appConfig.semantic?.retrieval?.fts?.indexName,
|
|
2058
|
+
measurePhase: async (phaseName, fn) => {
|
|
2059
|
+
const substage = `materialize.${phaseName}`;
|
|
2060
|
+
const phaseTotal = providerFirstMaterializePhaseTotal(phaseName, materializedRows, activeMaterializationPlan);
|
|
2066
2061
|
emitProviderFirstProgress(onProgress, substage, {
|
|
2067
|
-
stageCurrent:
|
|
2062
|
+
stageCurrent: 0,
|
|
2068
2063
|
stageTotal: phaseTotal,
|
|
2069
|
-
message: `provider materialize ${phaseName}
|
|
2064
|
+
message: `provider materialize ${phaseName} start`,
|
|
2070
2065
|
});
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
}
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2066
|
+
const startedAt = Date.now();
|
|
2067
|
+
try {
|
|
2068
|
+
return await fn();
|
|
2069
|
+
}
|
|
2070
|
+
finally {
|
|
2071
|
+
recordProviderFirstPhaseTiming(`materialize.${phaseName}`, Date.now() - startedAt);
|
|
2072
|
+
emitProviderFirstProgress(onProgress, substage, {
|
|
2073
|
+
stageCurrent: phaseTotal,
|
|
2074
|
+
stageTotal: phaseTotal,
|
|
2075
|
+
message: `provider materialize ${phaseName} done`,
|
|
2076
|
+
});
|
|
2077
|
+
}
|
|
2078
|
+
},
|
|
2079
|
+
});
|
|
2080
|
+
}
|
|
2081
|
+
else if (!scopedSourceFileListActive &&
|
|
2082
|
+
!providerFirstIncrementalActive) {
|
|
2083
|
+
await ladybugDb.withTransaction(conn, async (txConn) => {
|
|
2084
|
+
await ladybugDb.pruneStaleScipExternalSymbols(txConn, repoId, []);
|
|
2085
|
+
});
|
|
2086
|
+
}
|
|
2087
|
+
if (!providerFirstRemovedFilesDeleted &&
|
|
2088
|
+
providerScan.removedFileIds.length > 0) {
|
|
2089
|
+
await ladybugDb.withTransaction(conn, async (txConn) => {
|
|
2090
|
+
await ladybugDb.deleteFilesByIds(txConn, providerScan.removedFileIds);
|
|
2091
|
+
});
|
|
2092
|
+
}
|
|
2093
|
+
if (activeProviderInputHash) {
|
|
2094
|
+
await ladybugDb.mergeScipIngestionRecord(conn, {
|
|
2095
|
+
id: hashValue({
|
|
2096
|
+
repoId,
|
|
2097
|
+
indexPath: PROVIDER_FIRST_ACTIVE_INPUT_RECORD_PATH,
|
|
2098
|
+
}),
|
|
2090
2099
|
repoId,
|
|
2091
2100
|
indexPath: PROVIDER_FIRST_ACTIVE_INPUT_RECORD_PATH,
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2101
|
+
contentHash: activeProviderInputHash,
|
|
2102
|
+
ingestedAt: new Date().toISOString(),
|
|
2103
|
+
ledgerVersion: providerResult.generationId,
|
|
2104
|
+
symbolCount: materializedRows.symbols.length,
|
|
2105
|
+
edgeCount: activeMaterializationPlan.writeEdges
|
|
2106
|
+
? materializedRows.edges.length
|
|
2107
|
+
: 0,
|
|
2108
|
+
externalSymbolCount: materializedRows.externalSymbols.length,
|
|
2109
|
+
truncated: !activeMaterializationPlan.writeEdges,
|
|
2110
|
+
});
|
|
2111
|
+
}
|
|
2112
|
+
}, postIndexSessionTimeoutMs);
|
|
2113
|
+
});
|
|
2114
|
+
invalidateIndexResultCaches(repoId);
|
|
2115
|
+
providerFirstScipMaterialized = true;
|
|
2116
|
+
if (activeMaterializationPlan.reuseExistingProviderRows) {
|
|
2117
|
+
providerFirstMaterializedFiles = 0;
|
|
2118
|
+
providerFirstMaterializedSymbols = 0;
|
|
2119
|
+
providerFirstMaterializedEdges = 0;
|
|
2120
|
+
}
|
|
2121
|
+
else {
|
|
2122
|
+
providerFirstMaterializedFiles = materializedRows.files.length;
|
|
2123
|
+
providerFirstMaterializedSymbols =
|
|
2124
|
+
materializedRows.symbols.length +
|
|
2125
|
+
materializedRows.externalSymbols.length;
|
|
2126
|
+
providerFirstMaterializedEdges = materializedRows.edges.length;
|
|
2127
|
+
for (const fileId of materializedRows.changedFileIds) {
|
|
2128
|
+
providerFirstChangedFileIds.add(fileId);
|
|
2105
2129
|
}
|
|
2106
|
-
}, postIndexSessionTimeoutMs);
|
|
2107
|
-
});
|
|
2108
|
-
invalidateIndexResultCaches(repoId);
|
|
2109
|
-
providerFirstScipMaterialized = true;
|
|
2110
|
-
if (activeMaterializationPlan.reuseExistingProviderRows) {
|
|
2111
|
-
providerFirstMaterializedFiles = 0;
|
|
2112
|
-
providerFirstMaterializedSymbols = 0;
|
|
2113
|
-
providerFirstMaterializedEdges = 0;
|
|
2114
|
-
}
|
|
2115
|
-
else {
|
|
2116
|
-
providerFirstMaterializedFiles = materializedRows.files.length;
|
|
2117
|
-
providerFirstMaterializedSymbols =
|
|
2118
|
-
materializedRows.symbols.length +
|
|
2119
|
-
materializedRows.externalSymbols.length;
|
|
2120
|
-
providerFirstMaterializedEdges = materializedRows.edges.length;
|
|
2121
|
-
for (const fileId of materializedRows.changedFileIds) {
|
|
2122
|
-
providerFirstChangedFileIds.add(fileId);
|
|
2123
2130
|
}
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
.
|
|
2130
|
-
executionReasons.push(`SCIP provider ignored ${scanScopedProvider.ignoredProviderPaths.length} repo-relative file(s) outside the configured scan scope: ${sample}`);
|
|
2131
|
-
}
|
|
2132
|
-
if (activeMaterializationPlan.reuseExistingProviderRows) {
|
|
2133
|
-
executionReasons.push(`active provider rows reused for ${materializedRows.symbols.length} large existing symbol row(s); clean rebuild or shadow activation is required to physically retire stale provider rows`);
|
|
2134
|
-
}
|
|
2135
|
-
if (scopedSourceFileListActive) {
|
|
2136
|
-
executionReasons.push("repo.sourceFileListPath is set; provider-first active row reuse and shadow activation are disabled so this subset run cannot masquerade as a complete graph");
|
|
2137
|
-
}
|
|
2138
|
-
if (coverageReport.fallbackPaths.size > 0) {
|
|
2139
|
-
if (legacyFallbackPlan.runLegacyFallback) {
|
|
2140
|
-
executionReasons.push(`legacy fallback indexed ${legacyFallbackPlan.parsedFiles} uncovered or provider-unusable file(s) after provider-first materialization`);
|
|
2131
|
+
const executionReasons = [...coverageReport.reasons];
|
|
2132
|
+
if (scanScopedProvider.ignoredProviderPaths.length > 0) {
|
|
2133
|
+
const sample = scanScopedProvider.ignoredProviderPaths
|
|
2134
|
+
.slice(0, 5)
|
|
2135
|
+
.join(", ");
|
|
2136
|
+
executionReasons.push(`SCIP provider ignored ${scanScopedProvider.ignoredProviderPaths.length} repo-relative file(s) outside the configured scan scope: ${sample}`);
|
|
2141
2137
|
}
|
|
2142
|
-
if (
|
|
2143
|
-
executionReasons.push(
|
|
2138
|
+
if (activeMaterializationPlan.reuseExistingProviderRows) {
|
|
2139
|
+
executionReasons.push(`active provider rows reused for ${materializedRows.symbols.length} large existing symbol row(s); clean rebuild or shadow activation is required to physically retire stale provider rows`);
|
|
2144
2140
|
}
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
...coverageReport.summary,
|
|
2148
|
-
fallbackFiles: legacyFallbackPlan.parsedFiles,
|
|
2149
|
-
};
|
|
2150
|
-
if (legacyFallbackPlan.skippedFiles > 0) {
|
|
2151
|
-
coverageSummary.legacyFallbackSkippedFiles =
|
|
2152
|
-
legacyFallbackPlan.skippedFiles;
|
|
2153
|
-
coverageSummary.legacyFallbackFileLimit =
|
|
2154
|
-
legacyFallbackPlan.fileLimit;
|
|
2155
|
-
if (legacyFallbackPlan.semanticEligibleFallbackFiles !== undefined &&
|
|
2156
|
-
legacyFallbackPlan.semanticEligibleFileLimit !== undefined) {
|
|
2157
|
-
coverageSummary.semanticEligibleFallbackFiles =
|
|
2158
|
-
legacyFallbackPlan.semanticEligibleFallbackFiles;
|
|
2159
|
-
coverageSummary.semanticEligibleFallbackFileLimit =
|
|
2160
|
-
legacyFallbackPlan.semanticEligibleFileLimit;
|
|
2141
|
+
if (scopedSourceFileListActive) {
|
|
2142
|
+
executionReasons.push("repo.sourceFileListPath is set; provider-first active row reuse and shadow activation are disabled so this subset run cannot masquerade as a complete graph");
|
|
2161
2143
|
}
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
symbolsIndexed: materializedRows.symbols.length +
|
|
2173
|
-
materializedRows.externalSymbols.length,
|
|
2174
|
-
edgesCreated: materializedRows.edges.length,
|
|
2175
|
-
externalSymbolsIndexed: materializedRows.externalSymbols.length,
|
|
2176
|
-
shadowBuild,
|
|
2177
|
-
coverage: coverageSummary,
|
|
2178
|
-
reasons: executionReasons,
|
|
2179
|
-
};
|
|
2180
|
-
if (legacyFallbackPlan.runLegacyFallback) {
|
|
2181
|
-
logger.info("indexRepo: provider-first materialized; using legacy fallback for uncovered files", {
|
|
2182
|
-
repoId,
|
|
2144
|
+
if (coverageReport.fallbackPaths.size > 0) {
|
|
2145
|
+
if (legacyFallbackPlan.runLegacyFallback) {
|
|
2146
|
+
executionReasons.push(`legacy fallback indexed ${legacyFallbackPlan.parsedFiles} uncovered or provider-unusable file(s) after provider-first materialization`);
|
|
2147
|
+
}
|
|
2148
|
+
if (skippedLegacyFallbackReason) {
|
|
2149
|
+
executionReasons.push(skippedLegacyFallbackReason);
|
|
2150
|
+
}
|
|
2151
|
+
}
|
|
2152
|
+
const coverageSummary = {
|
|
2153
|
+
...coverageReport.summary,
|
|
2183
2154
|
fallbackFiles: legacyFallbackPlan.parsedFiles,
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2155
|
+
};
|
|
2156
|
+
if (legacyFallbackPlan.skippedFiles > 0) {
|
|
2157
|
+
coverageSummary.legacyFallbackSkippedFiles =
|
|
2158
|
+
legacyFallbackPlan.skippedFiles;
|
|
2159
|
+
coverageSummary.legacyFallbackFileLimit =
|
|
2160
|
+
legacyFallbackPlan.fileLimit;
|
|
2161
|
+
if (legacyFallbackPlan.semanticEligibleFallbackFiles !== undefined &&
|
|
2162
|
+
legacyFallbackPlan.semanticEligibleFileLimit !== undefined) {
|
|
2163
|
+
coverageSummary.semanticEligibleFallbackFiles =
|
|
2164
|
+
legacyFallbackPlan.semanticEligibleFallbackFiles;
|
|
2165
|
+
coverageSummary.semanticEligibleFallbackFileLimit =
|
|
2166
|
+
legacyFallbackPlan.semanticEligibleFileLimit;
|
|
2167
|
+
}
|
|
2189
2168
|
}
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2169
|
+
if (scanScopedProvider.ignoredProviderPaths.length > 0) {
|
|
2170
|
+
coverageSummary.ignoredProviderFiles =
|
|
2171
|
+
scanScopedProvider.ignoredProviderPaths.length;
|
|
2172
|
+
coverageSummary.ignoredProviderFileSamples =
|
|
2173
|
+
scanScopedProvider.ignoredProviderPaths.slice(0, 5);
|
|
2174
|
+
}
|
|
2175
|
+
providerFirstExecutedSummary = {
|
|
2176
|
+
...providerResult.summary,
|
|
2177
|
+
filesProcessed: materializedRows.files.length,
|
|
2178
|
+
symbolsIndexed: materializedRows.symbols.length +
|
|
2179
|
+
materializedRows.externalSymbols.length,
|
|
2180
|
+
edgesCreated: materializedRows.edges.length,
|
|
2181
|
+
externalSymbolsIndexed: materializedRows.externalSymbols.length,
|
|
2182
|
+
shadowBuild,
|
|
2183
|
+
coverage: coverageSummary,
|
|
2184
|
+
reasons: executionReasons,
|
|
2185
|
+
};
|
|
2186
|
+
if (legacyFallbackPlan.runLegacyFallback) {
|
|
2187
|
+
logger.info("indexRepo: provider-first materialized; using legacy fallback for uncovered files", {
|
|
2203
2188
|
repoId,
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
});
|
|
2207
|
-
const semanticRefresh = await runProviderFirstDeferredSemanticRefresh({
|
|
2208
|
-
versionId,
|
|
2209
|
-
semanticDeferred,
|
|
2189
|
+
fallbackFiles: legacyFallbackPlan.parsedFiles,
|
|
2190
|
+
reasons: executionReasons,
|
|
2210
2191
|
});
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2192
|
+
providerFirstScan = filterProviderFirstFallbackScan(providerScan, legacyFallbackPaths);
|
|
2193
|
+
for (const file of providerFirstScan.files) {
|
|
2194
|
+
providerFirstFallbackPaths.add(file.path);
|
|
2195
|
+
}
|
|
2196
|
+
providerFirstLegacyFallbackComplete =
|
|
2197
|
+
isProviderFirstLegacyFallbackPlanComplete(legacyFallbackPlan);
|
|
2198
|
+
providerFirstLegacyFallbackFileCount =
|
|
2199
|
+
legacyFallbackPlan.parsedFiles;
|
|
2200
|
+
providerFirstLegacyFallbackSamplePaths = [
|
|
2201
|
+
...legacyFallbackPaths,
|
|
2202
|
+
].slice(0, 10);
|
|
2203
|
+
providerFirstLegacyFallbackStartedAt = Date.now();
|
|
2204
|
+
}
|
|
2205
|
+
else {
|
|
2206
|
+
if (activeMaterializationPlan.reuseExistingProviderRows) {
|
|
2207
|
+
const versionId = await resolveActiveGraphVersionId("Provider-first SCIP active rows reused");
|
|
2208
|
+
let semanticDeferred = await markProviderFirstSemanticReadinessDeferred({
|
|
2209
|
+
repoId,
|
|
2210
|
+
versionId,
|
|
2211
|
+
appConfig,
|
|
2212
|
+
});
|
|
2213
|
+
const semanticRefresh = await runProviderFirstDeferredSemanticRefresh({
|
|
2214
|
+
versionId,
|
|
2215
|
+
semanticDeferred,
|
|
2216
|
+
});
|
|
2217
|
+
semanticDeferred = semanticRefresh.semanticDeferred;
|
|
2218
|
+
providerFirstExecutedSummary = withProviderFirstPhaseTimings(providerFirstExecutedSummary);
|
|
2219
|
+
const resultScip = "generatedIndexes" in providerResult
|
|
2220
|
+
? {
|
|
2221
|
+
generatedIndexes: providerResult.generatedIndexes,
|
|
2222
|
+
failures: providerResult.failures,
|
|
2223
|
+
generatorCache: activeScipPreRefresh?.cache,
|
|
2224
|
+
}
|
|
2225
|
+
: undefined;
|
|
2226
|
+
const result = {
|
|
2227
|
+
versionId,
|
|
2228
|
+
filesProcessed: 0,
|
|
2229
|
+
changedFiles: 0,
|
|
2230
|
+
removedFiles: 0,
|
|
2231
|
+
symbolsIndexed: 0,
|
|
2232
|
+
edgesCreated: 0,
|
|
2233
|
+
clustersComputed: 0,
|
|
2234
|
+
processesTraced: 0,
|
|
2235
|
+
durationMs: Date.now() - startTime,
|
|
2236
|
+
pass1Engine: emptyPass1EngineTelemetry(),
|
|
2237
|
+
scip: resultScip,
|
|
2238
|
+
providerFirst,
|
|
2239
|
+
providerFirstExecution: providerFirstExecutedSummary,
|
|
2240
|
+
semanticDeferred: semanticDeferred || undefined,
|
|
2241
|
+
algorithmRefresh: skippedDerivedStateResult("provider-first active rows reused").algorithmRefresh,
|
|
2242
|
+
};
|
|
2243
|
+
clearProviderGraphRowsForGc(materializedRows);
|
|
2244
|
+
return result;
|
|
2245
|
+
}
|
|
2246
|
+
const versionId = await createOrReuseVersion(providerFirstLspExecutor(providerResult.summary.executor)
|
|
2247
|
+
? providerFirstIncrementalActive
|
|
2248
|
+
? "Provider-first LSP incremental index"
|
|
2249
|
+
: "Provider-first LSP index"
|
|
2250
|
+
: providerFirstIncrementalActive
|
|
2251
|
+
? "Provider-first SCIP incremental index"
|
|
2252
|
+
: "Provider-first SCIP index", true);
|
|
2253
|
+
const pass1Engine = emptyPass1EngineTelemetry();
|
|
2254
|
+
const scip = "generatedIndexes" in providerResult
|
|
2214
2255
|
? {
|
|
2215
2256
|
generatedIndexes: providerResult.generatedIndexes,
|
|
2216
2257
|
failures: providerResult.failures,
|
|
2217
2258
|
generatorCache: activeScipPreRefresh?.cache,
|
|
2218
2259
|
}
|
|
2219
2260
|
: undefined;
|
|
2261
|
+
const post = await runPostIndexFinalization({
|
|
2262
|
+
versionId,
|
|
2263
|
+
indexMode: providerFirstIncrementalActive ? "incremental" : "full",
|
|
2264
|
+
filesTotal: providerFirstIncrementalActive
|
|
2265
|
+
? providerCoverageScan.files.length
|
|
2266
|
+
: materializedRows.files.length,
|
|
2267
|
+
filesScanned: materializedRows.files.length,
|
|
2268
|
+
symbolsExtracted: materializedRows.symbols.length +
|
|
2269
|
+
materializedRows.externalSymbols.length,
|
|
2270
|
+
edgesExtracted: materializedRows.edges.length,
|
|
2271
|
+
changedFileIdsForFinalize: scopedSourceFileListActive || providerFirstIncrementalActive
|
|
2272
|
+
? materializedRows.changedFileIds
|
|
2273
|
+
: undefined,
|
|
2274
|
+
changedTestFilePathsForFinalize: new Set(),
|
|
2275
|
+
changedFileIdsForMemory: materializedRows.changedFileIds,
|
|
2276
|
+
hasIndexMutations: true,
|
|
2277
|
+
callResolutionTelemetry: createCallResolutionTelemetry({
|
|
2278
|
+
repoId,
|
|
2279
|
+
mode: providerFirstIncrementalActive ? "incremental" : "full",
|
|
2280
|
+
pass2EligibleFileCount: 0,
|
|
2281
|
+
registeredResolvers: [],
|
|
2282
|
+
}),
|
|
2283
|
+
pass1Engine,
|
|
2284
|
+
scip,
|
|
2285
|
+
preloadedFileSummarySymbolFactsByFile: buildPreloadedFileSummarySymbolFactsFromRows({
|
|
2286
|
+
files: materializedRows.files,
|
|
2287
|
+
symbols: materializedRows.symbols,
|
|
2288
|
+
}),
|
|
2289
|
+
deferSemanticRefresh: true,
|
|
2290
|
+
skipDerivedStateReason: providerFirstSkipDerivedStateReason,
|
|
2291
|
+
});
|
|
2292
|
+
providerFirstExecutedSummary = {
|
|
2293
|
+
...providerFirstExecutedSummary,
|
|
2294
|
+
shadowBuild: await finalizeProviderFirstShadowBuild(providerFirstExecutedSummary.shadowBuild, versionId, post.semanticDeferred),
|
|
2295
|
+
};
|
|
2296
|
+
const semanticRefresh = await runProviderFirstDeferredSemanticRefresh({
|
|
2297
|
+
versionId,
|
|
2298
|
+
semanticDeferred: post.semanticDeferred,
|
|
2299
|
+
});
|
|
2300
|
+
providerFirstExecutedSummary = withProviderFirstPhaseTimings(providerFirstExecutedSummary);
|
|
2220
2301
|
const result = {
|
|
2221
2302
|
versionId,
|
|
2222
|
-
filesProcessed:
|
|
2223
|
-
changedFiles:
|
|
2224
|
-
removedFiles:
|
|
2225
|
-
symbolsIndexed:
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2303
|
+
filesProcessed: materializedRows.files.length,
|
|
2304
|
+
changedFiles: materializedRows.changedFileIds.size,
|
|
2305
|
+
removedFiles: providerScan.removedFiles,
|
|
2306
|
+
symbolsIndexed: materializedRows.symbols.length +
|
|
2307
|
+
materializedRows.externalSymbols.length,
|
|
2308
|
+
edgesCreated: materializedRows.edges.length,
|
|
2309
|
+
clustersComputed: post.clustersComputed,
|
|
2310
|
+
processesTraced: post.processesTraced,
|
|
2229
2311
|
durationMs: Date.now() - startTime,
|
|
2230
|
-
|
|
2231
|
-
|
|
2312
|
+
summaryStats: semanticRefresh.summaryStats ?? post.summaryStats,
|
|
2313
|
+
timings: phaseTimings
|
|
2314
|
+
? {
|
|
2315
|
+
totalMs: Date.now() - startTime,
|
|
2316
|
+
phases: phaseTimings,
|
|
2317
|
+
}
|
|
2318
|
+
: undefined,
|
|
2319
|
+
pass1Engine,
|
|
2320
|
+
scip,
|
|
2232
2321
|
providerFirst,
|
|
2233
2322
|
providerFirstExecution: providerFirstExecutedSummary,
|
|
2234
|
-
semanticDeferred: semanticDeferred
|
|
2235
|
-
algorithmRefresh:
|
|
2323
|
+
semanticDeferred: semanticRefresh.semanticDeferred,
|
|
2324
|
+
algorithmRefresh: post.algorithmRefresh,
|
|
2236
2325
|
};
|
|
2237
|
-
clearProviderGraphRowsForGc(materializedRows);
|
|
2238
2326
|
return result;
|
|
2239
2327
|
}
|
|
2240
|
-
const versionId = await createOrReuseVersion(providerFirstLspExecutor(providerResult.summary.executor)
|
|
2241
|
-
? providerFirstIncrementalActive
|
|
2242
|
-
? "Provider-first LSP incremental index"
|
|
2243
|
-
: "Provider-first LSP index"
|
|
2244
|
-
: providerFirstIncrementalActive
|
|
2245
|
-
? "Provider-first SCIP incremental index"
|
|
2246
|
-
: "Provider-first SCIP index", true);
|
|
2247
|
-
const pass1Engine = emptyPass1EngineTelemetry();
|
|
2248
|
-
const scip = "generatedIndexes" in providerResult
|
|
2249
|
-
? {
|
|
2250
|
-
generatedIndexes: providerResult.generatedIndexes,
|
|
2251
|
-
failures: providerResult.failures,
|
|
2252
|
-
generatorCache: activeScipPreRefresh?.cache,
|
|
2253
|
-
}
|
|
2254
|
-
: undefined;
|
|
2255
|
-
const post = await runPostIndexFinalization({
|
|
2256
|
-
versionId,
|
|
2257
|
-
indexMode: providerFirstIncrementalActive ? "incremental" : "full",
|
|
2258
|
-
filesTotal: providerFirstIncrementalActive
|
|
2259
|
-
? providerCoverageScan.files.length
|
|
2260
|
-
: materializedRows.files.length,
|
|
2261
|
-
filesScanned: materializedRows.files.length,
|
|
2262
|
-
symbolsExtracted: materializedRows.symbols.length +
|
|
2263
|
-
materializedRows.externalSymbols.length,
|
|
2264
|
-
edgesExtracted: materializedRows.edges.length,
|
|
2265
|
-
changedFileIdsForFinalize: scopedSourceFileListActive || providerFirstIncrementalActive
|
|
2266
|
-
? materializedRows.changedFileIds
|
|
2267
|
-
: undefined,
|
|
2268
|
-
changedTestFilePathsForFinalize: new Set(),
|
|
2269
|
-
changedFileIdsForMemory: materializedRows.changedFileIds,
|
|
2270
|
-
hasIndexMutations: true,
|
|
2271
|
-
callResolutionTelemetry: createCallResolutionTelemetry({
|
|
2272
|
-
repoId,
|
|
2273
|
-
mode: providerFirstIncrementalActive ? "incremental" : "full",
|
|
2274
|
-
pass2EligibleFileCount: 0,
|
|
2275
|
-
registeredResolvers: [],
|
|
2276
|
-
}),
|
|
2277
|
-
pass1Engine,
|
|
2278
|
-
scip,
|
|
2279
|
-
preloadedFileSummarySymbolFactsByFile: buildPreloadedFileSummarySymbolFactsFromRows({
|
|
2280
|
-
files: materializedRows.files,
|
|
2281
|
-
symbols: materializedRows.symbols,
|
|
2282
|
-
}),
|
|
2283
|
-
deferSemanticRefresh: true,
|
|
2284
|
-
skipDerivedStateReason: providerFirstSkipDerivedStateReason,
|
|
2285
|
-
});
|
|
2286
|
-
providerFirstExecutedSummary = {
|
|
2287
|
-
...providerFirstExecutedSummary,
|
|
2288
|
-
shadowBuild: await finalizeProviderFirstShadowBuild(providerFirstExecutedSummary.shadowBuild, versionId, post.semanticDeferred),
|
|
2289
|
-
};
|
|
2290
|
-
const semanticRefresh = await runProviderFirstDeferredSemanticRefresh({
|
|
2291
|
-
versionId,
|
|
2292
|
-
semanticDeferred: post.semanticDeferred,
|
|
2293
|
-
});
|
|
2294
|
-
providerFirstExecutedSummary = withProviderFirstPhaseTimings(providerFirstExecutedSummary);
|
|
2295
|
-
const result = {
|
|
2296
|
-
versionId,
|
|
2297
|
-
filesProcessed: materializedRows.files.length,
|
|
2298
|
-
changedFiles: materializedRows.changedFileIds.size,
|
|
2299
|
-
removedFiles: providerScan.removedFiles,
|
|
2300
|
-
symbolsIndexed: materializedRows.symbols.length +
|
|
2301
|
-
materializedRows.externalSymbols.length,
|
|
2302
|
-
edgesCreated: materializedRows.edges.length,
|
|
2303
|
-
clustersComputed: post.clustersComputed,
|
|
2304
|
-
processesTraced: post.processesTraced,
|
|
2305
|
-
durationMs: Date.now() - startTime,
|
|
2306
|
-
summaryStats: semanticRefresh.summaryStats ?? post.summaryStats,
|
|
2307
|
-
timings: phaseTimings
|
|
2308
|
-
? {
|
|
2309
|
-
totalMs: Date.now() - startTime,
|
|
2310
|
-
phases: phaseTimings,
|
|
2311
|
-
}
|
|
2312
|
-
: undefined,
|
|
2313
|
-
pass1Engine,
|
|
2314
|
-
scip,
|
|
2315
|
-
providerFirst,
|
|
2316
|
-
providerFirstExecution: providerFirstExecutedSummary,
|
|
2317
|
-
semanticDeferred: semanticRefresh.semanticDeferred,
|
|
2318
|
-
algorithmRefresh: post.algorithmRefresh,
|
|
2319
|
-
};
|
|
2320
|
-
return result;
|
|
2321
2328
|
}
|
|
2322
2329
|
}
|
|
2323
2330
|
}
|
|
2324
2331
|
}
|
|
2332
|
+
catch (err) {
|
|
2333
|
+
if (err instanceof ProviderFirstGraphValidationError)
|
|
2334
|
+
throw err;
|
|
2335
|
+
if (providerFirst.requestedMode !== "auto")
|
|
2336
|
+
throw err;
|
|
2337
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
2338
|
+
const fallbackReason = formatProviderFirstAutoFallbackReason({
|
|
2339
|
+
reason,
|
|
2340
|
+
providerFirstIncrementalActive,
|
|
2341
|
+
});
|
|
2342
|
+
logger.warn("indexRepo: provider-first phase failed (coverage scan or collection), using legacy fallback", { repoId, reason: fallbackReason });
|
|
2343
|
+
providerFirstExecutionFallback = providerFirstFallbackSummary([
|
|
2344
|
+
fallbackReason,
|
|
2345
|
+
]);
|
|
2346
|
+
// providerFirstScan stays undefined; legacy scanRepoForIndex will run below.
|
|
2347
|
+
}
|
|
2325
2348
|
}
|
|
2326
2349
|
const { files, existingByPath, removedFiles, removedFileIds, allFilesUnchanged: scanAllFilesUnchanged, } = providerFirstScan ??
|
|
2327
2350
|
(await measurePhase("scanRepo", () => scanRepoForIndex({
|
|
@@ -2904,6 +2927,11 @@ async function indexRepoImpl(repoId, mode, onProgress, signal, options, scipPreR
|
|
|
2904
2927
|
onProgress,
|
|
2905
2928
|
}));
|
|
2906
2929
|
recordFinalizeIndexingSubphaseTimings(finalizeResult.timings);
|
|
2930
|
+
await measurePhase("ensureCriticalSymbolFts", async () => {
|
|
2931
|
+
await ensureCriticalSymbolFtsIndex({
|
|
2932
|
+
recordTiming: recordIndexSubphaseTiming,
|
|
2933
|
+
});
|
|
2934
|
+
});
|
|
2907
2935
|
// Refresh read connection again after version/metrics writes.
|
|
2908
2936
|
freshConn = await getLadybugConn();
|
|
2909
2937
|
const derivedResult = providerFirstSkipDerivedStateReason
|