sdl-mcp 0.11.12 → 0.11.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,7 +11,7 @@ import { finalizeDerivedState } from "./finalize-derived-state.js";
11
11
  import { watchRepositoryWithIndexer } from "./watcher.js";
12
12
  import { IndexingConfigSchema, } from "../config/types.js";
13
13
  import { loadConfig } from "../config/loadConfig.js";
14
- import { buildDeferredIndexes, closeLadybugDb, flushStaleFinalizers, getLadybugConn, getLadybugDbPath, initLadybugDb, preIndexCheckpoint, withWriteConn, } from "../db/ladybug.js";
14
+ import { buildDeferredIndexes, closeLadybugDb, ensureCriticalSymbolFtsIndex, flushStaleFinalizers, getLadybugConn, getLadybugDbPath, initLadybugDb, preIndexCheckpoint, withWriteConn, } from "../db/ladybug.js";
15
15
  import { withPostIndexWriteSession } from "../db/write-session.js";
16
16
  import * as ladybugDb from "../db/ladybug-queries.js";
17
17
  import { derivedStateIsStale, getDerivedState, markDerivedStateDirty, recordDerivedStateError, } from "../db/ladybug-derived-state.js";
@@ -1461,6 +1461,11 @@ async function indexRepoImpl(repoId, mode, onProgress, signal, options, scipPreR
1461
1461
  onProgress,
1462
1462
  }));
1463
1463
  recordFinalizeIndexingSubphaseTimings(finalizeResult.timings);
1464
+ await measurePhase("ensureCriticalSymbolFts", async () => {
1465
+ await ensureCriticalSymbolFtsIndex({
1466
+ recordTiming: recordIndexSubphaseTiming,
1467
+ });
1468
+ });
1464
1469
  const freshConn = await getLadybugConn();
1465
1470
  const derivedResult = params.skipDerivedStateReason
1466
1471
  ? await measurePhase("skipDerivedState", async () => {
@@ -1655,673 +1660,691 @@ async function indexRepoImpl(repoId, mode, onProgress, signal, options, scipPreR
1655
1660
  providerFirstExecutor &&
1656
1661
  providerFirstIncrementalBootstrapped) {
1657
1662
  providerFirstTimingStartedAt = Date.now();
1658
- emitProviderFirstProgress(onProgress, "coverageScan", {
1659
- message: "scanning repository for provider-first coverage",
1660
- });
1661
- const providerCoverageScan = await measureProviderFirstPhase("coverageScan", "providerFirstCoverageScan", () => scanRepoForIndex({
1662
- repoId,
1663
- repoRoot: repoRow.rootPath,
1664
- config,
1665
- onProgress,
1666
- deleteRemovedFiles: false,
1667
- }));
1668
- emitProviderFirstProgress(onProgress, "coverageScan", {
1669
- stageCurrent: providerCoverageScan.files.length,
1670
- stageTotal: providerCoverageScan.files.length,
1671
- message: `scanned ${providerCoverageScan.files.length} file(s) for provider-first coverage`,
1672
- });
1673
- let providerFirstRemovedFilesDeleted = false;
1674
- if (providerFirstIncrementalActive &&
1675
- providerCoverageScan.removedFileIds.length > 0) {
1676
- await measureProviderFirstPhase("deleteRemovedFiles", "providerFirstIncrementalDeleteRemovedFiles", () => withWriteConn(async (conn) => {
1677
- await ladybugDb.withTransaction(conn, async (txConn) => {
1678
- await ladybugDb.deleteFilesByIds(txConn, providerCoverageScan.removedFileIds);
1679
- });
1680
- }, postIndexSessionTimeoutMs));
1681
- providerFirstRemovedFilesDeleted = true;
1682
- }
1683
- const providerChangedFiles = providerFirstIncrementalActive
1684
- ? providerFirstIncrementalChangedFiles(providerCoverageScan)
1685
- : providerCoverageScan.files;
1686
- const providerExecutionScan = providerFirstIncrementalActive
1687
- ? filterProviderFirstIncrementalScan(providerCoverageScan, providerChangedFiles)
1688
- : providerCoverageScan;
1689
- if (providerFirstIncrementalActive && providerChangedFiles.length === 0) {
1690
- providerFirstScan = providerCoverageScan;
1663
+ try {
1691
1664
  emitProviderFirstProgress(onProgress, "coverageScan", {
1692
- stageCurrent: 0,
1693
- stageTotal: 0,
1694
- message: "no changed files for provider-first incremental execution",
1665
+ message: "scanning repository for provider-first coverage",
1695
1666
  });
1696
- }
1697
- let providerResult;
1698
- if (!providerFirstIncrementalActive || providerChangedFiles.length > 0) {
1699
- try {
1700
- let providerScipPreRefresh = activeScipPreRefresh;
1701
- let providerIncrementalTempPaths = [];
1667
+ const providerCoverageScan = await measureProviderFirstPhase("coverageScan", "providerFirstCoverageScan", () => scanRepoForIndex({
1668
+ repoId,
1669
+ repoRoot: repoRow.rootPath,
1670
+ config,
1671
+ onProgress,
1672
+ deleteRemovedFiles: false,
1673
+ }));
1674
+ emitProviderFirstProgress(onProgress, "coverageScan", {
1675
+ stageCurrent: providerCoverageScan.files.length,
1676
+ stageTotal: providerCoverageScan.files.length,
1677
+ message: `scanned ${providerCoverageScan.files.length} file(s) for provider-first coverage`,
1678
+ });
1679
+ let providerFirstRemovedFilesDeleted = false;
1680
+ if (providerFirstIncrementalActive &&
1681
+ providerCoverageScan.removedFileIds.length > 0) {
1682
+ await measureProviderFirstPhase("deleteRemovedFiles", "providerFirstIncrementalDeleteRemovedFiles", () => withWriteConn(async (conn) => {
1683
+ await ladybugDb.withTransaction(conn, async (txConn) => {
1684
+ await ladybugDb.deleteFilesByIds(txConn, providerCoverageScan.removedFileIds);
1685
+ });
1686
+ }, postIndexSessionTimeoutMs));
1687
+ providerFirstRemovedFilesDeleted = true;
1688
+ }
1689
+ const providerChangedFiles = providerFirstIncrementalActive
1690
+ ? providerFirstIncrementalChangedFiles(providerCoverageScan)
1691
+ : providerCoverageScan.files;
1692
+ const providerExecutionScan = providerFirstIncrementalActive
1693
+ ? filterProviderFirstIncrementalScan(providerCoverageScan, providerChangedFiles)
1694
+ : providerCoverageScan;
1695
+ if (providerFirstIncrementalActive && providerChangedFiles.length === 0) {
1696
+ providerFirstScan = providerCoverageScan;
1697
+ emitProviderFirstProgress(onProgress, "coverageScan", {
1698
+ stageCurrent: 0,
1699
+ stageTotal: 0,
1700
+ message: "no changed files for provider-first incremental execution",
1701
+ });
1702
+ }
1703
+ let providerResult;
1704
+ if (!providerFirstIncrementalActive || providerChangedFiles.length > 0) {
1702
1705
  try {
1703
- if (providerFirstExecutor === "scipIncremental") {
1704
- const scopedScip = await measureProviderFirstPhase("scipIncrementalGenerate", "providerFirstScipIncrementalGenerate", () => runProviderFirstIncrementalScipIo({
1705
- repoId,
1706
- repoRoot: repoRow.rootPath,
1707
- repoConfig: config,
1708
- appConfig,
1709
- changedFiles: providerChangedFiles,
1710
- signal,
1711
- }));
1712
- providerScipPreRefresh = scopedScip.diagnostics;
1713
- activeScipPreRefresh = scopedScip.diagnostics;
1714
- providerIncrementalTempPaths = scopedScip.tempPaths;
1715
- }
1716
- providerResult = await measureProviderFirstPhase("providerCollection", providerFirstLspExecutor(providerFirstExecutor)
1717
- ? providerFirstIncrementalActive
1718
- ? "providerFirstLspIncremental"
1719
- : "providerFirstLspFull"
1720
- : providerFirstIncrementalActive
1721
- ? "providerFirstScipIncremental"
1722
- : "providerFirstScipFull", () => {
1723
- if (providerFirstExecutor === "lspFull") {
1724
- return executeProviderFirstLspFull({
1706
+ let providerScipPreRefresh = activeScipPreRefresh;
1707
+ let providerIncrementalTempPaths = [];
1708
+ try {
1709
+ if (providerFirstExecutor === "scipIncremental") {
1710
+ const scopedScip = await measureProviderFirstPhase("scipIncrementalGenerate", "providerFirstScipIncrementalGenerate", () => runProviderFirstIncrementalScipIo({
1725
1711
  repoId,
1726
1712
  repoRoot: repoRow.rootPath,
1727
- config: appConfig,
1728
- scannedFiles: providerExecutionScan.files,
1729
- recordPhaseTiming: recordProviderFirstPhaseTiming,
1713
+ repoConfig: config,
1714
+ appConfig,
1715
+ changedFiles: providerChangedFiles,
1730
1716
  signal,
1731
- });
1717
+ }));
1718
+ providerScipPreRefresh = scopedScip.diagnostics;
1719
+ activeScipPreRefresh = scopedScip.diagnostics;
1720
+ providerIncrementalTempPaths = scopedScip.tempPaths;
1732
1721
  }
1733
- if (providerFirstExecutor === "lspIncremental") {
1734
- return executeProviderFirstLspIncremental({
1722
+ providerResult = await measureProviderFirstPhase("providerCollection", providerFirstLspExecutor(providerFirstExecutor)
1723
+ ? providerFirstIncrementalActive
1724
+ ? "providerFirstLspIncremental"
1725
+ : "providerFirstLspFull"
1726
+ : providerFirstIncrementalActive
1727
+ ? "providerFirstScipIncremental"
1728
+ : "providerFirstScipFull", () => {
1729
+ if (providerFirstExecutor === "lspFull") {
1730
+ return executeProviderFirstLspFull({
1731
+ repoId,
1732
+ repoRoot: repoRow.rootPath,
1733
+ config: appConfig,
1734
+ scannedFiles: providerExecutionScan.files,
1735
+ recordPhaseTiming: recordProviderFirstPhaseTiming,
1736
+ signal,
1737
+ });
1738
+ }
1739
+ if (providerFirstExecutor === "lspIncremental") {
1740
+ return executeProviderFirstLspIncremental({
1741
+ repoId,
1742
+ repoRoot: repoRow.rootPath,
1743
+ config: appConfig,
1744
+ scannedFiles: providerExecutionScan.files,
1745
+ recordPhaseTiming: recordProviderFirstPhaseTiming,
1746
+ signal,
1747
+ });
1748
+ }
1749
+ const scipParams = {
1735
1750
  repoId,
1736
1751
  repoRoot: repoRow.rootPath,
1737
1752
  config: appConfig,
1753
+ generatedIndexes: providerScipPreRefresh?.generatedIndexes,
1754
+ generatorFailures: providerScipPreRefresh?.failures,
1755
+ generatorCacheKey: providerScipPreRefresh?.cache?.key,
1756
+ scannedPaths: providerExecutionScan.files.map((file) => file.path),
1738
1757
  scannedFiles: providerExecutionScan.files,
1739
1758
  recordPhaseTiming: recordProviderFirstPhaseTiming,
1759
+ onProgress,
1740
1760
  signal,
1741
- });
1742
- }
1743
- const scipParams = {
1744
- repoId,
1745
- repoRoot: repoRow.rootPath,
1746
- config: appConfig,
1747
- generatedIndexes: providerScipPreRefresh?.generatedIndexes,
1748
- generatorFailures: providerScipPreRefresh?.failures,
1749
- generatorCacheKey: providerScipPreRefresh?.cache?.key,
1750
- scannedPaths: providerExecutionScan.files.map((file) => file.path),
1751
- scannedFiles: providerExecutionScan.files,
1752
- recordPhaseTiming: recordProviderFirstPhaseTiming,
1753
- onProgress,
1754
- signal,
1755
- };
1756
- return providerFirstExecutor === "scipIncremental"
1757
- ? executeProviderFirstScipIncremental(scipParams)
1758
- : executeProviderFirstScipFull(scipParams);
1759
- });
1760
- }
1761
- finally {
1762
- await cleanupProviderFirstIncrementalTempPaths(providerIncrementalTempPaths);
1763
- }
1764
- }
1765
- catch (err) {
1766
- if (err instanceof ProviderFirstGraphValidationError) {
1767
- throw err;
1761
+ };
1762
+ return providerFirstExecutor === "scipIncremental"
1763
+ ? executeProviderFirstScipIncremental(scipParams)
1764
+ : executeProviderFirstScipFull(scipParams);
1765
+ });
1766
+ }
1767
+ finally {
1768
+ await cleanupProviderFirstIncrementalTempPaths(providerIncrementalTempPaths);
1769
+ }
1768
1770
  }
1769
- const reason = err instanceof Error
1770
- ? err.message
1771
- : `provider-first SCIP execution failed: ${String(err)}`;
1772
- if (providerFirst.requestedMode === "auto") {
1773
- const fallbackReason = formatProviderFirstAutoFallbackReason({
1774
- reason,
1775
- providerFirstIncrementalActive,
1776
- });
1777
- logger.warn("indexRepo: provider-first execution failed, using legacy fallback", {
1778
- repoId,
1779
- reason: fallbackReason,
1780
- });
1781
- providerFirstExecutionFallback = providerFirstFallbackSummary([
1782
- fallbackReason,
1783
- ]);
1784
- if (providerFirstIncrementalActive) {
1785
- // Keep provider-first incremental failures scoped; otherwise the
1786
- // legacy safety fallback degenerates into a full repo pass.
1787
- providerFirstScan = providerExecutionScan;
1788
- for (const file of providerExecutionScan.files) {
1789
- providerFirstFallbackPaths.add(file.path);
1771
+ catch (err) {
1772
+ if (err instanceof ProviderFirstGraphValidationError) {
1773
+ throw err;
1774
+ }
1775
+ const reason = err instanceof Error
1776
+ ? err.message
1777
+ : `provider-first SCIP execution failed: ${String(err)}`;
1778
+ if (providerFirst.requestedMode === "auto") {
1779
+ const fallbackReason = formatProviderFirstAutoFallbackReason({
1780
+ reason,
1781
+ providerFirstIncrementalActive,
1782
+ });
1783
+ logger.warn("indexRepo: provider-first execution failed, using legacy fallback", {
1784
+ repoId,
1785
+ reason: fallbackReason,
1786
+ });
1787
+ providerFirstExecutionFallback = providerFirstFallbackSummary([
1788
+ fallbackReason,
1789
+ ]);
1790
+ if (providerFirstIncrementalActive) {
1791
+ // Keep provider-first incremental failures scoped; otherwise the
1792
+ // legacy safety fallback degenerates into a full repo pass.
1793
+ providerFirstScan = providerExecutionScan;
1794
+ for (const file of providerExecutionScan.files) {
1795
+ providerFirstFallbackPaths.add(file.path);
1796
+ }
1797
+ providerFirstLegacyFallbackComplete = true;
1798
+ providerFirstLegacyFallbackFileCount = providerExecutionScan.files.length;
1799
+ providerFirstLegacyFallbackSamplePaths = providerExecutionScan.files
1800
+ .map((file) => file.path)
1801
+ .slice(0, 10);
1802
+ providerFirstLegacyFallbackStartedAt = Date.now();
1790
1803
  }
1791
- providerFirstLegacyFallbackComplete = true;
1792
- providerFirstLegacyFallbackFileCount = providerExecutionScan.files.length;
1793
- providerFirstLegacyFallbackSamplePaths = providerExecutionScan.files
1794
- .map((file) => file.path)
1795
- .slice(0, 10);
1796
- providerFirstLegacyFallbackStartedAt = Date.now();
1797
1804
  }
1798
- }
1799
- else {
1800
- throw err;
1805
+ else {
1806
+ throw err;
1807
+ }
1801
1808
  }
1802
1809
  }
1803
- }
1804
- if (providerResult) {
1805
- const executionFailureReasons = providerFirstFatalFailureReasons({
1806
- failures: "failures" in providerResult ? providerResult.failures : [],
1807
- providerRowsAvailable: providerResult.rows.files.length > 0,
1808
- });
1809
- if (executionFailureReasons.length > 0) {
1810
- if (providerFirst.requestedMode === "auto") {
1811
- logger.warn("indexRepo: provider-first execution had failures, using legacy fallback", {
1812
- repoId,
1813
- reasons: executionFailureReasons,
1814
- });
1815
- providerFirstExecutionFallback = providerFirstFallbackSummary(executionFailureReasons);
1810
+ if (providerResult) {
1811
+ const executionFailureReasons = providerFirstFatalFailureReasons({
1812
+ failures: "failures" in providerResult ? providerResult.failures : [],
1813
+ providerRowsAvailable: providerResult.rows.files.length > 0,
1814
+ });
1815
+ if (executionFailureReasons.length > 0) {
1816
+ if (providerFirst.requestedMode === "auto") {
1817
+ logger.warn("indexRepo: provider-first execution had failures, using legacy fallback", {
1818
+ repoId,
1819
+ reasons: executionFailureReasons,
1820
+ });
1821
+ providerFirstExecutionFallback = providerFirstFallbackSummary(executionFailureReasons);
1822
+ }
1823
+ else {
1824
+ throw new Error(`Provider-first indexing cannot execute for ${repoId}: ${executionFailureReasons.join("; ")}`);
1825
+ }
1816
1826
  }
1817
1827
  else {
1818
- throw new Error(`Provider-first indexing cannot execute for ${repoId}: ${executionFailureReasons.join("; ")}`);
1819
- }
1820
- }
1821
- else {
1822
- const providerScan = providerExecutionScan;
1823
- providerFirstScan = providerScan;
1824
- if (providerFactsShouldDropCoveragePayloads(providerResult.facts)) {
1825
- clearProviderFactPayloadsForCoverageAnalysis(providerResult.facts);
1828
+ const providerScan = providerExecutionScan;
1829
+ providerFirstScan = providerScan;
1830
+ if (providerFactsShouldDropCoveragePayloads(providerResult.facts)) {
1831
+ clearProviderFactPayloadsForCoverageAnalysis(providerResult.facts);
1832
+ emitProviderFirstProgress(onProgress, "coverageAnalyze", {
1833
+ message: "large provider occurrence payloads released",
1834
+ });
1835
+ }
1836
+ const scanScopedProvider = filterProviderFirstDataToScannedScope({
1837
+ rows: providerResult.rows,
1838
+ facts: providerResult.facts,
1839
+ scannedPaths: providerScan.files.map((file) => file.path),
1840
+ });
1826
1841
  emitProviderFirstProgress(onProgress, "coverageAnalyze", {
1827
- message: "large provider occurrence payloads released",
1842
+ message: "provider rows filtered to scan scope",
1828
1843
  });
1829
- }
1830
- const scanScopedProvider = filterProviderFirstDataToScannedScope({
1831
- rows: providerResult.rows,
1832
- facts: providerResult.facts,
1833
- scannedPaths: providerScan.files.map((file) => file.path),
1834
- });
1835
- emitProviderFirstProgress(onProgress, "coverageAnalyze", {
1836
- message: "provider rows filtered to scan scope",
1837
- });
1838
- const scanScopedProviderPaths = scanScopedProvider.rows.files.map((file) => file.relPath);
1839
- const scanHasCppSemanticPaths = providerScan.files.some((file) => isCppSemanticScanPath(file.path));
1840
- const semanticEligiblePaths = scanHasCppSemanticPaths
1841
- ? await resolveProviderFirstSemanticEligiblePaths({
1842
- repoRoot: repoRow.rootPath,
1844
+ const scanScopedProviderPaths = scanScopedProvider.rows.files.map((file) => file.relPath);
1845
+ const scanHasCppSemanticPaths = providerScan.files.some((file) => isCppSemanticScanPath(file.path));
1846
+ const semanticEligiblePaths = scanHasCppSemanticPaths
1847
+ ? await resolveProviderFirstSemanticEligiblePaths({
1848
+ repoRoot: repoRow.rootPath,
1849
+ scannedPaths: providerScan.files.map((file) => file.path),
1850
+ providerPaths: scanScopedProviderPaths,
1851
+ })
1852
+ : undefined;
1853
+ const coverageReport = analyzeProviderFirstCoverage({
1843
1854
  scannedPaths: providerScan.files.map((file) => file.path),
1855
+ semanticEligiblePaths: semanticEligiblePaths && semanticEligiblePaths.size > 0
1856
+ ? semanticEligiblePaths
1857
+ : undefined,
1844
1858
  providerPaths: scanScopedProviderPaths,
1845
- })
1846
- : undefined;
1847
- const coverageReport = analyzeProviderFirstCoverage({
1848
- scannedPaths: providerScan.files.map((file) => file.path),
1849
- semanticEligiblePaths: semanticEligiblePaths && semanticEligiblePaths.size > 0
1850
- ? semanticEligiblePaths
1851
- : undefined,
1852
- providerPaths: scanScopedProviderPaths,
1853
- coverage: scanScopedProvider.facts.coverage,
1854
- symbols: scanScopedProvider.facts.symbols,
1855
- occurrences: scanScopedProvider.facts.occurrences,
1856
- sourceLinesByPath: scanScopedProvider.facts.sourceLinesByPath,
1857
- });
1858
- emitProviderFirstProgress(onProgress, "coverageAnalyze", {
1859
- message: "provider coverage analyzed",
1860
- });
1861
- if (coverageReport.fatalReasons.length > 0) {
1862
- throw new Error(`Provider-first indexing cannot execute for ${repoId}: ${coverageReport.fatalReasons.join("; ")}`);
1863
- }
1864
- else {
1865
- const legacyFallbackPlan = resolveProviderFirstLegacyFallbackPlan({
1866
- fallbackFileCount: coverageReport.fallbackPaths.size,
1867
- semanticEligibleFallbackFileCount: coverageReport.summary.semanticEligibilityGap?.totalFiles,
1868
- maxLegacyFallbackFiles: providerFirstConfig.maxLegacyFallbackFiles,
1869
- maxSemanticEligibleFallbackFiles: providerFirstConfig.maxSemanticEligibleFallbackFiles,
1859
+ coverage: scanScopedProvider.facts.coverage,
1860
+ symbols: scanScopedProvider.facts.symbols,
1861
+ occurrences: scanScopedProvider.facts.occurrences,
1862
+ sourceLinesByPath: scanScopedProvider.facts.sourceLinesByPath,
1870
1863
  });
1871
- const skippedLegacyFallbackReason = skippedProviderFirstLegacyFallbackReason(legacyFallbackPlan);
1872
- const readinessGates = resolveProviderFirstReadinessGates({
1873
- callProofSkipReason: callProofSkipDerivedStateReason(coverageReport.callProofIncompletePaths),
1874
- skippedLegacyFallbackReason,
1864
+ emitProviderFirstProgress(onProgress, "coverageAnalyze", {
1865
+ message: "provider coverage analyzed",
1875
1866
  });
1876
- providerFirstSkipDerivedStateReason =
1877
- readinessGates.skipDerivedStateReason;
1878
- providerFirstShadowStagingSkipReason =
1879
- readinessGates.shadowStagingSkipReason;
1880
- if (scopedSourceFileListActive) {
1881
- providerFirstShadowStagingSkipReason =
1882
- providerFirstShadowStagingSkipReason
1883
- ? `${providerFirstShadowStagingSkipReason}; ${scopedSourceFileListReason}`
1884
- : scopedSourceFileListReason;
1885
- }
1886
- if (providerFirstShadowRepoScopeSkipReason) {
1887
- // ponytail: revisit multi-repo shadow DBs when shadow builds can copy
1888
- // a complete DB, including non-target repos, before activation.
1889
- providerFirstShadowStagingSkipReason =
1890
- providerFirstShadowStagingSkipReason
1891
- ? `${providerFirstShadowStagingSkipReason}; ${providerFirstShadowRepoScopeSkipReason}`
1892
- : providerFirstShadowRepoScopeSkipReason;
1867
+ if (coverageReport.fatalReasons.length > 0) {
1868
+ throw new Error(`Provider-first indexing cannot execute for ${repoId}: ${coverageReport.fatalReasons.join("; ")}`);
1893
1869
  }
1894
- if (providerFirstIncrementalActive) {
1895
- const incrementalShadowSkipReason = "shadow staging skipped because provider-first incremental materializes only changed files";
1870
+ else {
1871
+ const legacyFallbackPlan = resolveProviderFirstLegacyFallbackPlan({
1872
+ fallbackFileCount: coverageReport.fallbackPaths.size,
1873
+ semanticEligibleFallbackFileCount: coverageReport.summary.semanticEligibilityGap?.totalFiles,
1874
+ maxLegacyFallbackFiles: providerFirstConfig.maxLegacyFallbackFiles,
1875
+ maxSemanticEligibleFallbackFiles: providerFirstConfig.maxSemanticEligibleFallbackFiles,
1876
+ });
1877
+ const skippedLegacyFallbackReason = skippedProviderFirstLegacyFallbackReason(legacyFallbackPlan);
1878
+ const readinessGates = resolveProviderFirstReadinessGates({
1879
+ callProofSkipReason: callProofSkipDerivedStateReason(coverageReport.callProofIncompletePaths),
1880
+ skippedLegacyFallbackReason,
1881
+ });
1882
+ providerFirstSkipDerivedStateReason =
1883
+ readinessGates.skipDerivedStateReason;
1896
1884
  providerFirstShadowStagingSkipReason =
1897
- providerFirstShadowStagingSkipReason
1898
- ? `${providerFirstShadowStagingSkipReason}; ${incrementalShadowSkipReason}`
1899
- : incrementalShadowSkipReason;
1900
- }
1901
- const legacyFallbackPaths = selectProviderFirstLegacyFallbackPaths({
1902
- fallbackPaths: coverageReport.fallbackPaths,
1903
- semanticEligiblePaths,
1904
- parsedFiles: legacyFallbackPlan.parsedFiles,
1905
- });
1906
- const providerRowsExcludedByLegacyFallback = legacyFallbackPlan.runLegacyFallback
1907
- ? selectProviderRowsExcludedByLegacyFallback({
1908
- fallbackPaths: legacyFallbackPaths,
1909
- coverage: scanScopedProvider.facts.coverage,
1910
- })
1911
- : new Set();
1912
- const materializedRows = filterProviderRowsForFallback(scanScopedProvider.rows, providerRowsExcludedByLegacyFallback);
1913
- await measureProviderFirstPhase("persistProvenance", "providerFirstPersistProvenance", () => withWriteConn((conn) => persistProviderFirstProvenance(conn, scanScopedProvider.facts)));
1914
- // The provider fact set can include every SCIP occurrence and source
1915
- // line. Drop it before legacy fallback and versioning so large repos
1916
- // do not carry decoded provider payloads into post-index finalization.
1917
- clearProviderFactPayloadsForGc(providerResult.facts);
1918
- if (scanScopedProvider.facts !== providerResult.facts) {
1919
- clearProviderFactPayloadsForGc(scanScopedProvider.facts);
1920
- }
1921
- if (providerResult.rows !== materializedRows) {
1922
- clearProviderGraphRowsForGc(providerResult.rows);
1923
- }
1924
- if (scanScopedProvider.rows !== materializedRows &&
1925
- scanScopedProvider.rows !== providerResult.rows) {
1926
- clearProviderGraphRowsForGc(scanScopedProvider.rows);
1927
- }
1928
- await measureProviderFirstPhase("postProviderGc", "providerFirstPostProviderGc", () => flushProviderFirstPayloadFinalizers());
1929
- emitProviderFirstProgress(onProgress, "coverageAnalyze", {
1930
- message: "provider payload finalizers flushed",
1931
- });
1932
- providerFirstProviderRows = materializedRows;
1933
- providerFirstGenerationId = providerResult.generationId;
1934
- applyScannedFileMetadataToProviderRows({
1935
- rows: materializedRows,
1936
- scannedFiles: providerScan.files,
1937
- });
1938
- let shadowBuild;
1939
- if (!legacyFallbackPlan.runLegacyFallback) {
1940
- if (providerFirstShadowStagingSkipReason) {
1941
- const shadowStageTotal = providerFirstGraphRowTotal(materializedRows);
1942
- emitProviderFirstProgress(onProgress, "shadowStage", {
1943
- stageCurrent: 0,
1944
- stageTotal: shadowStageTotal,
1945
- message: `shadow staging skipped: ${providerFirstShadowStagingSkipReason}`,
1946
- });
1947
- shadowBuild = skippedProviderFirstShadowBuild({
1948
- generationId: providerResult.generationId,
1949
- activation: providerFirstConfig.activation,
1950
- requestedFormat: providerFirstConfig.stagingFormat,
1951
- rows: materializedRows,
1952
- reason: providerFirstShadowStagingSkipReason,
1953
- });
1885
+ readinessGates.shadowStagingSkipReason;
1886
+ if (scopedSourceFileListActive) {
1887
+ providerFirstShadowStagingSkipReason =
1888
+ providerFirstShadowStagingSkipReason
1889
+ ? `${providerFirstShadowStagingSkipReason}; ${scopedSourceFileListReason}`
1890
+ : scopedSourceFileListReason;
1954
1891
  }
1955
- else {
1956
- const shadowStageTotal = providerFirstGraphRowTotal(materializedRows);
1957
- emitProviderFirstProgress(onProgress, "shadowStage", {
1958
- stageCurrent: 0,
1959
- stageTotal: shadowStageTotal,
1960
- message: "staging provider rows for shadow bulk load",
1961
- });
1962
- shadowBuild = await measureProviderFirstPhase("shadowStage", "providerFirstShadowStage", () => stageProviderFirstShadowBuild({
1963
- repoId,
1964
- generationId: providerResult.generationId,
1965
- activation: providerFirstConfig.activation,
1966
- requestedFormat: providerFirstConfig.stagingFormat,
1967
- activeDbPath: getLadybugDbPath(),
1968
- repoRoot: repoRow.rootPath,
1969
- repoConfigJson: repoRow.configJson,
1970
- rows: materializedRows,
1971
- }));
1972
- shadowBuild.activationResult =
1973
- summarizeProviderFirstShadowActivationReadiness({
1974
- shadowBuild,
1975
- fallbackFiles: 0,
1976
- graphDerivedStateReady: providerFirstSkipDerivedStateReason === undefined,
1977
- shadowContainsFinalizedGraph: false,
1978
- finalizedGraphReasons: providerFirstSkipDerivedStateReason
1979
- ? [providerFirstSkipDerivedStateReason]
1980
- : undefined,
1892
+ if (providerFirstShadowRepoScopeSkipReason) {
1893
+ // ponytail: revisit multi-repo shadow DBs when shadow builds can copy
1894
+ // a complete DB, including non-target repos, before activation.
1895
+ providerFirstShadowStagingSkipReason =
1896
+ providerFirstShadowStagingSkipReason
1897
+ ? `${providerFirstShadowStagingSkipReason}; ${providerFirstShadowRepoScopeSkipReason}`
1898
+ : providerFirstShadowRepoScopeSkipReason;
1899
+ }
1900
+ if (providerFirstIncrementalActive) {
1901
+ const incrementalShadowSkipReason = "shadow staging skipped because provider-first incremental materializes only changed files";
1902
+ providerFirstShadowStagingSkipReason =
1903
+ providerFirstShadowStagingSkipReason
1904
+ ? `${providerFirstShadowStagingSkipReason}; ${incrementalShadowSkipReason}`
1905
+ : incrementalShadowSkipReason;
1906
+ }
1907
+ const legacyFallbackPaths = selectProviderFirstLegacyFallbackPaths({
1908
+ fallbackPaths: coverageReport.fallbackPaths,
1909
+ semanticEligiblePaths,
1910
+ parsedFiles: legacyFallbackPlan.parsedFiles,
1911
+ });
1912
+ const providerRowsExcludedByLegacyFallback = legacyFallbackPlan.runLegacyFallback
1913
+ ? selectProviderRowsExcludedByLegacyFallback({
1914
+ fallbackPaths: legacyFallbackPaths,
1915
+ coverage: scanScopedProvider.facts.coverage,
1916
+ })
1917
+ : new Set();
1918
+ const materializedRows = filterProviderRowsForFallback(scanScopedProvider.rows, providerRowsExcludedByLegacyFallback);
1919
+ await measureProviderFirstPhase("persistProvenance", "providerFirstPersistProvenance", () => withWriteConn((conn) => persistProviderFirstProvenance(conn, scanScopedProvider.facts)));
1920
+ // The provider fact set can include every SCIP occurrence and source
1921
+ // line. Drop it before legacy fallback and versioning so large repos
1922
+ // do not carry decoded provider payloads into post-index finalization.
1923
+ clearProviderFactPayloadsForGc(providerResult.facts);
1924
+ if (scanScopedProvider.facts !== providerResult.facts) {
1925
+ clearProviderFactPayloadsForGc(scanScopedProvider.facts);
1926
+ }
1927
+ if (providerResult.rows !== materializedRows) {
1928
+ clearProviderGraphRowsForGc(providerResult.rows);
1929
+ }
1930
+ if (scanScopedProvider.rows !== materializedRows &&
1931
+ scanScopedProvider.rows !== providerResult.rows) {
1932
+ clearProviderGraphRowsForGc(scanScopedProvider.rows);
1933
+ }
1934
+ await measureProviderFirstPhase("postProviderGc", "providerFirstPostProviderGc", () => flushProviderFirstPayloadFinalizers());
1935
+ emitProviderFirstProgress(onProgress, "coverageAnalyze", {
1936
+ message: "provider payload finalizers flushed",
1937
+ });
1938
+ providerFirstProviderRows = materializedRows;
1939
+ providerFirstGenerationId = providerResult.generationId;
1940
+ applyScannedFileMetadataToProviderRows({
1941
+ rows: materializedRows,
1942
+ scannedFiles: providerScan.files,
1943
+ });
1944
+ let shadowBuild;
1945
+ if (!legacyFallbackPlan.runLegacyFallback) {
1946
+ if (providerFirstShadowStagingSkipReason) {
1947
+ const shadowStageTotal = providerFirstGraphRowTotal(materializedRows);
1948
+ emitProviderFirstProgress(onProgress, "shadowStage", {
1949
+ stageCurrent: 0,
1950
+ stageTotal: shadowStageTotal,
1951
+ message: `shadow staging skipped: ${providerFirstShadowStagingSkipReason}`,
1981
1952
  });
1982
- if (shadowBuild.status === "staged") {
1983
- const stagedTotal = providerFirstShadowStageTotal(shadowBuild.counts);
1953
+ shadowBuild = skippedProviderFirstShadowBuild({
1954
+ generationId: providerResult.generationId,
1955
+ activation: providerFirstConfig.activation,
1956
+ requestedFormat: providerFirstConfig.stagingFormat,
1957
+ rows: materializedRows,
1958
+ reason: providerFirstShadowStagingSkipReason,
1959
+ });
1960
+ }
1961
+ else {
1962
+ const shadowStageTotal = providerFirstGraphRowTotal(materializedRows);
1984
1963
  emitProviderFirstProgress(onProgress, "shadowStage", {
1985
- stageCurrent: stagedTotal,
1986
- stageTotal: stagedTotal,
1987
- message: "provider rows staged for shadow bulk load",
1964
+ stageCurrent: 0,
1965
+ stageTotal: shadowStageTotal,
1966
+ message: "staging provider rows for shadow bulk load",
1988
1967
  });
1968
+ shadowBuild = await measureProviderFirstPhase("shadowStage", "providerFirstShadowStage", () => stageProviderFirstShadowBuild({
1969
+ repoId,
1970
+ generationId: providerResult.generationId,
1971
+ activation: providerFirstConfig.activation,
1972
+ requestedFormat: providerFirstConfig.stagingFormat,
1973
+ activeDbPath: getLadybugDbPath(),
1974
+ repoRoot: repoRow.rootPath,
1975
+ repoConfigJson: repoRow.configJson,
1976
+ rows: materializedRows,
1977
+ }));
1978
+ shadowBuild.activationResult =
1979
+ summarizeProviderFirstShadowActivationReadiness({
1980
+ shadowBuild,
1981
+ fallbackFiles: 0,
1982
+ graphDerivedStateReady: providerFirstSkipDerivedStateReason === undefined,
1983
+ shadowContainsFinalizedGraph: false,
1984
+ finalizedGraphReasons: providerFirstSkipDerivedStateReason
1985
+ ? [providerFirstSkipDerivedStateReason]
1986
+ : undefined,
1987
+ });
1988
+ if (shadowBuild.status === "staged") {
1989
+ const stagedTotal = providerFirstShadowStageTotal(shadowBuild.counts);
1990
+ emitProviderFirstProgress(onProgress, "shadowStage", {
1991
+ stageCurrent: stagedTotal,
1992
+ stageTotal: stagedTotal,
1993
+ message: "provider rows staged for shadow bulk load",
1994
+ });
1995
+ }
1989
1996
  }
1990
1997
  }
1991
- }
1992
- const activeProviderInputHash = scopedSourceFileListActive || providerFirstIncrementalActive
1993
- ? null
1994
- : "generatedIndexes" in providerResult
1995
- ? providerFirstActiveInputFingerprint(providerResult.generatedIndexes)
1998
+ const activeProviderInputHash = scopedSourceFileListActive || providerFirstIncrementalActive
1999
+ ? null
2000
+ : "generatedIndexes" in providerResult
2001
+ ? providerFirstActiveInputFingerprint(providerResult.generatedIndexes)
2002
+ : null;
2003
+ const activeProviderInputRecord = activeProviderInputHash
2004
+ ? await ladybugDb.getScipIngestionRecord(conn, repoId, PROVIDER_FIRST_ACTIVE_INPUT_RECORD_PATH)
1996
2005
  : null;
1997
- const activeProviderInputRecord = activeProviderInputHash
1998
- ? await ladybugDb.getScipIngestionRecord(conn, repoId, PROVIDER_FIRST_ACTIVE_INPUT_RECORD_PATH)
1999
- : null;
2000
- const activeProviderInputMatches = Boolean(activeProviderInputHash &&
2001
- activeProviderInputRecord?.contentHash ===
2002
- activeProviderInputHash &&
2003
- activeProviderInputRecord.truncated !== true);
2004
- const existingProviderSymbolCount = await countExistingScipProviderSymbols(conn, repoId);
2005
- const activeMaterializationPlan = providerFirstIncrementalActive
2006
- ? {
2007
- deleteExistingFileSymbols: true,
2008
- useKnownFreshWriters: true,
2009
- writeEdges: true,
2010
- reuseExistingProviderRows: false,
2011
- }
2012
- : resolveProviderFirstActiveMaterializationPlan({
2013
- existingProviderFileCount: countExistingProviderPrimaryFiles({
2014
- providerFiles: materializedRows.files,
2015
- existingByPath: providerScan.existingByPath,
2016
- }),
2017
- providerSymbolCount: materializedRows.symbols.length,
2018
- activeProviderInputMatches,
2019
- existingProviderSymbolCount,
2020
- });
2021
- if (activeMaterializationPlan.reuseExistingProviderRows) {
2022
- const deleteTotal = providerFirstMaterializePhaseTotal("deleteFileSymbols", materializedRows, activeMaterializationPlan);
2023
- emitProviderFirstProgress(onProgress, "materialize.deleteFileSymbols", {
2024
- stageCurrent: 0,
2025
- stageTotal: deleteTotal,
2026
- message: "provider active stale cleanup skipped for large symbol set",
2027
- });
2028
- const symbolTotal = providerFirstMaterializePhaseTotal("upsertSymbols", materializedRows, activeMaterializationPlan);
2029
- emitProviderFirstProgress(onProgress, "materialize.upsertSymbols", {
2030
- stageCurrent: symbolTotal,
2031
- stageTotal: symbolTotal,
2032
- message: "provider active rows reused for existing large symbol set",
2033
- });
2034
- }
2035
- await measureProviderFirstPhase("materialize", "providerFirstMaterialize", () => {
2006
+ const activeProviderInputMatches = Boolean(activeProviderInputHash &&
2007
+ activeProviderInputRecord?.contentHash ===
2008
+ activeProviderInputHash &&
2009
+ activeProviderInputRecord.truncated !== true);
2010
+ const existingProviderSymbolCount = await countExistingScipProviderSymbols(conn, repoId);
2011
+ const activeMaterializationPlan = providerFirstIncrementalActive
2012
+ ? {
2013
+ deleteExistingFileSymbols: true,
2014
+ useKnownFreshWriters: true,
2015
+ writeEdges: true,
2016
+ reuseExistingProviderRows: false,
2017
+ }
2018
+ : resolveProviderFirstActiveMaterializationPlan({
2019
+ existingProviderFileCount: countExistingProviderPrimaryFiles({
2020
+ providerFiles: materializedRows.files,
2021
+ existingByPath: providerScan.existingByPath,
2022
+ }),
2023
+ providerSymbolCount: materializedRows.symbols.length,
2024
+ activeProviderInputMatches,
2025
+ existingProviderSymbolCount,
2026
+ });
2036
2027
  if (activeMaterializationPlan.reuseExistingProviderRows) {
2037
- return Promise.resolve();
2028
+ const deleteTotal = providerFirstMaterializePhaseTotal("deleteFileSymbols", materializedRows, activeMaterializationPlan);
2029
+ emitProviderFirstProgress(onProgress, "materialize.deleteFileSymbols", {
2030
+ stageCurrent: 0,
2031
+ stageTotal: deleteTotal,
2032
+ message: "provider active stale cleanup skipped for large symbol set",
2033
+ });
2034
+ const symbolTotal = providerFirstMaterializePhaseTotal("upsertSymbols", materializedRows, activeMaterializationPlan);
2035
+ emitProviderFirstProgress(onProgress, "materialize.upsertSymbols", {
2036
+ stageCurrent: symbolTotal,
2037
+ stageTotal: symbolTotal,
2038
+ message: "provider active rows reused for existing large symbol set",
2039
+ });
2038
2040
  }
2039
- return withWriteConn(async (conn) => {
2040
- if (providerRowsHaveMaterialization(materializedRows)) {
2041
- // materializeProviderFacts owns its transaction. Wrapping it
2042
- // in another transaction makes large provider-first writes
2043
- // crash in LadybugDB's native transaction handling.
2044
- await materializeProviderFacts(conn, materializedRows, {
2045
- replaceFileSymbols: true,
2046
- deleteExistingFileSymbols: activeMaterializationPlan.deleteExistingFileSymbols,
2047
- useKnownFreshWriters: activeMaterializationPlan.useKnownFreshWriters,
2048
- writeEdges: activeMaterializationPlan.writeEdges,
2049
- pruneExternalSymbols: !scopedSourceFileListActive &&
2050
- !providerFirstIncrementalActive,
2051
- symbolFtsIndexName: appConfig.semantic?.retrieval?.fts?.indexName,
2052
- measurePhase: async (phaseName, fn) => {
2053
- const substage = `materialize.${phaseName}`;
2054
- const phaseTotal = providerFirstMaterializePhaseTotal(phaseName, materializedRows, activeMaterializationPlan);
2055
- emitProviderFirstProgress(onProgress, substage, {
2056
- stageCurrent: 0,
2057
- stageTotal: phaseTotal,
2058
- message: `provider materialize ${phaseName} start`,
2059
- });
2060
- const startedAt = Date.now();
2061
- try {
2062
- return await fn();
2063
- }
2064
- finally {
2065
- recordProviderFirstPhaseTiming(`materialize.${phaseName}`, Date.now() - startedAt);
2041
+ await measureProviderFirstPhase("materialize", "providerFirstMaterialize", () => {
2042
+ if (activeMaterializationPlan.reuseExistingProviderRows) {
2043
+ return Promise.resolve();
2044
+ }
2045
+ return withWriteConn(async (conn) => {
2046
+ if (providerRowsHaveMaterialization(materializedRows)) {
2047
+ // materializeProviderFacts owns its transaction. Wrapping it
2048
+ // in another transaction makes large provider-first writes
2049
+ // crash in LadybugDB's native transaction handling.
2050
+ await materializeProviderFacts(conn, materializedRows, {
2051
+ replaceFileSymbols: true,
2052
+ deleteExistingFileSymbols: activeMaterializationPlan.deleteExistingFileSymbols,
2053
+ useKnownFreshWriters: activeMaterializationPlan.useKnownFreshWriters,
2054
+ writeEdges: activeMaterializationPlan.writeEdges,
2055
+ pruneExternalSymbols: !scopedSourceFileListActive &&
2056
+ !providerFirstIncrementalActive,
2057
+ symbolFtsIndexName: appConfig.semantic?.retrieval?.fts?.indexName,
2058
+ measurePhase: async (phaseName, fn) => {
2059
+ const substage = `materialize.${phaseName}`;
2060
+ const phaseTotal = providerFirstMaterializePhaseTotal(phaseName, materializedRows, activeMaterializationPlan);
2066
2061
  emitProviderFirstProgress(onProgress, substage, {
2067
- stageCurrent: phaseTotal,
2062
+ stageCurrent: 0,
2068
2063
  stageTotal: phaseTotal,
2069
- message: `provider materialize ${phaseName} done`,
2064
+ message: `provider materialize ${phaseName} start`,
2070
2065
  });
2071
- }
2072
- },
2073
- });
2074
- }
2075
- else if (!scopedSourceFileListActive &&
2076
- !providerFirstIncrementalActive) {
2077
- await ladybugDb.withTransaction(conn, async (txConn) => {
2078
- await ladybugDb.pruneStaleScipExternalSymbols(txConn, repoId, []);
2079
- });
2080
- }
2081
- if (!providerFirstRemovedFilesDeleted &&
2082
- providerScan.removedFileIds.length > 0) {
2083
- await ladybugDb.withTransaction(conn, async (txConn) => {
2084
- await ladybugDb.deleteFilesByIds(txConn, providerScan.removedFileIds);
2085
- });
2086
- }
2087
- if (activeProviderInputHash) {
2088
- await ladybugDb.mergeScipIngestionRecord(conn, {
2089
- id: hashValue({
2066
+ const startedAt = Date.now();
2067
+ try {
2068
+ return await fn();
2069
+ }
2070
+ finally {
2071
+ recordProviderFirstPhaseTiming(`materialize.${phaseName}`, Date.now() - startedAt);
2072
+ emitProviderFirstProgress(onProgress, substage, {
2073
+ stageCurrent: phaseTotal,
2074
+ stageTotal: phaseTotal,
2075
+ message: `provider materialize ${phaseName} done`,
2076
+ });
2077
+ }
2078
+ },
2079
+ });
2080
+ }
2081
+ else if (!scopedSourceFileListActive &&
2082
+ !providerFirstIncrementalActive) {
2083
+ await ladybugDb.withTransaction(conn, async (txConn) => {
2084
+ await ladybugDb.pruneStaleScipExternalSymbols(txConn, repoId, []);
2085
+ });
2086
+ }
2087
+ if (!providerFirstRemovedFilesDeleted &&
2088
+ providerScan.removedFileIds.length > 0) {
2089
+ await ladybugDb.withTransaction(conn, async (txConn) => {
2090
+ await ladybugDb.deleteFilesByIds(txConn, providerScan.removedFileIds);
2091
+ });
2092
+ }
2093
+ if (activeProviderInputHash) {
2094
+ await ladybugDb.mergeScipIngestionRecord(conn, {
2095
+ id: hashValue({
2096
+ repoId,
2097
+ indexPath: PROVIDER_FIRST_ACTIVE_INPUT_RECORD_PATH,
2098
+ }),
2090
2099
  repoId,
2091
2100
  indexPath: PROVIDER_FIRST_ACTIVE_INPUT_RECORD_PATH,
2092
- }),
2093
- repoId,
2094
- indexPath: PROVIDER_FIRST_ACTIVE_INPUT_RECORD_PATH,
2095
- contentHash: activeProviderInputHash,
2096
- ingestedAt: new Date().toISOString(),
2097
- ledgerVersion: providerResult.generationId,
2098
- symbolCount: materializedRows.symbols.length,
2099
- edgeCount: activeMaterializationPlan.writeEdges
2100
- ? materializedRows.edges.length
2101
- : 0,
2102
- externalSymbolCount: materializedRows.externalSymbols.length,
2103
- truncated: !activeMaterializationPlan.writeEdges,
2104
- });
2101
+ contentHash: activeProviderInputHash,
2102
+ ingestedAt: new Date().toISOString(),
2103
+ ledgerVersion: providerResult.generationId,
2104
+ symbolCount: materializedRows.symbols.length,
2105
+ edgeCount: activeMaterializationPlan.writeEdges
2106
+ ? materializedRows.edges.length
2107
+ : 0,
2108
+ externalSymbolCount: materializedRows.externalSymbols.length,
2109
+ truncated: !activeMaterializationPlan.writeEdges,
2110
+ });
2111
+ }
2112
+ }, postIndexSessionTimeoutMs);
2113
+ });
2114
+ invalidateIndexResultCaches(repoId);
2115
+ providerFirstScipMaterialized = true;
2116
+ if (activeMaterializationPlan.reuseExistingProviderRows) {
2117
+ providerFirstMaterializedFiles = 0;
2118
+ providerFirstMaterializedSymbols = 0;
2119
+ providerFirstMaterializedEdges = 0;
2120
+ }
2121
+ else {
2122
+ providerFirstMaterializedFiles = materializedRows.files.length;
2123
+ providerFirstMaterializedSymbols =
2124
+ materializedRows.symbols.length +
2125
+ materializedRows.externalSymbols.length;
2126
+ providerFirstMaterializedEdges = materializedRows.edges.length;
2127
+ for (const fileId of materializedRows.changedFileIds) {
2128
+ providerFirstChangedFileIds.add(fileId);
2105
2129
  }
2106
- }, postIndexSessionTimeoutMs);
2107
- });
2108
- invalidateIndexResultCaches(repoId);
2109
- providerFirstScipMaterialized = true;
2110
- if (activeMaterializationPlan.reuseExistingProviderRows) {
2111
- providerFirstMaterializedFiles = 0;
2112
- providerFirstMaterializedSymbols = 0;
2113
- providerFirstMaterializedEdges = 0;
2114
- }
2115
- else {
2116
- providerFirstMaterializedFiles = materializedRows.files.length;
2117
- providerFirstMaterializedSymbols =
2118
- materializedRows.symbols.length +
2119
- materializedRows.externalSymbols.length;
2120
- providerFirstMaterializedEdges = materializedRows.edges.length;
2121
- for (const fileId of materializedRows.changedFileIds) {
2122
- providerFirstChangedFileIds.add(fileId);
2123
2130
  }
2124
- }
2125
- const executionReasons = [...coverageReport.reasons];
2126
- if (scanScopedProvider.ignoredProviderPaths.length > 0) {
2127
- const sample = scanScopedProvider.ignoredProviderPaths
2128
- .slice(0, 5)
2129
- .join(", ");
2130
- executionReasons.push(`SCIP provider ignored ${scanScopedProvider.ignoredProviderPaths.length} repo-relative file(s) outside the configured scan scope: ${sample}`);
2131
- }
2132
- if (activeMaterializationPlan.reuseExistingProviderRows) {
2133
- executionReasons.push(`active provider rows reused for ${materializedRows.symbols.length} large existing symbol row(s); clean rebuild or shadow activation is required to physically retire stale provider rows`);
2134
- }
2135
- if (scopedSourceFileListActive) {
2136
- executionReasons.push("repo.sourceFileListPath is set; provider-first active row reuse and shadow activation are disabled so this subset run cannot masquerade as a complete graph");
2137
- }
2138
- if (coverageReport.fallbackPaths.size > 0) {
2139
- if (legacyFallbackPlan.runLegacyFallback) {
2140
- executionReasons.push(`legacy fallback indexed ${legacyFallbackPlan.parsedFiles} uncovered or provider-unusable file(s) after provider-first materialization`);
2131
+ const executionReasons = [...coverageReport.reasons];
2132
+ if (scanScopedProvider.ignoredProviderPaths.length > 0) {
2133
+ const sample = scanScopedProvider.ignoredProviderPaths
2134
+ .slice(0, 5)
2135
+ .join(", ");
2136
+ executionReasons.push(`SCIP provider ignored ${scanScopedProvider.ignoredProviderPaths.length} repo-relative file(s) outside the configured scan scope: ${sample}`);
2141
2137
  }
2142
- if (skippedLegacyFallbackReason) {
2143
- executionReasons.push(skippedLegacyFallbackReason);
2138
+ if (activeMaterializationPlan.reuseExistingProviderRows) {
2139
+ executionReasons.push(`active provider rows reused for ${materializedRows.symbols.length} large existing symbol row(s); clean rebuild or shadow activation is required to physically retire stale provider rows`);
2144
2140
  }
2145
- }
2146
- const coverageSummary = {
2147
- ...coverageReport.summary,
2148
- fallbackFiles: legacyFallbackPlan.parsedFiles,
2149
- };
2150
- if (legacyFallbackPlan.skippedFiles > 0) {
2151
- coverageSummary.legacyFallbackSkippedFiles =
2152
- legacyFallbackPlan.skippedFiles;
2153
- coverageSummary.legacyFallbackFileLimit =
2154
- legacyFallbackPlan.fileLimit;
2155
- if (legacyFallbackPlan.semanticEligibleFallbackFiles !== undefined &&
2156
- legacyFallbackPlan.semanticEligibleFileLimit !== undefined) {
2157
- coverageSummary.semanticEligibleFallbackFiles =
2158
- legacyFallbackPlan.semanticEligibleFallbackFiles;
2159
- coverageSummary.semanticEligibleFallbackFileLimit =
2160
- legacyFallbackPlan.semanticEligibleFileLimit;
2141
+ if (scopedSourceFileListActive) {
2142
+ executionReasons.push("repo.sourceFileListPath is set; provider-first active row reuse and shadow activation are disabled so this subset run cannot masquerade as a complete graph");
2161
2143
  }
2162
- }
2163
- if (scanScopedProvider.ignoredProviderPaths.length > 0) {
2164
- coverageSummary.ignoredProviderFiles =
2165
- scanScopedProvider.ignoredProviderPaths.length;
2166
- coverageSummary.ignoredProviderFileSamples =
2167
- scanScopedProvider.ignoredProviderPaths.slice(0, 5);
2168
- }
2169
- providerFirstExecutedSummary = {
2170
- ...providerResult.summary,
2171
- filesProcessed: materializedRows.files.length,
2172
- symbolsIndexed: materializedRows.symbols.length +
2173
- materializedRows.externalSymbols.length,
2174
- edgesCreated: materializedRows.edges.length,
2175
- externalSymbolsIndexed: materializedRows.externalSymbols.length,
2176
- shadowBuild,
2177
- coverage: coverageSummary,
2178
- reasons: executionReasons,
2179
- };
2180
- if (legacyFallbackPlan.runLegacyFallback) {
2181
- logger.info("indexRepo: provider-first materialized; using legacy fallback for uncovered files", {
2182
- repoId,
2144
+ if (coverageReport.fallbackPaths.size > 0) {
2145
+ if (legacyFallbackPlan.runLegacyFallback) {
2146
+ executionReasons.push(`legacy fallback indexed ${legacyFallbackPlan.parsedFiles} uncovered or provider-unusable file(s) after provider-first materialization`);
2147
+ }
2148
+ if (skippedLegacyFallbackReason) {
2149
+ executionReasons.push(skippedLegacyFallbackReason);
2150
+ }
2151
+ }
2152
+ const coverageSummary = {
2153
+ ...coverageReport.summary,
2183
2154
  fallbackFiles: legacyFallbackPlan.parsedFiles,
2184
- reasons: executionReasons,
2185
- });
2186
- providerFirstScan = filterProviderFirstFallbackScan(providerScan, legacyFallbackPaths);
2187
- for (const file of providerFirstScan.files) {
2188
- providerFirstFallbackPaths.add(file.path);
2155
+ };
2156
+ if (legacyFallbackPlan.skippedFiles > 0) {
2157
+ coverageSummary.legacyFallbackSkippedFiles =
2158
+ legacyFallbackPlan.skippedFiles;
2159
+ coverageSummary.legacyFallbackFileLimit =
2160
+ legacyFallbackPlan.fileLimit;
2161
+ if (legacyFallbackPlan.semanticEligibleFallbackFiles !== undefined &&
2162
+ legacyFallbackPlan.semanticEligibleFileLimit !== undefined) {
2163
+ coverageSummary.semanticEligibleFallbackFiles =
2164
+ legacyFallbackPlan.semanticEligibleFallbackFiles;
2165
+ coverageSummary.semanticEligibleFallbackFileLimit =
2166
+ legacyFallbackPlan.semanticEligibleFileLimit;
2167
+ }
2189
2168
  }
2190
- providerFirstLegacyFallbackComplete =
2191
- isProviderFirstLegacyFallbackPlanComplete(legacyFallbackPlan);
2192
- providerFirstLegacyFallbackFileCount =
2193
- legacyFallbackPlan.parsedFiles;
2194
- providerFirstLegacyFallbackSamplePaths = [
2195
- ...legacyFallbackPaths,
2196
- ].slice(0, 10);
2197
- providerFirstLegacyFallbackStartedAt = Date.now();
2198
- }
2199
- else {
2200
- if (activeMaterializationPlan.reuseExistingProviderRows) {
2201
- const versionId = await resolveActiveGraphVersionId("Provider-first SCIP active rows reused");
2202
- let semanticDeferred = await markProviderFirstSemanticReadinessDeferred({
2169
+ if (scanScopedProvider.ignoredProviderPaths.length > 0) {
2170
+ coverageSummary.ignoredProviderFiles =
2171
+ scanScopedProvider.ignoredProviderPaths.length;
2172
+ coverageSummary.ignoredProviderFileSamples =
2173
+ scanScopedProvider.ignoredProviderPaths.slice(0, 5);
2174
+ }
2175
+ providerFirstExecutedSummary = {
2176
+ ...providerResult.summary,
2177
+ filesProcessed: materializedRows.files.length,
2178
+ symbolsIndexed: materializedRows.symbols.length +
2179
+ materializedRows.externalSymbols.length,
2180
+ edgesCreated: materializedRows.edges.length,
2181
+ externalSymbolsIndexed: materializedRows.externalSymbols.length,
2182
+ shadowBuild,
2183
+ coverage: coverageSummary,
2184
+ reasons: executionReasons,
2185
+ };
2186
+ if (legacyFallbackPlan.runLegacyFallback) {
2187
+ logger.info("indexRepo: provider-first materialized; using legacy fallback for uncovered files", {
2203
2188
  repoId,
2204
- versionId,
2205
- appConfig,
2206
- });
2207
- const semanticRefresh = await runProviderFirstDeferredSemanticRefresh({
2208
- versionId,
2209
- semanticDeferred,
2189
+ fallbackFiles: legacyFallbackPlan.parsedFiles,
2190
+ reasons: executionReasons,
2210
2191
  });
2211
- semanticDeferred = semanticRefresh.semanticDeferred;
2212
- providerFirstExecutedSummary = withProviderFirstPhaseTimings(providerFirstExecutedSummary);
2213
- const resultScip = "generatedIndexes" in providerResult
2192
+ providerFirstScan = filterProviderFirstFallbackScan(providerScan, legacyFallbackPaths);
2193
+ for (const file of providerFirstScan.files) {
2194
+ providerFirstFallbackPaths.add(file.path);
2195
+ }
2196
+ providerFirstLegacyFallbackComplete =
2197
+ isProviderFirstLegacyFallbackPlanComplete(legacyFallbackPlan);
2198
+ providerFirstLegacyFallbackFileCount =
2199
+ legacyFallbackPlan.parsedFiles;
2200
+ providerFirstLegacyFallbackSamplePaths = [
2201
+ ...legacyFallbackPaths,
2202
+ ].slice(0, 10);
2203
+ providerFirstLegacyFallbackStartedAt = Date.now();
2204
+ }
2205
+ else {
2206
+ if (activeMaterializationPlan.reuseExistingProviderRows) {
2207
+ const versionId = await resolveActiveGraphVersionId("Provider-first SCIP active rows reused");
2208
+ let semanticDeferred = await markProviderFirstSemanticReadinessDeferred({
2209
+ repoId,
2210
+ versionId,
2211
+ appConfig,
2212
+ });
2213
+ const semanticRefresh = await runProviderFirstDeferredSemanticRefresh({
2214
+ versionId,
2215
+ semanticDeferred,
2216
+ });
2217
+ semanticDeferred = semanticRefresh.semanticDeferred;
2218
+ providerFirstExecutedSummary = withProviderFirstPhaseTimings(providerFirstExecutedSummary);
2219
+ const resultScip = "generatedIndexes" in providerResult
2220
+ ? {
2221
+ generatedIndexes: providerResult.generatedIndexes,
2222
+ failures: providerResult.failures,
2223
+ generatorCache: activeScipPreRefresh?.cache,
2224
+ }
2225
+ : undefined;
2226
+ const result = {
2227
+ versionId,
2228
+ filesProcessed: 0,
2229
+ changedFiles: 0,
2230
+ removedFiles: 0,
2231
+ symbolsIndexed: 0,
2232
+ edgesCreated: 0,
2233
+ clustersComputed: 0,
2234
+ processesTraced: 0,
2235
+ durationMs: Date.now() - startTime,
2236
+ pass1Engine: emptyPass1EngineTelemetry(),
2237
+ scip: resultScip,
2238
+ providerFirst,
2239
+ providerFirstExecution: providerFirstExecutedSummary,
2240
+ semanticDeferred: semanticDeferred || undefined,
2241
+ algorithmRefresh: skippedDerivedStateResult("provider-first active rows reused").algorithmRefresh,
2242
+ };
2243
+ clearProviderGraphRowsForGc(materializedRows);
2244
+ return result;
2245
+ }
2246
+ const versionId = await createOrReuseVersion(providerFirstLspExecutor(providerResult.summary.executor)
2247
+ ? providerFirstIncrementalActive
2248
+ ? "Provider-first LSP incremental index"
2249
+ : "Provider-first LSP index"
2250
+ : providerFirstIncrementalActive
2251
+ ? "Provider-first SCIP incremental index"
2252
+ : "Provider-first SCIP index", true);
2253
+ const pass1Engine = emptyPass1EngineTelemetry();
2254
+ const scip = "generatedIndexes" in providerResult
2214
2255
  ? {
2215
2256
  generatedIndexes: providerResult.generatedIndexes,
2216
2257
  failures: providerResult.failures,
2217
2258
  generatorCache: activeScipPreRefresh?.cache,
2218
2259
  }
2219
2260
  : undefined;
2261
+ const post = await runPostIndexFinalization({
2262
+ versionId,
2263
+ indexMode: providerFirstIncrementalActive ? "incremental" : "full",
2264
+ filesTotal: providerFirstIncrementalActive
2265
+ ? providerCoverageScan.files.length
2266
+ : materializedRows.files.length,
2267
+ filesScanned: materializedRows.files.length,
2268
+ symbolsExtracted: materializedRows.symbols.length +
2269
+ materializedRows.externalSymbols.length,
2270
+ edgesExtracted: materializedRows.edges.length,
2271
+ changedFileIdsForFinalize: scopedSourceFileListActive || providerFirstIncrementalActive
2272
+ ? materializedRows.changedFileIds
2273
+ : undefined,
2274
+ changedTestFilePathsForFinalize: new Set(),
2275
+ changedFileIdsForMemory: materializedRows.changedFileIds,
2276
+ hasIndexMutations: true,
2277
+ callResolutionTelemetry: createCallResolutionTelemetry({
2278
+ repoId,
2279
+ mode: providerFirstIncrementalActive ? "incremental" : "full",
2280
+ pass2EligibleFileCount: 0,
2281
+ registeredResolvers: [],
2282
+ }),
2283
+ pass1Engine,
2284
+ scip,
2285
+ preloadedFileSummarySymbolFactsByFile: buildPreloadedFileSummarySymbolFactsFromRows({
2286
+ files: materializedRows.files,
2287
+ symbols: materializedRows.symbols,
2288
+ }),
2289
+ deferSemanticRefresh: true,
2290
+ skipDerivedStateReason: providerFirstSkipDerivedStateReason,
2291
+ });
2292
+ providerFirstExecutedSummary = {
2293
+ ...providerFirstExecutedSummary,
2294
+ shadowBuild: await finalizeProviderFirstShadowBuild(providerFirstExecutedSummary.shadowBuild, versionId, post.semanticDeferred),
2295
+ };
2296
+ const semanticRefresh = await runProviderFirstDeferredSemanticRefresh({
2297
+ versionId,
2298
+ semanticDeferred: post.semanticDeferred,
2299
+ });
2300
+ providerFirstExecutedSummary = withProviderFirstPhaseTimings(providerFirstExecutedSummary);
2220
2301
  const result = {
2221
2302
  versionId,
2222
- filesProcessed: 0,
2223
- changedFiles: 0,
2224
- removedFiles: 0,
2225
- symbolsIndexed: 0,
2226
- edgesCreated: 0,
2227
- clustersComputed: 0,
2228
- processesTraced: 0,
2303
+ filesProcessed: materializedRows.files.length,
2304
+ changedFiles: materializedRows.changedFileIds.size,
2305
+ removedFiles: providerScan.removedFiles,
2306
+ symbolsIndexed: materializedRows.symbols.length +
2307
+ materializedRows.externalSymbols.length,
2308
+ edgesCreated: materializedRows.edges.length,
2309
+ clustersComputed: post.clustersComputed,
2310
+ processesTraced: post.processesTraced,
2229
2311
  durationMs: Date.now() - startTime,
2230
- pass1Engine: emptyPass1EngineTelemetry(),
2231
- scip: resultScip,
2312
+ summaryStats: semanticRefresh.summaryStats ?? post.summaryStats,
2313
+ timings: phaseTimings
2314
+ ? {
2315
+ totalMs: Date.now() - startTime,
2316
+ phases: phaseTimings,
2317
+ }
2318
+ : undefined,
2319
+ pass1Engine,
2320
+ scip,
2232
2321
  providerFirst,
2233
2322
  providerFirstExecution: providerFirstExecutedSummary,
2234
- semanticDeferred: semanticDeferred || undefined,
2235
- algorithmRefresh: skippedDerivedStateResult("provider-first active rows reused").algorithmRefresh,
2323
+ semanticDeferred: semanticRefresh.semanticDeferred,
2324
+ algorithmRefresh: post.algorithmRefresh,
2236
2325
  };
2237
- clearProviderGraphRowsForGc(materializedRows);
2238
2326
  return result;
2239
2327
  }
2240
- const versionId = await createOrReuseVersion(providerFirstLspExecutor(providerResult.summary.executor)
2241
- ? providerFirstIncrementalActive
2242
- ? "Provider-first LSP incremental index"
2243
- : "Provider-first LSP index"
2244
- : providerFirstIncrementalActive
2245
- ? "Provider-first SCIP incremental index"
2246
- : "Provider-first SCIP index", true);
2247
- const pass1Engine = emptyPass1EngineTelemetry();
2248
- const scip = "generatedIndexes" in providerResult
2249
- ? {
2250
- generatedIndexes: providerResult.generatedIndexes,
2251
- failures: providerResult.failures,
2252
- generatorCache: activeScipPreRefresh?.cache,
2253
- }
2254
- : undefined;
2255
- const post = await runPostIndexFinalization({
2256
- versionId,
2257
- indexMode: providerFirstIncrementalActive ? "incremental" : "full",
2258
- filesTotal: providerFirstIncrementalActive
2259
- ? providerCoverageScan.files.length
2260
- : materializedRows.files.length,
2261
- filesScanned: materializedRows.files.length,
2262
- symbolsExtracted: materializedRows.symbols.length +
2263
- materializedRows.externalSymbols.length,
2264
- edgesExtracted: materializedRows.edges.length,
2265
- changedFileIdsForFinalize: scopedSourceFileListActive || providerFirstIncrementalActive
2266
- ? materializedRows.changedFileIds
2267
- : undefined,
2268
- changedTestFilePathsForFinalize: new Set(),
2269
- changedFileIdsForMemory: materializedRows.changedFileIds,
2270
- hasIndexMutations: true,
2271
- callResolutionTelemetry: createCallResolutionTelemetry({
2272
- repoId,
2273
- mode: providerFirstIncrementalActive ? "incremental" : "full",
2274
- pass2EligibleFileCount: 0,
2275
- registeredResolvers: [],
2276
- }),
2277
- pass1Engine,
2278
- scip,
2279
- preloadedFileSummarySymbolFactsByFile: buildPreloadedFileSummarySymbolFactsFromRows({
2280
- files: materializedRows.files,
2281
- symbols: materializedRows.symbols,
2282
- }),
2283
- deferSemanticRefresh: true,
2284
- skipDerivedStateReason: providerFirstSkipDerivedStateReason,
2285
- });
2286
- providerFirstExecutedSummary = {
2287
- ...providerFirstExecutedSummary,
2288
- shadowBuild: await finalizeProviderFirstShadowBuild(providerFirstExecutedSummary.shadowBuild, versionId, post.semanticDeferred),
2289
- };
2290
- const semanticRefresh = await runProviderFirstDeferredSemanticRefresh({
2291
- versionId,
2292
- semanticDeferred: post.semanticDeferred,
2293
- });
2294
- providerFirstExecutedSummary = withProviderFirstPhaseTimings(providerFirstExecutedSummary);
2295
- const result = {
2296
- versionId,
2297
- filesProcessed: materializedRows.files.length,
2298
- changedFiles: materializedRows.changedFileIds.size,
2299
- removedFiles: providerScan.removedFiles,
2300
- symbolsIndexed: materializedRows.symbols.length +
2301
- materializedRows.externalSymbols.length,
2302
- edgesCreated: materializedRows.edges.length,
2303
- clustersComputed: post.clustersComputed,
2304
- processesTraced: post.processesTraced,
2305
- durationMs: Date.now() - startTime,
2306
- summaryStats: semanticRefresh.summaryStats ?? post.summaryStats,
2307
- timings: phaseTimings
2308
- ? {
2309
- totalMs: Date.now() - startTime,
2310
- phases: phaseTimings,
2311
- }
2312
- : undefined,
2313
- pass1Engine,
2314
- scip,
2315
- providerFirst,
2316
- providerFirstExecution: providerFirstExecutedSummary,
2317
- semanticDeferred: semanticRefresh.semanticDeferred,
2318
- algorithmRefresh: post.algorithmRefresh,
2319
- };
2320
- return result;
2321
2328
  }
2322
2329
  }
2323
2330
  }
2324
2331
  }
2332
+ catch (err) {
2333
+ if (err instanceof ProviderFirstGraphValidationError)
2334
+ throw err;
2335
+ if (providerFirst.requestedMode !== "auto")
2336
+ throw err;
2337
+ const reason = err instanceof Error ? err.message : String(err);
2338
+ const fallbackReason = formatProviderFirstAutoFallbackReason({
2339
+ reason,
2340
+ providerFirstIncrementalActive,
2341
+ });
2342
+ logger.warn("indexRepo: provider-first phase failed (coverage scan or collection), using legacy fallback", { repoId, reason: fallbackReason });
2343
+ providerFirstExecutionFallback = providerFirstFallbackSummary([
2344
+ fallbackReason,
2345
+ ]);
2346
+ // providerFirstScan stays undefined; legacy scanRepoForIndex will run below.
2347
+ }
2325
2348
  }
2326
2349
  const { files, existingByPath, removedFiles, removedFileIds, allFilesUnchanged: scanAllFilesUnchanged, } = providerFirstScan ??
2327
2350
  (await measurePhase("scanRepo", () => scanRepoForIndex({
@@ -2904,6 +2927,11 @@ async function indexRepoImpl(repoId, mode, onProgress, signal, options, scipPreR
2904
2927
  onProgress,
2905
2928
  }));
2906
2929
  recordFinalizeIndexingSubphaseTimings(finalizeResult.timings);
2930
+ await measurePhase("ensureCriticalSymbolFts", async () => {
2931
+ await ensureCriticalSymbolFtsIndex({
2932
+ recordTiming: recordIndexSubphaseTiming,
2933
+ });
2934
+ });
2907
2935
  // Refresh read connection again after version/metrics writes.
2908
2936
  freshConn = await getLadybugConn();
2909
2937
  const derivedResult = providerFirstSkipDerivedStateReason