sdl-mcp 0.10.9 → 0.11.0

package/README.md

@@ -531,6 +531,12 @@ Each gateway tool accepts an `action` discriminator field (e.g., `{ action: "sym
 
 [Tool Gateway Deep Dive →](./docs/feature-deep-dives/tool-gateway.md)
 
+---
+
+### Observability Dashboard
+
+Built-in read-only dashboard surfaces every metric needed to diagnose SDL-MCP behaviour without parsing stderr logs. Cyberpunk-corporate dark UI at `/ui/observability` on the HTTP transport, plus REST + SSE APIs (`/api/observability/snapshot`, `/timeseries`, `/beam-explain`, `/stream`) for programmatic access. Surfaces cache hit rates, hybrid-retrieval breakdowns (FTS / vector / PPR / RRF), beam-search decision traces, indexing pipeline metrics, write-pool and drain saturation, packed-wire savings, SCIP ingest health, deterministic bottleneck classification, and OS-level resource samples. See [Observability Dashboard Deep Dive](./docs/feature-deep-dives/observability-dashboard.md).
+
 <br/>
 
 ---
@@ -539,12 +545,12 @@ Each gateway tool accepts an `action` discriminator field (e.g., `{ action: "sym
 
 ## Current Tool Surface at a Glance
 
-| Mode | Tool count | Composition |
-| :--- | :--------- | :---------- |
-| Flat | `33` | `2` universal + `31` flat tools |
-| Gateway | `6` | `2` universal + `4` gateway tools |
-| Gateway + legacy | `37` | `2` universal + `4` gateway + `31` flat tools |
-| Code Mode exclusive | `4` | `sdl.action.search`, `sdl.context`, `sdl.manual`, `sdl.workflow` |
+| Mode                | Tool count | Composition                                                      |
+| :------------------ | :--------- | :--------------------------------------------------------------- |
+| Flat                | `33`       | `2` universal + `31` flat tools                                  |
+| Gateway             | `6`        | `2` universal + `4` gateway tools                                |
+| Gateway + legacy    | `37`       | `2` universal + `4` gateway + `31` flat tools                    |
+| Code Mode exclusive | `4`        | `sdl.action.search`, `sdl.context`, `sdl.manual`, `sdl.workflow` |
 
 The generated source of truth is [`docs/generated/tool-inventory.md`](./docs/generated/tool-inventory.md).
 
@@ -627,21 +633,21 @@ The generated source of truth is [`docs/generated/tool-inventory.md`](./docs/gen
 
 ## CLI Commands
 
-| Command             | Description                                                                                    |
-| :------------------ | :--------------------------------------------------------------------------------------------- |
-| `sdl-mcp init`      | Bootstrap config, detect repo/languages, optionally auto-index                                 |
-| `sdl-mcp doctor`    | Validate runtime, config, DB, grammars, repo access                                            |
-| `sdl-mcp index`     | Index repositories (with optional `--watch` mode)                                              |
-| `sdl-mcp serve`     | Start MCP server (`--stdio` or `--http`)                                                       |
-| `sdl-mcp tool`      | Access 30 direct action aliases ([docs](./docs/feature-deep-dives/cli-tool-access.md))          |
-| `sdl-mcp info`      | Runtime diagnostics — version, Node.js, platform, database, config                             |
-| `sdl-mcp summary`   | Generate copy/paste context summaries from the CLI                                             |
-| `sdl-mcp health`    | Compute composite health score with badge/JSON output                                          |
-| `sdl-mcp benchmark` | Run CI regression benchmarks                                                                   |
-| `sdl-mcp export`    | Export sync artifact                                                                           |
-| `sdl-mcp import`    | Import sync artifact                                                                           |
-| `sdl-mcp pull`      | Pull by version/commit with fallback                                                           |
-| `sdl-mcp version`   | Show version and environment info                                                              |
+| Command             | Description                                                                            |
+| :------------------ | :------------------------------------------------------------------------------------- |
+| `sdl-mcp init`      | Bootstrap config, detect repo/languages, optionally auto-index                         |
+| `sdl-mcp doctor`    | Validate runtime, config, DB, grammars, repo access                                    |
+| `sdl-mcp index`     | Index repositories (with optional `--watch` mode)                                      |
+| `sdl-mcp serve`     | Start MCP server (`--stdio` or `--http`)                                               |
+| `sdl-mcp tool`      | Access 30 direct action aliases ([docs](./docs/feature-deep-dives/cli-tool-access.md)) |
+| `sdl-mcp info`      | Runtime diagnostics — version, Node.js, platform, database, config                     |
+| `sdl-mcp summary`   | Generate copy/paste context summaries from the CLI                                     |
+| `sdl-mcp health`    | Compute composite health score with badge/JSON output                                  |
+| `sdl-mcp benchmark` | Run CI regression benchmarks                                                           |
+| `sdl-mcp export`    | Export sync artifact                                                                   |
+| `sdl-mcp import`    | Import sync artifact                                                                   |
+| `sdl-mcp pull`      | Pull by version/commit with fallback                                                   |
+| `sdl-mcp version`   | Show version and environment info                                                      |
 
 [CLI Reference →](./docs/cli-reference.md) · [Configuration Reference →](./docs/configuration-reference.md)
 
@@ -731,16 +737,16 @@ flowchart TD
 
 ## Documentation
 
-| Document                                                          | Description                                                                     |
-| :---------------------------------------------------------------- | :------------------------------------------------------------------------------ |
-| [Getting Started](./docs/getting-started.md)                      | Installation, 5-minute setup, MCP client config                                 |
-| [MCP Tools Reference](./docs/mcp-tools-detailed.md)               | Detailed docs for the current flat, gateway, and Code Mode surfaces             |
-| [CLI Reference](./docs/cli-reference.md)                          | All CLI commands and options                                                    |
-| [Configuration Reference](./docs/configuration-reference.md)      | Every config option with defaults and guidance                                  |
-| [Agent Workflows](./docs/agent-workflows.md)                      | Workflow instructions for CLAUDE.md / AGENTS.md                                 |
-| [Architecture](./docs/architecture.md)                            | Tech stack, data flow, component diagram                                        |
-| [Iris Gate Ladder](./docs/feature-deep-dives/iris-gate-ladder.md) | Context escalation methodology                                                  |
-| [Troubleshooting](./docs/troubleshooting.md)                      | Common issues and fixes                                                         |
+| Document                                                          | Description                                                         |
+| :---------------------------------------------------------------- | :------------------------------------------------------------------ |
+| [Getting Started](./docs/getting-started.md)                      | Installation, 5-minute setup, MCP client config                     |
+| [MCP Tools Reference](./docs/mcp-tools-detailed.md)               | Detailed docs for the current flat, gateway, and Code Mode surfaces |
+| [CLI Reference](./docs/cli-reference.md)                          | All CLI commands and options                                        |
+| [Configuration Reference](./docs/configuration-reference.md)      | Every config option with defaults and guidance                      |
+| [Agent Workflows](./docs/agent-workflows.md)                      | Workflow instructions for CLAUDE.md / AGENTS.md                     |
+| [Architecture](./docs/architecture.md)                            | Tech stack, data flow, component diagram                            |
+| [Iris Gate Ladder](./docs/feature-deep-dives/iris-gate-ladder.md) | Context escalation methodology                                      |
+| [Troubleshooting](./docs/troubleshooting.md)                      | Common issues and fixes                                             |
 
 ### Feature Deep Dives
 

package/config/benchmark.ci.config.json

@@ -41,10 +41,10 @@
     },
     "performance": {
       "sliceBuildTimeMs": {
-        "maxMs": 350,
+        "maxMs": 500,
         "trend": "lower-is-better",
         "allowableIncreasePercent": 125,
-        "_comment": "Raised from 200 ms: TS call-resolver creates more edges (import aliases, barrel re-exports), making graph loading and beam-search proportionally slower."
+        "_comment": "Raised from 350 ms: hosted Linux runners have produced valid zod-oss native-addon samples around 420 ms after edge-density increases, while this still guards against subsecond slice regressions."
       },
       "avgSkeletonTimeMs": {
         "maxMs": 10,

package/config/sdlmcp.config.example.json

@@ -126,6 +126,12 @@
     "summaryMaxConcurrency": 5,
     "summaryBatchSize": 20,
     "embeddingConcurrency": 1,
+    "embeddingBatchSize": 32,
+    "fileSummaryEmbeddingBatchSize": 4,
+    "fileSummaryEmbeddingMaxChars": 4096,
+    "embeddingsSequential": false,
+    "modelVariant": "default",
+    "executionProviders": ["cpu"],
     "retrieval": {
       "mode": "hybrid",
       "extensionsOptional": true,
@@ -159,7 +165,7 @@
   "prefetch": {
     "enabled": true,
     "maxBudgetPercent": 20,
-    "warmTopN": 50
+    "warmTopN": 0
   },
   "tracing": {
     "enabled": true,
@@ -168,6 +174,19 @@
     "otlpEndpoint": null,
     "sampleRate": 1
   },
+  "observability": {
+    "enabled": true,
+    "sampleIntervalMs": 2000,
+    "retentionShortMinutes": 15,
+    "retentionLongHours": 24,
+    "pprMetricsEnabled": true,
+    "packedStatsEnabled": true,
+    "scipIngestMetrics": true,
+    "beamExplainCapacity": 128,
+    "beamExplainEntriesPerSlice": 512,
+    "sseHeartbeatMs": 15000,
+    "sseMaxStreamMs": 3600000
+  },
   "parallelScorer": {
     "enabled": true,
     "poolSize": null,
@@ -199,7 +218,14 @@
   },
   "httpAuth": {
     "enabled": false,
-    "token": null
+    "token": null,
+    "rateLimit": {
+      "bucketSize": 30,
+      "refillPerSec": 0.5
+    }
+  },
+  "http": {
+    "allowRemote": false
   },
   "codeMode": {
     "enabled": true,
@@ -232,7 +258,8 @@
       "binary": "scip-io",
       "args": [],
       "autoInstall": true,
-      "timeoutMs": 600000
+      "timeoutMs": 600000,
+      "cleanupAfterIngest": true
     }
   },
   "memory": {

package/config/sdlmcp.config.json

@@ -143,6 +143,8 @@
     "enabled": true,
     "alpha": 0.6,
     "provider": "mock",
+    "executionProviders": ["cpu"],
+    "modelVariant": "default",
     "model": "jina-embeddings-v2-base-code",
     "additionalModels": ["nomic-embed-text-v1.5"],
     "generateSummaries": false

package/config/sdlmcp.config.schema.json

@@ -597,11 +597,47 @@
         },
         "embeddingConcurrency": {
           "type": "integer",
-          "description": "Number of embedding batches to process concurrently during refreshSymbolEmbeddings(). Defaults to 1 (sequential). Increasing this can improve throughput on multi-core machines but ONNX Runtime's internal thread pool is shared across concurrent calls; consider reducing intraOpNumThreads proportionally when raising above 1. Maximum 4.",
+          "description": "Number of embedding batches to process concurrently during refreshSymbolEmbeddings(). Defaults to 1 (sequential). Increasing this can improve throughput on multi-core machines but ONNX Runtime's internal thread pool is shared across concurrent calls; consider reducing intraOpNumThreads proportionally when raising above 1. Maximum 8.",
           "minimum": 1,
-          "maximum": 4,
+          "maximum": 8,
           "default": 1
         },
+        "embeddingBatchSize": {
+          "type": "integer",
+          "description": "ONNX inference batch width for symbol embedding refresh. Default 32 matches LocalEmbeddingProvider's tokenizer + session expectations. Larger batches (64-128) amortise tokenizer + session bind/unbind costs across more rows but raise peak memory roughly with the longest sequence in the batch. Length-bucketing before splitting keeps tokenizer pad waste bounded.",
+          "minimum": 1,
+          "maximum": 128,
+          "default": 32
+        },
+        "fileSummaryEmbeddingBatchSize": {
+          "type": "integer",
+          "description": "ONNX inference batch width for hybrid FileSummary embedding refresh. File-level payloads are larger than symbol payloads, so this defaults lower than embeddingBatchSize to bound DirectML/ONNX memory use.",
+          "minimum": 1,
+          "maximum": 16,
+          "default": 4
+        },
+        "fileSummaryEmbeddingMaxChars": {
+          "type": "integer",
+          "description": "Maximum characters sent to the embedding provider for each FileSummary. Stored summaries and FTS searchText remain complete; this only bounds vector-generation resources.",
+          "minimum": 512,
+          "maximum": 32768,
+          "default": 4096
+        },
+        "embeddingsSequential": {
+          "type": "boolean",
+          "description": "When two or more embedding models are configured, run them in series instead of via Promise.all. Default false launches concurrently — best when ORT runs sessions on independent thread pools. Set true on systems where ORT serializes sessions at the thread-pool layer (observed alternation pattern in CLI progress); each model holds the full thread budget end-to-end, weights stay hot in L3 cache, and model-handoff scheduling overhead disappears.",
+          "default": false
+        },
+        "modelVariant": {
+          "type": "string",
+          "description": "ONNX model file variant to load. Common values: 'default' / 'int8' (~140-160MB, balanced), 'fp16' (~270-320MB, ~30% faster than fp32 with <0.5% accuracy loss), 'fp32' (~550-650MB, reference quality). nomic-embed-text-v1.5 also publishes 'uint8', 'q4', 'q4f16', 'bnb4'. When the chosen variant is not published for a given model, the registry falls back to that model's default variant with a warning."
+        },
+        "executionProviders": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "ONNX Runtime execution providers in priority order. ORT tries them left-to-right; first one that initialises wins. Bundled with default onnxruntime-node npm package: Windows x64 ['cpu', 'dml', 'webgpu'], macOS ['cpu', 'coreml'], Linux x64 ['cpu', 'cuda', 'tensorrt'] (CUDA EP requires NVIDIA GPU + CUDA 12 + cuDNN on the host), Linux arm64 ['cpu']. DirectML covers any DX12 GPU including AMD Radeon, NVIDIA GeForce, Intel Arc. ROCm (AMD on Linux), OpenVINO, QNN need a custom ORT build. Unsupported entries on a given platform are dropped with a warning; 'cpu' is auto-appended as final fallback so session creation never strands.",
+          "default": ["cpu"]
+        },
         "ann": {
           "type": "object",
           "description": "Deprecated — removed in v0.10.1. Silently ignored. Use retrieval.vector instead.",
@@ -811,14 +847,94 @@
         },
         "warmTopN": {
           "type": "integer",
-          "minimum": 1,
-          "default": 50
+          "minimum": 0,
+          "default": 0,
+          "description": "Number of top-fan-in symbols to prefetch on serve start. Default 0 disables startup warming; raise only when you have evidence the warm set is consumed within the 5-minute stale window."
         }
       },
       "default": {
         "enabled": true,
         "maxBudgetPercent": 20,
-        "warmTopN": 50
+        "warmTopN": 0
+      }
+    },
+    "observability": {
+      "type": "object",
+      "description": "Observability dashboard configuration (in-process metrics + SSE stream)",
+      "additionalProperties": false,
+      "properties": {
+        "enabled": {
+          "type": "boolean",
+          "default": true
+        },
+        "sampleIntervalMs": {
+          "type": "integer",
+          "minimum": 250,
+          "maximum": 60000,
+          "default": 2000
+        },
+        "retentionShortMinutes": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 60,
+          "default": 15
+        },
+        "retentionLongHours": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 168,
+          "default": 24
+        },
+        "pprMetricsEnabled": {
+          "type": "boolean",
+          "default": true
+        },
+        "packedStatsEnabled": {
+          "type": "boolean",
+          "default": true
+        },
+        "scipIngestMetrics": {
+          "type": "boolean",
+          "default": true
+        },
+        "beamExplainCapacity": {
+          "type": "integer",
+          "minimum": 8,
+          "maximum": 2048,
+          "default": 128
+        },
+        "beamExplainEntriesPerSlice": {
+          "type": "integer",
+          "minimum": 16,
+          "maximum": 8192,
+          "default": 512
+        },
+        "sseHeartbeatMs": {
+          "type": "integer",
+          "minimum": 1000,
+          "maximum": 60000,
+          "default": 15000
+        },
+        "sseMaxStreamMs": {
+          "type": "integer",
+          "minimum": 60000,
+          "maximum": 86400000,
+          "default": 3600000,
+          "description": "Maximum lifetime for a single observability SSE stream before the server closes it."
+        }
+      },
+      "default": {
+        "enabled": true,
+        "sampleIntervalMs": 2000,
+        "retentionShortMinutes": 15,
+        "retentionLongHours": 24,
+        "pprMetricsEnabled": true,
+        "packedStatsEnabled": true,
+        "scipIngestMetrics": true,
+        "beamExplainCapacity": 128,
+        "beamExplainEntriesPerSlice": 512,
+        "sseHeartbeatMs": 15000,
+        "sseMaxStreamMs": 3600000
       }
     },
     "tracing": {
@@ -1069,11 +1185,52 @@
           "description": "Static bearer token. When null a random token is generated at startup.",
           "minLength": 1,
           "default": null
+        },
+        "rateLimit": {
+          "type": "object",
+          "description": "Per-client token bucket for protected HTTP routes before bearer-token comparison",
+          "properties": {
+            "bucketSize": {
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 10000,
+              "default": 30
+            },
+            "refillPerSec": {
+              "type": "number",
+              "exclusiveMinimum": 0,
+              "maximum": 1000,
+              "default": 0.5
+            }
+          },
+          "default": {
+            "bucketSize": 30,
+            "refillPerSec": 0.5
+          }
         }
       },
       "default": {
         "enabled": false,
-        "token": null
+        "token": null,
+        "rateLimit": {
+          "bucketSize": 30,
+          "refillPerSec": 0.5
+        }
+      }
+    },
+    "http": {
+      "type": "object",
+      "description": "HTTP transport bind configuration.",
+      "additionalProperties": false,
+      "properties": {
+        "allowRemote": {
+          "type": "boolean",
+          "description": "Allow binding the HTTP transport to a non-loopback host such as 0.0.0.0 or ::.",
+          "default": false
+        }
+      },
+      "default": {
+        "allowRemote": false
       }
     },
     "codeMode": {
@@ -1164,6 +1321,13 @@
           "default": 0.95
         },
         "autoIngestOnRefresh": { "type": "boolean", "default": true },
+        "ingestConcurrency": {
+          "type": "integer",
+          "description": "Number of SCIP indexes to ingest concurrently. Decode/parse overlaps; writes still serialise via writeLimiter. Default 1 (sequential).",
+          "minimum": 1,
+          "maximum": 8,
+          "default": 1
+        },
         "generator": {
           "type": "object",
           "description": "Auto-generate index.scip via the scip-io CLI before each refresh. Has no effect unless scip.enabled is also true. When enabled and the binary is missing, sdl-mcp downloads it from GitHub releases into ~/.sdl-mcp/bin/ with mandatory SHA-256 verification against the release's SHA256SUMS.txt; installs are refused if the checksum file is absent.",
@@ -1181,6 +1345,11 @@
               "minimum": 1000,
               "maximum": 1800000,
               "default": 600000
+            },
+            "cleanupAfterIngest": {
+              "type": "boolean",
+              "description": "Delete <repoRoot>/index.scip after the post-refresh ingest consumes it. The file is regenerated every refresh, so keeping it just clutters the working tree. Only the default output location is cleaned up — passing --output via args opts you out of cleanup automatically.",
+              "default": true
             }
           },
           "default": {
@@ -1188,7 +1357,8 @@
             "binary": "scip-io",
             "args": [],
             "autoInstall": true,
-            "timeoutMs": 600000
+            "timeoutMs": 600000,
+            "cleanupAfterIngest": true
           }
         }
       }
@@ -1231,6 +1401,55 @@
         }
       },
       "additionalProperties": false
+    },
+    "wire": {
+      "type": "object",
+      "description": "Wire-format configuration (packed, auto-gate)",
+      "properties": {
+        "packed": {
+          "type": "object",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "description": "Master switch for packed wire format. Env var SDL_PACKED_ENABLED overrides.",
+              "default": true
+            },
+            "threshold": {
+              "type": "number",
+              "description": "Minimum byte-savings ratio before packed payload is emitted (0.0-1.0). Env var SDL_PACKED_THRESHOLD overrides.",
+              "default": 0.1,
+              "minimum": 0,
+              "maximum": 1
+            },
+            "tokenThreshold": {
+              "type": "number",
+              "description": "Minimum estimated-token-savings ratio before packed payload is emitted (0.0-1.0). Either threshold (bytes OR tokens) is sufficient. Env var SDL_PACKED_TOKEN_THRESHOLD overrides.",
+              "default": 0.2,
+              "minimum": 0,
+              "maximum": 1
+            },
+            "defaultFormat": {
+              "type": "string",
+              "enum": ["packed", "auto", "compact"],
+              "description": "Default wire format when client omits the field. Env var SDL_PACKED_DEFAULT_FORMAT overrides.",
+              "default": "auto"
+            },
+            "encoders": {
+              "type": "object",
+              "description": "Per-encoder enable map keyed by encoderId (sl1, ss1, ctx1)",
+              "additionalProperties": {
+                "type": "object",
+                "properties": {
+                  "enabled": { "type": "boolean", "default": true }
+                },
+                "additionalProperties": false
+              }
+            }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
     }
   }
 }