sigmap 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +68 -65
- package/CHANGELOG.md +14 -0
- package/README.md +36 -19
- package/gen-context.js +32 -6
- package/package.json +1 -1
- package/packages/cli/package.json +1 -1
- package/packages/core/package.json +1 -1
- package/src/format/benchmark-report.js +443 -0
- package/src/mcp/server.js +1 -1
package/AGENTS.md
CHANGED
|
@@ -12,7 +12,7 @@ Use this marker block for all appendable context files:
|
|
|
12
12
|
## Auto-generated signatures
|
|
13
13
|
<!-- Updated by gen-context.js -->
|
|
14
14
|
You are a coding assistant with full knowledge of this codebase.
|
|
15
|
-
Below are the code signatures extracted by SigMap v5.2.0 on 2026-04-
|
|
15
|
+
Below are the code signatures extracted by SigMap v5.2.0 on 2026-04-16T23:13:56.540Z.
|
|
16
16
|
|
|
17
17
|
Use these signatures to answer questions about the code accurately.
|
|
18
18
|
|
|
@@ -23,16 +23,51 @@ Use these signatures to answer questions about the code accurately.
|
|
|
23
23
|
|
|
24
24
|
# Code signatures
|
|
25
25
|
|
|
26
|
-
## changes (last 5 commits —
|
|
26
|
+
## changes (last 5 commits — 46 minutes ago)
|
|
27
27
|
```
|
|
28
28
|
src/config/loader.js +loadBaseConfig ~loadConfig ~deepClone
|
|
29
29
|
src/format/dashboard.js ~computeExtractorCoverage ~readBenchmarkTrend
|
|
30
|
-
src/judge/judge-engine.js +tokenize +groundedness +judge
|
|
31
|
-
src/
|
|
30
|
+
src/judge/judge-engine.js +tokenize +groundedness +extractContextFiles +judge
|
|
31
|
+
src/learning/weights.js +weightsPath +clampMultiplier +normalizeFile +sanitizeWeights
|
|
32
|
+
src/mcp/handlers.js ~queryContext ~getImpact
|
|
33
|
+
src/retrieval/ranker.js ~scoreFile ~rank
|
|
34
|
+
packages/core/index.js ~extract
|
|
32
35
|
```
|
|
33
36
|
|
|
34
37
|
## packages
|
|
35
38
|
|
|
39
|
+
### packages/core/README.md
|
|
40
|
+
```
|
|
41
|
+
h1 sigmap-core
|
|
42
|
+
h2 Installation
|
|
43
|
+
h2 Quick start
|
|
44
|
+
h2 API reference
|
|
45
|
+
h3 `extract(src, language)` → `string[]`
|
|
46
|
+
h3 `rank(query, sigIndex, opts?)` → `Result[]`
|
|
47
|
+
h3 `buildSigIndex(cwd)` → `Map<string, string[]>`
|
|
48
|
+
h3 `scan(sigs, filePath)` → `{ safe: string[], redacted: boolean }`
|
|
49
|
+
h3 `score(cwd)` → `HealthResult`
|
|
50
|
+
h2 Migration from v2.3 and earlier
|
|
51
|
+
h2 v3.0 — Multi-Adapter Architecture (released)
|
|
52
|
+
h2 Zero dependencies
|
|
53
|
+
code-fence bash
|
|
54
|
+
code-fence plain
|
|
55
|
+
code-fence js
|
|
56
|
+
code-fence ---
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### packages/core/index.js
|
|
60
|
+
```
|
|
61
|
+
module.exports = { extract, rank, buildSigIndex, scan, score, adapt }
|
|
62
|
+
function _resolveExtractor(language)
|
|
63
|
+
function extract(src, language) → string[]
|
|
64
|
+
function rank(query, sigIndex, opts) → { file: string, score: nu
|
|
65
|
+
function buildSigIndex(cwd) → Map<string, string[]>
|
|
66
|
+
function scan(sigs, filePath) → { safe: string[], redacte
|
|
67
|
+
function score(cwd) → { * score: number, * grad
|
|
68
|
+
function adapt(context, adapterName, opts = {}) → string
|
|
69
|
+
```
|
|
70
|
+
|
|
36
71
|
### packages/adapters/claude.js
|
|
37
72
|
```
|
|
38
73
|
module.exports = { name, format, outputPath, write }
|
|
@@ -115,38 +150,6 @@ module.exports = { CLI_ENTRY, run }
|
|
|
115
150
|
function run(argv, cwd) → void
|
|
116
151
|
```
|
|
117
152
|
|
|
118
|
-
### packages/core/README.md
|
|
119
|
-
```
|
|
120
|
-
h1 sigmap-core
|
|
121
|
-
h2 Installation
|
|
122
|
-
h2 Quick start
|
|
123
|
-
h2 API reference
|
|
124
|
-
h3 `extract(src, language)` → `string[]`
|
|
125
|
-
h3 `rank(query, sigIndex, opts?)` → `Result[]`
|
|
126
|
-
h3 `buildSigIndex(cwd)` → `Map<string, string[]>`
|
|
127
|
-
h3 `scan(sigs, filePath)` → `{ safe: string[], redacted: boolean }`
|
|
128
|
-
h3 `score(cwd)` → `HealthResult`
|
|
129
|
-
h2 Migration from v2.3 and earlier
|
|
130
|
-
h2 v3.0 — Multi-Adapter Architecture (released)
|
|
131
|
-
h2 Zero dependencies
|
|
132
|
-
code-fence bash
|
|
133
|
-
code-fence plain
|
|
134
|
-
code-fence js
|
|
135
|
-
code-fence ---
|
|
136
|
-
```
|
|
137
|
-
|
|
138
|
-
### packages/core/index.js
|
|
139
|
-
```
|
|
140
|
-
module.exports = { extract, rank, buildSigIndex, scan, score, adapt }
|
|
141
|
-
function _resolveExtractor(language)
|
|
142
|
-
function extract(src, language) → string[]
|
|
143
|
-
function rank(query, sigIndex, opts) → { file: string, score: nu
|
|
144
|
-
function buildSigIndex(cwd) → Map<string, string[]>
|
|
145
|
-
function scan(sigs, filePath) → { safe: string[], redacte
|
|
146
|
-
function score(cwd) → { * score: number, * grad
|
|
147
|
-
function adapt(context, adapterName, opts = {}) → string
|
|
148
|
-
```
|
|
149
|
-
|
|
150
153
|
## src
|
|
151
154
|
|
|
152
155
|
### src/config/loader.js
|
|
@@ -187,6 +190,35 @@ function extractContextFiles(context, cwd)
|
|
|
187
190
|
function judge(response, context, opts = {})
|
|
188
191
|
```
|
|
189
192
|
|
|
193
|
+
### src/learning/weights.js
|
|
194
|
+
```
|
|
195
|
+
module.exports = { BASELINE, DECAY, MAX_MULT, MIN_MULT, weightsPath, clampMultiplier, normalizeFile, loadWeights, saveWeights, updateWeights, boostFiles, penalizeFiles, resetWeights }
|
|
196
|
+
function weightsPath(cwd)
|
|
197
|
+
function clampMultiplier(value)
|
|
198
|
+
function normalizeFile(cwd, filePath)
|
|
199
|
+
function sanitizeWeights(cwd, weights)
|
|
200
|
+
function loadWeights(cwd)
|
|
201
|
+
function saveWeights(cwd, weights)
|
|
202
|
+
function updateWeights(cwd, opts = {})
|
|
203
|
+
function boostFiles(cwd, files, amount = 0.15)
|
|
204
|
+
function penalizeFiles(cwd, files, amount = 0.10)
|
|
205
|
+
function resetWeights(cwd)
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
### src/mcp/handlers.js
|
|
209
|
+
```
|
|
210
|
+
module.exports = { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules, queryContext, getImpact }
|
|
211
|
+
function readContext(args, cwd)
|
|
212
|
+
function searchSignatures(args, cwd)
|
|
213
|
+
function getMap(args, cwd)
|
|
214
|
+
function createCheckpoint(args, cwd)
|
|
215
|
+
function getRouting(args, cwd)
|
|
216
|
+
function explainFile(args, cwd)
|
|
217
|
+
function listModules(args, cwd)
|
|
218
|
+
function queryContext(args, cwd)
|
|
219
|
+
function getImpact(args, cwd)
|
|
220
|
+
```
|
|
221
|
+
|
|
190
222
|
### src/mcp/server.js
|
|
191
223
|
```
|
|
192
224
|
module.exports = { start }
|
|
@@ -604,21 +636,6 @@ module.exports = { score }
|
|
|
604
636
|
function score(cwd) → { * score: number, * grad
|
|
605
637
|
```
|
|
606
638
|
|
|
607
|
-
### src/learning/weights.js
|
|
608
|
-
```
|
|
609
|
-
module.exports = { BASELINE, DECAY, MAX_MULT, MIN_MULT, weightsPath, clampMultiplier, normalizeFile, loadWeights, saveWeights, updateWeights, boostFiles, penalizeFiles, resetWeights }
|
|
610
|
-
function weightsPath(cwd)
|
|
611
|
-
function clampMultiplier(value)
|
|
612
|
-
function normalizeFile(cwd, filePath)
|
|
613
|
-
function sanitizeWeights(cwd, weights)
|
|
614
|
-
function loadWeights(cwd)
|
|
615
|
-
function saveWeights(cwd, weights)
|
|
616
|
-
function updateWeights(cwd, opts = {})
|
|
617
|
-
function boostFiles(cwd, files, amount = 0.15)
|
|
618
|
-
function penalizeFiles(cwd, files, amount = 0.10)
|
|
619
|
-
function resetWeights(cwd)
|
|
620
|
-
```
|
|
621
|
-
|
|
622
639
|
### src/map/class-hierarchy.js
|
|
623
640
|
```
|
|
624
641
|
module.exports = { analyze }
|
|
@@ -641,20 +658,6 @@ function shouldSkipFile(rel)
|
|
|
641
658
|
function analyze(files, cwd)
|
|
642
659
|
```
|
|
643
660
|
|
|
644
|
-
### src/mcp/handlers.js
|
|
645
|
-
```
|
|
646
|
-
module.exports = { readContext, searchSignatures, getMap, createCheckpoint, getRouting, explainFile, listModules, queryContext, getImpact }
|
|
647
|
-
function readContext(args, cwd)
|
|
648
|
-
function searchSignatures(args, cwd)
|
|
649
|
-
function getMap(args, cwd)
|
|
650
|
-
function createCheckpoint(args, cwd)
|
|
651
|
-
function getRouting(args, cwd)
|
|
652
|
-
function explainFile(args, cwd)
|
|
653
|
-
function listModules(args, cwd)
|
|
654
|
-
function queryContext(args, cwd)
|
|
655
|
-
function getImpact(args, cwd)
|
|
656
|
-
```
|
|
657
|
-
|
|
658
661
|
### src/mcp/tools.js
|
|
659
662
|
```
|
|
660
663
|
module.exports = { TOOLS }
|
package/CHANGELOG.md
CHANGED
|
@@ -10,6 +10,20 @@ Format: [Semantic Versioning](https://semver.org/)
|
|
|
10
10
|
|
|
11
11
|
---
|
|
12
12
|
|
|
13
|
+
## [5.3.0] — 2026-04-17
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
- **MCP auto-wire: Windsurf** — `sigmap --setup` now registers the MCP server in `.windsurf/mcp.json` (project-level) and `~/.codeium/windsurf/mcp_config.json` (global) using the standard `mcpServers` shape.
|
|
18
|
+
- **MCP auto-wire: Zed** — `sigmap --setup` now registers a context server in `~/.config/zed/settings.json` using Zed's `context_servers` shape (`command.path` / `command.args`).
|
|
19
|
+
- **Updated `--setup` snippet** — help output now prints manual config snippets for all four tools: Claude, Cursor, Windsurf, and Zed.
|
|
20
|
+
|
|
21
|
+
### Changed
|
|
22
|
+
|
|
23
|
+
- `registerMcp()` skips each target when the file does not exist and never overwrites an already-registered `sigmap` entry (idempotent).
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
13
27
|
## [5.2.0] — 2026-04-17
|
|
14
28
|
|
|
15
29
|
### Added
|
package/README.md
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
</div>
|
|
13
13
|
|
|
14
14
|
<div align="center">
|
|
15
|
-
<img src="docs/impact-banner.svg" alt="SigMap —
|
|
15
|
+
<img src="docs/impact-banner.svg" alt="SigMap — grounded AI coding context with fewer prompts and smaller context windows" width="760" />
|
|
16
16
|
</div>
|
|
17
17
|
|
|
18
18
|
```sh
|
|
@@ -21,11 +21,28 @@ npx sigmap # 10 seconds. zero config. your AI never reads the wrong file again
|
|
|
21
21
|
|
|
22
22
|
**What you get in ~10 seconds**
|
|
23
23
|
- A compact signature map of your codebase
|
|
24
|
-
- The right file in context far more often (
|
|
25
|
-
- Fewer retries (1.
|
|
24
|
+
- The right file in context far more often (78.9% hit@5 vs 13.6% random)
|
|
25
|
+
- Fewer retries (1.69 vs 2.84 prompts per task)
|
|
26
26
|
- Far smaller context (~2K–4K tokens instead of ~80K)
|
|
27
27
|
|
|
28
|
-
> Latest: **
|
|
28
|
+
> Latest: **v5.3.0** — Learning engine + workflow-first release. Use `ask`, `validate`, `judge`, `learn`, `weights`, `compare`, and `share` on top of the core signature pipeline.
|
|
29
|
+
|
|
30
|
+
**What is new in v5.2**
|
|
31
|
+
- `sigmap ask` creates task-focused context in one step
|
|
32
|
+
- `sigmap validate` checks config health and query coverage
|
|
33
|
+
- `sigmap judge` scores groundedness against the supplied context
|
|
34
|
+
- `sigmap learn` and `sigmap weights` add safe local-only ranking feedback
|
|
35
|
+
- `node scripts/run-benchmark-matrix.mjs --save --skip-clone` now writes an HTML benchmark dashboard
|
|
36
|
+
|
|
37
|
+
**Daily workflow**
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
npx sigmap
|
|
41
|
+
sigmap ask "explain the auth flow"
|
|
42
|
+
sigmap validate --query "auth login token"
|
|
43
|
+
sigmap judge --response response.txt --context .context/query-context.md
|
|
44
|
+
sigmap weights
|
|
45
|
+
```
|
|
29
46
|
|
|
30
47
|
<div align="center">
|
|
31
48
|
<img src="demo.gif" alt="SigMap demo — reducing 80K tokens to 4K in under 10 seconds" width="760" />
|
|
@@ -61,11 +78,11 @@ npx sigmap # 10 seconds. zero config. your AI never reads the wrong file again
|
|
|
61
78
|
|
|
62
79
|
| | Without SigMap | With SigMap |
|
|
63
80
|
|---|:---:|:---:|
|
|
64
|
-
| Task success | 10% | **
|
|
65
|
-
| Prompts per task | 2.84 | **1.
|
|
81
|
+
| Task success | 10% | **52.2%** |
|
|
82
|
+
| Prompts per task | 2.84 | **1.69** |
|
|
66
83
|
| Tokens per session | ~80,000 | **~2,000** |
|
|
67
|
-
| Right file found | 13.6% | **
|
|
68
|
-
|
|
|
84
|
+
| Right file found | 13.6% | **78.9%** |
|
|
85
|
+
| Hidden-symbol risk | 74.7% | **context surfaced locally** |
|
|
69
86
|
|
|
70
87
|
Measured on 90 coding tasks across 18 real public repos. Full methodology and raw benchmark pages are linked below.
|
|
71
88
|
|
|
@@ -82,7 +99,7 @@ Measured on 90 coding tasks across 18 real public repos. Full methodology and ra
|
|
|
82
99
|
| [Standalone binaries](docs/readmes/binaries.md) | macOS, Linux, Windows — no Node required |
|
|
83
100
|
| [VS Code extension](#-vs-code-extension) | Status bar, stale alerts, commands |
|
|
84
101
|
| [JetBrains plugin](#-jetbrains-plugin) | IntelliJ IDEA, WebStorm, PyCharm support |
|
|
85
|
-
| [Languages supported](#-languages-supported) |
|
|
102
|
+
| [Languages supported](#-languages-supported) | 29 languages |
|
|
86
103
|
| [Context strategies](#-context-strategies) | full / per-module / hot-cold |
|
|
87
104
|
| [MCP server](#-mcp-server) | 8 on-demand tools |
|
|
88
105
|
| [CLI reference](#-cli-reference) | All flags |
|
|
@@ -105,7 +122,7 @@ SigMap scans your source files and extracts only the **function and class signat
|
|
|
105
122
|
Your codebase
|
|
106
123
|
│
|
|
107
124
|
▼
|
|
108
|
-
sigmap ─────────► extracts signatures from
|
|
125
|
+
sigmap ─────────► extracts signatures from 29 languages
|
|
109
126
|
│
|
|
110
127
|
▼
|
|
111
128
|
.github/copilot-instructions.md ◄── auto-read by Copilot / Claude / Cursor
|
|
@@ -126,7 +143,7 @@ AI agent session starts with full context
|
|
|
126
143
|
| **SigMap signatures** | **~4,000** | **95%** |
|
|
127
144
|
| SigMap + MCP (`hot-cold`) | ~200 | **99.75%** |
|
|
128
145
|
|
|
129
|
-
> **
|
|
146
|
+
> **98.1% fewer tokens in the latest saved benchmark snapshot.**
|
|
130
147
|
|
|
131
148
|
### Benchmark: real-world repos
|
|
132
149
|
|
|
@@ -153,7 +170,7 @@ Reproduced with `node scripts/run-benchmark.mjs` on public repos:
|
|
|
153
170
|
| fastify | JavaScript | 54.4K | 2.6K | **95.3%** |
|
|
154
171
|
| fastapi | Python | 178.4K | 5.2K | **97.1%** |
|
|
155
172
|
|
|
156
|
-
**Average: 97.6% reduction across 18 repos (16 languages).** See [`benchmarks/reports/token-reduction.md`](benchmarks/reports/token-reduction.md) or reproduce with `node scripts/run-benchmark.mjs`.
|
|
173
|
+
**Average: 97.6% reduction across 18 repos (16 languages).** See [`benchmarks/reports/token-reduction.md`](benchmarks/reports/token-reduction.md), open `benchmarks/reports/benchmark-report.html` after a matrix run, or reproduce with `node scripts/run-benchmark.mjs`.
|
|
157
174
|
|
|
158
175
|
---
|
|
159
176
|
|
|
@@ -503,12 +520,12 @@ Compatible with **IntelliJ IDEA 2024.1+** (Community & Ultimate), **WebStorm**,
|
|
|
503
520
|
|
|
504
521
|
## 🌐 Languages supported
|
|
505
522
|
|
|
506
|
-
>
|
|
523
|
+
> 29 languages and formats. All implemented with zero external dependencies — pure regex + Node built-ins.
|
|
507
524
|
>
|
|
508
525
|
> Also includes lightweight config/doc extraction for `.toml`, `.properties`, `.xml`, and `.md` to improve real-repo coverage beyond source-code files.
|
|
509
526
|
|
|
510
527
|
<details>
|
|
511
|
-
<summary><strong>Show all
|
|
528
|
+
<summary><strong>Show all 29 languages</strong></summary>
|
|
512
529
|
|
|
513
530
|
| Language | Extensions | Extracts |
|
|
514
531
|
|---|---|---|
|
|
@@ -737,7 +754,7 @@ Copy `gen-context.config.json.example` to `gen-context.config.json`:
|
|
|
737
754
|
- **`secretScan`** — redact secrets (AWS keys, tokens, etc.) from output
|
|
738
755
|
- **`strategy`** — output mode: `full` (default) | `per-module` | `hot-cold`
|
|
739
756
|
|
|
740
|
-
**Token budget (
|
|
757
|
+
**Token budget (auto-scaling):**
|
|
741
758
|
|
|
742
759
|
| Key | Default | Description |
|
|
743
760
|
|---|---|---|
|
|
@@ -788,13 +805,13 @@ If `output` is omitted, the default `.github/copilot-instructions.md` is used.
|
|
|
788
805
|
|
|
789
806
|
## 📊 Observability
|
|
790
807
|
|
|
791
|
-
### Coverage score
|
|
808
|
+
### Coverage score
|
|
792
809
|
|
|
793
810
|
Every run now prints a coverage line alongside token reduction:
|
|
794
811
|
|
|
795
812
|
```
|
|
796
813
|
───────────────────────────────────────────
|
|
797
|
-
SigMap
|
|
814
|
+
SigMap v5.3.0
|
|
798
815
|
Files scanned : 76
|
|
799
816
|
Symbols found : 332
|
|
800
817
|
Token reduction: 94% (65,227 → 4,103)
|
|
@@ -813,7 +830,7 @@ sigmap --report
|
|
|
813
830
|
|
|
814
831
|
```
|
|
815
832
|
[sigmap] report:
|
|
816
|
-
version :
|
|
833
|
+
version : 5.3.0
|
|
817
834
|
files processed : 76
|
|
818
835
|
reduction : 93.7%
|
|
819
836
|
coverage : A (97%) — 76 of 78 source files included
|
|
@@ -857,7 +874,7 @@ sigmap --health --json
|
|
|
857
874
|
Every output file now carries a metadata line so you can inspect freshness at a glance:
|
|
858
875
|
|
|
859
876
|
```
|
|
860
|
-
<!-- sigmap: version=
|
|
877
|
+
<!-- sigmap: version=5.3.0 confidence=HIGH coverage=97% dropped=2 commit=8540612 -->
|
|
861
878
|
```
|
|
862
879
|
|
|
863
880
|
### Diff risk score
|
package/gen-context.js
CHANGED
|
@@ -4853,7 +4853,7 @@ __factories["./src/mcp/server"] = function(module, exports) {
|
|
|
4853
4853
|
|
|
4854
4854
|
const SERVER_INFO = {
|
|
4855
4855
|
name: 'sigmap',
|
|
4856
|
-
version: '5.
|
|
4856
|
+
version: '5.3.0',
|
|
4857
4857
|
description: 'SigMap MCP server — code signatures on demand',
|
|
4858
4858
|
};
|
|
4859
4859
|
|
|
@@ -6571,7 +6571,7 @@ const path = require('path');
|
|
|
6571
6571
|
const os = require('os');
|
|
6572
6572
|
const { execSync } = require('child_process');
|
|
6573
6573
|
|
|
6574
|
-
const VERSION = '5.
|
|
6574
|
+
const VERSION = '5.3.0';
|
|
6575
6575
|
const MARKER = '\n\n## Auto-generated signatures\n<!-- Updated by gen-context.js -->\n';
|
|
6576
6576
|
|
|
6577
6577
|
function requireSourceOrBundled(key) {
|
|
@@ -8241,9 +8241,13 @@ function registerMcp(cwd, scriptPath) {
|
|
|
8241
8241
|
args: [path.resolve(scriptPath), '--mcp'],
|
|
8242
8242
|
};
|
|
8243
8243
|
|
|
8244
|
+
// mcpServers shape: Claude (.claude/settings.json), Cursor (.cursor/mcp.json),
|
|
8245
|
+
// Windsurf project (.windsurf/mcp.json) and global (~/.codeium/windsurf/mcp_config.json)
|
|
8244
8246
|
const targets = [
|
|
8245
8247
|
path.join(cwd, '.claude', 'settings.json'),
|
|
8246
8248
|
path.join(cwd, '.cursor', 'mcp.json'),
|
|
8249
|
+
path.join(cwd, '.windsurf', 'mcp.json'),
|
|
8250
|
+
path.join(os.homedir(), '.codeium', 'windsurf', 'mcp_config.json'),
|
|
8247
8251
|
];
|
|
8248
8252
|
|
|
8249
8253
|
for (const settingsPath of targets) {
|
|
@@ -8255,15 +8259,37 @@ function registerMcp(cwd, scriptPath) {
|
|
|
8255
8259
|
if (settings.mcpServers['sigmap']) continue; // already registered
|
|
8256
8260
|
settings.mcpServers['sigmap'] = serverEntry;
|
|
8257
8261
|
fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + '\n');
|
|
8258
|
-
console.warn(`[sigmap] registered MCP server in ${path.relative(cwd, settingsPath)}`);
|
|
8262
|
+
console.warn(`[sigmap] registered MCP server in ${settingsPath.startsWith(os.homedir()) ? '~' + settingsPath.slice(os.homedir().length) : path.relative(cwd, settingsPath)}`);
|
|
8259
8263
|
} catch (err) {
|
|
8260
8264
|
console.warn(`[sigmap] could not update ${path.relative(cwd, settingsPath)}: ${err.message}`);
|
|
8261
8265
|
}
|
|
8262
8266
|
}
|
|
8263
8267
|
|
|
8264
|
-
//
|
|
8265
|
-
|
|
8266
|
-
|
|
8268
|
+
// Zed uses context_servers (different shape from mcpServers)
|
|
8269
|
+
const zedSettingsPath = path.join(os.homedir(), '.config', 'zed', 'settings.json');
|
|
8270
|
+
if (fs.existsSync(zedSettingsPath)) {
|
|
8271
|
+
try {
|
|
8272
|
+
const raw = fs.readFileSync(zedSettingsPath, 'utf8');
|
|
8273
|
+
const settings = JSON.parse(raw);
|
|
8274
|
+
if (!settings.context_servers) settings.context_servers = {};
|
|
8275
|
+
if (!settings.context_servers['sigmap']) {
|
|
8276
|
+
settings.context_servers['sigmap'] = {
|
|
8277
|
+
command: { path: 'node', args: [path.resolve(scriptPath), '--mcp'] },
|
|
8278
|
+
};
|
|
8279
|
+
fs.writeFileSync(zedSettingsPath, JSON.stringify(settings, null, 2) + '\n');
|
|
8280
|
+
console.warn('[sigmap] registered context server in ~/.config/zed/settings.json');
|
|
8281
|
+
}
|
|
8282
|
+
} catch (err) {
|
|
8283
|
+
console.warn(`[sigmap] could not update ~/.config/zed/settings.json: ${err.message}`);
|
|
8284
|
+
}
|
|
8285
|
+
}
|
|
8286
|
+
|
|
8287
|
+
// Print manual snippets for all 4 tools
|
|
8288
|
+
console.warn('[sigmap] MCP / context server config snippets:');
|
|
8289
|
+
console.warn(' Claude / Cursor / Windsurf (.claude/settings.json | .cursor/mcp.json | .windsurf/mcp.json):');
|
|
8290
|
+
console.warn(JSON.stringify({ mcpServers: { sigmap: serverEntry } }, null, 2));
|
|
8291
|
+
console.warn(' Zed (~/.config/zed/settings.json):');
|
|
8292
|
+
console.warn(JSON.stringify({ context_servers: { sigmap: { command: { path: 'node', args: [path.resolve(scriptPath), '--mcp'] } } } }, null, 2));
|
|
8267
8293
|
}
|
|
8268
8294
|
|
|
8269
8295
|
// ---------------------------------------------------------------------------
|
package/package.json
CHANGED
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
|
|
6
|
+
function escapeHtml(value) {
|
|
7
|
+
return String(value == null ? '' : value)
|
|
8
|
+
.replace(/&/g, '&')
|
|
9
|
+
.replace(/</g, '<')
|
|
10
|
+
.replace(/>/g, '>')
|
|
11
|
+
.replace(/"/g, '"');
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function formatInt(value) {
|
|
15
|
+
const n = Number(value);
|
|
16
|
+
if (!Number.isFinite(n)) return 'n/a';
|
|
17
|
+
return Math.round(n).toLocaleString('en-US');
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function formatCompact(value) {
|
|
21
|
+
const n = Number(value);
|
|
22
|
+
if (!Number.isFinite(n)) return 'n/a';
|
|
23
|
+
if (Math.abs(n) >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
|
|
24
|
+
if (Math.abs(n) >= 1_000) return `${(n / 1_000).toFixed(1)}K`;
|
|
25
|
+
return String(Math.round(n));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function formatPct(value, digits = 1) {
|
|
29
|
+
const n = Number(value);
|
|
30
|
+
if (!Number.isFinite(n)) return 'n/a';
|
|
31
|
+
return `${n.toFixed(digits)}%`;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function formatMaybePct(value, digits = 1) {
|
|
35
|
+
const n = Number(value);
|
|
36
|
+
if (!Number.isFinite(n)) return 'n/a';
|
|
37
|
+
return `${n.toFixed(digits)}%`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function formatRatio(value, digits = 1) {
|
|
41
|
+
const n = Number(value);
|
|
42
|
+
if (!Number.isFinite(n)) return 'n/a';
|
|
43
|
+
return `${n.toFixed(digits)}x`;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function formatMoney(value) {
|
|
47
|
+
const n = Number(value);
|
|
48
|
+
if (!Number.isFinite(n)) return 'n/a';
|
|
49
|
+
return `$${n.toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function durationLabel(ms) {
|
|
53
|
+
const n = Number(ms);
|
|
54
|
+
if (!Number.isFinite(n)) return 'n/a';
|
|
55
|
+
const sec = n / 1000;
|
|
56
|
+
if (sec < 60) return `${sec.toFixed(1)}s`;
|
|
57
|
+
const min = Math.floor(sec / 60);
|
|
58
|
+
const rem = sec - (min * 60);
|
|
59
|
+
return `${min}m ${rem.toFixed(1)}s`;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function maxOrZero(values) {
|
|
63
|
+
if (!Array.isArray(values) || values.length === 0) return 0;
|
|
64
|
+
return Math.max(...values.map((v) => (Number.isFinite(v) ? v : 0)));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function readJson(filePath) {
|
|
68
|
+
try {
|
|
69
|
+
if (!fs.existsSync(filePath)) return null;
|
|
70
|
+
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
71
|
+
} catch (_) {
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function loadBenchmarkReports(cwd) {
|
|
77
|
+
const reportsDir = path.join(cwd, 'benchmarks', 'reports');
|
|
78
|
+
return {
|
|
79
|
+
reportsDir,
|
|
80
|
+
token: readJson(path.join(reportsDir, 'token-reduction.json')),
|
|
81
|
+
retrieval: readJson(path.join(reportsDir, 'retrieval.json')),
|
|
82
|
+
quality: readJson(path.join(reportsDir, 'quality.json')),
|
|
83
|
+
task: readJson(path.join(reportsDir, 'task-benchmark.json')),
|
|
84
|
+
matrix: readJson(path.join(reportsDir, 'benchmark-matrix.json')),
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function buildRetrievalSummary(retrieval) {
|
|
89
|
+
if (!retrieval || !Array.isArray(retrieval.repos) || retrieval.repos.length === 0) return null;
|
|
90
|
+
let totalTasks = 0;
|
|
91
|
+
let weightedHit = 0;
|
|
92
|
+
let weightedRand = 0;
|
|
93
|
+
let correct = 0;
|
|
94
|
+
let partial = 0;
|
|
95
|
+
let wrong = 0;
|
|
96
|
+
let repoCount = 0;
|
|
97
|
+
|
|
98
|
+
for (const repo of retrieval.repos) {
|
|
99
|
+
const tasks = Number(repo.tasks) || 0;
|
|
100
|
+
repoCount++;
|
|
101
|
+
totalTasks += tasks;
|
|
102
|
+
weightedHit += (Number(repo.hitAt5) || 0) * tasks;
|
|
103
|
+
weightedRand += (Number(repo.randomBaseline) || 0) * tasks;
|
|
104
|
+
correct += Number(repo.tiers && repo.tiers.correct) || 0;
|
|
105
|
+
partial += Number(repo.tiers && repo.tiers.partial) || 0;
|
|
106
|
+
wrong += Number(repo.tiers && repo.tiers.wrong) || 0;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const hitAt5 = totalTasks > 0 ? (weightedHit / totalTasks) * 100 : null;
|
|
110
|
+
const randomBaseline = totalTasks > 0 ? (weightedRand / totalTasks) * 100 : null;
|
|
111
|
+
const lift = hitAt5 && randomBaseline ? hitAt5 / randomBaseline : null;
|
|
112
|
+
|
|
113
|
+
return {
|
|
114
|
+
repoCount,
|
|
115
|
+
totalTasks,
|
|
116
|
+
hitAt5,
|
|
117
|
+
randomBaseline,
|
|
118
|
+
lift,
|
|
119
|
+
correct,
|
|
120
|
+
partial,
|
|
121
|
+
wrong,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function buildBenchmarkSummary(reports, matrixSummary) {
|
|
126
|
+
const missing = [];
|
|
127
|
+
if (!reports.token) missing.push('token-reduction.json');
|
|
128
|
+
if (!reports.retrieval) missing.push('retrieval.json');
|
|
129
|
+
if (!reports.quality) missing.push('quality.json');
|
|
130
|
+
if (!reports.task) missing.push('task-benchmark.json');
|
|
131
|
+
|
|
132
|
+
const retrievalSummary = buildRetrievalSummary(reports.retrieval);
|
|
133
|
+
const qualitySummary = reports.quality && reports.quality.summary ? reports.quality.summary : null;
|
|
134
|
+
const tokenSummary = reports.token && reports.token.summary ? reports.token.summary : null;
|
|
135
|
+
const taskSummary = reports.task && reports.task.summary ? reports.task.summary : null;
|
|
136
|
+
const matrix = matrixSummary || reports.matrix || null;
|
|
137
|
+
|
|
138
|
+
const generatedCandidates = [
|
|
139
|
+
matrix && matrix.generated,
|
|
140
|
+
reports.task && reports.task.generated,
|
|
141
|
+
reports.retrieval && reports.retrieval.generated,
|
|
142
|
+
reports.quality && reports.quality.timestamp,
|
|
143
|
+
reports.token && reports.token.timestamp,
|
|
144
|
+
].filter(Boolean);
|
|
145
|
+
const generatedAt = generatedCandidates
|
|
146
|
+
.map((value) => ({ value, time: Date.parse(value) }))
|
|
147
|
+
.filter((item) => Number.isFinite(item.time))
|
|
148
|
+
.sort((a, b) => b.time - a.time)[0];
|
|
149
|
+
|
|
150
|
+
return {
|
|
151
|
+
generatedAt: (generatedAt && generatedAt.value) || generatedCandidates[0] || new Date().toISOString(),
|
|
152
|
+
missing,
|
|
153
|
+
tokenSummary,
|
|
154
|
+
retrievalSummary,
|
|
155
|
+
qualitySummary,
|
|
156
|
+
taskSummary,
|
|
157
|
+
matrix,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function renderCard(label, value, hint, tone) {
|
|
162
|
+
const toneClass = tone ? ` ${tone}` : '';
|
|
163
|
+
return [
|
|
164
|
+
`<article class="card${toneClass}">`,
|
|
165
|
+
`<div class="label">${escapeHtml(label)}</div>`,
|
|
166
|
+
`<div class="value">${escapeHtml(value)}</div>`,
|
|
167
|
+
`<div class="hint">${escapeHtml(hint || '')}</div>`,
|
|
168
|
+
'</article>',
|
|
169
|
+
].join('');
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function renderProgress(label, value, max, suffix) {
|
|
173
|
+
const safeValue = Number.isFinite(value) ? value : 0;
|
|
174
|
+
const safeMax = Math.max(1, Number.isFinite(max) ? max : 1);
|
|
175
|
+
const width = Math.max(2, Math.min(100, (safeValue / safeMax) * 100));
|
|
176
|
+
return [
|
|
177
|
+
'<div class="progress-row">',
|
|
178
|
+
`<div class="progress-label">${escapeHtml(label)}</div>`,
|
|
179
|
+
'<div class="progress-bar"><span style="width:',
|
|
180
|
+
String(width.toFixed(1)),
|
|
181
|
+
'%"></span></div>',
|
|
182
|
+
`<div class="progress-value">${escapeHtml(`${safeValue}${suffix || ''}`)}</div>`,
|
|
183
|
+
'</div>',
|
|
184
|
+
].join('');
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function renderMatrixSection(matrix) {
|
|
188
|
+
if (!matrix || !Array.isArray(matrix.steps) || matrix.steps.length === 0) return '';
|
|
189
|
+
const rows = matrix.steps.map((step) => {
|
|
190
|
+
const status = step.ok ? 'ok' : 'fail';
|
|
191
|
+
return [
|
|
192
|
+
'<tr>',
|
|
193
|
+
`<td>${escapeHtml(step.name)}</td>`,
|
|
194
|
+
`<td><span class="badge ${status}">${escapeHtml(step.ok ? 'ok' : `exit ${step.status}`)}</span></td>`,
|
|
195
|
+
`<td>${escapeHtml(durationLabel(step.durationMs))}</td>`,
|
|
196
|
+
`<td><code>${escapeHtml(['node', step.script].concat(step.args || []).join(' '))}</code></td>`,
|
|
197
|
+
'</tr>',
|
|
198
|
+
].join('');
|
|
199
|
+
}).join('');
|
|
200
|
+
|
|
201
|
+
return [
|
|
202
|
+
'<section>',
|
|
203
|
+
'<h2>Run matrix</h2>',
|
|
204
|
+
'<p class="section-copy">This shows which benchmark jobs ran, whether they succeeded, and how long each step took.</p>',
|
|
205
|
+
'<table>',
|
|
206
|
+
'<thead><tr><th>Step</th><th>Status</th><th>Duration</th><th>Command</th></tr></thead>',
|
|
207
|
+
`<tbody>${rows}</tbody>`,
|
|
208
|
+
'</table>',
|
|
209
|
+
'</section>',
|
|
210
|
+
].join('');
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function renderTokenSection(token) {
|
|
214
|
+
if (!token || !Array.isArray(token.repos) || token.repos.length === 0) return '';
|
|
215
|
+
const rows = token.repos
|
|
216
|
+
.slice()
|
|
217
|
+
.sort((a, b) => (b.reductionPct || 0) - (a.reductionPct || 0))
|
|
218
|
+
.map((repo) => [
|
|
219
|
+
'<tr>',
|
|
220
|
+
`<td>${escapeHtml(repo.repo)}</td>`,
|
|
221
|
+
`<td>${escapeHtml(repo.language || 'n/a')}</td>`,
|
|
222
|
+
`<td>${escapeHtml(formatCompact(repo.rawTokens))}</td>`,
|
|
223
|
+
`<td>${escapeHtml(formatCompact(repo.finalTokens))}</td>`,
|
|
224
|
+
`<td>${escapeHtml(formatMaybePct(repo.reductionPct, 1))}</td>`,
|
|
225
|
+
'</tr>',
|
|
226
|
+
].join(''))
|
|
227
|
+
.join('');
|
|
228
|
+
|
|
229
|
+
return [
|
|
230
|
+
'<section>',
|
|
231
|
+
'<h2>Token reduction</h2>',
|
|
232
|
+
'<p class="section-copy">Raw repository tokens versus SigMap output size across the benchmark repos.</p>',
|
|
233
|
+
'<table>',
|
|
234
|
+
'<thead><tr><th>Repo</th><th>Language</th><th>Raw tokens</th><th>Final tokens</th><th>Reduction</th></tr></thead>',
|
|
235
|
+
`<tbody>${rows}</tbody>`,
|
|
236
|
+
'</table>',
|
|
237
|
+
'</section>',
|
|
238
|
+
].join('');
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function renderRetrievalSection(retrieval) {
|
|
242
|
+
if (!retrieval || !Array.isArray(retrieval.repos) || retrieval.repos.length === 0) return '';
|
|
243
|
+
const rows = retrieval.repos.map((repo) => {
|
|
244
|
+
const lift = repo.randomBaseline > 0 ? (repo.hitAt5 / repo.randomBaseline) : null;
|
|
245
|
+
return [
|
|
246
|
+
'<tr>',
|
|
247
|
+
`<td>${escapeHtml(repo.repo)}</td>`,
|
|
248
|
+
`<td>${escapeHtml(formatMaybePct((repo.randomBaseline || 0) * 100, 1))}</td>`,
|
|
249
|
+
`<td>${escapeHtml(formatMaybePct((repo.hitAt5 || 0) * 100, 1))}</td>`,
|
|
250
|
+
`<td>${escapeHtml(formatRatio(lift, 1))}</td>`,
|
|
251
|
+
`<td>${escapeHtml(String((repo.tiers && repo.tiers.correct) || 0))}</td>`,
|
|
252
|
+
`<td>${escapeHtml(String((repo.tiers && repo.tiers.partial) || 0))}</td>`,
|
|
253
|
+
`<td>${escapeHtml(String((repo.tiers && repo.tiers.wrong) || 0))}</td>`,
|
|
254
|
+
'</tr>',
|
|
255
|
+
].join('');
|
|
256
|
+
}).join('');
|
|
257
|
+
|
|
258
|
+
return [
|
|
259
|
+
'<section>',
|
|
260
|
+
'<h2>Retrieval quality</h2>',
|
|
261
|
+
'<p class="section-copy">Hit@5 performance against the random baseline, plus the quality-tier mix that drives the task benchmark.</p>',
|
|
262
|
+
'<table>',
|
|
263
|
+
'<thead><tr><th>Repo</th><th>Random hit@5</th><th>SigMap hit@5</th><th>Lift</th><th>Correct</th><th>Partial</th><th>Wrong</th></tr></thead>',
|
|
264
|
+
`<tbody>${rows}</tbody>`,
|
|
265
|
+
'</table>',
|
|
266
|
+
'</section>',
|
|
267
|
+
].join('');
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function renderQualitySection(quality) {
|
|
271
|
+
if (!quality || !Array.isArray(quality.repos) || quality.repos.length === 0) return '';
|
|
272
|
+
const rows = quality.repos.map((repo) => {
|
|
273
|
+
const overflow = (repo.rawTokens || 0) > 128000 ? 'overflow' : 'fits';
|
|
274
|
+
return [
|
|
275
|
+
'<tr>',
|
|
276
|
+
`<td>${escapeHtml(repo.repo)}</td>`,
|
|
277
|
+
`<td>${escapeHtml(formatInt(repo.groundedSymbols))}</td>`,
|
|
278
|
+
`<td>${escapeHtml(formatInt(repo.darkSymbols))}</td>`,
|
|
279
|
+
`<td>${escapeHtml(formatMaybePct(repo.groundingPct, 0))}</td>`,
|
|
280
|
+
`<td>${escapeHtml(String(repo.filesHiddenRaw || 0))}</td>`,
|
|
281
|
+
`<td><span class="badge ${overflow === 'overflow' ? 'warn' : 'ok'}">${escapeHtml(overflow)}</span></td>`,
|
|
282
|
+
'</tr>',
|
|
283
|
+
].join('');
|
|
284
|
+
}).join('');
|
|
285
|
+
|
|
286
|
+
return [
|
|
287
|
+
'<section>',
|
|
288
|
+
'<h2>Quality and hallucination surface</h2>',
|
|
289
|
+
'<p class="section-copy">How much code stays visible to the model, plus the overflow and dark-symbol risk by repo.</p>',
|
|
290
|
+
'<table>',
|
|
291
|
+
'<thead><tr><th>Repo</th><th>Grounded symbols</th><th>Dark symbols</th><th>Grounding</th><th>Hidden files (raw)</th><th>GPT-4o 128K</th></tr></thead>',
|
|
292
|
+
`<tbody>${rows}</tbody>`,
|
|
293
|
+
'</table>',
|
|
294
|
+
'</section>',
|
|
295
|
+
].join('');
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function renderTaskSection(task) {
|
|
299
|
+
if (!task || !Array.isArray(task.repos) || task.repos.length === 0 || !task.summary) return '';
|
|
300
|
+
const summary = task.summary;
|
|
301
|
+
const maxReduction = maxOrZero(task.repos.map((repo) => Number(repo.reductionPct) || 0));
|
|
302
|
+
const repoBars = task.repos
|
|
303
|
+
.slice()
|
|
304
|
+
.sort((a, b) => (b.reductionPct || 0) - (a.reductionPct || 0))
|
|
305
|
+
.slice(0, 10)
|
|
306
|
+
.map((repo) => renderProgress(repo.repo, Number(repo.reductionPct) || 0, maxReduction, '%'))
|
|
307
|
+
.join('');
|
|
308
|
+
|
|
309
|
+
return [
|
|
310
|
+
'<section>',
|
|
311
|
+
'<h2>Task benchmark</h2>',
|
|
312
|
+
'<p class="section-copy">A prompt-reduction proxy derived from retrieval quality tiers. Lower prompts means the right file surfaces sooner.</p>',
|
|
313
|
+
'<div class="split">',
|
|
314
|
+
'<div class="panel">',
|
|
315
|
+
'<h3>Answer quality tiers</h3>',
|
|
316
|
+
renderProgress('Correct', Number(summary.correctPct) || 0, 100, '%'),
|
|
317
|
+
renderProgress('Partial', Number(summary.partialPct) || 0, 100, '%'),
|
|
318
|
+
renderProgress('Wrong', Number(summary.wrongPct) || 0, 100, '%'),
|
|
319
|
+
'</div>',
|
|
320
|
+
'<div class="panel">',
|
|
321
|
+
'<h3>Best prompt reduction by repo</h3>',
|
|
322
|
+
repoBars,
|
|
323
|
+
'</div>',
|
|
324
|
+
'</div>',
|
|
325
|
+
'</section>',
|
|
326
|
+
].join('');
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function generateBenchmarkReportHtml(reports, opts = {}) {
|
|
330
|
+
const summary = buildBenchmarkSummary(reports, opts.matrixSummary);
|
|
331
|
+
const cards = [];
|
|
332
|
+
cards.push(renderCard(
|
|
333
|
+
'Token reduction',
|
|
334
|
+
summary.tokenSummary ? formatPct(summary.tokenSummary.overallReductionPct, 1) : 'n/a',
|
|
335
|
+
summary.tokenSummary ? `${formatInt(summary.tokenSummary.repoCount)} repos • ${formatCompact(summary.tokenSummary.totalRawTokens)} raw -> ${formatCompact(summary.tokenSummary.totalFinalTokens)} final` : 'token-reduction.json missing',
|
|
336
|
+
'cool'
|
|
337
|
+
));
|
|
338
|
+
cards.push(renderCard(
|
|
339
|
+
'Retrieval hit@5',
|
|
340
|
+
summary.retrievalSummary ? formatPct(summary.retrievalSummary.hitAt5, 1) : 'n/a',
|
|
341
|
+
summary.retrievalSummary ? `${formatPct(summary.retrievalSummary.randomBaseline, 1)} random baseline • ${formatRatio(summary.retrievalSummary.lift, 1)} lift` : 'retrieval.json missing',
|
|
342
|
+
'warm'
|
|
343
|
+
));
|
|
344
|
+
cards.push(renderCard(
|
|
345
|
+
'Prompt reduction',
|
|
346
|
+
summary.taskSummary ? formatPct(summary.taskSummary.avgReductionPct, 0) : 'n/a',
|
|
347
|
+
summary.taskSummary ? `${summary.taskSummary.avgPromptsWithout} -> ${summary.taskSummary.avgPromptsWith} prompts • ${formatInt(summary.taskSummary.totalTasks)} tasks` : 'task-benchmark.json missing',
|
|
348
|
+
'neutral'
|
|
349
|
+
));
|
|
350
|
+
cards.push(renderCard(
|
|
351
|
+
'Overflow risk',
|
|
352
|
+
summary.qualitySummary ? `${formatInt(summary.qualitySummary.overflowGPT4oCount)} repos` : 'n/a',
|
|
353
|
+
summary.qualitySummary ? `${formatInt(summary.qualitySummary.totalHiddenFiles)} hidden raw files • ${formatMoney(summary.qualitySummary.gpt4oSavedPerMonth)}/month saved` : 'quality.json missing',
|
|
354
|
+
summary.qualitySummary && summary.qualitySummary.overflowGPT4oCount > 0 ? 'warn' : 'ok'
|
|
355
|
+
));
|
|
356
|
+
|
|
357
|
+
const missingHtml = summary.missing.length > 0
|
|
358
|
+
? `<div class="notice">Missing source reports: ${escapeHtml(summary.missing.join(', '))}. The page still renders whatever data is available.</div>`
|
|
359
|
+
: '';
|
|
360
|
+
|
|
361
|
+
return [
|
|
362
|
+
'<!doctype html>',
|
|
363
|
+
'<html lang="en">',
|
|
364
|
+
'<head>',
|
|
365
|
+
'<meta charset="utf-8" />',
|
|
366
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1" />',
|
|
367
|
+
'<title>SigMap Benchmark Report</title>',
|
|
368
|
+
'<style>',
|
|
369
|
+
':root { color-scheme: light; --bg:#f5f1e8; --panel:#fffaf2; --ink:#1f1b16; --muted:#6a6258; --line:#dccfbf; --gold:#c87f2a; --green:#2f6f52; --blue:#2f5f8f; --red:#9f4f43; --shadow:0 18px 40px rgba(54,38,14,.10);} ',
|
|
370
|
+
'*{box-sizing:border-box} body{margin:0;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;background:linear-gradient(180deg,#f3ecdf 0%,#f7f3ed 100%);color:var(--ink)}',
|
|
371
|
+
'.page{max-width:1240px;margin:0 auto;padding:28px 20px 56px}',
|
|
372
|
+
'header{display:flex;justify-content:space-between;gap:24px;align-items:flex-end;margin-bottom:24px}',
|
|
373
|
+
'h1{margin:0;font-size:clamp(2rem,4vw,3.6rem);line-height:1.02;letter-spacing:-.04em}',
|
|
374
|
+
'.lede{max-width:760px;color:var(--muted);font-size:1rem;line-height:1.6;margin-top:10px}',
|
|
375
|
+
'.stamp{font-size:.92rem;color:var(--muted);text-align:right}',
|
|
376
|
+
'.grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px;margin:20px 0 24px}',
|
|
377
|
+
'.card,.panel,.notice,section{background:var(--panel);border:1px solid var(--line);box-shadow:var(--shadow);border-radius:18px}',
|
|
378
|
+
'.card{padding:18px 18px 16px}.card.cool{background:#f7f5ff}.card.warm{background:#fff4eb}.card.warn{background:#fff1eb}.card.ok{background:#eff8f1}',
|
|
379
|
+
'.label{font-size:.84rem;text-transform:uppercase;letter-spacing:.08em;color:var(--muted)}',
|
|
380
|
+
'.value{font-size:2rem;font-weight:700;letter-spacing:-.04em;margin-top:8px}',
|
|
381
|
+
'.hint{font-size:.95rem;color:var(--muted);margin-top:8px;line-height:1.5}',
|
|
382
|
+
'.notice{padding:14px 16px;margin-bottom:20px;color:var(--muted)}',
|
|
383
|
+
'section{padding:20px;margin-top:18px}',
|
|
384
|
+
'h2{margin:0 0 6px;font-size:1.4rem;letter-spacing:-.03em}',
|
|
385
|
+
'h3{margin:0 0 14px;font-size:1rem}',
|
|
386
|
+
'.section-copy{margin:0 0 16px;color:var(--muted);line-height:1.6}',
|
|
387
|
+
'table{width:100%;border-collapse:collapse;font-size:.95rem}',
|
|
388
|
+
'th,td{padding:10px 12px;border-bottom:1px solid var(--line);text-align:left;vertical-align:top}',
|
|
389
|
+
'th{font-size:.82rem;text-transform:uppercase;letter-spacing:.06em;color:var(--muted)}',
|
|
390
|
+
'tbody tr:hover{background:rgba(200,127,42,.06)}',
|
|
391
|
+
'.badge{display:inline-flex;align-items:center;padding:4px 8px;border-radius:999px;font-size:.78rem;font-weight:600;text-transform:uppercase;letter-spacing:.04em}',
|
|
392
|
+
'.badge.ok{background:#e6f4ea;color:#21573f}.badge.warn{background:#fff0de;color:#8a4a17}.badge.fail{background:#fde8e5;color:#8a2e23}',
|
|
393
|
+
'.split{display:grid;grid-template-columns:1fr 1fr;gap:16px}',
|
|
394
|
+
'.panel{padding:16px}',
|
|
395
|
+
'.progress-row{display:grid;grid-template-columns:140px 1fr 60px;gap:12px;align-items:center;margin:10px 0}',
|
|
396
|
+
'.progress-label,.progress-value{font-size:.92rem}',
|
|
397
|
+
'.progress-bar{height:10px;border-radius:999px;background:#efe4d5;overflow:hidden}',
|
|
398
|
+
'.progress-bar span{display:block;height:100%;border-radius:999px;background:linear-gradient(90deg,var(--gold),#ebbb61)}',
|
|
399
|
+
'code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:.85rem}',
|
|
400
|
+
'@media (max-width: 1020px){.grid{grid-template-columns:repeat(2,minmax(0,1fr))}.split{grid-template-columns:1fr}header{flex-direction:column;align-items:flex-start}.stamp{text-align:left}}',
|
|
401
|
+
'@media (max-width: 640px){.grid{grid-template-columns:1fr}.progress-row{grid-template-columns:110px 1fr 52px}th:nth-child(n+5),td:nth-child(n+5){display:none}}',
|
|
402
|
+
'</style>',
|
|
403
|
+
'</head>',
|
|
404
|
+
'<body>',
|
|
405
|
+
'<div class="page">',
|
|
406
|
+
'<header>',
|
|
407
|
+
'<div>',
|
|
408
|
+
'<h1>SigMap Benchmark Report</h1>',
|
|
409
|
+
'<p class="lede">A self-contained view of token reduction, retrieval quality, hallucination surface, and task-level prompt reduction. This page reads the saved JSON benchmark artifacts so it stays easy to regenerate locally.</p>',
|
|
410
|
+
'</div>',
|
|
411
|
+
`<div class="stamp">Generated: ${escapeHtml(summary.generatedAt)}<br />Source directory: <code>benchmarks/reports</code></div>`,
|
|
412
|
+
'</header>',
|
|
413
|
+
missingHtml,
|
|
414
|
+
`<div class="grid">${cards.join('')}</div>`,
|
|
415
|
+
renderMatrixSection(summary.matrix),
|
|
416
|
+
renderTokenSection(reports.token),
|
|
417
|
+
renderRetrievalSection(reports.retrieval),
|
|
418
|
+
renderQualitySection(reports.quality),
|
|
419
|
+
renderTaskSection(reports.task),
|
|
420
|
+
'</div>',
|
|
421
|
+
'</body>',
|
|
422
|
+
'</html>',
|
|
423
|
+
].join('');
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
function writeBenchmarkReport(cwd, opts = {}) {
|
|
427
|
+
const reports = loadBenchmarkReports(cwd);
|
|
428
|
+
const html = generateBenchmarkReportHtml(reports, opts);
|
|
429
|
+
const filePath = path.join(reports.reportsDir, opts.fileName || 'benchmark-report.html');
|
|
430
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
431
|
+
fs.writeFileSync(filePath, html, 'utf8');
|
|
432
|
+
return {
|
|
433
|
+
file: filePath,
|
|
434
|
+
summary: buildBenchmarkSummary(reports, opts.matrixSummary),
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
module.exports = {
|
|
439
|
+
loadBenchmarkReports,
|
|
440
|
+
buildBenchmarkSummary,
|
|
441
|
+
generateBenchmarkReportHtml,
|
|
442
|
+
writeBenchmarkReport,
|
|
443
|
+
};
|
package/src/mcp/server.js
CHANGED