pi-fovea 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/cli.ts +52 -3
- package/package.json +1 -1
- package/src/core/anchors.ts +36 -4
- package/src/core/build.ts +42 -9
- package/src/core/discover.ts +188 -0
- package/src/core/join.ts +1 -1
- package/src/core/sync.ts +11 -6
- package/src/core/types.ts +1 -0
package/README.md
CHANGED
|
@@ -134,6 +134,19 @@ File-convention routers never write a route string at all — those anchors are
|
|
|
134
134
|
|
|
135
135
|
**Known blind spots** (deliberate, logged in `src/core/anchors.ts`): Rust proc-macro attribute routers (actix `#[get("/x")]`, rocket) — ast-grep patterns can't parameterize attribute paths; frameworks with constructor-assigned prefixes (Flask Blueprint, FastAPI `APIRouter(prefix=…)`, chi `Mount`, Express `Router` mounts) — variable binding tracking is out of band; `scope`/`namespace` nesting in Phoenix/Rails/Django `include()` — prefixes across blocks aren't composed; tRPC/GraphQL/gRPC — no path token exists to anchor on.
|
|
136
136
|
|
|
137
|
+
### Tier 3: discovery mode
|
|
138
|
+
|
|
139
|
+
Unknown shapes self-heal. During the literal pass fovea harvests a per-repo histogram of *call-shape signatures* — `(lang · shape · callee · argIdx)` with a path-precision — and promotes statistically significant unknown ones into **implicit rules**: exact-arity ast-grep patterns, synthesized automatically, wired into the graph at **half hub gravity** with a `△` sigil in `fovea anchors` output. Sync reports discovered churn but never lets an unconfirmed hypothesis escalate to red; a hub upgrades to first-class the moment any site matches a non-implicit rule.
|
|
140
|
+
|
|
141
|
+
Promotion needs: ≥4 path-carrying sites, spread over ≥2 files, and a Jeffreys-smoothed posterior `p̂ = (pathN + .5) / (n + 1) ≥ 0.55`. Measured against 8 cloned projects, junk bands land below p̂≈0.27 and real shapes above p̂≈0.75 — the line is not tuned to the corpus, it sits mid-cliff. Frameworks already known to the static pack are never re-promoted.
|
|
142
|
+
|
|
143
|
+
```sh
|
|
144
|
+
fovea anchors <root> --discovered # only the △ hypothesis hubs
|
|
145
|
+
fovea rules <root> # promoted rules + evidence, ready to paste into .fovea/rules.json
|
|
146
|
+
fovea rules <root> --sigs # every path-touching signature, by precision (audit the corpus)
|
|
147
|
+
fovea rules <root> --adopt # write them into the repo's rule pack explicitly
|
|
148
|
+
```
|
|
149
|
+
|
|
137
150
|
Drop `.fovea/rules.json` in a repo to extend anchor detection beyond the built-ins:
|
|
138
151
|
|
|
139
152
|
```json
|
package/cli.ts
CHANGED
|
@@ -6,12 +6,16 @@
|
|
|
6
6
|
// fovea dwell [root] [factor] [budget] (deepens the in-process focus)
|
|
7
7
|
// fovea impact [root] [--files a,b] [--symbols x,y] [--base ref] [--no-uncommitted] [budget]
|
|
8
8
|
// fovea anchors [root] [filter] (every feature anchor, sorted)
|
|
9
|
+
// fovea rules [root] (tier-3 discovered shape hypotheses)
|
|
9
10
|
//
|
|
10
11
|
// The CLI is stateless across invocations (dwell needs a prior focus in the
|
|
11
12
|
// same process — combine ops inside pi, where sessions persist); stdout is
|
|
12
13
|
// the rendered field, nothing else, so it composes with head/grep/$().
|
|
13
14
|
|
|
15
|
+
import { statSync } from "node:fs";
|
|
14
16
|
import { ensureState, sketch, focus, dwell, impact } from "./src/core/ops.js";
|
|
17
|
+
import { aggregateFiles, posterior, promote } from "./src/core/discover.js";
|
|
18
|
+
import { DEFAULT_PACK } from "./src/core/anchors.js";
|
|
15
19
|
|
|
16
20
|
const [, , cmd = "status", ...argv] = process.argv;
|
|
17
21
|
|
|
@@ -37,9 +41,15 @@ const numAt = (i: number): number | undefined => {
|
|
|
37
41
|
return Number.isFinite(n) && n > 0 ? n : undefined;
|
|
38
42
|
};
|
|
39
43
|
// Root is the first positional that names a path; everything else is arg data.
|
|
44
|
+
// Existing directories count as paths even when bare ("next", "kernel").
|
|
40
45
|
const rootAt = (i: number): string => {
|
|
41
46
|
const p = pos[i];
|
|
42
|
-
|
|
47
|
+
if (p === undefined) return ".";
|
|
48
|
+
if (p.includes("/") || p === ".") return p;
|
|
49
|
+
try {
|
|
50
|
+
if (statSync(p).isDirectory()) return p;
|
|
51
|
+
} catch { /* not a path */ }
|
|
52
|
+
return ".";
|
|
43
53
|
};
|
|
44
54
|
|
|
45
55
|
try {
|
|
@@ -76,10 +86,49 @@ try {
|
|
|
76
86
|
const root = rootAt(0);
|
|
77
87
|
const filter = pos.find((p) => p !== root);
|
|
78
88
|
const rows = ensureState(root).graph.anchors
|
|
79
|
-
.map((a) => `${a.kind}\t${a.id}\t${a.file}:${a.line}`)
|
|
80
|
-
.filter((r) => !filter || r.includes(filter))
|
|
89
|
+
.map((a) => `${a.implicit ? "△" : " "}\t${a.kind}\t${a.id}\t${a.file}:${a.line}`)
|
|
90
|
+
.filter((r) => (!filter || r.includes(filter)) && (!flags.has("discovered") || r.startsWith("△")))
|
|
81
91
|
.sort();
|
|
82
92
|
out = rows.join("\n");
|
|
93
|
+
} else if (cmd === "rules") {
|
|
94
|
+
const root = rootAt(0);
|
|
95
|
+
const st = ensureState(root);
|
|
96
|
+
const sigs = aggregateFiles(Object.fromEntries(Object.entries(st.facts).map(([k, v]) => [k, v.sigs])));
|
|
97
|
+
const promoted = promote(sigs, DEFAULT_PACK);
|
|
98
|
+
if (flags.has("adopt") && promoted.length) {
|
|
99
|
+
const { mkdirSync, writeFileSync, readFileSync } = await import("node:fs");
|
|
100
|
+
const { join } = await import("node:path");
|
|
101
|
+
mkdirSync(join(root, ".fovea"), { recursive: true });
|
|
102
|
+
const rulesFile = join(root, ".fovea", "rules.json");
|
|
103
|
+
let existing = { rules: [] as unknown[] };
|
|
104
|
+
try { existing = JSON.parse(readFileSync(rulesFile, "utf8")); } catch { /* new */ }
|
|
105
|
+
const stamp = promoted.map((r) => ({
|
|
106
|
+
id: r.id.slice("implicit:".length),
|
|
107
|
+
langs: r.langs,
|
|
108
|
+
pattern: r.patterns[0],
|
|
109
|
+
methods: r.methods,
|
|
110
|
+
kind: r.kind,
|
|
111
|
+
}));
|
|
112
|
+
existing.rules = [...existing.rules, ...stamp];
|
|
113
|
+
writeFileSync(rulesFile, JSON.stringify(existing, null, 2) + "\n");
|
|
114
|
+
out = `wrote ${stamp.length} discovered rule(s) to .fovea/rules.json`;
|
|
115
|
+
} else if (flags.has("sigs")) {
|
|
116
|
+
out = sigs.filter((s) => s.pathN > 0)
|
|
117
|
+
.sort((a, b) => posterior(b.pathN, b.n) - posterior(a.pathN, a.n))
|
|
118
|
+
.map((s) => `${posterior(s.pathN, s.n).toFixed(2)} ${s.pathN}/${s.n} across ${s.files} files ${s.key}`)
|
|
119
|
+
.join("\n");
|
|
120
|
+
} else if (!promoted.length) {
|
|
121
|
+
out = "(no unknown shape passes the promotion floor — tier-1/2 coverage is doing fine)";
|
|
122
|
+
} else {
|
|
123
|
+
out = promoted.map((r) => JSON.stringify({
|
|
124
|
+
id: r.id.slice("implicit:".length),
|
|
125
|
+
langs: r.langs,
|
|
126
|
+
pattern: r.patterns[0],
|
|
127
|
+
methods: r.methods.replace(/\(\?i\)/, ""),
|
|
128
|
+
kind: r.kind,
|
|
129
|
+
_evidence: `p̂=${r.evidence.posterior.toFixed(2)} (${r.evidence.pathN}/${r.evidence.n} sites, ${r.evidence.files} files)`,
|
|
130
|
+
})).join("\n");
|
|
131
|
+
}
|
|
83
132
|
} else {
|
|
84
133
|
console.error(`unknown command: ${cmd}`);
|
|
85
134
|
process.exit(2);
|
package/package.json
CHANGED
package/src/core/anchors.ts
CHANGED
|
@@ -26,7 +26,8 @@ import type { Anchor } from "./types.js";
|
|
|
26
26
|
export interface AnchorRule {
|
|
27
27
|
id: string;
|
|
28
28
|
langs: string[];
|
|
29
|
-
pattern
|
|
29
|
+
/** Single ast-grep pattern; tier-3 synthesized rules use `patterns` instead. */
|
|
30
|
+
pattern?: string;
|
|
30
31
|
methods: string; // regex tested against the captured method metavar
|
|
31
32
|
kind: string;
|
|
32
33
|
/**
|
|
@@ -39,6 +40,10 @@ export interface AnchorRule {
|
|
|
39
40
|
verbFrom?: string;
|
|
40
41
|
/** Idiom writes paths mount-relative (Django `path("users/")`): root them. */
|
|
41
42
|
mountRoot?: boolean;
|
|
43
|
+
/** Synthesized tier-3 rules ship as variant lists (exact arity + trailing $$$H). */
|
|
44
|
+
patterns?: string[];
|
|
45
|
+
/** Discovered rules: half hub gravity until a real join upgrades them. */
|
|
46
|
+
implicit?: boolean;
|
|
42
47
|
}
|
|
43
48
|
|
|
44
49
|
const HTTP_VERB_RE = /^(?i:get|post|put|delete|patch|head|options)$/;
|
|
@@ -59,12 +64,19 @@ const PLACEHOLDER_ONLY = /^(:[A-Za-z_]\w*|\{[A-Za-z_]\w*\}|\[[A-Za-z_]\w*\])$/;
|
|
|
59
64
|
// Method names that are mounts, not verbs: Django urls, Rails match/root,
|
|
60
65
|
// Spring's umbrella RequestMapping. They anchor as ANY so the hub exists
|
|
61
66
|
// without pretending a verb was declared.
|
|
62
|
-
|
|
67
|
+
// Method names that mount a target rather than declare a verb. fetch(url)
|
|
68
|
+
// and redirect targets both resolve to GET; Django urlconfs and Rails mounts
|
|
69
|
+
// accept any verb.
|
|
70
|
+
const METHOD_ALIASES: Record<string, string> = {
|
|
71
|
+
PATH: "ANY", RE_PATH: "ANY", URL: "ANY", MATCH: "ANY", ROOT: "ANY",
|
|
72
|
+
REQUESTMAPPING: "ANY", RESOURCES: "ANY", FORWARD: "ANY",
|
|
73
|
+
FETCH: "GET", REDIRECT: "GET", RESPONDREDIRECT: "GET", REDIRECT_TO: "GET",
|
|
74
|
+
};
|
|
63
75
|
|
|
64
76
|
const deriveVerb = (method: string): string => {
|
|
65
77
|
let up = method.toUpperCase();
|
|
66
78
|
if (up.endsWith("MAPPING")) up = up.slice(0, -"MAPPING".length); // Spring GetMapping → GET
|
|
67
|
-
return
|
|
79
|
+
return METHOD_ALIASES[up] ?? up;
|
|
68
80
|
};
|
|
69
81
|
|
|
70
82
|
export const DEFAULT_PACK: AnchorRule[] = [
|
|
@@ -199,6 +211,24 @@ export const DEFAULT_PACK: AnchorRule[] = [
|
|
|
199
211
|
methods: "^add_url_rule$",
|
|
200
212
|
kind: "route",
|
|
201
213
|
},
|
|
214
|
+
{
|
|
215
|
+
// Client fetch with no receiver: fetch("/api/x"). Precision-audited by
|
|
216
|
+
// discovery (~93% path precision in the next.js clone corpus).
|
|
217
|
+
id: "fetch-bare",
|
|
218
|
+
langs: ["TypeScript", "Tsx", "JavaScript"],
|
|
219
|
+
pattern: "$M($P, $$$H)", // trailing $$$H absorbs the options bag; zero-arg tail matches fetch("/x") too
|
|
220
|
+
methods: "^fetch$",
|
|
221
|
+
kind: "route",
|
|
222
|
+
},
|
|
223
|
+
{
|
|
224
|
+
// Response-side route linkage: ktor respondRedirect("/myfiles") references
|
|
225
|
+
// an existing route without declaring it. Discovery found it at p̂≈0.81.
|
|
226
|
+
id: "ktor-respond-redirect",
|
|
227
|
+
langs: ["Kotlin"],
|
|
228
|
+
pattern: "$R.$M($P, $$$H)",
|
|
229
|
+
methods: "^respondRedirect$",
|
|
230
|
+
kind: "route",
|
|
231
|
+
},
|
|
202
232
|
{
|
|
203
233
|
id: "rust-router-chain",
|
|
204
234
|
langs: ["Rust"],
|
|
@@ -238,7 +268,8 @@ export const extractAnchors = (
|
|
|
238
268
|
if (p !== undefined && !prefixes.has(pm.file)) prefixes.set(pm.file, unquote(p));
|
|
239
269
|
}
|
|
240
270
|
}
|
|
241
|
-
|
|
271
|
+
const matchSets = patternRunAll(rule.patterns ?? [rule.pattern!], lang, langFiles, cwd);
|
|
272
|
+
for (const m of matchSets) {
|
|
242
273
|
const method = m.single.M;
|
|
243
274
|
const pathLike = m.single.P;
|
|
244
275
|
if (!method || !pathLike || !methodRe.test(method)) continue;
|
|
@@ -273,6 +304,7 @@ export const extractAnchors = (
|
|
|
273
304
|
nodeId: enclosing ?? `file:${m.file}`,
|
|
274
305
|
file: m.file,
|
|
275
306
|
line: m.line,
|
|
307
|
+
...(rule.implicit ? { implicit: true } : {}),
|
|
276
308
|
});
|
|
277
309
|
}
|
|
278
310
|
}
|
package/src/core/build.ts
CHANGED
|
@@ -8,10 +8,11 @@ import { existsSync, readFileSync, readdirSync, writeFileSync, mkdirSync } from
|
|
|
8
8
|
import { tmpdir } from "node:os";
|
|
9
9
|
import { basename, dirname, join as joinPath, posix } from "node:path";
|
|
10
10
|
import { spawnSync } from "node:child_process";
|
|
11
|
-
import { LANG_BY_EXT, isBinaryExt, isConfigFile } from "./astgrep.js";
|
|
11
|
+
import { LANG_BY_EXT, isBinaryExt, isConfigFile, langOf } from "./astgrep.js";
|
|
12
12
|
import { extractCalls, extractImports, extractLiterals, extractSymbols, isTestFile } from "./extract.js";
|
|
13
13
|
import { buildJoinIndex } from "./join.js";
|
|
14
14
|
import { extractAnchors, extractFileRoutes, loadRepoRules } from "./anchors.js";
|
|
15
|
+
import { aggregateFiles, harvestFile, promote, type FileSigs, type SynthesizedRule } from "./discover.js";
|
|
15
16
|
import type { CallSite, Edge, Graph, ImportSite, LiteralSite, NodeRec, SymbolRec } from "./types.js";
|
|
16
17
|
import type { AnchorDraft } from "./anchors.js";
|
|
17
18
|
import { coChangePairs } from "./cochange.js";
|
|
@@ -23,9 +24,13 @@ export interface FileFacts {
|
|
|
23
24
|
calls: CallSite[];
|
|
24
25
|
literals: LiteralSite[];
|
|
25
26
|
anchors: AnchorDraft[];
|
|
27
|
+
// Tier-3 discovery: per-file histogram of call-shape signatures
|
|
28
|
+
// (sig -> [totalSites, pathSites]); aggregated repo-wide at load to promote
|
|
29
|
+
// statistically significant unknown shapes into implicit half-weight rules.
|
|
30
|
+
sigs?: FileSigs;
|
|
26
31
|
}
|
|
27
32
|
|
|
28
|
-
const CACHE_VERSION =
|
|
33
|
+
const CACHE_VERSION = 6; // bump when extractor semantics change
|
|
29
34
|
const IGNORE_DIRS = new Set([".git", "node_modules", "dist", "vendor", ".venv", "venv", "target", "coverage", ".next", "build", "__pycache__", ".pi", ".pi-fovea", "deps", "_build", ".tox", "Pods"]);
|
|
30
35
|
const MAX_FILES = 24000;
|
|
31
36
|
// Generated dependency manifests are enormous and carry no first-class routes.
|
|
@@ -103,8 +108,12 @@ export const loadFacts = (root: string, files: string[]): Record<string, FileFac
|
|
|
103
108
|
} catch {
|
|
104
109
|
cached = undefined;
|
|
105
110
|
}
|
|
106
|
-
const { pack:
|
|
107
|
-
const
|
|
111
|
+
const { pack: basePack, fileRoutes, sha: baseRulesSha } = loadRepoRules(root);
|
|
112
|
+
const anchorPack = [...basePack];
|
|
113
|
+
// Implicit rules promote AFTER the first fact pass: their evidence lives in
|
|
114
|
+
// freshly harvested sigs, so the pack can only be finalized once facts exist.
|
|
115
|
+
let implicitRules: SynthesizedRule[] = [];
|
|
116
|
+
let rulesSha = baseRulesSha;
|
|
108
117
|
if (cached && (cached.version !== CACHE_VERSION || cached.root !== root)) cached = undefined;
|
|
109
118
|
const facts: Record<string, FileFacts> = {};
|
|
110
119
|
const dirty: string[] = [];
|
|
@@ -133,12 +142,31 @@ export const loadFacts = (root: string, files: string[]): Record<string, FileFac
|
|
|
133
142
|
putByFile(extractImports(code, root), (f, v) => f.imports.push(v));
|
|
134
143
|
putByFile(extractCalls(code, root), (f, v) => f.calls.push(v));
|
|
135
144
|
putByFile(extractLiterals(dirty, root), (f, v) => f.literals.push(v));
|
|
145
|
+
// Tier-3 harvest: regex histogram of call-shape signatures per file. Cheap
|
|
146
|
+
// (line scan, no ast-grep) and cached alongside the other facts.
|
|
147
|
+
for (const f of code) {
|
|
148
|
+
const lang = langOf(f);
|
|
149
|
+
if (!lang) continue;
|
|
150
|
+
let text = "";
|
|
151
|
+
try { text = readFileSync(joinPath(root, f), "utf8"); } catch { continue; }
|
|
152
|
+
const sigs = harvestFile(lang, text);
|
|
153
|
+
if (Object.keys(sigs).length) facts[f]!.sigs = sigs;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
// Tier-3: promote harvested signatures into implicit rules BEFORE anchors
|
|
157
|
+
// run, and fold their ids into the rules hash so a promotion change rebuilds
|
|
158
|
+
// anchors exactly like a rules.json edit would.
|
|
159
|
+
implicitRules = promote(aggregateFiles(Object.fromEntries(Object.entries(facts).map(([k, v]) => [k, v.sigs]))), anchorPack);
|
|
160
|
+
if (implicitRules.length) {
|
|
161
|
+
anchorPack.push(...implicitRules);
|
|
162
|
+
rulesSha = createHash("sha1").update(baseRulesSha).update(JSON.stringify(implicitRules.map((r) => r.id).sort())).digest("hex");
|
|
136
163
|
}
|
|
164
|
+
const rulesChangedFinal = cached !== undefined && cached.rulesSha !== rulesSha;
|
|
137
165
|
// Anchors need enclosing-symbol resolution. Symbols, imports, calls and
|
|
138
166
|
// literals are rules-independent — when only the rule pack changed, re-run
|
|
139
167
|
// anchor extraction over every code file against cached symbols and keep
|
|
140
168
|
// every other fact (green-node reuse one level up).
|
|
141
|
-
const anchorTargets =
|
|
169
|
+
const anchorTargets = rulesChangedFinal
|
|
142
170
|
? files.filter((f) => !isConfigFile(f))
|
|
143
171
|
: dirty.filter((f) => !isConfigFile(f));
|
|
144
172
|
if (anchorTargets.length) {
|
|
@@ -151,7 +179,7 @@ export const loadFacts = (root: string, files: string[]): Record<string, FileFac
|
|
|
151
179
|
const symsByFile = new Map<string, SymbolRec[]>();
|
|
152
180
|
for (const rel of anchorTargets) {
|
|
153
181
|
symsByFile.set(rel, facts[rel]?.symbols.length ? facts[rel]!.symbols : (cached?.facts[rel]?.symbols ?? []));
|
|
154
|
-
if (
|
|
182
|
+
if (rulesChangedFinal) facts[rel] && (facts[rel]!.anchors = []);
|
|
155
183
|
}
|
|
156
184
|
const enclosingId = (file: string, line: number): string | undefined => {
|
|
157
185
|
const syms = symsByFile.get(file) ?? [];
|
|
@@ -391,13 +419,18 @@ export const assembleGraph = (root: string, files: string[], facts: Record<strin
|
|
|
391
419
|
for (const [label, sites] of draftsByLabel) {
|
|
392
420
|
const first = sites[0]!;
|
|
393
421
|
const filesOf = [...new Set(sites.map((s) => s.file))];
|
|
394
|
-
|
|
422
|
+
// A hub is implicit only when EVERY site came from a discovered rule — a
|
|
423
|
+
// match by any real rule upgrades it back to first-class instantly.
|
|
424
|
+
const hubImplicit = sites.every((s) => s.implicit === true);
|
|
425
|
+
anchors.push({ id: label, kind: first.kind, label: sites.length > 1 ? `${label} · ${sites.length} sites` : label, nodeId: first.nodeId, file: first.file, line: first.line, ...(hubImplicit ? { implicit: true } : {}) });
|
|
395
426
|
const idx = addNode(nodes, seen, {
|
|
396
427
|
id: `anchor:${label}`, name: label, kind: "anchor", file: first.file, line: first.line,
|
|
397
|
-
sig: sites.length > 1 ? `${label} (${sites.length} sites)` : label
|
|
428
|
+
sig: `${hubImplicit ? "(△ discovered) " : ""}${sites.length > 1 ? `${label} (${sites.length} sites)` : label}`, lang: "anchor",
|
|
398
429
|
});
|
|
399
430
|
(byFile.get(first.file) ?? byFile.set(first.file, []).get(first.file)!).push(idx);
|
|
400
|
-
|
|
431
|
+
// Tier-3 hubs prove themselves at half conductance; a later literal join
|
|
432
|
+
// against a first-class hub can still warm them via the channel edges.
|
|
433
|
+
const w = (hubImplicit ? 0.5 : 1) / Math.sqrt(sites.length);
|
|
401
434
|
for (const s of sites) {
|
|
402
435
|
const handler = seen.get(s.nodeId) ?? fileIdx.get(s.file)!;
|
|
403
436
|
pushEdge(idx, handler, "anchors", w);
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
// Tier-3: autonomy for route extraction. The static pack catches known port
|
|
2
|
+
// shapes; this module watches the literal channel for shapes the pack does NOT
|
|
3
|
+
// declare, promotes statistically significant ones into synthesized "implicit"
|
|
4
|
+
// rules at half hub gravity, and lets confirmatory joins against real hubs
|
|
5
|
+
// upgrade them to full first-class weight.
|
|
6
|
+
//
|
|
7
|
+
// The promotion statistic is a per-argument conditional: given the string arg
|
|
8
|
+
// at position i of call-shape (lang·shape·callee), how often does it classify
|
|
9
|
+
// as a path? A Jeffreys-smoothed posterior with global floors decides — not raw
|
|
10
|
+
// frequency, under which chaff like `assertEquals(...)` dwarfs every real shape.
|
|
11
|
+
//
|
|
12
|
+
// Corpus audit (8 repos, 183 sigs n≥4): junk bands sit below p̂≈0.27, real
|
|
13
|
+
// shapes above p̂≈0.75 — the 0.55 line is a cliff, not a gradient.
|
|
14
|
+
//
|
|
15
|
+
// Harvest is line-regex over source text — no ast-grep pass needed. Per file we
|
|
16
|
+
// store a compact signature histogram (sig -> [sites, pathSites]); promotions
|
|
17
|
+
// aggregate across ALL files of the repo, so a dependency update or a style
|
|
18
|
+
// drift can only flip a marginal signature when its repo-wide evidence moves.
|
|
19
|
+
|
|
20
|
+
import { classifyLiteral } from "./join.js";
|
|
21
|
+
|
|
22
|
+
export type Shape = "recv" | "bare" | "dec";
|
|
23
|
+
|
|
24
|
+
// Compacted per-file histogram: sigKey -> [totalSites, pathSites]
|
|
25
|
+
export type FileSigs = Record<string, [number, number]>;
|
|
26
|
+
|
|
27
|
+
const QUOTED = /^[rbfuRBFU]{0,3}(["'`])([\s\S]*)\1$/;
|
|
28
|
+
const unquote = (s: string): string | undefined => {
|
|
29
|
+
const m = QUOTED.exec(s.trim());
|
|
30
|
+
return m ? m[2] : undefined;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
// Line-local call scan: grabs the callee shape (recv chain, bare, decorator)
|
|
34
|
+
// and the raw arg list of every `X(...)` on the line. Multi-line calls still
|
|
35
|
+
// land because route strings overwhelmingly sit on the call's first line.
|
|
36
|
+
const CALL_LINE_RE = /(?<at>@)?(?<recv>(?:[A-Za-z_$][\w$]*\.)+)?(?<method>[A-Za-z_$][\w$]*)\s*\((?<args>[^()]*)\)/g;
|
|
37
|
+
const STRING_RE = /([rbfuRBFU]{0,3})(["'`])((?:\\.|(?!\2)[^\\])*)\2/g;
|
|
38
|
+
|
|
39
|
+
// Callees that are noise even at 100% precision (env/fs access, string ctors,
|
|
40
|
+
// module loading — dynamic import()/require() hits the path channel but the
|
|
41
|
+
// import edge already covers it, an anchor would double-count the site).
|
|
42
|
+
const CALLEE_DENY = new Set([
|
|
43
|
+
"join", "resolve", "dirname", "basename", "expand_path",
|
|
44
|
+
"readFile", "readFileSync", "existsSync", "open", "load", "loads",
|
|
45
|
+
"import", "require",
|
|
46
|
+
// String predicates: membership tests hit the path column at high rate but
|
|
47
|
+
// never refer to routes. (Distinguishing cause from noise is callee-semantic.)
|
|
48
|
+
"startsWith", "endsWith", "contains", "includes", "equals", "equalsIgnoreCase",
|
|
49
|
+
"matches", "matchesPattern", "useParams", "matchPath",
|
|
50
|
+
]);
|
|
51
|
+
|
|
52
|
+
export const harvestFile = (lang: string, text: string): FileSigs => {
|
|
53
|
+
const sigs: FileSigs = {};
|
|
54
|
+
for (const line of text.split("\n")) {
|
|
55
|
+
if (!line.includes("(") || !(line.includes('"') || line.includes("'") || line.includes("`"))) continue;
|
|
56
|
+
CALL_LINE_RE.lastIndex = 0;
|
|
57
|
+
let c: RegExpExecArray | null;
|
|
58
|
+
while ((c = CALL_LINE_RE.exec(line))) {
|
|
59
|
+
const method = c.groups?.method;
|
|
60
|
+
if (!method || CALLEE_DENY.has(method)) continue;
|
|
61
|
+
const shape: Shape = c.groups?.at ? "dec" : c.groups?.recv ? "recv" : "bare";
|
|
62
|
+
const argsText = c.groups?.args ?? "";
|
|
63
|
+
if (!argsText) continue;
|
|
64
|
+
// True argument index of each string literal (split tolerantly on commas).
|
|
65
|
+
const args = argsText.split(",");
|
|
66
|
+
let idx = 0;
|
|
67
|
+
for (const raw of args) {
|
|
68
|
+
STRING_RE.lastIndex = 0;
|
|
69
|
+
const sm = STRING_RE.exec(raw);
|
|
70
|
+
const idxNow = idx++;
|
|
71
|
+
if (!sm) continue;
|
|
72
|
+
const lit = sm[3];
|
|
73
|
+
if (lit === undefined || lit === "") continue;
|
|
74
|
+
const key = `${lang}|${shape}|${method}|${idxNow}`;
|
|
75
|
+
const rec = sigs[key] ?? [0, 0];
|
|
76
|
+
rec[0]++;
|
|
77
|
+
if (classifyLiteral(lit) === "path") rec[1]++;
|
|
78
|
+
sigs[key] = rec;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return sigs;
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
export interface SigStats {
|
|
86
|
+
key: string;
|
|
87
|
+
lang: string;
|
|
88
|
+
shape: Shape;
|
|
89
|
+
callee: string;
|
|
90
|
+
argIdx: number;
|
|
91
|
+
n: number;
|
|
92
|
+
pathN: number;
|
|
93
|
+
files: number;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export const aggregateFiles = (perFile: Record<string, FileSigs | undefined>): SigStats[] => {
|
|
97
|
+
const agg = new Map<string, SigStats>();
|
|
98
|
+
for (const sigs of Object.values(perFile)) {
|
|
99
|
+
if (!sigs) continue;
|
|
100
|
+
for (const [key, [n, p]] of Object.entries(sigs)) {
|
|
101
|
+
const stat = agg.get(key);
|
|
102
|
+
if (stat) {
|
|
103
|
+
stat.n += n;
|
|
104
|
+
stat.pathN += p;
|
|
105
|
+
stat.files++;
|
|
106
|
+
} else {
|
|
107
|
+
const [lang, shape, callee, argIdx] = key.split("|");
|
|
108
|
+
agg.set(key, { key, lang: lang!, shape: shape as Shape, callee: callee!, argIdx: Number(argIdx), n, pathN: p, files: 1 });
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return [...agg.values()];
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
/** Jeffreys-ish posterior: p̂ = (pathN + .5) / (n + 1). */
|
|
116
|
+
export const posterior = (pathN: number, n: number): number => (pathN + 0.5) / (n + 1);
|
|
117
|
+
|
|
118
|
+
export const MIN_SITES = 4;
|
|
119
|
+
export const MIN_FILES = 2;
|
|
120
|
+
export const MIN_POSTERIOR = 0.55;
|
|
121
|
+
|
|
122
|
+
const escapeRe = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
123
|
+
|
|
124
|
+
// Pattern synthesis per shape and arg position: dummy metavars for the slots
|
|
125
|
+
// the corpus says are not the path, $P at the proven position. Two variants
|
|
126
|
+
// per site — exact arity and with a trailing $$$H absorb — because some
|
|
127
|
+
// dialects only match with explicit tail holes (Python `$X, $P, $$$H`).
|
|
128
|
+
export interface SynthesizedRule {
|
|
129
|
+
id: string;
|
|
130
|
+
langs: string[];
|
|
131
|
+
patterns: string[]; // patternRunAll variants
|
|
132
|
+
methods: string;
|
|
133
|
+
kind: string;
|
|
134
|
+
implicit: true;
|
|
135
|
+
evidence: { n: number; pathN: number; files: number; posterior: number };
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export const synthesize = (s: SigStats): SynthesizedRule | undefined => {
|
|
139
|
+
const slots: string[] = [];
|
|
140
|
+
for (let i = 0; i <= s.argIdx; i++) slots.push(i === s.argIdx ? "$P" : `$X${i}`);
|
|
141
|
+
const inner = slots.join(", ");
|
|
142
|
+
let variants: string[];
|
|
143
|
+
switch (s.shape) {
|
|
144
|
+
case "recv": variants = [`$R.$M(${inner})`, `$R.$M(${inner}, $$$H)`]; break;
|
|
145
|
+
case "bare": variants = [`$M(${inner})`, `$M(${inner}, $$$H)`]; break;
|
|
146
|
+
case "dec": variants = [`@$M(${inner})`, `@$M(${inner}, $$$H)`]; break;
|
|
147
|
+
}
|
|
148
|
+
return {
|
|
149
|
+
id: `implicit:${s.lang.toLowerCase()}:${s.shape}:${s.callee}:${s.argIdx}`,
|
|
150
|
+
langs: [s.lang],
|
|
151
|
+
patterns: variants,
|
|
152
|
+
methods: `^(?i:${escapeRe(s.callee)})$`,
|
|
153
|
+
kind: "route",
|
|
154
|
+
implicit: true,
|
|
155
|
+
evidence: { n: s.n, pathN: s.pathN, files: s.files, posterior: posterior(s.pathN, s.n) },
|
|
156
|
+
};
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
// Shape-shape compatibility: a discovery is only NEW when no existing rule
|
|
160
|
+
// already binds the callee with a compatible pattern shape in that language.
|
|
161
|
+
// e.g. Java's @GetMapping is in the pack, so java:dec:GetMapping promotes nothing.
|
|
162
|
+
const shapeCompatPatterns: Record<Shape, RegExp> = {
|
|
163
|
+
recv: /^\$R\.\$M\(/,
|
|
164
|
+
bare: /^\$M[(\s]/,
|
|
165
|
+
dec: /^@\$(R\.)?M\(/,
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
const isCovered = (sig: SigStats, pack: Array<{ langs: string[]; methods: string; pattern?: string; patterns?: string[] }>): boolean => {
|
|
169
|
+
const mre = (methods: string, callee: string): boolean => new RegExp(methods.replace(/^\^\(\?i\)/, "^(?i:")).test(callee);
|
|
170
|
+
return pack.some((r) => {
|
|
171
|
+
if (!r.langs.includes(sig.lang)) return false;
|
|
172
|
+
if (!mre(r.methods, sig.callee)) return false;
|
|
173
|
+
const pats = r.patterns ?? (r.pattern ? [r.pattern] : []);
|
|
174
|
+
return pats.some((p) => shapeCompatPatterns[sig.shape].test(p));
|
|
175
|
+
});
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
export const promote = (sigs: SigStats[], pack: Array<{ id: string; langs: string[]; methods: string; pattern?: string; patterns?: string[] }> = []): SynthesizedRule[] => {
|
|
179
|
+
const out: SynthesizedRule[] = [];
|
|
180
|
+
for (const s of sigs) {
|
|
181
|
+
if (s.n < MIN_SITES || s.files < MIN_FILES) continue;
|
|
182
|
+
if (posterior(s.pathN, s.n) < MIN_POSTERIOR) continue;
|
|
183
|
+
if (isCovered(s, pack)) continue;
|
|
184
|
+
const r = synthesize(s);
|
|
185
|
+
if (r) out.push(r);
|
|
186
|
+
}
|
|
187
|
+
return out;
|
|
188
|
+
};
|
package/src/core/join.ts
CHANGED
|
@@ -14,7 +14,7 @@ export interface LitOccurrence { node: number; line: number; file: string; }
|
|
|
14
14
|
|
|
15
15
|
export interface JoinEdge { a: number; b: number; w: number; }
|
|
16
16
|
|
|
17
|
-
const PLACEHOLDER_SEGMENT = /^(?::[^/]+|\{[^}/]*\}|\$\{[^}/]*\}
|
|
17
|
+
const PLACEHOLDER_SEGMENT = /^(?::[^/]+|\{[^}/]*\}|\$\{[^}/]*\}|\$[A-Za-z_]\w*|<[^/>]+>|\*+)$/; // $x = Kotlin template shorthand
|
|
18
18
|
const WORD_RE = /^[A-Za-z][\w$.\-]{6,63}$/;
|
|
19
19
|
const URLISH_RE = /^(?:https?|wss?):\/\/[^/]+/;
|
|
20
20
|
|
package/src/core/sync.ts
CHANGED
|
@@ -64,10 +64,14 @@ export const sync = (root: string, params: SyncParams, now?: RepoState): SyncOut
|
|
|
64
64
|
};
|
|
65
65
|
}
|
|
66
66
|
|
|
67
|
-
// Version drifted: measure what moved.
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
const
|
|
67
|
+
// Version drifted: measure what moved. Implicit (tier-3 discovered) hubs
|
|
68
|
+
// churn is reported but NEVER escalates alone — hypotheses with no first-class
|
|
69
|
+
// backing don't get to wake the model with a red verdict.
|
|
70
|
+
const current = new Map(state.graph.anchors.map((a) => [a.id, a.implicit === true]));
|
|
71
|
+
const currentIds = new Set(current.keys());
|
|
72
|
+
const added = [...currentIds].filter((id) => !prev.anchors.has(id));
|
|
73
|
+
const removed = [...prev.anchors].filter((id) => !currentIds.has(id));
|
|
74
|
+
const newlyImplicit = added.filter((id) => current.get(id));
|
|
71
75
|
|
|
72
76
|
const session = getSession(root);
|
|
73
77
|
const disclosedFiles = new Set<string>();
|
|
@@ -92,7 +96,7 @@ export const sync = (root: string, params: SyncParams, now?: RepoState): SyncOut
|
|
|
92
96
|
|
|
93
97
|
baselines.set(root, { ...snapshot(state), warmed: warmNow });
|
|
94
98
|
|
|
95
|
-
const red = added.length + removed.length > 0 || warmNew.length >= Math.max(1, params.warmFileThreshold);
|
|
99
|
+
const red = (added.length - newlyImplicit.length) + removed.length > 0 || warmNew.length >= Math.max(1, params.warmFileThreshold);
|
|
96
100
|
if (!red) {
|
|
97
101
|
return {
|
|
98
102
|
structural: true, red: false, tokens: 0,
|
|
@@ -102,7 +106,8 @@ export const sync = (root: string, params: SyncParams, now?: RepoState): SyncOut
|
|
|
102
106
|
|
|
103
107
|
const lines: string[] = [];
|
|
104
108
|
lines.push(`fovea sync · v ${state.version} · edit cascade did not stay local`);
|
|
105
|
-
for (const id of added.slice(0, 12)) lines.push(` ⚑=new ${id}`);
|
|
109
|
+
for (const id of added.filter((a) => !newlyImplicit.includes(a)).slice(0, 12)) lines.push(` ⚑=new ${id}`);
|
|
110
|
+
for (const id of newlyImplicit.slice(0, 6)) lines.push(` △ newly discovered hub ${id}`);
|
|
106
111
|
for (const id of removed.slice(0, 12)) lines.push(` ⚑-removed ${id}`);
|
|
107
112
|
if (warmNew.length) {
|
|
108
113
|
lines.push(` newly warm undisclosed files (revisit with fovea_focus):`);
|
package/src/core/types.ts
CHANGED