turntax 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +242 -138
- package/dist/cli.js +166 -11
- package/dist/core/advice.js +17 -7
- package/dist/core/audit.js +40 -6
- package/dist/core/diff.js +63 -0
- package/dist/core/fix.js +431 -0
- package/dist/core/mcp.js +227 -0
- package/dist/core/render.js +170 -5
- package/dist/core/user.js +80 -0
- package/dist/node/apply.js +43 -0
- package/dist/node/discover.js +80 -2
- package/dist/node/git.js +46 -0
- package/dist/node/mcp.js +212 -0
- package/package.json +64 -63
package/dist/core/advice.js
CHANGED
|
@@ -3,14 +3,16 @@ import { countO200k } from './tokenize.js';
|
|
|
3
3
|
import { TOOLS } from './tools.js';
|
|
4
4
|
const DEFAULT_SECTION_TOKENS = 600;
|
|
5
5
|
export function buildAdvice(units, files, opts = {}) {
|
|
6
|
+
// User-level files take part in checks across files; moving sections or checking paths only makes sense in the repository.
|
|
7
|
+
const repo = units.filter((u) => !u.scope);
|
|
6
8
|
const advice = [
|
|
7
9
|
...misconfigured(units),
|
|
8
10
|
...sizeLimits(units, files),
|
|
9
11
|
...duplicates(units),
|
|
10
12
|
...contradictions(units),
|
|
11
|
-
...sections(
|
|
13
|
+
...sections(repo, opts.sectionTokens ?? DEFAULT_SECTION_TOKENS),
|
|
12
14
|
...nonEnglish(units, opts.english),
|
|
13
|
-
...staleReferences(
|
|
15
|
+
...staleReferences(repo, opts),
|
|
14
16
|
];
|
|
15
17
|
const order = { high: 0, medium: 1, low: 2 };
|
|
16
18
|
return advice.sort((a, b) => order[a.severity] - order[b.severity] || (b.savingTokens ?? 0) - (a.savingTokens ?? 0));
|
|
@@ -263,11 +265,8 @@ function sections(units, threshold) {
|
|
|
263
265
|
const tools = toolsWith(units, u.path, 'always');
|
|
264
266
|
for (const s of splitSections(u.text)) {
|
|
265
267
|
const tokens = countO200k(s.text);
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
const top = mostCommon(dirs);
|
|
269
|
-
if (tokens >= 100 && top && top.count >= 2 && top.count / dirs.length >= 0.6) {
|
|
270
|
-
const scope = commonDir(refs.filter((r) => r.startsWith(top.value + '/')));
|
|
268
|
+
const scope = sectionScope(s.text);
|
|
269
|
+
if (scope) {
|
|
271
270
|
out.push({
|
|
272
271
|
kind: 'scopable-section',
|
|
273
272
|
severity: 'medium',
|
|
@@ -295,6 +294,17 @@ function sections(units, threshold) {
|
|
|
295
294
|
}
|
|
296
295
|
return out;
|
|
297
296
|
}
|
|
297
|
+
/** The directory a section is mostly about (for example `packages/api`), or null when it is not scopable. */
|
|
298
|
+
export function sectionScope(text) {
|
|
299
|
+
if (countO200k(text) < 100)
|
|
300
|
+
return null;
|
|
301
|
+
const refs = pathRefs(text).filter((r) => !r.startsWith('.'));
|
|
302
|
+
const dirs = refs.map((r) => r.split('/')[0]);
|
|
303
|
+
const top = mostCommon(dirs);
|
|
304
|
+
if (!top || top.count < 2 || top.count / dirs.length < 0.6)
|
|
305
|
+
return null;
|
|
306
|
+
return commonDir(refs.filter((r) => r.startsWith(top.value + '/')));
|
|
307
|
+
}
|
|
298
308
|
/** Longest directory prefix shared by all refs, e.g. `packages/api` for `packages/api/db.ts` and `packages/api/handlers/`. */
|
|
299
309
|
function commonDir(refs) {
|
|
300
310
|
const dirs = refs.map((r) => (r.endsWith('/') ? r.slice(0, -1) : r.slice(0, r.lastIndexOf('/'))).split('/'));
|
package/dist/core/audit.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { buildAdvice } from './advice.js';
|
|
2
2
|
import { classify, detectTools } from './classify.js';
|
|
3
|
+
import { defersTools, serverKey } from './mcp.js';
|
|
3
4
|
import { countO200k, offlineCounter } from './tokenize.js';
|
|
4
5
|
import { TOOLS } from './tools.js';
|
|
6
|
+
import { classifyUser } from './user.js';
|
|
5
7
|
export function modelsForTool(tool, models) {
|
|
6
8
|
const spec = TOOLS.find((t) => t.id === tool);
|
|
7
9
|
return spec.families.length ? models.filter((m) => spec.families.includes(m.family)) : models;
|
|
@@ -15,9 +17,12 @@ function counterFor(model, opts) {
|
|
|
15
17
|
}
|
|
16
18
|
export async function audit(files, opts) {
|
|
17
19
|
const turns = opts.turns ?? 100;
|
|
18
|
-
const allUnits = classify(files);
|
|
19
|
-
const
|
|
20
|
+
const allUnits = [...classify(files), ...classifyUser(opts.userFiles ?? [])];
|
|
21
|
+
const servers = opts.mcp?.servers ?? [];
|
|
22
|
+
const toolIds = opts.tools ??
|
|
23
|
+
[...new Set([...detectTools(allUnits), ...servers.filter((s) => !s.disabled).map((s) => s.tool)])].sort((a, b) => TOOLS.findIndex((t) => t.id === a) - TOOLS.findIndex((t) => t.id === b));
|
|
20
24
|
const units = allUnits.filter((u) => toolIds.includes(u.tool));
|
|
25
|
+
const mcp = servers.filter((s) => toolIds.includes(s.tool)).map((s) => mcpRow(s, opts.mcp?.measured));
|
|
21
26
|
// Count each distinct text once per model, only for the tools that run that model.
|
|
22
27
|
const counts = new Map();
|
|
23
28
|
for (const model of opts.models) {
|
|
@@ -39,6 +44,8 @@ export async function audit(files, opts) {
|
|
|
39
44
|
};
|
|
40
45
|
if (u.importedFrom)
|
|
41
46
|
row.importedFrom = u.importedFrom;
|
|
47
|
+
if (u.scope)
|
|
48
|
+
row.scope = u.scope;
|
|
42
49
|
if (u.mode !== 'ignored')
|
|
43
50
|
for (const m of modelsForTool(u.tool, opts.models)) {
|
|
44
51
|
const c = counts.get(m.id)?.get(u.text);
|
|
@@ -51,21 +58,48 @@ export async function audit(files, opts) {
|
|
|
51
58
|
const spec = TOOLS.find((t) => t.id === id);
|
|
52
59
|
const rows = files_.filter((r) => r.tool === id);
|
|
53
60
|
const always = rows.filter((r) => r.mode === 'always');
|
|
54
|
-
const
|
|
61
|
+
const repoAlways = always.filter((r) => !r.scope);
|
|
62
|
+
const userAlways = always.filter((r) => r.scope === 'user');
|
|
63
|
+
const onDemand = rows.filter((r) => r.mode === 'on-demand' && !r.scope);
|
|
55
64
|
const models = modelsForTool(id, opts.models).map((m) => modelCost(m, always, turns));
|
|
65
|
+
const toolMcp = mcp.filter((r) => r.tool === id && r.status !== 'disabled');
|
|
66
|
+
const mcpTokens = (scope) => sum(toolMcp.filter((r) => r.scope === scope).map((r) => r.o200k ?? 0));
|
|
67
|
+
const perTurnO200k = sum(repoAlways.map((r) => r.o200k));
|
|
68
|
+
const userPerTurnO200k = sum(userAlways.map((r) => r.o200k)) + mcpTokens('user');
|
|
69
|
+
const mcpPerTurnO200k = mcpTokens('project');
|
|
56
70
|
return {
|
|
57
71
|
tool: id,
|
|
58
72
|
name: spec.name,
|
|
59
|
-
alwaysFiles: new Set(
|
|
73
|
+
alwaysFiles: new Set(repoAlways.map((r) => r.path)).size,
|
|
60
74
|
onDemandFiles: new Set(onDemand.map((r) => r.path)).size,
|
|
61
|
-
perTurnO200k
|
|
75
|
+
perTurnO200k,
|
|
62
76
|
onDemandO200k: sum(onDemand.map((r) => r.o200k)),
|
|
77
|
+
userAlwaysFiles: new Set(userAlways.map((r) => r.path)).size,
|
|
78
|
+
userPerTurnO200k,
|
|
79
|
+
mcpServers: toolMcp.length,
|
|
80
|
+
mcpPerTurnO200k,
|
|
81
|
+
mcpMeasured: toolMcp.every((r) => r.status === 'measured'),
|
|
82
|
+
totalPerTurnO200k: perTurnO200k + mcpPerTurnO200k + userPerTurnO200k,
|
|
63
83
|
models,
|
|
64
84
|
notes: spec.notes,
|
|
65
85
|
sources: spec.sources,
|
|
66
86
|
};
|
|
67
87
|
});
|
|
68
|
-
|
|
88
|
+
const adviceFiles = [...files, ...(opts.userFiles ?? [])];
|
|
89
|
+
return { turns, tools, files: files_, advice: buildAdvice(units, adviceFiles, opts.advice), models: opts.models, mcp };
|
|
90
|
+
}
|
|
91
|
+
function mcpRow(s, measured) {
|
|
92
|
+
const row = { name: s.name, tool: s.tool, configPath: s.configPath, scope: s.scope, transport: s.transport, status: 'not-measured' };
|
|
93
|
+
if (s.disabled)
|
|
94
|
+
return { ...row, status: 'disabled' };
|
|
95
|
+
const m = measured?.get(serverKey(s));
|
|
96
|
+
if (!m)
|
|
97
|
+
return row;
|
|
98
|
+
if ('error' in m)
|
|
99
|
+
return { ...row, status: 'failed', error: m.error };
|
|
100
|
+
if (defersTools(s))
|
|
101
|
+
return { ...row, status: 'measured', tools: m.tools, o200k: m.namesO200k, deferredO200k: m.o200k };
|
|
102
|
+
return { ...row, status: 'measured', tools: m.tools, o200k: m.o200k };
|
|
69
103
|
}
|
|
70
104
|
function modelCost(model, always, turns) {
|
|
71
105
|
const counts = always.map((r) => r.counts[model.id]);
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
export function diffReports(before, after, base) {
|
|
2
|
+
const ids = [...new Set([...after.tools.map((t) => t.tool), ...before.tools.map((t) => t.tool)])];
|
|
3
|
+
const tools = ids.map((id) => {
|
|
4
|
+
const a = after.tools.find((t) => t.tool === id);
|
|
5
|
+
const b = before.tools.find((t) => t.tool === id);
|
|
6
|
+
const av = a?.perTurnO200k ?? null;
|
|
7
|
+
const bv = b?.perTurnO200k ?? null;
|
|
8
|
+
return { tool: id, name: (a ?? b).name, before: bv, after: av, delta: (av ?? 0) - (bv ?? 0) };
|
|
9
|
+
});
|
|
10
|
+
const alwaysFiles = (r) => {
|
|
11
|
+
const m = new Map();
|
|
12
|
+
for (const f of r.files) {
|
|
13
|
+
if (f.mode !== 'always' || f.scope === 'user')
|
|
14
|
+
continue;
|
|
15
|
+
const key = `${f.path}\0${f.part}`;
|
|
16
|
+
const prev = m.get(key);
|
|
17
|
+
if (prev)
|
|
18
|
+
prev.tools.add(f.tool);
|
|
19
|
+
else
|
|
20
|
+
m.set(key, { path: f.path, part: f.part, o200k: f.o200k, tools: new Set([f.tool]) });
|
|
21
|
+
}
|
|
22
|
+
return m;
|
|
23
|
+
};
|
|
24
|
+
const bf = alwaysFiles(before);
|
|
25
|
+
const af = alwaysFiles(after);
|
|
26
|
+
const files = [];
|
|
27
|
+
for (const key of new Set([...bf.keys(), ...af.keys()])) {
|
|
28
|
+
const b = bf.get(key);
|
|
29
|
+
const a = af.get(key);
|
|
30
|
+
const delta = (a?.o200k ?? 0) - (b?.o200k ?? 0);
|
|
31
|
+
if (a && b && delta === 0)
|
|
32
|
+
continue;
|
|
33
|
+
files.push({
|
|
34
|
+
path: (a ?? b).path,
|
|
35
|
+
part: (a ?? b).part,
|
|
36
|
+
before: b?.o200k ?? null,
|
|
37
|
+
after: a?.o200k ?? null,
|
|
38
|
+
delta,
|
|
39
|
+
tools: [...new Set([...(b?.tools ?? []), ...(a?.tools ?? [])])],
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
files.sort((x, y) => Math.abs(y.delta) - Math.abs(x.delta) || x.path.localeCompare(y.path));
|
|
43
|
+
const servers = (r) => new Set((r.mcp ?? []).filter((s) => s.scope === 'project' && s.status !== 'disabled').map((s) => `${s.name} (${s.tool})`));
|
|
44
|
+
const bs = servers(before);
|
|
45
|
+
const as = servers(after);
|
|
46
|
+
return {
|
|
47
|
+
base,
|
|
48
|
+
tools,
|
|
49
|
+
files,
|
|
50
|
+
mcp: { added: [...as].filter((s) => !bs.has(s)), removed: [...bs].filter((s) => !as.has(s)) },
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
/** Largest per-turn increase across agents (0 when nothing grew). */
|
|
54
|
+
export function maxIncrease(diff) {
|
|
55
|
+
return Math.max(0, ...diff.tools.map((t) => t.delta));
|
|
56
|
+
}
|
|
57
|
+
/** Validates a parsed JSON report enough to diff against it. */
|
|
58
|
+
export function asDiffInput(data) {
|
|
59
|
+
const r = data;
|
|
60
|
+
if (!r || !Array.isArray(r.tools) || !Array.isArray(r.files))
|
|
61
|
+
throw new Error('not a turntax --json report');
|
|
62
|
+
return r;
|
|
63
|
+
}
|
package/dist/core/fix.js
ADDED
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
import { sectionScope } from './advice.js';
|
|
2
|
+
import { classify } from './classify.js';
|
|
3
|
+
import { isInstructionFile } from './files.js';
|
|
4
|
+
import { basename, findImports, linesWithFences, parseFrontmatter, splitSections } from './markdown.js';
|
|
5
|
+
import { countO200k } from './tokenize.js';
|
|
6
|
+
import { TOOLS } from './tools.js';
|
|
7
|
+
const RANK = { ignored: 0, manual: 1, 'on-demand': 2, always: 3 };
|
|
8
|
+
const toolName = (id) => TOOLS.find((t) => t.id === id)?.name ?? id;
|
|
9
|
+
const names = (ids) => [...ids].map(toolName).join(', ');
|
|
10
|
+
/** Text with LF line endings, plus the line ending to write back. */
|
|
11
|
+
function lf(text) {
|
|
12
|
+
return { text: text.replace(/\r\n/g, '\n'), eol: text.includes('\r\n') ? '\r\n' : '\n' };
|
|
13
|
+
}
|
|
14
|
+
function contentUnits(units) {
|
|
15
|
+
return units.filter((u) => u.part !== 'description' && u.mode !== 'ignored');
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* A line as compared by the fixes: only list markers and whitespace are ignored. Case, underscores,
|
|
19
|
+
* asterisks and backticks can carry meaning (`create_user`, globs), so lines that differ there differ.
|
|
20
|
+
*/
|
|
21
|
+
function lineKey(line) {
|
|
22
|
+
return line.replace(/^\s*(?:[-*+]|\d+[.)])\s+/, '').replace(/\s+/g, ' ').trim();
|
|
23
|
+
}
|
|
24
|
+
/** Prose lines long enough to be meaningful (outside fenced code, not headings), by key. */
|
|
25
|
+
function proseLines(text) {
|
|
26
|
+
const out = new Map();
|
|
27
|
+
for (const { line, inFence } of linesWithFences(text)) {
|
|
28
|
+
const key = lineKey(line);
|
|
29
|
+
if (!inFence && key.length >= 30 && !key.startsWith('#'))
|
|
30
|
+
out.set(key, line);
|
|
31
|
+
}
|
|
32
|
+
return out;
|
|
33
|
+
}
|
|
34
|
+
function keysOf(text) {
|
|
35
|
+
return text
|
|
36
|
+
.split('\n')
|
|
37
|
+
.map(lineKey)
|
|
38
|
+
.filter((k) => k.replace(/[#\s]/g, '').length > 0);
|
|
39
|
+
}
|
|
40
|
+
function loadedLines(units) {
|
|
41
|
+
const out = new Map();
|
|
42
|
+
for (const u of contentUnits(units)) {
|
|
43
|
+
const lines = out.get(u.tool) ?? new Map();
|
|
44
|
+
out.set(u.tool, lines);
|
|
45
|
+
for (const key of keysOf(u.text))
|
|
46
|
+
lines.set(key, Math.max(lines.get(key) ?? 0, RANK[u.mode]));
|
|
47
|
+
}
|
|
48
|
+
return out;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Agents that would lose something: every line an agent read before must still be read at least as
|
|
52
|
+
* often, except the `edited` lines, which need only `need(before)`.
|
|
53
|
+
*/
|
|
54
|
+
function check(before, after, edited, need) {
|
|
55
|
+
const lost = [];
|
|
56
|
+
for (const [tool, lines] of before) {
|
|
57
|
+
const now = after.get(tool);
|
|
58
|
+
for (const [key, rank] of lines) {
|
|
59
|
+
if ((now?.get(key) ?? 0) < (edited.has(key) ? need(rank) : rank)) {
|
|
60
|
+
lost.push(tool);
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return lost;
|
|
66
|
+
}
|
|
67
|
+
class Workspace {
|
|
68
|
+
tools;
|
|
69
|
+
state;
|
|
70
|
+
constructor(input, tools) {
|
|
71
|
+
this.tools = tools;
|
|
72
|
+
this.state = this.build(new Map(input.map((f) => [f.path, f])));
|
|
73
|
+
}
|
|
74
|
+
build(files) {
|
|
75
|
+
const units = classify([...files.values()], this.tools);
|
|
76
|
+
return { files, units, loaded: loadedLines(units) };
|
|
77
|
+
}
|
|
78
|
+
get files() {
|
|
79
|
+
return this.state.files;
|
|
80
|
+
}
|
|
81
|
+
get units() {
|
|
82
|
+
return this.state.units;
|
|
83
|
+
}
|
|
84
|
+
/** The state with `edits` applied (null deletes a file). */
|
|
85
|
+
trial(edits) {
|
|
86
|
+
const files = new Map(this.state.files);
|
|
87
|
+
for (const [path, text] of edits) {
|
|
88
|
+
if (text === null)
|
|
89
|
+
files.delete(path);
|
|
90
|
+
else
|
|
91
|
+
files.set(path, { path, text });
|
|
92
|
+
}
|
|
93
|
+
return this.build(files);
|
|
94
|
+
}
|
|
95
|
+
check(next, edited, need) {
|
|
96
|
+
return check(this.state.loaded, next.loaded, new Set(edited), need);
|
|
97
|
+
}
|
|
98
|
+
commit(next) {
|
|
99
|
+
this.state = next;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
export function planFixes(input, opts = {}) {
|
|
103
|
+
// Only repository files are ever changed; user-level files (~/...) are not passed in.
|
|
104
|
+
const ws = new Workspace(input.filter((f) => !f.path.startsWith('~')), opts.tools);
|
|
105
|
+
// Files reached only through @imports (a README, docs) are for people too; they are never changed.
|
|
106
|
+
const readOnly = (path) => !isInstructionFile(path)
|
|
107
|
+
? `${path} is not an agent instruction file (it is @imported); it is left alone.`
|
|
108
|
+
: opts.readOnly?.(path)
|
|
109
|
+
? `${path} is a symlink, in a symlinked folder or the target of a symlink; it is left alone.`
|
|
110
|
+
: null;
|
|
111
|
+
const original = new Map(ws.files);
|
|
112
|
+
const renames = new Map();
|
|
113
|
+
const fixes = [];
|
|
114
|
+
const skipped = [];
|
|
115
|
+
if (!opts.tools || opts.tools.includes('cursor'))
|
|
116
|
+
renameCursorRules(ws, readOnly, renames, fixes, skipped);
|
|
117
|
+
// Moves run before duplicate removal: a move may relax how often lines load, so a line must not be
|
|
118
|
+
// removed elsewhere on the strength of a copy that is moved later.
|
|
119
|
+
moveScopableSections(ws, opts, readOnly, fixes, skipped);
|
|
120
|
+
removeDuplicates(ws, readOnly, fixes, skipped);
|
|
121
|
+
const renamedTo = new Map([...renames].map(([from, to]) => [to, from]));
|
|
122
|
+
const changes = [];
|
|
123
|
+
for (const [path, file] of ws.files) {
|
|
124
|
+
const from = renamedTo.get(path);
|
|
125
|
+
const before = original.get(from ?? path)?.text ?? null;
|
|
126
|
+
if (from)
|
|
127
|
+
changes.push({ path, before, after: file.text, renamedFrom: from });
|
|
128
|
+
else if (before !== file.text)
|
|
129
|
+
changes.push({ path, before, after: file.text });
|
|
130
|
+
}
|
|
131
|
+
changes.sort((a, b) => a.path.localeCompare(b.path));
|
|
132
|
+
return { fixes, skipped, changes };
|
|
133
|
+
}
|
|
134
|
+
/** `.md` files in `.cursor/rules` are ignored by Cursor; `.mdc` is the documented extension. */
|
|
135
|
+
function renameCursorRules(ws, readOnly, renames, fixes, skipped) {
|
|
136
|
+
for (const f of [...ws.files.values()]) {
|
|
137
|
+
if (!/^(?:.*\/)?\.cursor\/rules\/.+\.md$/.test(f.path))
|
|
138
|
+
continue;
|
|
139
|
+
const to = f.path.replace(/\.md$/, '.mdc');
|
|
140
|
+
if (readOnly(f.path) || ws.files.has(to)) {
|
|
141
|
+
skipped.push({ kind: 'misconfigured', paths: [f.path], reason: readOnly(f.path) ?? `${to} already exists.` });
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
const next = ws.trial(new Map([[f.path, null], [to, f.text]]));
|
|
145
|
+
const lost = ws.check(next, [], (r) => r);
|
|
146
|
+
if (lost.length) {
|
|
147
|
+
skipped.push({ kind: 'misconfigured', paths: [f.path], reason: `${names(lost)} would stop reading it.` });
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
const mode = next.units.find((u) => u.tool === 'cursor' && u.path === to && u.part !== 'description')?.mode;
|
|
151
|
+
ws.commit(next);
|
|
152
|
+
renames.set(f.path, to);
|
|
153
|
+
fixes.push({
|
|
154
|
+
kind: 'misconfigured',
|
|
155
|
+
paths: [f.path, to],
|
|
156
|
+
message: `Renamed ${f.path} to ${basename(to)} so Cursor reads it` +
|
|
157
|
+
(mode === 'manual' ? '; without frontmatter it applies only when @-mentioned (add alwaysApply, globs or description).' : '.'),
|
|
158
|
+
savingTokens: null,
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
/** Indexes of body lines (after any frontmatter) outside fenced code. */
|
|
163
|
+
function bodyLines(text) {
|
|
164
|
+
const start = text.split('\n').length - parseFrontmatter(text).body.split('\n').length;
|
|
165
|
+
return linesWithFences(text)
|
|
166
|
+
.map(({ line, inFence }, index) => ({ line, inFence, index }))
|
|
167
|
+
.filter((l) => !l.inFence && l.index >= start);
|
|
168
|
+
}
|
|
169
|
+
/** Drops the given line indexes; a blank line left next to another blank line where a line was removed goes too. */
|
|
170
|
+
function dropLines(lines, drop) {
|
|
171
|
+
const kept = [];
|
|
172
|
+
let removed = false;
|
|
173
|
+
lines.forEach((line, i) => {
|
|
174
|
+
if (drop.has(i)) {
|
|
175
|
+
removed = true;
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
if (removed && line.trim() === '' && (kept.length === 0 || kept[kept.length - 1].trim() === ''))
|
|
179
|
+
return;
|
|
180
|
+
removed = false;
|
|
181
|
+
kept.push(line);
|
|
182
|
+
});
|
|
183
|
+
return kept;
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Removes a line from a file when every agent that reads the file also reads the same line every turn
|
|
187
|
+
* from another file. Files read by fewer agents are trimmed first, so the broadest copy stays.
|
|
188
|
+
*/
|
|
189
|
+
function removeDuplicates(ws, readOnly, fixes, skipped) {
|
|
190
|
+
const readers = (path) => new Set(contentUnits(ws.units).filter((u) => u.path === path).map((u) => u.tool));
|
|
191
|
+
const order = [...ws.files.keys()].sort((a, b) => readers(a).size - readers(b).size || b.localeCompare(a));
|
|
192
|
+
for (const path of order) {
|
|
193
|
+
const tools = readers(path);
|
|
194
|
+
// Skill, command and subagent bodies are procedures or separate prompts; they are left whole.
|
|
195
|
+
if (!tools.size || ws.units.some((u) => u.path === path && u.part === 'body'))
|
|
196
|
+
continue;
|
|
197
|
+
const { text, eol } = lf(ws.files.get(path).text);
|
|
198
|
+
const elsewhere = (key) => contentUnits(ws.units).filter((u) => u.path !== path && u.mode === 'always' && proseLines(u.text).has(key));
|
|
199
|
+
// Lines this file shares with another file that some agent reads every turn.
|
|
200
|
+
const shared = new Map();
|
|
201
|
+
const blockers = new Set();
|
|
202
|
+
const removable = new Set();
|
|
203
|
+
for (const [key] of proseLines(text)) {
|
|
204
|
+
const others = elsewhere(key);
|
|
205
|
+
if (!others.length)
|
|
206
|
+
continue;
|
|
207
|
+
shared.set(key, [...new Set(others.map((u) => u.path))]);
|
|
208
|
+
const missing = [...tools].filter((t) => !others.some((u) => u.tool === t));
|
|
209
|
+
missing.forEach((t) => blockers.add(t));
|
|
210
|
+
if (!missing.length)
|
|
211
|
+
removable.add(key);
|
|
212
|
+
}
|
|
213
|
+
if (!shared.size)
|
|
214
|
+
continue;
|
|
215
|
+
const otherPaths = [...new Set([...shared.values()].flat())];
|
|
216
|
+
if (readOnly(path)) {
|
|
217
|
+
skipped.push({ kind: 'duplicate', paths: [path, ...otherPaths], reason: readOnly(path) });
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
if (removable.size) {
|
|
221
|
+
const lines = text.split('\n');
|
|
222
|
+
const drop = new Set(bodyLines(text).filter((l) => removable.has(lineKey(l.line))).map((l) => l.index));
|
|
223
|
+
const removed = [...drop].map((i) => lines[i]);
|
|
224
|
+
const next = ws.trial(new Map([[path, dropLines(lines, drop).join('\n').replace(/\n/g, eol)]]));
|
|
225
|
+
const lost = ws.check(next, removable, (r) => r);
|
|
226
|
+
if (lost.length) {
|
|
227
|
+
skipped.push({ kind: 'duplicate', paths: [path, ...otherPaths], reason: `Kept ${removable.size} line(s) in ${path}: ${names(lost)} would lose context without them (for example a file they @import).` });
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
const everyTurn = contentUnits(ws.units).some((u) => u.path === path && u.mode === 'always');
|
|
231
|
+
const from = [...new Set([...removable].flatMap((k) => shared.get(k)))];
|
|
232
|
+
ws.commit(next);
|
|
233
|
+
fixes.push({
|
|
234
|
+
kind: 'duplicate',
|
|
235
|
+
paths: [path, ...from],
|
|
236
|
+
message: `Removed ${removed.length} line(s) from ${path} that ${names(tools)} already read every turn from ${from.join(', ')}.`,
|
|
237
|
+
savingTokens: everyTurn ? countO200k(removed.join('\n')) : 0,
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
const kept = shared.size - removable.size;
|
|
242
|
+
if (kept && blockers.size)
|
|
243
|
+
skipped.push({
|
|
244
|
+
kind: 'duplicate',
|
|
245
|
+
paths: [path, ...otherPaths],
|
|
246
|
+
reason: `Kept ${kept} line(s) in ${path} that are also in ${otherPaths.join(', ')}: ${names(blockers)} would lose them.`,
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/** Where a section about `scope` can live so it loads only there: the same file name nested, AGENTS.md, or a Claude rule. */
|
|
251
|
+
function targets(source, scope, heading) {
|
|
252
|
+
const name = basename(source);
|
|
253
|
+
const slug = heading.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') || 'section';
|
|
254
|
+
const out = [];
|
|
255
|
+
if (['CLAUDE.md', 'AGENTS.md', 'GEMINI.md'].includes(name))
|
|
256
|
+
out.push({ path: `${scope}/${name}`, header: '' });
|
|
257
|
+
if (name !== 'AGENTS.md')
|
|
258
|
+
out.push({ path: `${scope}/AGENTS.md`, header: '' });
|
|
259
|
+
out.push({ path: `.claude/rules/${slug}.md`, header: `---\npaths:\n - "${scope}/**"\n---\n\n` });
|
|
260
|
+
return out;
|
|
261
|
+
}
|
|
262
|
+
const headingLevel = (line) => /^\s{0,3}(#{1,6})\s/.exec(line)?.[1].length ?? 0;
|
|
263
|
+
/** Level 2 and 3 sections of the LF text, each running to the next heading of the same or a higher level. */
|
|
264
|
+
function movableSections(text) {
|
|
265
|
+
const lines = text.split('\n');
|
|
266
|
+
const all = splitSections(text).filter((s) => s.heading !== '(top)');
|
|
267
|
+
return all
|
|
268
|
+
.map((s) => ({ s, level: headingLevel(lines[s.line - 1]) }))
|
|
269
|
+
.filter((x) => x.level >= 2)
|
|
270
|
+
.map(({ s, level }) => {
|
|
271
|
+
const next = all.find((o) => o.line > s.line && headingLevel(lines[o.line - 1]) <= level);
|
|
272
|
+
const end = next ? next.line - 1 : lines.length;
|
|
273
|
+
return { heading: s.heading, start: s.line - 1, end, body: lines.slice(s.line - 1, end).join('\n').replace(/\n+$/, '') };
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
function moveScopableSections(ws, opts, readOnly, fixes, skipped) {
|
|
277
|
+
const sources = [...new Set(ws.units.filter((u) => u.mode === 'always' && u.part === 'full').map((u) => u.path))].sort();
|
|
278
|
+
const tried = new Set();
|
|
279
|
+
for (const path of sources) {
|
|
280
|
+
// Sections are found in the raw file so that comments and formatting move as they are.
|
|
281
|
+
for (;;) {
|
|
282
|
+
const { text, eol } = lf(ws.files.get(path).text);
|
|
283
|
+
const next = movableSections(text)
|
|
284
|
+
.map((x) => ({ ...x, scope: sectionScope(x.body) }))
|
|
285
|
+
.find((x) => x.scope && !tried.has(`${path}\0${x.body}`));
|
|
286
|
+
if (!next)
|
|
287
|
+
break;
|
|
288
|
+
tried.add(`${path}\0${next.body}`);
|
|
289
|
+
const { heading, body, start, end } = next;
|
|
290
|
+
const scope = next.scope;
|
|
291
|
+
const reason = readOnly(path) ?? moveBlocker(body, scope, opts);
|
|
292
|
+
if (reason) {
|
|
293
|
+
skipped.push({ kind: 'scopable-section', paths: [path], reason: `Section "${heading}": ${reason}` });
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
const lines = text.split('\n');
|
|
297
|
+
const head = lines.slice(0, start);
|
|
298
|
+
const tail = lines.slice(end);
|
|
299
|
+
while (head.length && tail.length && head[head.length - 1].trim() === '' && tail[0].trim() === '')
|
|
300
|
+
tail.shift();
|
|
301
|
+
const rest = [...head, ...tail].join('\n').replace(/\n*$/, '\n');
|
|
302
|
+
let moved = false;
|
|
303
|
+
const refused = [];
|
|
304
|
+
for (const t of targets(path, scope, heading)) {
|
|
305
|
+
const existing = ws.files.get(t.path);
|
|
306
|
+
if (t.path === path || readOnly(t.path) || (existing && t.header))
|
|
307
|
+
continue;
|
|
308
|
+
const target = existing ? lf(existing.text) : { text: '', eol };
|
|
309
|
+
const targetText = existing ? `${target.text.replace(/\n+$/, '')}\n\n${body}\n` : `${t.header}${body}\n`;
|
|
310
|
+
const trial = ws.trial(new Map([
|
|
311
|
+
[path, rest.replace(/\n/g, eol)],
|
|
312
|
+
[t.path, targetText.replace(/\n/g, target.eol)],
|
|
313
|
+
]));
|
|
314
|
+
// Agents that loaded the section every turn may now load it only when working in that folder.
|
|
315
|
+
const lost = ws.check(trial, keysOf(body), (r) => Math.min(r, RANK['on-demand']));
|
|
316
|
+
if (lost.length) {
|
|
317
|
+
refused.push(`${t.path} (${names(lost)} would lose context)`);
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
const readers = [...new Set(ws.units.filter((u) => u.path === path && u.mode === 'always' && u.part !== 'description').map((u) => u.tool))];
|
|
321
|
+
const reads = (l, tool) => keysOf(body).some((k) => l.get(tool)?.has(k));
|
|
322
|
+
const added = [...trial.loaded.keys()].filter((tool) => !reads(ws.state.loaded, tool) && reads(trial.loaded, tool));
|
|
323
|
+
ws.commit(trial);
|
|
324
|
+
fixes.push({
|
|
325
|
+
kind: 'scopable-section',
|
|
326
|
+
paths: [path, t.path],
|
|
327
|
+
message: `Moved section "${heading}" from ${path} to ${t.path}${existing ? '' : ' (new file)'}; ${names(readers)} load it only when working in ${scope}/.` +
|
|
328
|
+
(added.length ? ` Now also read there by ${names(added)}.` : ''),
|
|
329
|
+
savingTokens: countO200k(body),
|
|
330
|
+
});
|
|
331
|
+
moved = true;
|
|
332
|
+
break;
|
|
333
|
+
}
|
|
334
|
+
if (!moved)
|
|
335
|
+
skipped.push({
|
|
336
|
+
kind: 'scopable-section',
|
|
337
|
+
paths: [path],
|
|
338
|
+
reason: `Section "${heading}": no target keeps every agent's context: ${refused.join('; ')}. Use --tools to choose the agents to keep it for.`,
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
function moveBlocker(body, scope, opts) {
|
|
344
|
+
if (scope.split('/').some((seg) => seg === '' || seg === '.' || seg === '..'))
|
|
345
|
+
return `unusual folder ${scope}.`;
|
|
346
|
+
if (opts.exists && !opts.exists(scope))
|
|
347
|
+
return `folder ${scope}/ does not exist.`;
|
|
348
|
+
if (findImports(body).length)
|
|
349
|
+
return 'it has @imports, which resolve relative to the file they are in.';
|
|
350
|
+
if (/\]\((?!https?:|#|\/|mailto:)[^)]+\)/.test(body))
|
|
351
|
+
return 'it has relative links, which resolve relative to the file they are in.';
|
|
352
|
+
return null;
|
|
353
|
+
}
|
|
354
|
+
/** Unified diff of the planned changes, in the format `git apply` accepts. */
|
|
355
|
+
export function unifiedDiff(changes) {
|
|
356
|
+
let out = '';
|
|
357
|
+
for (const c of changes) {
|
|
358
|
+
const from = c.renamedFrom ?? c.path;
|
|
359
|
+
out += `diff --git a/${from} b/${c.path}\n`;
|
|
360
|
+
if (c.renamedFrom) {
|
|
361
|
+
if (c.before === c.after)
|
|
362
|
+
out += 'similarity index 100%\n';
|
|
363
|
+
out += `rename from ${c.renamedFrom}\nrename to ${c.path}\n`;
|
|
364
|
+
if (c.before === c.after)
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
if (c.before === null)
|
|
368
|
+
out += 'new file mode 100644\n';
|
|
369
|
+
out += `--- ${c.before === null ? '/dev/null' : `a/${from}`}\n+++ b/${c.path}\n`;
|
|
370
|
+
out += hunks(c.before ?? '', c.after);
|
|
371
|
+
}
|
|
372
|
+
return out;
|
|
373
|
+
}
|
|
374
|
+
function split(text) {
|
|
375
|
+
if (text === '')
|
|
376
|
+
return [];
|
|
377
|
+
const lines = text.split('\n');
|
|
378
|
+
const noEol = lines[lines.length - 1] !== '';
|
|
379
|
+
if (!noEol)
|
|
380
|
+
lines.pop();
|
|
381
|
+
return lines.map((t, i) => ({ text: t, noEol: noEol && i === lines.length - 1 }));
|
|
382
|
+
}
|
|
383
|
+
/** Line diff by longest common subsequence; instruction files are small. */
|
|
384
|
+
function diffLines(a, b) {
|
|
385
|
+
const n = a.length;
|
|
386
|
+
const m = b.length;
|
|
387
|
+
const same = (i, j) => a[i].text === b[j].text && a[i].noEol === b[j].noEol;
|
|
388
|
+
if (n * m > 4_000_000)
|
|
389
|
+
return [...a.map((l) => ({ op: '-', ...l })), ...b.map((l) => ({ op: '+', ...l }))];
|
|
390
|
+
const lcs = Array.from({ length: n + 1 }, () => new Uint32Array(m + 1));
|
|
391
|
+
for (let i = n - 1; i >= 0; i--)
|
|
392
|
+
for (let j = m - 1; j >= 0; j--)
|
|
393
|
+
lcs[i][j] = same(i, j) ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
|
|
394
|
+
const out = [];
|
|
395
|
+
let i = 0;
|
|
396
|
+
let j = 0;
|
|
397
|
+
while (i < n || j < m) {
|
|
398
|
+
if (i < n && j < m && same(i, j)) {
|
|
399
|
+
out.push({ op: ' ', ...a[i++] });
|
|
400
|
+
j++;
|
|
401
|
+
}
|
|
402
|
+
else if (i < n && (j === m || lcs[i + 1][j] >= lcs[i][j + 1]))
|
|
403
|
+
out.push({ op: '-', ...a[i++] });
|
|
404
|
+
else
|
|
405
|
+
out.push({ op: '+', ...b[j++] });
|
|
406
|
+
}
|
|
407
|
+
return out;
|
|
408
|
+
}
|
|
409
|
+
function hunks(before, after, context = 3) {
|
|
410
|
+
const lines = diffLines(split(before), split(after));
|
|
411
|
+
const changed = lines.map((l, i) => (l.op === ' ' ? -1 : i)).filter((i) => i >= 0);
|
|
412
|
+
let out = '';
|
|
413
|
+
let k = 0;
|
|
414
|
+
while (k < changed.length) {
|
|
415
|
+
const start = Math.max(0, changed[k] - context);
|
|
416
|
+
let end = Math.min(lines.length, changed[k] + context + 1);
|
|
417
|
+
while (k + 1 < changed.length && changed[k + 1] - context <= end)
|
|
418
|
+
end = Math.min(lines.length, changed[++k] + context + 1);
|
|
419
|
+
k++;
|
|
420
|
+
const slice = lines.slice(start, end);
|
|
421
|
+
const prefix = lines.slice(0, start);
|
|
422
|
+
const oldStart = prefix.filter((l) => l.op !== '+').length;
|
|
423
|
+
const newStart = prefix.filter((l) => l.op !== '-').length;
|
|
424
|
+
const oldLen = slice.filter((l) => l.op !== '+').length;
|
|
425
|
+
const newLen = slice.filter((l) => l.op !== '-').length;
|
|
426
|
+
out += `@@ -${oldLen ? oldStart + 1 : oldStart},${oldLen} +${newLen ? newStart + 1 : newStart},${newLen} @@\n`;
|
|
427
|
+
for (const l of slice)
|
|
428
|
+
out += `${l.op}${l.text}\n${l.noEol ? '\\n' : ''}`;
|
|
429
|
+
}
|
|
430
|
+
return out;
|
|
431
|
+
}
|