tina4-nodejs 3.13.60 → 3.13.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +1 -1
- package/package.json +1 -1
- package/packages/core/src/context/chunker.ts +161 -0
- package/packages/core/src/context/index.ts +424 -0
- package/packages/core/src/devAdmin.ts +15 -0
- package/packages/core/src/index.ts +2 -0
- package/packages/core/src/mcp.ts +44 -1
- package/packages/orm/src/baseModel.ts +29 -1
package/CLAUDE.md
CHANGED
|
@@ -777,7 +777,7 @@ bindDatabase(await createAdapterFromUrl("postgres://localhost:5432/analytics"),
|
|
|
777
777
|
// silently falling back to the default. (initDatabase / the internal setAdapter are unchanged.)
|
|
778
778
|
```
|
|
779
779
|
|
|
780
|
-
**Soft delete:** set `static softDelete = true`.
|
|
780
|
+
**Soft delete:** set `static softDelete = true`. Server boot (`syncModels()`) adds the `is_deleted` INTEGER column (0/1) — but **`Model.createTable()` does not**, so declare it there yourself. `delete()` flips the flag, `forceDelete()` removes the row, `restore()` clears it.
|
|
781
781
|
|
|
782
782
|
## Module: QueryBuilder (`packages/orm/src/queryBuilder.ts`)
|
|
783
783
|
|
package/package.json
CHANGED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
// Tina4 Node.js
|
|
2
|
+
//
|
|
3
|
+
// Text folding + chunking for the Context subsystem.
|
|
4
|
+
//
|
|
5
|
+
// A thin, idiomatic TypeScript port of tina4-python's context/chunker.py
|
|
6
|
+
// (itself the proven slice of neemee's tokenizer/chunkers). Zero dependencies —
|
|
7
|
+
// nothing here touches SQLite; it produces the (folded body, raw text) pairs the
|
|
8
|
+
// FTS5 index stores.
|
|
9
|
+
|
|
10
|
+
// join comma-grouped numbers ("24,601" -> "24601") and split camelCase
|
|
11
|
+
// ("ForeignKeyField" -> "foreign key field") so a query token reaches a
|
|
12
|
+
// code identifier. snake_case already splits for free at the tokenizer.
|
|
13
|
+
const NUM_COMMA = /(?<=\d),(?=\d)/g;
|
|
14
|
+
const CAMEL = /(?<=[a-z0-9])(?=[A-Z])/g;
|
|
15
|
+
const WORD_RE = /[a-z0-9]+/g;
|
|
16
|
+
|
|
17
|
+
// Sentence boundary = end punctuation FOLLOWED BY whitespace/EOL, or a newline.
|
|
18
|
+
// NOT a bare '.', which would shred embedded code in prose ("db.fetch()" ->
|
|
19
|
+
// "db. fetch()"). Intra-token dots (method calls, module paths) stay intact.
|
|
20
|
+
const SENT = /(?<=[.!?])\s+|\n+/;
|
|
21
|
+
|
|
22
|
+
// Chunk boundaries across languages: python defs/classes/decorators, php/js
|
|
23
|
+
// functions and methods, ts exports/interfaces, php traits, and Object Pascal
|
|
24
|
+
// unit/type/routine headers (Delphi is case-insensitive, accept either case).
|
|
25
|
+
// Anchored at line start; tested per line (parity with python's re.match).
|
|
26
|
+
const TOPLEVEL =
|
|
27
|
+
/^(async def |def |class |@\w|\s*(?:public |private |protected |static |final |abstract )*function \w|(?:export )?(?:default )?(?:abstract )?class |interface |trait |export (?:const|function|interface|type|async) |[Uu]nit |[Pp]rocedure |[Ff]unction |[Cc]onstructor |[Dd]estructor |[Tt]ype$)/;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Lowercase, strip diacritics, join comma-grouped numbers, split camelCase.
|
|
31
|
+
*
|
|
32
|
+
* Applied symmetrically to the indexed `body` and to query tokens so matching
|
|
33
|
+
* is consistent: a query for `field` reaches `IntegerField`. The trailing
|
|
34
|
+
* NFKD-normalise + strip-non-ASCII mirrors python's
|
|
35
|
+
* `unicodedata.normalize("NFKD", ...).encode("ascii", "ignore")`.
|
|
36
|
+
*/
|
|
37
|
+
export function fold(text: string): string {
|
|
38
|
+
const split = text.replace(NUM_COMMA, "").replace(CAMEL, " ").toLowerCase();
|
|
39
|
+
// eslint-disable-next-line no-control-regex
|
|
40
|
+
return split.normalize("NFKD").replace(/[^\x00-\x7f]/g, "");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Fold simple plurals: strip one trailing 's', never from ss/us/is endings
|
|
45
|
+
* (class, status, axis). QUERY-side only — so `fields` in a question also
|
|
46
|
+
* reaches `Field` definitions.
|
|
47
|
+
*/
|
|
48
|
+
export function lightStem(token: string): string {
|
|
49
|
+
if (token.length > 3 && token.endsWith("s") && !/(?:ss|us|is)$/.test(token)) {
|
|
50
|
+
return token.slice(0, -1);
|
|
51
|
+
}
|
|
52
|
+
return token;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Accent-folded lowercase alphanumeric tokens (digits kept). */
|
|
56
|
+
export function terms(text: string): string[] {
|
|
57
|
+
return fold(text).match(WORD_RE) ?? [];
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Split into lines the way python's `str.splitlines()` does: on any line
|
|
62
|
+
* terminator, WITHOUT a trailing empty element for a final newline. A plain
|
|
63
|
+
* `text.split("\n")` would append a spurious "" for "a\n" — this drops it so
|
|
64
|
+
* chunk boundaries and line counts match the reference.
|
|
65
|
+
*/
|
|
66
|
+
function splitLines(text: string): string[] {
|
|
67
|
+
const lines = text.split(/\r\n|\r|\n/);
|
|
68
|
+
if (lines.length > 0 && lines[lines.length - 1] === "" && /[\r\n]$/.test(text)) {
|
|
69
|
+
lines.pop();
|
|
70
|
+
}
|
|
71
|
+
return lines;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Pack line-segments into chunks of at most `maxLines` lines, hard-splitting
|
|
76
|
+
* any single oversized segment.
|
|
77
|
+
*/
|
|
78
|
+
function pack(segments: string[][], maxLines: number): string[][] {
|
|
79
|
+
const chunks: string[][] = [];
|
|
80
|
+
let cur: string[] = [];
|
|
81
|
+
for (let seg of segments) {
|
|
82
|
+
while (seg.length > maxLines) {
|
|
83
|
+
if (cur.length) {
|
|
84
|
+
chunks.push(cur);
|
|
85
|
+
cur = [];
|
|
86
|
+
}
|
|
87
|
+
chunks.push(seg.slice(0, maxLines));
|
|
88
|
+
seg = seg.slice(maxLines);
|
|
89
|
+
}
|
|
90
|
+
if (cur.length && cur.length + seg.length > maxLines) {
|
|
91
|
+
chunks.push(cur);
|
|
92
|
+
cur = [];
|
|
93
|
+
}
|
|
94
|
+
cur = cur.concat(seg);
|
|
95
|
+
}
|
|
96
|
+
if (cur.length) {
|
|
97
|
+
chunks.push(cur);
|
|
98
|
+
}
|
|
99
|
+
return chunks;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Chunk source on top-level def/class/decorator boundaries (sentence chunking
|
|
104
|
+
* shreds code). Segments are packed up to `maxLines`, and every chunk starts
|
|
105
|
+
* with a `# file: <path>` line so the path's tokens are indexed — 'where is the
|
|
106
|
+
* router?' should match core/router.ts by name.
|
|
107
|
+
*
|
|
108
|
+
* Returns a list of `[index, chunkText]` pairs.
|
|
109
|
+
*/
|
|
110
|
+
export function chunkCode(text: string, path = "", maxLines = 60): Array<[number, string]> {
|
|
111
|
+
const lines = splitLines(text);
|
|
112
|
+
let bounds: number[] = [];
|
|
113
|
+
for (let i = 0; i < lines.length; i++) {
|
|
114
|
+
if (TOPLEVEL.test(lines[i])) bounds.push(i);
|
|
115
|
+
}
|
|
116
|
+
if (bounds.length === 0 || bounds[0] !== 0) {
|
|
117
|
+
bounds = [0, ...bounds];
|
|
118
|
+
}
|
|
119
|
+
const ends = [...bounds.slice(1), lines.length];
|
|
120
|
+
const segments: string[][] = [];
|
|
121
|
+
for (let k = 0; k < bounds.length; k++) {
|
|
122
|
+
segments.push(lines.slice(bounds[k], ends[k]));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const header = path ? `# file: ${path}` : null;
|
|
126
|
+
const packed = pack(segments, maxLines);
|
|
127
|
+
const out: Array<[number, string]> = [];
|
|
128
|
+
for (let i = 0; i < packed.length; i++) {
|
|
129
|
+
const body = (header ? [header, ...packed[i]] : packed[i]).join("\n");
|
|
130
|
+
out.push([i, body]);
|
|
131
|
+
}
|
|
132
|
+
return out;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Chunk prose/docs into sentence-packed windows of at most `maxWords` words.
|
|
137
|
+
* Returns a list of `[index, chunkText]` pairs.
|
|
138
|
+
*/
|
|
139
|
+
export function chunkText(text: string, maxWords = 350): Array<[number, string]> {
|
|
140
|
+
const sents = text
|
|
141
|
+
.split(SENT)
|
|
142
|
+
.map((s) => s.trim())
|
|
143
|
+
.filter((s) => s.length > 0);
|
|
144
|
+
const chunks: string[] = [];
|
|
145
|
+
let cur: string[] = [];
|
|
146
|
+
let curWords = 0;
|
|
147
|
+
for (const s of sents) {
|
|
148
|
+
const w = s.split(/\s+/).filter(Boolean).length;
|
|
149
|
+
if (cur.length && curWords + w > maxWords) {
|
|
150
|
+
chunks.push(cur.join(" "));
|
|
151
|
+
cur = [];
|
|
152
|
+
curWords = 0;
|
|
153
|
+
}
|
|
154
|
+
cur.push(s);
|
|
155
|
+
curWords += w;
|
|
156
|
+
}
|
|
157
|
+
if (cur.length) {
|
|
158
|
+
chunks.push(cur.join(" "));
|
|
159
|
+
}
|
|
160
|
+
return chunks.map((c, i): [number, string] => [i, c]);
|
|
161
|
+
}
|
|
@@ -0,0 +1,424 @@
|
|
|
1
|
+
// Tina4 Node.js
|
|
2
|
+
//
|
|
3
|
+
// Context — a native, zero-dependency code/doc grounding index.
|
|
4
|
+
//
|
|
5
|
+
// Lets a Tina4 app ground its own AI assistant on its own source, offline:
|
|
6
|
+
// it walks the project, chunks code on def/class boundaries and docs as prose,
|
|
7
|
+
// and answers keyword/fuzzy queries over a SQLite FTS5 index (Node's built-in
|
|
8
|
+
// `node:sqlite` — FTS5 + `bm25()` are compiled in, so NO new dependency; the
|
|
9
|
+
// same module the ORM/session/docstore subsystems already use).
|
|
10
|
+
//
|
|
11
|
+
// import { Context } from "@tina4/core";
|
|
12
|
+
// const ctx = new Context(".tina4/context.db");
|
|
13
|
+
// ctx.indexRoot("src");
|
|
14
|
+
// ctx.search("where is the auth token issued?", 5);
|
|
15
|
+
// // -> [{ path: "src/auth.ts", score: 2.31, snippet: "..." }, ...]
|
|
16
|
+
//
|
|
17
|
+
// A faithful TypeScript port of tina4-python's tina4_python/context/__init__.py
|
|
18
|
+
// (the proven slice of neemee's SqliteFTS + source-over-tests / definition-first
|
|
19
|
+
// reordering). It COMPLEMENTS the api_* reflection tools: api_* is exact
|
|
20
|
+
// structural lookup, Context is fuzzy/semantic FTS over source + docs.
|
|
21
|
+
//
|
|
22
|
+
// On-disk index defaults to `.tina4/context.db` (gitignored). Guards a sqlite
|
|
23
|
+
// build without FTS5: if absent, the Context degrades to safe no-ops rather than
|
|
24
|
+
// crashing the app.
|
|
25
|
+
//
|
|
26
|
+
// node:sqlite is synchronous and JavaScript is single-threaded, so — unlike the
|
|
27
|
+
// Python port — no lock is needed: every method runs to completion before the
|
|
28
|
+
// event loop can dispatch another.
|
|
29
|
+
|
|
30
|
+
import { DatabaseSync } from "node:sqlite";
|
|
31
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, realpathSync, statSync } from "node:fs";
|
|
32
|
+
import { basename, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
33
|
+
|
|
34
|
+
import { chunkCode, chunkText, fold, lightStem, terms } from "./chunker.js";
|
|
35
|
+
|
|
36
|
+
// File classification — mirrors neemee's repo walk.
|
|
37
|
+
const CODE_EXTS = new Set([
|
|
38
|
+
".py", ".php", ".js", ".mjs", ".ts", ".rb",
|
|
39
|
+
".pas", ".dpr", ".dpk", ".inc", ".dfm", ".fmx",
|
|
40
|
+
]);
|
|
41
|
+
const DOC_EXTS = new Set([".md", ".txt", ".rst", ".twig", ".html"]);
|
|
42
|
+
// deploy/CLI/env answers live in config files, not sources — chunk as code
|
|
43
|
+
// (line windows), since sentence chunking shreds YAML/Dockerfiles.
|
|
44
|
+
const CONFIG_EXTS = new Set([".toml", ".yml", ".yaml"]);
|
|
45
|
+
const SPECIAL_FILES = new Set([
|
|
46
|
+
"dockerfile", "makefile", "docker-compose.yml",
|
|
47
|
+
"package.json", "composer.json",
|
|
48
|
+
".env.example", ".env.sample",
|
|
49
|
+
]);
|
|
50
|
+
// Same dirs neemee skips, plus Tina4 runtime dirs that hold no source of truth
|
|
51
|
+
// (our own index/backups, session blobs, logs).
|
|
52
|
+
const SKIP_DIRS = new Set([
|
|
53
|
+
".git", "__pycache__", "node_modules", "vendor", "dist",
|
|
54
|
+
"build", "coverage", ".idea", ".venv", "venv", ".pytest_cache",
|
|
55
|
+
".tina4", "sessions", "logs",
|
|
56
|
+
]);
|
|
57
|
+
|
|
58
|
+
// generic question/code vocabulary that never NAMES a symbol — kept small.
|
|
59
|
+
const DEF_STOP = new Set(
|
|
60
|
+
("the and what how does can which where when who why list all available " +
|
|
61
|
+
"module class function functions method methods def get set new return " +
|
|
62
|
+
"import from with that this are is was for into use used").split(" "),
|
|
63
|
+
);
|
|
64
|
+
|
|
65
|
+
// a chunk that DEFINES a queried symbol ('def get_token', 'class Widget') should
|
|
66
|
+
// out-rank one that merely USES it. Matched against the folded body.
|
|
67
|
+
const DEF_KW = "(?:async def|def|class|function|fn|func|interface|trait)";
|
|
68
|
+
|
|
69
|
+
/** True if this build of node:sqlite supports FTS5. */
|
|
70
|
+
export function fts5Supported(): boolean {
|
|
71
|
+
try {
|
|
72
|
+
const conn = new DatabaseSync(":memory:");
|
|
73
|
+
try {
|
|
74
|
+
conn.exec("CREATE VIRTUAL TABLE _probe USING fts5(x)");
|
|
75
|
+
return true;
|
|
76
|
+
} finally {
|
|
77
|
+
conn.close();
|
|
78
|
+
}
|
|
79
|
+
} catch {
|
|
80
|
+
return false;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function escapeRegex(s: string): string {
|
|
85
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Resolve `abs` to its canonical (symlink-free) form, matching Python's
|
|
90
|
+
* `Path.resolve()`. If the file itself doesn't exist yet (a delete event), fall
|
|
91
|
+
* back to canonicalising the parent directory and re-joining the basename, then
|
|
92
|
+
* to the lexical path. Needed because macOS temp dirs (and /var) are symlinks,
|
|
93
|
+
* so a raw `process.cwd()` (already real) and a lexically-resolved index root
|
|
94
|
+
* would otherwise mismatch and drop every reindex as "outside root".
|
|
95
|
+
*/
|
|
96
|
+
function realResolve(abs: string): string {
|
|
97
|
+
try {
|
|
98
|
+
return realpathSync(abs);
|
|
99
|
+
} catch {
|
|
100
|
+
/* not created (deleted file) — fall through */
|
|
101
|
+
}
|
|
102
|
+
try {
|
|
103
|
+
return join(realpathSync(dirname(abs)), basename(abs));
|
|
104
|
+
} catch {
|
|
105
|
+
return abs;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export interface SearchHit {
|
|
110
|
+
path: string;
|
|
111
|
+
score: number;
|
|
112
|
+
snippet: string;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
interface ChunkRow {
|
|
116
|
+
path: string;
|
|
117
|
+
raw: string;
|
|
118
|
+
body: string;
|
|
119
|
+
s: number;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* A SQLite FTS5 index over a project's source + docs.
|
|
124
|
+
*
|
|
125
|
+
* - indexPath(file, label?) upsert one file (delete-by-path, re-chunk, insert)
|
|
126
|
+
* - indexRoot(root) walk a tree, index every eligible file
|
|
127
|
+
* - search(query, k) [{path, score, snippet}] ranked by bm25() with
|
|
128
|
+
* source-over-tests + definition-first reordering
|
|
129
|
+
* - reindexFile(changed) upsert one changed file against the indexed root
|
|
130
|
+
*/
|
|
131
|
+
export class Context {
|
|
132
|
+
path: string;
|
|
133
|
+
root: string | null = null;
|
|
134
|
+
available: boolean;
|
|
135
|
+
private conn: DatabaseSync | null = null;
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* @param dbPath on-disk index file (its parent dir is created).
|
|
139
|
+
* @param fts5Check overrides FTS5 detection (used by tests to exercise the
|
|
140
|
+
* graceful-degradation path); defaults to a real probe.
|
|
141
|
+
*/
|
|
142
|
+
constructor(dbPath = "./.tina4/context.db", fts5Check?: () => boolean) {
|
|
143
|
+
this.path = String(dbPath);
|
|
144
|
+
const check = fts5Check ?? fts5Supported;
|
|
145
|
+
this.available = Boolean(check());
|
|
146
|
+
if (!this.available) return;
|
|
147
|
+
const parent = dirname(this.path);
|
|
148
|
+
if (parent !== "" && parent !== ".") {
|
|
149
|
+
mkdirSync(parent, { recursive: true });
|
|
150
|
+
}
|
|
151
|
+
this.conn = new DatabaseSync(this.path);
|
|
152
|
+
this.ensureTable();
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Whether this Node build's node:sqlite supports FTS5. */
|
|
156
|
+
static fts5Available(): boolean {
|
|
157
|
+
return fts5Supported();
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
private ensureTable(): void {
|
|
161
|
+
// `body` holds fold(text) so query/index tokenization is symmetric; `raw`
|
|
162
|
+
// (UNINDEXED) keeps the original text to return as a snippet; `path`/`cid`
|
|
163
|
+
// (UNINDEXED) are metadata used for upsert + citation.
|
|
164
|
+
this.conn!.exec(
|
|
165
|
+
"CREATE VIRTUAL TABLE IF NOT EXISTS chunks " +
|
|
166
|
+
"USING fts5(cid UNINDEXED, path UNINDEXED, raw UNINDEXED, body)",
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Drop and recreate the index (full rebuild starting point). */
|
|
171
|
+
reset(): void {
|
|
172
|
+
if (!this.available) return;
|
|
173
|
+
this.conn!.exec("DROP TABLE IF EXISTS chunks");
|
|
174
|
+
this.ensureTable();
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// ── indexing ───────────────────────────────────────────────
|
|
178
|
+
private static chunksFor(label: string, text: string): Array<[number, string]> {
|
|
179
|
+
const ext = extname(label).toLowerCase();
|
|
180
|
+
const special = SPECIAL_FILES.has(basename(label).toLowerCase());
|
|
181
|
+
if (CODE_EXTS.has(ext) || CONFIG_EXTS.has(ext) || special) {
|
|
182
|
+
return chunkCode(text, label);
|
|
183
|
+
}
|
|
184
|
+
return chunkText(text);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* UPSERT one file into the index: delete this path's existing chunks,
|
|
189
|
+
* re-chunk the current contents, insert. `label` is the stored/citation path
|
|
190
|
+
* (defaults to `file`) and MUST be stable across calls for the same file so
|
|
191
|
+
* the delete targets the right rows. Returns rows inserted.
|
|
192
|
+
*/
|
|
193
|
+
indexPath(file: string, label?: string): number {
|
|
194
|
+
if (!this.available) return 0;
|
|
195
|
+
const stored = label != null ? String(label) : String(file);
|
|
196
|
+
let text: string;
|
|
197
|
+
try {
|
|
198
|
+
text = readFileSync(file, "utf-8");
|
|
199
|
+
} catch {
|
|
200
|
+
return 0;
|
|
201
|
+
}
|
|
202
|
+
const rows = Context.chunksFor(stored, text).map(
|
|
203
|
+
([i, chunk]): [string, string, string, string] => [`${stored}:${i}`, stored, chunk, fold(chunk)],
|
|
204
|
+
);
|
|
205
|
+
this.conn!.prepare("DELETE FROM chunks WHERE path = ?").run(stored);
|
|
206
|
+
if (rows.length) {
|
|
207
|
+
const ins = this.conn!.prepare("INSERT INTO chunks(cid, path, raw, body) VALUES (?, ?, ?, ?)");
|
|
208
|
+
for (const r of rows) ins.run(r[0], r[1], r[2], r[3]);
|
|
209
|
+
}
|
|
210
|
+
return rows.length;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* The per-file filter used by both indexRoot and reindexFile. Directory
|
|
215
|
+
* skipping is handled separately.
|
|
216
|
+
*/
|
|
217
|
+
private static eligible(filename: string): boolean {
|
|
218
|
+
const fn = filename.toLowerCase();
|
|
219
|
+
if (fn.endsWith(".min.js")) return false;
|
|
220
|
+
const ext = extname(fn);
|
|
221
|
+
return CODE_EXTS.has(ext) || DOC_EXTS.has(ext) || CONFIG_EXTS.has(ext) || SPECIAL_FILES.has(fn);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Walk `root`, indexing every eligible file (skips vendor/build/runtime
|
|
226
|
+
* dirs). Paths are stored RELATIVE to `root` for clean citations. Records
|
|
227
|
+
* `root` so reindexFile can relabel a changed file consistently. Returns the
|
|
228
|
+
* total number of chunks inserted.
|
|
229
|
+
*/
|
|
230
|
+
indexRoot(root: string): number {
|
|
231
|
+
if (!this.available) return 0;
|
|
232
|
+
const rootAbs = realResolve(resolve(String(root)));
|
|
233
|
+
this.root = rootAbs;
|
|
234
|
+
let total = 0;
|
|
235
|
+
const walk = (dir: string): void => {
|
|
236
|
+
const entries = readdirSync(dir, { withFileTypes: true });
|
|
237
|
+
const files: string[] = [];
|
|
238
|
+
const subdirs: string[] = [];
|
|
239
|
+
for (const e of entries) {
|
|
240
|
+
if (e.isDirectory()) {
|
|
241
|
+
if (!SKIP_DIRS.has(e.name) && !e.name.startsWith(".")) subdirs.push(e.name);
|
|
242
|
+
} else if (e.isFile()) {
|
|
243
|
+
files.push(e.name);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
files.sort();
|
|
247
|
+
for (const fn of files) {
|
|
248
|
+
if (!Context.eligible(fn)) continue;
|
|
249
|
+
const full = join(dir, fn);
|
|
250
|
+
const rel = relative(rootAbs, full);
|
|
251
|
+
total += this.indexPath(full, rel);
|
|
252
|
+
}
|
|
253
|
+
for (const d of subdirs) walk(join(dir, d));
|
|
254
|
+
};
|
|
255
|
+
walk(rootAbs);
|
|
256
|
+
return total;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Re-index a single changed file into the LIVE index — the hook the dev
|
|
261
|
+
* WebSocket reload trigger (POST /__dev/api/reload) calls so code_search
|
|
262
|
+
* tracks edits without a rebuild. Resolves `changedPath` against the indexed
|
|
263
|
+
* root, then: outside root / under a skip-or-dot dir / ineligible → skip (-1);
|
|
264
|
+
* deleted → drop its chunks (0); otherwise UPSERT (rows). No-op (-1) until
|
|
265
|
+
* indexRoot has run (nothing to keep fresh yet).
|
|
266
|
+
*/
|
|
267
|
+
reindexFile(changedPath: string): number {
|
|
268
|
+
if (!this.available || this.root === null) return -1;
|
|
269
|
+
const raw = String(changedPath);
|
|
270
|
+
// the reload trigger reports paths relative to the project root (cwd during
|
|
271
|
+
// `tina4 serve`); the index root may be a subdir like src/.
|
|
272
|
+
const abs = isAbsolute(raw) ? raw : join(process.cwd(), raw);
|
|
273
|
+
const resolved = realResolve(resolve(abs));
|
|
274
|
+
const rel = relative(this.root, resolved);
|
|
275
|
+
if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) {
|
|
276
|
+
return -1; // outside the indexed root
|
|
277
|
+
}
|
|
278
|
+
const parts = rel.split(/[\\/]/);
|
|
279
|
+
const dirParts = parts.slice(0, -1);
|
|
280
|
+
if (parts.some((seg) => SKIP_DIRS.has(seg)) || dirParts.some((seg) => seg.startsWith("."))) {
|
|
281
|
+
return -1; // under a skipped / dot dir
|
|
282
|
+
}
|
|
283
|
+
if (!Context.eligible(basename(rel))) return -1;
|
|
284
|
+
const stored = rel;
|
|
285
|
+
if (!existsSync(abs)) {
|
|
286
|
+
// deleted → drop its chunks
|
|
287
|
+
this.conn!.prepare("DELETE FROM chunks WHERE path = ?").run(stored);
|
|
288
|
+
return 0;
|
|
289
|
+
}
|
|
290
|
+
return this.indexPath(abs, stored);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// ── query ──────────────────────────────────────────────────
|
|
294
|
+
private matchExpr(query: string): { expr: string | null; toks: Set<string> } {
|
|
295
|
+
const toks = new Set<string>();
|
|
296
|
+
for (const t of terms(query)) {
|
|
297
|
+
toks.add(t);
|
|
298
|
+
toks.add(lightStem(t));
|
|
299
|
+
}
|
|
300
|
+
toks.delete("");
|
|
301
|
+
if (toks.size === 0) return { expr: null, toks };
|
|
302
|
+
// Quoting keeps each term injection-safe inside the FTS5 MATCH string.
|
|
303
|
+
const expr = [...toks].sort().map((t) => `"${t}"`).join(" OR ");
|
|
304
|
+
return { expr, toks };
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
private static isTestlike(path: string): boolean {
|
|
308
|
+
const s = (path || "").toLowerCase();
|
|
309
|
+
return ["/test", "test/", "test_", "_test.", ".test.", "/example", "example/"].some((p) => s.includes(p));
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
private static defines(foldedBody: string, symbols: Set<string>): boolean {
|
|
313
|
+
if (symbols.size === 0) return false;
|
|
314
|
+
const alt = [...symbols].map(escapeRegex).join("|");
|
|
315
|
+
const pat = new RegExp(`${DEF_KW}\\s+(?:${alt})(?![a-z0-9])`);
|
|
316
|
+
return pat.test(foldedBody);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* Return the top-`k` chunks as `[{path, score, snippet}]`, ranked by `bm25()`
|
|
321
|
+
* then reordered with two stable, proven passes:
|
|
322
|
+
* - source-over-tests: a test that merely mentions a symbol sinks below the
|
|
323
|
+
* source that defines it (skipped when the query is about tests);
|
|
324
|
+
* - definition-first: a chunk that DEFINES a queried symbol rises above
|
|
325
|
+
* chunks that only use it.
|
|
326
|
+
* Score is a higher-is-better float (sqlite's bm25 sign flipped).
|
|
327
|
+
*/
|
|
328
|
+
search(query: string, k = 5): SearchHit[] {
|
|
329
|
+
if (!this.available) return [];
|
|
330
|
+
const { expr } = this.matchExpr(query);
|
|
331
|
+
if (expr === null) return [];
|
|
332
|
+
const poolN = Math.max(k * 3, 15);
|
|
333
|
+
const rows = this.conn!
|
|
334
|
+
.prepare("SELECT path, raw, body, bm25(chunks) AS s FROM chunks WHERE chunks MATCH ? ORDER BY s LIMIT ?")
|
|
335
|
+
.all(expr, poolN) as unknown as ChunkRow[];
|
|
336
|
+
if (rows.length === 0) return [];
|
|
337
|
+
|
|
338
|
+
// candidate symbol names from the query (drop generic vocab).
|
|
339
|
+
const symbols = new Set(terms(query).filter((t) => t.length >= 3 && !DEF_STOP.has(t)));
|
|
340
|
+
const aboutTests = query.toLowerCase().includes("test");
|
|
341
|
+
|
|
342
|
+
const isTest = (p: string): number => (!aboutTests && Context.isTestlike(p) ? 1 : 0);
|
|
343
|
+
const isDef = (b: string): number => (Context.defines(b, symbols) ? 0 : 1);
|
|
344
|
+
|
|
345
|
+
const ordered = rows
|
|
346
|
+
.map((r, i) => ({ r, i }))
|
|
347
|
+
.sort((a, b) => {
|
|
348
|
+
const at = isTest(a.r.path);
|
|
349
|
+
const bt = isTest(b.r.path);
|
|
350
|
+
if (at !== bt) return at - bt;
|
|
351
|
+
const ad = isDef(a.r.body);
|
|
352
|
+
const bd = isDef(b.r.body);
|
|
353
|
+
if (ad !== bd) return ad - bd;
|
|
354
|
+
return a.i - b.i; // bm25 order breaks ties
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
return ordered.slice(0, k).map(({ r }) => ({
|
|
358
|
+
path: r.path,
|
|
359
|
+
score: Math.round(-Number(r.s) * 1e6) / 1e6, // bm25: more negative = better
|
|
360
|
+
snippet: Context.snippet(r.raw),
|
|
361
|
+
}));
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
private static snippet(raw: string, limit = 280): string {
|
|
365
|
+
const text = (raw || "").trim();
|
|
366
|
+
if (text.length <= limit) return text;
|
|
367
|
+
return text.slice(0, limit).replace(/\s+$/, "") + " ...";
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// ── misc ───────────────────────────────────────────────────
|
|
371
|
+
count(): number {
|
|
372
|
+
if (!this.available) return 0;
|
|
373
|
+
const row = this.conn!.prepare("SELECT count(*) AS c FROM chunks").get() as { c: number };
|
|
374
|
+
return row.c;
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
isEmpty(): boolean {
|
|
378
|
+
return this.count() === 0;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
close(): void {
|
|
382
|
+
if (this.conn !== null) {
|
|
383
|
+
this.conn.close();
|
|
384
|
+
this.conn = null;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// ── process-wide shared index ──────────────────────────────────
|
|
390
|
+
// code_search (dev MCP) and the dev-reload reindex hook must share ONE index so
|
|
391
|
+
// a saved file is immediately searchable. Keyed by resolved db path.
|
|
392
|
+
export const _sharedContexts = new Map<string, Context>();
|
|
393
|
+
|
|
394
|
+
function dbKey(db?: string): string {
|
|
395
|
+
return resolve(db ? String(db) : join(process.cwd(), ".tina4", "context.db"));
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Get (or create) the process-wide Context at `db` (default
|
|
400
|
+
* `<cwd>/.tina4/context.db`). If `root` is given and the index is empty, builds
|
|
401
|
+
* it once. This is what code_search uses so the reload hook can keep the SAME
|
|
402
|
+
* index fresh.
|
|
403
|
+
*/
|
|
404
|
+
export function defaultContext(root?: string, db?: string): Context {
|
|
405
|
+
const key = dbKey(db);
|
|
406
|
+
let ctx = _sharedContexts.get(key);
|
|
407
|
+
if (ctx === undefined) {
|
|
408
|
+
ctx = new Context(key);
|
|
409
|
+
_sharedContexts.set(key, ctx);
|
|
410
|
+
}
|
|
411
|
+
if (root != null && ctx.available && ctx.isEmpty()) {
|
|
412
|
+
ctx.indexRoot(root);
|
|
413
|
+
}
|
|
414
|
+
return ctx;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Return the already-created shared Context for `db` (or undefined). Used by the
|
|
419
|
+
* reload hook so a file change reindexes an EXISTING index but never creates one
|
|
420
|
+
* on its own (nothing to keep fresh until code_search runs).
|
|
421
|
+
*/
|
|
422
|
+
export function existingContext(db?: string): Context | undefined {
|
|
423
|
+
return _sharedContexts.get(dbKey(db));
|
|
424
|
+
}
|
|
@@ -703,6 +703,21 @@ const handleReload: RouteHandler = async (req, res) => {
|
|
|
703
703
|
console.error(` Re-discover on reload failed:`, err);
|
|
704
704
|
}
|
|
705
705
|
|
|
706
|
+
// Keep the code Context index LIVE on the same reload trigger: reindex just
|
|
707
|
+
// the changed file (UPSERT) so the dev-MCP code_search reflects the edit
|
|
708
|
+
// immediately. existingContext() only touches an already-built index (it never
|
|
709
|
+
// creates one — nothing to keep fresh until code_search has run), and the
|
|
710
|
+
// whole block is guarded so a context failure never breaks the reload.
|
|
711
|
+
try {
|
|
712
|
+
if (_reloadFile) {
|
|
713
|
+
const { existingContext } = await import("./context/index.js");
|
|
714
|
+
const ctx = existingContext();
|
|
715
|
+
if (ctx) ctx.reindexFile(_reloadFile);
|
|
716
|
+
}
|
|
717
|
+
} catch (err) {
|
|
718
|
+
console.error(` Context reindex on reload failed:`, err);
|
|
719
|
+
}
|
|
720
|
+
|
|
706
721
|
// WebSocket-primary reload: push an instant message to every browser
|
|
707
722
|
// connected on /__dev_reload. The toolbar client (and the dev-admin
|
|
708
723
|
// dashboard) act on this immediately — the mtime poll is only a fallback for
|
|
@@ -75,6 +75,8 @@ export { responseCache, clearCache, cacheStats, cacheGet, cacheSet, cacheDelete,
|
|
|
75
75
|
export type { ResponseCacheConfig, CacheBackend } from "./cache.js";
|
|
76
76
|
export { Api } from "./api.js";
|
|
77
77
|
export type { ApiResult } from "./api.js";
|
|
78
|
+
export { Context, defaultContext, existingContext, fts5Supported, _sharedContexts } from "./context/index.js";
|
|
79
|
+
export type { SearchHit } from "./context/index.js";
|
|
78
80
|
export { Events } from "./events.js";
|
|
79
81
|
export { DevAdmin, MessageLog, RequestInspector, ErrorTracker, DevMailboxStore, DevQueue, WsTracker, supervisorBaseUrl, devAdminLanguage } from "./devAdmin.js";
|
|
80
82
|
export {
|
package/packages/core/src/mcp.ts
CHANGED
|
@@ -1056,7 +1056,8 @@ function verifyNodeSyntax(absPath: string, relPath: string): string | null {
|
|
|
1056
1056
|
let _lastCacheStats: Record<string, unknown> | null = null;
|
|
1057
1057
|
|
|
1058
1058
|
/**
|
|
1059
|
-
* Register all
|
|
1059
|
+
* Register all built-in dev tools on the given McpServer (the api_* reflection
|
|
1060
|
+
* tools plus code_search, the fuzzy FTS grounding tool).
|
|
1060
1061
|
*/
|
|
1061
1062
|
export function registerDevTools(server: McpServer): void {
|
|
1062
1063
|
const projectRoot = path.resolve(process.cwd());
|
|
@@ -2070,6 +2071,48 @@ export function registerDevTools(server: McpServer): void {
|
|
|
2070
2071
|
{ name: "name", type: "string" },
|
|
2071
2072
|
]),
|
|
2072
2073
|
);
|
|
2074
|
+
|
|
2075
|
+
// ── Code/doc grounding (context subsystem) ──────────────────
|
|
2076
|
+
// Sibling of api_* but the DUAL of it: api_* is exact structural reflection
|
|
2077
|
+
// (class/method signatures); code_search is fuzzy/semantic FTS over the
|
|
2078
|
+
// project's own SOURCE + docs. Use code_search for "where/how is X done in
|
|
2079
|
+
// THIS codebase?" and api_* for "what's the signature of X?". Backed by a
|
|
2080
|
+
// zero-dependency node:sqlite FTS5 index at .tina4/context.db, held as a
|
|
2081
|
+
// PROCESS-WIDE shared Context (context.defaultContext) so the dev-reload hook
|
|
2082
|
+
// (handleReload) can keep the SAME index fresh on every file save.
|
|
2083
|
+
server.registerTool(
|
|
2084
|
+
"code_search",
|
|
2085
|
+
(args) => {
|
|
2086
|
+
try {
|
|
2087
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
2088
|
+
const { defaultContext } = req("./context/index.js") as typeof import("./context/index.js");
|
|
2089
|
+
// Index src/ when present (falls back to the project root), mirroring
|
|
2090
|
+
// the Python master's _code_root().
|
|
2091
|
+
const srcDir = path.join(projectRoot, "src");
|
|
2092
|
+
const root = fs.existsSync(srcDir) && fs.statSync(srcDir).isDirectory() ? srcDir : projectRoot;
|
|
2093
|
+
const ctx = defaultContext(root, path.join(projectRoot, ".tina4", "context.db"));
|
|
2094
|
+
if (!ctx.available) {
|
|
2095
|
+
return { error: "SQLite FTS5 is not available in this Node build; code_search is disabled." };
|
|
2096
|
+
}
|
|
2097
|
+
if (args.rebuild) {
|
|
2098
|
+
ctx.reset();
|
|
2099
|
+
ctx.indexRoot(root);
|
|
2100
|
+
}
|
|
2101
|
+
const k = parseInt(String(args.k ?? 5), 10) || 5;
|
|
2102
|
+
return ctx.search((args.query as string) || "", k);
|
|
2103
|
+
} catch (e) {
|
|
2104
|
+
return { error: (e as Error).message };
|
|
2105
|
+
}
|
|
2106
|
+
},
|
|
2107
|
+
"Fuzzy/semantic search over THIS project's own source + docs (node:sqlite FTS5, zero-dep). " +
|
|
2108
|
+
"Ranks definitions above tests that merely mention a symbol. Use for 'where is X done here?'; " +
|
|
2109
|
+
"use api_* for exact signatures. Returns [{path, score, snippet}].",
|
|
2110
|
+
schemaFromParams([
|
|
2111
|
+
{ name: "query", type: "string" },
|
|
2112
|
+
{ name: "k", type: "integer", default: 5 },
|
|
2113
|
+
{ name: "rebuild", type: "boolean", default: false },
|
|
2114
|
+
]),
|
|
2115
|
+
);
|
|
2073
2116
|
}
|
|
2074
2117
|
|
|
2075
2118
|
/** Alias for registerDevTools — parity with PHP/Ruby/Python. */
|
|
@@ -753,7 +753,35 @@ export class BaseModel {
|
|
|
753
753
|
typeof (db as any).getError === "function" ? (db as any).getError() :
|
|
754
754
|
typeof (db as any).getLastError === "function" ? (db as any).getLastError() :
|
|
755
755
|
null;
|
|
756
|
-
|
|
756
|
+
let cause = adapterErr || e?.message || String(e);
|
|
757
|
+
// ── DX hint (v3.13.60 parity with the Python master): turn a bare driver
|
|
758
|
+
// error into an actionable fix for the two commonest ORM write footguns.
|
|
759
|
+
// Matched case-insensitively (SQLite via node:sqlite: "no such table" /
|
|
760
|
+
// "no such column: is_deleted" / "has no column named is_deleted";
|
|
761
|
+
// Postgres/MySQL: "does not exist" / "doesn't exist" / "unknown column").
|
|
762
|
+
// Any OTHER error keeps its raw cause untouched so an unrelated failure
|
|
763
|
+
// (NOT NULL, duplicate PK) is never masked. ──
|
|
764
|
+
const low = cause.toLowerCase();
|
|
765
|
+
if (
|
|
766
|
+
ModelClass.softDelete && low.includes("is_deleted") &&
|
|
767
|
+
(low.includes("no such column") || low.includes("has no column") ||
|
|
768
|
+
low.includes("does not exist") || low.includes("doesn't exist") ||
|
|
769
|
+
low.includes("unknown column"))
|
|
770
|
+
) {
|
|
771
|
+
cause +=
|
|
772
|
+
" — softDelete=true needs an is_deleted column; declare it " +
|
|
773
|
+
"(is_deleted: { type: 'integer', default: 0 }), boot the server so " +
|
|
774
|
+
"syncModels() adds it, or run a migration";
|
|
775
|
+
} else if (
|
|
776
|
+
low.includes("no such table") ||
|
|
777
|
+
((low.includes("does not exist") || low.includes("doesn't exist")) &&
|
|
778
|
+
!low.includes("column"))
|
|
779
|
+
) {
|
|
780
|
+
cause +=
|
|
781
|
+
` — table '${ModelClass.tableName}' does not exist; call ` +
|
|
782
|
+
`${ModelClass.name}.createTable() or run a migration`;
|
|
783
|
+
}
|
|
784
|
+
this.lastError = cause;
|
|
757
785
|
Log.error(
|
|
758
786
|
`${ModelClass.name}.save() failed for table ` +
|
|
759
787
|
`'${ModelClass.tableName}': ${this.lastError}`,
|