tokenos 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +571 -0
  2. package/USAGE.md +451 -0
  3. package/dist/config.d.ts +22 -0
  4. package/dist/config.d.ts.map +1 -0
  5. package/dist/config.js +60 -0
  6. package/dist/config.js.map +1 -0
  7. package/dist/db/connection.d.ts +3 -0
  8. package/dist/db/connection.d.ts.map +1 -0
  9. package/dist/db/connection.js +78 -0
  10. package/dist/db/connection.js.map +1 -0
  11. package/dist/db/index.d.ts +4 -0
  12. package/dist/db/index.d.ts.map +1 -0
  13. package/dist/db/index.js +4 -0
  14. package/dist/db/index.js.map +1 -0
  15. package/dist/db/memory.d.ts +6 -0
  16. package/dist/db/memory.d.ts.map +1 -0
  17. package/dist/db/memory.js +62 -0
  18. package/dist/db/memory.js.map +1 -0
  19. package/dist/db/queries.d.ts +29 -0
  20. package/dist/db/queries.d.ts.map +1 -0
  21. package/dist/db/queries.js +215 -0
  22. package/dist/db/queries.js.map +1 -0
  23. package/dist/embeddings/client.d.ts +16 -0
  24. package/dist/embeddings/client.d.ts.map +1 -0
  25. package/dist/embeddings/client.js +70 -0
  26. package/dist/embeddings/client.js.map +1 -0
  27. package/dist/embeddings/index.d.ts +11 -0
  28. package/dist/embeddings/index.d.ts.map +1 -0
  29. package/dist/embeddings/index.js +37 -0
  30. package/dist/embeddings/index.js.map +1 -0
  31. package/dist/embeddings/similarity.d.ts +7 -0
  32. package/dist/embeddings/similarity.d.ts.map +1 -0
  33. package/dist/embeddings/similarity.js +31 -0
  34. package/dist/embeddings/similarity.js.map +1 -0
  35. package/dist/indexer/cli.d.ts +8 -0
  36. package/dist/indexer/cli.d.ts.map +1 -0
  37. package/dist/indexer/cli.js +21 -0
  38. package/dist/indexer/cli.js.map +1 -0
  39. package/dist/indexer/ignore.d.ts +4 -0
  40. package/dist/indexer/ignore.d.ts.map +1 -0
  41. package/dist/indexer/ignore.js +30 -0
  42. package/dist/indexer/ignore.js.map +1 -0
  43. package/dist/indexer/index.d.ts +5 -0
  44. package/dist/indexer/index.d.ts.map +1 -0
  45. package/dist/indexer/index.js +4 -0
  46. package/dist/indexer/index.js.map +1 -0
  47. package/dist/indexer/indexer.d.ts +13 -0
  48. package/dist/indexer/indexer.d.ts.map +1 -0
  49. package/dist/indexer/indexer.js +125 -0
  50. package/dist/indexer/indexer.js.map +1 -0
  51. package/dist/indexer/parser.d.ts +10 -0
  52. package/dist/indexer/parser.d.ts.map +1 -0
  53. package/dist/indexer/parser.js +444 -0
  54. package/dist/indexer/parser.js.map +1 -0
  55. package/dist/indexer/watcher.d.ts +7 -0
  56. package/dist/indexer/watcher.d.ts.map +1 -0
  57. package/dist/indexer/watcher.js +64 -0
  58. package/dist/indexer/watcher.js.map +1 -0
  59. package/dist/main.d.ts +3 -0
  60. package/dist/main.d.ts.map +1 -0
  61. package/dist/main.js +92 -0
  62. package/dist/main.js.map +1 -0
  63. package/dist/reset.d.ts +6 -0
  64. package/dist/reset.d.ts.map +1 -0
  65. package/dist/reset.js +23 -0
  66. package/dist/reset.js.map +1 -0
  67. package/dist/server/index.d.ts +2 -0
  68. package/dist/server/index.d.ts.map +1 -0
  69. package/dist/server/index.js +2 -0
  70. package/dist/server/index.js.map +1 -0
  71. package/dist/server/server.d.ts +4 -0
  72. package/dist/server/server.d.ts.map +1 -0
  73. package/dist/server/server.js +558 -0
  74. package/dist/server/server.js.map +1 -0
  75. package/dist/server/visualize.d.ts +2 -0
  76. package/dist/server/visualize.d.ts.map +1 -0
  77. package/dist/server/visualize.js +299 -0
  78. package/dist/server/visualize.js.map +1 -0
  79. package/dist/test-phase1.d.ts +13 -0
  80. package/dist/test-phase1.d.ts.map +1 -0
  81. package/dist/test-phase1.js +90 -0
  82. package/dist/test-phase1.js.map +1 -0
  83. package/dist/test-phase2.d.ts +13 -0
  84. package/dist/test-phase2.d.ts.map +1 -0
  85. package/dist/test-phase2.js +110 -0
  86. package/dist/test-phase2.js.map +1 -0
  87. package/dist/test-phase3.d.ts +12 -0
  88. package/dist/test-phase3.d.ts.map +1 -0
  89. package/dist/test-phase3.js +85 -0
  90. package/dist/test-phase3.js.map +1 -0
  91. package/dist/types.d.ts +73 -0
  92. package/dist/types.d.ts.map +1 -0
  93. package/dist/types.js +3 -0
  94. package/dist/types.js.map +1 -0
  95. package/dist/utils/cache.d.ts +12 -0
  96. package/dist/utils/cache.d.ts.map +1 -0
  97. package/dist/utils/cache.js +45 -0
  98. package/dist/utils/cache.js.map +1 -0
  99. package/dist/utils/logger.d.ts +16 -0
  100. package/dist/utils/logger.d.ts.map +1 -0
  101. package/dist/utils/logger.js +52 -0
  102. package/dist/utils/logger.js.map +1 -0
  103. package/dist/utils/scoring.d.ts +15 -0
  104. package/dist/utils/scoring.d.ts.map +1 -0
  105. package/dist/utils/scoring.js +17 -0
  106. package/dist/utils/scoring.js.map +1 -0
  107. package/dist/verify-parser.d.ts +6 -0
  108. package/dist/verify-parser.d.ts.map +1 -0
  109. package/dist/verify-parser.js +105 -0
  110. package/dist/verify-parser.js.map +1 -0
  111. package/package.json +52 -0
package/USAGE.md ADDED
@@ -0,0 +1,451 @@
1
+ # How to Use TokenOS — Token-Saving Guide
2
+
3
+ > **Problem**: Every new AI chat session analyzes your entire codebase from scratch — reading dozens of files, burning thousands of tokens, and wasting compute before any real work begins.
4
+ >
5
+ > **Solution**: TokenOS pre-indexes your codebase into a structured graph. The AI queries the graph to get exactly the context it needs in 1–2 tool calls instead of scanning every file.
6
+
7
+ ---
8
+
9
+ ## Table of Contents
10
+
11
+ 1. [Setup (One-Time)](#1-setup-one-time)
12
+ 2. [How It Saves Tokens](#2-how-it-saves-tokens)
13
+ 3. [Real-World Workflows](#3-real-world-workflows)
14
+ 4. [Tool-by-Tool Usage Guide](#4-tool-by-tool-usage-guide)
15
+ 5. [Tips for Maximum Token Savings](#5-tips-for-maximum-token-savings)
16
+ 6. [Adding to Your Project](#6-adding-to-your-project)
17
+ 7. [Troubleshooting](#7-troubleshooting)
18
+
19
+ ---
20
+
21
+ ## 1. Setup (One-Time)
22
+
23
+ ### Step 1: Configure your project path
24
+
25
+ Edit `tokenos.config.json` in the TokenOS directory:
26
+
27
+ ```json
28
+ {
29
+ "watchPath": "/Users/wripcode_/Desktop/myLab/swiss-knife-flow",
30
+ "ollama": {
31
+ "url": "http://localhost:11434",
32
+ "model": "mxbai-embed-large:latest"
33
+ }
34
+ }
35
+ ```
36
+
37
+ ### Step 2: Start the server
38
+
39
+ ```bash
40
+ cd /path/to/TokenOS
41
+ npm run dev
42
+ ```
43
+
44
+ The server indexes your codebase once, then watches for changes. It stays running in the background.
45
+
46
+ ### Step 3: Connect your AI client
47
+
48
+ Add to your MCP client config (Claude Desktop, Cursor, etc.):
49
+
50
+ ```json
51
+ {
52
+ "mcpServers": {
53
+ "tokenos": {
54
+ "command": "npx",
55
+ "args": ["tsx", "/absolute/path/to/TokenOS/src/main.ts"]
56
+ }
57
+ }
58
+ }
59
+ ```
60
+
61
+ **That's it.** The AI now has access to 6 query tools instead of scanning files.
62
+
63
+ ---
64
+
65
+ ## 2. How It Saves Tokens
66
+
67
+ ### Without TokenOS (the old way)
68
+
69
+ ```
70
+ You: "Add a new auth middleware to swiss-knife-flow"
71
+
72
+ AI: Let me understand your codebase...
73
+ → list_dir src/ (50 files)
74
+ → view_file src/app/layout.tsx
75
+ → view_file src/middleware.ts
76
+ → view_file src/lib/auth.ts
77
+ → view_file src/app/api/auth/route.ts
78
+ → view_file src/components/auth-provider.tsx
79
+ → view_file src/types/auth.ts
80
+ → grep_search "middleware" (30 results)
81
+ → view_file ... (3 more files)
82
+
83
+ Total: ~10-15 tool calls, ~8,000-15,000 tokens just to understand the codebase
84
+ ```
85
+
86
+ ### With TokenOS (the new way)
87
+
88
+ ```
89
+ You: "Add a new auth middleware to swiss-knife-flow"
90
+
91
+ AI: Let me find the relevant code...
92
+ → find_nodes { query: "auth middleware", mode: "semantic" }
93
+ → get_connections { id: "src/middleware.ts::middleware" }
94
+
95
+ Total: 2 tool calls, ~500-1,000 tokens to understand the relevant parts
96
+ ```
97
+
98
+ **Result: 80-95% token reduction for codebase understanding.**
99
+
100
+ ---
101
+
102
+ ## 3. Real-World Workflows
103
+
104
+ ### Scenario 1: Implementing a New Function
105
+
106
+ > *"I want to add a `validateWebhook()` function to my swiss-knife-flow project"*
107
+
108
+ **What the AI should do:**
109
+
110
+ 1. **Find related code** (1 tool call):
111
+ ```
112
+ find_nodes { query: "webhook validate", mode: "semantic" }
113
+ ```
114
+ → Returns existing webhook-related functions, their file paths, and importance scores.
115
+
116
+ 2. **Understand the context** (1 tool call):
117
+ ```
118
+ get_connections { id: "src/lib/webhooks.ts::processWebhook" }
119
+ ```
120
+ → Shows what calls `processWebhook`, what it imports, and related types.
121
+
122
+ 3. **Now implement** — The AI knows exactly where to put the new function and what patterns to follow, without reading every file.
123
+
124
+ **Tokens saved**: ~5,000–10,000
125
+
126
+ ---
127
+
128
+ ### Scenario 2: Debugging an Issue
129
+
130
+ > *"The login flow is broken after the last refactor"*
131
+
132
+ **What the AI should do:**
133
+
134
+ 1. **Find the entry point** (1 tool call):
135
+ ```
136
+ find_nodes { query: "login", type: "function" }
137
+ ```
138
+
139
+ 2. **Trace the call graph** (1 tool call):
140
+ ```
141
+ explore { id: "src/app/api/auth/login/route.ts::POST", depth: 2 }
142
+ ```
143
+ → Shows the full call chain: route → auth service → database → response.
144
+
145
+ 3. **Read only the relevant files** — Now the AI reads 2–3 specific files instead of scanning 50.
146
+
147
+ ---
148
+
149
+ ### Scenario 3: Understanding Unfamiliar Code
150
+
151
+ > *"What are the most important parts of this codebase?"*
152
+
153
+ **What the AI should do:**
154
+
155
+ 1. **Get the architecture overview** (1 tool call):
156
+ ```
157
+ top_nodes { limit: 15 }
158
+ ```
159
+ → Returns the 15 most connected/important nodes — the backbone of your app.
160
+
161
+ 2. **Drill into specific areas** as needed:
162
+ ```
163
+ get_connections { id: "src/lib/api-client.ts::ApiClient" }
164
+ ```
165
+
166
+ ---
167
+
168
+ ### Scenario 4: Adding a Feature That Touches Multiple Files
169
+
170
+ > *"Add dark mode support to the dashboard"*
171
+
172
+ **What the AI should do:**
173
+
174
+ 1. **Find all UI-related code** (1 tool call):
175
+ ```
176
+ find_nodes { query: "theme color style", mode: "semantic", type: "variable" }
177
+ ```
178
+
179
+ 2. **Find the component tree** (1 tool call):
180
+ ```
181
+ find_nodes { query: "dashboard", type: "function" }
182
+ ```
183
+
184
+ 3. **Trace dependencies** (1 tool call per key component):
185
+ ```
186
+ get_connections { id: "src/components/dashboard/layout.tsx::DashboardLayout" }
187
+ ```
188
+
189
+ ---
190
+
191
+ ### Scenario 5: Refactoring
192
+
193
+ > *"Rename the `fetchSites` function and update all callers"*
194
+
195
+ **What the AI should do:**
196
+
197
+ 1. **Find the function** (1 tool call):
198
+ ```
199
+ find_nodes { query: "fetchSites", type: "function" }
200
+ ```
201
+
202
+ 2. **Find all callers** (1 tool call):
203
+ ```
204
+ get_connections { id: "src/lib/webflow.ts::fetchSites" }
205
+ ```
206
+ → Shows every file that CALLS this function — no grep needed.
207
+
208
+ ---
209
+
210
+ ## 4. Tool-by-Tool Usage Guide
211
+
212
+ ### `top_nodes` — Start Here
213
+
214
+ **When**: Beginning of any new conversation about the codebase.
215
+
216
+ ```
217
+ top_nodes { limit: 20, response_format: "markdown" }
218
+ ```
219
+
220
+ Returns the most architecturally significant nodes. This gives the AI a mental map of your project in one call. **Always start here.**
221
+
222
+ | Param | Default | Description |
223
+ |-------|---------|-------------|
224
+ | `limit` | 20 | How many nodes (1–100) |
225
+ | `response_format` | json | `json` or `markdown` |
226
+
227
+ ---
228
+
229
+ ### `find_nodes` — Find Relevant Code
230
+
231
+ **When**: Looking for specific functions, classes, or concepts.
232
+
233
+ **Text mode** (fast, name-based):
234
+ ```
235
+ find_nodes { query: "auth", type: "function" }
236
+ ```
237
+ Finds functions with "auth" in the name. Use `type` to narrow results.
238
+
239
+ **Semantic mode** (concept-based, requires Ollama):
240
+ ```
241
+ find_nodes { query: "user authentication handler", mode: "semantic" }
242
+ ```
243
+ Finds code related to the concept even if the name doesn't match. E.g., `loginUser()` matches "authentication handler".
244
+
245
+ | Param | Default | Description |
246
+ |-------|---------|-------------|
247
+ | `query` | required | Search term or description |
248
+ | `type` | all | `function`, `class`, `file`, `import`, `variable` |
249
+ | `mode` | text | `text` or `semantic` |
250
+ | `limit` | 10 | Results per page (1–50) |
251
+ | `offset` | 0 | For pagination |
252
+
253
+ ---
254
+
255
+ ### `get_node` — Full Details
256
+
257
+ **When**: You have a node ID and want the complete picture (code snippet, importance, etc.).
258
+
259
+ ```
260
+ get_node { id: "src/lib/auth.ts::validateToken" }
261
+ ```
262
+
263
+ Returns: name, type, file path, code snippet (first 12 lines), importance score.
264
+
265
+ ---
266
+
267
+ ### `get_connections` — What's Connected?
268
+
269
+ **When**: Understanding what a function calls, what calls it, and what it imports.
270
+
271
+ ```
272
+ get_connections { id: "src/lib/auth.ts::validateToken" }
273
+ ```
274
+
275
+ Returns:
276
+ - **Edges**: `CALLS`, `IMPORTS`, `EXPORTS`, `EXTENDS`, `DEFINES`
277
+ - **Connected nodes**: sorted by importance
278
+
279
+ This is the key tool for understanding how code fits together without reading entire files.
280
+
281
+ ---
282
+
283
+ ### `explore` — Full Context Tree
284
+
285
+ **When**: You need the complete call/dependency tree around a node.
286
+
287
+ ```
288
+ explore { id: "src/app/api/route.ts::handler", depth: 2 }
289
+ ```
290
+
291
+ Returns a BFS traversal — all nodes and edges reachable within `depth` hops. Use `depth: 1` for large codebases to keep responses small.
292
+
293
+ | Param | Default | Description |
294
+ |-------|---------|-------------|
295
+ | `id` | required | Starting node |
296
+ | `depth` | 2 | 1–3 hops |
297
+
298
+ ---
299
+
300
+ ### `search` — Smart Search
301
+
302
+ **When**: You want the AI to automatically figure out the best search strategy.
303
+
304
+ ```
305
+ search { query: "how does the auth flow work?", response_format: "markdown" }
306
+ ```
307
+
308
+ Automatically detects intent, runs hybrid search, expands the graph, and includes relevant memories. Best for complex or exploratory queries.
309
+
310
+ ---
311
+
312
+ ## 5. Tips for Maximum Token Savings
313
+
314
+ ### DO ✅
315
+
316
+ - **Start every new chat with `top_nodes`** — gives the AI instant context
317
+ - **Use `type` filters** — `find_nodes { query: "auth", type: "function" }` returns only functions, not imports
318
+ - **Use semantic search for vague queries** — "error handling" finds `catchApiError()`, `handleException()`, etc.
319
+ - **Use `get_connections` before reading files** — often the edge list is enough to understand relationships
320
+ - **Use `response_format: "markdown"`** — more compact for AI consumption
321
+
322
+ ### DON'T ❌
323
+
324
+ - **Don't let the AI `list_dir` and `view_file` to "understand the codebase"** — TokenOS already has this information
325
+ - **Don't use `depth: 3` on `explore`** — the response can be huge. Start with `depth: 1`
326
+ - **Don't search without filters** — `find_nodes { query: "a" }` returns too many results
327
+ - **Don't re-index manually** — the file watcher handles changes automatically
328
+
329
+ ### Pro Tips 💡
330
+
331
+ 1. **Node IDs are `filePath::name`** — e.g., `src/lib/auth.ts::validateToken`. You can construct these from file paths you already know.
332
+
333
+ 2. **Importance score tells priority** — Higher score = more central to the architecture. Focus on high-importance nodes first.
334
+
335
+ 3. **Semantic search finds concepts** — If you're looking for "the function that handles form submissions" but don't know its name, semantic mode will find it.
336
+
337
+ 4. **The graph updates in real-time** — When you save a file, the watcher re-indexes it. Next query will have the latest data.
338
+
339
+ 5. **Combine tools in sequence**:
340
+ ```
341
+ 1. top_nodes { limit: 10 } → overview
342
+ 2. find_nodes { query: "..." } → find specific code
343
+ 3. get_connections { id: "..." } → understand connections
344
+ 4. view_file (only the specific file you need) → read actual code
345
+ ```
346
+
347
+ ---
348
+
349
+ ## 6. Adding to Your Project
350
+
351
+ ### Add `.tokenos/` to your project's `.gitignore`:
352
+
353
+ ```bash
354
+ echo ".tokenos/" >> /path/to/your/project/.gitignore
355
+ ```
356
+
357
+ This prevents the per-project database from being committed.
358
+
359
+ ### Switching Between Projects
360
+
361
+ Edit `tokenos.config.json` and change `watchPath`:
362
+
363
+ ```json
364
+ {
365
+ "watchPath": "/path/to/different/project"
366
+ }
367
+ ```
368
+
369
+ Then restart: `npm run dev`. Or if you want a fresh index:
370
+
371
+ ```bash
372
+ npm run reset
373
+ npm run dev
374
+ ```
375
+
376
+ ### Changing the Embedding Model
377
+
378
+ Edit `tokenos.config.json`:
379
+
380
+ ```json
381
+ {
382
+ "ollama": {
383
+ "model": "nomic-embed-text"
384
+ }
385
+ }
386
+ ```
387
+
388
+ **Important**: After changing models, run `npm run reset` then `npm run dev` — different models produce incompatible vectors.
389
+
390
+ ---
391
+
392
+ ## 7. Troubleshooting
393
+
394
+ ### "0 files indexed"
395
+
396
+ - Check that `watchPath` points to a directory containing `.ts` or `.tsx` files
397
+ - Check that the path exists and is accessible
398
+ - Check `.gitignore` — files matching gitignore patterns are skipped
399
+
400
+ ### "Embeddings skipped (Ollama offline)"
401
+
402
+ - Start Ollama: `ollama serve`
403
+ - Pull the model: `ollama pull mxbai-embed-large:latest`
404
+ - Semantic search falls back to text mode when Ollama is offline — everything else still works
405
+
406
+ ### "Semantic search returns no results"
407
+
408
+ - Make sure embeddings were generated (check the boot log for "embeddings ready: X updated")
409
+ - If you changed the model, run `npm run reset` and re-index
410
+
411
+ ### Stale data after refactoring
412
+
413
+ ```bash
414
+ npm run reset # delete the database
415
+ npm run dev # re-index from scratch
416
+ ```
417
+
418
+ ---
419
+
420
+ ## Quick Reference Card
421
+
422
+ ```
423
+ ┌──────────────────────────────────────────────────────────────┐
424
+ │ TOKENOS CHEAT SHEET │
425
+ ├──────────────────────────────────────────────────────────────┤
426
+ │ │
427
+ │ START HERE: │
428
+ │ top_nodes { limit: 15 } │
429
+ │ │
430
+ │ FIND CODE: │
431
+ │ find_nodes { query: "auth", type: "function" } │
432
+ │ find_nodes { query: "handles login", mode: "semantic" } │
433
+ │ │
434
+ │ UNDERSTAND CONNECTIONS: │
435
+ │ get_connections { id: "file::name" } │
436
+ │ │
437
+ │ EXPLORE CONTEXT: │
438
+ │ explore { id: "file::name", depth: 1 } │
439
+ │ │
440
+ │ FULL NODE DETAILS: │
441
+ │ get_node { id: "file::name" } │
442
+ │ │
443
+ │ SMART SEARCH: │
444
+ │ search { query: "how does auth work?" } │
445
+ │ │
446
+ │ COMMANDS: │
447
+ │ npm run dev → start server │
448
+ │ npm run reset → delete DB & re-index │
449
+ │ │
450
+ └──────────────────────────────────────────────────────────────┘
451
+ ```
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Config loader for TokenOS.
3
+ *
4
+ * Precedence: CLI args → config file → env vars → defaults.
5
+ * Config file: tokenos.config.json in the project root.
6
+ */
7
+ export interface TokenOSConfig {
8
+ watchPath: string;
9
+ ollama: {
10
+ url: string;
11
+ model: string;
12
+ };
13
+ ui: {
14
+ enabled: boolean;
15
+ port: number;
16
+ };
17
+ /** Derived: absolute path to the SQLite DB for the target project */
18
+ dbPath: string;
19
+ }
20
+ /** Singleton config — built once on import */
21
+ export declare const config: TokenOSConfig;
22
+ //# sourceMappingURL=config.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,EAAE,EAAE;QACF,OAAO,EAAE,OAAO,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;CAChB;AA+DD,8CAA8C;AAC9C,eAAO,MAAM,MAAM,EAAE,aAA6B,CAAC"}
package/dist/config.js ADDED
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Config loader for TokenOS.
3
+ *
4
+ * Precedence: CLI args → config file → env vars → defaults.
5
+ * Config file: tokenos.config.json in the project root.
6
+ */
7
+ import { readFileSync, existsSync } from "fs";
8
+ import { resolve, join, dirname } from "path";
9
+ import { fileURLToPath } from "url";
10
+ // ───── Loader ─────────────────────────────────────────────────────────────────
11
+ const __filename = fileURLToPath(import.meta.url);
12
+ const __dirname = dirname(__filename);
13
+ const PROJECT_ROOT = resolve(__dirname, "..");
14
+ function loadConfigFile() {
15
+ const configPath = join(PROJECT_ROOT, "tokenos.config.json");
16
+ if (!existsSync(configPath))
17
+ return {};
18
+ try {
19
+ const raw = readFileSync(configPath, "utf-8");
20
+ return JSON.parse(raw);
21
+ }
22
+ catch {
23
+ return {};
24
+ }
25
+ }
26
+ function buildConfig() {
27
+ const file = loadConfigFile();
28
+ const fileOllama = (file.ollama ?? {});
29
+ const fileUI = (file.ui ?? {});
30
+ // CLI args (backwards-compatible: first non-flag arg is watchPath, --ui flag)
31
+ const args = process.argv.slice(2);
32
+ const cliPath = args.find((a) => !a.startsWith("--"));
33
+ const cliUI = args.includes("--ui");
34
+ // Resolve watchPath: CLI > config > cwd
35
+ const rawPath = cliPath ?? file.watchPath ?? process.cwd();
36
+ const watchPath = resolve(rawPath);
37
+ // Ollama settings: env > config > defaults
38
+ const ollamaUrl = process.env.OLLAMA_URL ??
39
+ fileOllama.url ??
40
+ "http://localhost:11434";
41
+ const ollamaModel = process.env.EMBEDDING_MODEL ??
42
+ fileOllama.model ??
43
+ "nomic-embed-text";
44
+ // UI settings: CLI flag > config > defaults
45
+ const uiEnabled = cliUI || fileUI.enabled === true;
46
+ const uiPort = Number(process.env.GRAPH_UI_PORT) ||
47
+ fileUI.port ||
48
+ 3333;
49
+ // Per-project DB: <watchPath>/.tokenos/graph.db
50
+ const dbPath = join(watchPath, ".tokenos", "graph.db");
51
+ return {
52
+ watchPath,
53
+ ollama: { url: ollamaUrl, model: ollamaModel },
54
+ ui: { enabled: uiEnabled, port: uiPort },
55
+ dbPath,
56
+ };
57
+ }
58
+ /** Singleton config — built once on import */
59
+ export const config = buildConfig();
60
+ //# sourceMappingURL=config.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAkBpC,iFAAiF;AAEjF,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AACtC,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAE9C,SAAS,cAAc;IACrB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IAC7D,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,WAAW;IAClB,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAA4B,CAAC;IAClE,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAA4B,CAAC;IAE1D,8EAA8E;IAC9E,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEpC,wCAAwC;IACxC,MAAM,OAAO,GAAG,OAAO,IAAK,IAAI,CAAC,SAAgC,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACnF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEnC,2CAA2C;IAC3C,MAAM,SAAS,GACb,OAAO,CAAC,GAAG,CAAC,UAAU;QACrB,UAAU,CAAC,GAA0B;QACtC,wBAAwB,CAAC;IAE3B,MAAM,WAAW,GACf,OAAO,CAAC,GAAG,CAAC,eAAe;QAC1B,UAAU,CAAC,KAA4B;QACxC,kBAAkB,CAAC;IAErB,4CAA4C;IAC5C,MAAM,SAAS,GAAG,KAAK,IAAK,MAAM,CAAC,OAA+B,KAAK,IAAI,CAAC;IAC5E,MAAM,MAAM,GACV,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAChC,MAAM,CAAC,IAA2B;QACnC,IAAI,CAAC;IAEP,gDAAgD;IAChD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IAEvD,OAAO;QACL,SAAS;QACT,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE;QAC9C,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE;QACxC,MAAM;KACP,CAAC;AACJ,CAAC;AAED,8CAA8C;AAC9C,MAAM,CAAC,MAAM,MAAM,GAAkB,WAAW,EAAE,CAAC"}
@@ -0,0 +1,3 @@
1
+ import { type Database as DatabaseType } from "better-sqlite3";
2
+ export declare const db: DatabaseType;
3
+ //# sourceMappingURL=connection.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"connection.d.ts","sourceRoot":"","sources":["../../src/db/connection.ts"],"names":[],"mappings":"AAAA,OAAiB,EAAE,KAAK,QAAQ,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AASzE,eAAO,MAAM,EAAE,EAAE,YAA0C,CAAC"}
@@ -0,0 +1,78 @@
1
+ import Database from "better-sqlite3";
2
+ import { dirname } from "path";
3
+ import { mkdirSync } from "fs";
4
+ import { config } from "../config.js";
5
+ // Ensure the .tokenos directory exists in the target project
6
+ mkdirSync(dirname(config.dbPath), { recursive: true });
7
+ // Open or create the SQLite database at the per-project path
8
+ export const db = new Database(config.dbPath);
9
+ // Enable WAL mode for better concurrent read performance
10
+ db.pragma("journal_mode = WAL");
11
+ db.pragma("foreign_keys = ON");
12
+ // Run schema migrations on startup
13
+ db.exec(`
14
+ -- NODES TABLE
15
+ CREATE TABLE IF NOT EXISTS nodes (
16
+ id TEXT PRIMARY KEY,
17
+ type TEXT NOT NULL,
18
+ name TEXT NOT NULL,
19
+ file_path TEXT NOT NULL,
20
+
21
+ summary TEXT,
22
+ code_snippet TEXT,
23
+ embedding TEXT,
24
+
25
+ hash TEXT,
26
+ importance REAL DEFAULT 0,
27
+
28
+ created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
29
+ updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
30
+ );
31
+
32
+ -- EDGES TABLE
33
+ CREATE TABLE IF NOT EXISTS edges (
34
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
35
+ from_node TEXT NOT NULL,
36
+ to_node TEXT NOT NULL,
37
+ type TEXT NOT NULL,
38
+
39
+ FOREIGN KEY(from_node) REFERENCES nodes(id) ON DELETE CASCADE,
40
+ FOREIGN KEY(to_node) REFERENCES nodes(id) ON DELETE CASCADE
41
+ );
42
+
43
+ -- MEMORIES TABLE (Phase 5)
44
+ CREATE TABLE IF NOT EXISTS memories (
45
+ id TEXT PRIMARY KEY,
46
+ title TEXT NOT NULL,
47
+ summary TEXT NOT NULL,
48
+ key_points TEXT NOT NULL, -- JSON array of strings
49
+ tags TEXT NOT NULL, -- JSON array of strings
50
+ embedding TEXT, -- JSON array of numbers
51
+ created_at INTEGER NOT NULL
52
+ );
53
+
54
+ -- INDEXES
55
+ CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name);
56
+ CREATE INDEX IF NOT EXISTS idx_nodes_type ON nodes(type);
57
+ CREATE INDEX IF NOT EXISTS idx_nodes_file ON nodes(file_path);
58
+ CREATE INDEX IF NOT EXISTS idx_nodes_importance ON nodes(importance);
59
+
60
+ CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node);
61
+ CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node);
62
+ CREATE INDEX IF NOT EXISTS idx_edges_type ON edges(type);
63
+ `);
64
+ // ── Safe migration: add meta column if not already present ────────────────────
65
+ try {
66
+ db.exec(`ALTER TABLE nodes ADD COLUMN meta TEXT;`);
67
+ }
68
+ catch {
69
+ // Column already exists — ignore
70
+ }
71
+ // Unique edge constraint to prevent duplicates
72
+ try {
73
+ db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_edges_unique ON edges(from_node, to_node, type);`);
74
+ }
75
+ catch {
76
+ // Already exists — ignore
77
+ }
78
+ //# sourceMappingURL=connection.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"connection.js","sourceRoot":"","sources":["../../src/db/connection.ts"],"names":[],"mappings":"AAAA,OAAO,QAA2C,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC/B,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,6DAA6D;AAC7D,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAEvD,6DAA6D;AAC7D,MAAM,CAAC,MAAM,EAAE,GAAiB,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAE5D,yDAAyD;AACzD,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;AAChC,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;AAE/B,mCAAmC;AACnC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkDP,CAAC,CAAC;AAEH,iFAAiF;AACjF,IAAI,CAAC;IACH,EAAE,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;AACrD,CAAC;AAAC,MAAM,CAAC;IACP,iCAAiC;AACnC,CAAC;AAED,+CAA+C;AAC/C,IAAI,CAAC;IACH,EAAE,CAAC,IAAI,CAAC,wFAAwF,CAAC,CAAC;AACpG,CAAC;AAAC,MAAM,CAAC;IACP,0BAA0B;AAC5B,CAAC"}
@@ -0,0 +1,4 @@
1
+ export { db } from "./connection.js";
2
+ export * from "./queries.js";
3
+ export * from "./memory.js";
4
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC"}
@@ -0,0 +1,4 @@
1
+ export { db } from "./connection.js";
2
+ export * from "./queries.js";
3
+ export * from "./memory.js";
4
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC"}
@@ -0,0 +1,6 @@
1
+ import type { ConversationMemory } from "../types.js";
2
+ export declare function upsertMemory(m: ConversationMemory): void;
3
+ export declare function getMemory(id: string): ConversationMemory | undefined;
4
+ export declare function getAllMemories(): ConversationMemory[];
5
+ export declare function searchMemories(query: string): ConversationMemory[];
6
+ //# sourceMappingURL=memory.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../src/db/memory.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAatD,wBAAgB,YAAY,CAAC,CAAC,EAAE,kBAAkB,GAAG,IAAI,CAUxD;AAMD,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS,CASpE;AAMD,wBAAgB,cAAc,IAAI,kBAAkB,EAAE,CAOrD;AAQD,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,kBAAkB,EAAE,CAQlE"}