tokenos 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +571 -0
- package/USAGE.md +451 -0
- package/dist/config.d.ts +22 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +60 -0
- package/dist/config.js.map +1 -0
- package/dist/db/connection.d.ts +3 -0
- package/dist/db/connection.d.ts.map +1 -0
- package/dist/db/connection.js +78 -0
- package/dist/db/connection.js.map +1 -0
- package/dist/db/index.d.ts +4 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/db/index.js +4 -0
- package/dist/db/index.js.map +1 -0
- package/dist/db/memory.d.ts +6 -0
- package/dist/db/memory.d.ts.map +1 -0
- package/dist/db/memory.js +62 -0
- package/dist/db/memory.js.map +1 -0
- package/dist/db/queries.d.ts +29 -0
- package/dist/db/queries.d.ts.map +1 -0
- package/dist/db/queries.js +215 -0
- package/dist/db/queries.js.map +1 -0
- package/dist/embeddings/client.d.ts +16 -0
- package/dist/embeddings/client.d.ts.map +1 -0
- package/dist/embeddings/client.js +70 -0
- package/dist/embeddings/client.js.map +1 -0
- package/dist/embeddings/index.d.ts +11 -0
- package/dist/embeddings/index.d.ts.map +1 -0
- package/dist/embeddings/index.js +37 -0
- package/dist/embeddings/index.js.map +1 -0
- package/dist/embeddings/similarity.d.ts +7 -0
- package/dist/embeddings/similarity.d.ts.map +1 -0
- package/dist/embeddings/similarity.js +31 -0
- package/dist/embeddings/similarity.js.map +1 -0
- package/dist/indexer/cli.d.ts +8 -0
- package/dist/indexer/cli.d.ts.map +1 -0
- package/dist/indexer/cli.js +21 -0
- package/dist/indexer/cli.js.map +1 -0
- package/dist/indexer/ignore.d.ts +4 -0
- package/dist/indexer/ignore.d.ts.map +1 -0
- package/dist/indexer/ignore.js +30 -0
- package/dist/indexer/ignore.js.map +1 -0
- package/dist/indexer/index.d.ts +5 -0
- package/dist/indexer/index.d.ts.map +1 -0
- package/dist/indexer/index.js +4 -0
- package/dist/indexer/index.js.map +1 -0
- package/dist/indexer/indexer.d.ts +13 -0
- package/dist/indexer/indexer.d.ts.map +1 -0
- package/dist/indexer/indexer.js +125 -0
- package/dist/indexer/indexer.js.map +1 -0
- package/dist/indexer/parser.d.ts +10 -0
- package/dist/indexer/parser.d.ts.map +1 -0
- package/dist/indexer/parser.js +444 -0
- package/dist/indexer/parser.js.map +1 -0
- package/dist/indexer/watcher.d.ts +7 -0
- package/dist/indexer/watcher.d.ts.map +1 -0
- package/dist/indexer/watcher.js +64 -0
- package/dist/indexer/watcher.js.map +1 -0
- package/dist/main.d.ts +3 -0
- package/dist/main.d.ts.map +1 -0
- package/dist/main.js +92 -0
- package/dist/main.js.map +1 -0
- package/dist/reset.d.ts +6 -0
- package/dist/reset.d.ts.map +1 -0
- package/dist/reset.js +23 -0
- package/dist/reset.js.map +1 -0
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +2 -0
- package/dist/server/index.js.map +1 -0
- package/dist/server/server.d.ts +4 -0
- package/dist/server/server.d.ts.map +1 -0
- package/dist/server/server.js +558 -0
- package/dist/server/server.js.map +1 -0
- package/dist/server/visualize.d.ts +2 -0
- package/dist/server/visualize.d.ts.map +1 -0
- package/dist/server/visualize.js +299 -0
- package/dist/server/visualize.js.map +1 -0
- package/dist/test-phase1.d.ts +13 -0
- package/dist/test-phase1.d.ts.map +1 -0
- package/dist/test-phase1.js +90 -0
- package/dist/test-phase1.js.map +1 -0
- package/dist/test-phase2.d.ts +13 -0
- package/dist/test-phase2.d.ts.map +1 -0
- package/dist/test-phase2.js +110 -0
- package/dist/test-phase2.js.map +1 -0
- package/dist/test-phase3.d.ts +12 -0
- package/dist/test-phase3.d.ts.map +1 -0
- package/dist/test-phase3.js +85 -0
- package/dist/test-phase3.js.map +1 -0
- package/dist/types.d.ts +73 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/cache.d.ts +12 -0
- package/dist/utils/cache.d.ts.map +1 -0
- package/dist/utils/cache.js +45 -0
- package/dist/utils/cache.js.map +1 -0
- package/dist/utils/logger.d.ts +16 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +52 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/scoring.d.ts +15 -0
- package/dist/utils/scoring.d.ts.map +1 -0
- package/dist/utils/scoring.js +17 -0
- package/dist/utils/scoring.js.map +1 -0
- package/dist/verify-parser.d.ts +6 -0
- package/dist/verify-parser.d.ts.map +1 -0
- package/dist/verify-parser.js +105 -0
- package/dist/verify-parser.js.map +1 -0
- package/package.json +52 -0
package/USAGE.md
ADDED
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
# How to Use TokenOS — Token-Saving Guide
|
|
2
|
+
|
|
3
|
+
> **Problem**: Every new AI chat session analyzes your entire codebase from scratch — reading dozens of files, burning thousands of tokens, and wasting compute before any real work begins.
|
|
4
|
+
>
|
|
5
|
+
> **Solution**: TokenOS pre-indexes your codebase into a structured graph. The AI queries the graph to get exactly the context it needs in 1–2 tool calls instead of scanning every file.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Table of Contents
|
|
10
|
+
|
|
11
|
+
1. [Setup (One-Time)](#1-setup-one-time)
|
|
12
|
+
2. [How It Saves Tokens](#2-how-it-saves-tokens)
|
|
13
|
+
3. [Real-World Workflows](#3-real-world-workflows)
|
|
14
|
+
4. [Tool-by-Tool Usage Guide](#4-tool-by-tool-usage-guide)
|
|
15
|
+
5. [Tips for Maximum Token Savings](#5-tips-for-maximum-token-savings)
|
|
16
|
+
6. [Adding to Your Project](#6-adding-to-your-project)
|
|
17
|
+
7. [Troubleshooting](#7-troubleshooting)
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## 1. Setup (One-Time)
|
|
22
|
+
|
|
23
|
+
### Step 1: Configure your project path
|
|
24
|
+
|
|
25
|
+
Edit `tokenos.config.json` in the TokenOS directory:
|
|
26
|
+
|
|
27
|
+
```json
|
|
28
|
+
{
|
|
29
|
+
"watchPath": "/Users/wripcode_/Desktop/myLab/swiss-knife-flow",
|
|
30
|
+
"ollama": {
|
|
31
|
+
"url": "http://localhost:11434",
|
|
32
|
+
"model": "mxbai-embed-large:latest"
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
### Step 2: Start the server
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
cd /path/to/TokenOS
|
|
41
|
+
npm run dev
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
The server indexes your codebase once, then watches for changes. It stays running in the background.
|
|
45
|
+
|
|
46
|
+
### Step 3: Connect your AI client
|
|
47
|
+
|
|
48
|
+
Add to your MCP client config (Claude Desktop, Cursor, etc.):
|
|
49
|
+
|
|
50
|
+
```json
|
|
51
|
+
{
|
|
52
|
+
"mcpServers": {
|
|
53
|
+
"tokenos": {
|
|
54
|
+
"command": "npx",
|
|
55
|
+
"args": ["tsx", "/absolute/path/to/TokenOS/src/main.ts"]
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
**That's it.** The AI now has access to 6 query tools instead of scanning files.
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
## 2. How It Saves Tokens
|
|
66
|
+
|
|
67
|
+
### Without TokenOS (the old way)
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
You: "Add a new auth middleware to swiss-knife-flow"
|
|
71
|
+
|
|
72
|
+
AI: Let me understand your codebase...
|
|
73
|
+
→ list_dir src/ (50 files)
|
|
74
|
+
→ view_file src/app/layout.tsx
|
|
75
|
+
→ view_file src/middleware.ts
|
|
76
|
+
→ view_file src/lib/auth.ts
|
|
77
|
+
→ view_file src/app/api/auth/route.ts
|
|
78
|
+
→ view_file src/components/auth-provider.tsx
|
|
79
|
+
→ view_file src/types/auth.ts
|
|
80
|
+
→ grep_search "middleware" (30 results)
|
|
81
|
+
→ view_file ... (3 more files)
|
|
82
|
+
|
|
83
|
+
Total: ~10-15 tool calls, ~8,000-15,000 tokens just to understand the codebase
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### With TokenOS (the new way)
|
|
87
|
+
|
|
88
|
+
```
|
|
89
|
+
You: "Add a new auth middleware to swiss-knife-flow"
|
|
90
|
+
|
|
91
|
+
AI: Let me find the relevant code...
|
|
92
|
+
→ find_nodes { query: "auth middleware", mode: "semantic" }
|
|
93
|
+
→ get_connections { id: "src/middleware.ts::middleware" }
|
|
94
|
+
|
|
95
|
+
Total: 2 tool calls, ~500-1,000 tokens to understand the relevant parts
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
**Result: 80-95% token reduction for codebase understanding.**
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## 3. Real-World Workflows
|
|
103
|
+
|
|
104
|
+
### Scenario 1: Implementing a New Function
|
|
105
|
+
|
|
106
|
+
> *"I want to add a `validateWebhook()` function to my swiss-knife-flow project"*
|
|
107
|
+
|
|
108
|
+
**What the AI should do:**
|
|
109
|
+
|
|
110
|
+
1. **Find related code** (1 tool call):
|
|
111
|
+
```
|
|
112
|
+
find_nodes { query: "webhook validate", mode: "semantic" }
|
|
113
|
+
```
|
|
114
|
+
→ Returns existing webhook-related functions, their file paths, and importance scores.
|
|
115
|
+
|
|
116
|
+
2. **Understand the context** (1 tool call):
|
|
117
|
+
```
|
|
118
|
+
get_connections { id: "src/lib/webhooks.ts::processWebhook" }
|
|
119
|
+
```
|
|
120
|
+
→ Shows what calls `processWebhook`, what it imports, and related types.
|
|
121
|
+
|
|
122
|
+
3. **Now implement** — The AI knows exactly where to put the new function and what patterns to follow, without reading every file.
|
|
123
|
+
|
|
124
|
+
**Tokens saved**: ~5,000–10,000
|
|
125
|
+
|
|
126
|
+
---
|
|
127
|
+
|
|
128
|
+
### Scenario 2: Debugging an Issue
|
|
129
|
+
|
|
130
|
+
> *"The login flow is broken after the last refactor"*
|
|
131
|
+
|
|
132
|
+
**What the AI should do:**
|
|
133
|
+
|
|
134
|
+
1. **Find the entry point** (1 tool call):
|
|
135
|
+
```
|
|
136
|
+
find_nodes { query: "login", type: "function" }
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
2. **Trace the call graph** (1 tool call):
|
|
140
|
+
```
|
|
141
|
+
explore { id: "src/app/api/auth/login/route.ts::POST", depth: 2 }
|
|
142
|
+
```
|
|
143
|
+
→ Shows the full call chain: route → auth service → database → response.
|
|
144
|
+
|
|
145
|
+
3. **Read only the relevant files** — Now the AI reads 2–3 specific files instead of scanning 50.
|
|
146
|
+
|
|
147
|
+
---
|
|
148
|
+
|
|
149
|
+
### Scenario 3: Understanding Unfamiliar Code
|
|
150
|
+
|
|
151
|
+
> *"What are the most important parts of this codebase?"*
|
|
152
|
+
|
|
153
|
+
**What the AI should do:**
|
|
154
|
+
|
|
155
|
+
1. **Get the architecture overview** (1 tool call):
|
|
156
|
+
```
|
|
157
|
+
top_nodes { limit: 15 }
|
|
158
|
+
```
|
|
159
|
+
→ Returns the 15 most connected/important nodes — the backbone of your app.
|
|
160
|
+
|
|
161
|
+
2. **Drill into specific areas** as needed:
|
|
162
|
+
```
|
|
163
|
+
get_connections { id: "src/lib/api-client.ts::ApiClient" }
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
### Scenario 4: Adding a Feature That Touches Multiple Files
|
|
169
|
+
|
|
170
|
+
> *"Add dark mode support to the dashboard"*
|
|
171
|
+
|
|
172
|
+
**What the AI should do:**
|
|
173
|
+
|
|
174
|
+
1. **Find all UI-related code** (1 tool call):
|
|
175
|
+
```
|
|
176
|
+
find_nodes { query: "theme color style", mode: "semantic", type: "variable" }
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
2. **Find the component tree** (1 tool call):
|
|
180
|
+
```
|
|
181
|
+
find_nodes { query: "dashboard", type: "function" }
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
3. **Trace dependencies** (1 tool call per key component):
|
|
185
|
+
```
|
|
186
|
+
get_connections { id: "src/components/dashboard/layout.tsx::DashboardLayout" }
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
---
|
|
190
|
+
|
|
191
|
+
### Scenario 5: Refactoring
|
|
192
|
+
|
|
193
|
+
> *"Rename the `fetchSites` function and update all callers"*
|
|
194
|
+
|
|
195
|
+
**What the AI should do:**
|
|
196
|
+
|
|
197
|
+
1. **Find the function** (1 tool call):
|
|
198
|
+
```
|
|
199
|
+
find_nodes { query: "fetchSites", type: "function" }
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
2. **Find all callers** (1 tool call):
|
|
203
|
+
```
|
|
204
|
+
get_connections { id: "src/lib/webflow.ts::fetchSites" }
|
|
205
|
+
```
|
|
206
|
+
→ Shows every file that CALLS this function — no grep needed.
|
|
207
|
+
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
## 4. Tool-by-Tool Usage Guide
|
|
211
|
+
|
|
212
|
+
### `top_nodes` — Start Here
|
|
213
|
+
|
|
214
|
+
**When**: Beginning of any new conversation about the codebase.
|
|
215
|
+
|
|
216
|
+
```
|
|
217
|
+
top_nodes { limit: 20, response_format: "markdown" }
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
Returns the most architecturally significant nodes. This gives the AI a mental map of your project in one call. **Always start here.**
|
|
221
|
+
|
|
222
|
+
| Param | Default | Description |
|
|
223
|
+
|-------|---------|-------------|
|
|
224
|
+
| `limit` | 20 | How many nodes (1–100) |
|
|
225
|
+
| `response_format` | json | `json` or `markdown` |
|
|
226
|
+
|
|
227
|
+
---
|
|
228
|
+
|
|
229
|
+
### `find_nodes` — Find Relevant Code
|
|
230
|
+
|
|
231
|
+
**When**: Looking for specific functions, classes, or concepts.
|
|
232
|
+
|
|
233
|
+
**Text mode** (fast, name-based):
|
|
234
|
+
```
|
|
235
|
+
find_nodes { query: "auth", type: "function" }
|
|
236
|
+
```
|
|
237
|
+
Finds functions with "auth" in the name. Use `type` to narrow results.
|
|
238
|
+
|
|
239
|
+
**Semantic mode** (concept-based, requires Ollama):
|
|
240
|
+
```
|
|
241
|
+
find_nodes { query: "user authentication handler", mode: "semantic" }
|
|
242
|
+
```
|
|
243
|
+
Finds code related to the concept even if the name doesn't match. E.g., `loginUser()` matches "authentication handler".
|
|
244
|
+
|
|
245
|
+
| Param | Default | Description |
|
|
246
|
+
|-------|---------|-------------|
|
|
247
|
+
| `query` | required | Search term or description |
|
|
248
|
+
| `type` | all | `function`, `class`, `file`, `import`, `variable` |
|
|
249
|
+
| `mode` | text | `text` or `semantic` |
|
|
250
|
+
| `limit` | 10 | Results per page (1–50) |
|
|
251
|
+
| `offset` | 0 | For pagination |
|
|
252
|
+
|
|
253
|
+
---
|
|
254
|
+
|
|
255
|
+
### `get_node` — Full Details
|
|
256
|
+
|
|
257
|
+
**When**: You have a node ID and want the complete picture (code snippet, importance, etc.).
|
|
258
|
+
|
|
259
|
+
```
|
|
260
|
+
get_node { id: "src/lib/auth.ts::validateToken" }
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
Returns: name, type, file path, code snippet (first 12 lines), importance score.
|
|
264
|
+
|
|
265
|
+
---
|
|
266
|
+
|
|
267
|
+
### `get_connections` — What's Connected?
|
|
268
|
+
|
|
269
|
+
**When**: Understanding what a function calls, what calls it, and what it imports.
|
|
270
|
+
|
|
271
|
+
```
|
|
272
|
+
get_connections { id: "src/lib/auth.ts::validateToken" }
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
Returns:
|
|
276
|
+
- **Edges**: `CALLS`, `IMPORTS`, `EXPORTS`, `EXTENDS`, `DEFINES`
|
|
277
|
+
- **Connected nodes**: sorted by importance
|
|
278
|
+
|
|
279
|
+
This is the key tool for understanding how code fits together without reading entire files.
|
|
280
|
+
|
|
281
|
+
---
|
|
282
|
+
|
|
283
|
+
### `explore` — Full Context Tree
|
|
284
|
+
|
|
285
|
+
**When**: You need the complete call/dependency tree around a node.
|
|
286
|
+
|
|
287
|
+
```
|
|
288
|
+
explore { id: "src/app/api/route.ts::handler", depth: 2 }
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
Returns a BFS traversal — all nodes and edges reachable within `depth` hops. Use `depth: 1` for large codebases to keep responses small.
|
|
292
|
+
|
|
293
|
+
| Param | Default | Description |
|
|
294
|
+
|-------|---------|-------------|
|
|
295
|
+
| `id` | required | Starting node |
|
|
296
|
+
| `depth` | 2 | 1–3 hops |
|
|
297
|
+
|
|
298
|
+
---
|
|
299
|
+
|
|
300
|
+
### `search` — Smart Search
|
|
301
|
+
|
|
302
|
+
**When**: You want the AI to automatically figure out the best search strategy.
|
|
303
|
+
|
|
304
|
+
```
|
|
305
|
+
search { query: "how does the auth flow work?", response_format: "markdown" }
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
Automatically detects intent, runs hybrid search, expands the graph, and includes relevant memories. Best for complex or exploratory queries.
|
|
309
|
+
|
|
310
|
+
---
|
|
311
|
+
|
|
312
|
+
## 5. Tips for Maximum Token Savings
|
|
313
|
+
|
|
314
|
+
### DO ✅
|
|
315
|
+
|
|
316
|
+
- **Start every new chat with `top_nodes`** — gives the AI instant context
|
|
317
|
+
- **Use `type` filters** — `find_nodes { query: "auth", type: "function" }` returns only functions, not imports
|
|
318
|
+
- **Use semantic search for vague queries** — "error handling" finds `catchApiError()`, `handleException()`, etc.
|
|
319
|
+
- **Use `get_connections` before reading files** — often the edge list is enough to understand relationships
|
|
320
|
+
- **Use `response_format: "markdown"`** — more compact for AI consumption
|
|
321
|
+
|
|
322
|
+
### DON'T ❌
|
|
323
|
+
|
|
324
|
+
- **Don't let the AI `list_dir` and `view_file` to "understand the codebase"** — TokenOS already has this information
|
|
325
|
+
- **Don't use `depth: 3` on `explore`** — the response can be huge. Start with `depth: 1`
|
|
326
|
+
- **Don't search without filters** — `find_nodes { query: "a" }` returns too many results
|
|
327
|
+
- **Don't re-index manually** — the file watcher handles changes automatically
|
|
328
|
+
|
|
329
|
+
### Pro Tips 💡
|
|
330
|
+
|
|
331
|
+
1. **Node IDs are `filePath::name`** — e.g., `src/lib/auth.ts::validateToken`. You can construct these from file paths you already know.
|
|
332
|
+
|
|
333
|
+
2. **Importance score tells priority** — Higher score = more central to the architecture. Focus on high-importance nodes first.
|
|
334
|
+
|
|
335
|
+
3. **Semantic search finds concepts** — If you're looking for "the function that handles form submissions" but don't know its name, semantic mode will find it.
|
|
336
|
+
|
|
337
|
+
4. **The graph updates in real-time** — When you save a file, the watcher re-indexes it. Next query will have the latest data.
|
|
338
|
+
|
|
339
|
+
5. **Combine tools in sequence**:
|
|
340
|
+
```
|
|
341
|
+
1. top_nodes { limit: 10 } → overview
|
|
342
|
+
2. find_nodes { query: "..." } → find specific code
|
|
343
|
+
3. get_connections { id: "..." } → understand connections
|
|
344
|
+
4. view_file (only the specific file you need) → read actual code
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
---
|
|
348
|
+
|
|
349
|
+
## 6. Adding to Your Project
|
|
350
|
+
|
|
351
|
+
### Add `.tokenos/` to your project's `.gitignore`:
|
|
352
|
+
|
|
353
|
+
```bash
|
|
354
|
+
echo ".tokenos/" >> /path/to/your/project/.gitignore
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
This prevents the per-project database from being committed.
|
|
358
|
+
|
|
359
|
+
### Switching Between Projects
|
|
360
|
+
|
|
361
|
+
Edit `tokenos.config.json` and change `watchPath`:
|
|
362
|
+
|
|
363
|
+
```json
|
|
364
|
+
{
|
|
365
|
+
"watchPath": "/path/to/different/project"
|
|
366
|
+
}
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
Then restart: `npm run dev`. Or if you want a fresh index:
|
|
370
|
+
|
|
371
|
+
```bash
|
|
372
|
+
npm run reset
|
|
373
|
+
npm run dev
|
|
374
|
+
```
|
|
375
|
+
|
|
376
|
+
### Changing the Embedding Model
|
|
377
|
+
|
|
378
|
+
Edit `tokenos.config.json`:
|
|
379
|
+
|
|
380
|
+
```json
|
|
381
|
+
{
|
|
382
|
+
"ollama": {
|
|
383
|
+
"model": "nomic-embed-text"
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
```
|
|
387
|
+
|
|
388
|
+
**Important**: After changing models, run `npm run reset` then `npm run dev` — different models produce incompatible vectors.
|
|
389
|
+
|
|
390
|
+
---
|
|
391
|
+
|
|
392
|
+
## 7. Troubleshooting
|
|
393
|
+
|
|
394
|
+
### "0 files indexed"
|
|
395
|
+
|
|
396
|
+
- Check that `watchPath` points to a directory containing `.ts` or `.tsx` files
|
|
397
|
+
- Check that the path exists and is accessible
|
|
398
|
+
- Check `.gitignore` — files matching gitignore patterns are skipped
|
|
399
|
+
|
|
400
|
+
### "Embeddings skipped (Ollama offline)"
|
|
401
|
+
|
|
402
|
+
- Start Ollama: `ollama serve`
|
|
403
|
+
- Pull the model: `ollama pull mxbai-embed-large:latest`
|
|
404
|
+
- Semantic search falls back to text mode when Ollama is offline — everything else still works
|
|
405
|
+
|
|
406
|
+
### "Semantic search returns no results"
|
|
407
|
+
|
|
408
|
+
- Make sure embeddings were generated (check the boot log for "embeddings ready: X updated")
|
|
409
|
+
- If you changed the model, run `npm run reset` and re-index
|
|
410
|
+
|
|
411
|
+
### Stale data after refactoring
|
|
412
|
+
|
|
413
|
+
```bash
|
|
414
|
+
npm run reset # delete the database
|
|
415
|
+
npm run dev # re-index from scratch
|
|
416
|
+
```
|
|
417
|
+
|
|
418
|
+
---
|
|
419
|
+
|
|
420
|
+
## Quick Reference Card
|
|
421
|
+
|
|
422
|
+
```
|
|
423
|
+
┌──────────────────────────────────────────────────────────────┐
|
|
424
|
+
│ TOKENOS CHEAT SHEET │
|
|
425
|
+
├──────────────────────────────────────────────────────────────┤
|
|
426
|
+
│ │
|
|
427
|
+
│ START HERE: │
|
|
428
|
+
│ top_nodes { limit: 15 } │
|
|
429
|
+
│ │
|
|
430
|
+
│ FIND CODE: │
|
|
431
|
+
│ find_nodes { query: "auth", type: "function" } │
|
|
432
|
+
│ find_nodes { query: "handles login", mode: "semantic" } │
|
|
433
|
+
│ │
|
|
434
|
+
│ UNDERSTAND CONNECTIONS: │
|
|
435
|
+
│ get_connections { id: "file::name" } │
|
|
436
|
+
│ │
|
|
437
|
+
│ EXPLORE CONTEXT: │
|
|
438
|
+
│ explore { id: "file::name", depth: 1 } │
|
|
439
|
+
│ │
|
|
440
|
+
│ FULL NODE DETAILS: │
|
|
441
|
+
│ get_node { id: "file::name" } │
|
|
442
|
+
│ │
|
|
443
|
+
│ SMART SEARCH: │
|
|
444
|
+
│ search { query: "how does auth work?" } │
|
|
445
|
+
│ │
|
|
446
|
+
│ COMMANDS: │
|
|
447
|
+
│ npm run dev → start server │
|
|
448
|
+
│ npm run reset → delete DB & re-index │
|
|
449
|
+
│ │
|
|
450
|
+
└──────────────────────────────────────────────────────────────┘
|
|
451
|
+
```
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Config loader for TokenOS.
|
|
3
|
+
*
|
|
4
|
+
* Precedence: CLI args → config file → env vars → defaults.
|
|
5
|
+
* Config file: tokenos.config.json in the project root.
|
|
6
|
+
*/
|
|
7
|
+
export interface TokenOSConfig {
|
|
8
|
+
watchPath: string;
|
|
9
|
+
ollama: {
|
|
10
|
+
url: string;
|
|
11
|
+
model: string;
|
|
12
|
+
};
|
|
13
|
+
ui: {
|
|
14
|
+
enabled: boolean;
|
|
15
|
+
port: number;
|
|
16
|
+
};
|
|
17
|
+
/** Derived: absolute path to the SQLite DB for the target project */
|
|
18
|
+
dbPath: string;
|
|
19
|
+
}
|
|
20
|
+
/** Singleton config — built once on import */
|
|
21
|
+
export declare const config: TokenOSConfig;
|
|
22
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE;QACN,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;IACF,EAAE,EAAE;QACF,OAAO,EAAE,OAAO,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;CAChB;AA+DD,8CAA8C;AAC9C,eAAO,MAAM,MAAM,EAAE,aAA6B,CAAC"}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Config loader for TokenOS.
|
|
3
|
+
*
|
|
4
|
+
* Precedence: CLI args → config file → env vars → defaults.
|
|
5
|
+
* Config file: tokenos.config.json in the project root.
|
|
6
|
+
*/
|
|
7
|
+
import { readFileSync, existsSync } from "fs";
|
|
8
|
+
import { resolve, join, dirname } from "path";
|
|
9
|
+
import { fileURLToPath } from "url";
|
|
10
|
+
// ───── Loader ─────────────────────────────────────────────────────────────────
|
|
11
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
12
|
+
const __dirname = dirname(__filename);
|
|
13
|
+
const PROJECT_ROOT = resolve(__dirname, "..");
|
|
14
|
+
function loadConfigFile() {
|
|
15
|
+
const configPath = join(PROJECT_ROOT, "tokenos.config.json");
|
|
16
|
+
if (!existsSync(configPath))
|
|
17
|
+
return {};
|
|
18
|
+
try {
|
|
19
|
+
const raw = readFileSync(configPath, "utf-8");
|
|
20
|
+
return JSON.parse(raw);
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return {};
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
function buildConfig() {
|
|
27
|
+
const file = loadConfigFile();
|
|
28
|
+
const fileOllama = (file.ollama ?? {});
|
|
29
|
+
const fileUI = (file.ui ?? {});
|
|
30
|
+
// CLI args (backwards-compatible: first non-flag arg is watchPath, --ui flag)
|
|
31
|
+
const args = process.argv.slice(2);
|
|
32
|
+
const cliPath = args.find((a) => !a.startsWith("--"));
|
|
33
|
+
const cliUI = args.includes("--ui");
|
|
34
|
+
// Resolve watchPath: CLI > config > cwd
|
|
35
|
+
const rawPath = cliPath ?? file.watchPath ?? process.cwd();
|
|
36
|
+
const watchPath = resolve(rawPath);
|
|
37
|
+
// Ollama settings: env > config > defaults
|
|
38
|
+
const ollamaUrl = process.env.OLLAMA_URL ??
|
|
39
|
+
fileOllama.url ??
|
|
40
|
+
"http://localhost:11434";
|
|
41
|
+
const ollamaModel = process.env.EMBEDDING_MODEL ??
|
|
42
|
+
fileOllama.model ??
|
|
43
|
+
"nomic-embed-text";
|
|
44
|
+
// UI settings: CLI flag > config > defaults
|
|
45
|
+
const uiEnabled = cliUI || fileUI.enabled === true;
|
|
46
|
+
const uiPort = Number(process.env.GRAPH_UI_PORT) ||
|
|
47
|
+
fileUI.port ||
|
|
48
|
+
3333;
|
|
49
|
+
// Per-project DB: <watchPath>/.tokenos/graph.db
|
|
50
|
+
const dbPath = join(watchPath, ".tokenos", "graph.db");
|
|
51
|
+
return {
|
|
52
|
+
watchPath,
|
|
53
|
+
ollama: { url: ollamaUrl, model: ollamaModel },
|
|
54
|
+
ui: { enabled: uiEnabled, port: uiPort },
|
|
55
|
+
dbPath,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/** Singleton config — built once on import */
|
|
59
|
+
export const config = buildConfig();
|
|
60
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAkBpC,iFAAiF;AAEjF,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AACtC,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAE9C,SAAS,cAAc;IACrB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,qBAAqB,CAAC,CAAC;IAC7D,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,WAAW;IAClB,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC;IAC9B,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAA4B,CAAC;IAClE,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAA4B,CAAC;IAE1D,8EAA8E;IAC9E,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEpC,wCAAwC;IACxC,MAAM,OAAO,GAAG,OAAO,IAAK,IAAI,CAAC,SAAgC,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACnF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEnC,2CAA2C;IAC3C,MAAM,SAAS,GACb,OAAO,CAAC,GAAG,CAAC,UAAU;QACrB,UAAU,CAAC,GAA0B;QACtC,wBAAwB,CAAC;IAE3B,MAAM,WAAW,GACf,OAAO,CAAC,GAAG,CAAC,eAAe;QAC1B,UAAU,CAAC,KAA4B;QACxC,kBAAkB,CAAC;IAErB,4CAA4C;IAC5C,MAAM,SAAS,GAAG,KAAK,IAAK,MAAM,CAAC,OAA+B,KAAK,IAAI,CAAC;IAC5E,MAAM,MAAM,GACV,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAChC,MAAM,CAAC,IAA2B;QACnC,IAAI,CAAC;IAEP,gDAAgD;IAChD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IAEvD,OAAO;QACL,SAAS;QACT,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE;QAC9C,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE;QACxC,MAAM;KACP,CAAC;AACJ,CAAC;AAED,8CAA8C;AAC9C,MAAM,CAAC,MAAM,MAAM,GAAkB,WAAW,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connection.d.ts","sourceRoot":"","sources":["../../src/db/connection.ts"],"names":[],"mappings":"AAAA,OAAiB,EAAE,KAAK,QAAQ,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AASzE,eAAO,MAAM,EAAE,EAAE,YAA0C,CAAC"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
import { dirname } from "path";
|
|
3
|
+
import { mkdirSync } from "fs";
|
|
4
|
+
import { config } from "../config.js";
|
|
5
|
+
// Ensure the .tokenos directory exists in the target project
|
|
6
|
+
mkdirSync(dirname(config.dbPath), { recursive: true });
|
|
7
|
+
// Open or create the SQLite database at the per-project path
|
|
8
|
+
export const db = new Database(config.dbPath);
|
|
9
|
+
// Enable WAL mode for better concurrent read performance
|
|
10
|
+
db.pragma("journal_mode = WAL");
|
|
11
|
+
db.pragma("foreign_keys = ON");
|
|
12
|
+
// Run schema migrations on startup
|
|
13
|
+
db.exec(`
|
|
14
|
+
-- NODES TABLE
|
|
15
|
+
CREATE TABLE IF NOT EXISTS nodes (
|
|
16
|
+
id TEXT PRIMARY KEY,
|
|
17
|
+
type TEXT NOT NULL,
|
|
18
|
+
name TEXT NOT NULL,
|
|
19
|
+
file_path TEXT NOT NULL,
|
|
20
|
+
|
|
21
|
+
summary TEXT,
|
|
22
|
+
code_snippet TEXT,
|
|
23
|
+
embedding TEXT,
|
|
24
|
+
|
|
25
|
+
hash TEXT,
|
|
26
|
+
importance REAL DEFAULT 0,
|
|
27
|
+
|
|
28
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
29
|
+
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
30
|
+
);
|
|
31
|
+
|
|
32
|
+
-- EDGES TABLE
|
|
33
|
+
CREATE TABLE IF NOT EXISTS edges (
|
|
34
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
35
|
+
from_node TEXT NOT NULL,
|
|
36
|
+
to_node TEXT NOT NULL,
|
|
37
|
+
type TEXT NOT NULL,
|
|
38
|
+
|
|
39
|
+
FOREIGN KEY(from_node) REFERENCES nodes(id) ON DELETE CASCADE,
|
|
40
|
+
FOREIGN KEY(to_node) REFERENCES nodes(id) ON DELETE CASCADE
|
|
41
|
+
);
|
|
42
|
+
|
|
43
|
+
-- MEMORIES TABLE (Phase 5)
|
|
44
|
+
CREATE TABLE IF NOT EXISTS memories (
|
|
45
|
+
id TEXT PRIMARY KEY,
|
|
46
|
+
title TEXT NOT NULL,
|
|
47
|
+
summary TEXT NOT NULL,
|
|
48
|
+
key_points TEXT NOT NULL, -- JSON array of strings
|
|
49
|
+
tags TEXT NOT NULL, -- JSON array of strings
|
|
50
|
+
embedding TEXT, -- JSON array of numbers
|
|
51
|
+
created_at INTEGER NOT NULL
|
|
52
|
+
);
|
|
53
|
+
|
|
54
|
+
-- INDEXES
|
|
55
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name);
|
|
56
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_type ON nodes(type);
|
|
57
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_file ON nodes(file_path);
|
|
58
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_importance ON nodes(importance);
|
|
59
|
+
|
|
60
|
+
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node);
|
|
61
|
+
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node);
|
|
62
|
+
CREATE INDEX IF NOT EXISTS idx_edges_type ON edges(type);
|
|
63
|
+
`);
|
|
64
|
+
// ── Safe migration: add meta column if not already present ────────────────────
|
|
65
|
+
try {
|
|
66
|
+
db.exec(`ALTER TABLE nodes ADD COLUMN meta TEXT;`);
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
// Column already exists — ignore
|
|
70
|
+
}
|
|
71
|
+
// Unique edge constraint to prevent duplicates
|
|
72
|
+
try {
|
|
73
|
+
db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_edges_unique ON edges(from_node, to_node, type);`);
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// Already exists — ignore
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=connection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connection.js","sourceRoot":"","sources":["../../src/db/connection.ts"],"names":[],"mappings":"AAAA,OAAO,QAA2C,MAAM,gBAAgB,CAAC;AACzE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAC/B,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,6DAA6D;AAC7D,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAEvD,6DAA6D;AAC7D,MAAM,CAAC,MAAM,EAAE,GAAiB,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAE5D,yDAAyD;AACzD,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;AAChC,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;AAE/B,mCAAmC;AACnC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkDP,CAAC,CAAC;AAEH,iFAAiF;AACjF,IAAI,CAAC;IACH,EAAE,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;AACrD,CAAC;AAAC,MAAM,CAAC;IACP,iCAAiC;AACnC,CAAC;AAED,+CAA+C;AAC/C,IAAI,CAAC;IACH,EAAE,CAAC,IAAI,CAAC,wFAAwF,CAAC,CAAC;AACpG,CAAC;AAAC,MAAM,CAAC;IACP,0BAA0B;AAC5B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC"}
|
package/dist/db/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,iBAAiB,CAAC;AACrC,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { ConversationMemory } from "../types.js";
|
|
2
|
+
export declare function upsertMemory(m: ConversationMemory): void;
|
|
3
|
+
export declare function getMemory(id: string): ConversationMemory | undefined;
|
|
4
|
+
export declare function getAllMemories(): ConversationMemory[];
|
|
5
|
+
export declare function searchMemories(query: string): ConversationMemory[];
|
|
6
|
+
//# sourceMappingURL=memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../src/db/memory.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAatD,wBAAgB,YAAY,CAAC,CAAC,EAAE,kBAAkB,GAAG,IAAI,CAUxD;AAMD,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,kBAAkB,GAAG,SAAS,CASpE;AAMD,wBAAgB,cAAc,IAAI,kBAAkB,EAAE,CAOrD;AAQD,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,kBAAkB,EAAE,CAQlE"}
|