tiddlywiki-mcp-server 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +242 -0
  3. package/dist/embeddings/database.d.ts +53 -0
  4. package/dist/embeddings/database.d.ts.map +1 -0
  5. package/dist/embeddings/database.js +212 -0
  6. package/dist/embeddings/database.js.map +1 -0
  7. package/dist/embeddings/ollama-client.d.ts +39 -0
  8. package/dist/embeddings/ollama-client.d.ts.map +1 -0
  9. package/dist/embeddings/ollama-client.js +190 -0
  10. package/dist/embeddings/ollama-client.js.map +1 -0
  11. package/dist/embeddings/sync-worker.d.ts +49 -0
  12. package/dist/embeddings/sync-worker.d.ts.map +1 -0
  13. package/dist/embeddings/sync-worker.js +244 -0
  14. package/dist/embeddings/sync-worker.js.map +1 -0
  15. package/dist/filter-reference.d.ts +8 -0
  16. package/dist/filter-reference.d.ts.map +1 -0
  17. package/dist/filter-reference.js +159 -0
  18. package/dist/filter-reference.js.map +1 -0
  19. package/dist/index.d.ts +3 -0
  20. package/dist/index.d.ts.map +1 -0
  21. package/dist/index.js +450 -0
  22. package/dist/index.js.map +1 -0
  23. package/dist/logger.d.ts +17 -0
  24. package/dist/logger.d.ts.map +1 -0
  25. package/dist/logger.js +33 -0
  26. package/dist/logger.js.map +1 -0
  27. package/dist/service-discovery.d.ts +24 -0
  28. package/dist/service-discovery.d.ts.map +1 -0
  29. package/dist/service-discovery.js +82 -0
  30. package/dist/service-discovery.js.map +1 -0
  31. package/dist/tiddlywiki-http.d.ts +55 -0
  32. package/dist/tiddlywiki-http.d.ts.map +1 -0
  33. package/dist/tiddlywiki-http.js +253 -0
  34. package/dist/tiddlywiki-http.js.map +1 -0
  35. package/dist/tools/create-tiddler.d.ts +15 -0
  36. package/dist/tools/create-tiddler.d.ts.map +1 -0
  37. package/dist/tools/create-tiddler.js +61 -0
  38. package/dist/tools/create-tiddler.js.map +1 -0
  39. package/dist/tools/delete-tiddler.d.ts +9 -0
  40. package/dist/tools/delete-tiddler.d.ts.map +1 -0
  41. package/dist/tools/delete-tiddler.js +40 -0
  42. package/dist/tools/delete-tiddler.js.map +1 -0
  43. package/dist/tools/index.d.ts +6 -0
  44. package/dist/tools/index.d.ts.map +1 -0
  45. package/dist/tools/index.js +7 -0
  46. package/dist/tools/index.js.map +1 -0
  47. package/dist/tools/search-tiddlers.d.ts +12 -0
  48. package/dist/tools/search-tiddlers.d.ts.map +1 -0
  49. package/dist/tools/search-tiddlers.js +189 -0
  50. package/dist/tools/search-tiddlers.js.map +1 -0
  51. package/dist/tools/types.d.ts +101 -0
  52. package/dist/tools/types.d.ts.map +1 -0
  53. package/dist/tools/types.js +55 -0
  54. package/dist/tools/types.js.map +1 -0
  55. package/dist/tools/update-tiddler.d.ts +9 -0
  56. package/dist/tools/update-tiddler.d.ts.map +1 -0
  57. package/dist/tools/update-tiddler.js +90 -0
  58. package/dist/tools/update-tiddler.js.map +1 -0
  59. package/package.json +67 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Petru Paler <petru@paler.net>
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,242 @@
1
+ # TiddlyWiki MCP Server
2
+
3
+ A Model Context Protocol (MCP) server that provides AI assistants with access to TiddlyWiki wikis via the HTTP API. Supports semantic search using Ollama embeddings.
4
+
5
+ ## Features
6
+
7
+ ### MCP Tools
8
+
9
+ - **search_tiddlers** - Search tiddlers using TiddlyWiki filter syntax, semantic similarity, or hybrid (both combined)
10
+ - **create_tiddler** - Create new tiddlers with custom fields
11
+ - **update_tiddler** - Update existing tiddlers with diff preview
12
+ - **delete_tiddler** - Delete tiddlers with content preview
13
+
14
+ ### MCP Resources
15
+
16
+ - **filter-reference://syntax** - Complete TiddlyWiki filter syntax reference
17
+
18
+ ### Semantic Search
19
+
20
+ When Ollama is available, the server provides semantic search capabilities:
21
+
22
+ - Natural language queries find conceptually related tiddlers
23
+ - Uses `nomic-embed-text` embeddings model
24
+ - SQLite-vec for efficient vector similarity search
25
+ - Background sync keeps embeddings up-to-date
26
+ - Hybrid mode combines filter results with semantic reranking
27
+
28
+ ## Requirements
29
+
30
+ - Node.js 22+
31
+ - TiddlyWiki with HTTP API enabled (e.g., TiddlyWiki on Node.js with `listen` command)
32
+ - Ollama (optional, for semantic search)
33
+
34
+ ### Build Prerequisites
35
+
36
+ This project uses native SQLite modules that require compilation. You'll need:
37
+
38
+ - **Linux**: `build-essential`, Python 3
39
+ - **macOS**: Xcode Command Line Tools (`xcode-select --install`)
40
+ - **Windows**: Visual Studio Build Tools, Python 3
41
+
42
+ ## Installation
43
+
44
+ ```bash
45
+ npm install
46
+ npm run build
47
+ ```
48
+
49
+ ## Quick Start
50
+
51
+ ### 1. Start TiddlyWiki with HTTP API
52
+
53
+ ```bash
54
+ # Install TiddlyWiki if you haven't already
55
+ npm install -g tiddlywiki
56
+
57
+ # Create a new wiki and start it with HTTP API
58
+ tiddlywiki mywiki --init server
59
+ tiddlywiki mywiki --listen port=8080
60
+ ```
61
+
62
+ ### 2. (Optional) Set up Ollama for Semantic Search
63
+
64
+ ```bash
65
+ # Install Ollama from https://ollama.ai
66
+ # Then pull the embedding model:
67
+ ollama pull nomic-embed-text
68
+ ```
69
+
70
+ ### 3. Start the MCP Server
71
+
72
+ ```bash
73
+ TIDDLYWIKI_URL=http://localhost:8080 node dist/index.js
74
+ ```
75
+
76
+ ## Configuration
77
+
78
+ All configuration is via environment variables. See `.env.example` for a complete reference.
79
+
80
+ ### Required
81
+
82
+ | Variable | Description |
83
+ |----------|-------------|
84
+ | `TIDDLYWIKI_URL` | URL of your TiddlyWiki server (e.g., `http://localhost:8080`) |
85
+
86
+ ### Optional
87
+
88
+ | Variable | Default | Description |
89
+ |----------|---------|-------------|
90
+ | `MCP_TRANSPORT` | `stdio` | Transport mode: `stdio` or `http` |
91
+ | `MCP_PORT` | `3000` | HTTP server port (when using http transport) |
92
+ | `OLLAMA_URL` | `http://localhost:11434` | Ollama API URL |
93
+ | `OLLAMA_MODEL` | `nomic-embed-text` | Embedding model name |
94
+ | `EMBEDDINGS_ENABLED` | `true` | Enable/disable semantic search |
95
+ | `EMBEDDINGS_DB_PATH` | `./embeddings.db` | SQLite database path for embeddings |
96
+ | `AUTH_HEADER` | `X-Oidc-Username` | HTTP header for authentication (can be any header your TiddlyWiki expects) |
97
+ | `AUTH_USER` | `mcp-user` | Username for TiddlyWiki API requests |
98
+
99
+ ## Usage
100
+
101
+ ### stdio Mode (Claude Desktop)
102
+
103
+ Add to your Claude Desktop configuration (`claude_desktop_config.json`):
104
+
105
+ ```json
106
+ {
107
+ "mcpServers": {
108
+ "tiddlywiki": {
109
+ "command": "node",
110
+ "args": ["/path/to/tiddlywiki-mcp/dist/index.js"],
111
+ "env": {
112
+ "TIDDLYWIKI_URL": "http://localhost:8080"
113
+ }
114
+ }
115
+ }
116
+ }
117
+ ```
118
+
119
+ ### HTTP Mode
120
+
121
+ Start the server:
122
+
123
+ ```bash
124
+ TIDDLYWIKI_URL=http://localhost:8080 MCP_TRANSPORT=http MCP_PORT=3000 node dist/index.js
125
+ ```
126
+
127
+ The server exposes:
128
+ - `GET /health` - Health check endpoint
129
+ - `POST /mcp` - MCP JSON-RPC endpoint (stateless mode)
130
+
131
+ ### Example Tool Usage
132
+
133
+ **Filter search** (TiddlyWiki filter syntax):
134
+ ```json
135
+ {
136
+ "name": "search_tiddlers",
137
+ "arguments": {
138
+ "filter": "[tag[Journal]prefix[2025-01]]",
139
+ "includeText": true
140
+ }
141
+ }
142
+ ```
143
+
144
+ **Semantic search** (natural language):
145
+ ```json
146
+ {
147
+ "name": "search_tiddlers",
148
+ "arguments": {
149
+ "semantic": "times I felt anxious about work",
150
+ "limit": 10
151
+ }
152
+ }
153
+ ```
154
+
155
+ **Hybrid search** (filter + semantic reranking):
156
+ ```json
157
+ {
158
+ "name": "search_tiddlers",
159
+ "arguments": {
160
+ "filter": "[tag[Journal]]",
161
+ "semantic": "productivity tips",
162
+ "limit": 20
163
+ }
164
+ }
165
+ ```
166
+
167
+ ## Development
168
+
169
+ ### Setup
170
+
171
+ ```bash
172
+ npm install
173
+ ```
174
+
175
+ ### Running Tests
176
+
177
+ ```bash
178
+ npm test
179
+ ```
180
+
181
+ Tests run quickly (~1s) and include unit tests for all tool handlers.
182
+
183
+ ### Linting
184
+
185
+ ```bash
186
+ npm run lint # Check for issues
187
+ npm run format # Fix formatting
188
+ npm run format:check # Check formatting only
189
+ ```
190
+
191
+ ### Type Checking
192
+
193
+ ```bash
194
+ npm run typecheck
195
+ ```
196
+
197
+ ### Pre-commit Hooks
198
+
199
+ Pre-commit hooks are configured with lefthook and run automatically:
200
+
201
+ 1. Format check (Prettier)
202
+ 2. Lint (ESLint)
203
+ 3. Tests (Vitest)
204
+ 4. Type check (TypeScript)
205
+
206
+ ### Building
207
+
208
+ ```bash
209
+ npm run build
210
+ ```
211
+
212
+ ## Architecture
213
+
214
+ ```
215
+ src/
216
+ ├── index.ts # Entry point, transport setup, server lifecycle
217
+ ├── tiddlywiki-http.ts # TiddlyWiki HTTP API client
218
+ ├── service-discovery.ts # URL resolution (direct URLs, Consul SRV, hostname:port)
219
+ ├── filter-reference.ts # Filter syntax documentation
220
+ ├── logger.ts # Structured logging
221
+ ├── tools/ # MCP tool handlers
222
+ │ ├── types.ts # Shared types and Zod schemas
223
+ │ ├── search-tiddlers.ts
224
+ │ ├── create-tiddler.ts
225
+ │ ├── update-tiddler.ts
226
+ │ └── delete-tiddler.ts
227
+ └── embeddings/ # Semantic search infrastructure
228
+ ├── database.ts # SQLite-vec database
229
+ ├── ollama-client.ts # Ollama API client
230
+ └── sync-worker.ts # Background embedding sync
231
+ ```
232
+
233
+ ### Key Design Decisions
234
+
235
+ - **Stateless HTTP mode**: Each request gets its own Server/Transport instance to prevent request ID collisions with concurrent clients
236
+ - **Graceful degradation**: Semantic search is optional; the server works without Ollama
237
+ - **Token-aware responses**: Search results are validated against token limits with pagination suggestions
238
+ - **Background sync**: Embeddings are updated periodically without blocking requests
239
+
240
+ ## License
241
+
242
+ MIT
@@ -0,0 +1,53 @@
1
+ export interface EmbeddingMetadata {
2
+ created: string;
3
+ modified: string;
4
+ tags: string;
5
+ }
6
+ export interface SearchResult {
7
+ tiddler_title: string;
8
+ chunk_id: number;
9
+ chunk_text: string;
10
+ created: string;
11
+ modified: string;
12
+ tags: string;
13
+ distance: number;
14
+ }
15
+ export interface SyncStatus {
16
+ tiddler_title: string;
17
+ last_modified: string;
18
+ last_indexed: string;
19
+ total_chunks: number;
20
+ status: string;
21
+ error_message: string | null;
22
+ }
23
+ export declare class EmbeddingsDB {
24
+ private db;
25
+ private dbPath;
26
+ constructor(dbPath?: string);
27
+ private initSchema;
28
+ /**
29
+ * Migrate existing empty timestamps to sentinel value
30
+ * Fixes infinite re-indexing bug for tiddlers without modified timestamps
31
+ */
32
+ private migrateEmptyTimestamps;
33
+ /**
34
+ * Add status and error_message columns to sync_status table
35
+ * Enables tracking of empty/error tiddlers to prevent infinite re-indexing
36
+ */
37
+ private migrateAddStatusColumns;
38
+ insertEmbedding(tiddlerTitle: string, chunkId: number, embedding: number[], chunkText: string, metadata: EmbeddingMetadata): void;
39
+ searchSimilar(queryEmbedding: number[], limit?: number): SearchResult[];
40
+ updateSyncStatus(tiddlerTitle: string, lastModified: string, totalChunks: number, status?: string, errorMessage?: string | null): void;
41
+ getSyncStatus(tiddlerTitle: string): SyncStatus | null;
42
+ getAllSyncStatuses(): SyncStatus[];
43
+ getUnindexedTiddlers(tiddlerTitles: string[]): string[];
44
+ getOutdatedTiddlers(tiddlersWithModified: Array<{
45
+ title: string;
46
+ modified: string;
47
+ }>): string[];
48
+ deleteEmbeddingsForTiddler(tiddlerTitle: string): void;
49
+ getEmbeddingsCount(): number;
50
+ getIndexedTiddlersCount(): number;
51
+ close(): void;
52
+ }
53
+ //# sourceMappingURL=database.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../src/embeddings/database.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,EAAE,CAAoB;IAC9B,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,GAAE,MAA4D;IAiBhF,OAAO,CAAC,UAAU;IAiClB;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAgB9B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAmB/B,eAAe,CACb,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EAAE,EACnB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,iBAAiB,GAC1B,IAAI;IA2BP,aAAa,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE,KAAK,GAAE,MAAW,GAAG,YAAY,EAAE;IAqB3E,gBAAgB,CACd,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,MAAM,GAAE,MAAkB,EAC1B,YAAY,GAAE,MAAM,GAAG,IAAW,GACjC,IAAI;IASP,aAAa,CAAC,YAAY,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI;IAUtD,kBAAkB,IAAI,UAAU,EAAE;IAUlC,oBAAoB,CAAC,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAmBvD,mBAAmB,CAAC,oBAAoB,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,MAAM,EAAE;IAc/F,0BAA0B,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IA8BtD,kBAAkB,IAAI,MAAM;IAQ5B,uBAAuB,IAAI,MAAM;IAQjC,KAAK,IAAI,IAAI;CAGd"}
@@ -0,0 +1,212 @@
1
+ // ABOUTME: SQLite database for storing and querying vector embeddings
2
+ // ABOUTME: Uses sqlite-vec for efficient similarity search operations
3
+ import Database from 'better-sqlite3';
4
+ import * as sqliteVec from 'sqlite-vec';
5
+ export class EmbeddingsDB {
6
+ db;
7
+ dbPath;
8
+ constructor(dbPath = process.env.EMBEDDINGS_DB_PATH || './embeddings.db') {
9
+ this.dbPath = dbPath;
10
+ this.db = new Database(dbPath);
11
+ // Load sqlite-vec extension
12
+ sqliteVec.load(this.db);
13
+ // Initialize schema
14
+ this.initSchema();
15
+ // Migrate any existing empty timestamps to sentinel value
16
+ this.migrateEmptyTimestamps();
17
+ // Add status tracking columns if they don't exist
18
+ this.migrateAddStatusColumns();
19
+ }
20
+ initSchema() {
21
+ // Create vec0 virtual table for embeddings (vectors only)
22
+ this.db.exec(`
23
+ CREATE VIRTUAL TABLE IF NOT EXISTS entry_embeddings USING vec0(
24
+ embedding float[768]
25
+ );
26
+
27
+ CREATE TABLE IF NOT EXISTS embedding_metadata (
28
+ id INTEGER PRIMARY KEY,
29
+ tiddler_title TEXT NOT NULL,
30
+ chunk_id NOT NULL,
31
+ chunk_text TEXT NOT NULL,
32
+ created TEXT,
33
+ modified TEXT,
34
+ tags TEXT
35
+ );
36
+
37
+ CREATE TABLE IF NOT EXISTS sync_status (
38
+ tiddler_title TEXT PRIMARY KEY,
39
+ last_modified TEXT NOT NULL,
40
+ last_indexed TEXT NOT NULL,
41
+ total_chunks INTEGER NOT NULL DEFAULT 1,
42
+ indexed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
43
+ );
44
+
45
+ CREATE INDEX IF NOT EXISTS idx_sync_status_modified
46
+ ON sync_status(last_modified);
47
+
48
+ CREATE INDEX IF NOT EXISTS idx_embedding_metadata_tiddler
49
+ ON embedding_metadata(tiddler_title);
50
+ `);
51
+ }
52
+ /**
53
+ * Migrate existing empty timestamps to sentinel value
54
+ * Fixes infinite re-indexing bug for tiddlers without modified timestamps
55
+ */
56
+ migrateEmptyTimestamps() {
57
+ const MISSING_TIMESTAMP = '00000000000000000';
58
+ const stmt = this.db.prepare(`
59
+ UPDATE sync_status
60
+ SET last_modified = ?
61
+ WHERE last_modified = ''
62
+ `);
63
+ const result = stmt.run(MISSING_TIMESTAMP);
64
+ if (result.changes > 0) {
65
+ console.log(`[DB Migration] Updated ${result.changes} entries with missing timestamps`);
66
+ }
67
+ }
68
+ /**
69
+ * Add status and error_message columns to sync_status table
70
+ * Enables tracking of empty/error tiddlers to prevent infinite re-indexing
71
+ */
72
+ migrateAddStatusColumns() {
73
+ // Check if status column exists
74
+ const tableInfo = this.db.prepare('PRAGMA table_info(sync_status)').all();
75
+ const hasStatus = tableInfo.some((col) => col.name === 'status');
76
+ if (!hasStatus) {
77
+ console.log('[DB Migration] Adding status and error_message columns to sync_status table');
78
+ this.db.exec(`
79
+ ALTER TABLE sync_status ADD COLUMN status TEXT NOT NULL DEFAULT 'indexed';
80
+ ALTER TABLE sync_status ADD COLUMN error_message TEXT;
81
+ `);
82
+ console.log('[DB Migration] Status columns added successfully');
83
+ }
84
+ }
85
+ insertEmbedding(tiddlerTitle, chunkId, embedding, chunkText, metadata) {
86
+ const embeddingArray = new Float32Array(embedding);
87
+ // Insert embedding into vec0 table
88
+ const embStmt = this.db.prepare(`
89
+ INSERT INTO entry_embeddings(rowid, embedding)
90
+ VALUES (NULL, ?)
91
+ `);
92
+ const embResult = embStmt.run(Buffer.from(embeddingArray.buffer));
93
+ const rowid = embResult.lastInsertRowid;
94
+ // Insert metadata into regular table with same rowid
95
+ const metaStmt = this.db.prepare(`
96
+ INSERT INTO embedding_metadata(id, tiddler_title, chunk_id, chunk_text, created, modified, tags)
97
+ VALUES (?, ?, ?, ?, ?, ?, ?)
98
+ `);
99
+ metaStmt.run(rowid, tiddlerTitle, chunkId, chunkText, metadata.created, metadata.modified, metadata.tags);
100
+ }
101
+ searchSimilar(queryEmbedding, limit = 10) {
102
+ const embeddingArray = new Float32Array(queryEmbedding);
103
+ const stmt = this.db.prepare(`
104
+ SELECT
105
+ m.tiddler_title,
106
+ m.chunk_id,
107
+ m.chunk_text,
108
+ m.created,
109
+ m.modified,
110
+ m.tags,
111
+ e.distance
112
+ FROM entry_embeddings e
113
+ JOIN embedding_metadata m ON e.rowid = m.id
114
+ WHERE e.embedding MATCH ?
115
+ AND k = ?
116
+ `);
117
+ return stmt.all(Buffer.from(embeddingArray.buffer), limit);
118
+ }
119
+ updateSyncStatus(tiddlerTitle, lastModified, totalChunks, status = 'indexed', errorMessage = null) {
120
+ const stmt = this.db.prepare(`
121
+ INSERT OR REPLACE INTO sync_status(tiddler_title, last_modified, last_indexed, total_chunks, status, error_message)
122
+ VALUES (?, ?, datetime('now'), ?, ?, ?)
123
+ `);
124
+ stmt.run(tiddlerTitle, lastModified, totalChunks, status, errorMessage);
125
+ }
126
+ getSyncStatus(tiddlerTitle) {
127
+ const stmt = this.db.prepare(`
128
+ SELECT tiddler_title, last_modified, last_indexed, total_chunks, status, error_message
129
+ FROM sync_status
130
+ WHERE tiddler_title = ?
131
+ `);
132
+ return stmt.get(tiddlerTitle);
133
+ }
134
+ getAllSyncStatuses() {
135
+ const stmt = this.db.prepare(`
136
+ SELECT tiddler_title, last_modified, last_indexed, total_chunks, status, error_message
137
+ FROM sync_status
138
+ ORDER BY tiddler_title
139
+ `);
140
+ return stmt.all();
141
+ }
142
+ getUnindexedTiddlers(tiddlerTitles) {
143
+ if (tiddlerTitles.length === 0) {
144
+ return [];
145
+ }
146
+ const _placeholders = tiddlerTitles.map(() => '?').join(',');
147
+ const stmt = this.db.prepare(`
148
+ SELECT ? as title
149
+ FROM (SELECT ? as title ${tiddlerTitles
150
+ .slice(1)
151
+ .map(() => 'UNION ALL SELECT ?')
152
+ .join(' ')})
153
+ WHERE title NOT IN (SELECT tiddler_title FROM sync_status)
154
+ `);
155
+ const results = stmt.all(...tiddlerTitles);
156
+ return results.map((r) => r.title);
157
+ }
158
+ getOutdatedTiddlers(tiddlersWithModified) {
159
+ const outdated = [];
160
+ for (const tiddler of tiddlersWithModified) {
161
+ const syncStatus = this.getSyncStatus(tiddler.title);
162
+ if (!syncStatus || syncStatus.last_modified !== tiddler.modified) {
163
+ outdated.push(tiddler.title);
164
+ }
165
+ }
166
+ return outdated;
167
+ }
168
+ deleteEmbeddingsForTiddler(tiddlerTitle) {
169
+ // Get all rowids for this tiddler from metadata table
170
+ const getRowids = this.db.prepare(`
171
+ SELECT id FROM embedding_metadata WHERE tiddler_title = ?
172
+ `);
173
+ const rows = getRowids.all(tiddlerTitle);
174
+ // Delete from embedding_metadata
175
+ const deleteMetadata = this.db.prepare(`
176
+ DELETE FROM embedding_metadata WHERE tiddler_title = ?
177
+ `);
178
+ deleteMetadata.run(tiddlerTitle);
179
+ // Delete from entry_embeddings using rowids
180
+ if (rows.length > 0) {
181
+ const deleteEmbeddings = this.db.prepare(`
182
+ DELETE FROM entry_embeddings WHERE rowid = ?
183
+ `);
184
+ for (const row of rows) {
185
+ deleteEmbeddings.run(row.id);
186
+ }
187
+ }
188
+ // Delete sync status
189
+ const deleteSyncStatus = this.db.prepare(`
190
+ DELETE FROM sync_status WHERE tiddler_title = ?
191
+ `);
192
+ deleteSyncStatus.run(tiddlerTitle);
193
+ }
194
+ getEmbeddingsCount() {
195
+ const stmt = this.db.prepare(`
196
+ SELECT COUNT(*) as count FROM entry_embeddings
197
+ `);
198
+ const result = stmt.get();
199
+ return result.count;
200
+ }
201
+ getIndexedTiddlersCount() {
202
+ const stmt = this.db.prepare(`
203
+ SELECT COUNT(*) as count FROM sync_status
204
+ `);
205
+ const result = stmt.get();
206
+ return result.count;
207
+ }
208
+ close() {
209
+ this.db.close();
210
+ }
211
+ }
212
+ //# sourceMappingURL=database.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"database.js","sourceRoot":"","sources":["../../src/embeddings/database.ts"],"names":[],"mappings":"AAAA,sEAAsE;AACtE,sEAAsE;AAEtE,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,KAAK,SAAS,MAAM,YAAY,CAAC;AA2BxC,MAAM,OAAO,YAAY;IACf,EAAE,CAAoB;IACtB,MAAM,CAAS;IAEvB,YAAY,SAAiB,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,iBAAiB;QAC9E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAE/B,4BAA4B;QAC5B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAExB,oBAAoB;QACpB,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,0DAA0D;QAC1D,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,kDAAkD;QAClD,IAAI,CAAC,uBAAuB,EAAE,CAAC;IACjC,CAAC;IAEO,UAAU;QAChB,0DAA0D;QAC1D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA4BZ,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,sBAAsB;QAC5B,MAAM,iBAAiB,GAAG,mBAAmB,CAAC;QAE9C,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI5B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAE3C,IAAI,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,0BAA0B,MAAM,CAAC,OAAO,kCAAkC,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,uBAAuB;QAC7B,gCAAgC;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC,GAAG,EAErE,CAAC;QACH,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QAEjE,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;YAE3F,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;OAGZ,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,OAAe,EACf,SAAmB,EACnB,SAAiB,EACjB,QAA2B;QAE3B,MAAM,cAAc,GAAG,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;QAEnD,mCAAmC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAG/B,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,SAAS,CAAC,eAAe,CAAC;QAExC,qDAAqD;QACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAGhC,CAAC,CAAC;QACH,QAAQ,CAAC,GAAG,CACV,KAAK,EACL,YAAY,EACZ,OAAO,EACP,SAAS,EACT,QAAQ,CAAC,OAAO,EAChB,QAAQ,CAAC,QAAQ,EACjB,QAAQ,CAAC,IAAI,CACd,CAAC;IACJ,CAAC;IAED,aAAa,CAAC,cAAwB,EAAE,QAAgB,EAAE;QACxD,MAAM,cAAc,GAAG,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC;QAExD,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;;KAa5B,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,KAAK,CAAmB,CAAC;IAC/E,CAAC;IAED,gBAAgB,CACd,YAAoB,EACpB,YAAoB,EACpB,WAAmB,EACnB,SAAiB,SAAS,EAC1B,eAA8B,IAAI;QAElC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAG5B,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IAC1E,CAAC;IAED,aAAa,CAAC,YAAoB;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI5B,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,CAAsB,CAAC;IACrD,CAAC;IAED,kBAAkB;QAChB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI5B,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,GAAG,EAAkB,CAAC;IACpC,CAAC;IAED,oBAAoB,CAAC,aAAuB;QAC1C,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;gCAED,aAAa;aACpC,KAAK,CAAC,CAAC,CAAC;aACR,GAAG,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC;aAC/B,IAAI,CAAC,GAAG,CAAC;;KAEb,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,aAAa,CAA6B,CAAC;QACvE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,mBAAmB,CAAC,oBAAgE;QAClF,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,KAAK,MAAM,OAAO,IAAI,oBAAoB,EAAE,CAAC;YAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAErD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,aAAa,KAAK,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACjE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,0BAA0B,CAAC,YAAoB;QAC7C,sDAAsD;QACtD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAEjC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,YAAY,CAA0B,CAAC;QAElE,iCAAiC;QACjC,MAAM,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAEtC,CAAC,CAAC;QACH,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAEjC,4CAA4C;QAC5C,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,gBAAgB,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;OAExC,CAAC,CAAC;YACH,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,MAAM,gBAAgB,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAExC,CAAC,CAAC;QACH,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACrC,CAAC;IAED,kBAAkB;QAChB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAE5B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAuB,CAAC;QAC/C,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,uBAAuB;QACrB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAE5B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAuB,CAAC;QAC/C,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}
@@ -0,0 +1,39 @@
1
+ export declare class OllamaClient {
2
+ private serviceUrl;
3
+ private model;
4
+ private resolvedBaseUrl;
5
+ constructor(serviceUrl?: string, model?: string);
6
+ /**
7
+ * Get the resolved base URL, using service discovery if needed.
8
+ * Caches the result for subsequent calls.
9
+ */
10
+ private getBaseUrl;
11
+ generateEmbeddings(texts: string[]): Promise<number[][]>;
12
+ generateEmbedding(text: string): Promise<number[]>;
13
+ /**
14
+ * Generate embedding for a document with proper task prefix.
15
+ * Uses "search_document:" prefix as required by nomic-embed-text for RAG.
16
+ */
17
+ generateDocumentEmbedding(text: string): Promise<number[]>;
18
+ /**
19
+ * Generate embeddings for multiple documents with proper task prefix.
20
+ * Uses "search_document:" prefix as required by nomic-embed-text for RAG.
21
+ */
22
+ generateDocumentEmbeddings(texts: string[]): Promise<number[][]>;
23
+ /**
24
+ * Generate embedding for a query with proper task prefix.
25
+ * Uses "search_query:" prefix as required by nomic-embed-text for RAG.
26
+ */
27
+ generateQueryEmbedding(text: string): Promise<number[]>;
28
+ healthCheck(): Promise<boolean>;
29
+ /**
30
+ * Chunk text if it exceeds token limit.
31
+ * Splits at paragraph boundaries to maintain semantic coherence.
32
+ */
33
+ chunkText(text: string, maxTokens?: number): string[];
34
+ /**
35
+ * Count tokens in text using gpt-tokenizer (approximation for nomic-embed-text)
36
+ */
37
+ countTokens(text: string): number;
38
+ }
39
+ //# sourceMappingURL=ollama-client.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ollama-client.d.ts","sourceRoot":"","sources":["../../src/embeddings/ollama-client.ts"],"names":[],"mappings":"AA+CA,qBAAa,YAAY;IACvB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,eAAe,CAAuB;gBAG5C,UAAU,GAAE,MAA2D,EACvE,KAAK,GAAE,MAAuD;IAMhE;;;OAGG;YACW,UAAU;IAQlB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAiCxD,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKxD;;;OAGG;IACG,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKhE;;;OAGG;IACG,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAKtE;;;OAGG;IACG,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKvD,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAqBrC;;;OAGG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAa,GAAG,MAAM,EAAE;IAyE3D;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;CAGlC"}