tiddlywiki-mcp-server 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +242 -0
- package/dist/embeddings/database.d.ts +53 -0
- package/dist/embeddings/database.d.ts.map +1 -0
- package/dist/embeddings/database.js +212 -0
- package/dist/embeddings/database.js.map +1 -0
- package/dist/embeddings/ollama-client.d.ts +39 -0
- package/dist/embeddings/ollama-client.d.ts.map +1 -0
- package/dist/embeddings/ollama-client.js +190 -0
- package/dist/embeddings/ollama-client.js.map +1 -0
- package/dist/embeddings/sync-worker.d.ts +49 -0
- package/dist/embeddings/sync-worker.d.ts.map +1 -0
- package/dist/embeddings/sync-worker.js +244 -0
- package/dist/embeddings/sync-worker.js.map +1 -0
- package/dist/filter-reference.d.ts +8 -0
- package/dist/filter-reference.d.ts.map +1 -0
- package/dist/filter-reference.js +159 -0
- package/dist/filter-reference.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +450 -0
- package/dist/index.js.map +1 -0
- package/dist/logger.d.ts +17 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +33 -0
- package/dist/logger.js.map +1 -0
- package/dist/service-discovery.d.ts +24 -0
- package/dist/service-discovery.d.ts.map +1 -0
- package/dist/service-discovery.js +82 -0
- package/dist/service-discovery.js.map +1 -0
- package/dist/tiddlywiki-http.d.ts +55 -0
- package/dist/tiddlywiki-http.d.ts.map +1 -0
- package/dist/tiddlywiki-http.js +253 -0
- package/dist/tiddlywiki-http.js.map +1 -0
- package/dist/tools/create-tiddler.d.ts +15 -0
- package/dist/tools/create-tiddler.d.ts.map +1 -0
- package/dist/tools/create-tiddler.js +61 -0
- package/dist/tools/create-tiddler.js.map +1 -0
- package/dist/tools/delete-tiddler.d.ts +9 -0
- package/dist/tools/delete-tiddler.d.ts.map +1 -0
- package/dist/tools/delete-tiddler.js +40 -0
- package/dist/tools/delete-tiddler.js.map +1 -0
- package/dist/tools/index.d.ts +6 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +7 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/search-tiddlers.d.ts +12 -0
- package/dist/tools/search-tiddlers.d.ts.map +1 -0
- package/dist/tools/search-tiddlers.js +189 -0
- package/dist/tools/search-tiddlers.js.map +1 -0
- package/dist/tools/types.d.ts +101 -0
- package/dist/tools/types.d.ts.map +1 -0
- package/dist/tools/types.js +55 -0
- package/dist/tools/types.js.map +1 -0
- package/dist/tools/update-tiddler.d.ts +9 -0
- package/dist/tools/update-tiddler.d.ts.map +1 -0
- package/dist/tools/update-tiddler.js +90 -0
- package/dist/tools/update-tiddler.js.map +1 -0
- package/package.json +67 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Petru Paler <petru@paler.net>
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# TiddlyWiki MCP Server
|
|
2
|
+
|
|
3
|
+
A Model Context Protocol (MCP) server that provides AI assistants with access to TiddlyWiki wikis via the HTTP API. Supports semantic search using Ollama embeddings.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
### MCP Tools
|
|
8
|
+
|
|
9
|
+
- **search_tiddlers** - Search tiddlers using TiddlyWiki filter syntax, semantic similarity, or hybrid (both combined)
|
|
10
|
+
- **create_tiddler** - Create new tiddlers with custom fields
|
|
11
|
+
- **update_tiddler** - Update existing tiddlers with diff preview
|
|
12
|
+
- **delete_tiddler** - Delete tiddlers with content preview
|
|
13
|
+
|
|
14
|
+
### MCP Resources
|
|
15
|
+
|
|
16
|
+
- **filter-reference://syntax** - Complete TiddlyWiki filter syntax reference
|
|
17
|
+
|
|
18
|
+
### Semantic Search
|
|
19
|
+
|
|
20
|
+
When Ollama is available, the server provides semantic search capabilities:
|
|
21
|
+
|
|
22
|
+
- Natural language queries find conceptually related tiddlers
|
|
23
|
+
- Uses `nomic-embed-text` embeddings model
|
|
24
|
+
- SQLite-vec for efficient vector similarity search
|
|
25
|
+
- Background sync keeps embeddings up-to-date
|
|
26
|
+
- Hybrid mode combines filter results with semantic reranking
|
|
27
|
+
|
|
28
|
+
## Requirements
|
|
29
|
+
|
|
30
|
+
- Node.js 22+
|
|
31
|
+
- TiddlyWiki with HTTP API enabled (e.g., TiddlyWiki on Node.js with `listen` command)
|
|
32
|
+
- Ollama (optional, for semantic search)
|
|
33
|
+
|
|
34
|
+
### Build Prerequisites
|
|
35
|
+
|
|
36
|
+
This project uses native SQLite modules that require compilation. You'll need:
|
|
37
|
+
|
|
38
|
+
- **Linux**: `build-essential`, Python 3
|
|
39
|
+
- **macOS**: Xcode Command Line Tools (`xcode-select --install`)
|
|
40
|
+
- **Windows**: Visual Studio Build Tools, Python 3
|
|
41
|
+
|
|
42
|
+
## Installation
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
npm install
|
|
46
|
+
npm run build
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Quick Start
|
|
50
|
+
|
|
51
|
+
### 1. Start TiddlyWiki with HTTP API
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
# Install TiddlyWiki if you haven't already
|
|
55
|
+
npm install -g tiddlywiki
|
|
56
|
+
|
|
57
|
+
# Create a new wiki and start it with HTTP API
|
|
58
|
+
tiddlywiki mywiki --init server
|
|
59
|
+
tiddlywiki mywiki --listen port=8080
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
### 2. (Optional) Set up Ollama for Semantic Search
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
# Install Ollama from https://ollama.ai
|
|
66
|
+
# Then pull the embedding model:
|
|
67
|
+
ollama pull nomic-embed-text
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### 3. Start the MCP Server
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
TIDDLYWIKI_URL=http://localhost:8080 node dist/index.js
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Configuration
|
|
77
|
+
|
|
78
|
+
All configuration is via environment variables. See `.env.example` for a complete reference.
|
|
79
|
+
|
|
80
|
+
### Required
|
|
81
|
+
|
|
82
|
+
| Variable | Description |
|
|
83
|
+
|----------|-------------|
|
|
84
|
+
| `TIDDLYWIKI_URL` | URL of your TiddlyWiki server (e.g., `http://localhost:8080`) |
|
|
85
|
+
|
|
86
|
+
### Optional
|
|
87
|
+
|
|
88
|
+
| Variable | Default | Description |
|
|
89
|
+
|----------|---------|-------------|
|
|
90
|
+
| `MCP_TRANSPORT` | `stdio` | Transport mode: `stdio` or `http` |
|
|
91
|
+
| `MCP_PORT` | `3000` | HTTP server port (when using http transport) |
|
|
92
|
+
| `OLLAMA_URL` | `http://localhost:11434` | Ollama API URL |
|
|
93
|
+
| `OLLAMA_MODEL` | `nomic-embed-text` | Embedding model name |
|
|
94
|
+
| `EMBEDDINGS_ENABLED` | `true` | Enable/disable semantic search |
|
|
95
|
+
| `EMBEDDINGS_DB_PATH` | `./embeddings.db` | SQLite database path for embeddings |
|
|
96
|
+
| `AUTH_HEADER` | `X-Oidc-Username` | HTTP header for authentication (can be any header your TiddlyWiki expects) |
|
|
97
|
+
| `AUTH_USER` | `mcp-user` | Username for TiddlyWiki API requests |
|
|
98
|
+
|
|
99
|
+
## Usage
|
|
100
|
+
|
|
101
|
+
### stdio Mode (Claude Desktop)
|
|
102
|
+
|
|
103
|
+
Add to your Claude Desktop configuration (`claude_desktop_config.json`):
|
|
104
|
+
|
|
105
|
+
```json
|
|
106
|
+
{
|
|
107
|
+
"mcpServers": {
|
|
108
|
+
"tiddlywiki": {
|
|
109
|
+
"command": "node",
|
|
110
|
+
"args": ["/path/to/tiddlywiki-mcp/dist/index.js"],
|
|
111
|
+
"env": {
|
|
112
|
+
"TIDDLYWIKI_URL": "http://localhost:8080"
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
### HTTP Mode
|
|
120
|
+
|
|
121
|
+
Start the server:
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
TIDDLYWIKI_URL=http://localhost:8080 MCP_TRANSPORT=http MCP_PORT=3000 node dist/index.js
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
The server exposes:
|
|
128
|
+
- `GET /health` - Health check endpoint
|
|
129
|
+
- `POST /mcp` - MCP JSON-RPC endpoint (stateless mode)
|
|
130
|
+
|
|
131
|
+
### Example Tool Usage
|
|
132
|
+
|
|
133
|
+
**Filter search** (TiddlyWiki filter syntax):
|
|
134
|
+
```json
|
|
135
|
+
{
|
|
136
|
+
"name": "search_tiddlers",
|
|
137
|
+
"arguments": {
|
|
138
|
+
"filter": "[tag[Journal]prefix[2025-01]]",
|
|
139
|
+
"includeText": true
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
**Semantic search** (natural language):
|
|
145
|
+
```json
|
|
146
|
+
{
|
|
147
|
+
"name": "search_tiddlers",
|
|
148
|
+
"arguments": {
|
|
149
|
+
"semantic": "times I felt anxious about work",
|
|
150
|
+
"limit": 10
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
**Hybrid search** (filter + semantic reranking):
|
|
156
|
+
```json
|
|
157
|
+
{
|
|
158
|
+
"name": "search_tiddlers",
|
|
159
|
+
"arguments": {
|
|
160
|
+
"filter": "[tag[Journal]]",
|
|
161
|
+
"semantic": "productivity tips",
|
|
162
|
+
"limit": 20
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
## Development
|
|
168
|
+
|
|
169
|
+
### Setup
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
npm install
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
### Running Tests
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
npm test
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Tests run quickly (~1s) and include unit tests for all tool handlers.
|
|
182
|
+
|
|
183
|
+
### Linting
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
npm run lint # Check for issues
|
|
187
|
+
npm run format # Fix formatting
|
|
188
|
+
npm run format:check # Check formatting only
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
### Type Checking
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
npm run typecheck
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
### Pre-commit Hooks
|
|
198
|
+
|
|
199
|
+
Pre-commit hooks are configured with lefthook and run automatically:
|
|
200
|
+
|
|
201
|
+
1. Format check (Prettier)
|
|
202
|
+
2. Lint (ESLint)
|
|
203
|
+
3. Tests (Vitest)
|
|
204
|
+
4. Type check (TypeScript)
|
|
205
|
+
|
|
206
|
+
### Building
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
npm run build
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
## Architecture
|
|
213
|
+
|
|
214
|
+
```
|
|
215
|
+
src/
|
|
216
|
+
├── index.ts # Entry point, transport setup, server lifecycle
|
|
217
|
+
├── tiddlywiki-http.ts # TiddlyWiki HTTP API client
|
|
218
|
+
├── service-discovery.ts # URL resolution (direct URLs, Consul SRV, hostname:port)
|
|
219
|
+
├── filter-reference.ts # Filter syntax documentation
|
|
220
|
+
├── logger.ts # Structured logging
|
|
221
|
+
├── tools/ # MCP tool handlers
|
|
222
|
+
│ ├── types.ts # Shared types and Zod schemas
|
|
223
|
+
│ ├── search-tiddlers.ts
|
|
224
|
+
│ ├── create-tiddler.ts
|
|
225
|
+
│ ├── update-tiddler.ts
|
|
226
|
+
│ └── delete-tiddler.ts
|
|
227
|
+
└── embeddings/ # Semantic search infrastructure
|
|
228
|
+
├── database.ts # SQLite-vec database
|
|
229
|
+
├── ollama-client.ts # Ollama API client
|
|
230
|
+
└── sync-worker.ts # Background embedding sync
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
### Key Design Decisions
|
|
234
|
+
|
|
235
|
+
- **Stateless HTTP mode**: Each request gets its own Server/Transport instance to prevent request ID collisions with concurrent clients
|
|
236
|
+
- **Graceful degradation**: Semantic search is optional; the server works without Ollama
|
|
237
|
+
- **Token-aware responses**: Search results are validated against token limits with pagination suggestions
|
|
238
|
+
- **Background sync**: Embeddings are updated periodically without blocking requests
|
|
239
|
+
|
|
240
|
+
## License
|
|
241
|
+
|
|
242
|
+
MIT
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export interface EmbeddingMetadata {
|
|
2
|
+
created: string;
|
|
3
|
+
modified: string;
|
|
4
|
+
tags: string;
|
|
5
|
+
}
|
|
6
|
+
export interface SearchResult {
|
|
7
|
+
tiddler_title: string;
|
|
8
|
+
chunk_id: number;
|
|
9
|
+
chunk_text: string;
|
|
10
|
+
created: string;
|
|
11
|
+
modified: string;
|
|
12
|
+
tags: string;
|
|
13
|
+
distance: number;
|
|
14
|
+
}
|
|
15
|
+
export interface SyncStatus {
|
|
16
|
+
tiddler_title: string;
|
|
17
|
+
last_modified: string;
|
|
18
|
+
last_indexed: string;
|
|
19
|
+
total_chunks: number;
|
|
20
|
+
status: string;
|
|
21
|
+
error_message: string | null;
|
|
22
|
+
}
|
|
23
|
+
export declare class EmbeddingsDB {
|
|
24
|
+
private db;
|
|
25
|
+
private dbPath;
|
|
26
|
+
constructor(dbPath?: string);
|
|
27
|
+
private initSchema;
|
|
28
|
+
/**
|
|
29
|
+
* Migrate existing empty timestamps to sentinel value
|
|
30
|
+
* Fixes infinite re-indexing bug for tiddlers without modified timestamps
|
|
31
|
+
*/
|
|
32
|
+
private migrateEmptyTimestamps;
|
|
33
|
+
/**
|
|
34
|
+
* Add status and error_message columns to sync_status table
|
|
35
|
+
* Enables tracking of empty/error tiddlers to prevent infinite re-indexing
|
|
36
|
+
*/
|
|
37
|
+
private migrateAddStatusColumns;
|
|
38
|
+
insertEmbedding(tiddlerTitle: string, chunkId: number, embedding: number[], chunkText: string, metadata: EmbeddingMetadata): void;
|
|
39
|
+
searchSimilar(queryEmbedding: number[], limit?: number): SearchResult[];
|
|
40
|
+
updateSyncStatus(tiddlerTitle: string, lastModified: string, totalChunks: number, status?: string, errorMessage?: string | null): void;
|
|
41
|
+
getSyncStatus(tiddlerTitle: string): SyncStatus | null;
|
|
42
|
+
getAllSyncStatuses(): SyncStatus[];
|
|
43
|
+
getUnindexedTiddlers(tiddlerTitles: string[]): string[];
|
|
44
|
+
getOutdatedTiddlers(tiddlersWithModified: Array<{
|
|
45
|
+
title: string;
|
|
46
|
+
modified: string;
|
|
47
|
+
}>): string[];
|
|
48
|
+
deleteEmbeddingsForTiddler(tiddlerTitle: string): void;
|
|
49
|
+
getEmbeddingsCount(): number;
|
|
50
|
+
getIndexedTiddlersCount(): number;
|
|
51
|
+
close(): void;
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=database.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../src/embeddings/database.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,EAAE,CAAoB;IAC9B,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,GAAE,MAA4D;IAiBhF,OAAO,CAAC,UAAU;IAiClB;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAgB9B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAmB/B,eAAe,CACb,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EAAE,EACnB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,iBAAiB,GAC1B,IAAI;IA2BP,aAAa,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE,KAAK,GAAE,MAAW,GAAG,YAAY,EAAE;IAqB3E,gBAAgB,CACd,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,MAAM,GAAE,MAAkB,EAC1B,YAAY,GAAE,MAAM,GAAG,IAAW,GACjC,IAAI;IASP,aAAa,CAAC,YAAY,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI;IAUtD,kBAAkB,IAAI,UAAU,EAAE;IAUlC,oBAAoB,CAAC,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAmBvD,mBAAmB,CAAC,oBAAoB,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,MAAM,EAAE;IAc/F,0BAA0B,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IA8BtD,kBAAkB,IAAI,MAAM;IAQ5B,uBAAuB,IAAI,MAAM;IAQjC,KAAK,IAAI,IAAI;CAGd"}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
// ABOUTME: SQLite database for storing and querying vector embeddings
|
|
2
|
+
// ABOUTME: Uses sqlite-vec for efficient similarity search operations
|
|
3
|
+
import Database from 'better-sqlite3';
|
|
4
|
+
import * as sqliteVec from 'sqlite-vec';
|
|
5
|
+
export class EmbeddingsDB {
|
|
6
|
+
db;
|
|
7
|
+
dbPath;
|
|
8
|
+
constructor(dbPath = process.env.EMBEDDINGS_DB_PATH || './embeddings.db') {
|
|
9
|
+
this.dbPath = dbPath;
|
|
10
|
+
this.db = new Database(dbPath);
|
|
11
|
+
// Load sqlite-vec extension
|
|
12
|
+
sqliteVec.load(this.db);
|
|
13
|
+
// Initialize schema
|
|
14
|
+
this.initSchema();
|
|
15
|
+
// Migrate any existing empty timestamps to sentinel value
|
|
16
|
+
this.migrateEmptyTimestamps();
|
|
17
|
+
// Add status tracking columns if they don't exist
|
|
18
|
+
this.migrateAddStatusColumns();
|
|
19
|
+
}
|
|
20
|
+
initSchema() {
|
|
21
|
+
// Create vec0 virtual table for embeddings (vectors only)
|
|
22
|
+
this.db.exec(`
|
|
23
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS entry_embeddings USING vec0(
|
|
24
|
+
embedding float[768]
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
CREATE TABLE IF NOT EXISTS embedding_metadata (
|
|
28
|
+
id INTEGER PRIMARY KEY,
|
|
29
|
+
tiddler_title TEXT NOT NULL,
|
|
30
|
+
chunk_id NOT NULL,
|
|
31
|
+
chunk_text TEXT NOT NULL,
|
|
32
|
+
created TEXT,
|
|
33
|
+
modified TEXT,
|
|
34
|
+
tags TEXT
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
CREATE TABLE IF NOT EXISTS sync_status (
|
|
38
|
+
tiddler_title TEXT PRIMARY KEY,
|
|
39
|
+
last_modified TEXT NOT NULL,
|
|
40
|
+
last_indexed TEXT NOT NULL,
|
|
41
|
+
total_chunks INTEGER NOT NULL DEFAULT 1,
|
|
42
|
+
indexed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
CREATE INDEX IF NOT EXISTS idx_sync_status_modified
|
|
46
|
+
ON sync_status(last_modified);
|
|
47
|
+
|
|
48
|
+
CREATE INDEX IF NOT EXISTS idx_embedding_metadata_tiddler
|
|
49
|
+
ON embedding_metadata(tiddler_title);
|
|
50
|
+
`);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Migrate existing empty timestamps to sentinel value
|
|
54
|
+
* Fixes infinite re-indexing bug for tiddlers without modified timestamps
|
|
55
|
+
*/
|
|
56
|
+
migrateEmptyTimestamps() {
|
|
57
|
+
const MISSING_TIMESTAMP = '00000000000000000';
|
|
58
|
+
const stmt = this.db.prepare(`
|
|
59
|
+
UPDATE sync_status
|
|
60
|
+
SET last_modified = ?
|
|
61
|
+
WHERE last_modified = ''
|
|
62
|
+
`);
|
|
63
|
+
const result = stmt.run(MISSING_TIMESTAMP);
|
|
64
|
+
if (result.changes > 0) {
|
|
65
|
+
console.log(`[DB Migration] Updated ${result.changes} entries with missing timestamps`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Add status and error_message columns to sync_status table
|
|
70
|
+
* Enables tracking of empty/error tiddlers to prevent infinite re-indexing
|
|
71
|
+
*/
|
|
72
|
+
migrateAddStatusColumns() {
|
|
73
|
+
// Check if status column exists
|
|
74
|
+
const tableInfo = this.db.prepare('PRAGMA table_info(sync_status)').all();
|
|
75
|
+
const hasStatus = tableInfo.some((col) => col.name === 'status');
|
|
76
|
+
if (!hasStatus) {
|
|
77
|
+
console.log('[DB Migration] Adding status and error_message columns to sync_status table');
|
|
78
|
+
this.db.exec(`
|
|
79
|
+
ALTER TABLE sync_status ADD COLUMN status TEXT NOT NULL DEFAULT 'indexed';
|
|
80
|
+
ALTER TABLE sync_status ADD COLUMN error_message TEXT;
|
|
81
|
+
`);
|
|
82
|
+
console.log('[DB Migration] Status columns added successfully');
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
insertEmbedding(tiddlerTitle, chunkId, embedding, chunkText, metadata) {
|
|
86
|
+
const embeddingArray = new Float32Array(embedding);
|
|
87
|
+
// Insert embedding into vec0 table
|
|
88
|
+
const embStmt = this.db.prepare(`
|
|
89
|
+
INSERT INTO entry_embeddings(rowid, embedding)
|
|
90
|
+
VALUES (NULL, ?)
|
|
91
|
+
`);
|
|
92
|
+
const embResult = embStmt.run(Buffer.from(embeddingArray.buffer));
|
|
93
|
+
const rowid = embResult.lastInsertRowid;
|
|
94
|
+
// Insert metadata into regular table with same rowid
|
|
95
|
+
const metaStmt = this.db.prepare(`
|
|
96
|
+
INSERT INTO embedding_metadata(id, tiddler_title, chunk_id, chunk_text, created, modified, tags)
|
|
97
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
98
|
+
`);
|
|
99
|
+
metaStmt.run(rowid, tiddlerTitle, chunkId, chunkText, metadata.created, metadata.modified, metadata.tags);
|
|
100
|
+
}
|
|
101
|
+
searchSimilar(queryEmbedding, limit = 10) {
|
|
102
|
+
const embeddingArray = new Float32Array(queryEmbedding);
|
|
103
|
+
const stmt = this.db.prepare(`
|
|
104
|
+
SELECT
|
|
105
|
+
m.tiddler_title,
|
|
106
|
+
m.chunk_id,
|
|
107
|
+
m.chunk_text,
|
|
108
|
+
m.created,
|
|
109
|
+
m.modified,
|
|
110
|
+
m.tags,
|
|
111
|
+
e.distance
|
|
112
|
+
FROM entry_embeddings e
|
|
113
|
+
JOIN embedding_metadata m ON e.rowid = m.id
|
|
114
|
+
WHERE e.embedding MATCH ?
|
|
115
|
+
AND k = ?
|
|
116
|
+
`);
|
|
117
|
+
return stmt.all(Buffer.from(embeddingArray.buffer), limit);
|
|
118
|
+
}
|
|
119
|
+
updateSyncStatus(tiddlerTitle, lastModified, totalChunks, status = 'indexed', errorMessage = null) {
|
|
120
|
+
const stmt = this.db.prepare(`
|
|
121
|
+
INSERT OR REPLACE INTO sync_status(tiddler_title, last_modified, last_indexed, total_chunks, status, error_message)
|
|
122
|
+
VALUES (?, ?, datetime('now'), ?, ?, ?)
|
|
123
|
+
`);
|
|
124
|
+
stmt.run(tiddlerTitle, lastModified, totalChunks, status, errorMessage);
|
|
125
|
+
}
|
|
126
|
+
getSyncStatus(tiddlerTitle) {
|
|
127
|
+
const stmt = this.db.prepare(`
|
|
128
|
+
SELECT tiddler_title, last_modified, last_indexed, total_chunks, status, error_message
|
|
129
|
+
FROM sync_status
|
|
130
|
+
WHERE tiddler_title = ?
|
|
131
|
+
`);
|
|
132
|
+
return stmt.get(tiddlerTitle);
|
|
133
|
+
}
|
|
134
|
+
getAllSyncStatuses() {
|
|
135
|
+
const stmt = this.db.prepare(`
|
|
136
|
+
SELECT tiddler_title, last_modified, last_indexed, total_chunks, status, error_message
|
|
137
|
+
FROM sync_status
|
|
138
|
+
ORDER BY tiddler_title
|
|
139
|
+
`);
|
|
140
|
+
return stmt.all();
|
|
141
|
+
}
|
|
142
|
+
getUnindexedTiddlers(tiddlerTitles) {
|
|
143
|
+
if (tiddlerTitles.length === 0) {
|
|
144
|
+
return [];
|
|
145
|
+
}
|
|
146
|
+
const _placeholders = tiddlerTitles.map(() => '?').join(',');
|
|
147
|
+
const stmt = this.db.prepare(`
|
|
148
|
+
SELECT ? as title
|
|
149
|
+
FROM (SELECT ? as title ${tiddlerTitles
|
|
150
|
+
.slice(1)
|
|
151
|
+
.map(() => 'UNION ALL SELECT ?')
|
|
152
|
+
.join(' ')})
|
|
153
|
+
WHERE title NOT IN (SELECT tiddler_title FROM sync_status)
|
|
154
|
+
`);
|
|
155
|
+
const results = stmt.all(...tiddlerTitles);
|
|
156
|
+
return results.map((r) => r.title);
|
|
157
|
+
}
|
|
158
|
+
getOutdatedTiddlers(tiddlersWithModified) {
|
|
159
|
+
const outdated = [];
|
|
160
|
+
for (const tiddler of tiddlersWithModified) {
|
|
161
|
+
const syncStatus = this.getSyncStatus(tiddler.title);
|
|
162
|
+
if (!syncStatus || syncStatus.last_modified !== tiddler.modified) {
|
|
163
|
+
outdated.push(tiddler.title);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return outdated;
|
|
167
|
+
}
|
|
168
|
+
deleteEmbeddingsForTiddler(tiddlerTitle) {
|
|
169
|
+
// Get all rowids for this tiddler from metadata table
|
|
170
|
+
const getRowids = this.db.prepare(`
|
|
171
|
+
SELECT id FROM embedding_metadata WHERE tiddler_title = ?
|
|
172
|
+
`);
|
|
173
|
+
const rows = getRowids.all(tiddlerTitle);
|
|
174
|
+
// Delete from embedding_metadata
|
|
175
|
+
const deleteMetadata = this.db.prepare(`
|
|
176
|
+
DELETE FROM embedding_metadata WHERE tiddler_title = ?
|
|
177
|
+
`);
|
|
178
|
+
deleteMetadata.run(tiddlerTitle);
|
|
179
|
+
// Delete from entry_embeddings using rowids
|
|
180
|
+
if (rows.length > 0) {
|
|
181
|
+
const deleteEmbeddings = this.db.prepare(`
|
|
182
|
+
DELETE FROM entry_embeddings WHERE rowid = ?
|
|
183
|
+
`);
|
|
184
|
+
for (const row of rows) {
|
|
185
|
+
deleteEmbeddings.run(row.id);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
// Delete sync status
|
|
189
|
+
const deleteSyncStatus = this.db.prepare(`
|
|
190
|
+
DELETE FROM sync_status WHERE tiddler_title = ?
|
|
191
|
+
`);
|
|
192
|
+
deleteSyncStatus.run(tiddlerTitle);
|
|
193
|
+
}
|
|
194
|
+
getEmbeddingsCount() {
|
|
195
|
+
const stmt = this.db.prepare(`
|
|
196
|
+
SELECT COUNT(*) as count FROM entry_embeddings
|
|
197
|
+
`);
|
|
198
|
+
const result = stmt.get();
|
|
199
|
+
return result.count;
|
|
200
|
+
}
|
|
201
|
+
getIndexedTiddlersCount() {
|
|
202
|
+
const stmt = this.db.prepare(`
|
|
203
|
+
SELECT COUNT(*) as count FROM sync_status
|
|
204
|
+
`);
|
|
205
|
+
const result = stmt.get();
|
|
206
|
+
return result.count;
|
|
207
|
+
}
|
|
208
|
+
close() {
|
|
209
|
+
this.db.close();
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
//# sourceMappingURL=database.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"database.js","sourceRoot":"","sources":["../../src/embeddings/database.ts"],"names":[],"mappings":"AAAA,sEAAsE;AACtE,sEAAsE;AAEtE,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,KAAK,SAAS,MAAM,YAAY,CAAC;AA2BxC,MAAM,OAAO,YAAY;IACf,EAAE,CAAoB;IACtB,MAAM,CAAS;IAEvB,YAAY,SAAiB,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,iBAAiB;QAC9E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAE/B,4BAA4B;QAC5B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAExB,oBAAoB;QACpB,IAAI,CAAC,UAAU,EAAE,CAAC;QAElB,0DAA0D;QAC1D,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,kDAAkD;QAClD,IAAI,CAAC,uBAAuB,EAAE,CAAC;IACjC,CAAC;IAEO,UAAU;QAChB,0DAA0D;QAC1D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA4BZ,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,sBAAsB;QAC5B,MAAM,iBAAiB,GAAG,mBAAmB,CAAC;QAE9C,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI5B,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAE3C,IAAI,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,0BAA0B,MAAM,CAAC,OAAO,kCAAkC,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,uBAAuB;QAC7B,gCAAgC;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC,GAAG,EAErE,CAAC;QACH,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;QAEjE,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;YAE3F,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;OAGZ,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,OAAe,EACf,SAAmB,EACnB,SAAiB,EACjB,QAA2B;QAE3B,MAAM,cAAc,GAAG,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;QAEnD,mCAAmC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAG/B,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,SAAS,CAAC,eAAe,CAAC;QAExC,qDAAqD;QACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAGhC,CAAC,CAAC;QACH,QAAQ,CAAC,GAAG,CACV,KAAK,EACL,YAAY,EACZ,OAAO,EACP,SAAS,EACT,QAAQ,CAAC,OAAO,EAChB,QAAQ,CAAC,QAAQ,EACjB,QAAQ,CAAC,IAAI,CACd,CAAC;IACJ,CAAC;IAED,aAAa,CAAC,cAAwB,EAAE,QAAgB,EAAE;QACxD,MAAM,cAAc,GAAG,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC;QAExD,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;;;KAa5B,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,KAAK,CAAmB,CAAC;IAC/E,CAAC;IAED,gBAAgB,CACd,YAAoB,EACpB,YAAoB,EACpB,WAAmB,EACnB,SAAiB,SAAS,EAC1B,eAA8B,IAAI;QAElC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAG5B,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IAC1E,CAAC;IAED,aAAa,CAAC,YAAoB;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI5B,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,CAAsB,CAAC;IACrD,CAAC;IAED,kBAAkB;QAChB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI5B,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,GAAG,EAAkB,CAAC;IACpC,CAAC;IAED,oBAAoB,CAAC,aAAuB;QAC1C,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;gCAED,aAAa;aACpC,KAAK,CAAC,CAAC,CAAC;aACR,GAAG,CAAC,GAAG,EAAE,CAAC,oBAAoB,CAAC;aAC/B,IAAI,CAAC,GAAG,CAAC;;KAEb,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,aAAa,CAA6B,CAAC;QACvE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,mBAAmB,CAAC,oBAAgE;QAClF,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,KAAK,MAAM,OAAO,IAAI,oBAAoB,EAAE,CAAC;YAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAErD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,aAAa,KAAK,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACjE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,0BAA0B,CAAC,YAAoB;QAC7C,sDAAsD;QACtD,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAEjC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,YAAY,CAA0B,CAAC;QAElE,iCAAiC;QACjC,MAAM,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAEtC,CAAC,CAAC;QACH,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAEjC,4CAA4C;QAC5C,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,gBAAgB,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;OAExC,CAAC,CAAC;YACH,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,MAAM,gBAAgB,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAExC,CAAC,CAAC;QACH,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACrC,CAAC;IAED,kBAAkB;QAChB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAE5B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAuB,CAAC;QAC/C,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,uBAAuB;QACrB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;KAE5B,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAuB,CAAC;QAC/C,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
export declare class OllamaClient {
|
|
2
|
+
private serviceUrl;
|
|
3
|
+
private model;
|
|
4
|
+
private resolvedBaseUrl;
|
|
5
|
+
constructor(serviceUrl?: string, model?: string);
|
|
6
|
+
/**
|
|
7
|
+
* Get the resolved base URL, using service discovery if needed.
|
|
8
|
+
* Caches the result for subsequent calls.
|
|
9
|
+
*/
|
|
10
|
+
private getBaseUrl;
|
|
11
|
+
generateEmbeddings(texts: string[]): Promise<number[][]>;
|
|
12
|
+
generateEmbedding(text: string): Promise<number[]>;
|
|
13
|
+
/**
|
|
14
|
+
* Generate embedding for a document with proper task prefix.
|
|
15
|
+
* Uses "search_document:" prefix as required by nomic-embed-text for RAG.
|
|
16
|
+
*/
|
|
17
|
+
generateDocumentEmbedding(text: string): Promise<number[]>;
|
|
18
|
+
/**
|
|
19
|
+
* Generate embeddings for multiple documents with proper task prefix.
|
|
20
|
+
* Uses "search_document:" prefix as required by nomic-embed-text for RAG.
|
|
21
|
+
*/
|
|
22
|
+
generateDocumentEmbeddings(texts: string[]): Promise<number[][]>;
|
|
23
|
+
/**
|
|
24
|
+
* Generate embedding for a query with proper task prefix.
|
|
25
|
+
* Uses "search_query:" prefix as required by nomic-embed-text for RAG.
|
|
26
|
+
*/
|
|
27
|
+
generateQueryEmbedding(text: string): Promise<number[]>;
|
|
28
|
+
healthCheck(): Promise<boolean>;
|
|
29
|
+
/**
|
|
30
|
+
* Chunk text if it exceeds token limit.
|
|
31
|
+
* Splits at paragraph boundaries to maintain semantic coherence.
|
|
32
|
+
*/
|
|
33
|
+
chunkText(text: string, maxTokens?: number): string[];
|
|
34
|
+
/**
|
|
35
|
+
* Count tokens in text using gpt-tokenizer (approximation for nomic-embed-text)
|
|
36
|
+
*/
|
|
37
|
+
countTokens(text: string): number;
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=ollama-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama-client.d.ts","sourceRoot":"","sources":["../../src/embeddings/ollama-client.ts"],"names":[],"mappings":"AA+CA,qBAAa,YAAY;IACvB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,eAAe,CAAuB;gBAG5C,UAAU,GAAE,MAA2D,EACvE,KAAK,GAAE,MAAuD;IAMhE;;;OAGG;YACW,UAAU;IAQlB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAiCxD,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKxD;;;OAGG;IACG,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKhE;;;OAGG;IACG,0BAA0B,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAKtE;;;OAGG;IACG,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKvD,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAqBrC;;;OAGG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAa,GAAG,MAAM,EAAE;IAyE3D;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;CAGlC"}
|