tiddlywiki-mcp-server 1.0.0

package/dist/embeddings/ollama-client.js

@@ -0,0 +1,190 @@
+// ABOUTME: Client for Ollama API to generate text embeddings
+// ABOUTME: Handles chunking, token counting, and health checks
+import { encode } from 'gpt-tokenizer';
+import * as logger from '../logger.js';
+import { getServiceUrl } from '../service-discovery.js';
+// Timeout configuration (in milliseconds)
+const TIMEOUT_EMBEDDINGS = 120000; // 120 seconds for embeddings (can be slow)
+const TIMEOUT_HEALTH = 10000; // 10 seconds for health check
+/**
+ * Fetch with timeout using AbortController
+ */
+async function fetchWithTimeout(url, options, timeoutMs, operationName) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => {
+        logger.warn(`[Ollama] ${operationName} timed out after ${timeoutMs}ms`);
+        controller.abort();
+    }, timeoutMs);
+    try {
+        const response = await fetch(url, { ...options, signal: controller.signal });
+        return response;
+    }
+    catch (error) {
+        if (error instanceof Error && error.name === 'AbortError') {
+            throw new Error(`${operationName} timed out after ${timeoutMs}ms`);
+        }
+        throw error;
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+}
+export class OllamaClient {
+    serviceUrl;
+    model;
+    resolvedBaseUrl = null;
+    constructor(serviceUrl = process.env.OLLAMA_URL || 'http://localhost:11434', model = process.env.OLLAMA_MODEL || 'nomic-embed-text') {
+        this.serviceUrl = serviceUrl;
+        this.model = model;
+    }
+    /**
+     * Get the resolved base URL, using service discovery if needed.
+     * Caches the result for subsequent calls.
+     */
+    async getBaseUrl() {
+        if (this.resolvedBaseUrl) {
+            return this.resolvedBaseUrl;
+        }
+        this.resolvedBaseUrl = await getServiceUrl(this.serviceUrl, '');
+        return this.resolvedBaseUrl;
+    }
+    async generateEmbeddings(texts) {
+        if (texts.length === 0) {
+            return [];
+        }
+        const baseUrl = await this.getBaseUrl();
+        logger.debug(`[Ollama] generateEmbeddings: ${texts.length} text(s)`);
+        const response = await fetchWithTimeout(`${baseUrl}/api/embed`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                model: this.model,
+                input: texts,
+            }),
+        }, TIMEOUT_EMBEDDINGS, `generateEmbeddings(${texts.length} texts)`);
+        if (!response.ok) {
+            const errorText = await response.text();
+            logger.error(`[Ollama] generateEmbeddings failed: ${response.status} - ${errorText}`);
+            throw new Error(`Ollama API error (${response.status}): ${errorText}`);
+        }
+        const data = await response.json();
+        logger.debug(`[Ollama] generateEmbeddings: OK (${data.embeddings.length} embeddings)`);
+        return data.embeddings;
+    }
+    async generateEmbedding(text) {
+        const embeddings = await this.generateEmbeddings([text]);
+        return embeddings[0];
+    }
+    /**
+     * Generate embedding for a document with proper task prefix.
+     * Uses "search_document:" prefix as required by nomic-embed-text for RAG.
+     */
+    async generateDocumentEmbedding(text) {
+        const prefixedText = `search_document: ${text}`;
+        return this.generateEmbedding(prefixedText);
+    }
+    /**
+     * Generate embeddings for multiple documents with proper task prefix.
+     * Uses "search_document:" prefix as required by nomic-embed-text for RAG.
+     */
+    async generateDocumentEmbeddings(texts) {
+        const prefixedTexts = texts.map((text) => `search_document: ${text}`);
+        return this.generateEmbeddings(prefixedTexts);
+    }
+    /**
+     * Generate embedding for a query with proper task prefix.
+     * Uses "search_query:" prefix as required by nomic-embed-text for RAG.
+     */
+    async generateQueryEmbedding(text) {
+        const prefixedText = `search_query: ${text}`;
+        return this.generateEmbedding(prefixedText);
+    }
+    async healthCheck() {
+        try {
+            const baseUrl = await this.getBaseUrl();
+            logger.debug(`[Ollama] healthCheck: ${baseUrl}`);
+            const response = await fetchWithTimeout(baseUrl, { method: 'GET' }, TIMEOUT_HEALTH, 'healthCheck');
+            logger.debug(`[Ollama] healthCheck: ${response.ok ? 'OK' : 'FAILED'} (${response.status})`);
+            return response.ok;
+        }
+        catch (error) {
+            const errMsg = error instanceof Error ? error.message : String(error);
+            logger.error(`[Ollama] healthCheck failed: ${errMsg}`);
+            return false;
+        }
+    }
+    /**
+     * Chunk text if it exceeds token limit.
+     * Splits at paragraph boundaries to maintain semantic coherence.
+     */
+    chunkText(text, maxTokens = 6000) {
+        const tokens = encode(text);
+        // If text fits in one chunk, return as-is
+        if (tokens.length <= maxTokens) {
+            return [text];
+        }
+        // Split at paragraph boundaries (double newline)
+        const paragraphs = text.split(/\n\n+/);
+        const chunks = [];
+        let currentChunk = '';
+        let _currentTokens = 0;
+        for (const para of paragraphs) {
+            const paraTokens = encode(para);
+            const testChunk = currentChunk ? `${currentChunk}\n\n${para}` : para;
+            const testTokenCount = encode(testChunk).length;
+            // If adding this paragraph would exceed the limit and we have content, save current chunk
+            if (testTokenCount > maxTokens && currentChunk) {
+                chunks.push(currentChunk.trim());
+                currentChunk = para;
+                _currentTokens = paraTokens.length;
+            }
+            // If a single paragraph is too large, split it by sentences
+            else if (paraTokens.length > maxTokens) {
+                // Save any accumulated text first
+                if (currentChunk) {
+                    chunks.push(currentChunk.trim());
+                    currentChunk = '';
+                    _currentTokens = 0;
+                }
+                // Split large paragraph by sentences
+                const sentences = para.split(/[.!?]+\s+/);
+                let sentenceChunk = '';
+                for (const sentence of sentences) {
+                    const sentenceWithPunctuation = sentence + (sentence.match(/[.!?]$/) ? '' : '.');
+                    const testSentenceChunk = sentenceChunk
+                        ? `${sentenceChunk} ${sentenceWithPunctuation}`
+                        : sentenceWithPunctuation;
+                    const sentenceTokenCount = encode(testSentenceChunk).length;
+                    if (sentenceTokenCount > maxTokens && sentenceChunk) {
+                        chunks.push(sentenceChunk.trim());
+                        sentenceChunk = sentenceWithPunctuation;
+                    }
+                    else {
+                        sentenceChunk = testSentenceChunk;
+                    }
+                }
+                if (sentenceChunk) {
+                    currentChunk = sentenceChunk;
+                    _currentTokens = encode(sentenceChunk).length;
+                }
+            }
+            // Otherwise, accumulate the paragraph
+            else {
+                currentChunk = testChunk;
+                _currentTokens = testTokenCount;
+            }
+        }
+        // Add the final chunk if it exists
+        if (currentChunk) {
+            chunks.push(currentChunk.trim());
+        }
+        return chunks.filter((chunk) => chunk.length > 0);
+    }
+    /**
+     * Count tokens in text using gpt-tokenizer (approximation for nomic-embed-text)
+     */
+    countTokens(text) {
+        return encode(text).length;
+    }
+}
+//# sourceMappingURL=ollama-client.js.map

package/dist/embeddings/ollama-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ollama-client.js","sourceRoot":"","sources":["../../src/embeddings/ollama-client.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAC7D,+DAA+D;AAE/D,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACvC,OAAO,KAAK,MAAM,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAExD,0CAA0C;AAC1C,MAAM,kBAAkB,GAAG,MAAM,CAAC,CAAC,2CAA2C;AAC9E,MAAM,cAAc,GAAG,KAAK,CAAC,CAAC,8BAA8B;AAU5D;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,OAAoB,EACpB,SAAiB,EACjB,aAAqB;IAErB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;QAChC,MAAM,CAAC,IAAI,CAAC,YAAY,aAAa,oBAAoB,SAAS,IAAI,CAAC,CAAC;QACxE,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,EAAE,SAAS,CAAC,CAAC;IAEd,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7E,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CAAC,GAAG,aAAa,oBAAoB,SAAS,IAAI,CAAC,CAAC;QACrE,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,SAAS,CAAC,CAAC;IAC1B,CAAC;AACH,CAAC;AAED,MAAM,OAAO,YAAY;IACf,UAAU,CAAS;IACnB,KAAK,CAAS;IACd,eAAe,GAAkB,IAAI,CAAC;IAE9C,YACE,aAAqB,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,wBAAwB,EACvE,QAAgB,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,kBAAkB;QAE9D,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,UAAU;QACtB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAChE,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,KAAe;QACtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,gCAAgC,KAAK,CAAC,MAAM,UAAU,CAAC,CAAC;QAErE,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CACrC,GAAG,OAAO,YAAY,EACtB;YACE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,KAAK;aACb,CAAC;SACH,EACD,kBAAkB,EAClB,sBAAsB,KAAK,CAAC,MAAM,SAAS,CAC5C,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,CAAC,KAAK,CAAC,uCAAuC,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;YACtF,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,MAAM,IAAI,GAAwB,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxD,MAAM,CAAC,KAAK,CAAC,oCAAoC,IAAI,CAAC,UAAU,CAAC,MAAM,cAAc,CAAC,CAAC;QACvF,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,IAAY;QAClC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACzD,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,yBAAyB,CAAC,IAAY;QAC1C,MAAM,YAAY,GAAG,oBAAoB,IAAI,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,0BAA0B,CAAC,KAAe;QAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC;QACtE,OAAO,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,CAAC;IAChD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,sBAAsB,CAAC,IAAY;QACvC,MAAM,YAAY,GAAG,iBAAiB,IAAI,EAAE,CAAC;QAC7C,OAAO,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,MAAM,CAAC,KAAK,CAAC,yBAAyB,OAAO,EAAE,CAAC,CAAC;YAEjD,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CACrC,OAAO,EACP,EAAE,MAAM,EAAE,KAAK,EAAE,EACjB,cAAc,EACd,aAAa,CACd,CAAC;YAEF,MAAM,CAAC,KAAK,CAAC,yBAAyB,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YAC5F,OAAO,QAAQ,CAAC,EAAE,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACtE,MAAM,CAAC,KAAK,CAAC,gCAAgC,MAAM,EAAE,CAAC,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,IAAY,EAAE,YAAoB,IAAI;QAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAE5B,0CAA0C;QAC1C,IAAI,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC;QAED,iDAAiD;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,GAAG,YAAY,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YACrE,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;YAEhD,0FAA0F;YAC1F,IAAI,cAAc,GAAG,SAAS,IAAI,YAAY,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;gBACjC,YAAY,GAAG,IAAI,CAAC;gBACpB,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;YACrC,CAAC;YACD,4DAA4D;iBACvD,IAAI,UAAU,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;gBACvC,kCAAkC;gBAClC,IAAI,YAAY,EAAE,CAAC;oBACjB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;oBACjC,YAAY,GAAG,EAAE,CAAC;oBAClB,cAAc,GAAG,CAAC,CAAC;gBACrB,CAAC;gBAED,qCAAqC;gBACrC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBAC1C,IAAI,aAAa,GAAG,EAAE,CAAC;gBAEvB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACjC,MAAM,uBAAuB,GAAG,QAAQ,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBACjF,MAAM,iBAAiB,GAAG,aAAa;wBACrC,CAAC,CAAC,GAAG,aAAa,IAAI,uBAAuB,EAAE;wBAC/C,CAAC,CAAC,uBAAuB,CAAC;oBAC5B,MAAM,kBAAkB,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC;oBAE5D,IAAI,kBAAkB,GAAG,SAAS,IAAI,aAAa,EAAE,CAAC;wBACpD,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;wBAClC,aAAa,GAAG,uBAAuB,CAAC;oBAC1C,CAAC;yBAAM,CAAC;wBACN,aAAa,GAAG,iBAAiB,CAAC;oBACpC,CAAC;gBACH,CAAC;gBAED,IAAI,aAAa,EAAE,CAAC;oBAClB,YAAY,GAAG,aAAa,CAAC;oBAC7B,cAAc,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;gBAChD,CAAC;YACH,CAAC;YACD,sCAAsC;iBACjC,CAAC;gBACJ,YAAY,GAAG,SAAS,CAAC;gBACzB,cAAc,GAAG,cAAc,CAAC;YAClC,CAAC;QACH,CAAC;QAED,mCAAmC;QACnC,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,IAAY;QACtB,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC7B,CAAC;CACF"}

package/dist/embeddings/sync-worker.d.ts

@@ -0,0 +1,49 @@
+import { EmbeddingsDB } from './database.js';
+import { OllamaClient } from './ollama-client.js';
+export interface SyncWorkerConfig {
+    syncIntervalMs: number;
+    batchSize: number;
+    enabled: boolean;
+}
+export declare class SyncWorker {
+    private db;
+    private ollama;
+    private config;
+    private intervalId;
+    private isRunning;
+    private isSyncing;
+    constructor(db: EmbeddingsDB, ollama: OllamaClient, config?: Partial<SyncWorkerConfig>);
+    /**
+     * Start the sync worker
+     */
+    start(): Promise<void>;
+    /**
+     * Stop the sync worker
+     */
+    stop(): void;
+    /**
+     * Get sync worker status
+     */
+    getStatus(): {
+        running: boolean;
+        syncing: boolean;
+        indexedTiddlers: number;
+        totalEmbeddings: number;
+        syncInterval: number;
+        enabled: boolean;
+    };
+    /**
+     * Run a sync cycle
+     */
+    private runSync;
+    /**
+     * Index a single tiddler
+     * Returns status: 'indexed', 'empty', or 'error'
+     */
+    private indexTiddler;
+    /**
+     * Force a sync cycle (for manual triggering)
+     */
+    forceSync(): Promise<void>;
+}
+//# sourceMappingURL=sync-worker.d.ts.map

package/dist/embeddings/sync-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync-worker.d.ts","sourceRoot":"","sources":["../../src/embeddings/sync-worker.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAQlD,MAAM,WAAW,gBAAgB;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,EAAE,CAAe;IACzB,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,SAAS,CAAkB;gBAEvB,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAE,OAAO,CAAC,gBAAgB,CAAM;IAU1F;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAmC5B;;OAEG;IACH,IAAI,IAAI,IAAI;IAgBZ;;OAEG;IACH,SAAS;;;;;;;;IAWT;;OAEG;YACW,OAAO;IAyHrB;;;OAGG;YACW,YAAY;IA4F1B;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;CAGjC"}

package/dist/embeddings/sync-worker.js

@@ -0,0 +1,244 @@
+// ABOUTME: Background worker that syncs tiddler content to vector embeddings
+// ABOUTME: Periodically checks for changes and updates the embeddings database
+import { queryTiddlers } from '../tiddlywiki-http.js';
+import * as logger from '../logger.js';
+// Sentinel value for tiddlers without modified timestamp
+// Prevents infinite re-indexing loop for tiddlers that don't track modifications
+const MISSING_TIMESTAMP = '00000000000000000';
+export class SyncWorker {
+    db;
+    ollama;
+    config;
+    intervalId = null;
+    isRunning = false;
+    isSyncing = false;
+    constructor(db, ollama, config = {}) {
+        this.db = db;
+        this.ollama = ollama;
+        this.config = {
+            syncIntervalMs: config.syncIntervalMs || 5 * 60 * 1000, // 5 minutes
+            batchSize: config.batchSize || 5,
+            enabled: config.enabled ?? true,
+        };
+    }
+    /**
+     * Start the sync worker
+     */
+    async start() {
+        if (!this.config.enabled) {
+            logger.log('[SyncWorker] Disabled, not starting');
+            return;
+        }
+        if (this.isRunning) {
+            logger.log('[SyncWorker] Already running');
+            return;
+        }
+        logger.log('[SyncWorker] Starting...');
+        this.isRunning = true;
+        // Check Ollama health
+        const healthy = await this.ollama.healthCheck();
+        if (!healthy) {
+            logger.error('[SyncWorker] Ollama is not healthy, will retry on next sync');
+        }
+        // Run initial sync (non-blocking)
+        this.runSync().catch((error) => {
+            logger.error('[SyncWorker] Initial sync error:', error);
+        });
+        // Schedule periodic syncs
+        this.intervalId = setInterval(() => {
+            this.runSync().catch((error) => {
+                logger.error('[SyncWorker] Periodic sync error:', error);
+            });
+        }, this.config.syncIntervalMs);
+        logger.log(`[SyncWorker] Started with ${this.config.syncIntervalMs / 1000}s interval`);
+    }
+    /**
+     * Stop the sync worker
+     */
+    stop() {
+        if (!this.isRunning) {
+            return;
+        }
+        logger.log('[SyncWorker] Stopping...');
+        this.isRunning = false;
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+            this.intervalId = null;
+        }
+        logger.log('[SyncWorker] Stopped');
+    }
+    /**
+     * Get sync worker status
+     */
+    getStatus() {
+        return {
+            running: this.isRunning,
+            syncing: this.isSyncing,
+            indexedTiddlers: this.db.getIndexedTiddlersCount(),
+            totalEmbeddings: this.db.getEmbeddingsCount(),
+            syncInterval: this.config.syncIntervalMs / 1000,
+            enabled: this.config.enabled,
+        };
+    }
+    /**
+     * Run a sync cycle
+     */
+    async runSync() {
+        if (this.isSyncing) {
+            logger.debug('[SyncWorker] Sync already in progress, skipping');
+            return;
+        }
+        this.isSyncing = true;
+        const startTime = Date.now();
+        try {
+            logger.debug('[SyncWorker] Starting sync cycle...');
+            // Check Ollama health
+            const healthy = await this.ollama.healthCheck();
+            if (!healthy) {
+                logger.error('[SyncWorker] Ollama is not available, skipping sync');
+                return;
+            }
+            // Get all tiddlers (excluding system tiddlers)
+            const allTiddlers = await queryTiddlers('[!is[system]sort[title]]', false // Only need metadata for comparison
+            );
+            // Filter out filesystem paths (un-imported .tid files)
+            // Real tiddler titles don't contain full filesystem paths
+            const validTiddlers = allTiddlers.filter((t) => !t.title.startsWith('/') && !t.title.includes('.tid'));
+            logger.debug(`[SyncWorker] Found ${validTiddlers.length} total tiddlers (${allTiddlers.length - validTiddlers.length} filesystem paths filtered)`);
+            // Determine which tiddlers need indexing
+            const tiddlersToIndex = [];
+            const RETRY_ERROR_AFTER_MS = 24 * 60 * 60 * 1000; // 24 hours
+            for (const tiddler of validTiddlers) {
+                const syncStatus = this.db.getSyncStatus(tiddler.title);
+                // Normalize undefined modified to sentinel value for consistent comparison
+                const tiddlerModified = tiddler.modified || MISSING_TIMESTAMP;
+                // Decision logic for whether to index this tiddler
+                let shouldIndex = false;
+                let reason = '';
+                if (!syncStatus) {
+                    // Never indexed before
+                    shouldIndex = true;
+                    reason = 'NEW tiddler';
+                }
+                else if (syncStatus.last_modified !== tiddlerModified) {
+                    // Modified timestamp changed - always re-index
+                    shouldIndex = true;
+                    reason = `modified timestamp changed (stored="${syncStatus.last_modified}" vs current="${tiddlerModified}")`;
+                }
+                else if (syncStatus.status === 'error') {
+                    // For error status, retry after 24 hours
+                    const lastIndexedTime = new Date(syncStatus.last_indexed).getTime();
+                    const now = Date.now();
+                    const timeSinceError = now - lastIndexedTime;
+                    if (timeSinceError > RETRY_ERROR_AFTER_MS) {
+                        shouldIndex = true;
+                        reason = `retrying after error (${(timeSinceError / (1000 * 60 * 60)).toFixed(1)}h since last attempt)`;
+                    }
+                }
+                // Note: Empty tiddlers (status='empty') are only re-indexed if modified timestamp changes (handled above)
+                // Successful tiddlers (status='indexed') are only re-indexed if modified timestamp changes (handled above)
+                if (shouldIndex) {
+                    logger.debug(`[SyncWorker] Queuing ${tiddler.title}: ${reason}`);
+                    tiddlersToIndex.push(tiddler);
+                }
+            }
+            if (tiddlersToIndex.length === 0) {
+                logger.debug('[SyncWorker] No tiddlers need indexing');
+                return;
+            }
+            logger.debug(`[SyncWorker] Indexing ${tiddlersToIndex.length} tiddlers...`);
+            // Process tiddlers in batches and track results
+            const stats = {
+                indexed: 0,
+                empty: 0,
+                error: 0,
+            };
+            for (let i = 0; i < tiddlersToIndex.length; i += this.config.batchSize) {
+                const batch = tiddlersToIndex.slice(i, i + this.config.batchSize);
+                const results = await Promise.all(batch.map((tiddler) => this.indexTiddler(tiddler)));
+                // Count results by status
+                for (const status of results) {
+                    if (status === 'indexed')
+                        stats.indexed++;
+                    else if (status === 'empty')
+                        stats.empty++;
+                    else if (status === 'error')
+                        stats.error++;
+                }
+                const processed = i + batch.length;
+                logger.debug(`[SyncWorker] Progress: ${processed}/${tiddlersToIndex.length} tiddlers processed`);
+            }
+            const duration = ((Date.now() - startTime) / 1000).toFixed(1);
+            logger.log(`[SyncWorker] Sync cycle completed in ${duration}s. Results: ${stats.indexed} indexed, ${stats.empty} empty, ${stats.error} errors`);
+        }
+        catch (error) {
+            logger.error('[SyncWorker] Sync cycle error:', error);
+            throw error;
+        }
+        finally {
+            this.isSyncing = false;
+        }
+    }
+    /**
+     * Index a single tiddler
+     * Returns status: 'indexed', 'empty', or 'error'
+     */
+    async indexTiddler(tiddlerMetadata) {
+        try {
+            // Fetch full tiddler content
+            const [fullTiddler] = await queryTiddlers(`[title[${tiddlerMetadata.title}]]`, true // Include text
+            );
+            if (!fullTiddler || !fullTiddler.text) {
+                logger.warn(`[SyncWorker] Tiddler ${tiddlerMetadata.title} has no text, marking as empty to avoid re-processing`);
+                // Mark as empty to prevent re-indexing on every sync cycle
+                this.db.updateSyncStatus(tiddlerMetadata.title, tiddlerMetadata.modified || MISSING_TIMESTAMP, 0, // No chunks
+                'empty', null);
+                return 'empty';
+            }
+            // Delete existing embeddings for this tiddler
+            this.db.deleteEmbeddingsForTiddler(fullTiddler.title);
+            // Chunk the text if needed
+            const chunks = this.ollama.chunkText(fullTiddler.text);
+            try {
+                // Generate embeddings for all chunks with search_document prefix
+                const embeddings = await this.ollama.generateDocumentEmbeddings(chunks);
+                // Store embeddings
+                for (let i = 0; i < chunks.length; i++) {
+                    this.db.insertEmbedding(fullTiddler.title, i, // chunk_id
+                    embeddings[i], chunks[i], {
+                        created: fullTiddler.created || '',
+                        modified: fullTiddler.modified || '',
+                        tags: fullTiddler.tags || '',
+                    });
+                }
+                // Update sync status with success
+                this.db.updateSyncStatus(fullTiddler.title, fullTiddler.modified || MISSING_TIMESTAMP, chunks.length, 'indexed', null);
+                const tokenCount = this.ollama.countTokens(fullTiddler.text);
+                logger.debug(`[SyncWorker] Indexed ${fullTiddler.title} (${tokenCount} tokens, ${chunks.length} chunks)`);
+                return 'indexed';
+            }
+            catch (embeddingError) {
+                // Handle Ollama API errors (e.g., context length exceeded)
+                const errorMessage = embeddingError instanceof Error ? embeddingError.message : String(embeddingError);
+                logger.error(`[SyncWorker] Failed to generate embeddings for ${fullTiddler.title}: ${errorMessage}`);
+                // Mark as error to prevent infinite retry loop
+                this.db.updateSyncStatus(fullTiddler.title, fullTiddler.modified || MISSING_TIMESTAMP, 0, // No chunks successfully stored
+                'error', errorMessage);
+                // Don't re-throw - continue processing other tiddlers
+                return 'error';
+            }
+        }
+        catch (error) {
+            logger.error(`[SyncWorker] Error indexing tiddler ${tiddlerMetadata.title}:`, error);
+            // Return error status instead of throwing to allow batch processing to continue
+            return 'error';
+        }
+    }
+    /**
+     * Force a sync cycle (for manual triggering)
+     */
+    async forceSync() {
+        await this.runSync();
+    }
+}
+//# sourceMappingURL=sync-worker.js.map

package/dist/embeddings/sync-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync-worker.js","sourceRoot":"","sources":["../../src/embeddings/sync-worker.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,+EAA+E;AAI/E,OAAO,EAAE,aAAa,EAAW,MAAM,uBAAuB,CAAC;AAC/D,OAAO,KAAK,MAAM,MAAM,cAAc,CAAC;AAEvC,yDAAyD;AACzD,iFAAiF;AACjF,MAAM,iBAAiB,GAAG,mBAAmB,CAAC;AAQ9C,MAAM,OAAO,UAAU;IACb,EAAE,CAAe;IACjB,MAAM,CAAe;IACrB,MAAM,CAAmB;IACzB,UAAU,GAA0B,IAAI,CAAC;IACzC,SAAS,GAAY,KAAK,CAAC;IAC3B,SAAS,GAAY,KAAK,CAAC;IAEnC,YAAY,EAAgB,EAAE,MAAoB,EAAE,SAAoC,EAAE;QACxF,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG;YACZ,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,EAAE,YAAY;YACpE,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,CAAC;YAChC,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;SAChC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;YAClD,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;YAC3C,OAAO;QACT,CAAC;QAED,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,sBAAsB;QACtB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAChD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;QAC9E,CAAC;QAED,kCAAkC;QAClC,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YAC7B,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,0BAA0B;QAC1B,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE;YACjC,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC7B,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;YAC3D,CAAC,CAAC,CAAC;QACL,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAE/B,MAAM,CAAC,GAAG,CAAC,6BAA6B,IAAI,CAAC,MAAM,CAAC,cAAc,GAAG,IAAI,YAAY,CAAC,CAAC;IACzF,CAAC;IAED;;OAEG;IACH,IAAI;QACF,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,OAAO;QACT,CAAC;QAED,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAEvB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;QAED,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,SAAS;YACvB,OAAO,EAAE,IAAI,CAAC,SAAS;YACvB,eAAe,EAAE,IAAI,CAAC,EAAE,CAAC,uBAAuB,EAAE;YAClD,eAAe,EAAE,IAAI,CAAC,EAAE,CAAC,kBAAkB,EAAE;YAC7C,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,GAAG,IAAI;YAC/C,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;SAC7B,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,OAAO;QACnB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;YAChE,OAAO;QACT,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;YAEpD,sBAAsB;YACtB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAChD,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;gBACpE,OAAO;YACT,CAAC;YAED,+CAA+C;YAC/C,MAAM,WAAW,GAAG,MAAM,aAAa,CACrC,0BAA0B,EAC1B,KAAK,CAAC,oCAAoC;aAC3C,CAAC;YAEF,uDAAuD;YACvD,0DAA0D;YAC1D,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC7D,CAAC;YAEF,MAAM,CAAC,KAAK,CACV,sBAAsB,aAAa,CAAC,MAAM,oBAAoB,WAAW,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,6BAA6B,CACrI,CAAC;YAEF,yCAAyC;YACzC,MAAM,eAAe,GAAc,EAAE,CAAC;YACtC,MAAM,oBAAoB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW;YAE7D,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;gBACpC,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAExD,2EAA2E;gBAC3E,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAC;gBAE9D,mDAAmD;gBACnD,IAAI,WAAW,GAAG,KAAK,CAAC;gBACxB,IAAI,MAAM,GAAG,EAAE,CAAC;gBAEhB,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,uBAAuB;oBACvB,WAAW,GAAG,IAAI,CAAC;oBACnB,MAAM,GAAG,aAAa,CAAC;gBACzB,CAAC;qBAAM,IAAI,UAAU,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;oBACxD,+CAA+C;oBAC/C,WAAW,GAAG,IAAI,CAAC;oBACnB,MAAM,GAAG,uCAAuC,UAAU,CAAC,aAAa,iBAAiB,eAAe,IAAI,CAAC;gBAC/G,CAAC;qBAAM,IAAI,UAAU,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;oBACzC,yCAAyC;oBACzC,MAAM,eAAe,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;oBACpE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBACvB,MAAM,cAAc,GAAG,GAAG,GAAG,eAAe,CAAC;oBAE7C,IAAI,cAAc,GAAG,oBAAoB,EAAE,CAAC;wBAC1C,WAAW,GAAG,IAAI,CAAC;wBACnB,MAAM,GAAG,yBAAyB,CAAC,cAAc,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB,CAAC;oBAC1G,CAAC;gBACH,CAAC;gBACD,0GAA0G;gBAC1G,2GAA2G;gBAE3G,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,CAAC,KAAK,CAAC,wBAAwB,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC;oBACjE,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC;YAED,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;gBACvD,OAAO;YACT,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,yBAAyB,eAAe,CAAC,MAAM,cAAc,CAAC,CAAC;YAE5E,gDAAgD;YAChD,MAAM,KAAK,GAAG;gBACZ,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,CAAC;gBACR,KAAK,EAAE,CAAC;aACT,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBACvE,MAAM,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBAElE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAEtF,0BAA0B;gBAC1B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,MAAM,KAAK,SAAS;wBAAE,KAAK,CAAC,OAAO,EAAE,CAAC;yBACrC,IAAI,MAAM,KAAK,OAAO;wBAAE,KAAK,CAAC,KAAK,EAAE,CAAC;yBACtC,IAAI,MAAM,KAAK,OAAO;wBAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC7C,CAAC;gBAED,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;gBACnC,MAAM,CAAC,KAAK,CACV,0BAA0B,SAAS,IAAI,eAAe,CAAC,MAAM,qBAAqB,CACnF,CAAC;YACJ,CAAC;YAED,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC9D,MAAM,CAAC,GAAG,CACR,wCAAwC,QAAQ,eAAe,KAAK,CAAC,OAAO,aAAa,KAAK,CAAC,KAAK,WAAW,KAAK,CAAC,KAAK,SAAS,CACpI,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY,CAAC,eAAwB;QACjD,IAAI,CAAC;YACH,6BAA6B;YAC7B,MAAM,CAAC,WAAW,CAAC,GAAG,MAAM,aAAa,CACvC,UAAU,eAAe,CAAC,KAAK,IAAI,EACnC,IAAI,CAAC,eAAe;aACrB,CAAC;YAEF,IAAI,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;gBACtC,MAAM,CAAC,IAAI,CACT,wBAAwB,eAAe,CAAC,KAAK,uDAAuD,CACrG,CAAC;gBAEF,2DAA2D;gBAC3D,IAAI,CAAC,EAAE,CAAC,gBAAgB,CACtB,eAAe,CAAC,KAAK,EACrB,eAAe,CAAC,QAAQ,IAAI,iBAAiB,EAC7C,CAAC,EAAE,YAAY;gBACf,OAAO,EACP,IAAI,CACL,CAAC;gBAEF,OAAO,OAAO,CAAC;YACjB,CAAC;YAED,8CAA8C;YAC9C,IAAI,CAAC,EAAE,CAAC,0BAA0B,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAEtD,2BAA2B;YAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAEvD,IAAI,CAAC;gBACH,iEAAiE;gBACjE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC;gBAExE,mBAAmB;gBACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvC,IAAI,CAAC,EAAE,CAAC,eAAe,CACrB,WAAW,CAAC,KAAK,EACjB,CAAC,EAAE,WAAW;oBACd,UAAU,CAAC,CAAC,CAAC,EACb,MAAM,CAAC,CAAC,CAAC,EACT;wBACE,OAAO,EAAE,WAAW,CAAC,OAAO,IAAI,EAAE;wBAClC,QAAQ,EAAE,WAAW,CAAC,QAAQ,IAAI,EAAE;wBACpC,IAAI,EAAE,WAAW,CAAC,IAAI,IAAI,EAAE;qBAC7B,CACF,CAAC;gBACJ,CAAC;gBAED,kCAAkC;gBAClC,IAAI,CAAC,EAAE,CAAC,gBAAgB,CACtB,WAAW,CAAC,KAAK,EACjB,WAAW,CAAC,QAAQ,IAAI,iBAAiB,EACzC,MAAM,CAAC,MAAM,EACb,SAAS,EACT,IAAI,CACL,CAAC;gBAEF,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC7D,MAAM,CAAC,KAAK,CACV,wBAAwB,WAAW,CAAC,KAAK,KAAK,UAAU,YAAY,MAAM,CAAC,MAAM,UAAU,CAC5F,CAAC;gBAEF,OAAO,SAAS,CAAC;YACnB,CAAC;YAAC,OAAO,cAAc,EAAE,CAAC;gBACxB,2DAA2D;gBAC3D,MAAM,YAAY,GAChB,cAAc,YAAY,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBACpF,MAAM,CAAC,KAAK,CACV,kDAAkD,WAAW,CAAC,KAAK,KAAK,YAAY,EAAE,CACvF,CAAC;gBAEF,+CAA+C;gBAC/C,IAAI,CAAC,EAAE,CAAC,gBAAgB,CACtB,WAAW,CAAC,KAAK,EACjB,WAAW,CAAC,QAAQ,IAAI,iBAAiB,EACzC,CAAC,EAAE,gCAAgC;gBACnC,OAAO,EACP,YAAY,CACb,CAAC;gBAEF,sDAAsD;gBACtD,OAAO,OAAO,CAAC;YACjB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,uCAAuC,eAAe,CAAC,KAAK,GAAG,EAAE,KAAK,CAAC,CAAC;YACrF,gFAAgF;YAChF,OAAO,OAAO,CAAC;QACjB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS;QACb,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;IACvB,CAAC;CACF"}

package/dist/filter-reference.d.ts

@@ -0,0 +1,8 @@
+export declare const FILTER_REFERENCE = "# TiddlyWiki Filter Reference\n\n## Syntax Basics\n\nFilters are composed of **runs** enclosed in square brackets: `[operator[parameter]]`\nMultiple operators can be chained within a run.\nMultiple runs can be combined with different logic.\n\n## Logic Operators\n\n- **AND (implicit)**: Space between runs \u2192 `[run1] [run2]` (intersection)\n- **OR**: Comma + prefix \u2192 `[run1],[run2]` (union)\n- **NOT**: ! prefix \u2192 `[all[tiddlers]!tag[Exclude]]` (exclusion)\n\n## Selection Operators\n\n### all[]\nSelect tiddlers by type:\n- `[all[tiddlers]]` - all non-shadow tiddlers\n- `[all[shadows]]` - all shadow tiddlers\n- `[all[]]]` - all tiddlers (including shadows)\n\n### tag[]\nSelect by tag:\n- `[tag[Journal]]` - tiddlers tagged \"Journal\"\n- `[tag[OYS]tag[Planning]]` - tiddlers with BOTH tags (AND)\n- `[tag[OYS]],[tag[Planning]]` - tiddlers with EITHER tag (OR)\n- `[all[tiddlers]!tag[System]]` - exclude tagged tiddlers\n\n### prefix[] / suffix[]\nString matching on title:\n- `[prefix[2025-11]]` - titles starting with \"2025-11\"\n- `[suffix[.md]]` - titles ending with \".md\"\n- `[prefix[OYS ]]` - OYS posts (note the space!)\n\n### search[]\nFull-text search:\n- `[search[keyword]]` - search title and text\n- `[search:title[keyword]]` - search title only\n- `[search:text[keyword]]` - search text only\n- `[search:tags[keyword]]` - search tags\n- `[search[In\u00EAs]]` - find all mentions\n\n### field matching\nMatch by field value:\n- `[field:fieldname[value]]` - exact match\n- `[type[text/markdown]]` - tiddlers of specific type\n- `[creator[username]]` - created by user\n\n## Date Operators\n\n### days[]\nRelative date filtering:\n- `[!days:created[-7]]` - created in last 7 days\n- `[!days:modified[-30]]` - modified in last 30 days\n- `[days:created[1]]` - created tomorrow or before (future + past)\n- `[days:modified[0]]` - modified today only\n\n**Parameter D**: number of days from today\n- D=0: today only\n- D=1: tomorrow and everything before\n- D=-1: yesterday and everything after\n- D=-7: last week and everything after\n\n### sameday[]\nMatch specific date:\n- `[sameday:created[20251112]]` - created on Nov 12, 2025\n- `[sameday:modified[20251112]]` - modified on Nov 12, 2025\n\n**Date format**: YYYYMMDD or YYYYMMDDHHmmSSsss\n\n## Sorting Operators\n\n### sort[] / !sort[]\nSort results:\n- `[tag[Journal]sort[title]]` - alphabetical by title\n- `[tag[Journal]!sort[created]]` - newest first (! = descending)\n- `[tag[Journal]!sort[modified]]` - most recently modified first\n- `[all[tiddlers]nsort[title]]` - natural sort (handles numbers)\n\n### limit[]\nLimit results:\n- `[tag[Journal]!sort[modified]limit[10]]` - 10 most recent\n- `[tag[Journal]limit[5]]` - first 5 matches\n\n## Practical Examples\n\n### Recent journal entries\n```\n[tag[Journal]!days:modified[-7]!sort[modified]]\n```\nJournal entries modified in last 7 days, newest first\n\n### November 2025 entries\n```\n[tag[Journal]prefix[2025-11]sort[title]]\n```\nAll November 2025 journal entries, chronological\n\n### Search with context\n```\n[tag[Journal]search[In\u00EAs]!sort[modified]limit[10]]\n```\n10 most recent journal entries mentioning In\u00EAs\n\n### OYS posts\n```\n[tag[OYS]!sort[created]]\n```\nAll OYS posts, newest first\n\n### Entries without a tag\n```\n[tag[Journal]!tag[agent-generated]]\n```\nHuman-written journal entries only\n\n### Date range (last 30 days)\n```\n[tag[Journal]!days:created[-30]!sort[created]]\n```\nJournal entries created in last 30 days\n\n### Complex query\n```\n[tag[Journal]!days:modified[-14]search[exercise]!tag[Draft]!sort[modified]limit[5]]\n```\nLast 5 non-draft journal entries mentioning \"exercise\" from the past 2 weeks\n\n## Tips\n\n1. **Testing**: Test filters incrementally. Start with broad selection, then add constraints.\n2. **Performance**: More specific filters (prefix, tag) are faster than full-text search.\n3. **Case sensitivity**: Search is case-insensitive by default.\n4. **Timestamps**: TiddlyWiki uses milliseconds since epoch for created/modified fields.\n5. **Empty results**: Invalid syntax may return empty results rather than errors.\n\n## Common Patterns\n\n| Goal | Filter |\n|------|--------|\n| Last 7 days | `[!days:modified[-7]]` |\n| This month | `[prefix[2025-11]]` |\n| Last 10 entries | `[tag[Journal]!sort[modified]limit[10]]` |\n| Find keyword | `[search[keyword]]` |\n| Exclude system | `[all[tiddlers]!prefix[$:]]` |\n| By date | `[sameday:created[20251112]]` |\n";
+/**
+ * Get the filter reference as a structured object
+ */
+export declare function getFilterReference(): {
+    content: string;
+};
+//# sourceMappingURL=filter-reference.d.ts.map

package/dist/filter-reference.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filter-reference.d.ts","sourceRoot":"","sources":["../src/filter-reference.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,gBAAgB,i4IAmJ5B,CAAC;AAEF;;GAEG;AACH,wBAAgB,kBAAkB,IAAI;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,CAIxD"}