threadshelf 1.2.0

package/dist/src/validation.js

@@ -0,0 +1,250 @@
+export class ValidationError extends Error {
+    field;
+    constructor(message, { field } = {}) {
+        super(message);
+        this.name = 'ValidationError';
+        this.field = field;
+    }
+}
+const COLLECTION_PATTERN = /^[a-z0-9][a-z0-9_-]{0,62}$/;
+const RESERVED_COLLECTIONS = new Set(['all']);
+const PROTECTED_COLLECTIONS = new Set(['all', 'chunks', 'threadshelf_conversations']);
+export const MAX_QUERY_CHARS = 4000;
+export const MAX_STRING_CHARS = 1_000_000;
+export const MAX_SEARCH_N = 50;
+export const normalizeCollectionName = (value, { field = 'collection' } = {}) => {
+    if (value === undefined || value === null) {
+        throw new ValidationError(`Missing ${field}`, { field });
+    }
+    const slug = String(value)
+        .trim()
+        .replace(/[^a-z0-9_-]/gi, '_')
+        .replace(/_+/g, '_')
+        .replace(/^[_-]+|[_-]+$/g, '')
+        .toLowerCase()
+        .slice(0, 63);
+    if (!slug) {
+        throw new ValidationError(`Invalid ${field}: empty after normalization`, { field });
+    }
+    if (!COLLECTION_PATTERN.test(slug)) {
+        throw new ValidationError(`Invalid ${field}: must match ${COLLECTION_PATTERN}`, { field });
+    }
+    if (RESERVED_COLLECTIONS.has(slug)) {
+        throw new ValidationError(`Reserved ${field}: ${slug}`, { field });
+    }
+    return slug;
+};
+export const normalizeCollectionSelector = (value, { defaultValue = 'chunks', field = 'collection' } = {}) => {
+    if (value === undefined || value === null || value === '')
+        return defaultValue;
+    const raw = String(value).trim().toLowerCase();
+    if (raw === 'all')
+        return 'all';
+    return normalizeCollectionName(raw, { field });
+};
+export const assertDeletableCollection = (name) => {
+    const normalized = normalizeCollectionName(name);
+    if (PROTECTED_COLLECTIONS.has(normalized)) {
+        throw new ValidationError(`Collection cannot be deleted: ${normalized}`, {
+            field: 'collection',
+        });
+    }
+    return normalized;
+};
+export const assertClearableCollection = (name) => {
+    const normalized = normalizeCollectionName(name);
+    if (normalized === 'all' || normalized === 'threadshelf_conversations') {
+        throw new ValidationError(`Collection cannot be cleared: ${normalized}`, {
+            field: 'collection',
+        });
+    }
+    return normalized;
+};
+export const normalizeQuery = (value, { field = 'q', maxLength = MAX_QUERY_CHARS } = {}) => {
+    if (value === undefined || value === null) {
+        throw new ValidationError(`Missing ${field}`, { field });
+    }
+    if (typeof value !== 'string') {
+        throw new ValidationError(`Invalid ${field}: must be a string`, { field });
+    }
+    const trimmed = value.trim();
+    if (!trimmed) {
+        throw new ValidationError(`Invalid ${field}: empty`, { field });
+    }
+    if (trimmed.length > maxLength) {
+        throw new ValidationError(`Invalid ${field}: max ${maxLength} chars`, { field });
+    }
+    return trimmed;
+};
+export const normalizeCount = (value, { defaultValue, min = 1, max = MAX_SEARCH_N, field = 'n', } = {}) => {
+    if (value === undefined || value === null || value === '') {
+        if (defaultValue === undefined) {
+            throw new ValidationError(`Missing ${field}`, { field });
+        }
+        return defaultValue;
+    }
+    const num = typeof value === 'number' ? value : Number(value);
+    if (!Number.isFinite(num)) {
+        throw new ValidationError(`Invalid ${field}: not a number`, { field });
+    }
+    if (!Number.isInteger(num)) {
+        throw new ValidationError(`Invalid ${field}: must be an integer`, { field });
+    }
+    if (num < min) {
+        throw new ValidationError(`Invalid ${field}: minimum ${min}`, { field });
+    }
+    if (num > max) {
+        throw new ValidationError(`Invalid ${field}: maximum ${max}`, { field });
+    }
+    return num;
+};
+const ALLOWED_ROLES = new Set(['user', 'thinking', 'ai']);
+export const normalizeRoles = (value, { field = 'roles' } = {}) => {
+    if (value === undefined || value === null || value === '')
+        return null;
+    let list;
+    if (Array.isArray(value)) {
+        list = value;
+    }
+    else if (typeof value === 'string') {
+        list = value.split(',');
+    }
+    else {
+        throw new ValidationError(`Invalid ${field}: must be string or array`, { field });
+    }
+    const normalized = list.map((entry) => String(entry).trim().toLowerCase()).filter(Boolean);
+    if (normalized.length === 0)
+        return null;
+    for (const role of normalized) {
+        if (!ALLOWED_ROLES.has(role)) {
+            throw new ValidationError(`Invalid ${field}: unknown role "${role}"`, { field });
+        }
+    }
+    return [...new Set(normalized)];
+};
+export const normalizeOptionalString = (value, { field = 'value', maxLength = 200 } = {}) => {
+    if (value === undefined || value === null || value === '')
+        return undefined;
+    if (typeof value !== 'string') {
+        throw new ValidationError(`Invalid ${field}: must be a string`, { field });
+    }
+    const trimmed = value.trim();
+    if (!trimmed)
+        return undefined;
+    if (trimmed.length > maxLength) {
+        throw new ValidationError(`Invalid ${field}: max ${maxLength} chars`, { field });
+    }
+    return trimmed;
+};
+export const normalizeOptionalIsoDate = (value, { field = 'date', endOfDay = false } = {}) => {
+    if (value === undefined || value === null || value === '')
+        return undefined;
+    if (typeof value !== 'string') {
+        throw new ValidationError(`Invalid ${field}: must be a string`, { field });
+    }
+    const trimmed = value.trim();
+    if (!trimmed)
+        return undefined;
+    const normalized = /^\d{4}-\d{2}-\d{2}$/.test(trimmed)
+        ? `${trimmed}T${endOfDay ? '23:59:59.999' : '00:00:00.000'}Z`
+        : trimmed;
+    const timestamp = Date.parse(normalized);
+    if (Number.isNaN(timestamp)) {
+        throw new ValidationError(`Invalid ${field}: must be an ISO-like date string`, { field });
+    }
+    return new Date(timestamp).toISOString();
+};
+export const normalizeDateRange = (fromValue, toValue) => {
+    const from = normalizeOptionalIsoDate(fromValue, { field: 'from' });
+    const to = normalizeOptionalIsoDate(toValue, { field: 'to', endOfDay: true });
+    if (from && to && from > to) {
+        throw new ValidationError('Invalid date range: from must be before to', { field: 'from' });
+    }
+    return { from, to };
+};
+export const normalizeSearchMode = (value, { field = 'mode' } = {}) => {
+    if (value === undefined || value === null || value === '')
+        return 'semantic';
+    const mode = String(value).trim().toLowerCase();
+    if (mode === 'semantic' || mode === 'keyword')
+        return mode;
+    throw new ValidationError(`Invalid ${field}: expected "semantic" or "keyword"`, { field });
+};
+export const normalizeBoolean = (value, { defaultValue = false, field = 'flag' } = {}) => {
+    if (value === undefined || value === null || value === '')
+        return defaultValue;
+    if (typeof value === 'boolean')
+        return value;
+    if (typeof value === 'number')
+        return value !== 0;
+    if (typeof value === 'string') {
+        const v = value.trim().toLowerCase();
+        if (['1', 'true', 'yes', 'on'].includes(v))
+            return true;
+        if (['0', 'false', 'no', 'off'].includes(v))
+            return false;
+    }
+    throw new ValidationError(`Invalid ${field}: expected boolean`, { field });
+};
+export const validateTurn = (turn, { index } = {}) => {
+    if (!turn || typeof turn !== 'object' || Array.isArray(turn)) {
+        throw new ValidationError(`Invalid turn at index ${index}: must be an object`);
+    }
+    const t = turn;
+    const roleKeys = ['user', 'thinking', 'ai'].filter((key) => t[key] !== undefined);
+    if (roleKeys.length !== 1) {
+        throw new ValidationError(`Invalid turn at index ${index}: must have exactly one of user/thinking/ai (had ${roleKeys.join(', ') || 'none'})`);
+    }
+    const role = roleKeys[0];
+    if (typeof t[role] !== 'string') {
+        throw new ValidationError(`Invalid turn at index ${index}: ${role} must be a string`);
+    }
+    if (t[role].length === 0) {
+        throw new ValidationError(`Invalid turn at index ${index}: ${role} must not be empty`);
+    }
+    if (t.model !== undefined && (typeof t.model !== 'string' || !t.model.trim())) {
+        throw new ValidationError(`Invalid turn at index ${index}: model must be a non-empty string when present`);
+    }
+    if (t.createdAt !== undefined &&
+        (typeof t.createdAt !== 'string' || Number.isNaN(Date.parse(t.createdAt)))) {
+        throw new ValidationError(`Invalid turn at index ${index}: createdAt must be an ISO-like date string when present`);
+    }
+    if (t.createdInThreadShelf !== undefined && typeof t.createdInThreadShelf !== 'boolean') {
+        throw new ValidationError(`Invalid turn at index ${index}: createdInThreadShelf must be a boolean when present`);
+    }
+    if (t.generationProvider !== undefined &&
+        !['llama-cpp', 'openrouter'].includes(String(t.generationProvider))) {
+        throw new ValidationError(`Invalid turn at index ${index}: generationProvider must be llama-cpp or openrouter`);
+    }
+    const normalized = {
+        role,
+        text: t[role],
+        model: t.model,
+    };
+    if (t.createdAt !== undefined) {
+        return { ...normalized, createdAt: t.createdAt };
+    }
+    return normalized;
+};
+export const validateTurns = (turns) => {
+    if (!Array.isArray(turns)) {
+        throw new ValidationError('Invalid turns: expected an array');
+    }
+    turns.forEach((turn, index) => validateTurn(turn, { index }));
+    return turns;
+};
+export const isSafeRelativePath = (input) => {
+    if (typeof input !== 'string' || input.length === 0)
+        return false;
+    if (input.length > 1024)
+        return false;
+    if (input.includes('\0'))
+        return false;
+    const normalized = input.replace(/\\/g, '/');
+    if (normalized.startsWith('/'))
+        return false;
+    if (/^[a-zA-Z]:/.test(normalized))
+        return false;
+    const segments = normalized.split('/');
+    return segments.every((segment) => segment !== '..' && segment.length <= 255);
+};

package/dist/src/watch.js

@@ -0,0 +1,83 @@
+import { watch } from 'fs';
+import { stat } from 'fs/promises';
+import { basename, join, resolve } from 'path';
+import { ingestFiles, isExportFileName } from './ingest.js';
+// Collects change events and flushes a deduplicated batch once the folder has
+// been quiet for debounceMs. Flushes never overlap: changes arriving while a
+// flush runs queue up for the next one.
+export const createChangeBatcher = ({ debounceMs, onFlush, }) => {
+    const pending = new Set();
+    let timer = null;
+    let running = Promise.resolve();
+    const flush = () => {
+        timer = null;
+        const batch = [...pending];
+        pending.clear();
+        if (!batch.length)
+            return;
+        running = running.then(() => onFlush(batch)).catch(() => { });
+    };
+    return {
+        add(path) {
+            pending.add(path);
+            if (timer)
+                clearTimeout(timer);
+            timer = setTimeout(flush, debounceMs);
+        },
+        async settle() {
+            if (timer) {
+                clearTimeout(timer);
+                flush();
+            }
+            await running;
+        },
+        close() {
+            if (timer)
+                clearTimeout(timer);
+            timer = null;
+            pending.clear();
+        },
+    };
+};
+export const watchFolder = (collection, folderPath, opts = {}) => {
+    const resolved = resolve(folderPath);
+    const runIngest = opts.ingest ?? ((files) => ingestFiles(collection, files));
+    const batcher = createChangeBatcher({
+        debounceMs: opts.debounceMs ?? 2000,
+        onFlush: async (paths) => {
+            const files = [];
+            for (const path of paths) {
+                try {
+                    const info = await stat(path);
+                    if (info.isFile())
+                        files.push(path);
+                }
+                catch {
+                    // Deleted or unreadable between event and flush — keep the index as-is.
+                }
+            }
+            if (!files.length)
+                return;
+            try {
+                const result = await runIngest(files);
+                opts.onBatch?.(files, result);
+            }
+            catch (e) {
+                opts.onBatch?.(files, null, e);
+            }
+        },
+    });
+    const watcher = watch(resolved, { recursive: true }, (_event, filename) => {
+        if (!filename)
+            return;
+        if (!isExportFileName(basename(filename)))
+            return;
+        batcher.add(join(resolved, filename));
+    });
+    return {
+        close: async () => {
+            watcher.close();
+            await batcher.settle();
+        },
+    };
+};

package/package.json

@@ -0,0 +1,103 @@
+{
+  "name": "threadshelf",
+  "version": "1.2.0",
+  "description": "Local semantic search, backup, and RAG for your AI chats — ChatGPT, Claude, Google AI Studio, OpenRouter, LM Studio, and Grok — searchable from a web UI, HTTP API, or MCP",
+  "license": "MIT",
+  "type": "module",
+  "engines": {
+    "node": ">=20.19.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ChrystianSchutz/ThreadShelf.git"
+  },
+  "homepage": "https://github.com/ChrystianSchutz/ThreadShelf#readme",
+  "bugs": {
+    "url": "https://github.com/ChrystianSchutz/ThreadShelf/issues"
+  },
+  "keywords": [
+    "semantic-search",
+    "rag",
+    "local-first",
+    "chatgpt",
+    "claude",
+    "openrouter",
+    "lm-studio",
+    "grok",
+    "lancedb",
+    "mcp"
+  ],
+  "bin": {
+    "threadshelf": "./bin/threadshelf.js",
+    "threadshelf-mcp": "./bin/threadshelf-mcp.js"
+  },
+  "files": [
+    "bin/",
+    "dist/",
+    "public/",
+    "scripts/openrouter-export-all.js",
+    "scripts/openrouter-export-browser.js",
+    "README.md",
+    "CHANGELOG.md",
+    "LICENSE",
+    "SECURITY.md"
+  ],
+  "workspaces": [
+    "client"
+  ],
+  "overrides": {
+    "@huggingface/transformers": {
+      "sharp": "0.35.3"
+    },
+    "onnxruntime-node": {
+      "adm-zip": "0.6.0"
+    }
+  },
+  "scripts": {
+    "test": "node scripts/run-node-tests.js test",
+    "test:e2e": "node scripts/run-node-tests.js test/e2e",
+    "test:playwright": "playwright test",
+    "check:repo": "tsx scripts/check-repo-hygiene.ts",
+    "check": "npm run check:repo && npm run lint && npx tsc --noEmit && npm test && npm run test:e2e && npm run build:client && npm run test:playwright",
+    "parse": "tsx src/cli.ts",
+    "ingest": "tsx src/ingest-cli.ts",
+    "search": "tsx src/search-cli.ts",
+    "setup:llama": "tsx scripts/setup-llama-cpp.ts",
+    "start": "tsx src/server.ts",
+    "dev": "tsx --watch src/server.ts",
+    "dev:client": "npm run dev --workspace client",
+    "build": "npm run build:client && npm run build:server",
+    "build:client": "npm run build --workspace client",
+    "docs:screenshots": "tsx scripts/capture-docs-screenshots.ts",
+    "mcp": "tsx mcp/server.ts",
+    "lint": "eslint src/ mcp/ client/src/",
+    "format": "prettier --write \"src/**/*.ts\" \"client/src/**/*.{ts,tsx}\"",
+    "build:server": "tsc -p tsconfig.build.json",
+    "build:package": "npm run clean:dist && npm run build:client && npm run build:server",
+    "clean:dist": "node -e \"fs.rmSync('dist',{recursive:true,force:true})\"",
+    "prepack": "npm run build:package",
+    "pack:verify": "node scripts/verify-package.js"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^4.2.0",
+    "@lancedb/lancedb": "^0.29.0",
+    "express": "^5.2.1",
+    "multer": "^2.2.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@playwright/test": "^1.62.1",
+    "@types/express": "^5.0.6",
+    "@types/multer": "^2.1.0",
+    "@types/node": "^26.1.2",
+    "eslint": "^10.5.0",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-react-hooks": "^7.1.1",
+    "prettier": "^3.8.4",
+    "sass": "^1.101.0",
+    "tsx": "^4.22.4",
+    "typescript": "^6.0.3",
+    "typescript-eslint": "^8.61.1",
+    "vite": "^8.2.0"
+  }
+}