toolpack-sdk 1.0.0 → 1.1.0-SNAPSHOT

package/dist/knowledge/sources/json-source.js

@@ -0,0 +1,224 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JSONSource = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+const errors_js_1 = require("../errors.js");
+class JSONSource {
+    filePath;
+    options;
+    watcher = null;
+    lastHash = null;
+    constructor(filePath, options = {}) {
+        this.filePath = path.resolve(filePath);
+        this.options = options;
+    }
+    async *load() {
+        try {
+            const content = await fs.promises.readFile(this.filePath, 'utf-8');
+            this.lastHash = this.hashContent(content);
+            const data = JSON.parse(content);
+            const chunks = this.extractChunks(data);
+            for (const chunk of chunks) {
+                yield chunk;
+            }
+        }
+        catch (error) {
+            throw new errors_js_1.IngestionError(`Failed to parse JSON file: ${error.message}`, this.filePath, error);
+        }
+    }
+    async *watch() {
+        if (!this.options.watch) {
+            return;
+        }
+        const updateQueue = [];
+        let resolveNext = null;
+        const processChange = async () => {
+            try {
+                const content = await fs.promises.readFile(this.filePath, 'utf-8');
+                const newHash = this.hashContent(content);
+                if (newHash !== this.lastHash) {
+                    this.lastHash = newHash;
+                    const data = JSON.parse(content);
+                    const chunks = this.extractChunks(data);
+                    for (const chunk of chunks) {
+                        updateQueue.push({ type: 'update', chunk });
+                    }
+                    if (resolveNext && updateQueue.length > 0) {
+                        const update = updateQueue.shift();
+                        const resolve = resolveNext;
+                        resolveNext = null;
+                        resolve({ value: update, done: false });
+                    }
+                }
+            }
+            catch {
+                // File might be in the middle of being written
+            }
+        };
+        this.watcher = fs.watch(this.filePath, (eventType) => {
+            if (eventType === 'change') {
+                processChange();
+            }
+        });
+        while (true) {
+            if (updateQueue.length > 0) {
+                yield updateQueue.shift();
+            }
+            else {
+                yield await new Promise((resolve) => {
+                    resolveNext = (result) => resolve(result.value);
+                });
+            }
+        }
+    }
+    stop() {
+        if (this.watcher) {
+            this.watcher.close();
+            this.watcher = null;
+        }
+    }
+    extractChunks(data) {
+        const chunkBy = this.options.chunkBy ?? 'item';
+        const contentFields = this.options.contentFields ?? [];
+        const metadataFields = this.options.metadataFields ?? [];
+        let items;
+        if (chunkBy === 'item') {
+            items = Array.isArray(data) ? data : [data];
+        }
+        else if (chunkBy.startsWith('$.')) {
+            items = this.evaluateJSONPath(data, chunkBy);
+        }
+        else {
+            items = Array.isArray(data) ? data : [data];
+        }
+        return items.map((item, index) => this.itemToChunk(item, index, contentFields, metadataFields));
+    }
+    evaluateJSONPath(data, pathExpr) {
+        const path = pathExpr.slice(2);
+        const segments = path.split(/(?=\[)|\./).filter(Boolean);
+        let current = [data];
+        for (const segment of segments) {
+            const next = [];
+            for (const item of current) {
+                if (segment === '[*]') {
+                    if (Array.isArray(item)) {
+                        next.push(...item);
+                    }
+                    else if (typeof item === 'object' && item !== null) {
+                        next.push(...Object.values(item));
+                    }
+                }
+                else {
+                    const key = segment.replace(/^\[|\]$/g, '');
+                    if (item && typeof item === 'object' && key in item) {
+                        next.push(item[key]);
+                    }
+                }
+            }
+            current = next;
+        }
+        return current;
+    }
+    itemToChunk(item, index, contentFields, metadataFields) {
+        let content;
+        if (contentFields.length > 0) {
+            const contentParts = contentFields
+                .map((field) => this.getNestedValue(item, field))
+                .filter((v) => v !== undefined && v !== null)
+                .map((v) => String(v));
+            content = contentParts.join('\n\n');
+        }
+        else {
+            content = this.flattenToContent(item);
+        }
+        const metadata = {
+            ...(this.options.metadata ?? {}),
+            source: this.filePath,
+            index,
+        };
+        if (metadataFields.length > 0) {
+            for (const field of metadataFields) {
+                const value = this.getNestedValue(item, field);
+                if (value !== undefined) {
+                    metadata[field] = value;
+                }
+            }
+        }
+        const namespace = this.options.namespace ?? 'json';
+        const hash = crypto.createHash('md5').update(content).digest('hex').slice(0, 8);
+        const id = `${namespace}:${path.basename(this.filePath)}:${hash}:${index}`;
+        return { id, content, metadata };
+    }
+    getNestedValue(obj, path) {
+        const parts = path.split('.');
+        let current = obj;
+        for (const part of parts) {
+            if (current === null || current === undefined) {
+                return undefined;
+            }
+            current = current[part];
+        }
+        return current;
+    }
+    flattenToContent(obj, prefix = '') {
+        if (obj === null || obj === undefined) {
+            return '';
+        }
+        if (typeof obj !== 'object') {
+            return prefix ? `${prefix}: ${obj}` : String(obj);
+        }
+        if (Array.isArray(obj)) {
+            return obj
+                .map((item, i) => this.flattenToContent(item, prefix ? `${prefix}[${i}]` : `[${i}]`))
+                .filter(Boolean)
+                .join('\n');
+        }
+        return Object.entries(obj)
+            .map(([key, value]) => {
+            const newPrefix = prefix ? `${prefix}.${key}` : key;
+            return this.flattenToContent(value, newPrefix);
+        })
+            .filter(Boolean)
+            .join('\n');
+    }
+    hashContent(content) {
+        return crypto.createHash('md5').update(content).digest('hex');
+    }
+}
+exports.JSONSource = JSONSource;
+//# sourceMappingURL=json-source.js.map

package/dist/knowledge/sources/json-source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-source.js","sourceRoot":"","sources":["../../../src/knowledge/sources/json-source.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAEjC,4CAA8C;AAE9C,MAAa,UAAU;IACb,QAAQ,CAAS;IACjB,OAAO,CAAoB;IAC3B,OAAO,GAAwB,IAAI,CAAC;IACpC,QAAQ,GAAkB,IAAI,CAAC;IAEvC,YAAY,QAAgB,EAAE,UAA6B,EAAE;QAC3D,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,CAAC,IAAI;QACT,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;YACxC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,0BAAc,CACtB,8BAA+B,KAAe,CAAC,OAAO,EAAE,EACxD,IAAI,CAAC,QAAQ,EACb,KAAc,CACf,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,KAAK;QACV,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,MAAM,WAAW,GAAkB,EAAE,CAAC;QACtC,IAAI,WAAW,GAA0D,IAAI,CAAC;QAE9E,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;YAC/B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBACnE,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAE1C,IAAI,OAAO,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAC9B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;oBACxB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBACjC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;oBAExC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBAC3B,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;oBAC9C,CAAC;oBAED,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC1C,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,EAAG,CAAC;wBACpC,MAAM,OAAO,GAAG,WAAW,CAAC;wBAC5B,WAAW,GAAG,IAAI,CAAC;wBACnB,OAAO,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;oBAC1C,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,+CAA+C;YACjD,CAAC;QACH,CAAC,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,EAAE;YACnD,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAC3B,aAAa,EAAE,CAAC;YAClB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,WAAW,CAAC,KAAK,EAAG,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,IAAI,OAAO,CAAc,CAAC,OAAO,EAAE,EAAE;oBAC/C,WAAW,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAClD,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACrB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,IAAS;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC;QAC/C,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,EAAE,CAAC;QACvD,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC;QAEzD,IAAI,KAAY,CAAC;QAEjB,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YACvB,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC;aAAM,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC;IAClG,CAAC;IAEO,gBAAgB,CAAC,IAAS,EAAE,QAAgB;QAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEzD,IAAI,OAAO,GAAU,CAAC,IAAI,CAAC,CAAC;QAE5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAU,EAAE,CAAC;YAEvB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBAC3B,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;oBACtB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;wBACxB,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBACrB,CAAC;yBAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;wBACrD,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;oBACpC,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;oBAC5C,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;wBACpD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;oBACvB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,WAAW,CACjB,IAAS,EACT,KAAa,EACb,aAAuB,EACvB,cAAwB;QAExB,IAAI,OAAe,CAAC;QAEpB,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,YAAY,GAAG,aAAa;iBAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;iBAChD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,CAAC;iBAC5C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACzB,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtC,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,QAAQ,GAAwB;YACpC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;YAChC,MAAM,EAAE,IAAI,CAAC,QAAQ;YACrB,KAAK;SACN,CAAC;QAEF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;gBACnC,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC/C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,QAAQ,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;QACnD,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,EAAE,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;QAE3E,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IACnC,CAAC;IAEO,cAAc,CAAC,GAAQ,EAAE,IAAY;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,OAAO,GAAG,GAAG,CAAC;QAElB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC9C,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,gBAAgB,CAAC,GAAQ,EAAE,SAAiB,EAAE;QACpD,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtC,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;YAC5B,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACpD,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,GAAG;iBACP,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBACpF,MAAM,CAAC,OAAO,CAAC;iBACf,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC;QAED,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC;aACvB,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YACpB,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YACpD,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACjD,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAEO,WAAW,CAAC,OAAe;QACjC,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAChE,CAAC;CACF;AA1ND,gCA0NC"}

package/dist/knowledge/sources/markdown-source.d.ts

@@ -0,0 +1,27 @@
+import type { KnowledgeSource, Chunk, ChunkUpdate, MarkdownSourceOptions } from '../types.js';
+export declare class MarkdownSource implements KnowledgeSource {
+    private pattern;
+    private options;
+    private watcher;
+    private fileHashes;
+    constructor(pattern: string, options?: MarkdownSourceOptions);
+    load(): AsyncIterable<Chunk>;
+    watch(): AsyncIterable<ChunkUpdate>;
+    stop(): void;
+    private parseMarkdown;
+    private extractFrontmatter;
+    private splitByHeadings;
+    private splitLargeSection;
+    private getOverlapText;
+    private splitIntoSentences;
+    private estimateTokens;
+    private extractTags;
+    private extractWikilinks;
+    private generateChunkId;
+    private getRelativePath;
+    private getBaseDir;
+    private matchesPattern;
+    private hashContent;
+    private parseFrontmatterValue;
+}
+//# sourceMappingURL=markdown-source.d.ts.map

package/dist/knowledge/sources/markdown-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown-source.d.ts","sourceRoot":"","sources":["../../../src/knowledge/sources/markdown-source.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAiB9F,qBAAa,cAAe,YAAW,eAAe;IACpD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,UAAU,CAAkC;gBAExC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,qBAA0B;IAezD,IAAI,IAAI,aAAa,CAAC,KAAK,CAAC;IAuB5B,KAAK,IAAI,aAAa,CAAC,WAAW,CAAC;IA8E1C,IAAI,IAAI,IAAI;IAOZ,OAAO,CAAC,aAAa;IAqCrB,OAAO,CAAC,kBAAkB;IAwB1B,OAAO,CAAC,eAAe;IAmEvB,OAAO,CAAC,iBAAiB;IAqEzB,OAAO,CAAC,cAAc;IAiBtB,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,gBAAgB;IAYxB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,eAAe;IAKvB,OAAO,CAAC,UAAU;IAKlB,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,qBAAqB;CAsB9B"}

package/dist/knowledge/sources/markdown-source.js

@@ -0,0 +1,410 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MarkdownSource = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+const fast_glob_1 = __importDefault(require("fast-glob"));
+const errors_js_1 = require("../errors.js");
+const DEFAULT_MAX_CHUNK_SIZE = 2000;
+const DEFAULT_CHUNK_OVERLAP = 200;
+const DEFAULT_MIN_CHUNK_SIZE = 100;
+class MarkdownSource {
+    pattern;
+    options;
+    watcher = null;
+    fileHashes = new Map();
+    constructor(pattern, options = {}) {
+        // Auto-append **/*.md if pattern looks like a directory (no glob chars and no .md extension)
+        if (!pattern.includes('*') && !pattern.includes('?') && !pattern.endsWith('.md')) {
+            this.pattern = pattern.endsWith('/') ? `${pattern}**/*.md` : `${pattern}/**/*.md`;
+        }
+        else {
+            this.pattern = pattern;
+        }
+        this.options = {
+            maxChunkSize: DEFAULT_MAX_CHUNK_SIZE,
+            chunkOverlap: DEFAULT_CHUNK_OVERLAP,
+            minChunkSize: DEFAULT_MIN_CHUNK_SIZE,
+            ...options,
+        };
+    }
+    async *load() {
+        const files = await (0, fast_glob_1.default)(this.pattern, { absolute: true });
+        for (const filePath of files) {
+            try {
+                const content = await fs.promises.readFile(filePath, 'utf-8');
+                const hash = this.hashContent(content);
+                this.fileHashes.set(filePath, hash);
+                const chunks = this.parseMarkdown(filePath, content);
+                for (const chunk of chunks) {
+                    yield chunk;
+                }
+            }
+            catch (error) {
+                throw new errors_js_1.IngestionError(`Failed to parse markdown file: ${error.message}`, filePath, error);
+            }
+        }
+    }
+    async *watch() {
+        if (!this.options.watch) {
+            return;
+        }
+        const baseDir = this.getBaseDir();
+        const updateQueue = [];
+        let resolveNext = null;
+        const processFile = async (filePath, eventType) => {
+            const absolutePath = path.isAbsolute(filePath) ? filePath : path.join(baseDir, filePath);
+            if (!this.matchesPattern(absolutePath)) {
+                return;
+            }
+            if (eventType === 'unlink') {
+                const oldHash = this.fileHashes.get(absolutePath);
+                if (oldHash) {
+                    this.fileHashes.delete(absolutePath);
+                    const chunkId = this.generateChunkId(absolutePath, '', 0);
+                    updateQueue.push({
+                        type: 'delete',
+                        chunk: { id: chunkId, content: '', metadata: { source: absolutePath } },
+                    });
+                }
+            }
+            else {
+                try {
+                    const content = await fs.promises.readFile(absolutePath, 'utf-8');
+                    const newHash = this.hashContent(content);
+                    const oldHash = this.fileHashes.get(absolutePath);
+                    if (oldHash !== newHash) {
+                        this.fileHashes.set(absolutePath, newHash);
+                        const chunks = this.parseMarkdown(absolutePath, content);
+                        const updateType = oldHash ? 'update' : 'add';
+                        for (const chunk of chunks) {
+                            updateQueue.push({ type: updateType, chunk });
+                        }
+                    }
+                }
+                catch {
+                    // File might have been deleted between event and read
+                }
+            }
+            if (resolveNext && updateQueue.length > 0) {
+                const update = updateQueue.shift();
+                const resolve = resolveNext;
+                resolveNext = null;
+                resolve({ value: update, done: false });
+            }
+        };
+        this.watcher = fs.watch(baseDir, { recursive: true }, (eventType, filename) => {
+            if (filename && filename.endsWith('.md')) {
+                const fullPath = path.join(baseDir, filename);
+                if (eventType === 'rename') {
+                    fs.access(fullPath, fs.constants.F_OK, (err) => {
+                        processFile(fullPath, err ? 'unlink' : 'add');
+                    });
+                }
+                else {
+                    processFile(fullPath, 'change');
+                }
+            }
+        });
+        while (true) {
+            if (updateQueue.length > 0) {
+                yield updateQueue.shift();
+            }
+            else {
+                yield await new Promise((resolve) => {
+                    resolveNext = (result) => resolve(result.value);
+                });
+            }
+        }
+    }
+    stop() {
+        if (this.watcher) {
+            this.watcher.close();
+            this.watcher = null;
+        }
+    }
+    parseMarkdown(filePath, content) {
+        const { frontmatter, body } = this.extractFrontmatter(content);
+        const sections = this.splitByHeadings(body);
+        const chunks = [];
+        let chunkIndex = 0;
+        const relativePath = this.getRelativePath(filePath);
+        for (const section of sections) {
+            const sectionChunks = this.splitLargeSection(section);
+            const totalChunks = sectionChunks.length;
+            for (const sectionChunk of sectionChunks) {
+                const id = this.generateChunkId(filePath, sectionChunk.content, chunkIndex);
+                chunks.push({
+                    id,
+                    content: sectionChunk.content,
+                    metadata: {
+                        ...frontmatter,
+                        ...(this.options.metadata ?? {}),
+                        heading: sectionChunk.headings,
+                        hasCode: sectionChunk.hasCode,
+                        source: relativePath,
+                        chunkIndex,
+                        totalChunks,
+                        ...(this.extractTags(sectionChunk.content)),
+                        ...(this.extractWikilinks(sectionChunk.content)),
+                    },
+                });
+                chunkIndex++;
+            }
+        }
+        return chunks;
+    }
+    extractFrontmatter(content) {
+        const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n/);
+        if (!frontmatterMatch) {
+            return { frontmatter: {}, body: content };
+        }
+        const frontmatterStr = frontmatterMatch[1];
+        const body = content.slice(frontmatterMatch[0].length);
+        const frontmatter = {};
+        for (const line of frontmatterStr.split('\n')) {
+            const colonIndex = line.indexOf(':');
+            if (colonIndex > 0) {
+                const key = line.slice(0, colonIndex).trim();
+                const rawValue = line.slice(colonIndex + 1).trim();
+                const parsedValue = this.parseFrontmatterValue(rawValue);
+                frontmatter[key] = parsedValue;
+            }
+        }
+        return { frontmatter, body };
+    }
+    splitByHeadings(content) {
+        const lines = content.split('\n');
+        const sections = [];
+        let currentHeadings = [];
+        let currentContent = [];
+        let inCodeBlock = false;
+        for (const line of lines) {
+            if (line.startsWith('```')) {
+                inCodeBlock = !inCodeBlock;
+                currentContent.push(line);
+                continue;
+            }
+            if (!inCodeBlock && line.match(/^#{1,6}\s/)) {
+                if (currentContent.length > 0) {
+                    const contentStr = currentContent.join('\n').trim();
+                    if (contentStr) {
+                        sections.push({
+                            headings: [...currentHeadings],
+                            content: contentStr,
+                            hasCode: currentContent.some((l) => l.startsWith('```')),
+                            frontmatter: {},
+                        });
+                    }
+                }
+                const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
+                if (headingMatch) {
+                    const level = headingMatch[1].length;
+                    const headingText = headingMatch[2].trim();
+                    currentHeadings = currentHeadings.slice(0, level - 1);
+                    currentHeadings[level - 1] = headingText;
+                    currentHeadings = currentHeadings.filter(Boolean);
+                }
+                currentContent = [];
+            }
+            else {
+                currentContent.push(line);
+            }
+        }
+        if (currentContent.length > 0) {
+            const contentStr = currentContent.join('\n').trim();
+            if (contentStr) {
+                sections.push({
+                    headings: [...currentHeadings],
+                    content: contentStr,
+                    hasCode: currentContent.some((l) => l.startsWith('```')),
+                    frontmatter: {},
+                });
+            }
+        }
+        if (sections.length === 0 && content.trim()) {
+            sections.push({
+                headings: [],
+                content: content.trim(),
+                hasCode: content.includes('```'),
+                frontmatter: {},
+            });
+        }
+        return sections;
+    }
+    splitLargeSection(section) {
+        const maxSize = this.options.maxChunkSize;
+        const minSize = this.options.minChunkSize;
+        const overlap = this.options.chunkOverlap;
+        const tokens = this.estimateTokens(section.content);
+        if (tokens <= maxSize) {
+            return [section];
+        }
+        const paragraphs = section.content.split(/\n\n+/);
+        const chunks = [];
+        let currentChunk = [];
+        let currentTokens = 0;
+        for (const paragraph of paragraphs) {
+            const paragraphTokens = this.estimateTokens(paragraph);
+            if (currentTokens + paragraphTokens > maxSize && currentChunk.length > 0) {
+                chunks.push({
+                    ...section,
+                    content: currentChunk.join('\n\n'),
+                });
+                const overlapText = this.getOverlapText(currentChunk, overlap);
+                currentChunk = overlapText ? [overlapText] : [];
+                currentTokens = this.estimateTokens(currentChunk.join('\n\n'));
+            }
+            if (paragraphTokens > maxSize) {
+                const sentences = this.splitIntoSentences(paragraph);
+                for (const sentence of sentences) {
+                    const sentenceTokens = this.estimateTokens(sentence);
+                    if (currentTokens + sentenceTokens > maxSize && currentChunk.length > 0) {
+                        chunks.push({
+                            ...section,
+                            content: currentChunk.join(' '),
+                        });
+                        currentChunk = [];
+                        currentTokens = 0;
+                    }
+                    currentChunk.push(sentence);
+                    currentTokens += sentenceTokens;
+                }
+            }
+            else {
+                currentChunk.push(paragraph);
+                currentTokens += paragraphTokens;
+            }
+        }
+        if (currentChunk.length > 0) {
+            const content = currentChunk.join('\n\n');
+            if (this.estimateTokens(content) >= minSize || chunks.length === 0) {
+                chunks.push({
+                    ...section,
+                    content,
+                });
+            }
+            else if (chunks.length > 0) {
+                const lastChunk = chunks[chunks.length - 1];
+                lastChunk.content += '\n\n' + content;
+            }
+        }
+        return chunks;
+    }
+    getOverlapText(chunks, targetTokens) {
+        const result = [];
+        let tokens = 0;
+        for (let i = chunks.length - 1; i >= 0 && tokens < targetTokens; i--) {
+            const chunkTokens = this.estimateTokens(chunks[i]);
+            if (tokens + chunkTokens <= targetTokens) {
+                result.unshift(chunks[i]);
+                tokens += chunkTokens;
+            }
+            else {
+                break;
+            }
+        }
+        return result.join('\n\n');
+    }
+    splitIntoSentences(text) {
+        return text.split(/(?<=[.!?])\s+/).filter(Boolean);
+    }
+    estimateTokens(text) {
+        return Math.ceil(text.length / 4);
+    }
+    extractTags(content) {
+        const tagMatches = content.match(/#[\w-]+/g);
+        if (tagMatches) {
+            return { tags: [...new Set(tagMatches.map((t) => t.slice(1)))] };
+        }
+        return {};
+    }
+    extractWikilinks(content) {
+        const linkMatches = content.match(/\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g);
+        if (linkMatches) {
+            const links = linkMatches.map((l) => {
+                const match = l.match(/\[\[([^\]|]+)/);
+                return match ? match[1] : '';
+            }).filter(Boolean);
+            return { links: [...new Set(links)] };
+        }
+        return {};
+    }
+    generateChunkId(filePath, content, index) {
+        const namespace = this.options.namespace ?? 'default';
+        const relativePath = this.getRelativePath(filePath);
+        const hash = crypto.createHash('md5').update(content).digest('hex').slice(0, 8);
+        return `${namespace}:${relativePath}:${hash}:${index}`;
+    }
+    getRelativePath(filePath) {
+        const baseDir = this.getBaseDir();
+        return path.relative(baseDir, filePath);
+    }
+    getBaseDir() {
+        const patternParts = this.pattern.split(/[*?]/);
+        return patternParts[0].replace(/\/$/, '') || '.';
+    }
+    matchesPattern(filePath) {
+        return filePath.endsWith('.md');
+    }
+    hashContent(content) {
+        return crypto.createHash('md5').update(content).digest('hex');
+    }
+    parseFrontmatterValue(value) {
+        if (value.startsWith('[') && value.endsWith(']')) {
+            return value
+                .slice(1, -1)
+                .split(',')
+                .map((segment) => segment.trim().replace(/^["']|["']$/g, ''));
+        }
+        if (value === 'true') {
+            return true;
+        }
+        if (value === 'false') {
+            return false;
+        }
+        if (!Number.isNaN(Number(value)) && value !== '') {
+            return Number(value);
+        }
+        return value.replace(/^["']|["']$/g, '');
+    }
+}
+exports.MarkdownSource = MarkdownSource;
+//# sourceMappingURL=markdown-source.js.map