xtctx 0.3.2

package/dist/src/runtime/services.js

@@ -0,0 +1,227 @@
+import { readdir, readFile } from "node:fs/promises";
+import { extname, join, resolve } from "node:path";
+import { parse as parseYaml } from "yaml";
+import { KnowledgeRepository } from "../knowledge/repository.js";
+export class EmptySessionService {
+    async listRecentSessions(_limit, _toolFilter) {
+        return [];
+    }
+    async getSessionDetail(_sessionRef, _offset, _limit) {
+        return [];
+    }
+}
+export class FileConfigStore {
+    configRoot;
+    constructor(configRoot) {
+        this.configRoot = configRoot;
+    }
+    async list(type) {
+        const types = type ? [type] : ["skill", "command", "agent"];
+        const all = [];
+        for (const configType of types) {
+            const dir = join(this.configRoot, `${configType}s`);
+            let files;
+            try {
+                files = await readdir(dir);
+            }
+            catch {
+                continue;
+            }
+            for (const file of files) {
+                const parsed = await this.parseFile(join(dir, file), configType);
+                if (parsed) {
+                    all.push(parsed);
+                }
+            }
+        }
+        return all;
+    }
+    async get(type, name) {
+        const items = await this.list(type);
+        return items.find((item) => item.name === name) ?? null;
+    }
+    async toolPreferences(tool) {
+        const filePath = join(this.configRoot, "shared.yaml");
+        try {
+            const raw = await readFile(filePath, "utf-8");
+            const parsed = parseYaml(raw);
+            const prefs = parsed.toolPreferences;
+            if (prefs && typeof prefs === "object" && !Array.isArray(prefs)) {
+                const toolPrefs = prefs[tool];
+                if (toolPrefs && typeof toolPrefs === "object" && !Array.isArray(toolPrefs)) {
+                    return toolPrefs;
+                }
+            }
+        }
+        catch {
+            // fall through
+        }
+        return {};
+    }
+    async parseFile(filePath, type) {
+        try {
+            const raw = await readFile(filePath, "utf-8");
+            const name = filePath
+                .split(/[\\/]/)
+                .pop()
+                ?.replace(/\.(json|ya?ml|md|txt)$/i, "");
+            if (!name) {
+                return null;
+            }
+            const extension = extname(filePath).toLowerCase();
+            if (extension === ".json") {
+                return { type, name, content: JSON.parse(raw) };
+            }
+            if (extension === ".yaml" || extension === ".yml") {
+                const parsed = parseYaml(raw);
+                if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+                    return { type, name, content: parsed };
+                }
+            }
+            return { type, name, content: { text: raw } };
+        }
+        catch {
+            return null;
+        }
+    }
+}
+export async function createProjectServices(projectPath) {
+    const projectRoot = resolve(projectPath ?? process.cwd());
+    const xtctxDir = join(projectRoot, ".xtctx");
+    const storeDir = join(xtctxDir, ".store");
+    const knowledgeDir = join(xtctxDir, "knowledge");
+    const configRoot = join(xtctxDir, "tool-config");
+    const stateDir = join(xtctxDir, "state");
+    const config = await loadProjectConfig(xtctxDir);
+    const webPort = parseWebPort(config);
+    const apiSecurity = parseApiSecurityConfig(config);
+    const ingestion = parseIngestionConfig(config, projectRoot);
+    const knowledge = new KnowledgeRepository(knowledgeDir);
+    await knowledge.initialize();
+    return {
+        projectRoot,
+        xtctxDir,
+        storeDir,
+        knowledgeDir,
+        configRoot,
+        stateDir,
+        webPort,
+        apiSecurity,
+        ingestion,
+        knowledge,
+        sessions: new EmptySessionService(),
+        configs: new FileConfigStore(configRoot),
+    };
+}
+async function loadProjectConfig(xtctxDir) {
+    const configPath = join(xtctxDir, "config.yaml");
+    try {
+        const raw = await readFile(configPath, "utf-8");
+        const parsed = parseYaml(raw);
+        if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+            return parsed;
+        }
+    }
+    catch {
+        // ignore
+    }
+    return {};
+}
+function parseWebPort(config) {
+    const web = config.web;
+    if (web && typeof web === "object" && !Array.isArray(web)) {
+        const port = Number(web.port);
+        if (Number.isFinite(port) && port > 0 && port < 65536) {
+            return port;
+        }
+    }
+    return 3232;
+}
+function parseApiSecurityConfig(config) {
+    const api = config.api;
+    if (!api || typeof api !== "object" || Array.isArray(api)) {
+        return {};
+    }
+    const security = api.security;
+    if (!security || typeof security !== "object" || Array.isArray(security)) {
+        return {};
+    }
+    const value = security;
+    const parsed = {};
+    if (typeof value.token === "string") {
+        const token = value.token.trim();
+        parsed.apiToken = token.length > 0 ? token : null;
+    }
+    else if (value.token === null) {
+        parsed.apiToken = null;
+    }
+    if (Array.isArray(value.allowedOrigins)) {
+        parsed.allowedOrigins = value.allowedOrigins.filter((item) => typeof item === "string" && item.length > 0);
+    }
+    if (typeof value.allowLocalhostOrigins === "boolean") {
+        parsed.allowLocalhostOrigins = value.allowLocalhostOrigins;
+    }
+    const rateLimitWindowMs = Number(value.rateLimitWindowMs);
+    if (Number.isFinite(rateLimitWindowMs) && rateLimitWindowMs > 0) {
+        parsed.rateLimitWindowMs = Math.floor(rateLimitWindowMs);
+    }
+    const rateLimitMax = Number(value.rateLimitMax);
+    if (Number.isFinite(rateLimitMax) && rateLimitMax > 0) {
+        parsed.rateLimitMax = Math.floor(rateLimitMax);
+    }
+    return parsed;
+}
+function parseIngestionConfig(config, projectRoot) {
+    const ingestion = config.ingestion;
+    const defaults = {
+        scrapers: [],
+        watchPaths: [projectRoot],
+        pollIntervalMs: 30_000,
+        excludePatterns: ["node_modules/**", "dist/**"],
+    };
+    if (!ingestion || typeof ingestion !== "object" || Array.isArray(ingestion)) {
+        return defaults;
+    }
+    const value = ingestion;
+    const watchPaths = Array.isArray(value.watchPaths)
+        ? value.watchPaths
+            .filter((item) => typeof item === "string" && item.length > 0)
+            .map((item) => resolve(projectRoot, item))
+        : defaults.watchPaths;
+    const pollIntervalMs = Number(value.pollIntervalMs);
+    const excludePatterns = Array.isArray(value.excludePatterns)
+        ? value.excludePatterns.filter((item) => typeof item === "string" && item.length > 0)
+        : defaults.excludePatterns;
+    const scrapers = Array.isArray(value.scrapers)
+        ? value.scrapers
+            .map((item) => {
+            if (!item || typeof item !== "object" || Array.isArray(item)) {
+                return null;
+            }
+            const scraper = item;
+            const tool = typeof scraper.tool === "string" ? scraper.tool : "";
+            if (!tool) {
+                return null;
+            }
+            const enabled = scraper.enabled !== false;
+            if (typeof scraper.customStorePath === "string") {
+                return {
+                    tool,
+                    enabled,
+                    customStorePath: resolve(projectRoot, scraper.customStorePath),
+                };
+            }
+            return { tool, enabled };
+        })
+            .filter((item) => item !== null)
+        : defaults.scrapers;
+    return {
+        scrapers,
+        watchPaths: watchPaths.length > 0 ? watchPaths : defaults.watchPaths,
+        pollIntervalMs: Number.isFinite(pollIntervalMs) && pollIntervalMs > 0
+            ? Math.floor(pollIntervalMs)
+            : defaults.pollIntervalMs,
+        excludePatterns,
+    };
+}
+//# sourceMappingURL=services.js.map

package/dist/src/runtime/services.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"services.js","sourceRoot":"","sources":["../../../src/runtime/services.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAIjE,MAAM,OAAO,mBAAmB;IAC9B,KAAK,CAAC,kBAAkB,CAAC,MAAc,EAAE,WAAsB;QAC7D,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,WAAmB,EACnB,OAAe,EACf,MAAc;QAEd,OAAO,EAAE,CAAC;IACZ,CAAC;CACF;AAED,MAAM,OAAO,eAAe;IACG;IAA7B,YAA6B,UAAkB;QAAlB,eAAU,GAAV,UAAU,CAAQ;IAAG,CAAC;IAEnD,KAAK,CAAC,IAAI,CAAC,IAAiB;QAC1B,MAAM,KAAK,GAAiB,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC1E,MAAM,GAAG,GAAkB,EAAE,CAAC;QAE9B,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,UAAU,GAAG,CAAC,CAAC;YACpD,IAAI,KAAe,CAAC;YAEpB,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YAC7B,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;gBACjE,IAAI,MAAM,EAAE,CAAC;oBACX,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAgB,EAAE,IAAY;QACtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAAY;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;QAEtD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAA4B,CAAC;YACzD,MAAM,KAAK,GAAG,MAAM,CAAC,eAAe,CAAC;YAErC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChE,MAAM,SAAS,GAAI,KAAiC,CAAC,IAAI,CAAC,CAAC;gBAC3D,IAAI,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;oBAC5E,OAAO,SAAoC,CAAC;gBAC9C,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,eAAe;QACjB,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,QAAgB,EAAE,IAAgB;QACxD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAG,QAAQ;iBAClB,KAAK,CAAC,OAAO,CAAC;iBACd,GAAG,EAAE;gBACN,EAAE,OAAO,CAAC,yBAAyB,EAAE,EAAE,CAAC,CAAC;YAE3C,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;YAClD,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;gBAC1B,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,EAAE,CAAC;YAC7E,CAAC;YAED,IAAI,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gBAClD,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAY,CAAC;gBACzC,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;oBACnE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,MAAiC,EAAE,CAAC;gBACpE,CAAC;YACH,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;CACF;AA4BD,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,WAAoB;IAC9D,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,WAAW,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IACnD,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAE5D,MAAM,SAAS,GAAG,IAAI,mBAAmB,CAAC,YAAY,CAAC,CAAC;IACxD,MAAM,SAAS,CAAC,UAAU,EAAE,CAAC;IAE7B,OAAO;QACL,WAAW;QACX,QAAQ;QACR,QAAQ;QACR,YAAY;QACZ,UAAU;QACV,QAAQ;QACR,OAAO;QACP,WAAW;QACX,SAAS;QACT,SAAS;QACT,QAAQ,EAAE,IAAI,mBAAmB,EAAE;QACnC,OAAO,EAAE,IAAI,eAAe,CAAC,UAAU,CAAC;KACzC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAgB;IAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,OAAO,MAAiC,CAAC;QAC3C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,SAAS;IACX,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,YAAY,CAAC,MAA+B;IACnD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;IACvB,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,MAAM,CAAE,GAA+B,CAAC,IAAI,CAAC,CAAC;QAC3D,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;YACtD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,sBAAsB,CAAC,MAA+B;IAO7D,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;IACvB,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1D,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,QAAQ,GAAI,GAA+B,CAAC,QAAQ,CAAC;IAC3D,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzE,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAG,QAAmC,CAAC;IAClD,MAAM,MAAM,GAMR,EAAE,CAAC;IAEP,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IACpD,CAAC;SAAM,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QACxC,MAAM,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc,CAAC,MAAM,CACjD,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CACtE,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,KAAK,CAAC,qBAAqB,KAAK,SAAS,EAAE,CAAC;QACrD,MAAM,CAAC,qBAAqB,GAAG,KAAK,CAAC,qBAAqB,CAAC;IAC7D,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC1D,IAAI,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;QAChE,MAAM,CAAC,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAChD,IAAI,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,oBAAoB,CAC3B,MAA+B,EAC/B,WAAmB;IAOnB,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;IACnC,MAAM,QAAQ,GAAG;QACf,QAAQ,EAAE,EAAyE;QACnF,UAAU,EAAE,CAAC,WAAW,CAAC;QACzB,cAAc,EAAE,MAAM;QACtB,eAAe,EAAE,CAAC,iBAAiB,EAAE,SAAS,CAAC;KAChD,CAAC;IAEF,IAAI,CAAC,SAAS,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5E,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,SAAoC,CAAC;IAEnD,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC;QAChD,CAAC,CAAC,KAAK,CAAC,UAAU;aACb,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;aAC7E,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAC9C,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;IAExB,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IAEpD,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC;QAC1D,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAC1B,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CACtE;QACH,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC;IAE7B,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,CAAC,CAAC,KAAK,CAAC,QAAQ;aACX,GAAG,CAAC,CAAC,IAAI,EAAuE,EAAE;YACjF,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7D,OAAO,IAAI,CAAC;YACd,CAAC;YACD,MAAM,OAAO,GAAG,IAA+B,CAAC;YAChD,MAAM,IAAI,GAAG,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,KAAK,KAAK,CAAC;YAC1C,IAAI,OAAO,OAAO,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;gBAChD,OAAO;oBACL,IAAI;oBACJ,OAAO;oBACP,eAAe,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,eAAe,CAAC;iBAC/D,CAAC;YACJ,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC3B,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAwE,EAAE,CACrF,IAAI,KAAK,IAAI,CACd;QACL,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAEtB,OAAO;QACL,QAAQ;QACR,UAAU,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU;QACpE,cAAc,EACZ,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC;YACnD,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC;YAC5B,CAAC,CAAC,QAAQ,CAAC,cAAc;QAC7B,eAAe;KAChB,CAAC;AACJ,CAAC"}

package/dist/src/scrapers/base.d.ts

@@ -0,0 +1,10 @@
+import type { ScraperState } from "../types/scraper.js";
+export declare class ScraperStateManager {
+    private readonly stateDir;
+    constructor(stateDir: string);
+    load(tool: string): Promise<ScraperState>;
+    save(tool: string, state: ScraperState): Promise<void>;
+    private statePath;
+}
+export declare function estimateTokens(text: string): number;
+//# sourceMappingURL=base.d.ts.map

package/dist/src/scrapers/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../src/scrapers/base.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAExD,qBAAa,mBAAmB;IAClB,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,MAAM;IAEvC,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAezC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAM5D,OAAO,CAAC,SAAS;CAGlB;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD"}

package/dist/src/scrapers/base.js

@@ -0,0 +1,34 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+export class ScraperStateManager {
+    stateDir;
+    constructor(stateDir) {
+        this.stateDir = stateDir;
+    }
+    async load(tool) {
+        const path = this.statePath(tool);
+        try {
+            const raw = await readFile(path, "utf-8");
+            const data = JSON.parse(raw);
+            return {
+                ...data,
+                lastTimestamp: new Date(data.lastTimestamp),
+            };
+        }
+        catch {
+            return { lastTimestamp: new Date(0) };
+        }
+    }
+    async save(tool, state) {
+        const path = this.statePath(tool);
+        await mkdir(dirname(path), { recursive: true });
+        await writeFile(path, JSON.stringify(state, null, 2), "utf-8");
+    }
+    statePath(tool) {
+        return join(this.stateDir, `${tool}-state.json`);
+    }
+}
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+//# sourceMappingURL=base.js.map

package/dist/src/scrapers/base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../../src/scrapers/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAG1C,MAAM,OAAO,mBAAmB;IACD;IAA7B,YAA6B,QAAgB;QAAhB,aAAQ,GAAR,QAAQ,CAAQ;IAAG,CAAC;IAEjD,KAAK,CAAC,IAAI,CAAC,IAAY;QACrB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAElC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAiB,CAAC;YAC7C,OAAO;gBACL,GAAG,IAAI;gBACP,aAAa,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC;aAC5C,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,aAAa,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAY,EAAE,KAAmB;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACjE,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,aAAa,CAAC,CAAC;IACnD,CAAC;CACF;AAED,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC"}

package/dist/src/scrapers/claude-code.d.ts

@@ -0,0 +1,16 @@
+import type { ClaudeCodeChunk, ConversationScraper, ScraperState } from "../types/scraper.js";
+export declare class ClaudeCodeScraper implements ConversationScraper<ClaudeCodeChunk> {
+    private readonly claudeProjectsDir;
+    readonly tool = "claude-code";
+    private readonly stateManager;
+    constructor(claudeProjectsDir: string, stateDir: string);
+    detect(): Promise<boolean>;
+    getStorePaths(): string[];
+    scrape(since?: Date): AsyncIterable<ClaudeCodeChunk>;
+    fullSync(): AsyncIterable<ClaudeCodeChunk>;
+    parseRaw(raw: unknown): ClaudeCodeChunk;
+    getLastScrapedPosition(): Promise<ScraperState>;
+    saveScrapedPosition(state: ScraperState): Promise<void>;
+    private readAllSessions;
+}
+//# sourceMappingURL=claude-code.d.ts.map

package/dist/src/scrapers/claude-code.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-code.d.ts","sourceRoot":"","sources":["../../../src/scrapers/claude-code.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAEV,eAAe,EACf,mBAAmB,EACnB,YAAY,EACb,MAAM,qBAAqB,CAAC;AAW7B,qBAAa,iBAAkB,YAAW,mBAAmB,CAAC,eAAe,CAAC;IAK1E,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAJpC,QAAQ,CAAC,IAAI,iBAAiB;IAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsB;gBAGhC,iBAAiB,EAAE,MAAM,EAC1C,QAAQ,EAAE,MAAM;IAKZ,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAShC,aAAa,IAAI,MAAM,EAAE;IAIlB,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,GAAG,aAAa,CAAC,eAAe,CAAC;IAMpD,QAAQ,IAAI,aAAa,CAAC,eAAe,CAAC;IAIjD,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,eAAe;IAuBjC,sBAAsB,IAAI,OAAO,CAAC,YAAY,CAAC;IAI/C,mBAAmB,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;YAI9C,eAAe;CAyD/B"}

package/dist/src/scrapers/claude-code.js

@@ -0,0 +1,120 @@
+import { stat, readdir } from "node:fs/promises";
+import { join } from "node:path";
+import { createReadStream } from "node:fs";
+import { createInterface } from "node:readline";
+import { estimateTokens, ScraperStateManager } from "./base.js";
+const ROLE_MAP = {
+    human: "user",
+    assistant: "assistant",
+    system: "system",
+    tool_use: "tool",
+    tool_result: "tool",
+};
+export class ClaudeCodeScraper {
+    claudeProjectsDir;
+    tool = "claude-code";
+    stateManager;
+    constructor(claudeProjectsDir, stateDir) {
+        this.claudeProjectsDir = claudeProjectsDir;
+        this.stateManager = new ScraperStateManager(stateDir);
+    }
+    async detect() {
+        try {
+            const dirStat = await stat(this.claudeProjectsDir);
+            return dirStat.isDirectory();
+        }
+        catch {
+            return false;
+        }
+    }
+    getStorePaths() {
+        return [this.claudeProjectsDir];
+    }
+    async *scrape(since) {
+        const state = await this.getLastScrapedPosition();
+        const cutoff = since ?? state.lastTimestamp;
+        yield* this.readAllSessions(cutoff);
+    }
+    async *fullSync() {
+        yield* this.readAllSessions(new Date(0));
+    }
+    parseRaw(raw) {
+        const obj = raw;
+        const timestamp = new Date(obj.timestamp || Date.now());
+        const content = obj.content || "";
+        const type = obj.type || "unknown";
+        return {
+            tool: "claude-code",
+            sessionId: obj.sessionId || "unknown",
+            timestamp,
+            role: ROLE_MAP[type] || "system",
+            content,
+            metadata: {
+                messageIndex: obj.messageIndex || 0,
+                tokenEstimate: estimateTokens(content),
+                referencedFiles: [],
+                toolCalls: type === "tool_use" ? [obj.name || ""] : undefined,
+                costUsd: obj.costUsd,
+                sessionType: "interactive",
+            },
+        };
+    }
+    async getLastScrapedPosition() {
+        return this.stateManager.load(this.tool);
+    }
+    async saveScrapedPosition(state) {
+        await this.stateManager.save(this.tool, state);
+    }
+    async *readAllSessions(since) {
+        let projectDirs;
+        try {
+            projectDirs = await readdir(this.claudeProjectsDir);
+        }
+        catch {
+            return;
+        }
+        for (const projectHash of projectDirs) {
+            const projectDir = join(this.claudeProjectsDir, projectHash);
+            let files;
+            try {
+                files = await readdir(projectDir);
+            }
+            catch {
+                continue;
+            }
+            for (const file of files) {
+                if (!file.endsWith(".jsonl")) {
+                    continue;
+                }
+                const sessionId = file.replace(".jsonl", "");
+                const filePath = join(projectDir, file);
+                const reader = createInterface({
+                    input: createReadStream(filePath, { encoding: "utf8" }),
+                    crlfDelay: Infinity,
+                });
+                let messageIndex = 0;
+                for await (const line of reader) {
+                    if (!line.trim()) {
+                        continue;
+                    }
+                    try {
+                        const obj = JSON.parse(line);
+                        const timestamp = new Date(obj.timestamp ?? 0);
+                        if (timestamp <= since) {
+                            messageIndex += 1;
+                            continue;
+                        }
+                        obj.sessionId = sessionId;
+                        obj.messageIndex = messageIndex;
+                        messageIndex += 1;
+                        yield this.parseRaw(obj);
+                    }
+                    catch {
+                        // Skip malformed lines.
+                    }
+                }
+            }
+        }
+    }
+}
+//# sourceMappingURL=claude-code.js.map

package/dist/src/scrapers/claude-code.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude-code.js","sourceRoot":"","sources":["../../../src/scrapers/claude-code.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAOhD,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAEhE,MAAM,QAAQ,GAA4C;IACxD,KAAK,EAAE,MAAM;IACb,SAAS,EAAE,WAAW;IACtB,MAAM,EAAE,QAAQ;IAChB,QAAQ,EAAE,MAAM;IAChB,WAAW,EAAE,MAAM;CACpB,CAAC;AAEF,MAAM,OAAO,iBAAiB;IAKT;IAJV,IAAI,GAAG,aAAa,CAAC;IACb,YAAY,CAAsB;IAEnD,YACmB,iBAAyB,EAC1C,QAAgB;QADC,sBAAiB,GAAjB,iBAAiB,CAAQ;QAG1C,IAAI,CAAC,YAAY,GAAG,IAAI,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,MAAM;QACV,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACnD,OAAO,OAAO,CAAC,WAAW,EAAE,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,aAAa;QACX,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,KAAY;QACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,KAAK,IAAI,KAAK,CAAC,aAAa,CAAC;QAC5C,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,CAAC,QAAQ;QACb,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,QAAQ,CAAC,GAAY;QACnB,MAAM,GAAG,GAAG,GAA8B,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,IAAI,CAAE,GAAG,CAAC,SAAoB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACpE,MAAM,OAAO,GAAI,GAAG,CAAC,OAAkB,IAAI,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAI,GAAG,CAAC,IAAe,IAAI,SAAS,CAAC;QAE/C,OAAO;YACL,IAAI,EAAE,aAAa;YACnB,SAAS,EAAG,GAAG,CAAC,SAAoB,IAAI,SAAS;YACjD,SAAS;YACT,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,QAAQ;YAChC,OAAO;YACP,QAAQ,EAAE;gBACR,YAAY,EAAG,GAAG,CAAC,YAAuB,IAAI,CAAC;gBAC/C,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC;gBACtC,eAAe,EAAE,EAAE;gBACnB,SAAS,EAAE,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,IAAe,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;gBACzE,OAAO,EAAE,GAAG,CAAC,OAA6B;gBAC1C,WAAW,EAAE,aAAa;aACoB;SACjD,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,KAAmB;QAC3C,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,CAAC,eAAe,CAAC,KAAW;QACxC,IAAI,WAAqB,CAAC;QAE1B,IAAI,CAAC;YACH,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QAED,KAAK,MAAM,WAAW,IAAI,WAAW,EAAE,CAAC;YACtC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;YAC7D,IAAI,KAAe,CAAC;YAEpB,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7B,SAAS;gBACX,CAAC;gBAED,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;gBAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBACxC,MAAM,MAAM,GAAG,eAAe,CAAC;oBAC7B,KAAK,EAAE,gBAAgB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;oBACvD,SAAS,EAAE,QAAQ;iBACpB,CAAC,CAAC;gBAEH,IAAI,YAAY,GAAG,CAAC,CAAC;gBACrB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;oBAChC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;wBACjB,SAAS;oBACX,CAAC;oBAED,IAAI,CAAC;wBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;wBACxD,MAAM,SAAS,GAAG,IAAI,IAAI,CAAE,GAAG,CAAC,SAAoB,IAAI,CAAC,CAAC,CAAC;wBAE3D,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;4BACvB,YAAY,IAAI,CAAC,CAAC;4BAClB,SAAS;wBACX,CAAC;wBAED,GAAG,CAAC,SAAS,GAAG,SAAS,CAAC;wBAC1B,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC;wBAChC,YAAY,IAAI,CAAC,CAAC;wBAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBAC3B,CAAC;oBAAC,MAAM,CAAC;wBACP,wBAAwB;oBAC1B,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/src/scrapers/codex.d.ts

@@ -0,0 +1,17 @@
+import type { CodexChunk, ConversationScraper, ScraperState } from "../types/scraper.js";
+export declare class CodexCliScraper implements ConversationScraper<CodexChunk> {
+    private readonly codexSessionsPath;
+    readonly tool = "codex";
+    private readonly stateManager;
+    constructor(codexSessionsPath: string, stateDir: string);
+    detect(): Promise<boolean>;
+    getStorePaths(): string[];
+    scrape(since?: Date): AsyncIterable<CodexChunk>;
+    fullSync(): AsyncIterable<CodexChunk>;
+    parseRaw(raw: unknown): CodexChunk;
+    getLastScrapedPosition(): Promise<ScraperState>;
+    saveScrapedPosition(state: ScraperState): Promise<void>;
+    private readAllSessions;
+    private resolveJsonlFiles;
+}
+//# sourceMappingURL=codex.d.ts.map

package/dist/src/scrapers/codex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex.d.ts","sourceRoot":"","sources":["../../../src/scrapers/codex.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAazF,qBAAa,eAAgB,YAAW,mBAAmB,CAAC,UAAU,CAAC;IAKnE,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IAJpC,QAAQ,CAAC,IAAI,WAAW;IACxB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsB;gBAGhC,iBAAiB,EAAE,MAAM,EAC1C,QAAQ,EAAE,MAAM;IAKZ,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAShC,aAAa,IAAI,MAAM,EAAE;IAIlB,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC;IAM/C,QAAQ,IAAI,aAAa,CAAC,UAAU,CAAC;IAI5C,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,UAAU;IAqB5B,sBAAsB,IAAI,OAAO,CAAC,YAAY,CAAC;IAI/C,mBAAmB,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;YAI9C,eAAe;YAuChB,iBAAiB;CAehC"}

package/dist/src/scrapers/codex.js

@@ -0,0 +1,179 @@
+import { createReadStream } from "node:fs";
+import { readdir, stat } from "node:fs/promises";
+import { basename, join } from "node:path";
+import { createInterface } from "node:readline";
+import { estimateTokens, ScraperStateManager } from "./base.js";
+const ROLE_MAP = {
+    user: "user",
+    human: "user",
+    assistant: "assistant",
+    system: "system",
+    tool: "tool",
+};
+export class CodexCliScraper {
+    codexSessionsPath;
+    tool = "codex";
+    stateManager;
+    constructor(codexSessionsPath, stateDir) {
+        this.codexSessionsPath = codexSessionsPath;
+        this.stateManager = new ScraperStateManager(stateDir);
+    }
+    async detect() {
+        try {
+            const target = await stat(this.codexSessionsPath);
+            return target.isDirectory() || target.isFile();
+        }
+        catch {
+            return false;
+        }
+    }
+    getStorePaths() {
+        return [this.codexSessionsPath];
+    }
+    async *scrape(since) {
+        const state = await this.getLastScrapedPosition();
+        const cutoff = since ?? state.lastTimestamp;
+        yield* this.readAllSessions(cutoff);
+    }
+    async *fullSync() {
+        yield* this.readAllSessions(new Date(0));
+    }
+    parseRaw(raw) {
+        const value = raw;
+        const content = toStringValue(value.content) ?? "";
+        const role = normalizeRole(toStringValue(value.role) ?? toStringValue(value.type));
+        return {
+            tool: "codex",
+            sessionId: toStringValue(value.sessionId) ?? "unknown",
+            timestamp: toDate(value.timestamp),
+            role,
+            content,
+            metadata: {
+                messageIndex: toMessageIndex(value.messageIndex),
+                tokenEstimate: estimateTokens(content),
+                referencedFiles: [],
+                approvalMode: normalizeApprovalMode(toStringValue(value.approvalMode)),
+                sandboxed: toBoolean(value.sandboxed),
+            },
+        };
+    }
+    async getLastScrapedPosition() {
+        return this.stateManager.load(this.tool);
+    }
+    async saveScrapedPosition(state) {
+        await this.stateManager.save(this.tool, state);
+    }
+    async *readAllSessions(since) {
+        const files = await this.resolveJsonlFiles();
+        for (const filePath of files) {
+            const sessionId = inferSessionId(filePath);
+            const reader = createInterface({
+                input: createReadStream(filePath, { encoding: "utf8" }),
+                crlfDelay: Infinity,
+            });
+            let messageIndex = 0;
+            for await (const line of reader) {
+                if (!line.trim()) {
+                    continue;
+                }
+                try {
+                    const parsed = JSON.parse(line);
+                    const timestamp = toDate(parsed.timestamp ?? parsed.created_at ?? parsed.createdAt);
+                    if (timestamp <= since) {
+                        messageIndex += 1;
+                        continue;
+                    }
+                    yield this.parseRaw({
+                        ...parsed,
+                        sessionId: parsed.sessionId ?? parsed.session_id ?? sessionId,
+                        messageIndex,
+                        timestamp,
+                        approvalMode: parsed.approvalMode ?? parsed.approval_mode,
+                    });
+                    messageIndex += 1;
+                }
+                catch {
+                    // Skip malformed lines.
+                }
+            }
+        }
+    }
+    async resolveJsonlFiles() {
+        try {
+            const target = await stat(this.codexSessionsPath);
+            if (target.isFile()) {
+                return this.codexSessionsPath.endsWith(".jsonl") ? [this.codexSessionsPath] : [];
+            }
+            const files = await readdir(this.codexSessionsPath);
+            return files
+                .filter((file) => file.endsWith(".jsonl"))
+                .map((file) => join(this.codexSessionsPath, file));
+        }
+        catch {
+            return [];
+        }
+    }
+}
+function normalizeRole(value) {
+    if (!value) {
+        return "system";
+    }
+    return ROLE_MAP[value.toLowerCase()] ?? "system";
+}
+function normalizeApprovalMode(value) {
+    switch (value) {
+        case "auto-edit":
+            return "auto-edit";
+        case "full-auto":
+            return "full-auto";
+        default:
+            return "suggest";
+    }
+}
+function inferSessionId(filePath) {
+    return basename(filePath).replace(".jsonl", "") || "unknown";
+}
+function toStringValue(value) {
+    if (typeof value !== "string") {
+        return undefined;
+    }
+    return value;
+}
+function toMessageIndex(value) {
+    const parsed = Number(value);
+    if (Number.isFinite(parsed) && parsed >= 0) {
+        return Math.floor(parsed);
+    }
+    return 0;
+}
+function toBoolean(value) {
+    if (typeof value === "boolean") {
+        return value;
+    }
+    if (typeof value === "string") {
+        return value.toLowerCase() === "true";
+    }
+    return false;
+}
+function toDate(value) {
+    if (value instanceof Date) {
+        return value;
+    }
+    if (typeof value === "number" && Number.isFinite(value)) {
+        const millis = value > 10_000_000_000 ? value : value * 1000;
+        return new Date(millis);
+    }
+    if (typeof value === "string") {
+        const numeric = Number(value);
+        if (Number.isFinite(numeric) && value.trim() !== "") {
+            const millis = numeric > 10_000_000_000 ? numeric : numeric * 1000;
+            return new Date(millis);
+        }
+        const parsed = new Date(value);
+        if (!Number.isNaN(parsed.getTime())) {
+            return parsed;
+        }
+    }
+    return new Date(0);
+}
+//# sourceMappingURL=codex.js.map

package/dist/src/scrapers/codex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"codex.js","sourceRoot":"","sources":["../../../src/scrapers/codex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAEhE,MAAM,QAAQ,GAAuC;IACnD,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,MAAM;IACb,SAAS,EAAE,WAAW;IACtB,MAAM,EAAE,QAAQ;IAChB,IAAI,EAAE,MAAM;CACb,CAAC;AAIF,MAAM,OAAO,eAAe;IAKP;IAJV,IAAI,GAAG,OAAO,CAAC;IACP,YAAY,CAAsB;IAEnD,YACmB,iBAAyB,EAC1C,QAAgB;QADC,sBAAiB,GAAjB,iBAAiB,CAAQ;QAG1C,IAAI,CAAC,YAAY,GAAG,IAAI,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,MAAM;QACV,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAClD,OAAO,MAAM,CAAC,WAAW,EAAE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,aAAa;QACX,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,KAAY;QACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,KAAK,IAAI,KAAK,CAAC,aAAa,CAAC;QAC5C,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,CAAC,QAAQ;QACb,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,QAAQ,CAAC,GAAY;QACnB,MAAM,KAAK,GAAG,GAA8B,CAAC;QAC7C,MAAM,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACnD,MAAM,IAAI,GAAG,aAAa,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAEnF,OAAO;YACL,IAAI,EAAE,OAAO;YACb,SAAS,EAAE,aAAa,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,SAAS;YACtD,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;YAClC,IAAI;YACJ,OAAO;YACP,QAAQ,EAAE;gBACR,YAAY,EAAE,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC;gBAChD,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC;gBACtC,eAAe,EAAE,EAAE;gBACnB,YAAY,EAAE,qBAAqB,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBACtE,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC;aACtC;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB;QAC1B,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,KAAmB;QAC3C,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,CAAC,eAAe,CAAC,KAAW;QACxC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE7C,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,eAAe,CAAC;gBAC7B,KAAK,EAAE,gBAAgB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;gBACvD,SAAS,EAAE,QAAQ;aACpB,CAAC,CAAC;YAEH,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;oBACjB,SAAS;gBACX,CAAC;gBAED,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;oBAC3D,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;oBACpF,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;wBACvB,YAAY,IAAI,CAAC,CAAC;wBAClB,SAAS;oBACX,CAAC;oBAED,MAAM,IAAI,CAAC,QAAQ,CAAC;wBAClB,GAAG,MAAM;wBACT,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,UAAU,IAAI,SAAS;wBAC7D,YAAY;wBACZ,SAAS;wBACT,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,aAAa;qBAC1D,CAAC,CAAC;oBACH,YAAY,IAAI,CAAC,CAAC;gBACpB,CAAC;gBAAC,MAAM,CAAC;oBACP,wBAAwB;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAClD,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpB,OAAO,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACnF,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACpD,OAAO,KAAK;iBACT,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;iBACzC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,QAAQ,CAAC;AACnD,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,WAAW;YACd,OAAO,WAAW,CAAC;QACrB,KAAK,WAAW;YACd,OAAO,WAAW,CAAC;QACrB;YACE,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,QAAgB;IACtC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC;AAC/D,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;IACxC,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACxD,MAAM,MAAM,GAAG,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC;QAC7D,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACpD,MAAM,MAAM,GAAG,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC;YACnE,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YACpC,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;AACrB,CAAC"}