xtctx 0.3.2 → 0.11.3

package/dist/src/scrapers/copilot-cli.js

@@ -0,0 +1,342 @@
+import { createReadStream } from "node:fs";
+import { readdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { createInterface } from "node:readline";
+import { pathMatchesProject } from "../utils/project-scope.js";
+import { AbstractScraper, estimateTokens, toDate } from "./base.js";
+const SCRAPER_NAME = "copilot-cli";
+/**
+ * Mutation shapes the copilot-cli scraper tolerates silently. Anything
+ * outside this whitelist that drops records must warn.
+ */
+export const ACCEPTED_DEGRADATIONS = {
+    /** Sessions root missing — Copilot CLI not installed. */
+    missingSessionRoot: "~/.copilot/session-state directory absent",
+    /** Session directory has no events.jsonl file yet — fresh session. */
+    missingEventsJsonl: "session directory has no events.jsonl",
+    /** Blank line in events.jsonl. */
+    blankJsonlLine: "blank line in events.jsonl",
+    /** Event with no extractable role — many event types (status, tool_call,
+     *  subagent meta, etc.) legitimately have no role. */
+    noRole: "event has no role — non-conversation event",
+    /** Event with no extractable text content. */
+    noContent: "event has role but no text content",
+    /** Forward-compat unknown keys alongside known fields. */
+    unknownFieldsAlongside: "extra keys alongside known event schema",
+};
+const ROLE_MAP = {
+    user: "user",
+    human: "user",
+    assistant: "assistant",
+    ai: "assistant",
+    system: "system",
+    tool: "tool",
+};
+/**
+ * Current Copilot CLI writes typed events ("user.message",
+ * "assistant.message", "system.message") whose payload lives under `data`.
+ * The event type itself carries the role.
+ */
+const EVENT_TYPE_ROLE_MAP = {
+    "user.message": "user",
+    "assistant.message": "assistant",
+    "system.message": "system",
+};
+function warnDrift(sourcePath, surprise, recordsAffected) {
+    console.warn(`[${SCRAPER_NAME}] schema-drift surprise at ${sourcePath}: ${surprise} ` +
+        `(records affected: ${recordsAffected})`);
+}
+export class CopilotCliScraper extends AbstractScraper {
+    sessionStateDir;
+    projectRoot;
+    tool = "copilot-cli";
+    constructor(sessionStateDir, stateDir, projectRoot) {
+        super(stateDir);
+        this.sessionStateDir = sessionStateDir;
+        this.projectRoot = projectRoot;
+    }
+    async detect() {
+        try {
+            const target = await stat(this.sessionStateDir);
+            return target.isDirectory();
+        }
+        catch {
+            return false;
+        }
+    }
+    getStorePaths() {
+        return [this.sessionStateDir];
+    }
+    async *scrape(since) {
+        const state = await this.getLastScrapedPosition();
+        const cutoff = since ?? state.lastTimestamp;
+        yield* this.readAllSessions(cutoff);
+    }
+    async *fullSync() {
+        yield* this.readAllSessions(new Date(0));
+    }
+    async *readAllSessions(since) {
+        let sessionDirs;
+        try {
+            sessionDirs = await readdir(this.sessionStateDir);
+        }
+        catch {
+            // ACCEPTED_DEGRADATIONS.missingSessionRoot
+            return;
+        }
+        for (const sessionId of sessionDirs) {
+            const sessionDir = join(this.sessionStateDir, sessionId);
+            let dirStat;
+            try {
+                dirStat = await stat(sessionDir);
+            }
+            catch {
+                continue;
+            }
+            if (!dirStat.isDirectory())
+                continue;
+            const eventsPath = join(sessionDir, "events.jsonl");
+            try {
+                const eventsStat = await stat(eventsPath);
+                if (!eventsStat.isFile())
+                    continue;
+            }
+            catch {
+                // ACCEPTED_DEGRADATIONS.missingEventsJsonl
+                continue;
+            }
+            try {
+                yield* this.readEventsFile(eventsPath, sessionId, since);
+            }
+            catch (err) {
+                // One unreadable session must not abort the remaining sessions.
+                warnDrift(eventsPath, `unreadable transcript file: ${err.message}`, 0);
+            }
+        }
+    }
+    async *readEventsFile(filePath, sessionId, since) {
+        const reader = createInterface({
+            input: createReadStream(filePath, { encoding: "utf8" }),
+            crlfDelay: Infinity,
+        });
+        let messageIndex = 0;
+        let lineNo = 0;
+        // null = no session.start context seen yet; only consulted when scoped.
+        let projectMatch = null;
+        for await (const line of reader) {
+            lineNo++;
+            if (!line.trim()) {
+                // ACCEPTED_DEGRADATIONS.blankJsonlLine
+                continue;
+            }
+            let event;
+            try {
+                event = JSON.parse(line);
+            }
+            catch (err) {
+                warnDrift(`${filePath}:${lineNo}`, `events.jsonl line is not valid JSON: ${err.message}`, 1);
+                continue;
+            }
+            if (!isRecord(event)) {
+                warnDrift(`${filePath}:${lineNo}`, `events.jsonl line is not an object (got ${describeType(event)})`, 1);
+                continue;
+            }
+            if (this.projectRoot && event.type === "session.start") {
+                projectMatch = sessionStartMatchesProject(event, this.projectRoot);
+                if (projectMatch === false) {
+                    // Another project's session; nothing in this file belongs here.
+                    return;
+                }
+                continue;
+            }
+            const role = extractRole(event);
+            const content = extractContent(event);
+            if (!role) {
+                // If the event looks like a conversation message (has extractable
+                // content AND a 'message' or 'role'-shaped sibling that we COULDN'T
+                // route), warn — likely a rename. Routine non-conversation events
+                // (status, tool_call, etc.) carry no content and are silent.
+                if (content) {
+                    const looksLikeMessage = event.type === "message" ||
+                        "role" in event ||
+                        (isRecord(event.message) && "role" in event.message);
+                    if (looksLikeMessage) {
+                        warnDrift(`${filePath}:${lineNo}`, "event has content but no readable role — likely role-field rename", 1);
+                    }
+                }
+                // ACCEPTED_DEGRADATIONS.noRole — non-conversation events are
+                // routine in events.jsonl. Skip silently.
+                continue;
+            }
+            if (!content) {
+                // Role is present but no content extractable. If a 'content' key
+                // exists in an unexpected shape (null, number, etc.), warn — silent
+                // skip would hide a content-field retype. A genuinely-missing content
+                // key on a role'd event is unusual but not necessarily drift.
+                if ("content" in event && typeof event.content !== "string" && !Array.isArray(event.content)) {
+                    warnDrift(`${filePath}:${lineNo}`, `event has role but 'content' is unexpected type ${describeType(event.content)}`, 1);
+                }
+                // ACCEPTED_DEGRADATIONS.noContent
+                continue;
+            }
+            if (this.projectRoot && projectMatch === null) {
+                // Fail closed: without session.start context this session cannot be
+                // attributed to a project, so scoped indexing must not include it.
+                warnDrift(filePath, "session has no session.start context; cannot attribute to a project — skipped under project scoping", 1);
+                return;
+            }
+            const tsValue = event.timestamp ?? event.created_at ?? event.createdAt ?? event.time;
+            const timestamp = toDate(tsValue);
+            if (since.getTime() > 0 && timestamp <= since) {
+                messageIndex++;
+                continue;
+            }
+            const eventType = typeof event.type === "string" ? event.type : undefined;
+            yield {
+                tool: "copilot-cli",
+                sessionId,
+                timestamp,
+                role,
+                content,
+                metadata: {
+                    messageIndex,
+                    tokenEstimate: estimateTokens(content),
+                    referencedFiles: [],
+                    eventType,
+                },
+            };
+            messageIndex++;
+        }
+    }
+}
+function extractRole(event) {
+    // Try event.role first.
+    if (typeof event.role === "string") {
+        const mapped = ROLE_MAP[event.role.toLowerCase()];
+        if (mapped)
+            return mapped;
+    }
+    // Try event.message.role.
+    const message = event.message;
+    if (isRecord(message) && typeof message.role === "string") {
+        const mapped = ROLE_MAP[message.role.toLowerCase()];
+        if (mapped)
+            return mapped;
+    }
+    // Typed events: event.data.role, then the role implied by the event type.
+    const data = event.data;
+    if (isRecord(data) && typeof data.role === "string") {
+        const mapped = ROLE_MAP[data.role.toLowerCase()];
+        if (mapped)
+            return mapped;
+    }
+    if (typeof event.type === "string") {
+        const mapped = EVENT_TYPE_ROLE_MAP[event.type];
+        if (mapped)
+            return mapped;
+    }
+    return null;
+}
+function sessionStartMatchesProject(event, projectRoot) {
+    const data = event.data;
+    if (!isRecord(data))
+        return false;
+    const context = data.context;
+    if (!isRecord(context))
+        return false;
+    const candidates = [context.cwd, context.gitRoot].filter((value) => typeof value === "string" && value.length > 0);
+    return candidates.some((candidate) => pathMatchesProject(candidate, projectRoot));
+}
+function extractContent(event) {
+    // 1. event.content as string.
+    if (typeof event.content === "string") {
+        const trimmed = event.content.trim();
+        if (trimmed.length > 0)
+            return trimmed;
+    }
+    // 2. event.text as string.
+    if (typeof event.text === "string") {
+        const trimmed = event.text.trim();
+        if (trimmed.length > 0)
+            return trimmed;
+    }
+    // 3. event.message.content as string or array of {type, text}.
+    const message = event.message;
+    if (isRecord(message)) {
+        const inner = message.content;
+        if (typeof inner === "string") {
+            const trimmed = inner.trim();
+            if (trimmed.length > 0)
+                return trimmed;
+        }
+        if (Array.isArray(inner)) {
+            const texts = inner
+                .map((item) => {
+                if (typeof item === "string")
+                    return item;
+                if (isRecord(item) && typeof item.text === "string")
+                    return item.text;
+                return "";
+            })
+                .filter((t) => t.length > 0);
+            const joined = texts.join("\n").trim();
+            if (joined.length > 0)
+                return joined;
+        }
+    }
+    // 4. event.content as array of {type, text}.
+    if (Array.isArray(event.content)) {
+        const texts = event.content
+            .map((item) => {
+            if (typeof item === "string")
+                return item;
+            if (isRecord(item) && typeof item.text === "string")
+                return item.text;
+            return "";
+        })
+            .filter((t) => t.length > 0);
+        const joined = texts.join("\n").trim();
+        if (joined.length > 0)
+            return joined;
+    }
+    // 5. Typed events: event.data.content / event.data.text.
+    const data = event.data;
+    if (isRecord(data)) {
+        if (typeof data.content === "string") {
+            const trimmed = data.content.trim();
+            if (trimmed.length > 0)
+                return trimmed;
+        }
+        if (Array.isArray(data.content)) {
+            const texts = data.content
+                .map((item) => {
+                if (typeof item === "string")
+                    return item;
+                if (isRecord(item) && typeof item.text === "string")
+                    return item.text;
+                return "";
+            })
+                .filter((t) => t.length > 0);
+            const joined = texts.join("\n").trim();
+            if (joined.length > 0)
+                return joined;
+        }
+        if (typeof data.text === "string") {
+            const trimmed = data.text.trim();
+            if (trimmed.length > 0)
+                return trimmed;
+        }
+    }
+    return undefined;
+}
+function isRecord(value) {
+    return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+function describeType(value) {
+    if (value === null)
+        return "null";
+    if (Array.isArray(value))
+        return "array";
+    return typeof value;
+}
+//# sourceMappingURL=copilot-cli.js.map

package/dist/src/scrapers/copilot-cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"copilot-cli.js","sourceRoot":"","sources":["../../../src/scrapers/copilot-cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAEpE,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,yDAAyD;IACzD,kBAAkB,EAAE,2CAA2C;IAC/D,sEAAsE;IACtE,kBAAkB,EAAE,uCAAuC;IAC3D,kCAAkC;IAClC,cAAc,EAAE,4BAA4B;IAC5C;0DACsD;IACtD,MAAM,EAAE,4CAA4C;IACpD,8CAA8C;IAC9C,SAAS,EAAE,oCAAoC;IAC/C,0DAA0D;IAC1D,sBAAsB,EAAE,yCAAyC;CAClE,CAAC;AAEF,MAAM,QAAQ,GAA4C;IACxD,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,MAAM;IACb,SAAS,EAAE,WAAW;IACtB,EAAE,EAAE,WAAW;IACf,MAAM,EAAE,QAAQ;IAChB,IAAI,EAAE,MAAM;CACb,CAAC;AAEF;;;;GAIG;AACH,MAAM,mBAAmB,GAA4C;IACnE,cAAc,EAAE,MAAM;IACtB,mBAAmB,EAAE,WAAW;IAChC,gBAAgB,EAAE,QAAQ;CAC3B,CAAC;AAEF,SAAS,SAAS,CAAC,UAAkB,EAAE,QAAgB,EAAE,eAAuB;IAC9E,OAAO,CAAC,IAAI,CACV,IAAI,YAAY,8BAA8B,UAAU,KAAK,QAAQ,GAAG;QACtE,sBAAsB,eAAe,GAAG,CAC3C,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,iBAAkB,SAAQ,eAAgC;IAIlD;IAEA;IALV,IAAI,GAAG,aAAa,CAAC;IAE9B,YACmB,eAAuB,EACxC,QAAgB,EACC,WAAoB;QAErC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAJC,oBAAe,GAAf,eAAe,CAAQ;QAEvB,gBAAW,GAAX,WAAW,CAAS;IAGvC,CAAC;IAED,KAAK,CAAC,MAAM;QACV,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAChD,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,aAAa;QACX,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,KAAY;QACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,KAAK,IAAI,KAAK,CAAC,aAAa,CAAC;QAC5C,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,CAAC,QAAQ;QACb,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,CAAC,eAAe,CAAC,KAAW;QACxC,IAAI,WAAqB,CAAC;QAC1B,IAAI,CAAC;YACH,WAAW,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACP,2CAA2C;YAC3C,OAAO;QACT,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,WAAW,EAAE,CAAC;YACpC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;YACzD,IAAI,OAAO,CAAC;YACZ,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;YACnC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;gBAAE,SAAS;YAErC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;YACpD,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC1C,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE;oBAAE,SAAS;YACrC,CAAC;YAAC,MAAM,CAAC;gBACP,2CAA2C;gBAC3C,SAAS;YACX,CAAC;YAED,IAAI,CAAC;gBACH,KAAK,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YAC3D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,gEAAgE;gBAChE,SAAS,CAAC,UAAU,EAAE,+BAAgC,GAAa,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YACpF,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,CAAC,cAAc,CAC3B,QAAgB,EAChB,SAAiB,EACjB,KAAW;QAEX,MAAM,MAAM,GAAG,eAAe,CAAC;YAC7B,KAAK,EAAE,gBAAgB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;YACvD,SAAS,EAAE,QAAQ;SACpB,CAAC,CAAC;QAEH,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,wEAAwE;QACxE,IAAI,YAAY,GAAmB,IAAI,CAAC;QAExC,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;YAChC,MAAM,EAAE,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACjB,uCAAuC;gBACvC,SAAS;YACX,CAAC;YAED,IAAI,KAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;YACtD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,CACP,GAAG,QAAQ,IAAI,MAAM,EAAE,EACvB,wCAAyC,GAAa,CAAC,OAAO,EAAE,EAChE,CAAC,CACF,CAAC;gBACF,SAAS;YACX,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACrB,SAAS,CACP,GAAG,QAAQ,IAAI,MAAM,EAAE,EACvB,2CAA2C,YAAY,CAAC,KAAK,CAAC,GAAG,EACjE,CAAC,CACF,CAAC;gBACF,SAAS;YACX,CAAC;YAED,IAAI,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;gBACvD,YAAY,GAAG,0BAA0B,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;gBACnE,IAAI,YAAY,KAAK,KAAK,EAAE,CAAC;oBAC3B,gEAAgE;oBAChE,OAAO;gBACT,CAAC;gBACD,SAAS;YACX,CAAC;YAED,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;YAChC,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YAEtC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,kEAAkE;gBAClE,oEAAoE;gBACpE,kEAAkE;gBAClE,6DAA6D;gBAC7D,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,gBAAgB,GACpB,KAAK,CAAC,IAAI,KAAK,SAAS;wBACxB,MAAM,IAAI,KAAK;wBACf,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;oBACvD,IAAI,gBAAgB,EAAE,CAAC;wBACrB,SAAS,CACP,GAAG,QAAQ,IAAI,MAAM,EAAE,EACvB,mEAAmE,EACnE,CAAC,CACF,CAAC;oBACJ,CAAC;gBACH,CAAC;gBACD,6DAA6D;gBAC7D,0CAA0C;gBAC1C,SAAS;YACX,CAAC;YAED,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,iEAAiE;gBACjE,oEAAoE;gBACpE,sEAAsE;gBACtE,8DAA8D;gBAC9D,IAAI,SAAS,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC7F,SAAS,CACP,GAAG,QAAQ,IAAI,MAAM,EAAE,EACvB,mDAAmD,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,EAChF,CAAC,CACF,CAAC;gBACJ,CAAC;gBACD,kCAAkC;gBAClC,SAAS;YACX,CAAC;YAED,IAAI,IAAI,CAAC,WAAW,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;gBAC9C,oEAAoE;gBACpE,mEAAmE;gBACnE,SAAS,CACP,QAAQ,EACR,qGAAqG,EACrG,CAAC,CACF,CAAC;gBACF,OAAO;YACT,CAAC;YAED,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC;YACrF,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;gBAC9C,YAAY,EAAE,CAAC;gBACf,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YAE1E,MAAM;gBACJ,IAAI,EAAE,aAAa;gBACnB,SAAS;gBACT,SAAS;gBACT,IAAI;gBACJ,OAAO;gBACP,QAAQ,EAAE;oBACR,YAAY;oBACZ,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC;oBACtC,eAAe,EAAE,EAAE;oBACnB,SAAS;iBACV;aACF,CAAC;YACF,YAAY,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;CACF;AAED,SAAS,WAAW,CAAC,KAA8B;IACjD,wBAAwB;IACxB,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAClD,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;IAC5B,CAAC;IAED,0BAA0B;IAC1B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC9B,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1D,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACpD,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;IAC5B,CAAC;IAED,0EAA0E;IAC1E,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxB,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACjD,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;IAC5B,CAAC;IACD,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;IAC5B,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,0BAA0B,CACjC,KAA8B,EAC9B,WAAmB;IAEnB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC7B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,UAAU,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CACtD,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAC1E,CAAC;IACF,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;AACpF,CAAC;AAED,SAAS,cAAc,CAAC,KAA8B;IACpD,8BAA8B;IAC9B,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,OAAO,CAAC;IACzC,CAAC;IAED,2BAA2B;IAC3B,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,OAAO,CAAC;IACzC,CAAC;IAED,+DAA+D;IAC/D,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC9B,IAAI,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC;QAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,OAAO,CAAC;QACzC,CAAC;QACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,KAAK;iBAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,OAAO,IAAI,KAAK,QAAQ;oBAAE,OAAO,IAAI,CAAC;gBAC1C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;oBAAE,OAAO,IAAI,CAAC,IAAI,CAAC;gBACtE,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;iBACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,MAAM,CAAC;QACvC,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO;aACxB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,IAAI,OAAO,IAAI,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAC;YAC1C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAC,IAAI,CAAC;YACtE,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,MAAM,CAAC;IACvC,CAAC;IAED,yDAAyD;IACzD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxB,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,OAAO,CAAC;QACzC,CAAC;QACD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO;iBACvB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,IAAI,OAAO,IAAI,KAAK,QAAQ;oBAAE,OAAO,IAAI,CAAC;gBAC1C,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;oBAAE,OAAO,IAAI,CAAC,IAAI,CAAC;gBACtE,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;iBACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,MAAM,CAAC;QACvC,CAAC;QACD,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,OAAO,CAAC;QACzC,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IAClC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACzC,OAAO,OAAO,KAAK,CAAC;AACtB,CAAC"}

package/dist/src/scrapers/copilot.d.ts

@@ -1,17 +1,50 @@
-import type { ConversationScraper, CopilotChunk, ScraperState } from "../types/scraper.js";
-export declare class CopilotScraper implements ConversationScraper<CopilotChunk> {
-    private readonly copilotHistoryPath;
+import type { CopilotChunk } from "../types/scraper.js";
+import { AbstractScraper } from "./base.js";
+/**
+ * Shapes that the Copilot scraper tolerates silently without logging.
+ * Each entry documents the shape being accepted and why it is not drift.
+ * Anything not listed here that falls outside the happy path MUST warn
+ * (or throw, for required-schema violations).
+ */
+export declare const ACCEPTED_DEGRADATIONS: {
+    /** Workspace has no state.vscdb yet — pristine VS Code install. */
+    missingStateVscdb: string;
+    /** workspaceStorage root missing — VS Code not installed. */
+    missingWorkspaceStorage: string;
+    /** better-sqlite3 native module absent — Copilot ingestion is opt-in. */
+    missingSqliteBinding: string;
+    /** State.vscdb has no interactive.sessions row — workspace never used Copilot chat. */
+    noInteractiveSessionsKey: string;
+    /** ItemTable shape varies across VS Code versions; tolerate open/query failure. */
+    unreadableItemTable: string;
+    /** A canceled request is intentionally skipped — not drift. */
+    canceledRequest: string;
+    /** Some sessions legitimately have no user-text (agent-only runs); skip silently. */
+    emptyUserText: string;
+    /** Empty assistant response (thinking timeout etc.) — not drift. */
+    emptyAssistantText: string;
+    /** Extra/unknown fields alongside known ones are forward-compatible. */
+    unknownFieldsAlongside: string;
+};
+export declare class CopilotScraper extends AbstractScraper<CopilotChunk> {
+    /** Path to %APPDATA%/Code/User/workspaceStorage (or a test stand-in). */
+    private readonly workspaceStoragePath;
+    private readonly projectRoot?;
     readonly tool = "copilot";
-    private readonly stateManager;
-    constructor(copilotHistoryPath: string, stateDir: string);
+    constructor(
+    /** Path to %APPDATA%/Code/User/workspaceStorage (or a test stand-in). */
+    workspaceStoragePath: string, stateDir: string, projectRoot?: string | undefined);
     detect(): Promise<boolean>;
     getStorePaths(): string[];
     scrape(since?: Date): AsyncIterable<CopilotChunk>;
     fullSync(): AsyncIterable<CopilotChunk>;
     parseRaw(raw: unknown): CopilotChunk;
-    getLastScrapedPosition(): Promise<ScraperState>;
-    saveScrapedPosition(state: ScraperState): Promise<void>;
     private readAllMessages;
-    private resolveJsonFiles;
+    private readFromDb;
+    /**
+     * Discovers VS Code workspaceStorage SQLite databases.
+     * Pattern: &lt;workspaceStoragePath&gt;/[hash]/state.vscdb
+     */
+    private resolveDbPaths;
 }
 //# sourceMappingURL=copilot.d.ts.map

package/dist/src/scrapers/copilot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"copilot.d.ts","sourceRoot":"","sources":["../../../src/scrapers/copilot.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAoB3F,qBAAa,cAAe,YAAW,mBAAmB,CAAC,YAAY,CAAC;IAKpE,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IAJrC,QAAQ,CAAC,IAAI,aAAa;IAC1B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsB;gBAGhC,kBAAkB,EAAE,MAAM,EAC3C,QAAQ,EAAE,MAAM;IAKZ,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAShC,aAAa,IAAI,MAAM,EAAE;IAIlB,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,GAAG,aAAa,CAAC,YAAY,CAAC;IAMjD,QAAQ,IAAI,aAAa,CAAC,YAAY,CAAC;IAI9C,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,YAAY;IAoB9B,sBAAsB,IAAI,OAAO,CAAC,YAAY,CAAC;IAI/C,mBAAmB,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;YAI9C,eAAe;YAmChB,gBAAgB;CAe/B"}
+{"version":3,"file":"copilot.d.ts","sourceRoot":"","sources":["../../../src/scrapers/copilot.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,eAAe,EAA0B,MAAM,WAAW,CAAC;AAQpE;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB;IAChC,mEAAmE;;IAEnE,6DAA6D;;IAE7D,yEAAyE;;IAEzE,uFAAuF;;IAEvF,mFAAmF;;IAEnF,+DAA+D;;IAE/D,qFAAqF;;IAErF,oEAAoE;;IAEpE,wEAAwE;;CAEzE,CAAC;AAyBF,qBAAa,cAAe,SAAQ,eAAe,CAAC,YAAY,CAAC;IAI7D,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,oBAAoB;IAErC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;IAN/B,QAAQ,CAAC,IAAI,aAAa;;IAGxB,yEAAyE;IACxD,oBAAoB,EAAE,MAAM,EAC7C,QAAQ,EAAE,MAAM,EACC,WAAW,CAAC,EAAE,MAAM,YAAA;IAKjC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAShC,aAAa,IAAI,MAAM,EAAE;IAIlB,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,GAAG,aAAa,CAAC,YAAY,CAAC;IAIjD,QAAQ,IAAI,aAAa,CAAC,YAAY,CAAC;IAI9C,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,YAAY;YAoBrB,eAAe;IAmC9B,OAAO,CAAE,UAAU;IAoKnB;;;OAGG;YACW,cAAc;CAkC7B"}