visionclaw 0.1.129 → 0.1.130

package/dist/data-processing/restructure.js

@@ -0,0 +1,276 @@
+/**
+ * Stage 2: Restructure VisionClaw logs into task-oriented records.
+ *
+ * Loads log entries, segments into wake cycles, optionally enriches with
+ * SDK transcripts, and writes three separate JSONL output files:
+ *   - restructured-clean.jsonl
+ *   - restructured-interrupted.jsonl
+ *   - restructured-incomplete.jsonl
+ */
+import fs from "node:fs";
+import path from "node:path";
+import { loadAllEntries } from "./loader.js";
+import { segmentWakeCycles, classifyCycles } from "./segmenter.js";
+import { loadSdkTranscript, segmentSdkWakeCycles, buildSdkCycleIndex, matchSdkCycle, } from "./sdk-transcript.js";
+import { buildRecord } from "./record-builder.js";
+function parseRestructureArgs(args) {
+    let logsDir = "";
+    let transcriptDir;
+    let outputDir = ".";
+    let samples = false;
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+        if ((arg === "-t" || arg === "--transcript-dir") && i + 1 < args.length) {
+            transcriptDir = args[++i];
+        }
+        else if ((arg === "-o" || arg === "--output-dir") && i + 1 < args.length) {
+            outputDir = args[++i];
+        }
+        else if (arg === "--samples") {
+            samples = true;
+        }
+        else if (!arg.startsWith("-") && !logsDir) {
+            logsDir = arg;
+        }
+    }
+    if (!logsDir) {
+        console.error("Usage: visionclaw restructure-logs <logs-dir> [-t transcript-dir] [-o output-dir] [--samples]");
+        process.exit(1);
+    }
+    return { logsDir, transcriptDir, outputDir, samples };
+}
+// ---------------------------------------------------------------------------
+// JSONL writer
+// ---------------------------------------------------------------------------
+function writeJsonl(filePath, records) {
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    const fd = fs.openSync(filePath, "w");
+    try {
+        for (const rec of records) {
+            const line = JSON.stringify(rec);
+            fs.writeSync(fd, line + "\n");
+        }
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+}
+function writeSamples(filePath, records, maxSamples = 3) {
+    const samples = records.slice(0, maxSamples);
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.writeFileSync(filePath, JSON.stringify(samples, null, 2));
+}
+// ---------------------------------------------------------------------------
+// Compaction-only detection
+// ---------------------------------------------------------------------------
+function isCompactionOnly(record) {
+    return record.trajectory.some((t) => t.role === "user_message" &&
+        t.content.includes("<local-command-stdout>Compacted"));
+}
+// ---------------------------------------------------------------------------
+// No meaningful work: trajectory has no non-finish tool calls
+// ---------------------------------------------------------------------------
+function lacksRealToolCall(record) {
+    return !record.trajectory.some((t) => t.role === "tool_call" &&
+        t.metadata?.name !== "mcp__visionclaw__finish");
+}
+// ---------------------------------------------------------------------------
+// Process a list of cycles into records
+// ---------------------------------------------------------------------------
+function processCycles(cycles, sdkIndex, sdkCycles, hasTranscript) {
+    let matched = 0;
+    let unmatched = 0;
+    let compactionOnly = 0;
+    let noRealToolCall = 0;
+    const records = [];
+    for (const c of cycles) {
+        let sdkCycle;
+        if (hasTranscript) {
+            sdkCycle = matchSdkCycle(c, sdkIndex, sdkCycles);
+            if (sdkCycle)
+                matched++;
+            else
+                unmatched++;
+        }
+        const record = buildRecord(c, sdkCycle);
+        if (isCompactionOnly(record)) {
+            compactionOnly++;
+            continue;
+        }
+        if (lacksRealToolCall(record)) {
+            noRealToolCall++;
+            continue;
+        }
+        records.push(record);
+    }
+    return { records, matched, unmatched, compactionOnly, noRealToolCall };
+}
+// ---------------------------------------------------------------------------
+// Print stats for a set of records
+// ---------------------------------------------------------------------------
+function printRecordStats(label, records, matched, unmatched, hasTranscript, compactionOnly, noRealToolCall) {
+    if (records.length === 0 && compactionOnly === 0 && noRealToolCall === 0)
+        return;
+    const totalTraj = records.reduce((sum, r) => sum + r.trajectory.length, 0);
+    const withPrompts = records.filter((r) => r.prompt !== "heartbeat").length;
+    const withInterrupts = records.filter((r) => r.has_interrupts).length;
+    console.log(`\n  ${label}:`);
+    console.log(`    Records:              ${records.length}`);
+    if (compactionOnly > 0) {
+        console.log(`    Compaction-only (filtered): ${compactionOnly}`);
+    }
+    if (noRealToolCall > 0) {
+        console.log(`    No real tool call (filtered): ${noRealToolCall}`);
+    }
+    console.log(`    With user prompt:     ${withPrompts}`);
+    if (withInterrupts > 0) {
+        console.log(`    With interrupts:      ${withInterrupts}`);
+    }
+    console.log(`    Total trajectory:     ${totalTraj}`);
+    if (records.length > 0) {
+        console.log(`    Avg trajectory len:   ${(totalTraj / records.length).toFixed(1)}`);
+    }
+    if (hasTranscript) {
+        console.log(`    SDK matched:          ${matched}`);
+        console.log(`    Log fallback:         ${unmatched}`);
+    }
+}
+// ---------------------------------------------------------------------------
+// Pick diverse samples
+// ---------------------------------------------------------------------------
+const MIN_MEANINGFUL_TRAJECTORY = 5;
+function pickDiverseSamples(records, maxSamples = 3) {
+    const samples = [];
+    // 1. With compaction summary + user prompt
+    for (const r of records) {
+        if (r.prompt !== "heartbeat" &&
+            r.prior_context?.compaction_summary &&
+            r.trajectory.length >= MIN_MEANINGFUL_TRAJECTORY) {
+            samples.push(r);
+            break;
+        }
+    }
+    // 2. Interrupted cycle (if any)
+    for (const r of records) {
+        if (r.has_interrupts &&
+            r.interrupts &&
+            r.trajectory.length >= MIN_MEANINGFUL_TRAJECTORY) {
+            samples.push(r);
+            break;
+        }
+    }
+    // 3. Compacted fresh start with real work
+    for (const r of records) {
+        if (samples.includes(r))
+            continue;
+        if (r.prior_context?.compacted_fresh_start &&
+            r.trajectory.length >= MIN_MEANINGFUL_TRAJECTORY) {
+            samples.push(r);
+            break;
+        }
+    }
+    // Fill remaining — prefer records with meaningful trajectories
+    for (const r of records) {
+        if (samples.length >= maxSamples)
+            break;
+        if (!samples.includes(r) && r.trajectory.length >= MIN_MEANINGFUL_TRAJECTORY) {
+            samples.push(r);
+        }
+    }
+    // Last resort: fill from any records if not enough yet
+    for (const r of records) {
+        if (samples.length >= maxSamples)
+            break;
+        if (!samples.includes(r))
+            samples.push(r);
+    }
+    return samples.slice(0, maxSamples);
+}
+// ---------------------------------------------------------------------------
+// Public entry point
+// ---------------------------------------------------------------------------
+export async function runRestructure(args) {
+    const opts = parseRestructureArgs(args);
+    // Load logs
+    const entries = await loadAllEntries(opts.logsDir);
+    // Segment and classify
+    const cycles = segmentWakeCycles(entries);
+    const { clean, interrupted, incomplete } = classifyCycles(cycles);
+    console.log(`\nWake cycles: ${cycles.length}`);
+    console.log(`  Clean (no interrupts, complete):  ${clean.length}`);
+    console.log(`  Interrupted (has injections):      ${interrupted.length}`);
+    console.log(`  Incomplete (no result):            ${incomplete.length}`);
+    const heartbeat = clean.filter((c) => c.trigger === "heartbeat").length;
+    const message = clean.filter((c) => c.trigger === "message").length;
+    console.log(`\nClean cycles breakdown:`);
+    console.log(`  Heartbeat-triggered:  ${heartbeat}`);
+    console.log(`  Message-triggered:    ${message}`);
+    // Load SDK transcript if provided
+    let sdkIndex = new Map();
+    let sdkCyclesList = [];
+    const hasTranscript = !!opts.transcriptDir;
+    if (opts.transcriptDir) {
+        console.log(`\nLoading SDK transcript...`);
+        const sdkMessages = await loadSdkTranscript(opts.transcriptDir);
+        sdkCyclesList = segmentSdkWakeCycles(sdkMessages);
+        sdkIndex = buildSdkCycleIndex(sdkCyclesList);
+        console.log(`  SDK wake cycles: ${sdkCyclesList.length}`);
+    }
+    // Process each category
+    const cleanResult = processCycles(clean, sdkIndex, sdkCyclesList, hasTranscript);
+    const interruptedResult = processCycles(interrupted, sdkIndex, sdkCyclesList, hasTranscript);
+    const incompleteResult = processCycles(incomplete, sdkIndex, sdkCyclesList, hasTranscript);
+    // Print stats
+    const totalMatched = cleanResult.matched + interruptedResult.matched + incompleteResult.matched;
+    const totalUnmatched = cleanResult.unmatched +
+        interruptedResult.unmatched +
+        incompleteResult.unmatched;
+    if (hasTranscript) {
+        console.log(`\nSDK transcript matching:`);
+        console.log(`  Matched:   ${totalMatched}`);
+        console.log(`  Unmatched: ${totalUnmatched} (using log-based trajectory)`);
+    }
+    printRecordStats("Clean", cleanResult.records, cleanResult.matched, cleanResult.unmatched, hasTranscript, cleanResult.compactionOnly, cleanResult.noRealToolCall);
+    printRecordStats("Interrupted", interruptedResult.records, interruptedResult.matched, interruptedResult.unmatched, hasTranscript, interruptedResult.compactionOnly, interruptedResult.noRealToolCall);
+    printRecordStats("Incomplete", incompleteResult.records, incompleteResult.matched, incompleteResult.unmatched, hasTranscript, incompleteResult.compactionOnly, incompleteResult.noRealToolCall);
+    // Write output files
+    const outDir = opts.outputDir;
+    const cleanPath = path.join(outDir, "restructured-clean.jsonl");
+    const interruptedPath = path.join(outDir, "restructured-interrupted.jsonl");
+    const incompletePath = path.join(outDir, "restructured-incomplete.jsonl");
+    writeJsonl(cleanPath, cleanResult.records);
+    writeJsonl(interruptedPath, interruptedResult.records);
+    writeJsonl(incompletePath, incompleteResult.records);
+    console.log(`\nWrote output files to ${outDir}:`);
+    console.log(`  ${cleanPath} (${cleanResult.records.length} records)`);
+    console.log(`  ${interruptedPath} (${interruptedResult.records.length} records)`);
+    console.log(`  ${incompletePath} (${incompleteResult.records.length} records)`);
+    // Write samples if requested
+    if (opts.samples) {
+        const allRecords = [
+            ...cleanResult.records,
+            ...interruptedResult.records,
+            ...incompleteResult.records,
+        ];
+        if (cleanResult.records.length > 0) {
+            const samplesPath = path.join(outDir, "samples-clean.json");
+            writeSamples(samplesPath, pickDiverseSamples(cleanResult.records));
+            console.log(`  ${samplesPath}`);
+        }
+        if (interruptedResult.records.length > 0) {
+            const samplesPath = path.join(outDir, "samples-interrupted.json");
+            writeSamples(samplesPath, pickDiverseSamples(interruptedResult.records));
+            console.log(`  ${samplesPath}`);
+        }
+        if (incompleteResult.records.length > 0) {
+            const samplesPath = path.join(outDir, "samples-incomplete.json");
+            writeSamples(samplesPath, pickDiverseSamples(incompleteResult.records));
+            console.log(`  ${samplesPath}`);
+        }
+        // Also write a combined samples file
+        const combinedPath = path.join(outDir, "samples-all.json");
+        writeSamples(combinedPath, pickDiverseSamples(allRecords));
+        console.log(`  ${combinedPath}`);
+    }
+}
+//# sourceMappingURL=restructure.js.map

package/dist/data-processing/restructure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"restructure.js","sourceRoot":"","sources":["../../src/data-processing/restructure.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAM7B,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,kBAAkB,EAClB,aAAa,GACd,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAalD,SAAS,oBAAoB,CAAC,IAAc;IAC1C,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,aAAiC,CAAC;IACtC,IAAI,SAAS,GAAG,GAAG,CAAC;IACpB,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACxE,aAAa,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,CAAC;aAAM,IAAI,CAAC,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC3E,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,GAAG,KAAK,WAAW,EAAE,CAAC;YAC/B,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;aAAM,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAC5C,OAAO,GAAG,GAAG,CAAC;QAChB,CAAC;IACH,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CACX,+FAA+F,CAChG,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;AACxD,CAAC;AAED,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E,SAAS,UAAU,CAAC,QAAgB,EAAE,OAAuB;IAC3D,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACtC,IAAI,CAAC;QACH,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACjC,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CACnB,QAAgB,EAChB,OAAuB,EACvB,UAAU,GAAG,CAAC;IAEd,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC7C,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,SAAS,gBAAgB,CAAC,MAAoB;IAC5C,OAAO,MAAM,CAAC,UAAU,CAAC,IAAI,CAC3B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,cAAc;QACzB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CACxD,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,8DAA8D;AAC9D,8EAA8E;AAE9E,SAAS,iBAAiB,CAAC,MAAoB;IAC7C,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAC5B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,WAAW;QACtB,CAAC,CAAC,QAAQ,EAAE,IAAI,KAAK,yBAAyB,CACjD,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,wCAAwC;AACxC,8EAA8E;AAE9E,SAAS,aAAa,CACpB,MAAmB,EACnB,QAAmC,EACnC,SAAyB,EACzB,aAAsB;IAEtB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,MAAM,OAAO,GAAmB,EAAE,CAAC;IAEnC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,QAAkC,CAAC;QACvC,IAAI,aAAa,EAAE,CAAC;YAClB,QAAQ,GAAG,aAAa,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;YACjD,IAAI,QAAQ;gBAAE,OAAO,EAAE,CAAC;;gBACnB,SAAS,EAAE,CAAC;QACnB,CAAC;QACD,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACxC,IAAI,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,cAAc,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QACD,IAAI,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,cAAc,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,cAAc,EAAE,CAAC;AACzE,CAAC;AAED,8EAA8E;AAC9E,mCAAmC;AACnC,8EAA8E;AAE9E,SAAS,gBAAgB,CACvB,KAAa,EACb,OAAuB,EACvB,OAAe,EACf,SAAiB,EACjB,aAAsB,EACtB,cAAsB,EACtB,cAAsB;IAEtB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,KAAK,CAAC,IAAI,cAAc,KAAK,CAAC;QAAE,OAAO;IAEjF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAC9B,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EACrC,CAAC,CACF,CAAC;IACF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAChC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAChC,CAAC,MAAM,CAAC;IACT,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC;IAEtE,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,GAAG,CAAC,CAAC;IAC7B,OAAO,CAAC,GAAG,CAAC,6BAA6B,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3D,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,mCAAmC,cAAc,EAAE,CAAC,CAAC;IACnE,CAAC;IACD,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,qCAAqC,cAAc,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAC;IACxD,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,6BAA6B,cAAc,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,6BAA6B,SAAS,EAAE,CAAC,CAAC;IACtD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CACT,6BAA6B,CAAC,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACvE,CAAC;IACJ,CAAC;IACD,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,6BAA6B,OAAO,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,6BAA6B,SAAS,EAAE,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAEpC,SAAS,kBAAkB,CACzB,OAAuB,EACvB,UAAU,GAAG,CAAC;IAEd,MAAM,OAAO,GAAmB,EAAE,CAAC;IAEnC,2CAA2C;IAC3C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IACE,CAAC,CAAC,MAAM,KAAK,WAAW;YACxB,CAAC,CAAC,aAAa,EAAE,kBAAkB;YACnC,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB,EAChD,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,MAAM;QACR,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IACE,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB,EAChD,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,MAAM;QACR,CAAC;IACH,CAAC;IAED,0CAA0C;IAC1C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,SAAS;QAClC,IACE,CAAC,CAAC,aAAa,EAAE,qBAAqB;YACtC,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB,EAChD,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,MAAM;QACR,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;YAAE,MAAM;QACxC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB,EAAE,CAAC;YAC7E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU;YAAE,MAAM;QACxC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;AACtC,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAc;IACjD,MAAM,IAAI,GAAG,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAExC,YAAY;IACZ,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEnD,uBAAuB;IACvB,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAElE,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,uCAAuC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,wCAAwC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1E,OAAO,CAAC,GAAG,CAAC,wCAAwC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAEzE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IACpE,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,2BAA2B,OAAO,EAAE,CAAC,CAAC;IAElD,kCAAkC;IAClC,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,IAAI,aAAa,GAAmB,EAAE,CAAC;IACvC,MAAM,aAAa,GAAG,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;IAE3C,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAChE,aAAa,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC;QAClD,QAAQ,GAAG,kBAAkB,CAAC,aAAa,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,sBAAsB,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,wBAAwB;IACxB,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;IACjF,MAAM,iBAAiB,GAAG,aAAa,CACrC,WAAW,EACX,QAAQ,EACR,aAAa,EACb,aAAa,CACd,CAAC;IACF,MAAM,gBAAgB,GAAG,aAAa,CACpC,UAAU,EACV,QAAQ,EACR,aAAa,EACb,aAAa,CACd,CAAC;IAEF,cAAc;IACd,MAAM,YAAY,GAChB,WAAW,CAAC,OAAO,GAAG,iBAAiB,CAAC,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC;IAC7E,MAAM,cAAc,GAClB,WAAW,CAAC,SAAS;QACrB,iBAAiB,CAAC,SAAS;QAC3B,gBAAgB,CAAC,SAAS,CAAC;IAE7B,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,gBAAgB,YAAY,EAAE,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,gBAAgB,cAAc,+BAA+B,CAAC,CAAC;IAC7E,CAAC;IAED,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,cAAc,EAAE,WAAW,CAAC,cAAc,CAAC,CAAC;IAClK,gBAAgB,CAAC,aAAa,EAAE,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,CAAC,SAAS,EAAE,aAAa,EAAE,iBAAiB,CAAC,cAAc,EAAE,iBAAiB,CAAC,cAAc,CAAC,CAAC;IACtM,gBAAgB,CAAC,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,gBAAgB,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,EAAE,aAAa,EAAE,gBAAgB,CAAC,cAAc,EAAE,gBAAgB,CAAC,cAAc,CAAC,CAAC;IAEhM,qBAAqB;IACrB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;IAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAAC;IAChE,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,gCAAgC,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,+BAA+B,CAAC,CAAC;IAE1E,UAAU,CAAC,SAAS,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IAC3C,UAAU,CAAC,eAAe,EAAE,iBAAiB,CAAC,OAAO,CAAC,CAAC;IACvD,UAAU,CAAC,cAAc,EAAE,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAErD,OAAO,CAAC,GAAG,CAAC,2BAA2B,MAAM,GAAG,CAAC,CAAC;IAClD,OAAO,CAAC,GAAG,CAAC,KAAK,SAAS,KAAK,WAAW,CAAC,OAAO,CAAC,MAAM,WAAW,CAAC,CAAC;IACtE,OAAO,CAAC,GAAG,CACT,KAAK,eAAe,KAAK,iBAAiB,CAAC,OAAO,CAAC,MAAM,WAAW,CACrE,CAAC;IACF,OAAO,CAAC,GAAG,CACT,KAAK,cAAc,KAAK,gBAAgB,CAAC,OAAO,CAAC,MAAM,WAAW,CACnE,CAAC;IAEF,6BAA6B;IAC7B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,MAAM,UAAU,GAAG;YACjB,GAAG,WAAW,CAAC,OAAO;YACtB,GAAG,iBAAiB,CAAC,OAAO;YAC5B,GAAG,gBAAgB,CAAC,OAAO;SAC5B,CAAC;QAEF,IAAI,WAAW,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;YAC5D,YAAY,CAAC,WAAW,EAAE,kBAAkB,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;YACnE,OAAO,CAAC,GAAG,CAAC,KAAK,WAAW,EAAE,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,iBAAiB,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAAC;YAClE,YAAY,CAAC,WAAW,EAAE,kBAAkB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;YACzE,OAAO,CAAC,GAAG,CAAC,KAAK,WAAW,EAAE,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,gBAAgB,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC;YACjE,YAAY,CACV,WAAW,EACX,kBAAkB,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAC7C,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,KAAK,WAAW,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,qCAAqC;QACrC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;QAC3D,YAAY,CAAC,YAAY,EAAE,kBAAkB,CAAC,UAAU,CAAC,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,KAAK,YAAY,EAAE,CAAC,CAAC;IACnC,CAAC;AACH,CAAC"}

package/dist/data-processing/sanitize.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Stage 1: Sanitize VisionClaw JSONL logs in-place.
+ *
+ * Applies:
+ *  1. Remove lines containing noise substrings ([compact], [image-pruner])
+ *  2. Remove lines whose message starts with specific prefixes
+ *  3. Replace leaked API key strings
+ *  4. Redact obs domain URLs
+ *  5. Redact embedded base64 image payloads
+ */
+export declare function runSanitize(args: string[]): void;
+//# sourceMappingURL=sanitize.d.ts.map

package/dist/data-processing/sanitize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sanitize.d.ts","sourceRoot":"","sources":["../../src/data-processing/sanitize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAiMH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAkEhD"}

package/dist/data-processing/sanitize.js

@@ -0,0 +1,229 @@
+/**
+ * Stage 1: Sanitize VisionClaw JSONL logs in-place.
+ *
+ * Applies:
+ *  1. Remove lines containing noise substrings ([compact], [image-pruner])
+ *  2. Remove lines whose message starts with specific prefixes
+ *  3. Replace leaked API key strings
+ *  4. Redact obs domain URLs
+ *  5. Redact embedded base64 image payloads
+ */
+import fs from "node:fs";
+import path from "node:path";
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const APIKEY_OLD = "apiKey=58Io52IKCOO1cir4WKDHNu0Cq9CF4HzQ";
+const APIKEY_NEW = "apiKey=58Io52IKCOO1xxxxxxxxxx";
+const REMOVE_SUBSTRINGS = ["[compact]", "[image-pruner]"];
+const DROP_MESSAGE_PREFIXES = ["Version check", "Image resized for vision"];
+const REDACT_OBS_DOMAIN = "visionclawobs.com";
+const REDACT_OBS_REPLACEMENT = "<obsurl>";
+const REDACTED_IMAGE_DATA = "<base64-redacted>";
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function parseMessageField(line) {
+    const trimmed = line.trim();
+    if (!trimmed)
+        return null;
+    try {
+        const obj = JSON.parse(trimmed);
+        const msg = obj.message;
+        return typeof msg === "string" ? msg : null;
+    }
+    catch {
+        return null;
+    }
+}
+function redactImageBase64InObj(obj) {
+    let redactions = 0;
+    if (Array.isArray(obj)) {
+        for (const item of obj) {
+            redactions += redactImageBase64InObj(item);
+        }
+    }
+    else if (obj !== null && typeof obj === "object") {
+        const record = obj;
+        if (record.type === "image") {
+            const src = record.source;
+            if (src !== null &&
+                typeof src === "object" &&
+                !Array.isArray(src)) {
+                const source = src;
+                if (typeof source.data === "string" && source.data) {
+                    source.data = REDACTED_IMAGE_DATA;
+                    redactions += 1;
+                }
+            }
+        }
+        for (const v of Object.values(record)) {
+            redactions += redactImageBase64InObj(v);
+        }
+    }
+    return redactions;
+}
+function redactImageBase64InEmbeddedJsonStrings(obj) {
+    let redactions = 0;
+    if (Array.isArray(obj)) {
+        for (const item of obj) {
+            redactions += redactImageBase64InEmbeddedJsonStrings(item);
+        }
+    }
+    else if (obj !== null && typeof obj === "object") {
+        const record = obj;
+        for (const [key, val] of Object.entries(record)) {
+            if (typeof val === "string") {
+                const s = val.trim();
+                if (s && (s.startsWith("{") || s.startsWith("["))) {
+                    try {
+                        const embedded = JSON.parse(s);
+                        const r1 = redactImageBase64InObj(embedded);
+                        const r2 = redactImageBase64InEmbeddedJsonStrings(embedded);
+                        if (r1 || r2) {
+                            redactions += r1 + r2;
+                            record[key] = JSON.stringify(embedded);
+                            continue;
+                        }
+                    }
+                    catch {
+                        // Not valid JSON — skip
+                    }
+                }
+            }
+            redactions += redactImageBase64InEmbeddedJsonStrings(val);
+        }
+    }
+    return redactions;
+}
+function cleanLines(lines) {
+    const stats = {
+        removedSubstring: 0,
+        removedPrefix: 0,
+        replaced: 0,
+        redactedImageBase64: 0,
+        redactedObsDomain: 0,
+    };
+    // 1. Remove lines with noise substrings
+    let stage = lines.filter((line) => {
+        if (REMOVE_SUBSTRINGS.some((s) => line.includes(s))) {
+            stats.removedSubstring++;
+            return false;
+        }
+        return true;
+    });
+    // 2. Replace API key + obs domain
+    stage = stage.map((line) => {
+        if (line.includes(APIKEY_OLD)) {
+            stats.replaced += countOccurrences(line, APIKEY_OLD);
+            line = line.replaceAll(APIKEY_OLD, APIKEY_NEW);
+        }
+        if (line.includes(REDACT_OBS_DOMAIN)) {
+            stats.redactedObsDomain += countOccurrences(line, REDACT_OBS_DOMAIN);
+            line = line.replaceAll(REDACT_OBS_DOMAIN, REDACT_OBS_REPLACEMENT);
+        }
+        return line;
+    });
+    // 3. Drop lines by message prefix
+    stage = stage.filter((line) => {
+        const msg = parseMessageField(line);
+        if (msg !== null &&
+            DROP_MESSAGE_PREFIXES.some((p) => msg.startsWith(p))) {
+            stats.removedPrefix++;
+            return false;
+        }
+        return true;
+    });
+    // 4. Redact base64 image payloads
+    const final = stage.map((line) => {
+        const trimmed = line.trim();
+        if (!trimmed)
+            return line;
+        let obj;
+        try {
+            obj = JSON.parse(trimmed);
+        }
+        catch {
+            return line;
+        }
+        let redactions = redactImageBase64InObj(obj);
+        redactions += redactImageBase64InEmbeddedJsonStrings(obj);
+        if (redactions) {
+            stats.redactedImageBase64 += redactions;
+            return JSON.stringify(obj);
+        }
+        return line;
+    });
+    return { cleaned: final, stats };
+}
+function countOccurrences(str, sub) {
+    let count = 0;
+    let idx = str.indexOf(sub);
+    while (idx !== -1) {
+        count++;
+        idx = str.indexOf(sub, idx + sub.length);
+    }
+    return count;
+}
+// ---------------------------------------------------------------------------
+// Public entry point
+// ---------------------------------------------------------------------------
+export function runSanitize(args) {
+    const logsDir = args[0];
+    if (!logsDir) {
+        console.error("Usage: visionclaw sanitize-logs <logs-dir>");
+        process.exit(1);
+    }
+    if (!fs.existsSync(logsDir) || !fs.statSync(logsDir).isDirectory()) {
+        console.error(`Not a directory: ${logsDir}`);
+        process.exit(1);
+    }
+    const files = fs
+        .readdirSync(logsDir)
+        .filter((f) => f.endsWith(".jsonl"))
+        .sort()
+        .map((f) => path.join(logsDir, f));
+    if (files.length === 0) {
+        console.error(`No .jsonl files found in ${logsDir}`);
+        process.exit(1);
+    }
+    const totals = {
+        filesChanged: 0,
+        removedSubstring: 0,
+        removedPrefix: 0,
+        replaced: 0,
+        redactedImageBase64: 0,
+        redactedObsDomain: 0,
+        linesBefore: 0,
+        linesAfter: 0,
+    };
+    for (const filePath of files) {
+        const originalText = fs.readFileSync(filePath, "utf-8");
+        const lines = originalText.split("\n");
+        const { cleaned, stats } = cleanLines(lines);
+        totals.linesBefore += lines.length;
+        totals.linesAfter += cleaned.length;
+        totals.removedSubstring += stats.removedSubstring;
+        totals.removedPrefix += stats.removedPrefix;
+        totals.replaced += stats.replaced;
+        totals.redactedImageBase64 += stats.redactedImageBase64;
+        totals.redactedObsDomain += stats.redactedObsDomain;
+        const newText = cleaned.join("\n");
+        if (newText !== originalText) {
+            fs.writeFileSync(filePath, newText, "utf-8");
+            totals.filesChanged++;
+        }
+    }
+    console.log("Sanitize complete:", {
+        files_processed: files.length,
+        files_changed: totals.filesChanged,
+        lines_before: totals.linesBefore,
+        lines_after: totals.linesAfter,
+        removed_substring: totals.removedSubstring,
+        removed_prefix: totals.removedPrefix,
+        replaced_apikey: totals.replaced,
+        redacted_image_base64: totals.redactedImageBase64,
+        redacted_obs_domain: totals.redactedObsDomain,
+    });
+}
+//# sourceMappingURL=sanitize.js.map

package/dist/data-processing/sanitize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sanitize.js","sourceRoot":"","sources":["../../src/data-processing/sanitize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,MAAM,UAAU,GAAG,yCAAyC,CAAC;AAC7D,MAAM,UAAU,GAAG,+BAA+B,CAAC;AAEnD,MAAM,iBAAiB,GAAG,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;AAC1D,MAAM,qBAAqB,GAAG,CAAC,eAAe,EAAE,0BAA0B,CAAC,CAAC;AAE5E,MAAM,iBAAiB,GAAG,mBAAmB,CAAC;AAC9C,MAAM,sBAAsB,GAAG,UAAU,CAAC;AAE1C,MAAM,mBAAmB,GAAG,mBAAmB,CAAC;AAEhD,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,SAAS,iBAAiB,CAAC,IAAY;IACrC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAA4B,CAAC;QAC3D,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC;QACxB,OAAO,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,GAAY;IAC1C,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;YACvB,UAAU,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;SAAM,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACnD,MAAM,MAAM,GAAG,GAA8B,CAAC;QAC9C,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;YAC1B,IACE,GAAG,KAAK,IAAI;gBACZ,OAAO,GAAG,KAAK,QAAQ;gBACvB,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EACnB,CAAC;gBACD,MAAM,MAAM,GAAG,GAA8B,CAAC;gBAC9C,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBACnD,MAAM,CAAC,IAAI,GAAG,mBAAmB,CAAC;oBAClC,UAAU,IAAI,CAAC,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACtC,UAAU,IAAI,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,sCAAsC,CAAC,GAAY;IAC1D,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;YACvB,UAAU,IAAI,sCAAsC,CAAC,IAAI,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;SAAM,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACnD,MAAM,MAAM,GAAG,GAA8B,CAAC;QAC9C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAChD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;gBACrB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;oBAClD,IAAI,CAAC;wBACH,MAAM,QAAQ,GAAY,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;wBACxC,MAAM,EAAE,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;wBAC5C,MAAM,EAAE,GAAG,sCAAsC,CAAC,QAAQ,CAAC,CAAC;wBAC5D,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;4BACb,UAAU,IAAI,EAAE,GAAG,EAAE,CAAC;4BACtB,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;4BACvC,SAAS;wBACX,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,wBAAwB;oBAC1B,CAAC;gBACH,CAAC;YACH,CAAC;YACD,UAAU,IAAI,sCAAsC,CAAC,GAAG,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAcD,SAAS,UAAU,CAAC,KAAe;IACjC,MAAM,KAAK,GAAe;QACxB,gBAAgB,EAAE,CAAC;QACnB,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,CAAC;QACX,mBAAmB,EAAE,CAAC;QACtB,iBAAiB,EAAE,CAAC;KACrB,CAAC;IAEF,wCAAwC;IACxC,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QAChC,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,KAAK,CAAC,gBAAgB,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,kCAAkC;IAClC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,QAAQ,IAAI,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACrD,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACrC,KAAK,CAAC,iBAAiB,IAAI,gBAAgB,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;YACrE,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE,sBAAsB,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,kCAAkC;IAClC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QAC5B,MAAM,GAAG,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACpC,IACE,GAAG,KAAK,IAAI;YACZ,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EACpD,CAAC;YACD,KAAK,CAAC,aAAa,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,kCAAkC;IAClC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAC1B,IAAI,GAAY,CAAC;QACjB,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,UAAU,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAC;QAC7C,UAAU,IAAI,sCAAsC,CAAC,GAAG,CAAC,CAAC;QAC1D,IAAI,UAAU,EAAE,CAAC;YACf,KAAK,CAAC,mBAAmB,IAAI,UAAU,CAAC;YACxC,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,GAAW;IAChD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC3B,OAAO,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;QAClB,KAAK,EAAE,CAAC;QACR,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,UAAU,WAAW,CAAC,IAAc;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACxB,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QACnE,OAAO,CAAC,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,EAAE;SACb,WAAW,CAAC,OAAO,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SACnC,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAErC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG;QACb,YAAY,EAAE,CAAC;QACf,gBAAgB,EAAE,CAAC;QACnB,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,CAAC;QACX,mBAAmB,EAAE,CAAC;QACtB,iBAAiB,EAAE,CAAC;QACpB,WAAW,EAAE,CAAC;QACd,UAAU,EAAE,CAAC;KACd,CAAC;IAEF,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACxD,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEvC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAE7C,MAAM,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC;QACnC,MAAM,CAAC,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;QACpC,MAAM,CAAC,gBAAgB,IAAI,KAAK,CAAC,gBAAgB,CAAC;QAClD,MAAM,CAAC,aAAa,IAAI,KAAK,CAAC,aAAa,CAAC;QAC5C,MAAM,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC;QAClC,MAAM,CAAC,mBAAmB,IAAI,KAAK,CAAC,mBAAmB,CAAC;QACxD,MAAM,CAAC,iBAAiB,IAAI,KAAK,CAAC,iBAAiB,CAAC;QAEpD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;YAC7B,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,MAAM,CAAC,YAAY,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE;QAChC,eAAe,EAAE,KAAK,CAAC,MAAM;QAC7B,aAAa,EAAE,MAAM,CAAC,YAAY;QAClC,YAAY,EAAE,MAAM,CAAC,WAAW;QAChC,WAAW,EAAE,MAAM,CAAC,UAAU;QAC9B,iBAAiB,EAAE,MAAM,CAAC,gBAAgB;QAC1C,cAAc,EAAE,MAAM,CAAC,aAAa;QACpC,eAAe,EAAE,MAAM,CAAC,QAAQ;QAChC,qBAAqB,EAAE,MAAM,CAAC,mBAAmB;QACjD,mBAAmB,EAAE,MAAM,CAAC,iBAAiB;KAC9C,CAAC,CAAC;AACL,CAAC"}

package/dist/data-processing/sdk-transcript.d.ts

@@ -0,0 +1,13 @@
+/**
+ * SDK transcript loading, segmentation, prior context extraction,
+ * and cycle matching.
+ */
+import type { SDKMessage, SDKWakeCycle, TrajectoryEntry, WakeCycle, PriorContext } from "./types.js";
+export declare function loadSdkTranscript(transcriptDir: string): Promise<SDKMessage[]>;
+export declare function segmentSdkWakeCycles(messages: SDKMessage[]): SDKWakeCycle[];
+export declare function buildPriorContext(sdkCycle: SDKWakeCycle): PriorContext | null;
+export declare function convertSdkMessages(messages: SDKMessage[]): TrajectoryEntry[];
+export declare function buildSdkTrajectory(sdkCycle: SDKWakeCycle): TrajectoryEntry[];
+export declare function buildSdkCycleIndex(sdkCycles: SDKWakeCycle[]): Map<string, SDKWakeCycle>;
+export declare function matchSdkCycle(cycle: WakeCycle, sdkIndex: Map<string, SDKWakeCycle>, sdkCycles: SDKWakeCycle[]): SDKWakeCycle | undefined;
+//# sourceMappingURL=sdk-transcript.d.ts.map

package/dist/data-processing/sdk-transcript.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sdk-transcript.d.ts","sourceRoot":"","sources":["../../src/data-processing/sdk-transcript.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,EACV,UAAU,EACV,YAAY,EACZ,eAAe,EACf,SAAS,EACT,YAAY,EACb,MAAM,YAAY,CAAC;AAqEpB,wBAAsB,iBAAiB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAmDpF;AAMD,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,UAAU,EAAE,GACrB,YAAY,EAAE,CA6ChB;AAiBD,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,YAAY,GACrB,YAAY,GAAG,IAAI,CAgDrB;AAMD,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,UAAU,EAAE,GACrB,eAAe,EAAE,CAwGnB;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,YAAY,GAAG,eAAe,EAAE,CAE5E;AAMD,wBAAgB,kBAAkB,CAChC,SAAS,EAAE,YAAY,EAAE,GACxB,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAM3B;AAcD,wBAAgB,aAAa,CAC3B,KAAK,EAAE,SAAS,EAChB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EACnC,SAAS,EAAE,YAAY,EAAE,GACxB,YAAY,GAAG,SAAS,CAgC1B"}