selftune 0.2.22 → 0.2.24

package/cli/selftune/sync.ts

@@ -7,6 +7,7 @@
  * - Codex rollout logs
  * - OpenCode session history
  * - OpenClaw session history
+ * - Pi session history
  *
  * After syncing raw session/query/telemetry records, it rebuilds the repaired
  * skill-usage overlay from Claude transcripts and Codex rollouts so monitoring,
@@ -25,6 +26,8 @@ import {
   OPENCLAW_AGENTS_DIR,
   OPENCLAW_INGEST_MARKER,
   OPENCODE_INGEST_MARKER,
+  PI_INGEST_MARKER,
+  PI_SESSIONS_DIR,
   QUERY_LOG,
   REPAIRED_SKILL_LOG,
   REPAIRED_SKILL_SESSIONS_MARKER,
@@ -56,7 +59,14 @@ import {
   readSessionsFromSqlite,
   writeSession as writeOpenCodeSession,
 } from "./ingestors/opencode-ingest.js";
+import {
+  findPiSessions,
+  findPiSkillNames,
+  parsePiSession,
+  writeSession as writePiSession,
+} from "./ingestors/pi-ingest.js";
 import { getDb } from "./localdb/db.js";
+import { writeCronRunToDb } from "./localdb/direct-write.js";
 import { querySkillUsageRecords } from "./localdb/queries.js";
 import {
   persistRepairedSkillUsageToDb,
@@ -91,6 +101,7 @@ export interface SyncResult {
     codex: SyncStepResult;
     opencode: SyncStepResult;
     openclaw: SyncStepResult;
+    pi: SyncStepResult;
   };
   repair: {
     ran: boolean;
@@ -113,6 +124,7 @@ export interface SyncOptions {
   codexHome: string;
   opencodeDataDir: string;
   openclawAgentsDir: string;
+  piSessionsDir: string;
   skillLogPath: string;
   repairedSkillLogPath: string;
   repairedSessionsPath: string;
@@ -123,6 +135,7 @@ export interface SyncOptions {
   syncCodex: boolean;
   syncOpenCode: boolean;
   syncOpenClaw: boolean;
+  syncPi: boolean;
   rebuildSkillUsage: boolean;
 }
 
@@ -133,6 +146,7 @@ export interface SyncDeps {
   syncCodex?: (options: SyncOptions) => SyncStepResult;
   syncOpenCode?: (options: SyncOptions) => SyncStepResult;
   syncOpenClaw?: (options: SyncOptions) => SyncStepResult;
+  syncPi?: (options: SyncOptions) => SyncStepResult;
   rebuildSkillUsage?: (options: SyncOptions) => {
     repairedSessions: number;
     repairedRecords: number;
@@ -154,6 +168,7 @@ export function createDefaultSyncOptions(overrides: Partial<SyncOptions> = {}):
     codexHome: DEFAULT_CODEX_HOME,
     opencodeDataDir: DEFAULT_OPENCODE_DATA_DIR,
     openclawAgentsDir: OPENCLAW_AGENTS_DIR,
+    piSessionsDir: PI_SESSIONS_DIR,
     skillLogPath: SKILL_LOG,
     repairedSkillLogPath: REPAIRED_SKILL_LOG,
     repairedSessionsPath: REPAIRED_SKILL_SESSIONS_MARKER,
@@ -163,6 +178,7 @@ export function createDefaultSyncOptions(overrides: Partial<SyncOptions> = {}):
     syncCodex: true,
     syncOpenCode: true,
     syncOpenClaw: true,
+    syncPi: true,
     rebuildSkillUsage: true,
     ...overrides,
   };
@@ -356,6 +372,45 @@ function syncOpenClawSource(
   };
 }
 
+function syncPiSource(options: SyncOptions, onProgress?: SyncProgressCallback): SyncStepResult {
+  if (!existsSync(options.piSessionsDir)) {
+    return { available: false, scanned: 0, synced: 0, skipped: 0 };
+  }
+
+  onProgress?.("scanning Pi sessions...");
+  const sinceTs = options.since ? options.since.getTime() : null;
+  const allSessions = findPiSessions(options.piSessionsDir, sinceTs);
+  const skillNames = findPiSkillNames();
+  const alreadyIngested = options.force ? new Set<string>() : loadMarker(PI_INGEST_MARKER);
+  const pending = allSessions.filter((session) => !alreadyIngested.has(session.sessionId));
+  onProgress?.(`found ${allSessions.length} sessions, ${pending.length} pending`);
+  const newIngested = new Set<string>();
+  let synced = 0;
+  let skipped = 0;
+
+  for (const sessionFile of pending) {
+    const session = parsePiSession(sessionFile.filePath, skillNames);
+    if (!session.session_id || !session.timestamp) {
+      skipped += 1;
+      continue;
+    }
+    writePiSession(session, options.dryRun);
+    newIngested.add(sessionFile.sessionId);
+    synced += 1;
+  }
+
+  if (!options.dryRun && newIngested.size > 0) {
+    saveMarker(PI_INGEST_MARKER, new Set([...alreadyIngested, ...newIngested]));
+  }
+
+  return {
+    available: true,
+    scanned: allSessions.length,
+    synced,
+    skipped,
+  };
+}
+
 function rebuildSkillUsageOverlay(
   options: SyncOptions,
   onProgress?: SyncProgressCallback,
@@ -445,6 +500,7 @@ export function syncSources(
   const runCodex = deps.syncCodex;
   const runOpenCode = deps.syncOpenCode;
   const runOpenClaw = deps.syncOpenClaw;
+  const runPi = deps.syncPi;
   const runRepair = deps.rebuildSkillUsage;
   const runCreatorContributions = deps.stageCreatorContributions;
   const db = getDb();
@@ -485,6 +541,10 @@ export function syncSources(
       )
     : disabledStep;
 
+  const pi = options.syncPi
+    ? timePhase("pi", () => (runPi ? runPi(options) : syncPiSource(options, onProgress)), timings)
+    : disabledStep;
+
   const repair = options.rebuildSkillUsage
     ? timePhase(
         "repair",
@@ -512,10 +572,10 @@ export function syncSources(
 
   const totalElapsed = Math.round(performance.now() - totalStart);
 
-  return {
+  const syncResult: SyncResult = {
     since: options.since ? options.since.toISOString() : null,
     dry_run: options.dryRun,
-    sources: { claude, codex, opencode, openclaw },
+    sources: { claude, codex, opencode, openclaw, pi },
     repair: {
       ran: options.rebuildSkillUsage,
       repaired_sessions: repair.repairedSessions,
@@ -526,6 +586,8 @@ export function syncSources(
     timings,
     total_elapsed_ms: totalElapsed,
   };
+
+  return syncResult;
 }
 
 function formatMs(ms: number): string {
@@ -549,6 +611,7 @@ export async function cliMain(): Promise<void> {
       "codex-home": { type: "string", default: DEFAULT_CODEX_HOME },
       "opencode-data-dir": { type: "string", default: DEFAULT_OPENCODE_DATA_DIR },
       "openclaw-agents-dir": { type: "string", default: OPENCLAW_AGENTS_DIR },
+      "pi-sessions-dir": { type: "string", default: PI_SESSIONS_DIR },
       "skill-log": { type: "string", default: SKILL_LOG },
       "repaired-skill-log": { type: "string", default: REPAIRED_SKILL_LOG },
       "repaired-sessions-marker": { type: "string", default: REPAIRED_SKILL_SESSIONS_MARKER },
@@ -559,6 +622,7 @@ export async function cliMain(): Promise<void> {
       "no-codex": { type: "boolean", default: false },
       "no-opencode": { type: "boolean", default: false },
       "no-openclaw": { type: "boolean", default: false },
+      "no-pi": { type: "boolean", default: false },
       "no-repair": { type: "boolean", default: false },
       json: { type: "boolean", default: false },
       help: { type: "boolean", short: "h", default: false },
@@ -577,6 +641,7 @@ Options:
   --codex-home <dir>               Codex home directory (default: ~/.codex)
   --opencode-data-dir <dir>        OpenCode data directory
   --openclaw-agents-dir <dir>      OpenClaw agents directory
+  --pi-sessions-dir <dir>          Pi sessions directory
   --skill-log <path>               Raw skill usage log path
   --repaired-skill-log <path>      Repaired overlay log path
   --repaired-sessions-marker <p>   Repaired session marker path
@@ -587,6 +652,7 @@ Options:
   --no-codex                       Skip Codex rollout ingest
   --no-opencode                    Skip OpenCode ingest
   --no-openclaw                    Skip OpenClaw ingest
+  --no-pi                          Skip Pi ingest
   --no-repair                      Skip rebuilt skill-usage overlay
   --json                           Output raw JSON instead of human-readable summary
   -h, --help                       Show this help`);
@@ -622,27 +688,64 @@ Options:
     process.stderr.write(`selftune sync${flags.length ? ` ${flags.join(" ")}` : ""}\n`);
   }
 
-  const result = syncSources(
-    createDefaultSyncOptions({
-      projectsDir: values["projects-dir"] ?? CLAUDE_CODE_PROJECTS_DIR,
-      codexHome: values["codex-home"] ?? DEFAULT_CODEX_HOME,
-      opencodeDataDir: values["opencode-data-dir"] ?? DEFAULT_OPENCODE_DATA_DIR,
-      openclawAgentsDir: values["openclaw-agents-dir"] ?? OPENCLAW_AGENTS_DIR,
-      skillLogPath: values["skill-log"] ?? SKILL_LOG,
-      repairedSkillLogPath: values["repaired-skill-log"] ?? REPAIRED_SKILL_LOG,
-      repairedSessionsPath: values["repaired-sessions-marker"] ?? REPAIRED_SKILL_SESSIONS_MARKER,
-      since,
-      dryRun: values["dry-run"] ?? false,
-      force: values.force ?? false,
-      syncClaude: !(values["no-claude"] ?? false),
-      syncCodex: !(values["no-codex"] ?? false),
-      syncOpenCode: !(values["no-opencode"] ?? false),
-      syncOpenClaw: !(values["no-openclaw"] ?? false),
-      rebuildSkillUsage: !(values["no-repair"] ?? false),
-    }),
-    {},
-    onProgress,
-  );
+  const syncStartedAt = new Date();
+  const syncStart = performance.now();
+  let result: SyncResult;
+  try {
+    result = syncSources(
+      createDefaultSyncOptions({
+        projectsDir: values["projects-dir"] ?? CLAUDE_CODE_PROJECTS_DIR,
+        codexHome: values["codex-home"] ?? DEFAULT_CODEX_HOME,
+        opencodeDataDir: values["opencode-data-dir"] ?? DEFAULT_OPENCODE_DATA_DIR,
+        openclawAgentsDir: values["openclaw-agents-dir"] ?? OPENCLAW_AGENTS_DIR,
+        piSessionsDir: values["pi-sessions-dir"] ?? PI_SESSIONS_DIR,
+        skillLogPath: values["skill-log"] ?? SKILL_LOG,
+        repairedSkillLogPath: values["repaired-skill-log"] ?? REPAIRED_SKILL_LOG,
+        repairedSessionsPath: values["repaired-sessions-marker"] ?? REPAIRED_SKILL_SESSIONS_MARKER,
+        since,
+        dryRun: values["dry-run"] ?? false,
+        force: values.force ?? false,
+        syncClaude: !(values["no-claude"] ?? false),
+        syncCodex: !(values["no-codex"] ?? false),
+        syncOpenCode: !(values["no-opencode"] ?? false),
+        syncOpenClaw: !(values["no-openclaw"] ?? false),
+        syncPi: !(values["no-pi"] ?? false),
+        rebuildSkillUsage: !(values["no-repair"] ?? false),
+      }),
+      {},
+      onProgress,
+    );
+  } catch (err) {
+    const syncElapsed = Math.round(performance.now() - syncStart);
+    const message = err instanceof Error ? err.message : String(err);
+    writeCronRunToDb(getDb(), {
+      jobName: "sync",
+      startedAt: syncStartedAt.toISOString(),
+      elapsedMs: syncElapsed,
+      status: "error",
+      error: message,
+    });
+    throw err;
+  }
+
+  // Log successful sync run to unified cron_runs timeline
+  const syncElapsed = Math.round(performance.now() - syncStart);
+  const s = result.sources;
+  writeCronRunToDb(getDb(), {
+    jobName: "sync",
+    startedAt: syncStartedAt.toISOString(),
+    elapsedMs: syncElapsed,
+    status: "success",
+    metrics: {
+      total_synced:
+        s.claude.synced + s.codex.synced + s.opencode.synced + s.openclaw.synced + s.pi.synced,
+      claude_synced: s.claude.synced,
+      codex_synced: s.codex.synced,
+      opencode_synced: s.opencode.synced,
+      openclaw_synced: s.openclaw.synced,
+      pi_synced: s.pi.synced,
+    },
+  });
 
   if (jsonOutput) {
     console.log(JSON.stringify(result, null, 2));
@@ -662,6 +765,7 @@ Options:
     process.stderr.write(
       `${formatStepLine("OpenClaw", result.sources.openclaw, timingMap.get("openclaw"))}\n`,
     );
+    process.stderr.write(`${formatStepLine("Pi", result.sources.pi, timingMap.get("pi"))}\n`);
 
     if (result.repair.ran) {
       const repairTiming = timingMap.get("repair");