zozul-cli 0.1.0

package/src/sync/index.ts

@@ -0,0 +1,207 @@
+import type { SessionRow } from "../storage/db.js";
+import type { SessionRepo } from "../storage/repo.js";
+import type { ZozulApiClient } from "./client.js";
+import {
+  transformSession, transformTurn, transformToolUse,
+  transformHookEvent, transformTaskTag,
+  transformOtelMetric, transformOtelEvent,
+  type TurnLookup, type SessionSyncPayload,
+} from "./transform.js";
+
+const BATCH_SIZE = 500;
+
+export interface SyncCounts {
+  synced: number;
+  failed: number;
+}
+
+export interface SyncResult {
+  sessions: SyncCounts;
+  otel_metrics: SyncCounts;
+  otel_events: SyncCounts;
+}
+
+interface SyncOptions {
+  verbose?: boolean;
+  dryRun?: boolean;
+}
+
+function zeroCounts(): SyncCounts {
+  return { synced: 0, failed: 0 };
+}
+
+/**
+ * Build the sync payload for a single session — gathers all child entities
+ * (turns, tool_uses, hook_events, task_tags) and transforms them.
+ */
+function buildSessionPayload(
+  repo: SessionRepo,
+  session: SessionRow,
+): SessionSyncPayload {
+  const turns = repo.getSessionTurns(session.id);
+  const toolUses = repo.getSessionToolUses(session.id) as import("../storage/db.js").ToolUseRow[];
+  const hookEvents = repo.getSessionHookEvents(session.id) as import("../storage/db.js").HookEventRow[];
+
+  // Build turn_id → turn_index lookup for this session
+  const turnLookup: TurnLookup = new Map(turns.map(t => [t.id, t.turn_index]));
+
+  // Get task tags for all turns in this session
+  const taskTags: import("../storage/db.js").TaskTagRow[] = [];
+  for (const turn of turns) {
+    const tags = repo.getTaskTagsForTurn(turn.id);
+    taskTags.push(...tags);
+  }
+
+  return {
+    session: transformSession(session),
+    turns: turns.map(transformTurn),
+    tool_uses: toolUses.map(row => transformToolUse(row, turnLookup)),
+    task_tags: taskTags.map(row => transformTaskTag(row, turnLookup)),
+    hook_events: hookEvents.map(transformHookEvent),
+  };
+}
+
+export async function runSync(
+  repo: SessionRepo,
+  client: ZozulApiClient,
+  opts: SyncOptions = {},
+): Promise<SyncResult> {
+  const result: SyncResult = {
+    sessions: zeroCounts(),
+    otel_metrics: zeroCounts(),
+    otel_events: zeroCounts(),
+  };
+
+  // ── Phase 1: Determine sessions to sync ──
+  const sessionsWatermark = repo.getSyncWatermark("sessions");
+  const turnsWatermark = repo.getSyncWatermark("turns");
+
+  // New sessions (by rowid)
+  const newSessions = repo.getUnsyncedSessions(sessionsWatermark);
+  const sessionIdsToSync = new Set(newSessions.map(s => s.id));
+
+  // Sessions with new turns
+  const peekTurns = repo.getTurnsAfter(turnsWatermark, BATCH_SIZE);
+  for (const t of peekTurns) sessionIdsToSync.add(t.session_id);
+
+  // ── Phase 2: Sync sessions (one request per session, includes all child data) ──
+  if (sessionIdsToSync.size > 0) {
+    const idsFromTurns = [...sessionIdsToSync].filter(id => !newSessions.some(s => s.id === id));
+    const extraSessions = repo.getSessionsByIds(idsFromTurns);
+    const allSessions = [...newSessions, ...extraSessions];
+
+    let maxSyncedRowid = sessionsWatermark;
+    let maxSyncedTurnId = turnsWatermark;
+
+    for (const session of allSessions) {
+      if (opts.dryRun) {
+        result.sessions.synced++;
+        continue;
+      }
+
+      const payload = buildSessionPayload(repo, session);
+
+      try {
+        const resp = await client.syncSession(session.id, payload);
+        result.sessions.synced++;
+
+        // Track watermarks
+        const rowid = (session as { _rowid?: number })._rowid;
+        if (rowid != null && rowid > maxSyncedRowid) maxSyncedRowid = rowid;
+
+        // Track max turn id for this session's turns
+        const sessionTurns = repo.getSessionTurns(session.id);
+        for (const t of sessionTurns) {
+          if (t.id > maxSyncedTurnId) maxSyncedTurnId = t.id;
+        }
+
+        if (opts.verbose) {
+          console.log(
+            `  ${session.id}: ${resp.turns_synced} turns, ` +
+            `${resp.tool_uses_synced} tool_uses, ${resp.task_tags_synced} tags, ` +
+            `${resp.hook_events_synced} hook_events`
+          );
+        }
+      } catch (err) {
+        result.sessions.failed++;
+        if (opts.verbose) {
+          console.error(`  ${session.id}: failed — ${err instanceof Error ? err.message : err}`);
+        }
+      }
+    }
+
+    if (!opts.dryRun) {
+      if (maxSyncedRowid > sessionsWatermark) {
+        repo.setSyncWatermark("sessions", maxSyncedRowid);
+      }
+      if (maxSyncedTurnId > turnsWatermark) {
+        repo.setSyncWatermark("turns", maxSyncedTurnId);
+      }
+    }
+  }
+
+  // ── Phase 3: Sync OTel data (not session-scoped, uses separate bulk endpoints) ──
+  result.otel_metrics = await syncBulkTable(
+    "otel_metrics", repo,
+    (minId, limit) => repo.getOtelMetricsAfter(minId, limit),
+    transformOtelMetric,
+    (items) => client.postOtelMetricsBulk(items),
+    opts,
+  );
+
+  result.otel_events = await syncBulkTable(
+    "otel_events", repo,
+    (minId, limit) => repo.getOtelEventsAfter(minId, limit),
+    transformOtelEvent,
+    (items) => client.postOtelEventsBulk(items),
+    opts,
+  );
+
+  return result;
+}
+
+/**
+ * Generic helper for append-only bulk tables (otel_metrics, otel_events).
+ */
+async function syncBulkTable<TRow extends { id: number }, TApi>(
+  tableName: string,
+  repo: SessionRepo,
+  readAfter: (minId: number, limit: number) => TRow[],
+  transform: (row: TRow) => TApi,
+  postBulk: (items: TApi[]) => Promise<void>,
+  opts: SyncOptions,
+): Promise<SyncCounts> {
+  const counts = zeroCounts();
+  let watermark = repo.getSyncWatermark(tableName);
+
+  for (;;) {
+    const rows = readAfter(watermark, BATCH_SIZE);
+    if (rows.length === 0) break;
+
+    if (opts.dryRun) {
+      counts.synced += rows.length;
+      watermark = rows[rows.length - 1].id;
+      continue;
+    }
+
+    const payload = rows.map(transform);
+    try {
+      await postBulk(payload);
+      const newWatermark = rows[rows.length - 1].id;
+      repo.setSyncWatermark(tableName, newWatermark);
+      watermark = newWatermark;
+      counts.synced += rows.length;
+      if (opts.verbose) {
+        console.log(`  ${tableName}: synced ${rows.length} rows (watermark → ${newWatermark})`);
+      }
+    } catch (err) {
+      counts.failed += rows.length;
+      if (opts.verbose) {
+        console.error(`  ${tableName}: batch failed — ${err instanceof Error ? err.message : err}`);
+      }
+      break;
+    }
+  }
+
+  return counts;
+}

package/src/sync/sync.test.ts

@@ -0,0 +1,447 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import http from "node:http";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { getDb } from "../storage/db.js";
+import { SessionRepo } from "../storage/repo.js";
+import { ZozulApiClient } from "./client.js";
+import { runSync } from "./index.js";
+
+// ── Mock server that collects POSTed payloads ──
+
+function createMockServer() {
+  const received: Record<string, unknown[]> = {};
+
+  const server = http.createServer(async (req, res) => {
+    if (req.method !== "POST") {
+      res.writeHead(405).end();
+      return;
+    }
+
+    const chunks: Buffer[] = [];
+    for await (const chunk of req) chunks.push(chunk as Buffer);
+    const body = JSON.parse(Buffer.concat(chunks).toString());
+
+    const route = req.url!;
+    if (!received[route]) received[route] = [];
+
+    if (Array.isArray(body)) {
+      received[route].push(...body);
+    } else {
+      received[route].push(body);
+    }
+
+    // Return a sync-style response for session sync endpoints
+    if (route.includes("/sync")) {
+      const payload = body as { turns?: unknown[]; tool_uses?: unknown[]; task_tags?: unknown[]; hook_events?: unknown[] };
+      res.writeHead(200, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({
+        session_id: route.split("/")[4], // /api/v1/sessions/{id}/sync
+        turns_synced: payload.turns?.length ?? 0,
+        tool_uses_synced: payload.tool_uses?.length ?? 0,
+        task_tags_synced: payload.task_tags?.length ?? 0,
+        hook_events_synced: payload.hook_events?.length ?? 0,
+      }));
+      return;
+    }
+
+    res.writeHead(200, { "Content-Type": "application/json" });
+    res.end(JSON.stringify({ ok: true }));
+  });
+
+  return { server, received };
+}
+
+function listen(server: http.Server): Promise<number> {
+  return new Promise((resolve) => {
+    server.listen(0, "127.0.0.1", () => {
+      const addr = server.address() as { port: number };
+      resolve(addr.port);
+    });
+  });
+}
+
+function close(server: http.Server): Promise<void> {
+  return new Promise((resolve) => server.close(() => resolve()));
+}
+
+// ── Test suite ──
+
+describe("zozul sync e2e", () => {
+  let dbPath: string;
+  let repo: SessionRepo;
+  let db: ReturnType<typeof getDb>;
+  let server: http.Server;
+  let received: Record<string, unknown[]>;
+  let client: ZozulApiClient;
+
+  beforeEach(async () => {
+    dbPath = path.join(os.tmpdir(), `zozul-test-${Date.now()}.db`);
+    db = getDb(dbPath);
+    repo = new SessionRepo(db);
+
+    const mock = createMockServer();
+    server = mock.server;
+    received = mock.received;
+    const port = await listen(server);
+    client = new ZozulApiClient({
+      apiUrl: `http://127.0.0.1:${port}`,
+      apiKey: "test-key",
+    });
+  });
+
+  afterEach(async () => {
+    db.close();
+    await close(server);
+    try { fs.unlinkSync(dbPath); } catch {}
+    try { fs.unlinkSync(dbPath + "-wal"); } catch {}
+    try { fs.unlinkSync(dbPath + "-shm"); } catch {}
+  });
+
+  it("syncs a session with all child data in one request", async () => {
+    // Seed session
+    repo.upsertSession({
+      id: "sess-001",
+      project_path: "/projects/test",
+      started_at: "2026-03-28T10:00:00Z",
+      total_input_tokens: 100,
+      total_output_tokens: 50,
+      total_cache_read_tokens: 20,
+      total_cache_creation_tokens: 5,
+      total_cost_usd: 0.01,
+      total_turns: 2,
+      total_duration_ms: 5000,
+      model: "claude-sonnet-4-6",
+    });
+
+    // Seed turns
+    const turnId = repo.insertTurn({
+      session_id: "sess-001",
+      turn_index: 0,
+      role: "human",
+      timestamp: "2026-03-28T10:00:01Z",
+      input_tokens: 50,
+      output_tokens: 0,
+      cache_read_tokens: 10,
+      cache_creation_tokens: 0,
+      cost_usd: 0.005,
+      duration_ms: 1000,
+      model: "claude-sonnet-4-6",
+      content_text: "Hello",
+      tool_calls: null,
+      is_real_user: 1,
+    });
+
+    repo.insertTurn({
+      session_id: "sess-001",
+      turn_index: 1,
+      role: "assistant",
+      timestamp: "2026-03-28T10:00:02Z",
+      input_tokens: 50,
+      output_tokens: 50,
+      cache_read_tokens: 10,
+      cache_creation_tokens: 5,
+      cost_usd: 0.005,
+      duration_ms: 4000,
+      model: "claude-sonnet-4-6",
+      content_text: "Hi there!",
+      tool_calls: JSON.stringify([{ toolName: "Read", toolInput: { path: "/tmp" } }]),
+      is_real_user: 0,
+    });
+
+    // Seed tool use
+    repo.insertToolUse({
+      session_id: "sess-001",
+      turn_id: turnId,
+      tool_name: "Read",
+      tool_input: JSON.stringify({ path: "/tmp" }),
+      tool_result: "file contents",
+      success: 1,
+      duration_ms: 200,
+      timestamp: "2026-03-28T10:00:02Z",
+    });
+
+    // Seed hook event
+    repo.insertHookEvent({
+      session_id: "sess-001",
+      event_name: "UserPromptSubmit",
+      timestamp: "2026-03-28T10:00:01Z",
+      payload: JSON.stringify({ prompt: "Hello" }),
+    });
+
+    // Seed otel metric
+    repo.insertOtelMetric({
+      name: "claude_code.token.usage",
+      value: 100,
+      attributes: JSON.stringify({ type: "input" }),
+      session_id: "sess-001",
+      model: "claude-sonnet-4-6",
+      timestamp: "2026-03-28T10:00:02Z",
+    });
+
+    // Seed otel event
+    repo.insertOtelEvent({
+      event_name: "claude_code.conversation.turn",
+      attributes: JSON.stringify({ role: "human" }),
+      session_id: "sess-001",
+      prompt_id: "prompt-001",
+      timestamp: "2026-03-28T10:00:01Z",
+    });
+
+    // Seed task tag
+    repo.tagTurn(turnId, "FEAT-123");
+
+    // ── Run sync ──
+    const result = await runSync(repo, client, { verbose: false });
+
+    // ── Verify counts ──
+    expect(result.sessions.synced).toBe(1);
+    expect(result.sessions.failed).toBe(0);
+    expect(result.otel_metrics.synced).toBe(1);
+    expect(result.otel_events.synced).toBe(1);
+
+    // ── Verify session sync payload ──
+    const syncPayloads = received["/api/v1/sessions/sess-001/sync"];
+    expect(syncPayloads).toHaveLength(1);
+    const payload = syncPayloads[0] as Record<string, unknown>;
+
+    // Session
+    expect(payload.session).toMatchObject({ id: "sess-001", model: "claude-sonnet-4-6" });
+
+    // Turns — uses turn_index, no session_id (session is implicit)
+    const turns = payload.turns as Record<string, unknown>[];
+    expect(turns).toHaveLength(2);
+    expect(turns[0]).toMatchObject({ turn_index: 0, is_real_user: true });
+    expect(turns[1]).toMatchObject({ turn_index: 1, is_real_user: false });
+    expect(turns[1].tool_calls).toEqual([{ toolName: "Read", toolInput: { path: "/tmp" } }]);
+
+    // Tool uses — uses turn_index instead of turn_id
+    const toolUses = payload.tool_uses as Record<string, unknown>[];
+    expect(toolUses).toHaveLength(1);
+    expect(toolUses[0]).toMatchObject({ tool_name: "Read", success: true, turn_index: 0 });
+    expect(toolUses[0].tool_input).toEqual({ path: "/tmp" });
+    expect(toolUses[0]).not.toHaveProperty("turn_id");
+    expect(toolUses[0]).not.toHaveProperty("session_id");
+
+    // Hook events
+    const hookEvents = payload.hook_events as Record<string, unknown>[];
+    expect(hookEvents).toHaveLength(1);
+    expect(hookEvents[0]).toMatchObject({ event_name: "UserPromptSubmit" });
+    expect(hookEvents[0].payload).toEqual({ prompt: "Hello" });
+
+    // Task tags — uses turn_index
+    const taskTags = payload.task_tags as Record<string, unknown>[];
+    expect(taskTags).toHaveLength(1);
+    expect(taskTags[0]).toMatchObject({ task: "FEAT-123", turn_index: 0 });
+
+    // OTel sent separately
+    const otelMetrics = received["/api/v1/otel/metrics/bulk"] as Record<string, unknown>[];
+    expect(otelMetrics).toHaveLength(1);
+    expect(otelMetrics[0]).toMatchObject({ name: "claude_code.token.usage", value: 100 });
+
+    const otelEvents = received["/api/v1/otel/events/bulk"] as Record<string, unknown>[];
+    expect(otelEvents).toHaveLength(1);
+    expect(otelEvents[0]).toMatchObject({ event_name: "claude_code.conversation.turn" });
+  });
+
+  it("is idempotent — second sync sends nothing", async () => {
+    repo.upsertSession({
+      id: "sess-002",
+      project_path: null,
+      started_at: "2026-03-28T11:00:00Z",
+      total_input_tokens: 10,
+      total_output_tokens: 10,
+      total_cache_read_tokens: 0,
+      total_cache_creation_tokens: 0,
+      total_cost_usd: 0.001,
+      total_turns: 1,
+      total_duration_ms: 500,
+      model: "claude-haiku-4-5",
+    });
+
+    repo.insertTurn({
+      session_id: "sess-002",
+      turn_index: 0,
+      role: "human",
+      timestamp: "2026-03-28T11:00:01Z",
+      input_tokens: 10,
+      output_tokens: 10,
+      cache_read_tokens: 0,
+      cache_creation_tokens: 0,
+      cost_usd: 0.001,
+      duration_ms: 500,
+      model: "claude-haiku-4-5",
+      content_text: "test",
+      tool_calls: null,
+      is_real_user: 1,
+    });
+
+    // First sync
+    const r1 = await runSync(repo, client, {});
+    expect(r1.sessions.synced).toBe(1);
+
+    // Clear received payloads
+    for (const key of Object.keys(received)) delete received[key];
+
+    // Second sync
+    const r2 = await runSync(repo, client, {});
+    expect(r2.sessions.synced).toBe(0);
+    expect(r2.otel_metrics.synced).toBe(0);
+    expect(r2.otel_events.synced).toBe(0);
+    expect(Object.keys(received)).toHaveLength(0);
+  });
+
+  it("incremental sync picks up only new sessions", async () => {
+    repo.upsertSession({
+      id: "sess-A",
+      project_path: null,
+      started_at: "2026-03-28T12:00:00Z",
+      total_input_tokens: 10,
+      total_output_tokens: 10,
+      total_cache_read_tokens: 0,
+      total_cache_creation_tokens: 0,
+      total_cost_usd: 0.001,
+      total_turns: 1,
+      total_duration_ms: 100,
+      model: null,
+    });
+    repo.insertTurn({
+      session_id: "sess-A",
+      turn_index: 0,
+      role: "human",
+      timestamp: "2026-03-28T12:00:01Z",
+      input_tokens: 10,
+      output_tokens: 10,
+      cache_read_tokens: 0,
+      cache_creation_tokens: 0,
+      cost_usd: 0.001,
+      duration_ms: 100,
+      model: null,
+      content_text: "first",
+      tool_calls: null,
+      is_real_user: 1,
+    });
+
+    await runSync(repo, client, {});
+    for (const key of Object.keys(received)) delete received[key];
+
+    // Add second session
+    repo.upsertSession({
+      id: "sess-B",
+      project_path: "/new",
+      started_at: "2026-03-28T13:00:00Z",
+      total_input_tokens: 20,
+      total_output_tokens: 20,
+      total_cache_read_tokens: 0,
+      total_cache_creation_tokens: 0,
+      total_cost_usd: 0.002,
+      total_turns: 1,
+      total_duration_ms: 200,
+      model: "claude-opus-4-6",
+    });
+    repo.insertTurn({
+      session_id: "sess-B",
+      turn_index: 0,
+      role: "human",
+      timestamp: "2026-03-28T13:00:01Z",
+      input_tokens: 20,
+      output_tokens: 20,
+      cache_read_tokens: 0,
+      cache_creation_tokens: 0,
+      cost_usd: 0.002,
+      duration_ms: 200,
+      model: "claude-opus-4-6",
+      content_text: "second",
+      tool_calls: null,
+      is_real_user: 1,
+    });
+
+    const r2 = await runSync(repo, client, {});
+    expect(r2.sessions.synced).toBe(1);
+
+    // Only sess-B should have been synced
+    expect(received["/api/v1/sessions/sess-B/sync"]).toHaveLength(1);
+    expect(received["/api/v1/sessions/sess-A/sync"]).toBeUndefined();
+  });
+
+  it("dry-run does not send data or update watermarks", async () => {
+    repo.upsertSession({
+      id: "sess-dry",
+      project_path: null,
+      started_at: "2026-03-28T14:00:00Z",
+      total_input_tokens: 5,
+      total_output_tokens: 5,
+      total_cache_read_tokens: 0,
+      total_cache_creation_tokens: 0,
+      total_cost_usd: 0.0005,
+      total_turns: 1,
+      total_duration_ms: 50,
+      model: null,
+    });
+    repo.insertTurn({
+      session_id: "sess-dry",
+      turn_index: 0,
+      role: "human",
+      timestamp: "2026-03-28T14:00:01Z",
+      input_tokens: 5,
+      output_tokens: 5,
+      cache_read_tokens: 0,
+      cache_creation_tokens: 0,
+      cost_usd: 0.0005,
+      duration_ms: 50,
+      model: null,
+      content_text: "dry",
+      tool_calls: null,
+      is_real_user: 1,
+    });
+
+    const result = await runSync(repo, client, { dryRun: true });
+    expect(result.sessions.synced).toBe(1);
+    expect(Object.keys(received)).toHaveLength(0);
+    expect(repo.getSyncWatermark("sessions")).toBe(0);
+
+    // Real sync after dry run should still work
+    const r2 = await runSync(repo, client, {});
+    expect(r2.sessions.synced).toBe(1);
+    expect(received["/api/v1/sessions/sess-dry/sync"]).toHaveLength(1);
+  });
+
+  it("handles server errors gracefully", async () => {
+    repo.upsertSession({
+      id: "sess-err",
+      project_path: null,
+      started_at: "2026-03-28T15:00:00Z",
+      total_input_tokens: 10,
+      total_output_tokens: 10,
+      total_cache_read_tokens: 0,
+      total_cache_creation_tokens: 0,
+      total_cost_usd: 0.001,
+      total_turns: 1,
+      total_duration_ms: 100,
+      model: null,
+    });
+    repo.insertTurn({
+      session_id: "sess-err",
+      turn_index: 0,
+      role: "human",
+      timestamp: "2026-03-28T15:00:01Z",
+      input_tokens: 10,
+      output_tokens: 10,
+      cache_read_tokens: 0,
+      cache_creation_tokens: 0,
+      cost_usd: 0.001,
+      duration_ms: 100,
+      model: null,
+      content_text: "error test",
+      tool_calls: null,
+      is_real_user: 1,
+    });
+
+    await close(server);
+
+    const result = await runSync(repo, client, {});
+    expect(result.sessions.failed).toBe(1);
+    expect(repo.getSyncWatermark("sessions")).toBe(0);
+  });
+});