selftune 0.2.6 → 0.2.9

package/cli/selftune/alpha-upload/flush.ts

@@ -0,0 +1,191 @@
+/**
+ * Alpha upload flush engine.
+ *
+ * Drains the local upload queue by reading pending items, uploading
+ * them via the HTTP client, and updating their status. Implements
+ * retry with exponential backoff for transient (5xx/network) failures.
+ *
+ * Special status handling:
+ *   - 409 (duplicate push_id) is treated as success
+ *   - 401/403 (auth failures) are non-retryable with descriptive errors
+ *   - 4xx (client errors) are not retried
+ */
+
+import type { FlushSummary, QueueOperations } from "../alpha-upload-contract.js";
+import { uploadPushPayload } from "./client.js";
+
+// ---------------------------------------------------------------------------
+// Options
+// ---------------------------------------------------------------------------
+
+/** Options for the flush engine. */
+export interface FlushOptions {
+  /** Maximum number of items to read per flush batch (default: 50). */
+  batchSize?: number;
+  /** Maximum upload attempts per item before marking permanently failed (default: 5). */
+  maxRetries?: number;
+  /** When true, log what would be sent without making HTTP calls (default: false). */
+  dryRun?: boolean;
+  /** API key for Bearer auth on the cloud endpoint. */
+  apiKey?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const DEFAULT_BATCH_SIZE = 50;
+const DEFAULT_MAX_RETRIES = 5;
+const INITIAL_BACKOFF_MS = 1_000;
+const MAX_BACKOFF_MS = 16_000;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/** Returns true for HTTP status codes that are transient and worth retrying. */
+function isRetryable(status: number): boolean {
+  return status === 0 || status === 429 || status >= 500;
+}
+
+/** Returns true for auth errors that should not be retried. */
+function isAuthError(status: number): boolean {
+  return status === 401 || status === 403;
+}
+
+/** Sleep for the given number of milliseconds. */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** Calculate exponential backoff with cap. */
+function backoffMs(attempt: number): number {
+  const ms = INITIAL_BACKOFF_MS * 2 ** attempt;
+  return Math.min(ms, MAX_BACKOFF_MS);
+}
+
+/** Extract HTTP status from result. */
+function getStatus(result: Record<string, unknown>): number {
+  return (result as { _status?: number })._status ?? (result.success ? 200 : 0);
+}
+
+// ---------------------------------------------------------------------------
+// Flush engine
+// ---------------------------------------------------------------------------
+
+/**
+ * Flush the upload queue -- read pending items, upload them, update status.
+ */
+export async function flushQueue(
+  queue: QueueOperations,
+  endpoint: string,
+  options?: FlushOptions,
+): Promise<FlushSummary> {
+  const batchSize = options?.batchSize ?? DEFAULT_BATCH_SIZE;
+  const maxRetries = options?.maxRetries ?? DEFAULT_MAX_RETRIES;
+  const dryRun = options?.dryRun ?? false;
+  const apiKey = options?.apiKey;
+
+  const summary: FlushSummary = { sent: 0, failed: 0, skipped: 0 };
+
+  const items = queue.getPending(batchSize);
+
+  if (items.length === 0) {
+    return summary;
+  }
+
+  for (const item of items) {
+    const markFailedSafely = (message: string): void => {
+      if (!queue.markFailed(item.id, message)) {
+        console.error(`[alpha upload] Failed to persist queue failure state for item ${item.id}`);
+      }
+    };
+
+    if (item.attempts >= maxRetries) {
+      markFailedSafely("exhausted retries");
+      summary.failed++;
+      continue;
+    }
+
+    if (dryRun) {
+      summary.skipped++;
+      continue;
+    }
+
+    let payload: Record<string, unknown>;
+    try {
+      payload = JSON.parse(item.payload_json) as Record<string, unknown>;
+    } catch {
+      markFailedSafely("corrupt payload JSON");
+      summary.failed++;
+      continue;
+    }
+
+    if (!queue.markSending(item.id)) {
+      console.error(`[alpha upload] Failed to mark queue item ${item.id} as sending`);
+      summary.failed++;
+      continue;
+    }
+
+    let succeeded = false;
+    const attemptsRemaining = maxRetries - item.attempts;
+
+    for (let attempt = 0; attempt < attemptsRemaining; attempt++) {
+      if (attempt > 0) {
+        await sleep(backoffMs(attempt - 1));
+      }
+
+      const result = await uploadPushPayload(payload, endpoint, apiKey);
+      const status = getStatus(result as unknown as Record<string, unknown>);
+
+      if (result.success) {
+        if (!queue.markSent(item.id)) {
+          markFailedSafely("local queue state update failed after successful upload");
+          summary.failed++;
+        } else {
+          summary.sent++;
+        }
+        succeeded = true;
+        break;
+      }
+
+      // 409 Conflict = duplicate push_id, treat as success
+      if (status === 409) {
+        if (!queue.markSent(item.id)) {
+          markFailedSafely("local queue state update failed after duplicate upload");
+          summary.failed++;
+        } else {
+          summary.sent++;
+        }
+        succeeded = true;
+        break;
+      }
+
+      // Auth errors are non-retryable
+      if (isAuthError(status)) {
+        const authMessage =
+          status === 401
+            ? "Authentication failed: invalid or missing API key. Run 'selftune init --alpha --alpha-email <email>' to re-authenticate via browser."
+            : "Authorization denied: your API key does not have permission to upload. Run 'selftune doctor' to verify enrollment and cloud link, then re-run 'selftune init --alpha --alpha-email <email> --force' to re-authenticate.";
+        markFailedSafely(authMessage);
+        summary.failed++;
+        succeeded = true;
+        break;
+      }
+
+      if (!isRetryable(status)) {
+        markFailedSafely(result.errors[0] ?? `Upload failed with HTTP ${status}`);
+        summary.failed++;
+        succeeded = true;
+        break;
+      }
+    }
+
+    if (!succeeded) {
+      markFailedSafely("exhausted retries");
+      summary.failed++;
+    }
+  }
+
+  return summary;
+}

package/cli/selftune/alpha-upload/index.ts

@@ -0,0 +1,194 @@
+/**
+ * Alpha upload orchestration module.
+ *
+ * Coordinates the full upload cycle:
+ *   1. Stage canonical records from JSONL + evolution evidence into staging table
+ *   2. Read new staged records since watermark via single cursor
+ *   3. Build a V2 canonical push payload
+ *   4. Enqueue it in the local upload queue
+ *   5. Flush the queue to POST /api/v1/push
+ *
+ * Guards:
+ *   - Only runs when alpha enrolled (config.alpha?.enrolled === true)
+ *   - Fail-open: never throws, returns empty summary on errors
+ *   - Reads endpoint from config or SELFTUNE_ALPHA_ENDPOINT env var
+ */
+
+import type { Database } from "bun:sqlite";
+
+import type {
+  QueueItem as ContractQueueItem,
+  FlushSummary,
+  QueueOperations,
+} from "../alpha-upload-contract.js";
+import { buildV2PushPayload } from "./build-payloads.js";
+import { flushQueue } from "./flush.js";
+import {
+  enqueueUpload,
+  getPendingUploads,
+  markFailed,
+  markSending,
+  markSent,
+  readWatermark,
+  writeWatermark,
+} from "./queue.js";
+import { stageCanonicalRecords } from "./stage-canonical.js";
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const DEFAULT_ENDPOINT = "https://api.selftune.dev/api/v1/push";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface PrepareResult {
+  enqueued: number;
+  types: string[];
+}
+
+export interface UploadCycleOptions {
+  enrolled: boolean;
+  userId?: string;
+  agentType?: string;
+  selftuneVersion?: string;
+  endpoint?: string;
+  dryRun?: boolean;
+  apiKey?: string;
+  /** Override canonical log path (for testing). */
+  canonicalLogPath?: string;
+}
+
+export interface UploadCycleSummary {
+  enrolled: boolean;
+  prepared: number;
+  sent: number;
+  failed: number;
+  skipped: number;
+}
+
+// ---------------------------------------------------------------------------
+// prepareUploads -- stage, build V2 payload, enqueue
+// ---------------------------------------------------------------------------
+
+/**
+ * Stage canonical records, read new staged rows since watermark,
+ * build a single V2 push payload, and enqueue it. Never throws.
+ */
+export function prepareUploads(
+  db: Database,
+  _userId: string,
+  _agentType: string,
+  _selftuneVersion: string,
+  canonicalLogPath?: string,
+): PrepareResult {
+  const result: PrepareResult = { enqueued: 0, types: [] };
+
+  try {
+    // Step 1: Stage canonical records from JSONL + evolution evidence
+    stageCanonicalRecords(db, canonicalLogPath);
+
+    // Step 2: Read watermark (single cursor for all record types)
+    const afterSeq = readWatermark(db, "canonical") ?? undefined;
+
+    // Step 3: Build payload from staging table
+    const build = buildV2PushPayload(db, afterSeq);
+
+    if (!build) return result;
+
+    // Step 4: Enqueue the payload + advance watermark atomically
+    const tx = db.transaction(() => {
+      const ok = enqueueUpload(db, "push", JSON.stringify(build.payload));
+      if (!ok) {
+        throw new Error("enqueueUpload failed");
+      }
+
+      if (!writeWatermark(db, "canonical", build.lastSeq)) {
+        throw new Error("writeWatermark failed");
+      }
+
+      result.enqueued = 1;
+      result.types.push("canonical");
+    });
+    tx();
+  } catch (err) {
+    if (process.env.DEBUG || process.env.NODE_ENV === "development") {
+      console.error("[alpha-upload] prepareUploads failed:", err);
+    }
+  }
+
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// runUploadCycle -- the full cycle: prepare -> flush -> return summary
+// ---------------------------------------------------------------------------
+
+/**
+ * Run a full upload cycle: stage + read new data, enqueue it, flush to remote.
+ * Guards on enrollment -- returns empty summary if not enrolled.
+ * Never throws.
+ */
+export async function runUploadCycle(
+  db: Database,
+  options: UploadCycleOptions,
+): Promise<UploadCycleSummary> {
+  const emptySummary: UploadCycleSummary = {
+    enrolled: options.enrolled,
+    prepared: 0,
+    sent: 0,
+    failed: 0,
+    skipped: 0,
+  };
+
+  // Guard: must be enrolled
+  if (!options.enrolled) {
+    return emptySummary;
+  }
+
+  try {
+    const userId = options.userId ?? "unknown";
+    const agentType = options.agentType ?? "unknown";
+    const selftuneVersion = options.selftuneVersion ?? "0.0.0";
+    const endpoint = process.env.SELFTUNE_ALPHA_ENDPOINT ?? options.endpoint ?? DEFAULT_ENDPOINT;
+    const dryRun = options.dryRun ?? false;
+    const apiKey = options.apiKey;
+
+    // Step 1: Prepare -- stage, build V2 payload, enqueue
+    const prepared = prepareUploads(
+      db,
+      userId,
+      agentType,
+      selftuneVersion,
+      options.canonicalLogPath,
+    );
+
+    // Step 2: Flush -- drain the queue to the remote endpoint
+    const queueOps: QueueOperations = {
+      getPending: (limit: number) => getPendingUploads(db, limit) as ContractQueueItem[],
+      markSending: (id: number) => markSending(db, [id]),
+      markSent: (id: number) => markSent(db, [id]),
+      markFailed: (id: number, error?: string) => markFailed(db, id, error ?? "unknown"),
+    };
+
+    const flush: FlushSummary = await flushQueue(queueOps, endpoint, {
+      dryRun,
+      apiKey,
+    });
+
+    return {
+      enrolled: true,
+      prepared: prepared.enqueued,
+      sent: flush.sent,
+      failed: flush.failed,
+      skipped: flush.skipped,
+    };
+  } catch (err) {
+    if (process.env.DEBUG || process.env.NODE_ENV === "development") {
+      console.error("[alpha-upload] runUploadCycle failed:", err);
+    }
+    return emptySummary;
+  }
+}

package/cli/selftune/alpha-upload/queue.ts

@@ -0,0 +1,252 @@
+/**
+ * Alpha upload queue — local queue and watermark storage layer.
+ *
+ * Queues payload items for upload to the alpha remote endpoint.
+ * No HTTP code — this module only manages the SQLite queue state.
+ *
+ * All public functions follow the fail-open pattern from direct-write.ts:
+ * they catch errors internally and return boolean success / safe defaults.
+ */
+
+import type { Database } from "bun:sqlite";
+
+// -- Types --------------------------------------------------------------------
+
+export interface QueueItem {
+  id: number;
+  payload_type: string;
+  payload_json: string;
+  status: string;
+  attempts: number;
+  created_at: string;
+  updated_at: string;
+  last_error: string | null;
+}
+
+export interface QueueStats {
+  pending: number;
+  sending: number;
+  sent: number;
+  failed: number;
+}
+
+// -- Queue operations ---------------------------------------------------------
+
+/**
+ * Insert a new pending item into the upload queue.
+ * Returns true on success, false on failure (fail-open).
+ */
+export function enqueueUpload(db: Database, payloadType: string, payloadJson: string): boolean {
+  try {
+    const now = new Date().toISOString();
+    db.run(
+      `INSERT INTO upload_queue (payload_type, payload_json, status, attempts, created_at, updated_at)
+       VALUES (?, ?, 'pending', 0, ?, ?)`,
+      [payloadType, payloadJson, now, now],
+    );
+    return true;
+  } catch (err) {
+    if (process.env.DEBUG || process.env.NODE_ENV === "development") {
+      console.error("[alpha-upload/queue] enqueueUpload failed:", err);
+    }
+    return false;
+  }
+}
+
+/**
+ * Get pending upload items, oldest first.
+ * Default limit is 50.
+ */
+export function getPendingUploads(db: Database, limit = 50): QueueItem[] {
+  try {
+    return db
+      .query(
+        `SELECT id, payload_type, payload_json, status, attempts, created_at, updated_at, last_error
+         FROM upload_queue
+         WHERE status = 'pending'
+         ORDER BY id ASC
+         LIMIT ?`,
+      )
+      .all(limit) as QueueItem[];
+  } catch (err) {
+    if (process.env.DEBUG || process.env.NODE_ENV === "development") {
+      console.error("[alpha-upload/queue] getPendingUploads failed:", err);
+    }
+    return [];
+  }
+}
+
+/**
+ * Transition pending items to sending status.
+ * Only transitions items that are currently 'pending'.
+ */
+export function markSending(db: Database, ids: number[]): boolean {
+  if (ids.length === 0) return true;
+  try {
+    const now = new Date().toISOString();
+    const placeholders = ids.map(() => "?").join(",");
+    db.run(
+      `UPDATE upload_queue
+       SET status = 'sending', updated_at = ?
+       WHERE id IN (${placeholders}) AND status = 'pending'`,
+      [now, ...ids],
+    );
+    return true;
+  } catch (err) {
+    if (process.env.DEBUG || process.env.NODE_ENV === "development") {
+      console.error("[alpha-upload/queue] markSending failed:", err);
+    }
+    return false;
+  }
+}
+
+/**
+ * Transition sending items to sent status.
+ * Also updates the watermark per payload_type to the max id in the batch.
+ */
+export function markSent(db: Database, ids: number[]): boolean {
+  if (ids.length === 0) return true;
+  try {
+    const now = new Date().toISOString();
+    const placeholders = ids.map(() => "?").join(",");
+
+    db.run("BEGIN TRANSACTION");
+    try {
+      const sendingRows = db
+        .query(
+          `SELECT id, payload_type
+           FROM upload_queue
+           WHERE id IN (${placeholders}) AND status = 'sending'`,
+        )
+        .all(...ids) as Array<{ id: number; payload_type: string }>;
+
+      // Mark items as sent
+      db.run(
+        `UPDATE upload_queue
+         SET status = 'sent', updated_at = ?
+         WHERE id IN (${placeholders}) AND status = 'sending'`,
+        [now, ...ids],
+      );
+
+      // Update watermarks only for rows that actually transitioned from "sending".
+      const maxByType = new Map<string, number>();
+      for (const row of sendingRows) {
+        const current = maxByType.get(row.payload_type) ?? 0;
+        if (row.id > current) {
+          maxByType.set(row.payload_type, row.id);
+        }
+      }
+
+      for (const [payloadType, maxId] of maxByType.entries()) {
+        db.run(
+          `INSERT INTO upload_watermarks (payload_type, last_uploaded_id, updated_at)
+           VALUES (?, ?, ?)
+           ON CONFLICT(payload_type) DO UPDATE SET
+             last_uploaded_id = excluded.last_uploaded_id,
+             updated_at = excluded.updated_at`,
+          [payloadType, maxId, now],
+        );
+      }
+
+      db.run("COMMIT");
+    } catch (err) {
+      db.run("ROLLBACK");
+      throw err;
+    }
+    return true;
+  } catch (err) {
+    if (process.env.DEBUG || process.env.NODE_ENV === "development") {
+      console.error("[alpha-upload/queue] markSent failed:", err);
+    }
+    return false;
+  }
+}
+
+/**
+ * Transition a sending item to failed status.
+ * Increments the attempts counter and records the error message.
+ */
+export function markFailed(db: Database, id: number, error: string): boolean {
+  try {
+    const now = new Date().toISOString();
+    db.run(
+      `UPDATE upload_queue
+       SET status = 'failed', attempts = attempts + 1, last_error = ?, updated_at = ?
+       WHERE id = ? AND status = 'sending'`,
+      [error, now, id],
+    );
+    return true;
+  } catch (err) {
+    if (process.env.DEBUG || process.env.NODE_ENV === "development") {
+      console.error("[alpha-upload/queue] markFailed failed:", err);
+    }
+    return false;
+  }
+}
+
+/**
+ * Get counts of items by status.
+ */
+export function getQueueStats(db: Database): QueueStats {
+  try {
+    const row = db
+      .query(
+        `SELECT
+           COALESCE(SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END), 0) as pending,
+           COALESCE(SUM(CASE WHEN status = 'sending' THEN 1 ELSE 0 END), 0) as sending,
+           COALESCE(SUM(CASE WHEN status = 'sent'    THEN 1 ELSE 0 END), 0) as sent,
+           COALESCE(SUM(CASE WHEN status = 'failed'  THEN 1 ELSE 0 END), 0) as failed
+         FROM upload_queue`,
+      )
+      .get() as QueueStats;
+    return row;
+  } catch (err) {
+    if (process.env.DEBUG || process.env.NODE_ENV === "development") {
+      console.error("[alpha-upload/queue] getQueueStats failed:", err);
+    }
+    return { pending: 0, sending: 0, sent: 0, failed: 0 };
+  }
+}
+
+// -- Watermark operations -----------------------------------------------------
+
+/**
+ * Read the last uploaded ID for a given payload type.
+ * Returns null if no watermark exists.
+ */
+export function readWatermark(db: Database, payloadType: string): number | null {
+  try {
+    const row = db
+      .query("SELECT last_uploaded_id FROM upload_watermarks WHERE payload_type = ?")
+      .get(payloadType) as { last_uploaded_id: number } | null;
+    return row?.last_uploaded_id ?? null;
+  } catch (err) {
+    if (process.env.DEBUG || process.env.NODE_ENV === "development") {
+      console.error("[alpha-upload/queue] readWatermark failed:", err);
+    }
+    return null;
+  }
+}
+
+/**
+ * Upsert the watermark for a given payload type.
+ */
+export function writeWatermark(db: Database, payloadType: string, lastId: number): boolean {
+  try {
+    const now = new Date().toISOString();
+    db.run(
+      `INSERT INTO upload_watermarks (payload_type, last_uploaded_id, updated_at)
+       VALUES (?, ?, ?)
+       ON CONFLICT(payload_type) DO UPDATE SET
+         last_uploaded_id = excluded.last_uploaded_id,
+         updated_at = excluded.updated_at`,
+      [payloadType, lastId, now],
+    );
+    return true;
+  } catch (err) {
+    if (process.env.DEBUG || process.env.NODE_ENV === "development") {
+      console.error("[alpha-upload/queue] writeWatermark failed:", err);
+    }
+    return false;
+  }
+}