target-supabase-sdk 0.2.12 → 0.2.13

package/src/shared/log/README.md

@@ -0,0 +1,206 @@
+# Log module (`src/shared/log`)
+
+Structured logging, optional file-backed persistence, and Supabase `LogBatch` upload for L3 services.
+
+## Layout
+
+```text
+src/shared/log/
+  README.md
+  index.ts                 # browser-safe public API (re-exports core + upload types)
+
+  core/                    # in-process logging (browser + Node)
+    log-manager.ts         # LogManager singleton, LogEntry, levels
+    create-logger.ts       # createLogger({ module, scope, … })
+    log-scope.ts           # traceId / labels / patchScope
+    log-min-level.ts       # LOG_MIN_LEVEL env
+    log-batch.ts           # decode persisted LogBatch Lists (read side)
+
+  upload/                  # shared Supabase upload pipeline (Node)
+    env.ts                 # LOG_PERSIST_ENABLED, SERVICE, PROCESS, runtime dir
+    interface.ts           # LogBatch meta, loader keys, lane type
+    batch-id.ts            # computeLogBatchIdempotencyKey
+    flush.ts               # buildLogListDraft, postLogBatch
+    hook.ts                # LogManager → spool writer offer bridge
+    logger.ts              # internal pipeline loggers (no feedback loop)
+
+  spool/                   # file-backed persistence (Node only)
+    enable.ts              # ensureLogSpoolFromEnv, spawn env helpers
+    service-lifecycle.ts   # isLogSpoolEnabled, enableLogSpoolFromEnvInChild, shutdownLogSpool
+    writer.ts              # per-process memory buffer → .tmp files
+    collector.ts           # guard: upload .tmp, rename .json, periodic GC
+    paths.ts               # {RUNTIME_DATA_DIR}/log-spool/{serviceId}/…
+    file.ts                # batch filename parse/build
+    process-roles.ts       # main|guard|scheduler|worker + extras
+    config.ts              # writer / GC / collect interval env
+    coordinator.ts         # createLogSpoolCoordinator facade
+    register-collect-log-runner.ts
+```
+
+**Package exports**
+
+| Entry | Scope |
+|-------|--------|
+| `target-supabase-sdk` / `index.ts` | `core/` + decode types from `upload/interface` |
+| `target-supabase-sdk/node` | spool + upload symbols used by L3 services |
+
+Node-only code must not be imported from the browser bundle. Spool files use `node:fs`.
+
+---
+
+## Data flow
+
+```text
+App code
+  → createLogger / logManager
+  → offerToLogPersist (when spool enabled)
+  → LogSpoolWriter (memory buffer)
+  → {spoolRoot}/{serviceId}/{role}/*.tmp
+
+Guard collect-log runner (1 min tick)
+  → read *.tmp → postLogBatch → rename *.json
+  → GC old tmp/json (once per day; 7 d retention)
+
+log-service (downstream)
+  → decodeLogBatch / LogTrace merge by traceId
+```
+
+**One-line rule:** producers flush `LogEntry[]` to `.tmp`; only the guard uploads `LogBatch` and renames to `.json`.
+
+---
+
+## Environment
+
+| Key | Role |
+|-----|------|
+| `LOG_PERSIST_ENABLED` | Master switch (`true` → file spool) |
+| `LOG_PERSIST_SERVICE` | Logical service value in List meta (e.g. `watch-service`) |
+| `LOG_PERSIST_PROCESS` | This process role: `main` \| `guard` \| `scheduler` \| `worker` or extra |
+| `LOG_SPOOL_SERVICE_ID` | Instance id — set by main after registry claim |
+| `LOG_SPOOL_EXTRA_PROCESS_ROLES` | Comma-separated extras (e.g. `chrome-sidecar`) |
+| `RUNTIME_DATA_DIR` | Runtime root; spool lives at `{dir}/log-spool/` |
+| `LOG_SPOOL_DIR` | Optional override for spool root parent |
+
+### Writer tuning
+
+| Key | Default |
+|-----|---------|
+| `LOG_SPOOL_MAX_ENTRIES` | 2000 |
+| `LOG_SPOOL_MAX_BYTES` | 2 MB |
+| `LOG_SPOOL_FLUSH_DEBOUNCE_MS` | 3 s (after count/byte threshold) |
+| `LOG_SPOOL_MAX_AGE_MS` | 60 s (flush buffered logs even when below threshold) |
+| `LOG_PERSIST_POST_TIMEOUT_MS` | 30 s |
+
+### Collector / GC
+
+| Key | Default |
+|-----|---------|
+| `LOG_SPOOL_COLLECT_INTERVAL_MS` | 60 s |
+| `LOG_SPOOL_GC_INTERVAL_MS` | 24 h (how often guard runs tmp/json GC) |
+| `LOG_SPOOL_SYNCED_RETENTION_MS` | 7 d (delete `.json` when mtime older than this) |
+| `LOG_SPOOL_TMP_RETENTION_MS` | 7 d (delete stale `.tmp` by filename timestamp) |
+
+---
+
+## On-disk layout
+
+```text
+{RUNTIME_DATA_DIR}/log-spool/{serviceId}/
+  main/main.{YYYYMMDDHHmmss}.{hash8}.tmp   → .json after upload
+  guard/…
+  scheduler/…
+  worker/…
+  {extra}/…
+```
+
+Upload state: `.tmp` = pending, `.json` = uploaded (no central index file).
+
+Batch file JSON: `{ entries: LogEntry[], meta?: { serviceId, processRole, serviceValue } }`.
+
+Uploaded List meta:
+
+- `meta.process` = `{serviceId}:{processRole}` (e.g. `abc123:worker`)
+- `meta.service` = logical value (`watch-service`)
+- `meta.lane` = `"slow"` (legacy transport field; required for decode today)
+
+---
+
+## L3 integration
+
+### Preload (`runServicePreload`)
+
+Sync phases set `LOG_PERSIST_SERVICE`, `RUNTIME_DATA_DIR`, validate env, merge `logSpoolExtraProcessRoles`.  
+**Do not** call `ensureLogSpoolFromEnv` in preload — it is async and needs `LOG_SPOOL_SERVICE_ID`.
+
+### Main (`createServiceHost`)
+
+After registry claim, main sets `LOG_SPOOL_SERVICE_ID` and calls `ensureLogSpoolFromEnv({ processRole: "main", … })`.
+
+### Child processes
+
+Guard / scheduler / worker `main()`:
+
+```typescript
+import { enableLogSpoolFromEnvInChild } from "target-supabase-sdk/node";
+
+await enableLogSpoolFromEnvInChild();
+```
+
+### Shutdown
+
+```typescript
+import { shutdownLogSpool } from "target-supabase-sdk/node";
+
+await shutdownLogSpool();
+```
+
+Flushes remaining memory buffer to `.tmp`.
+
+### Extra process roles (e.g. download-service `chrome-sidecar`)
+
+```javascript
+// scripts/preload.mjs
+runServicePreload({
+  // …
+  logSpoolExtraProcessRoles: ["chrome-sidecar"],
+});
+```
+
+---
+
+## Public Node API (curated)
+
+| Symbol | Role |
+|--------|------|
+| `ensureLogSpoolFromEnv` | Enable writer in current process |
+| `enableLogSpoolFromEnvInChild` | Child entry: no-op when disabled, else `ensureLogSpoolFromEnv` |
+| `isLogSpoolEnabled` | Read master switch (`LOG_PERSIST_ENABLED`) |
+| `getMainLogSpoolWriterStats` | Writer stats or `null` when disabled (main observability) |
+| `shutdownLogSpool` | No-op when disabled, else flush + disable |
+| `shutdownLogSpoolFromEnv` | Flush buffer + disable (unconditional) |
+| `getLogSpoolStats` | Writer buffer stats |
+| `buildLogSpoolSpawnEnv` | Manual child env (usually unnecessary) |
+| `validateLogSpoolPreloadEnv` | Preload validation |
+| `logSpoolEnabledFromEnv` | Read master switch |
+| `createLogSpoolCoordinator` | Service-bound lifecycle facade |
+| `registerCollectLogRunner` | Guard TriggerNode runner (via `ServiceGuardNode`) |
+| `runCollectLogTick` | Manual / tests |
+| `resolveLogSpoolRoot` | Spool directory path |
+
+Internal helpers (`resolveLogSpoolSpawnEnvForLabel`, `resolveAllLogSpoolProcessRoles`, etc.) are not exported from `node` — use preload `logSpoolExtraProcessRoles` or env instead.
+
+---
+
+## Concurrency (Windows)
+
+- One writer per `.tmp` file — producers only create new files.
+- Guard renames after successful upload; does not RMW batch contents.
+- GC scans directories; no shared JSON index file.
+
+---
+
+## Related skills
+
+- [log-spool](../../../.cursor/skills/log-spool/SKILL.md) — agent quick reference
+- [service-preload](../../../.cursor/skills/service-preload/SKILL.md) — preload protocol
+- [l3-service-host](../../../.cursor/skills/l3-service-host/SKILL.md) — `createServiceHost` integration

package/src/shared/log/{log-batch.ts → core/log-batch.ts}

@@ -1,12 +1,12 @@
 /**
  * Read side of the log-persist pipeline: decode persisted `LogBatch` Lists into
  * structured batches and flat entries. Pure and defensive — malformed rows are
- * skipped rather than thrown. Mirrors the write side in `log-persist-flush.ts`.
+ * skipped rather than thrown. Mirrors the write side in `upload/flush.ts`.
  */
 
-import type { List } from "../../list/list.interface";
-import type { LogEntry } from "./log-manager";
-import { LOG_PERSIST_LOADER_KEY, type LogBatchMeta, type LogPersistLane } from "./log-persist.interface";
+import type { List } from "../../../list/list.interface";
+import type { LogEntry } from "../core/log-manager";
+import { LOG_PERSIST_LOADER_KEY, type LogBatchMeta, type LogPersistLane } from "../upload/interface";
 
 const LANES: ReadonlySet<string> = new Set<LogPersistLane>(["fast", "medium", "slow"]);
 

package/src/shared/log/{log-manager.ts → core/log-manager.ts}

@@ -1,6 +1,6 @@
-import { isDevEnvironment } from "../../core.utils";
-import { generateUniqueId } from "../utils/id.utils";
-import { offerToLogPersist } from "./log-persist-hook";
+import { isDevEnvironment } from "../../../core.utils";
+import { generateUniqueId } from "../../utils/id.utils";
+import { offerToLogPersist } from "../upload/hook";
 import { formatScopeLabels, type LoggerResetScopePatch, type LogScope, patchScope, resolveLogData } from "./log-scope";
 
 /**

package/src/shared/log/{log-min-level.ts → core/log-min-level.ts}

@@ -1,4 +1,4 @@
-import { isDevEnvironment } from "../../core.utils";
+import { isDevEnvironment } from "../../../core.utils";
 import { LogLevel } from "./log-manager";
 
 /** Env key for global {@link LogManager} min level (console + log-persist when enabled). */

package/src/shared/log/{log-scope.ts → core/log-scope.ts}

@@ -1,4 +1,4 @@
-import { generateUniqueId } from "../utils/id.utils";
+import { generateUniqueId } from "../../utils/id.utils";
 
 /** Logger-bound scope: module + trace chain + optional labels. */
 export interface LogScope {

package/src/shared/log/index.ts

@@ -1,16 +1,14 @@
 /**
- * Log domain public API — curated re-exports only.
+ * Log domain public API — curated re-exports only (browser-safe).
  */
 
 export type {
     CreateLoggerFromModuleInput,
     CreateLoggerFromScopeInput,
     CreateLoggerInput,
-} from "./create-logger";
-// create-logger
-export { createLogger } from "./create-logger";
-export type { DecodedLogBatch, FlatLogEntry } from "./log-batch";
-// log-batch (persisted LogBatch decode — read side of log-persist)
+} from "./core/create-logger";
+export { createLogger } from "./core/create-logger";
+export type { DecodedLogBatch, FlatLogEntry } from "./core/log-batch";
 export {
     decodeLogBatch,
     decodeLogBatchList,
@@ -19,19 +17,15 @@ export {
     isLogBatchList,
     isLogEntry,
     parseLogBatchMeta,
-} from "./log-batch";
-export type { LogEntry, LoggerWithScope, LogOptions, LogRestParams, WithScopeOptions } from "./log-manager";
-// log-manager
-export { LogLevel, LogManager, logManager } from "./log-manager";
-export type { ResolveLogMinLevelOptions } from "./log-min-level";
-// log-min-level
+} from "./core/log-batch";
+export type { LogEntry, LoggerWithScope, LogOptions, LogRestParams, WithScopeOptions } from "./core/log-manager";
+export { LogLevel, LogManager, logManager } from "./core/log-manager";
+export type { ResolveLogMinLevelOptions } from "./core/log-min-level";
 export {
     LOG_MIN_LEVEL_ENV_KEY,
     logMinLevelFromEnv,
     resolveLogMinLevel,
-} from "./log-min-level";
-export type { LogBatchMeta, LogPersistLane } from "./log-persist.interface";
-export { LOG_BATCH_LOADER_KEY, LOG_PERSIST_LOADER_KEY } from "./log-persist.interface";
+} from "./core/log-min-level";
 export type {
     CreateScopeInput,
     LoggerResetScopePatch,
@@ -39,8 +33,7 @@ export type {
     LogScopeInput,
     LogScopePatch,
     PatchScopeInput,
-} from "./log-scope";
-// log-scope
+} from "./core/log-scope";
 export {
     createScope,
     formatScopeLabels,
@@ -48,4 +41,6 @@ export {
     patchScope,
     resolveLogData,
     withModule,
-} from "./log-scope";
+} from "./core/log-scope";
+export type { LogBatchMeta, LogPersistLane } from "./upload/interface";
+export { LOG_BATCH_LOADER_KEY, LOG_PERSIST_LOADER_KEY } from "./upload/interface";

package/src/shared/log/spool/collector.ts

@@ -0,0 +1,330 @@
+import { access, constants, readdir, readFile, rename, stat, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { isLogEntry } from "../core/log-batch";
+import { computeLogBatchIdempotencyKey } from "../upload/batch-id";
+import { buildLogListDraft, postLogBatch } from "../upload/flush";
+import { spoolLogger } from "../upload/logger";
+import { type DEFAULT_LOG_SPOOL_CONFIG, resolveLogSpoolConfig } from "./config";
+import { parseLogSpoolBatchFilename } from "./file";
+import type { LogSpoolBatchFile, LogSpoolProcessRole } from "./interface";
+import { logSpoolProcessDir, resolveLogSpoolRoot } from "./paths";
+import { resolveAllLogSpoolProcessRoles } from "./process-roles";
+
+const UPLOAD_LANE = "slow";
+
+/** Per serviceId — guard collect-log tick runs GC at most once per `gc.intervalMs`. */
+const lastGcAtMsByServiceId = new Map<string, number>();
+
+export interface RunCollectLogTickInput {
+    serviceId: string;
+    serviceValue: string;
+    spoolRoot?: string;
+    config?: Partial<typeof DEFAULT_LOG_SPOOL_CONFIG>;
+    nowMs?: number;
+}
+
+export interface RunCollectLogTickResult {
+    uploaded: number;
+    skipped: number;
+    failed: number;
+    gcTmpRemoved: number;
+    gcSyncedRemoved: number;
+}
+
+async function pathExists(filePath: string): Promise<boolean> {
+    try {
+        await access(filePath, constants.F_OK);
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+async function readBatchTmpFile(tmpPath: string): Promise<LogSpoolBatchFile | null> {
+    try {
+        const raw = await readFile(tmpPath, "utf8");
+        const parsed = JSON.parse(raw) as LogSpoolBatchFile;
+        if (!Array.isArray(parsed.entries) || parsed.entries.length === 0) {
+            return null;
+        }
+        const entries = parsed.entries.filter(isLogEntry);
+        if (entries.length === 0) {
+            return null;
+        }
+        return { entries, meta: parsed.meta };
+    } catch {
+        return null;
+    }
+}
+
+function processMetaKey(serviceId: string, processRole: LogSpoolProcessRole): string {
+    return `${serviceId}:${processRole}`;
+}
+
+async function uploadTmpBatch(input: {
+    serviceId: string;
+    serviceValue: string;
+    processRole: LogSpoolProcessRole;
+    tmpPath: string;
+    fileName: string;
+    spoolRoot: string;
+    postTimeoutMs: number;
+}): Promise<"uploaded" | "skipped" | "failed"> {
+    const parsed = parseLogSpoolBatchFilename(input.fileName);
+    if (parsed == null) {
+        spoolLogger.warn("collect-log skipped — invalid tmp filename", { fileName: input.fileName });
+        return "failed";
+    }
+
+    const jsonFileName = input.fileName.replace(/\.tmp$/, ".json");
+    const jsonPath = join(logSpoolProcessDir(input.spoolRoot, input.serviceId, input.processRole), jsonFileName);
+
+    if (await pathExists(jsonPath)) {
+        try {
+            await unlink(input.tmpPath);
+            spoolLogger.info("collect-log removed orphan tmp (json exists)", {
+                tmpPath: input.tmpPath,
+                jsonPath,
+            });
+        } catch {
+            // best-effort
+        }
+        return "skipped";
+    }
+
+    const batch = await readBatchTmpFile(input.tmpPath);
+    if (batch == null) {
+        spoolLogger.warn("collect-log skipped — tmp failed validation", {
+            tmpPath: input.tmpPath,
+            fileName: input.fileName,
+        });
+        try {
+            await unlink(input.tmpPath);
+            spoolLogger.info("collect-log removed invalid tmp", { tmpPath: input.tmpPath });
+        } catch {
+            // best-effort
+        }
+        return "failed";
+    }
+
+    const processKey = processMetaKey(input.serviceId, input.processRole);
+    // TODO: `lane` is legacy transport metadata — scheduled for removal; fixed to "slow" for downstream decode compatibility.
+    const idempotencyKey = computeLogBatchIdempotencyKey({
+        service: input.serviceValue,
+        process: processKey,
+        lane: UPLOAD_LANE,
+        entries: batch.entries,
+    });
+
+    const draft = buildLogListDraft({
+        service: input.serviceValue,
+        process: processKey,
+        lane: UPLOAD_LANE,
+        idempotencyKey,
+        entries: batch.entries,
+    });
+
+    try {
+        await postLogBatch(draft, input.postTimeoutMs);
+    } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        spoolLogger.error("collect-log upload failed", {
+            fileName: input.fileName,
+            processRole: input.processRole,
+            error: message,
+        });
+        return "failed";
+    }
+
+    try {
+        await rename(input.tmpPath, jsonPath);
+    } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        spoolLogger.error("collect-log rename tmp→json failed after upload", {
+            tmpPath: input.tmpPath,
+            jsonPath,
+            error: message,
+        });
+        return "failed";
+    }
+
+    spoolLogger.info("collect-log uploaded batch", {
+        processRole: input.processRole,
+        fileName: jsonFileName,
+        entryCount: batch.entries.length,
+        idempotencyKey,
+    });
+
+    return "uploaded";
+}
+
+async function gcTmpFiles(input: {
+    spoolRoot: string;
+    serviceId: string;
+    processRole: LogSpoolProcessRole;
+    tmpRetentionMs: number;
+    nowMs: number;
+}): Promise<number> {
+    const processDir = logSpoolProcessDir(input.spoolRoot, input.serviceId, input.processRole);
+    let removed = 0;
+
+    try {
+        const names = await readdir(processDir);
+        for (const name of names) {
+            if (!name.endsWith(".tmp")) {
+                continue;
+            }
+            const parsed = parseLogSpoolBatchFilename(name);
+            if (parsed == null) {
+                continue;
+            }
+            if (input.nowMs - parsed.timestampMs <= input.tmpRetentionMs) {
+                continue;
+            }
+            try {
+                await unlink(join(processDir, name));
+                removed += 1;
+                spoolLogger.info("collect-log gc removed stale tmp", {
+                    fileName: name,
+                    processRole: input.processRole,
+                });
+            } catch {
+                // best-effort
+            }
+        }
+    } catch {
+        // directory may not exist yet
+    }
+
+    return removed;
+}
+
+async function gcSyncedJsonFiles(input: {
+    spoolRoot: string;
+    serviceId: string;
+    syncedRetentionMs: number;
+    nowMs: number;
+}): Promise<number> {
+    let removed = 0;
+
+    for (const processRole of resolveAllLogSpoolProcessRoles()) {
+        const processDir = logSpoolProcessDir(input.spoolRoot, input.serviceId, processRole);
+        let names: string[] = [];
+        try {
+            names = await readdir(processDir);
+        } catch {
+            continue;
+        }
+
+        for (const name of names) {
+            if (!name.endsWith(".json")) {
+                continue;
+            }
+            if (parseLogSpoolBatchFilename(name) == null) {
+                continue;
+            }
+
+            const jsonPath = join(processDir, name);
+            try {
+                const fileStat = await stat(jsonPath);
+                if (input.nowMs - fileStat.mtimeMs <= input.syncedRetentionMs) {
+                    continue;
+                }
+                await unlink(jsonPath);
+                removed += 1;
+                spoolLogger.info("collect-log gc removed synced json", {
+                    fileName: name,
+                    processRole,
+                });
+            } catch {
+                // best-effort
+            }
+        }
+    }
+
+    return removed;
+}
+
+function shouldRunGc(serviceId: string, nowMs: number, intervalMs: number): boolean {
+    const lastAt = lastGcAtMsByServiceId.get(serviceId) ?? 0;
+    if (nowMs - lastAt < intervalMs) {
+        return false;
+    }
+    lastGcAtMsByServiceId.set(serviceId, nowMs);
+    return true;
+}
+
+/**
+ * Guard-only: upload pending `.tmp` batches, rename to `.json`, then periodic GC.
+ * Upload completion is implied by `.json` presence; no sync-state file.
+ */
+export async function runCollectLogTick(input: RunCollectLogTickInput): Promise<RunCollectLogTickResult> {
+    const config = resolveLogSpoolConfig(input.config);
+    const spoolRoot = input.spoolRoot ?? resolveLogSpoolRoot();
+    const nowMs = input.nowMs ?? Date.now();
+
+    const result: RunCollectLogTickResult = {
+        uploaded: 0,
+        skipped: 0,
+        failed: 0,
+        gcTmpRemoved: 0,
+        gcSyncedRemoved: 0,
+    };
+
+    for (const processRole of resolveAllLogSpoolProcessRoles()) {
+        const processDir = logSpoolProcessDir(spoolRoot, input.serviceId, processRole);
+        let names: string[] = [];
+        try {
+            names = await readdir(processDir);
+        } catch {
+            continue;
+        }
+
+        for (const name of names) {
+            if (!name.endsWith(".tmp")) {
+                continue;
+            }
+            const status = await uploadTmpBatch({
+                serviceId: input.serviceId,
+                serviceValue: input.serviceValue,
+                processRole,
+                tmpPath: join(processDir, name),
+                fileName: name,
+                spoolRoot,
+                postTimeoutMs: config.writer.postTimeoutMs,
+            });
+            if (status === "uploaded") {
+                result.uploaded += 1;
+            } else if (status === "skipped") {
+                result.skipped += 1;
+            } else {
+                result.failed += 1;
+            }
+        }
+    }
+
+    if (shouldRunGc(input.serviceId, nowMs, config.gc.intervalMs)) {
+        for (const processRole of resolveAllLogSpoolProcessRoles()) {
+            result.gcTmpRemoved += await gcTmpFiles({
+                spoolRoot,
+                serviceId: input.serviceId,
+                processRole,
+                tmpRetentionMs: config.gc.tmpRetentionMs,
+                nowMs,
+            });
+        }
+
+        result.gcSyncedRemoved = await gcSyncedJsonFiles({
+            spoolRoot,
+            serviceId: input.serviceId,
+            syncedRetentionMs: config.gc.syncedRetentionMs,
+            nowMs,
+        });
+    }
+
+    if (result.uploaded > 0 || result.failed > 0) {
+        spoolLogger.info("collect-log tick complete", { ...result, serviceId: input.serviceId });
+    }
+
+    return result;
+}