target-supabase-sdk 0.2.0 → 0.2.2

package/src/shared/log/log-persist-flush.ts

@@ -0,0 +1,128 @@
+import { isCreateTargetAlreadyExistsError } from "../../core.api";
+import { postLogListCreate } from "../../list/list.api";
+import { CategoryList } from "../../list/list.interface";
+import type { LogEntry } from "./log-manager";
+import { LOG_PERSIST_LOADER_KEY } from "./log-persist.config";
+import type { BuildLogListDraftInput, LogListDraft } from "./log-persist.interface";
+import { persistLogger } from "./log-persist-logger";
+
+export function estimateEntriesBytes(entries: LogEntry[]): number {
+    if (entries.length === 0) {
+        return 0;
+    }
+    try {
+        return Buffer.byteLength(JSON.stringify(entries), "utf8");
+    } catch {
+        return entries.length * 512;
+    }
+}
+
+export function buildLogListDraft(input: BuildLogListDraftInput): LogListDraft {
+    const { service, process, lane, idempotencyKey, entries } = input;
+    const first = entries[0];
+    const last = entries[entries.length - 1];
+
+    return {
+        name: `log-batch:${service}:${process}:${lane}`,
+        value: idempotencyKey,
+        category: CategoryList.LIST,
+        tagList: ["log", lane, service],
+        details: {
+            manifestVersion: 0,
+            loaderKey: LOG_PERSIST_LOADER_KEY,
+            meta: JSON.stringify({
+                service,
+                process,
+                pid: globalThis.process.pid,
+                lane,
+                count: entries.length,
+                from: first?.timestamp ?? null,
+                to: last?.timestamp ?? null,
+                flushedAt: Date.now(),
+                idempotencyKey,
+            }),
+            preview: first?.message ?? "",
+            items: entries,
+        },
+    };
+}
+
+async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, label: string): Promise<T> {
+    let timer: ReturnType<typeof setTimeout> | null = null;
+    try {
+        return await Promise.race([
+            promise,
+            new Promise<T>((_resolve, reject) => {
+                timer = setTimeout(() => {
+                    reject(new Error(`${label} timed out after ${timeoutMs}ms`));
+                }, timeoutMs);
+            }),
+        ]);
+    } finally {
+        if (timer != null) {
+            clearTimeout(timer);
+        }
+    }
+}
+
+export async function postLogBatch(draft: LogListDraft, postTimeoutMs: number): Promise<void> {
+    try {
+        const result = await withTimeout(
+            postLogListCreate({
+                name: draft.name,
+                value: draft.value,
+                category: draft.category,
+                details: draft.details,
+            }),
+            postTimeoutMs,
+            "postLogListCreate",
+        );
+
+        if (!result.success) {
+            const message = result.error?.message ?? "postLogListCreate failed";
+            persistLogger.error("post log batch failed", {
+                idempotencyKey: draft.value,
+                name: draft.name,
+                error: message,
+            });
+            throw new Error(message);
+        }
+    } catch (error) {
+        if (isCreateTargetAlreadyExistsError(error)) {
+            persistLogger.error("post log batch idempotent duplicate — treating as success", {
+                idempotencyKey: draft.value,
+                name: draft.name,
+            });
+            return;
+        }
+        throw error;
+    }
+}
+
+/** Take a prefix of entries that fits within maxBytes. */
+export function takeEntriesWithinBytes(entries: LogEntry[], maxBytes: number): LogEntry[] {
+    if (entries.length === 0) {
+        return [];
+    }
+
+    const taken: LogEntry[] = [];
+    for (const entry of entries) {
+        const candidate = [...taken, entry];
+        if (taken.length > 0 && estimateEntriesBytes(candidate) > maxBytes) {
+            break;
+        }
+        taken.push(entry);
+    }
+
+    const result = taken.length > 0 ? taken : [entries[0]];
+    if (result.length < entries.length) {
+        persistLogger.warn("trimmed batch to maxBytes", {
+            requested: entries.length,
+            taken: result.length,
+            maxBytes,
+            takenBytes: estimateEntriesBytes(result),
+        });
+    }
+
+    return result;
+}

package/src/shared/log/log-persist-hook.ts

@@ -0,0 +1,13 @@
+import type { LogEntry } from "./log-manager";
+
+type LogPersistOfferFn = (entry: LogEntry) => void;
+
+let offerFn: LogPersistOfferFn | null = null;
+
+export function registerLogPersistOffer(fn: LogPersistOfferFn | null): void {
+    offerFn = fn;
+}
+
+export function offerToLogPersist(entry: LogEntry): void {
+    offerFn?.(entry);
+}

package/src/shared/log/log-persist-logger.ts

@@ -0,0 +1,33 @@
+import { createLogger } from "./create-logger";
+
+/** User-facing / lifecycle logs — may be persisted like normal app logs. */
+export const LOG_PERSIST_TOPIC = "log-persist";
+
+/** Operational logs from the persist pipeline — never re-offered to avoid feedback loops. */
+export const LOG_PERSIST_INTERNAL_TOPIC = "log-persist-internal";
+
+const logger = createLogger({ module: "log-persist" });
+
+export function isLogPersistInternalTopic(topic: string | undefined): boolean {
+    return topic === LOG_PERSIST_INTERNAL_TOPIC;
+}
+
+type PersistLogData = Record<string, unknown>;
+
+function write(level: "debug" | "info" | "warn" | "error", message: string, data?: PersistLogData): void {
+    logger[level](message, {
+        topic: LOG_PERSIST_INTERNAL_TOPIC,
+        ...(data != null ? { data } : {}),
+    });
+}
+
+export const persistLogger = {
+    debug: (message: string, data?: PersistLogData) => write("debug", message, data),
+    info: (message: string, data?: PersistLogData) => write("info", message, data),
+    warn: (message: string, data?: PersistLogData) => write("warn", message, data),
+    error: (message: string, data?: PersistLogData) => write("error", message, data),
+};
+
+export function patchPersistLoggerScope(labels: Record<string, string>): void {
+    logger.resetScope({ labels });
+}

package/src/shared/log/log-persist-registry.ts

@@ -0,0 +1,252 @@
+import { mkdir, readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { createJsonFileStateStore } from "../../node/fs/json-state-store";
+import type { LogPersistProcessRecord, LogPersistRegistryState } from "./log-persist.interface";
+import { persistLogger } from "./log-persist-logger";
+
+const DEFAULT_REGISTRY: LogPersistRegistryState = {
+    updatedAt: new Date(0).toISOString(),
+    service: "",
+    processes: {},
+};
+
+/** Per-process shard — each process owns one file; no cross-process RMW. */
+interface LogPersistProcessShard extends LogPersistProcessRecord {
+    service: string;
+    updatedAt: string;
+}
+
+const DEFAULT_SHARD: LogPersistProcessShard = {
+    service: "",
+    updatedAt: new Date(0).toISOString(),
+    pid: 0,
+    enabledAt: new Date(0).toISOString(),
+    lastHeartbeatAt: new Date(0).toISOString(),
+};
+
+const legacyStores = new Map<
+    string,
+    ReturnType<typeof createJsonFileStateStore<LogPersistRegistryState, "processes">>
+>();
+
+const shardStores = new Map<string, ReturnType<typeof createJsonFileStateStore<LogPersistProcessShard>>>();
+
+function isLegacyRegistryFile(registryPath: string): boolean {
+    return registryPath.endsWith(".json");
+}
+
+function legacyStoreFor(filePath: string) {
+    let store = legacyStores.get(filePath);
+    if (store == null) {
+        store = createJsonFileStateStore<LogPersistRegistryState, "processes">({
+            filePath,
+            defaultState: DEFAULT_REGISTRY,
+            nestedKeys: ["processes"],
+            updatedAtKey: "updatedAt",
+        });
+        legacyStores.set(filePath, store);
+    }
+    return store;
+}
+
+function shardStoreFor(shardFilePath: string) {
+    let store = shardStores.get(shardFilePath);
+    if (store == null) {
+        store = createJsonFileStateStore<LogPersistProcessShard>({
+            filePath: shardFilePath,
+            defaultState: DEFAULT_SHARD,
+            updatedAtKey: "updatedAt",
+        });
+        shardStores.set(shardFilePath, store);
+    }
+    return store;
+}
+
+function shardFilePath(registryDir: string, process: string): string {
+    return join(registryDir, `${process}.json`);
+}
+
+/** Registry root directory — one shard file per process under this path. */
+export function defaultLogPersistRegistryPath(registryDir: string): string {
+    return join(registryDir, "log-persist-registry");
+}
+
+async function readShardedLogPersistRegistry(registryDir: string): Promise<LogPersistRegistryState> {
+    await mkdir(registryDir, { recursive: true });
+
+    let entries: string[];
+    try {
+        entries = await readdir(registryDir);
+    } catch {
+        return { ...DEFAULT_REGISTRY };
+    }
+
+    const processes: Record<string, LogPersistProcessRecord> = {};
+    let service = "";
+    let updatedAt = new Date(0).toISOString();
+
+    for (const name of entries) {
+        if (!name.endsWith(".json")) {
+            continue;
+        }
+
+        const processName = name.slice(0, -".json".length);
+        if (processName === "" || processName.startsWith("_")) {
+            continue;
+        }
+
+        try {
+            const raw = await readFile(join(registryDir, name), "utf8");
+            const shard = JSON.parse(raw) as Partial<LogPersistProcessShard>;
+            if (shard.service != null && shard.service !== "") {
+                service = shard.service;
+            }
+            if (shard.updatedAt != null && shard.updatedAt > updatedAt) {
+                updatedAt = shard.updatedAt;
+            }
+            processes[processName] = {
+                pid: shard.pid ?? 0,
+                enabledAt: shard.enabledAt ?? new Date(0).toISOString(),
+                lastHeartbeatAt: shard.lastHeartbeatAt ?? new Date(0).toISOString(),
+            };
+        } catch {
+            persistLogger.warn("registry shard read skipped — corrupt or missing", {
+                registryDir,
+                shard: name,
+            });
+        }
+    }
+
+    return { updatedAt, service, processes };
+}
+
+export async function registerLogPersistProcess(input: {
+    registryFilePath: string;
+    service: string;
+    process: string;
+}): Promise<void> {
+    const now = new Date().toISOString();
+    const record: LogPersistProcessRecord = {
+        pid: process.pid,
+        enabledAt: now,
+        lastHeartbeatAt: now,
+    };
+
+    if (isLegacyRegistryFile(input.registryFilePath)) {
+        await legacyStoreFor(input.registryFilePath).write({
+            service: input.service,
+            processes: {
+                [input.process]: record,
+            },
+        });
+    } else {
+        await shardStoreFor(shardFilePath(input.registryFilePath, input.process)).write({
+            service: input.service,
+            pid: record.pid,
+            enabledAt: record.enabledAt,
+            lastHeartbeatAt: record.lastHeartbeatAt,
+        });
+    }
+
+    persistLogger.info("registry process registered", {
+        registryFilePath: input.registryFilePath,
+        service: input.service,
+        process: input.process,
+        pid: record.pid,
+    });
+}
+
+export async function heartbeatLogPersistProcess(input: { registryFilePath: string; process: string }): Promise<void> {
+    if (isLegacyRegistryFile(input.registryFilePath)) {
+        const current = await legacyStoreFor(input.registryFilePath).read();
+        const existing = current.processes[input.process];
+        if (existing == null) {
+            persistLogger.warn("registry heartbeat skipped — process not registered", {
+                registryFilePath: input.registryFilePath,
+                process: input.process,
+            });
+            return;
+        }
+
+        await legacyStoreFor(input.registryFilePath).write({
+            processes: {
+                [input.process]: {
+                    ...existing,
+                    lastHeartbeatAt: new Date().toISOString(),
+                },
+            },
+        });
+        return;
+    }
+
+    const store = shardStoreFor(shardFilePath(input.registryFilePath, input.process));
+    const current = await store.read();
+    if (current.pid === 0) {
+        persistLogger.warn("registry heartbeat skipped — process not registered", {
+            registryFilePath: input.registryFilePath,
+            process: input.process,
+        });
+        return;
+    }
+
+    await store.write({
+        lastHeartbeatAt: new Date().toISOString(),
+    });
+}
+
+export async function unregisterLogPersistProcess(input: { registryFilePath: string; process: string }): Promise<void> {
+    if (isLegacyRegistryFile(input.registryFilePath)) {
+        const current = await legacyStoreFor(input.registryFilePath).read();
+        const existing = current.processes[input.process];
+        if (existing == null) {
+            persistLogger.warn("registry unregister skipped — process not found", {
+                registryFilePath: input.registryFilePath,
+                process: input.process,
+            });
+            return;
+        }
+
+        await legacyStoreFor(input.registryFilePath).write({
+            processes: {
+                [input.process]: {
+                    ...existing,
+                    pid: 0,
+                    lastHeartbeatAt: new Date(0).toISOString(),
+                },
+            },
+        });
+
+        persistLogger.info("registry process unregistered", {
+            registryFilePath: input.registryFilePath,
+            process: input.process,
+        });
+        return;
+    }
+
+    const store = shardStoreFor(shardFilePath(input.registryFilePath, input.process));
+    const current = await store.read();
+    if (current.pid === 0) {
+        persistLogger.warn("registry unregister skipped — process not found", {
+            registryFilePath: input.registryFilePath,
+            process: input.process,
+        });
+        return;
+    }
+
+    await store.write({
+        pid: 0,
+        lastHeartbeatAt: new Date(0).toISOString(),
+    });
+
+    persistLogger.info("registry process unregistered", {
+        registryFilePath: input.registryFilePath,
+        process: input.process,
+    });
+}
+
+export async function readLogPersistRegistry(registryFilePath: string): Promise<LogPersistRegistryState> {
+    if (isLegacyRegistryFile(registryFilePath)) {
+        return legacyStoreFor(registryFilePath).read();
+    }
+    return readShardedLogPersistRegistry(registryFilePath);
+}

package/src/shared/log/log-persist.config.ts

@@ -0,0 +1,97 @@
+import { envBool, envInt, envMs } from "../../node/env/env-parsers";
+import type { LogPersistConfig } from "./log-persist.interface";
+
+export const LOG_PERSIST_LOADER_KEY = "LogBatch.0";
+
+export const DEFAULT_LOG_PERSIST_CONFIG: LogPersistConfig = {
+    fast: {
+        maxBufferEntries: 500,
+    },
+    medium: {
+        debounceMs: 5_000,
+        maxEntries: 20,
+        maxBytes: 512 * 1024,
+        maxBufferEntries: 1_000,
+    },
+    slow: {
+        maxEntries: 400,
+        maxBytes: 512 * 1024,
+        maxAgeMs: 120_000,
+        maxBufferEntries: 2_000,
+    },
+    postTimeoutMs: 30_000,
+    shutdownDrainTimeoutMs: 15_000,
+};
+
+export function resolveLogPersistConfig(partial?: Partial<LogPersistConfig>): LogPersistConfig {
+    const defaults = DEFAULT_LOG_PERSIST_CONFIG;
+    return {
+        fast: { ...defaults.fast, ...partial?.fast },
+        medium: { ...defaults.medium, ...partial?.medium },
+        slow: { ...defaults.slow, ...partial?.slow },
+        postTimeoutMs: partial?.postTimeoutMs ?? defaults.postTimeoutMs,
+        shutdownDrainTimeoutMs: partial?.shutdownDrainTimeoutMs ?? defaults.shutdownDrainTimeoutMs,
+    };
+}
+
+export function logPersistEnabledFromEnv(env: NodeJS.ProcessEnv = process.env): boolean {
+    return envBool("LOG_PERSIST_ENABLED", false, env);
+}
+
+export function logPersistServiceFromEnv(env: NodeJS.ProcessEnv = process.env): string | undefined {
+    const raw = env.LOG_PERSIST_SERVICE?.trim();
+    return raw != null && raw !== "" ? raw : undefined;
+}
+
+export function logPersistProcessFromEnv(env: NodeJS.ProcessEnv = process.env): string | undefined {
+    const raw = env.LOG_PERSIST_PROCESS?.trim();
+    return raw != null && raw !== "" ? raw : undefined;
+}
+
+export function logPersistRegistryDirFromEnv(env: NodeJS.ProcessEnv = process.env): string | undefined {
+    const configured = env.LOG_PERSIST_REGISTRY_DIR?.trim();
+    if (configured != null && configured !== "") {
+        return configured;
+    }
+    const runtime = env.RUNTIME_DATA_DIR?.trim();
+    return runtime != null && runtime !== "" ? runtime : undefined;
+}
+
+export function logPersistConfigFromEnv(env: NodeJS.ProcessEnv = process.env): LogPersistConfig {
+    return resolveLogPersistConfig({
+        fast: {
+            maxBufferEntries: envInt("LOG_FAST_MAX_BUFFER_ENTRIES", DEFAULT_LOG_PERSIST_CONFIG.fast.maxBufferEntries, {
+                env,
+                min: 1,
+            }),
+        },
+        medium: {
+            debounceMs: envMs("LOG_MEDIUM_DEBOUNCE_MS", DEFAULT_LOG_PERSIST_CONFIG.medium.debounceMs, { env }),
+            maxEntries: envInt("LOG_MEDIUM_MAX_ENTRIES", DEFAULT_LOG_PERSIST_CONFIG.medium.maxEntries, {
+                env,
+                min: 1,
+            }),
+            maxBytes: envInt("LOG_MEDIUM_MAX_BYTES", DEFAULT_LOG_PERSIST_CONFIG.medium.maxBytes, { env, min: 1 }),
+            maxBufferEntries: envInt(
+                "LOG_MEDIUM_MAX_BUFFER_ENTRIES",
+                DEFAULT_LOG_PERSIST_CONFIG.medium.maxBufferEntries,
+                { env, min: 1 },
+            ),
+        },
+        slow: {
+            maxEntries: envInt("LOG_SLOW_MAX_ENTRIES", DEFAULT_LOG_PERSIST_CONFIG.slow.maxEntries, { env, min: 1 }),
+            maxBytes: envInt("LOG_SLOW_MAX_BYTES", DEFAULT_LOG_PERSIST_CONFIG.slow.maxBytes, { env, min: 1 }),
+            maxAgeMs: envMs("LOG_SLOW_MAX_AGE_MS", DEFAULT_LOG_PERSIST_CONFIG.slow.maxAgeMs, { env }),
+            maxBufferEntries: envInt("LOG_SLOW_MAX_BUFFER_ENTRIES", DEFAULT_LOG_PERSIST_CONFIG.slow.maxBufferEntries, {
+                env,
+                min: 1,
+            }),
+        },
+        postTimeoutMs: envMs("LOG_PERSIST_POST_TIMEOUT_MS", DEFAULT_LOG_PERSIST_CONFIG.postTimeoutMs, { env }),
+        shutdownDrainTimeoutMs: envMs(
+            "LOG_PERSIST_SHUTDOWN_DRAIN_MS",
+            DEFAULT_LOG_PERSIST_CONFIG.shutdownDrainTimeoutMs,
+            { env },
+        ),
+    });
+}

package/src/shared/log/log-persist.interface.ts

@@ -0,0 +1,132 @@
+import type { TargetDraft } from "../../core.interface";
+import type { List } from "../../list/list.interface";
+import type { LogEntry } from "./log-manager";
+
+/** Persistence lane — maps to List tagList and flush policy. */
+export type LogPersistLane = "fast" | "medium" | "slow";
+
+export interface LogPersistFastConfig {
+    maxBufferEntries: number;
+}
+
+export interface LogPersistMediumConfig {
+    debounceMs: number;
+    maxEntries: number;
+    maxBytes: number;
+    maxBufferEntries: number;
+}
+
+export interface LogPersistSlowConfig {
+    maxEntries: number;
+    maxBytes: number;
+    maxAgeMs: number;
+    maxBufferEntries: number;
+}
+
+export interface LogPersistConfig {
+    fast: LogPersistFastConfig;
+    medium: LogPersistMediumConfig;
+    slow: LogPersistSlowConfig;
+    /** postListCreate / network ceiling per batch */
+    postTimeoutMs: number;
+    /** Graceful shutdown drain ceiling */
+    shutdownDrainTimeoutMs: number;
+}
+
+export interface EnableLogPersistOptions {
+    service: string;
+    process: string;
+    /** Shared registry root directory — all processes use the same path; each writes its own shard file. */
+    registryFilePath?: string;
+    config?: Partial<LogPersistConfig>;
+}
+
+export interface EnsureLogPersistFromEnvOptions {
+    /** Overrides LOG_PERSIST_PROCESS when set. */
+    process?: string;
+    /** Overrides LOG_PERSIST_SERVICE when set. */
+    service?: string;
+    registryFilePath?: string;
+    config?: Partial<LogPersistConfig>;
+}
+
+export interface LogPersistProcessRecord {
+    pid: number;
+    enabledAt: string;
+    lastHeartbeatAt: string;
+}
+
+export interface LogPersistRegistryState {
+    updatedAt: string;
+    service: string;
+    processes: Record<string, LogPersistProcessRecord>;
+}
+
+export interface LogPersistQueueStats {
+    fast: number;
+    fastRetry: number;
+    medium: number;
+    slow: number;
+}
+
+export interface LogPersistQueueBytesStats {
+    fast: number;
+    fastRetry: number;
+    medium: number;
+    slow: number;
+}
+
+export interface LogPersistStats {
+    enabled: boolean;
+    draining: boolean;
+    service: string | null;
+    process: string | null;
+    queues: LogPersistQueueStats;
+    queueBytes: LogPersistQueueBytesStats;
+    mediumDebouncePending: boolean;
+    laneFlushing: Record<LogPersistLane, boolean>;
+    consecutiveFailures: Record<LogPersistLane, number>;
+    config: LogPersistConfig;
+    lastFlushAt: Record<LogPersistLane, string | null>;
+    lastError: string | null;
+    lastErrorAt: string | null;
+}
+
+export interface WaitForLogPersistReadyOptions {
+    registryFilePath: string;
+    service: string;
+    expectedProcesses: string[];
+    timeoutMs?: number;
+    pollIntervalMs?: number;
+    /** Process heartbeat older than this is treated as missing. */
+    staleMs?: number;
+}
+
+export interface LogPersistReadySnapshot {
+    ok: boolean;
+    missing: string[];
+    stale: string[];
+    registered: string[];
+}
+
+export interface LogBatchMeta {
+    service: string;
+    process: string;
+    pid: number;
+    lane: LogPersistLane;
+    count: number;
+    from: number | null;
+    to: number | null;
+    flushedAt: number;
+    idempotencyKey: string;
+}
+
+export interface BuildLogListDraftInput {
+    service: string;
+    process: string;
+    lane: LogPersistLane;
+    idempotencyKey: string;
+    entries: LogEntry[];
+}
+
+export type LogListDraft = TargetDraft<List>;