target-supabase-sdk 0.2.12 → 0.2.13

package/src/shared/log/spool/writer.ts

@@ -0,0 +1,273 @@
+import { mkdir, rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import type { LogEntry } from "../core/log-manager";
+import { estimateEntriesBytes, takeEntriesWithinBytes } from "../upload/flush";
+import { registerLogPersistOffer } from "../upload/hook";
+import { isLogPersistInternalTopic, spoolLogger } from "../upload/logger";
+import { DEFAULT_LOG_SPOOL_CONFIG, resolveLogSpoolConfig } from "./config";
+import { buildLogSpoolBatchBaseName, computeLogSpoolShortHash } from "./file";
+import type { EnableLogSpoolOptions, LogSpoolBatchFile, LogSpoolProcessRole, LogSpoolWriterStats } from "./interface";
+import { logSpoolProcessDir, resolveLogSpoolRoot } from "./paths";
+
+class LogSpoolWriter {
+    private static instance: LogSpoolWriter | null = null;
+
+    private enabled = false;
+    private buffer: LogEntry[] = [];
+    private flushInProgress = false;
+    private debounceTimer: ReturnType<typeof setTimeout> | null = null;
+    private ageTimer: ReturnType<typeof setTimeout> | null = null;
+    private serviceId: string | null = null;
+    private processRole: LogSpoolProcessRole | null = null;
+    private serviceValue: string | null = null;
+    private spoolRoot: string | null = null;
+    private config = DEFAULT_LOG_SPOOL_CONFIG.writer;
+
+    static getInstance(): LogSpoolWriter {
+        LogSpoolWriter.instance ??= new LogSpoolWriter();
+        return LogSpoolWriter.instance;
+    }
+
+    async enable(options: EnableLogSpoolOptions): Promise<void> {
+        if (this.enabled) {
+            spoolLogger.warn("log spool writer already enabled", {
+                serviceId: this.serviceId,
+                processRole: this.processRole,
+            });
+            return;
+        }
+
+        this.serviceId = options.serviceId;
+        this.processRole = options.processRole;
+        this.serviceValue = options.serviceValue;
+        this.spoolRoot = options.spoolRoot ?? resolveLogSpoolRoot();
+        this.config = resolveLogSpoolConfig(options.config).writer;
+        this.enabled = true;
+
+        registerLogPersistOffer((entry) => this.offer(entry));
+
+        spoolLogger.info("log spool writer enabled", {
+            serviceId: options.serviceId,
+            processRole: options.processRole,
+            serviceValue: options.serviceValue,
+            spoolRoot: this.spoolRoot,
+            config: this.config,
+        });
+    }
+
+    async shutdown(): Promise<void> {
+        this.cancelDebounce();
+        this.cancelAgeFlush();
+        registerLogPersistOffer(null);
+        this.enabled = false;
+
+        while (this.buffer.length > 0) {
+            await this.flushChunk({ reason: "shutdown" });
+        }
+
+        spoolLogger.info("log spool writer shutdown complete", {
+            serviceId: this.serviceId,
+            processRole: this.processRole,
+        });
+    }
+
+    getStats(): LogSpoolWriterStats {
+        return {
+            enabled: this.enabled,
+            bufferedEntries: this.buffer.length,
+            bufferedBytes: estimateEntriesBytes(this.buffer),
+            flushInProgress: this.flushInProgress,
+            debouncePending: this.debounceTimer != null,
+            ageFlushPending: this.ageTimer != null,
+            serviceId: this.serviceId,
+            processRole: this.processRole,
+        };
+    }
+
+    offer(entry: LogEntry): void {
+        if (isLogPersistInternalTopic(entry.topic)) {
+            return;
+        }
+
+        if (!this.enabled) {
+            return;
+        }
+
+        if (this.buffer.length === 0) {
+            this.scheduleAgeFlush();
+        }
+
+        this.buffer.push(entry);
+
+        if (this.shouldScheduleDebouncedFlush()) {
+            this.scheduleDebouncedFlush();
+        }
+    }
+
+    private shouldScheduleDebouncedFlush(): boolean {
+        return (
+            this.buffer.length >= this.config.maxEntries || estimateEntriesBytes(this.buffer) >= this.config.maxBytes
+        );
+    }
+
+    private scheduleDebouncedFlush(): void {
+        if (this.debounceTimer != null) {
+            return;
+        }
+
+        spoolLogger.debug("log spool flush debounce scheduled", {
+            debounceMs: this.config.flushDebounceMs,
+            bufferedEntries: this.buffer.length,
+        });
+
+        this.debounceTimer = setTimeout(() => {
+            this.debounceTimer = null;
+            void this.flushChunk({ reason: "debounce" });
+        }, this.config.flushDebounceMs);
+        this.debounceTimer.unref?.();
+    }
+
+    private scheduleAgeFlush(): void {
+        if (this.ageTimer != null) {
+            return;
+        }
+
+        spoolLogger.debug("log spool age flush scheduled", {
+            maxAgeMs: this.config.maxAgeMs,
+        });
+
+        this.ageTimer = setTimeout(() => {
+            this.ageTimer = null;
+            void this.flushChunk({ reason: "maxAge" });
+        }, this.config.maxAgeMs);
+        this.ageTimer.unref?.();
+    }
+
+    private cancelDebounce(): void {
+        if (this.debounceTimer != null) {
+            clearTimeout(this.debounceTimer);
+            this.debounceTimer = null;
+        }
+    }
+
+    private cancelAgeFlush(): void {
+        if (this.ageTimer != null) {
+            clearTimeout(this.ageTimer);
+            this.ageTimer = null;
+        }
+    }
+
+    private async flushChunk(input: { reason: string }): Promise<void> {
+        if (this.flushInProgress || this.buffer.length === 0) {
+            return;
+        }
+        if (this.serviceId == null || this.processRole == null || this.serviceValue == null || this.spoolRoot == null) {
+            return;
+        }
+
+        this.cancelDebounce();
+        this.cancelAgeFlush();
+        this.flushInProgress = true;
+        try {
+            const byCount = this.buffer.slice(0, this.config.maxEntries);
+            const entries = takeEntriesWithinBytes(byCount, this.config.maxBytes);
+            this.buffer = this.buffer.slice(entries.length);
+
+            if (entries.length === 0) {
+                if (this.buffer.length > 0) {
+                    this.scheduleAgeFlush();
+                } else {
+                    this.cancelAgeFlush();
+                }
+                return;
+            }
+
+            const batchFile = await this.writeBatchTmp({
+                entries,
+                serviceId: this.serviceId,
+                processRole: this.processRole,
+                serviceValue: this.serviceValue,
+                spoolRoot: this.spoolRoot,
+            });
+
+            spoolLogger.info("log spool flushed to tmp", {
+                reason: input.reason,
+                fileName: batchFile.fileName,
+                entryCount: entries.length,
+                remainingBuffered: this.buffer.length,
+            });
+
+            if (this.buffer.length > 0) {
+                this.scheduleAgeFlush();
+                if (this.shouldScheduleDebouncedFlush()) {
+                    this.scheduleDebouncedFlush();
+                }
+            } else {
+                this.cancelAgeFlush();
+            }
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            spoolLogger.error("log spool flush failed", { reason: input.reason, error: message });
+            if (this.buffer.length > 0) {
+                this.scheduleAgeFlush();
+            }
+        } finally {
+            this.flushInProgress = false;
+        }
+    }
+
+    private async writeBatchTmp(input: {
+        entries: LogEntry[];
+        serviceId: string;
+        processRole: LogSpoolProcessRole;
+        serviceValue: string;
+        spoolRoot: string;
+    }): Promise<{ fileName: string; tmpPath: string }> {
+        const first = input.entries[0];
+        const shortHash = computeLogSpoolShortHash({
+            serviceValue: input.serviceValue,
+            serviceId: input.serviceId,
+            processRole: input.processRole,
+            entries: input.entries,
+        });
+        const baseName = buildLogSpoolBatchBaseName({
+            processRole: input.processRole,
+            firstEntryTimestamp: first.timestamp,
+            shortHash,
+        });
+        const fileName = `${baseName}.tmp`;
+        const processDir = logSpoolProcessDir(input.spoolRoot, input.serviceId, input.processRole);
+        await mkdir(processDir, { recursive: true });
+
+        const tmpPath = join(processDir, fileName);
+        const partPath = `${tmpPath}.part`;
+        const payload: LogSpoolBatchFile = {
+            entries: input.entries,
+            meta: {
+                serviceId: input.serviceId,
+                processRole: input.processRole,
+                serviceValue: input.serviceValue,
+            },
+        };
+        const content = `${JSON.stringify(payload)}\n`;
+
+        await writeFile(partPath, content, "utf8");
+        await rename(partPath, tmpPath);
+
+        return { fileName, tmpPath };
+    }
+}
+
+export { LogSpoolWriter };
+
+export async function enableLogSpoolWriter(options: EnableLogSpoolOptions): Promise<void> {
+    await LogSpoolWriter.getInstance().enable(options);
+}
+
+export async function shutdownLogSpoolWriter(): Promise<void> {
+    await LogSpoolWriter.getInstance().shutdown();
+}
+
+export function getLogSpoolWriterStats(): LogSpoolWriterStats {
+    return LogSpoolWriter.getInstance().getStats();
+}

package/src/shared/log/{log-persist-batch-id.ts → upload/batch-id.ts}

@@ -1,6 +1,6 @@
 import { createHash } from "node:crypto";
-import type { LogEntry } from "./log-manager";
-import type { LogPersistLane } from "./log-persist.interface";
+import type { LogEntry } from "../core/log-manager";
+import type { LogPersistLane } from "./interface";
 
 const BATCH_ID_VERSION = 1;
 

package/src/shared/log/upload/env.ts

@@ -0,0 +1,26 @@
+import { envBool } from "../../../node/env/env-parsers";
+
+/** Master switch — env name kept for compatibility (`LOG_PERSIST_ENABLED`). */
+export function logSpoolEnabledFromEnv(env: NodeJS.ProcessEnv = process.env): boolean {
+    return envBool("LOG_PERSIST_ENABLED", false, env);
+}
+
+export function logPersistServiceFromEnv(env: NodeJS.ProcessEnv = process.env): string | undefined {
+    const raw = env.LOG_PERSIST_SERVICE?.trim();
+    return raw != null && raw !== "" ? raw : undefined;
+}
+
+export function logPersistProcessFromEnv(env: NodeJS.ProcessEnv = process.env): string | undefined {
+    const raw = env.LOG_PERSIST_PROCESS?.trim();
+    return raw != null && raw !== "" ? raw : undefined;
+}
+
+/** `LOG_PERSIST_REGISTRY_DIR` or `RUNTIME_DATA_DIR`. */
+export function runtimeDataDirFromEnv(env: NodeJS.ProcessEnv = process.env): string | undefined {
+    const configured = env.LOG_PERSIST_REGISTRY_DIR?.trim();
+    if (configured != null && configured !== "") {
+        return configured;
+    }
+    const runtime = env.RUNTIME_DATA_DIR?.trim();
+    return runtime != null && runtime !== "" ? runtime : undefined;
+}

package/src/shared/log/{log-persist-flush.ts → upload/flush.ts}

@@ -1,10 +1,10 @@
-import { isCreateTargetAlreadyExistsError } from "../../core.api";
-import { postLogListCreate } from "../../list/list.api";
-import { CategoryList } from "../../list/list.interface";
-import type { LogEntry } from "./log-manager";
-import type { BuildLogListDraftInput, LogListDraft } from "./log-persist.interface";
-import { LOG_PERSIST_LOADER_KEY } from "./log-persist.interface";
-import { persistLogger } from "./log-persist-logger";
+import { isCreateTargetAlreadyExistsError } from "../../../core.api";
+import { postLogListCreate } from "../../../list/list.api";
+import { CategoryList } from "../../../list/list.interface";
+import type { LogEntry } from "../core/log-manager";
+import type { BuildLogListDraftInput, LogListDraft } from "./interface";
+import { LOG_PERSIST_LOADER_KEY } from "./interface";
+import { persistLogger } from "./logger";
 
 export function estimateEntriesBytes(entries: LogEntry[]): number {
     if (entries.length === 0) {
@@ -19,6 +19,7 @@ export function estimateEntriesBytes(entries: LogEntry[]): number {
 
 export function buildLogListDraft(input: BuildLogListDraftInput): LogListDraft {
     const { service, process, lane, idempotencyKey, entries } = input;
+    // TODO: `lane` is legacy transport metadata — scheduled for removal; callers still pass a fixed value for decode compatibility.
     const first = entries[0];
     const last = entries[entries.length - 1];
 

package/src/shared/log/{log-persist-hook.ts → upload/hook.ts}

@@ -1,4 +1,4 @@
-import type { LogEntry } from "./log-manager";
+import type { LogEntry } from "../core/log-manager";
 
 type LogPersistOfferFn = (entry: LogEntry) => void;
 

package/src/shared/log/upload/interface.ts

@@ -0,0 +1,38 @@
+import type { TargetDraft } from "../../../core.interface";
+import type { List } from "../../../list/list.interface";
+import type { LogEntry } from "../core/log-manager";
+
+/**
+ * `ListDetails.loaderKey` stamped on every persisted log batch (write + read side).
+ * Business type: **LogBatch** (transport flush). Pairs with log-service **LogTrace**
+ * (`LogTrace.0`) after rekeying by `traceId`.
+ */
+export const LOG_PERSIST_LOADER_KEY = "LogBatch.0";
+
+/** Alias for {@link LOG_PERSIST_LOADER_KEY} — preferred when contrasting with LogTrace. */
+export const LOG_BATCH_LOADER_KEY = LOG_PERSIST_LOADER_KEY;
+
+/** Legacy transport metadata — fixed `"slow"` for spool uploads; required for downstream decode today. */
+export type LogPersistLane = "fast" | "medium" | "slow";
+
+export interface LogBatchMeta {
+    service: string;
+    process: string;
+    pid: number;
+    lane: LogPersistLane;
+    count: number;
+    from: number | null;
+    to: number | null;
+    flushedAt: number;
+    idempotencyKey: string;
+}
+
+export interface BuildLogListDraftInput {
+    service: string;
+    process: string;
+    lane: LogPersistLane;
+    idempotencyKey: string;
+    entries: LogEntry[];
+}
+
+export type LogListDraft = TargetDraft<List>;

package/src/shared/log/upload/logger.ts

@@ -0,0 +1,37 @@
+import { createLogger } from "../core/create-logger";
+
+/** User-facing / lifecycle logs — may be persisted like normal app logs. */
+export const LOG_PERSIST_TOPIC = "log-persist";
+
+/** Operational logs from the persist pipeline — never re-offered to avoid feedback loops. */
+export const LOG_PERSIST_INTERNAL_TOPIC = "log-persist-internal";
+
+type PipelineLogData = Record<string, unknown>;
+
+function createPipelineLogger(module: string) {
+    const logger = createLogger({ module });
+
+    function write(level: "debug" | "info" | "warn" | "error", message: string, data?: PipelineLogData): void {
+        logger[level](message, {
+            topic: LOG_PERSIST_INTERNAL_TOPIC,
+            ...(data != null ? { data } : {}),
+        });
+    }
+
+    return {
+        debug: (message: string, data?: PipelineLogData) => write("debug", message, data),
+        info: (message: string, data?: PipelineLogData) => write("info", message, data),
+        warn: (message: string, data?: PipelineLogData) => write("warn", message, data),
+        error: (message: string, data?: PipelineLogData) => write("error", message, data),
+    };
+}
+
+/** Supabase upload path (`postLogBatch`). */
+export const persistLogger = createPipelineLogger("log-upload");
+
+/** File spool writer + collector. */
+export const spoolLogger = createPipelineLogger("log-spool");
+
+export function isLogPersistInternalTopic(topic: string | undefined): boolean {
+    return topic === LOG_PERSIST_INTERNAL_TOPIC;
+}