ravenflow 0.1.0 → 0.2.0

package/dist/index.d.ts

@@ -1,15 +1,9 @@
-import { Job, ConnectionOptions, Queue, QueueEvents } from 'bullmq';
-import * as pino from 'pino';
+import { z } from 'zod';
+import { Job, Queue, QueueEvents, ConnectionOptions, Worker } from 'bullmq';
+import pino from 'pino';
 
-interface RavenflowConfig<TQueues extends string = string, TJobs extends string = string> {
-    defaultQueue: TQueues;
-    queues: TQueues[];
-    jobs: Record<TJobs, JobConfig>;
-}
-type JobNames<T extends RavenflowConfig> = T extends RavenflowConfig<infer _, infer TJobs> ? TJobs : never;
-type QueueNames<T extends RavenflowConfig> = T extends RavenflowConfig<infer TQueues, infer _> ? TQueues : never;
 interface BaseJobConfig {
-    queueName: string;
+    queue?: string;
     cron?: string;
     attempts?: number;
     backoff?: {
@@ -17,67 +11,207 @@ interface BaseJobConfig {
         delay: number;
     };
 }
+declare const zRateLimitConfig: z.ZodUnion<readonly [z.ZodObject<{
+    perSecond: z.ZodNumber;
+}, z.core.$strip>, z.ZodObject<{
+    perMinute: z.ZodNumber;
+}, z.core.$strip>]>;
+type RateLimitConfig = z.infer<typeof zRateLimitConfig>;
+interface MemoryEtlStepConfig {
+    rateLimit?: RateLimitConfig;
+}
+interface StepConfig extends MemoryEtlStepConfig {
+    attempts?: number;
+    backoff?: {
+        type: "fixed" | "exponential";
+        delay: number;
+    };
+    queue?: string;
+}
+interface EtlExtractResult<TRaw, TCursor> {
+    data: TRaw[];
+    cursor: TCursor | null;
+    hasMore: boolean;
+}
+declare const zJobStatus: z.ZodEnum<{
+    pending: "pending";
+    in_progress: "in_progress";
+    completed: "completed";
+    failed: "failed";
+    stalled: "stalled";
+}>;
+type JobStatus = z.infer<typeof zJobStatus>;
+
 interface SimpleJobConfig<TData, TReturn> extends BaseJobConfig {
-    type: "job";
+    type: "simple";
     handler: (job: Job, params: TData) => Promise<TReturn>;
 }
-interface EtlJobConfig<TRaw, TProcessed> extends BaseJobConfig {
-    type: "etl";
-    extract: (page: number) => Promise<{
-        data: TRaw[];
-        hasNextPage?: boolean;
-    }>;
+declare function defineSimpleJob<TData, TReturn>(config: Omit<SimpleJobConfig<TData, TReturn>, "type">): SimpleJobConfig<TData, TReturn>;
+
+interface MemoryEtlJobConfig<TRaw, TProcessed, TCursor = number> extends BaseJobConfig {
+    type: "memory-etl";
+    extract: (cursor: TCursor | null) => Promise<EtlExtractResult<TRaw, TCursor>>;
+    transform: (data: TRaw) => Promise<TProcessed>;
+    load: (data: TProcessed[]) => Promise<void>;
+    steps?: {
+        extract?: MemoryEtlStepConfig;
+        transform?: MemoryEtlStepConfig;
+        load?: MemoryEtlStepConfig;
+    };
+}
+declare function defineMemoryEtlJob<TRaw, TProcessed, TCursor = number>(config: Omit<MemoryEtlJobConfig<TRaw, TProcessed, TCursor>, "type">): MemoryEtlJobConfig<TRaw, TProcessed, TCursor>;
+
+interface DistributedEtlJobConfig<TRaw, TProcessed, TCursor = number> extends BaseJobConfig {
+    type: "distributed-etl";
+    extract: (cursor: TCursor | null) => Promise<EtlExtractResult<TRaw, TCursor>>;
+    transform: (data: TRaw) => Promise<TProcessed>;
+    load: (data: TProcessed[]) => Promise<void>;
+    steps?: {
+        extract?: StepConfig;
+        transform?: StepConfig;
+        load?: StepConfig;
+    };
+}
+declare function defineDistributedEtlJob<TRaw, TProcessed, TCursor = number>(config: Omit<DistributedEtlJobConfig<TRaw, TProcessed, TCursor>, "type">): DistributedEtlJobConfig<TRaw, TProcessed, TCursor>;
+
+interface StatefulEtlJobConfig<TRaw, TProcessed, TCursor = string> extends BaseJobConfig {
+    type: "stateful-etl";
+    extract: (cursor: TCursor | null) => Promise<EtlExtractResult<TRaw, TCursor>>;
     transform: (data: TRaw) => Promise<TProcessed>;
     load: (data: TProcessed[]) => Promise<void>;
-    transformQueue?: string;
+    /**
+     * When true, only one run of this job can be active at a time.
+     * If a run is already pending or in_progress (with a live heartbeat), the new trigger is skipped.
+     * Defaults to false — each trigger creates an independent run.
+     */
+    singletonRun?: boolean;
+    stallTimeoutMs?: number;
+    maxStallRetries?: number;
+    steps?: {
+        extract?: StepConfig;
+        transform?: StepConfig;
+        load?: StepConfig;
+    };
+}
+declare function defineStatefulEtlJob<TRaw, TProcessed, TCursor = string>(config: Omit<StatefulEtlJobConfig<TRaw, TProcessed, TCursor>, "type">): StatefulEtlJobConfig<TRaw, TProcessed, TCursor>;
+
+interface TokenBucketJobConfig<TItem, TResult> extends BaseJobConfig {
+    type: "token-bucket";
+    cron: string;
+    rate: {
+        amount: number;
+        interval: "minute" | "hour" | "day";
+    };
+    activeHours?: {
+        start: number;
+        end: number;
+        timezone?: string;
+    };
+    maxPerBeat?: number;
+    getItems: (budget: number) => Promise<TItem[]>;
+    process: (item: TItem) => Promise<TResult>;
+    steps?: {
+        process?: StepConfig;
+    };
+}
+declare function defineTokenBucketJob<TItem, TResult>(config: Omit<TokenBucketJobConfig<TItem, TResult>, "type">): TokenBucketJobConfig<TItem, TResult>;
+
+type JobConfig = SimpleJobConfig<any, any> | MemoryEtlJobConfig<any, any, any> | DistributedEtlJobConfig<any, any, any> | StatefulEtlJobConfig<any, any, any> | TokenBucketJobConfig<any, any>;
+
+interface RetentionConfig {
+    /** Number of completed jobs to keep per queue. Set to 0 to remove immediately. */
+    completed?: number;
+    /** Number of failed jobs to keep per queue. Set to 0 to remove immediately. */
+    failed?: number;
+}
+interface RavenflowConfig<TJobs extends string = string> {
+    defaultQueue: string;
+    jobs: Record<TJobs, JobConfig>;
+}
+type JobNames<T extends RavenflowConfig> = T extends RavenflowConfig<infer TJobs> ? TJobs : never;
+declare function defineConfig<const T extends RavenflowConfig>(config: T): T;
+
+interface EtlResult {
+    extractedCount: number;
+    transformedCount: number;
+    loadedCount: number;
 }
-type JobConfig = SimpleJobConfig<any, any> | EtlJobConfig<any, any>;
 type JobProxy<T extends JobConfig> = T extends SimpleJobConfig<infer TData, infer TReturn> ? {
     send: (params: TData) => Promise<Job<TData, TReturn>>;
-} : T extends EtlJobConfig<any, any> ? {
-    send: (params?: {
-        startPage?: number;
-        batchSize?: number;
-    }) => Promise<void>;
+    sendAndWait: (params: TData) => Promise<TReturn>;
+} : T extends MemoryEtlJobConfig<any, any, any> ? {
+    send: () => Promise<void>;
+    sendAndWait: () => Promise<EtlResult>;
+} : T extends DistributedEtlJobConfig<any, any> ? {
+    send: () => Promise<void>;
+    sendAndWait: () => Promise<EtlResult>;
+} : T extends StatefulEtlJobConfig<any, any, any> ? {
+    send: () => Promise<void>;
+    sendAndWait: () => Promise<void>;
+} : T extends TokenBucketJobConfig<any, any> ? {
+    send: () => Promise<void>;
+    sendAndWait: () => Promise<void>;
 } : never;
 type JobsProxy<TJobs extends Record<string, JobConfig>> = {
     [K in keyof TJobs]: JobProxy<TJobs[K]>;
 };
 
+type Logger = pino.Logger;
 interface LoggerOptions {
     level?: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
     pretty?: boolean;
 }
-declare function createLogger(options?: LoggerOptions): pino.Logger<never>;
-type Logger = ReturnType<typeof createLogger>;
+declare function initLogger(options?: LoggerOptions): Logger;
+declare function getLogger(): Logger;
+
+type QueueStore = Record<string, {
+    queue: Queue;
+    events: QueueEvents;
+}>;
+
+declare class StallWatcher {
+    private readonly workerQueue;
+    private readonly statefulEtlConfigs;
+    private readonly userQueues;
+    constructor(jobConfigs: Record<string, JobConfig>, userQueues: Record<string, {
+        queue: Queue;
+    }>, defaultQueue: string, workerQueue: Queue);
+    canProcess(jobName: string): boolean;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    process(job: Job): Promise<void>;
+}
 
+interface MongoConnectionObject {
+    host: string;
+    port?: number;
+    username?: string;
+    password?: string;
+    database?: string;
+}
+interface RavenflowAppOptions {
+    redis: ConnectionOptions;
+    mongo?: string | MongoConnectionObject;
+    retention?: RetentionConfig;
+    logger?: LoggerOptions;
+}
 declare class RavenflowApp<TConfig extends RavenflowConfig = RavenflowConfig> {
-    protected connection: ConnectionOptions;
+    protected redisConnection: ConnectionOptions;
     protected defaultConcurrency: number;
-    protected logger: Logger;
     protected config: TConfig;
+    protected mongoUri: string | null;
     jobs: JobsProxy<TConfig["jobs"]>;
     protected jobConfigs: Record<JobNames<TConfig>, JobConfig>;
-    protected queues: Record<QueueNames<TConfig>, {
-        queue: Queue;
-        events: QueueEvents;
-    }>;
-    constructor(connection: ConnectionOptions, config: TConfig, loggerOptions?: LoggerOptions);
-    private initQueues;
-    private initJobs;
-    private sendJob;
-    startWorker(queueName: QueueNames<TConfig>, options?: {
+    protected queues: QueueStore;
+    protected workers: Worker[];
+    protected stallWatcher: StallWatcher | null;
+    constructor(config: TConfig, options: RavenflowAppOptions);
+    startWorker(queueName: string, options?: {
         concurrency?: number;
-    }): void;
-    private printWorkerBanner;
-    private setupWorkerEvents;
-    private syncJobSchedulers;
-    processJob(job: Job): Promise<any>;
+    }): Promise<void>;
+    close(): Promise<void>;
 }
 
-declare function createRavenflowApp<TConfig extends RavenflowConfig>(config: TConfig, connection: ConnectionOptions, loggerOptions?: {
-    level?: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
-    pretty?: boolean;
-}): RavenflowApp<TConfig>;
+declare function createRavenflowApp<TConfig extends RavenflowConfig>(config: TConfig, options: RavenflowAppOptions): RavenflowApp<TConfig>;
 
-export { type EtlJobConfig, type JobConfig, type Logger, type LoggerOptions, RavenflowApp, type RavenflowConfig, type SimpleJobConfig, createLogger, createRavenflowApp };
+export { type DistributedEtlJobConfig, type EtlExtractResult, type EtlResult, type JobConfig, type JobStatus, type Logger, type LoggerOptions, type MemoryEtlJobConfig, type MemoryEtlStepConfig, type MongoConnectionObject, type RateLimitConfig, RavenflowApp, type RavenflowAppOptions, type RavenflowConfig, type RetentionConfig, type SimpleJobConfig, type StatefulEtlJobConfig, type StepConfig, type TokenBucketJobConfig, createRavenflowApp, defineConfig, defineDistributedEtlJob, defineMemoryEtlJob, defineSimpleJob, defineStatefulEtlJob, defineTokenBucketJob, getLogger, initLogger };