ravenflow 0.1.0 → 0.2.0

package/README.md

@@ -1,49 +1,313 @@
-# Ravenflow
+# ravenflow
 
-Type-safe job queue management for TypeScript.
+Type-safe job queue framework for TypeScript, built on [BullMQ](https://docs.bullmq.io/) and [Redis](https://redis.io/).
 
-## Installation
+Define jobs as plain functions, ravenflow handles scheduling, retries, rate limiting, distributed processing, and crash recovery.
+
+## Install
 
 ```bash
-npm install ravenflow ioredis
+npm install ravenflow
+# or
+pnpm add ravenflow
 ```
 
+**Requirements:** Redis 6+, Node.js 18+. MongoDB required for stateful-etl and token-bucket jobs.
+
 ## Quick Start
 
-```typescript
-import { createRavenflowApp } from "ravenflow";
-
-const app = createRavenflowApp(
-  {
-    defaultQueue: "default",
-    queues: ["default"],
-    jobs: {
-      sendEmail: {
-        type: "job",
-        queueName: "default",
-        handler: async (job, params: { to: string; subject: string }) => {
-          console.log(`Sending email to ${params.to}`);
-          return { sent: true };
-        },
+```ts
+import {
+  createRavenflowApp,
+  defineConfig,
+  defineSimpleJob,
+} from "ravenflow";
+
+// 1. Define your jobs
+const config = defineConfig({
+  defaultQueue: "default",
+  jobs: {
+    sendEmail: defineSimpleJob({
+      handler: async (job, params: { to: string; subject: string }) => {
+        await sendEmail(params.to, params.subject);
+        return { sent: true };
       },
+    }),
+  },
+});
+
+// 2. Create the app
+const app = createRavenflowApp(config, {
+  redis: { host: "localhost", port: 6379 },
+});
+
+// 3. Start the worker
+await app.startWorker("default", { concurrency: 10 });
+
+// 4. Send jobs
+await app.jobs.sendEmail.send({ to: "user@example.com", subject: "Hello" });
+
+// Or send and wait for the result
+const result = await app.jobs.sendEmail.sendAndWait({ to: "user@example.com", subject: "Hello" });
+console.log(result); // { sent: true }
+```
+
+## Job Types
+
+### Simple Job
+
+One-shot function execution. Ideal for notifications, cleanup tasks, webhooks.
+
+```ts
+import { defineSimpleJob } from "ravenflow";
+
+const sendEmail = defineSimpleJob({
+  cron: "*/5 * * * *", // optional: run every 5 minutes
+  handler: async (job, params: { to: string; text: string }) => {
+    const result = await emailService.send(params.to, params.text);
+    return { messageId: result.id };
+  },
+});
+```
+
+### Memory ETL
+
+Extract-Transform-Load pipeline that runs entirely in-memory. Cursor-based pagination, parallel transforms with rate limiting.
+
+```ts
+import { defineMemoryEtlJob } from "ravenflow";
+
+const syncUsers = defineMemoryEtlJob({
+  cron: "0 2 * * *", // daily at 2 AM
+
+  extract: async (cursor: number | null) => {
+    const offset = cursor ?? 0;
+    const users = await api.getUsers({ offset, limit: 100 });
+    return {
+      data: users,
+      cursor: offset + 100,
+      hasMore: users.length === 100,
+    };
+  },
+
+  transform: async (user) => ({
+    id: user.id,
+    email: user.email,
+    normalizedName: user.name.trim().toLowerCase(),
+  }),
+
+  load: async (users) => {
+    await db.users.insertMany(users);
+  },
+
+  steps: {
+    transform: {
+      rateLimit: { perSecond: 50 }, // even spacing: 1 every 20ms
     },
   },
-  {
-    host: "localhost",
-    port: 6379,
-  }
-);
+});
+```
+
+### Distributed ETL
 
-// Enqueue job
-await app.jobs.sendEmail.send({
-  to: "user@example.com",
-  subject: "Hello",
+Same extract/transform/load pattern, but transforms are dispatched as independent BullMQ sub-jobs for parallel processing across workers.
+
+```ts
+import { defineDistributedEtlJob } from "ravenflow";
+
+const processOrders = defineDistributedEtlJob({
+  cron: "*/10 * * * *",
+
+  extract: async (cursor: number | null) => {
+    const orders = await db.orders
+      .find({ status: "pending" })
+      .skip(cursor ?? 0)
+      .limit(50)
+      .toArray();
+    return {
+      data: orders,
+      cursor: (cursor ?? 0) + 50,
+      hasMore: orders.length === 50,
+    };
+  },
+
+  transform: async (order) => {
+    const enriched = await paymentGateway.verify(order.paymentId);
+    return { ...order, verified: enriched.status === "ok" };
+  },
+
+  load: async (orders) => {
+    await db.processedOrders.insertMany(orders);
+  },
+
+  steps: {
+    transform: {
+      rateLimit: { perSecond: 10 },
+      attempts: 3,
+      backoff: { type: "exponential", delay: 1000 },
+    },
+  },
 });
+```
+
+### Stateful ETL
+
+Distributed ETL with MongoDB-backed checkpoints and automatic crash recovery. If a worker dies mid-processing, the Stall Watcher detects it and resumes from the last checkpoint.
+
+```ts
+import { defineStatefulEtlJob } from "ravenflow";
+
+const migrateLegacyData = defineStatefulEtlJob({
+  cron: "0 3 * * *",
+  singletonRun: true, // only one run at a time
+  stallTimeoutMs: 60_000, // detect stall after 60s without heartbeat
+  maxStallRetries: 3,
+
+  extract: async (cursor: string | null) => {
+    const page = await legacyApi.fetch({ after: cursor, limit: 100 });
+    return {
+      data: page.items,
+      cursor: page.nextCursor,
+      hasMore: page.hasMore,
+    };
+  },
 
-// Start worker
-app.startWorker("default");
+  transform: async (item) => {
+    return mapToNewSchema(item);
+  },
+
+  load: async (items) => {
+    await newDb.collection("records").insertMany(items);
+  },
+
+  steps: {
+    transform: {
+      rateLimit: { perSecond: 5 },
+      attempts: 3,
+      backoff: { type: "exponential", delay: 2000 },
+    },
+  },
+});
 ```
 
+**How crash recovery works:**
+1. During processing, the worker sends heartbeats to MongoDB
+2. A `__stall_check` cron job monitors all active runs
+3. If a heartbeat goes stale (exceeds `stallTimeoutMs`), the run is claimed as "stalled"
+4. A `__stall_resume` job re-enqueues the original job with the last checkpoint
+5. Processing resumes from where it left off (no duplicate work)
+
+## Configuration
+
+### App Options
+
+```ts
+const app = createRavenflowApp(config, {
+  // Required: Redis connection
+  redis: { host: "localhost", port: 6379, password: "secret", db: 0 },
+
+  // Optional: MongoDB (required for stateful-etl and token-bucket)
+  // String form:
+  mongo: "mongodb://localhost:27017/myapp",
+  // Or object form:
+  mongo: { host: "localhost", port: 27017, username: "admin", password: "secret", database: "myapp" },
+
+  // Optional: job retention policy
+  retention: {
+    completed: 1000, // keep last 1000 completed jobs per queue
+    failed: 5000,    // keep last 5000 failed jobs per queue
+  },
+
+  // Optional: logger config (pino)
+  logger: { level: "info", pretty: true },
+});
+```
+
+### Rate Limiting
+
+Even-spacing rate limiter. `perSecond: 10` means 1 request every 100ms (not a burst of 10 then wait).
+
+```ts
+steps: {
+  extract: { rateLimit: { perMinute: 60 } },  // 1 per second
+  transform: { rateLimit: { perSecond: 20 } }, // 1 every 50ms
+  load: { rateLimit: { perSecond: 5 } },       // 1 every 200ms
+}
+```
+
+### Step Config (Distributed & Stateful ETL)
+
+Each step can have its own retry strategy, rate limit, and target queue:
+
+```ts
+steps: {
+  transform: {
+    rateLimit: { perSecond: 10 },
+    attempts: 3,
+    backoff: { type: "exponential", delay: 1000 },
+    queue: "heavy-transforms", // route to a different queue
+  },
+}
+```
+
+### Cron Scheduling
+
+Jobs with a `cron` field are automatically scheduled when the worker starts:
+
+```ts
+defineSimpleJob({
+  cron: "*/5 * * * *",  // every 5 minutes
+  handler: async () => { /* ... */ },
+});
+```
+
+## Sending Jobs
+
+```ts
+// Fire and forget — returns BullMQ Job object
+const job = await app.jobs.sendEmail.send({ to: "user@example.com", subject: "Hi" });
+
+// Wait for the result
+const result = await app.jobs.sendEmail.sendAndWait({ to: "user@example.com", subject: "Hi" });
+
+// ETL jobs don't take parameters
+await app.jobs.syncUsers.send();
+
+// ETL sendAndWait returns counts
+const counts = await app.jobs.syncUsers.sendAndWait();
+// { extractedCount: 500, transformedCount: 500, loadedCount: 500 }
+```
+
+## Worker
+
+```ts
+// Start processing jobs from a queue
+await app.startWorker("default", { concurrency: 20 });
+
+// Graceful shutdown
+await app.close();
+```
+
+When `startWorker` is called:
+- Cron schedules are synced to Redis
+- The Stall Watcher starts automatically if stateful-etl jobs are registered
+- The worker begins processing jobs
+
+## Config Validation
+
+Ravenflow validates your config at startup and throws clear errors:
+
+```
+[ravenflow] Config validation failed:
+  - "migrateLegacy" (stateful-etl) requires a mongo connection.
+  - "rateLimiter" (token-bucket) requires a cron schedule.
+```
+
+## Companion Packages
+
+- [`@ravenflow/mongo`](https://www.npmjs.com/package/@ravenflow/mongo) — Typed MongoDB collections, base repository, Zod schema integration
+- [`@ravenflow/http`](https://www.npmjs.com/package/@ravenflow/http) — HTTP client utilities built on undici
+
 ## License
 
 MIT

package/dist/index.d.mts

@@ -1,15 +1,9 @@
-import { Job, ConnectionOptions, Queue, QueueEvents } from 'bullmq';
-import * as pino from 'pino';
+import { z } from 'zod';
+import { Job, Queue, QueueEvents, ConnectionOptions, Worker } from 'bullmq';
+import pino from 'pino';
 
-interface RavenflowConfig<TQueues extends string = string, TJobs extends string = string> {
-    defaultQueue: TQueues;
-    queues: TQueues[];
-    jobs: Record<TJobs, JobConfig>;
-}
-type JobNames<T extends RavenflowConfig> = T extends RavenflowConfig<infer _, infer TJobs> ? TJobs : never;
-type QueueNames<T extends RavenflowConfig> = T extends RavenflowConfig<infer TQueues, infer _> ? TQueues : never;
 interface BaseJobConfig {
-    queueName: string;
+    queue?: string;
     cron?: string;
     attempts?: number;
     backoff?: {
@@ -17,67 +11,207 @@ interface BaseJobConfig {
         delay: number;
     };
 }
+declare const zRateLimitConfig: z.ZodUnion<readonly [z.ZodObject<{
+    perSecond: z.ZodNumber;
+}, z.core.$strip>, z.ZodObject<{
+    perMinute: z.ZodNumber;
+}, z.core.$strip>]>;
+type RateLimitConfig = z.infer<typeof zRateLimitConfig>;
+interface MemoryEtlStepConfig {
+    rateLimit?: RateLimitConfig;
+}
+interface StepConfig extends MemoryEtlStepConfig {
+    attempts?: number;
+    backoff?: {
+        type: "fixed" | "exponential";
+        delay: number;
+    };
+    queue?: string;
+}
+interface EtlExtractResult<TRaw, TCursor> {
+    data: TRaw[];
+    cursor: TCursor | null;
+    hasMore: boolean;
+}
+declare const zJobStatus: z.ZodEnum<{
+    pending: "pending";
+    in_progress: "in_progress";
+    completed: "completed";
+    failed: "failed";
+    stalled: "stalled";
+}>;
+type JobStatus = z.infer<typeof zJobStatus>;
+
 interface SimpleJobConfig<TData, TReturn> extends BaseJobConfig {
-    type: "job";
+    type: "simple";
     handler: (job: Job, params: TData) => Promise<TReturn>;
 }
-interface EtlJobConfig<TRaw, TProcessed> extends BaseJobConfig {
-    type: "etl";
-    extract: (page: number) => Promise<{
-        data: TRaw[];
-        hasNextPage?: boolean;
-    }>;
+declare function defineSimpleJob<TData, TReturn>(config: Omit<SimpleJobConfig<TData, TReturn>, "type">): SimpleJobConfig<TData, TReturn>;
+
+interface MemoryEtlJobConfig<TRaw, TProcessed, TCursor = number> extends BaseJobConfig {
+    type: "memory-etl";
+    extract: (cursor: TCursor | null) => Promise<EtlExtractResult<TRaw, TCursor>>;
+    transform: (data: TRaw) => Promise<TProcessed>;
+    load: (data: TProcessed[]) => Promise<void>;
+    steps?: {
+        extract?: MemoryEtlStepConfig;
+        transform?: MemoryEtlStepConfig;
+        load?: MemoryEtlStepConfig;
+    };
+}
+declare function defineMemoryEtlJob<TRaw, TProcessed, TCursor = number>(config: Omit<MemoryEtlJobConfig<TRaw, TProcessed, TCursor>, "type">): MemoryEtlJobConfig<TRaw, TProcessed, TCursor>;
+
+interface DistributedEtlJobConfig<TRaw, TProcessed, TCursor = number> extends BaseJobConfig {
+    type: "distributed-etl";
+    extract: (cursor: TCursor | null) => Promise<EtlExtractResult<TRaw, TCursor>>;
+    transform: (data: TRaw) => Promise<TProcessed>;
+    load: (data: TProcessed[]) => Promise<void>;
+    steps?: {
+        extract?: StepConfig;
+        transform?: StepConfig;
+        load?: StepConfig;
+    };
+}
+declare function defineDistributedEtlJob<TRaw, TProcessed, TCursor = number>(config: Omit<DistributedEtlJobConfig<TRaw, TProcessed, TCursor>, "type">): DistributedEtlJobConfig<TRaw, TProcessed, TCursor>;
+
+interface StatefulEtlJobConfig<TRaw, TProcessed, TCursor = string> extends BaseJobConfig {
+    type: "stateful-etl";
+    extract: (cursor: TCursor | null) => Promise<EtlExtractResult<TRaw, TCursor>>;
     transform: (data: TRaw) => Promise<TProcessed>;
     load: (data: TProcessed[]) => Promise<void>;
-    transformQueue?: string;
+    /**
+     * When true, only one run of this job can be active at a time.
+     * If a run is already pending or in_progress (with a live heartbeat), the new trigger is skipped.
+     * Defaults to false — each trigger creates an independent run.
+     */
+    singletonRun?: boolean;
+    stallTimeoutMs?: number;
+    maxStallRetries?: number;
+    steps?: {
+        extract?: StepConfig;
+        transform?: StepConfig;
+        load?: StepConfig;
+    };
+}
+declare function defineStatefulEtlJob<TRaw, TProcessed, TCursor = string>(config: Omit<StatefulEtlJobConfig<TRaw, TProcessed, TCursor>, "type">): StatefulEtlJobConfig<TRaw, TProcessed, TCursor>;
+
+interface TokenBucketJobConfig<TItem, TResult> extends BaseJobConfig {
+    type: "token-bucket";
+    cron: string;
+    rate: {
+        amount: number;
+        interval: "minute" | "hour" | "day";
+    };
+    activeHours?: {
+        start: number;
+        end: number;
+        timezone?: string;
+    };
+    maxPerBeat?: number;
+    getItems: (budget: number) => Promise<TItem[]>;
+    process: (item: TItem) => Promise<TResult>;
+    steps?: {
+        process?: StepConfig;
+    };
+}
+declare function defineTokenBucketJob<TItem, TResult>(config: Omit<TokenBucketJobConfig<TItem, TResult>, "type">): TokenBucketJobConfig<TItem, TResult>;
+
+type JobConfig = SimpleJobConfig<any, any> | MemoryEtlJobConfig<any, any, any> | DistributedEtlJobConfig<any, any, any> | StatefulEtlJobConfig<any, any, any> | TokenBucketJobConfig<any, any>;
+
+interface RetentionConfig {
+    /** Number of completed jobs to keep per queue. Set to 0 to remove immediately. */
+    completed?: number;
+    /** Number of failed jobs to keep per queue. Set to 0 to remove immediately. */
+    failed?: number;
+}
+interface RavenflowConfig<TJobs extends string = string> {
+    defaultQueue: string;
+    jobs: Record<TJobs, JobConfig>;
+}
+type JobNames<T extends RavenflowConfig> = T extends RavenflowConfig<infer TJobs> ? TJobs : never;
+declare function defineConfig<const T extends RavenflowConfig>(config: T): T;
+
+interface EtlResult {
+    extractedCount: number;
+    transformedCount: number;
+    loadedCount: number;
 }
-type JobConfig = SimpleJobConfig<any, any> | EtlJobConfig<any, any>;
 type JobProxy<T extends JobConfig> = T extends SimpleJobConfig<infer TData, infer TReturn> ? {
     send: (params: TData) => Promise<Job<TData, TReturn>>;
-} : T extends EtlJobConfig<any, any> ? {
-    send: (params?: {
-        startPage?: number;
-        batchSize?: number;
-    }) => Promise<void>;
+    sendAndWait: (params: TData) => Promise<TReturn>;
+} : T extends MemoryEtlJobConfig<any, any, any> ? {
+    send: () => Promise<void>;
+    sendAndWait: () => Promise<EtlResult>;
+} : T extends DistributedEtlJobConfig<any, any> ? {
+    send: () => Promise<void>;
+    sendAndWait: () => Promise<EtlResult>;
+} : T extends StatefulEtlJobConfig<any, any, any> ? {
+    send: () => Promise<void>;
+    sendAndWait: () => Promise<void>;
+} : T extends TokenBucketJobConfig<any, any> ? {
+    send: () => Promise<void>;
+    sendAndWait: () => Promise<void>;
 } : never;
 type JobsProxy<TJobs extends Record<string, JobConfig>> = {
     [K in keyof TJobs]: JobProxy<TJobs[K]>;
 };
 
+type Logger = pino.Logger;
 interface LoggerOptions {
     level?: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
     pretty?: boolean;
 }
-declare function createLogger(options?: LoggerOptions): pino.Logger<never>;
-type Logger = ReturnType<typeof createLogger>;
+declare function initLogger(options?: LoggerOptions): Logger;
+declare function getLogger(): Logger;
+
+type QueueStore = Record<string, {
+    queue: Queue;
+    events: QueueEvents;
+}>;
+
+declare class StallWatcher {
+    private readonly workerQueue;
+    private readonly statefulEtlConfigs;
+    private readonly userQueues;
+    constructor(jobConfigs: Record<string, JobConfig>, userQueues: Record<string, {
+        queue: Queue;
+    }>, defaultQueue: string, workerQueue: Queue);
+    canProcess(jobName: string): boolean;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    process(job: Job): Promise<void>;
+}
 
+interface MongoConnectionObject {
+    host: string;
+    port?: number;
+    username?: string;
+    password?: string;
+    database?: string;
+}
+interface RavenflowAppOptions {
+    redis: ConnectionOptions;
+    mongo?: string | MongoConnectionObject;
+    retention?: RetentionConfig;
+    logger?: LoggerOptions;
+}
 declare class RavenflowApp<TConfig extends RavenflowConfig = RavenflowConfig> {
-    protected connection: ConnectionOptions;
+    protected redisConnection: ConnectionOptions;
     protected defaultConcurrency: number;
-    protected logger: Logger;
     protected config: TConfig;
+    protected mongoUri: string | null;
     jobs: JobsProxy<TConfig["jobs"]>;
     protected jobConfigs: Record<JobNames<TConfig>, JobConfig>;
-    protected queues: Record<QueueNames<TConfig>, {
-        queue: Queue;
-        events: QueueEvents;
-    }>;
-    constructor(connection: ConnectionOptions, config: TConfig, loggerOptions?: LoggerOptions);
-    private initQueues;
-    private initJobs;
-    private sendJob;
-    startWorker(queueName: QueueNames<TConfig>, options?: {
+    protected queues: QueueStore;
+    protected workers: Worker[];
+    protected stallWatcher: StallWatcher | null;
+    constructor(config: TConfig, options: RavenflowAppOptions);
+    startWorker(queueName: string, options?: {
         concurrency?: number;
-    }): void;
-    private printWorkerBanner;
-    private setupWorkerEvents;
-    private syncJobSchedulers;
-    processJob(job: Job): Promise<any>;
+    }): Promise<void>;
+    close(): Promise<void>;
 }
 
-declare function createRavenflowApp<TConfig extends RavenflowConfig>(config: TConfig, connection: ConnectionOptions, loggerOptions?: {
-    level?: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
-    pretty?: boolean;
-}): RavenflowApp<TConfig>;
+declare function createRavenflowApp<TConfig extends RavenflowConfig>(config: TConfig, options: RavenflowAppOptions): RavenflowApp<TConfig>;
 
-export { type EtlJobConfig, type JobConfig, type Logger, type LoggerOptions, RavenflowApp, type RavenflowConfig, type SimpleJobConfig, createLogger, createRavenflowApp };
+export { type DistributedEtlJobConfig, type EtlExtractResult, type EtlResult, type JobConfig, type JobStatus, type Logger, type LoggerOptions, type MemoryEtlJobConfig, type MemoryEtlStepConfig, type MongoConnectionObject, type RateLimitConfig, RavenflowApp, type RavenflowAppOptions, type RavenflowConfig, type RetentionConfig, type SimpleJobConfig, type StatefulEtlJobConfig, type StepConfig, type TokenBucketJobConfig, createRavenflowApp, defineConfig, defineDistributedEtlJob, defineMemoryEtlJob, defineSimpleJob, defineStatefulEtlJob, defineTokenBucketJob, getLogger, initLogger };