ravenflow 0.1.0 → 0.2.0

package/dist/index.js

@@ -1,57 +1,25 @@
 'use strict';
 
-var bullmq = require('bullmq');
 var pino = require('pino');
+var bullmq = require('bullmq');
+var zod = require('zod');
+var mongodb = require('mongodb');
 
 function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
 
 var pino__default = /*#__PURE__*/_interopDefault(pino);
 
-// src/app.ts
-
-// src/processor.ts
-async function processEtlJob(job, etlConfig, transformQueue, transformQueueEvents) {
-  let currentPage = 1;
-  let run = true;
-  const metadata = {
-    count: 0,
-    pages: 0,
-    currentPage,
-    startedAt: /* @__PURE__ */ new Date(),
-    endedAt: void 0
-  };
-  const pagination = {
-    maxPages: Infinity};
-  while (run && currentPage <= (pagination.maxPages || Infinity)) {
-    metadata.currentPage = currentPage;
-    metadata.pages = currentPage;
-    const {
-      data: rawElements,
-      hasNextPage: nextPageExists = false
-    } = await etlConfig.extract(currentPage);
-    const processedElements = [];
-    for (const element of rawElements) {
-      const transformJobName = `${job.name}:transform`;
-      const transformJob = await transformQueue.add(transformJobName, element, {
-        attempts: etlConfig.attempts,
-        backoff: etlConfig.backoff
-      });
-      const processedElement = await transformJob.waitUntilFinished(transformQueueEvents);
-      processedElements.push(processedElement);
-    }
-    await etlConfig.load(processedElements);
-    metadata.count += processedElements.length;
-    run = nextPageExists;
-    if (run) {
-      currentPage++;
-    }
-  }
-  return metadata;
-}
-function createLogger(options = {}) {
+var __defProp = Object.defineProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var logger = null;
+function initLogger(options = {}) {
+  if (logger) return logger;
   const isDev = process.env.NODE_ENV !== "production";
   const usePretty = options.pretty ?? isDev;
-  return pino__default.default({
+  logger = pino__default.default({
     level: options.level || (isDev ? "debug" : "info"),
     transport: usePretty ? {
       target: "pino-pretty",
@@ -62,6 +30,983 @@ function createLogger(options = {}) {
       }
     } : void 0
   });
+  return logger;
+}
+function getLogger() {
+  if (!logger) {
+    throw new Error("[ravenflow] Logger not initialized. Call initLogger() first.");
+  }
+  return logger;
+}
+function buildDefaultJobOptions(retention) {
+  if (!retention) return void 0;
+  const opts = {};
+  if (retention.completed !== void 0) {
+    opts.removeOnComplete = { count: retention.completed };
+  }
+  if (retention.failed !== void 0) {
+    opts.removeOnFail = { count: retention.failed };
+  }
+  return Object.keys(opts).length > 0 ? opts : void 0;
+}
+function initQueues(config, connection, retention) {
+  const queueNames = /* @__PURE__ */ new Set([config.defaultQueue]);
+  for (const job of Object.values(config.jobs)) {
+    if (job.queue) queueNames.add(job.queue);
+  }
+  const defaultJobOptions = buildDefaultJobOptions(retention);
+  const queues = {};
+  for (const name of queueNames) {
+    queues[name] = {
+      queue: new bullmq.Queue(name, { connection, defaultJobOptions }),
+      events: new bullmq.QueueEvents(name, { connection })
+    };
+  }
+  return queues;
+}
+async function sendJob(jobName, params, queues, queueName) {
+  const queueData = queues[queueName];
+  if (!queueData) {
+    throw new Error(`Queue ${queueName} not found`);
+  }
+  return await queueData.queue.add(jobName, params);
+}
+async function sendJobAndWait(jobName, params, queues, queueName) {
+  const queueData = queues[queueName];
+  if (!queueData) {
+    throw new Error(`Queue ${queueName} not found`);
+  }
+  const job = await queueData.queue.add(jobName, params);
+  return await job.waitUntilFinished(queueData.events);
+}
+async function closeQueues(queues) {
+  await Promise.all(
+    Object.values(queues).map(
+      ({ queue, events }) => Promise.all([queue.close(), events.close()])
+    )
+  );
+}
+
+// src/processors/token-bucket-job.processor/constants.ts
+var TOKEN_BUCKET_PROCESS_SUFFIX = ":process";
+var INTERVAL_MS = {
+  minute: 6e4,
+  hour: 36e5,
+  day: 864e5
+};
+
+// src/processors/constants.ts
+var ETL_EXTRACT_SUFFIX = ":extract";
+var ETL_TRANSFORM_SUFFIX = ":transform";
+var ETL_LOAD_SUFFIX = ":load";
+
+// src/processors/simple-job.processor.ts
+async function processSimpleJob(job, jobConfig) {
+  return jobConfig.handler(job, job.data);
+}
+var zRateLimitConfig = zod.z.union([
+  zod.z.object({ perSecond: zod.z.number().positive() }),
+  zod.z.object({ perMinute: zod.z.number().positive() })
+]);
+var zJobStatus = zod.z.enum([
+  "pending",
+  "in_progress",
+  "completed",
+  "failed",
+  "stalled"
+]);
+var zBaseJob = zod.z.object({
+  _id: zod.z.string(),
+  type: zod.z.string(),
+  jobId: zod.z.string(),
+  jobName: zod.z.string(),
+  status: zJobStatus,
+  params: zod.z.unknown().nullable(),
+  result: zod.z.unknown().nullable(),
+  error: zod.z.string().nullable(),
+  timestamps: zod.z.object({
+    createdAt: zod.z.date(),
+    updatedAt: zod.z.date(),
+    startedAt: zod.z.date().nullable(),
+    completedAt: zod.z.date().nullable()
+  })
+});
+function defineSimpleJob(config) {
+  return { ...config, type: "simple" };
+}
+var zSimpleJob = zBaseJob.extend({
+  type: zod.z.literal("simple")
+});
+var zMemoryEtlJob = zBaseJob.extend({
+  type: zod.z.literal("memory-etl"),
+  progress: zod.z.object({
+    extractedCount: zod.z.number().int().nonnegative(),
+    transformedCount: zod.z.number().int().nonnegative(),
+    loadedCount: zod.z.number().int().nonnegative()
+  })
+});
+function defineMemoryEtlJob(config) {
+  return { ...config, type: "memory-etl" };
+}
+var zDistributedEtlJob = zBaseJob.extend({
+  type: zod.z.literal("distributed-etl"),
+  progress: zod.z.object({
+    extractedCount: zod.z.number().int().nonnegative(),
+    transformedCount: zod.z.number().int().nonnegative(),
+    loadedCount: zod.z.number().int().nonnegative()
+  })
+});
+function defineDistributedEtlJob(config) {
+  return { ...config, type: "distributed-etl" };
+}
+var zStatefulEtlJob = zBaseJob.extend({
+  type: zod.z.literal("stateful-etl"),
+  stallTimeoutMs: zod.z.number().int().positive(),
+  maxStallRetries: zod.z.number().int().nonnegative(),
+  progress: zod.z.object({
+    transformedCount: zod.z.number().int().nonnegative(),
+    extractedCount: zod.z.number().int().nonnegative(),
+    loadedCount: zod.z.number().int().nonnegative(),
+    checkpoint: zod.z.unknown().nullable(),
+    lastHeartbeatAt: zod.z.date().nullable(),
+    stallRetries: zod.z.number().int().nonnegative()
+  })
+});
+function defineStatefulEtlJob(config) {
+  return { ...config, type: "stateful-etl" };
+}
+zod.z.object({
+  _id: zod.z.string(),
+  accumulatedCredit: zod.z.number(),
+  lastTickAt: zod.z.date(),
+  windowDispatched: zod.z.number().int().nonnegative(),
+  windowResetAt: zod.z.date()
+});
+function defineTokenBucketJob(config) {
+  return { ...config, type: "token-bucket" };
+}
+zod.z.discriminatedUnion("type", [
+  zSimpleJob,
+  zMemoryEtlJob,
+  zDistributedEtlJob,
+  zStatefulEtlJob
+]);
+
+// src/types/config.ts
+function defineConfig(config) {
+  return config;
+}
+
+// src/rate-limiter.ts
+var RateLimiter = class {
+  constructor() {
+    this.slots = /* @__PURE__ */ new Map();
+  }
+  // key → nextAvailableAt
+  /**
+   * Block until a slot is available. Combines getDelay + wait in a loop.
+   */
+  async waitForSlot(key, config) {
+    let delay = this.getDelay(key, config);
+    while (delay > 0) {
+      await this.wait(delay);
+      delay = this.getDelay(key, config);
+    }
+  }
+  /**
+   * Check if a slot is available. Returns 0 if allowed (and consumes the slot),
+   * or the number of ms to wait before retrying.
+   *
+   * Slots are evenly spaced: perSecond: 10 → one slot every 100ms.
+   */
+  getDelay(key, config) {
+    const intervalMs = resolve(config);
+    const now = Date.now();
+    const nextAvailableAt = this.slots.get(key);
+    if (nextAvailableAt === void 0 || now >= nextAvailableAt) {
+      this.slots.set(key, now + intervalMs);
+      return 0;
+    }
+    return nextAvailableAt - now;
+  }
+  /**
+   * Wait for the given number of ms.
+   */
+  wait(ms) {
+    if (ms <= 0) return Promise.resolve();
+    return new Promise((resolve2) => setTimeout(resolve2, ms));
+  }
+};
+function resolve(config) {
+  zRateLimitConfig.parse(config);
+  if ("perSecond" in config) return 1e3 / config.perSecond;
+  return 6e4 / config.perMinute;
+}
+
+// src/processors/memory-etl-job.processor.ts
+async function processMemoryEtlJob(job, config) {
+  const logger2 = getLogger();
+  const jobName = job.name;
+  const counts = { extractedCount: 0, transformedCount: 0, loadedCount: 0 };
+  const limiter = config.steps ? new RateLimiter() : null;
+  let cursor = null;
+  let hasMore = true;
+  while (hasMore) {
+    await applyRateLimit(limiter, jobName, "extract", config);
+    const page = await config.extract(cursor);
+    hasMore = page.hasMore;
+    cursor = page.cursor;
+    if (page.data.length === 0) continue;
+    counts.extractedCount += page.data.length;
+    const processed = await transformParallel(
+      page.data,
+      config.transform,
+      limiter,
+      jobName,
+      config
+    );
+    counts.transformedCount += processed.length;
+    await applyRateLimit(limiter, jobName, "load", config);
+    await config.load(processed);
+    counts.loadedCount += processed.length;
+    logger2.debug({ jobName, ...counts }, "[memory-etl] batch done");
+  }
+  logger2.info({ jobName, ...counts }, "[memory-etl] completed");
+  return counts;
+}
+async function transformParallel(batch, transform, limiter, jobName, config) {
+  const pending = [];
+  for (const item of batch) {
+    await applyRateLimit(limiter, jobName, "transform", config);
+    pending.push(transform(item));
+  }
+  return Promise.all(pending);
+}
+async function applyRateLimit(limiter, jobName, step, config) {
+  const rlConfig = config.steps?.[step]?.rateLimit;
+  if (!limiter || !rlConfig) return;
+  await limiter.waitForSlot(`ravenflow:rl:${jobName}:${step}`, rlConfig);
+}
+
+// src/processors/sub-job.ts
+async function dispatchSubJob(queue, queueEvents, name, data, options) {
+  const job = await queue.add(name, data, {
+    attempts: options?.attempts,
+    backoff: options?.backoff
+  });
+  return await job.waitUntilFinished(queueEvents);
+}
+async function dispatchSubJobsFireAndForget(queue, name, items, options) {
+  return queue.addBulk(
+    items.map((data) => ({
+      name,
+      data,
+      opts: {
+        attempts: options?.attempts,
+        backoff: options?.backoff
+      }
+    }))
+  );
+}
+
+// src/processors/etl-context.ts
+var EtlContext = class {
+  constructor(jobName, config, queues, initialCounts) {
+    this.jobName = jobName;
+    this.logger = getLogger();
+    this.stepsConfig = config.steps;
+    this.limiter = config.steps ? new RateLimiter() : null;
+    this.counts = initialCounts ? { ...initialCounts } : { extractedCount: 0, transformedCount: 0, loadedCount: 0 };
+    const defaultQueue = queues[config.queue];
+    this.steps = this.buildStepContexts(config, queues, defaultQueue);
+  }
+  // ---------------------------------------------------------------------------
+  // Rate limiting
+  // ---------------------------------------------------------------------------
+  async applyRateLimit(step) {
+    const config = this.stepsConfig?.[step]?.rateLimit;
+    if (!this.limiter || !config) return;
+    await this.limiter.waitForSlot(
+      `ravenflow:rl:${this.jobName}:${step}`,
+      config
+    );
+  }
+  // ---------------------------------------------------------------------------
+  // Sub-job dispatch primitives
+  // ---------------------------------------------------------------------------
+  async dispatchExtract(cursor) {
+    await this.applyRateLimit("extract");
+    const { queue, queueEvents, options } = this.steps.extract;
+    return dispatchSubJob(
+      queue,
+      queueEvents,
+      `${this.jobName}${ETL_EXTRACT_SUFFIX}`,
+      { cursor },
+      options
+    );
+  }
+  async dispatchTransform(item) {
+    const { queue, queueEvents, options } = this.steps.transform;
+    return dispatchSubJob(
+      queue,
+      queueEvents,
+      `${this.jobName}${ETL_TRANSFORM_SUFFIX}`,
+      item,
+      options
+    );
+  }
+  async transformParallel(rawItems) {
+    const pending = [];
+    for (const item of rawItems) {
+      await this.applyRateLimit("transform");
+      pending.push(this.dispatchTransform(item));
+    }
+    return Promise.all(pending);
+  }
+  async dispatchLoad(processedItems) {
+    await this.applyRateLimit("load");
+    const { queue, queueEvents, options } = this.steps.load;
+    await dispatchSubJob(
+      queue,
+      queueEvents,
+      `${this.jobName}${ETL_LOAD_SUFFIX}`,
+      { data: processedItems, jobName: this.jobName },
+      options
+    );
+  }
+  // ---------------------------------------------------------------------------
+  // Step context setup
+  // ---------------------------------------------------------------------------
+  buildStepContexts(config, queues, defaultQueue) {
+    const resolve2 = (step) => {
+      const q = step?.queue ? queues[step.queue] : defaultQueue;
+      if (!q) throw new Error(`Queue "${step?.queue}" not found`);
+      return {
+        queue: q.queue,
+        queueEvents: q.events,
+        options: {
+          attempts: step?.attempts ?? config.attempts,
+          backoff: step?.backoff ?? config.backoff
+        }
+      };
+    };
+    return {
+      extract: resolve2(config.steps?.extract),
+      transform: resolve2(config.steps?.transform),
+      load: resolve2(config.steps?.load)
+    };
+  }
+};
+
+// src/processors/distributed-etl-job.processor.ts
+async function processDistributedEtlJob(job, config, queues) {
+  const ctx = new EtlContext(job.name, config, queues);
+  let cursor = null;
+  let hasMore = true;
+  while (hasMore) {
+    const page = await ctx.dispatchExtract(cursor);
+    hasMore = page.hasMore;
+    cursor = page.cursor;
+    if (page.data.length === 0) continue;
+    ctx.counts.extractedCount += page.data.length;
+    const processed = await ctx.transformParallel(page.data);
+    ctx.counts.transformedCount += processed.length;
+    await ctx.dispatchLoad(processed);
+    ctx.counts.loadedCount += processed.length;
+    ctx.logger.debug({ jobName: ctx.jobName, ...ctx.counts }, "[distributed-etl] batch done");
+  }
+  ctx.logger.info({ jobName: ctx.jobName, ...ctx.counts }, "[distributed-etl] completed");
+  return ctx.counts;
+}
+var COLLECTION_PREFIX = "ravenflow_";
+var client = null;
+var db = null;
+async function connectMongo(uri) {
+  if (db) return db;
+  client = new mongodb.MongoClient(uri);
+  await client.connect();
+  db = client.db();
+  return db;
+}
+function getDb() {
+  if (!db) {
+    throw new Error("[ravenflow] MongoDB not connected. Pass a mongo URI to createRavenflowApp().");
+  }
+  return db;
+}
+function collectionName(name) {
+  return `${COLLECTION_PREFIX}${name}`;
+}
+async function closeMongo() {
+  if (client) {
+    await client.close();
+    client = null;
+    db = null;
+  }
+}
+
+// src/mongo/run.repo.ts
+var run_repo_exports = {};
+__export(run_repo_exports, {
+  claimStalledRun: () => claimStalledRun,
+  createRun: () => createRun,
+  findActiveRun: () => findActiveRun,
+  findById: () => findById,
+  findStalled: () => findStalled,
+  findStalledCandidates: () => findStalledCandidates,
+  markCompleted: () => markCompleted,
+  markFailed: () => markFailed,
+  markInProgress: () => markInProgress,
+  markResuming: () => markResuming,
+  updateCheckpoint: () => updateCheckpoint,
+  updateHeartbeat: () => updateHeartbeat
+});
+var RUNS_COLLECTION = "runs";
+function collection() {
+  return getDb().collection(collectionName(RUNS_COLLECTION));
+}
+async function findActiveRun(jobName) {
+  return await collection().findOne(
+    { jobName, status: { $in: ["pending", "in_progress", "stalled"] } },
+    { sort: { "timestamps.createdAt": -1 } }
+  );
+}
+async function findById(runId) {
+  return await collection().findOne({ _id: runId });
+}
+async function createRun(jobName, params, jobId, stallTimeoutMs, maxStallRetries) {
+  const now = /* @__PURE__ */ new Date();
+  const id = new mongodb.ObjectId().toString();
+  await collection().insertOne({
+    _id: id,
+    type: "stateful-etl",
+    jobId,
+    jobName,
+    status: "pending",
+    params: params ?? null,
+    result: null,
+    error: null,
+    stallTimeoutMs,
+    maxStallRetries,
+    progress: {
+      extractedCount: 0,
+      transformedCount: 0,
+      loadedCount: 0,
+      checkpoint: null,
+      lastHeartbeatAt: null,
+      stallRetries: 0
+    },
+    timestamps: {
+      createdAt: now,
+      updatedAt: now,
+      startedAt: null,
+      completedAt: null
+    }
+  });
+  return id;
+}
+async function markInProgress(runId) {
+  const now = /* @__PURE__ */ new Date();
+  await collection().updateOne(
+    { _id: runId },
+    {
+      $set: {
+        status: "in_progress",
+        "timestamps.startedAt": now,
+        "timestamps.updatedAt": now,
+        "progress.lastHeartbeatAt": now
+      }
+    }
+  );
+}
+async function updateHeartbeat(runId) {
+  const now = /* @__PURE__ */ new Date();
+  await collection().updateOne(
+    { _id: runId },
+    {
+      $set: {
+        "progress.lastHeartbeatAt": now,
+        "timestamps.updatedAt": now
+      }
+    }
+  );
+}
+async function updateCheckpoint(runId, checkpoint, counts) {
+  await collection().updateOne(
+    { _id: runId },
+    {
+      $set: {
+        "progress.checkpoint": checkpoint,
+        "progress.extractedCount": counts.extractedCount,
+        "progress.transformedCount": counts.transformedCount,
+        "progress.loadedCount": counts.loadedCount,
+        "timestamps.updatedAt": /* @__PURE__ */ new Date()
+      }
+    }
+  );
+}
+async function markCompleted(runId) {
+  const now = /* @__PURE__ */ new Date();
+  await collection().updateOne(
+    { _id: runId },
+    {
+      $set: {
+        status: "completed",
+        "timestamps.completedAt": now,
+        "timestamps.updatedAt": now,
+        "progress.lastHeartbeatAt": null
+      }
+    }
+  );
+}
+async function markFailed(runId, error) {
+  const now = /* @__PURE__ */ new Date();
+  await collection().updateOne(
+    { _id: runId },
+    {
+      $set: {
+        status: "failed",
+        error,
+        "timestamps.updatedAt": now,
+        "progress.lastHeartbeatAt": null
+      }
+    }
+  );
+}
+async function findStalled(stallTimeoutMs) {
+  const threshold = new Date(Date.now() - stallTimeoutMs);
+  return await collection().find({
+    status: "in_progress",
+    "progress.lastHeartbeatAt": { $lt: threshold }
+  }).toArray();
+}
+async function findStalledCandidates() {
+  const now = /* @__PURE__ */ new Date();
+  return await collection().find({
+    status: "in_progress",
+    "progress.lastHeartbeatAt": { $ne: null },
+    $expr: {
+      $lt: [
+        "$progress.lastHeartbeatAt",
+        { $subtract: [now, "$stallTimeoutMs"] }
+      ]
+    }
+  }).toArray();
+}
+async function claimStalledRun(runId) {
+  const result = await collection().findOneAndUpdate(
+    {
+      _id: runId,
+      status: "in_progress"
+    },
+    {
+      $set: {
+        status: "stalled",
+        "progress.lastHeartbeatAt": null,
+        "timestamps.updatedAt": /* @__PURE__ */ new Date()
+      },
+      $inc: {
+        "progress.stallRetries": 1
+      }
+    },
+    { returnDocument: "after" }
+  );
+  return result ?? null;
+}
+async function markResuming(runId) {
+  const now = /* @__PURE__ */ new Date();
+  await collection().updateOne(
+    { _id: runId },
+    {
+      $set: {
+        status: "in_progress",
+        "progress.lastHeartbeatAt": now,
+        "timestamps.updatedAt": now
+      }
+    }
+  );
+}
+
+// src/mongo/token-bucket.repo.ts
+var token_bucket_repo_exports = {};
+__export(token_bucket_repo_exports, {
+  getOrCreateState: () => getOrCreateState,
+  updateState: () => updateState
+});
+var COLLECTION = "token_bucket_state";
+function collection2() {
+  return getDb().collection(collectionName(COLLECTION));
+}
+async function getOrCreateState(jobName, now, windowResetAt) {
+  const result = await collection2().findOneAndUpdate(
+    { _id: jobName },
+    {
+      $setOnInsert: {
+        _id: jobName,
+        accumulatedCredit: 0,
+        lastTickAt: now,
+        windowDispatched: 0,
+        windowResetAt
+      }
+    },
+    { upsert: true, returnDocument: "after" }
+  );
+  return result;
+}
+async function updateState(jobName, accumulatedCredit, lastTickAt, windowDispatched, windowResetAt) {
+  await collection2().updateOne(
+    { _id: jobName },
+    {
+      $set: {
+        accumulatedCredit,
+        lastTickAt,
+        windowDispatched,
+        windowResetAt
+      }
+    }
+  );
+}
+
+// src/processors/stateful-etl-job.processor/constants.ts
+var DEFAULT_STALL_TIMEOUT_MS = 3e5;
+var HEARTBEAT_INTERVAL_MS = 3e4;
+var DEFAULT_MAX_STALL_RETRIES = 3;
+
+// src/processors/stateful-etl-job.processor/heartbeat.ts
+function startHeartbeat(runId) {
+  const logger2 = getLogger();
+  const timer = setInterval(async () => {
+    try {
+      await run_repo_exports.updateHeartbeat(runId);
+    } catch (err) {
+      logger2.warn(
+        { runId, error: err },
+        "[stateful-etl] heartbeat update failed"
+      );
+    }
+    logger2.debug({ runId }, "[stateful-etl] heartbeat");
+  }, HEARTBEAT_INTERVAL_MS);
+  timer.unref();
+  return timer;
+}
+
+// src/processors/stateful-etl-job.processor/paginate.ts
+async function paginateAndProcess(jobName, runId, jobConfig, queues, initialCursor, initialCounts) {
+  const ctx = new EtlContext(
+    jobName,
+    jobConfig,
+    queues,
+    initialCounts
+  );
+  let cursor = initialCursor;
+  let hasMore = true;
+  while (hasMore) {
+    const page = await ctx.dispatchExtract(cursor);
+    hasMore = page.hasMore;
+    cursor = page.cursor;
+    if (page.data.length === 0) continue;
+    const processed = await ctx.transformParallel(page.data);
+    await ctx.dispatchLoad(processed);
+    ctx.counts.extractedCount += page.data.length;
+    ctx.counts.transformedCount += processed.length;
+    ctx.counts.loadedCount += processed.length;
+    await run_repo_exports.updateCheckpoint(runId, cursor, ctx.counts);
+    ctx.logger.debug({ jobName, ...ctx.counts }, "[stateful-etl] batch done");
+  }
+}
+
+// src/processors/stateful-etl-job.processor/process.ts
+function isResumeData(data) {
+  return typeof data === "object" && data !== null && "__ravenflow_resume" in data && data.__ravenflow_resume === true;
+}
+async function processStatefulEtlJob(job, jobConfig, queues) {
+  const logger2 = getLogger();
+  const jobName = job.name;
+  if (isResumeData(job.data)) {
+    return resumeRun(job.data, jobName, jobConfig, queues, logger2);
+  }
+  return freshRun(job, jobName, jobConfig, queues, logger2);
+}
+async function resumeRun(data, jobName, jobConfig, queues, logger2) {
+  const { runId, checkpoint, counts } = data;
+  logger2.info({ jobName, runId }, "[stateful-etl] resuming stalled run");
+  await run_repo_exports.markResuming(runId);
+  const heartbeatTimer = startHeartbeat(runId);
+  try {
+    await paginateAndProcess(
+      jobName,
+      runId,
+      jobConfig,
+      queues,
+      checkpoint,
+      counts
+    );
+    await run_repo_exports.markCompleted(runId);
+    logger2.info({ jobName, runId }, "[stateful-etl] resumed run completed");
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    await run_repo_exports.markFailed(runId, message);
+    logger2.error({ jobName, runId, error: message }, "[stateful-etl] resumed run failed");
+    throw error;
+  } finally {
+    clearInterval(heartbeatTimer);
+  }
+}
+async function freshRun(job, jobName, jobConfig, queues, logger2) {
+  const params = job.data ?? null;
+  if (jobConfig.singletonRun) {
+    await ensureNotAlreadyRunning(jobName);
+  }
+  const stallTimeoutMs = jobConfig.stallTimeoutMs ?? DEFAULT_STALL_TIMEOUT_MS;
+  const maxStallRetries = jobConfig.maxStallRetries ?? DEFAULT_MAX_STALL_RETRIES;
+  const runId = await run_repo_exports.createRun(jobName, params, job.id, stallTimeoutMs, maxStallRetries);
+  await run_repo_exports.markInProgress(runId);
+  const heartbeatTimer = startHeartbeat(runId);
+  try {
+    await paginateAndProcess(
+      jobName,
+      runId,
+      jobConfig,
+      queues,
+      null,
+      { extractedCount: 0, transformedCount: 0, loadedCount: 0 }
+    );
+    await run_repo_exports.markCompleted(runId);
+    logger2.info({ jobName }, "[stateful-etl] run completed");
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    await run_repo_exports.markFailed(runId, message);
+    logger2.error({ jobName, error: message }, "[stateful-etl] run failed");
+    throw error;
+  } finally {
+    clearInterval(heartbeatTimer);
+  }
+}
+async function ensureNotAlreadyRunning(jobName) {
+  const activeRun = await run_repo_exports.findActiveRun(jobName);
+  if (activeRun) {
+    throw new Error(
+      `[stateful-etl] job "${jobName}" already has an active run (status: ${activeRun.status})`
+    );
+  }
+}
+
+// src/processors/token-bucket-job.processor/active-hours.ts
+function isWithinActiveHours(config, now) {
+  if (!config.activeHours) return true;
+  const { start, end, timezone } = config.activeHours;
+  const hour = getHourInTimezone(now, timezone);
+  if (start <= end) {
+    return hour >= start && hour < end;
+  }
+  return hour >= start || hour < end;
+}
+function getActiveWindowMs(config) {
+  if (!config.activeHours) {
+    return getFullIntervalMs(config.rate.interval);
+  }
+  const { start, end } = config.activeHours;
+  let hours;
+  if (start <= end) {
+    hours = end - start;
+  } else {
+    hours = 24 - start + end;
+  }
+  if (hours <= 0) hours = 24;
+  if (config.rate.interval === "day") {
+    return hours * 36e5;
+  }
+  return getFullIntervalMs(config.rate.interval);
+}
+function getFullIntervalMs(interval) {
+  switch (interval) {
+    case "minute":
+      return 6e4;
+    case "hour":
+      return 36e5;
+    case "day":
+      return 864e5;
+  }
+}
+function getHourInTimezone(date, timezone) {
+  if (!timezone) return date.getUTCHours();
+  const formatted = new Intl.DateTimeFormat("en-US", {
+    hour: "numeric",
+    hour12: false,
+    timeZone: timezone
+  }).format(date);
+  return parseInt(formatted, 10);
+}
+
+// src/processors/token-bucket-job.processor/budget.ts
+function calculateBudget(state, config, now) {
+  const nowMs = now.getTime();
+  const windowResetMs = state.windowResetAt.getTime();
+  let windowDispatched = state.windowDispatched;
+  let windowResetAt = state.windowResetAt;
+  if (nowMs >= windowResetMs) {
+    windowDispatched = 0;
+    windowResetAt = computeNextWindowReset(now, config.rate.interval);
+  }
+  const elapsedMs = nowMs - state.lastTickAt.getTime();
+  if (elapsedMs <= 0) {
+    return { budget: 0, newCredit: state.accumulatedCredit, newWindowDispatched: windowDispatched, newWindowResetAt: windowResetAt };
+  }
+  const activeWindowMs = getActiveWindowMs(config);
+  const msPerItem = activeWindowMs / config.rate.amount;
+  const earnedCredit = elapsedMs / msPerItem;
+  let credit = state.accumulatedCredit + earnedCredit;
+  const maxPerBeat = config.maxPerBeat ?? computeDefaultMaxPerBeat(config);
+  credit = Math.min(credit, maxPerBeat);
+  const remaining = Math.max(0, config.rate.amount - windowDispatched);
+  const budget = Math.min(Math.floor(credit), remaining);
+  return {
+    budget,
+    newCredit: credit - budget,
+    newWindowDispatched: windowDispatched + budget,
+    newWindowResetAt: windowResetAt
+  };
+}
+function computeDefaultMaxPerBeat(config) {
+  return Math.max(1, Math.ceil(config.rate.amount / 10));
+}
+function computeNextWindowReset(now, interval) {
+  return new Date(now.getTime() + INTERVAL_MS[interval]);
+}
+
+// src/processors/token-bucket-job.processor/process.ts
+async function processTokenBucketJob(job, config, queues) {
+  const logger2 = getLogger();
+  const jobName = job.name;
+  const now = /* @__PURE__ */ new Date();
+  if (!isWithinActiveHours(config, now)) {
+    logger2.debug({ jobName }, "[token-bucket] outside active hours, skipping");
+    return { dispatched: 0 };
+  }
+  const initialWindowReset = new Date(now.getTime() + INTERVAL_MS[config.rate.interval]);
+  const state = await token_bucket_repo_exports.getOrCreateState(jobName, now, initialWindowReset);
+  const { budget, newCredit, newWindowDispatched, newWindowResetAt } = calculateBudget(state, config, now);
+  if (budget <= 0) {
+    await token_bucket_repo_exports.updateState(jobName, newCredit, now, newWindowDispatched, newWindowResetAt);
+    logger2.debug({ jobName, credit: newCredit }, "[token-bucket] no budget, skipping");
+    return { dispatched: 0 };
+  }
+  const items = await config.getItems(budget);
+  if (items.length === 0) {
+    await token_bucket_repo_exports.updateState(jobName, newCredit, now, newWindowDispatched, newWindowResetAt);
+    logger2.debug({ jobName }, "[token-bucket] no items to dispatch");
+    return { dispatched: 0 };
+  }
+  const queueData = queues[config.queue];
+  if (!queueData) {
+    throw new Error(`[token-bucket] queue "${config.queue}" not found`);
+  }
+  const subJobOptions = config.steps?.process ? { attempts: config.steps.process.attempts, backoff: config.steps.process.backoff } : void 0;
+  const subJobName = `${jobName}${TOKEN_BUCKET_PROCESS_SUFFIX}`;
+  await dispatchSubJobsFireAndForget(queueData.queue, subJobName, items, subJobOptions);
+  const actualDispatched = items.length;
+  const adjustedCredit = Math.max(0, newCredit - Math.max(0, budget - actualDispatched));
+  const adjustedWindowDispatched = newWindowDispatched - (budget - actualDispatched);
+  await token_bucket_repo_exports.updateState(jobName, adjustedCredit, now, adjustedWindowDispatched, newWindowResetAt);
+  logger2.info(
+    { jobName, dispatched: actualDispatched, credit: adjustedCredit, windowDispatched: adjustedWindowDispatched },
+    "[token-bucket] dispatched items"
+  );
+  return { dispatched: actualDispatched };
+}
+
+// src/app/job-registry.ts
+function initJobs(config, queues) {
+  const jobsProxy = {};
+  const jobConfigs = {};
+  for (const [jobName, jobConfig] of Object.entries(config.jobs)) {
+    jobConfigs[jobName] = jobConfig;
+    const resolvedQueue = jobConfig.queue ?? config.defaultQueue;
+    jobsProxy[jobName] = {
+      send: async (params) => {
+        return sendJob(jobName, params, queues, resolvedQueue);
+      },
+      sendAndWait: async (params) => {
+        return sendJobAndWait(jobName, params, queues, resolvedQueue);
+      }
+    };
+  }
+  return {
+    jobsProxy,
+    jobConfigs
+  };
+}
+var ALL_SUFFIXES = [
+  ETL_EXTRACT_SUFFIX,
+  ETL_TRANSFORM_SUFFIX,
+  ETL_LOAD_SUFFIX,
+  TOKEN_BUCKET_PROCESS_SUFFIX
+];
+function resolveJobName(name, jobConfigs) {
+  for (const suffix of ALL_SUFFIXES) {
+    if (!name.endsWith(suffix)) continue;
+    const baseName = name.slice(0, -suffix.length);
+    if (jobConfigs[baseName]) {
+      return { baseName, suffix };
+    }
+  }
+  return { baseName: name, suffix: null };
+}
+async function processJob(job, jobConfigs, queues, defaultQueue) {
+  const { baseName, suffix } = resolveJobName(job.name, jobConfigs);
+  const jobConfig = getJobConfig(baseName, jobConfigs);
+  const resolvedQueue = jobConfig.queue ?? defaultQueue;
+  if (suffix !== null) {
+    if (jobConfig.type === "distributed-etl") {
+      if (suffix === ETL_EXTRACT_SUFFIX) {
+        return await jobConfig.extract(job.data.cursor);
+      }
+      if (suffix === ETL_TRANSFORM_SUFFIX) {
+        return await jobConfig.transform(job.data);
+      }
+      if (suffix === ETL_LOAD_SUFFIX) {
+        return await jobConfig.load(job.data.data);
+      }
+    }
+    if (jobConfig.type === "stateful-etl") {
+      if (suffix === ETL_EXTRACT_SUFFIX) {
+        return await jobConfig.extract(job.data.cursor);
+      }
+      if (suffix === ETL_TRANSFORM_SUFFIX) {
+        return await jobConfig.transform(job.data);
+      }
+      if (suffix === ETL_LOAD_SUFFIX) {
+        return await jobConfig.load(job.data.data);
+      }
+    }
+    if (jobConfig.type === "token-bucket") {
+      if (suffix === TOKEN_BUCKET_PROCESS_SUFFIX) {
+        return await jobConfig.process(job.data);
+      }
+    }
+    throw new Error(`Unexpected sub-job suffix "${suffix}" for ${baseName} (type: ${jobConfig.type})`);
+  }
+  if (jobConfig.type === "simple") {
+    return await processSimpleJob(job, jobConfig);
+  }
+  if (jobConfig.type === "memory-etl") {
+    return await processMemoryEtlJob(job, jobConfig);
+  }
+  if (jobConfig.type === "distributed-etl") {
+    return await processDistributedEtlJob(job, { ...jobConfig, queue: resolvedQueue }, queues);
+  }
+  if (jobConfig.type === "stateful-etl") {
+    return await processStatefulEtlJob(job, { ...jobConfig, queue: resolvedQueue }, queues);
+  }
+  if (jobConfig.type === "token-bucket") {
+    return await processTokenBucketJob(job, { ...jobConfig, queue: resolvedQueue }, queues);
+  }
+  throw new Error(`Unknown job type for: ${job.name}`);
+}
+function getJobConfig(jobName, jobConfigs) {
+  const config = jobConfigs[jobName];
+  if (!config) throw new Error(`Job handler not found: ${jobName}`);
+  return config;
 }
 
 // src/banner.ts
@@ -71,16 +1016,47 @@ var RAVENFLOW_ASCII = `
  \u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557  \u2588\u2588\u2551     \u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2557 \u2588\u2588\u2551
  \u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u255A\u2588\u2588\u2557 \u2588\u2588\u2554\u255D\u2588\u2588\u2554\u2550\u2550\u255D  \u2588\u2588\u2551\u255A\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255D  \u2588\u2588\u2551     \u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551\u2588\u2588\u2588\u2557\u2588\u2588\u2551
  \u2588\u2588\u2551  \u2588\u2588\u2551\u2588\u2588\u2551  \u2588\u2588\u2551 \u255A\u2588\u2588\u2588\u2588\u2554\u255D \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u255A\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551     \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u255A\u2588\u2588\u2588\u2554\u2588\u2588\u2588\u2554\u255D
- \u255A\u2550\u255D  \u255A\u2550\u255D\u255A\u2550\u255D  \u255A\u2550\u255D  \u255A\u2550\u2550\u2550\u255D  \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D\u255A\u2550\u255D  \u255A\u2550\u2550\u2550\u255D\u255A\u2550\u255D     \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D  \u255A\u2550\u2550\u255D\u255A\u2550\u2550\u255D 
+ \u255A\u2550\u255D  \u255A\u2550\u255D\u255A\u2550\u255D  \u255A\u2550\u255D  \u255A\u2550\u2550\u2550\u255D  \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D\u255A\u2550\u255D  \u255A\u2550\u2550\u2550\u255D\u255A\u2550\u255D     \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D  \u255A\u2550\u2550\u255D\u255A\u2550\u2550\u255D
+`;
+var RAVENFLOW_ASCII_ART = `                                        :==++==-:::..
+                             :--:::.*%%%%###+====--:::::.
+                           ===--:+@@%%%##%##*==-=%%%###*##:
+                         :+===-*@@%%%######*#*#%@@%
+                        =++==*@@%%%#########@@@@%.
+                       -*+==@@@%%%######*%@@@@%*
+                       **+#@@%%%#######%@@@@%%*=
+                      =**@@%%%%#######@@@@%%%**
+                      **@@%%%%#%####@@@@@%%#**+
+                     +#@%%%%%%#%##%@@@@@%%#***-
+                     %%%%%%#%#%##@@@@@%%%*+***
+                   .@%%%%%#%%%#%@@@@%%%%+=++*:
+                  =%%%%%#%%%##@@@@@%%%%===++=
+                 #%%%%%%%%##@@@@@%%%%*-=====
+               =%%%%%%%%%#@@@@@%%%%%::-===
+             =%%%#%%%#%@@@@@@@%%%=.:::-:
+           +******#%%%%%%%#*=:   . .
 `;
+function censorUri(uri) {
+  try {
+    const url = new URL(uri);
+    if (url.password) url.password = "***";
+    return url.toString();
+  } catch {
+    return uri;
+  }
+}
 function printWorkerBanner(config) {
-  const { queueName, concurrency, redis, jobs } = config;
+  const { queueName, concurrency, redis, mongo, jobs } = config;
+  console.log("\n\n" + RAVENFLOW_ASCII_ART);
   console.log("\n" + RAVENFLOW_ASCII);
   console.log("\u2550".repeat(85));
   console.log("\n[Config]");
   console.log(`  -> queue:       ${queueName}`);
   console.log(`  -> concurrency: ${concurrency}`);
   console.log(`  -> redis:       ${redis.host}:${redis.port}/${redis.db}`);
+  if (mongo) {
+    console.log(`  -> mongo:       ${censorUri(mongo)}`);
+  }
   console.log("\n[Registered Jobs]");
   const regularJobs = [];
   const cronJobs = [];
@@ -105,198 +1081,388 @@ function printWorkerBanner(config) {
   console.log("Ready to process jobs...\n");
 }
 
-// src/app.ts
-var RavenflowApp = class {
-  constructor(connection, config, loggerOptions) {
-    this.defaultConcurrency = 5;
-    this.connection = connection;
-    this.config = config;
-    this.logger = createLogger(loggerOptions);
-    if (!config.queues.includes(config.defaultQueue)) {
-      throw new Error(
-        `Default queue ${config.defaultQueue} is not in the list of queues`
-      );
+// src/app/worker-manager.ts
+function createWorker(queueName, connection, concurrency, processJob2) {
+  const worker = new bullmq.Worker(queueName, processJob2, {
+    connection,
+    autorun: false,
+    concurrency
+  });
+  return worker;
+}
+function setupWorkerEvents(worker, queueName) {
+  const logger2 = getLogger();
+  worker.on("active", (job) => {
+    logger2.info(`[${queueName}] ${job.name} - active`);
+  });
+  worker.on("completed", (job) => {
+    logger2.info(`[${queueName}] ${job.name} - completed`);
+  });
+  worker.on("failed", (job, error) => {
+    logger2.error(`[${queueName}] ${job?.name} - failed: ${error.message}`);
+  });
+}
+function printBanner(queueName, concurrency, connection, jobConfigs, mongoUri) {
+  const redisInfo = {
+    host: typeof connection === "object" && "host" in connection ? connection.host : "localhost",
+    port: typeof connection === "object" && "port" in connection ? connection.port : 6379,
+    db: typeof connection === "object" && "db" in connection ? connection.db : 0
+  };
+  printWorkerBanner({
+    queueName,
+    concurrency,
+    redis: redisInfo,
+    mongo: mongoUri,
+    jobs: jobConfigs
+  });
+}
+async function closeWorkers(workers) {
+  await Promise.all(workers.map((w) => w.close()));
+}
+
+// src/app/scheduler.ts
+async function syncJobSchedulers(queue, jobConfigs) {
+  const logger2 = getLogger();
+  logger2.info("[Scheduled Jobs]");
+  const jobConfigsWithCron = {};
+  for (const jobName in jobConfigs) {
+    if (jobConfigs[jobName].cron) {
+      jobConfigsWithCron[jobName] = jobConfigs[jobName];
     }
-    const { jobsProxy, jobConfigs } = this.initJobs(config);
-    this.jobs = jobsProxy;
-    this.jobConfigs = jobConfigs;
-    this.queues = this.initQueues(config);
-  }
-  initQueues(config) {
-    const queues = {};
-    for (const queueName of config.queues) {
-      queues[queueName] = {
-        queue: new bullmq.Queue(queueName, {
-          connection: this.connection
-        }),
-        events: new bullmq.QueueEvents(queueName, {
-          connection: this.connection
-        })
-      };
+  }
+  const registeredJobSchedulers = await queue.getJobSchedulers();
+  const existingSchedulerKeys = new Set(registeredJobSchedulers.map((s) => s.key));
+  const removed = [];
+  for (const jobScheduler of registeredJobSchedulers) {
+    if (!jobConfigsWithCron[jobScheduler.key]) {
+      removed.push(jobScheduler.key);
+      await queue.removeJobScheduler(jobScheduler.key);
     }
-    return queues;
   }
-  initJobs(config) {
-    const jobsProxy = {};
-    const jobConfigs = {};
-    for (const [jobName, jobConfig] of Object.entries(config.jobs)) {
-      jobConfigs[jobName] = jobConfig;
-      jobsProxy[jobName] = {
-        send: async (params) => {
-          return this.sendJob(jobName, params);
-        }
-      };
+  const newJobs = [];
+  const updated = [];
+  for (const jobName in jobConfigsWithCron) {
+    const jobConfig = jobConfigsWithCron[jobName];
+    if (!jobConfig?.cron) continue;
+    if (existingSchedulerKeys.has(jobName)) {
+      updated.push(jobName);
+    } else {
+      newJobs.push(jobName);
     }
-    return {
-      jobsProxy,
-      jobConfigs
-    };
+    await queue.upsertJobScheduler(jobName, { pattern: jobConfig.cron });
+  }
+  for (const name of newJobs) {
+    logger2.info(`  + ${name} \u2192 ${jobConfigsWithCron[name].cron}`);
   }
-  async sendJob(jobName, params) {
-    const jobConfig = this.jobConfigs[jobName];
-    if (!jobConfig) {
-      throw new Error(`Job ${jobName} not found`);
+  for (const name of updated) {
+    logger2.info(`  \u21BB ${name} \u2192 ${jobConfigsWithCron[name].cron}`);
+  }
+  for (const name of removed) {
+    logger2.info(`  - ${name}`);
+  }
+  if (newJobs.length === 0 && updated.length === 0 && removed.length === 0) {
+    logger2.info("  (no scheduled jobs)");
+  }
+}
+
+// src/internal-jobs/stall-watcher/types.ts
+var STALL_CHECK = "__stall_check";
+var STALL_RESUME = "__stall_resume";
+
+// src/internal-jobs/stall-watcher/check-stalled-runs.ts
+async function checkStalledRuns(statefulEtlConfigs, userQueues, workerQueue) {
+  const logger2 = getLogger();
+  const candidates = await run_repo_exports.findStalledCandidates();
+  for (const candidate of candidates) {
+    const entry = statefulEtlConfigs.get(candidate.jobName);
+    if (!entry) {
+      logger2.warn(
+        { runId: candidate._id, jobName: candidate.jobName },
+        "[stall-watcher] unknown job, marking failed"
+      );
+      await run_repo_exports.markFailed(
+        candidate._id,
+        "Job config not found during stall recovery"
+      );
+      continue;
     }
-    const queueData = this.queues[jobConfig.queueName];
+    const queueData = userQueues[entry.queue];
     if (!queueData) {
-      throw new Error(`Queue ${jobConfig.queueName} not found`);
+      logger2.error(
+        { queueName: entry.queue, runId: candidate._id },
+        "[stall-watcher] queue not found"
+      );
+      await run_repo_exports.markFailed(
+        candidate._id,
+        `Queue "${entry.queue}" not found during stall recovery`
+      );
+      continue;
     }
-    return await queueData.queue.add(jobName, params);
-  }
-  startWorker(queueName, options) {
-    const queueConfig = this.queues[queueName];
-    if (!queueConfig) {
-      throw new Error(`Queue ${queueName} not found in config.`);
+    if (await isJobActiveInRedis(queueData.queue, candidate.jobId)) {
+      logger2.debug(
+        { runId: candidate._id, jobId: candidate.jobId },
+        "[stall-watcher] job still active in Redis, skipping"
+      );
+      continue;
     }
-    const workerConcurrency = options?.concurrency || this.defaultConcurrency;
-    this.printWorkerBanner(queueName, workerConcurrency);
-    this.syncJobSchedulers(queueName);
-    const worker = new bullmq.Worker(queueName, (job) => this.processJob(job), {
-      connection: this.connection,
-      autorun: false,
-      concurrency: workerConcurrency
+    const claimed = await run_repo_exports.claimStalledRun(candidate._id);
+    if (!claimed) continue;
+    await workerQueue.add(STALL_RESUME, {
+      runId: claimed._id,
+      jobName: claimed.jobName,
+      queueName: entry.queue,
+      maxStallRetries: claimed.maxStallRetries,
+      stallRetries: claimed.progress.stallRetries,
+      checkpoint: claimed.progress.checkpoint,
+      extractedCount: claimed.progress.extractedCount,
+      transformedCount: claimed.progress.transformedCount,
+      loadedCount: claimed.progress.loadedCount
     });
-    this.setupWorkerEvents(worker, queueName);
-    worker.run();
   }
-  printWorkerBanner(queueName, concurrency) {
-    const redisInfo = {
-      host: typeof this.connection === "object" && "host" in this.connection ? this.connection.host : "localhost",
-      port: typeof this.connection === "object" && "port" in this.connection ? this.connection.port : 6379,
-      db: typeof this.connection === "object" && "db" in this.connection ? this.connection.db : 0
-    };
-    printWorkerBanner({
-      queueName,
-      concurrency,
-      redis: redisInfo,
-      jobs: this.jobConfigs
-    });
+}
+async function isJobActiveInRedis(queue, jobId) {
+  try {
+    const state = await queue.getJobState(jobId);
+    return state === "active";
+  } catch {
+    return false;
   }
-  setupWorkerEvents(worker, queueName) {
-    worker.on("active", (job) => {
-      this.logger.info(`[${queueName}] ${job.name} - active`);
-    });
-    worker.on("completed", (job) => {
-      this.logger.info(`[${queueName}] ${job.name} - completed`);
-    });
-    worker.on("failed", (job, error) => {
-      this.logger.error(
-        `[${queueName}] ${job?.name} - failed: ${error.message}`
-      );
-    });
+}
+
+// src/internal-jobs/stall-watcher/resume-stalled-run.ts
+async function resumeStalledRun(data, userQueues) {
+  const logger2 = getLogger();
+  if (data.stallRetries > data.maxStallRetries) {
+    logger2.warn(
+      {
+        runId: data.runId,
+        jobName: data.jobName,
+        stallRetries: data.stallRetries
+      },
+      "[stall-watcher] max stall retries exceeded, marking failed"
+    );
+    await run_repo_exports.markFailed(
+      data.runId,
+      `Stall recovery exhausted after ${data.stallRetries} retries`
+    );
+    return;
   }
-  async syncJobSchedulers(queueName) {
-    const queueData = this.queues[queueName];
-    if (!queueData) {
-      throw new Error(`Queue ${queueName} not found in config.`);
-    }
-    const queue = queueData.queue;
-    console.log("[Scheduled Jobs]");
-    const jobConfigsWithCron = {};
-    for (const jobName in this.jobConfigs) {
-      const jobConfig = this.jobConfigs[jobName];
-      if (jobConfig.cron) {
-        jobConfigsWithCron[jobName] = jobConfig;
-      }
-    }
-    const registeredJobSchedulers = await queue.getJobSchedulers();
-    const existingSchedulerKeys = new Set(
-      registeredJobSchedulers.map((s) => s.key)
+  const queueData = userQueues[data.queueName];
+  if (!queueData) {
+    logger2.error(
+      { queueName: data.queueName, runId: data.runId },
+      "[stall-watcher] queue not found for resume"
     );
-    const removed = [];
-    for (const jobScheduler of registeredJobSchedulers) {
-      if (!jobConfigsWithCron[jobScheduler.key]) {
-        removed.push(jobScheduler.key);
-        await queue.removeJobScheduler(jobScheduler.key);
-      }
+    await run_repo_exports.markFailed(
+      data.runId,
+      `Queue "${data.queueName}" not found`
+    );
+    return;
+  }
+  logger2.info(
+    {
+      runId: data.runId,
+      jobName: data.jobName,
+      stallRetries: data.stallRetries,
+      checkpoint: data.checkpoint
+    },
+    "[stall-watcher] re-enqueuing stalled run"
+  );
+  await queueData.queue.add(data.jobName, {
+    __ravenflow_resume: true,
+    runId: data.runId,
+    checkpoint: data.checkpoint,
+    counts: {
+      extractedCount: data.extractedCount,
+      transformedCount: data.transformedCount,
+      loadedCount: data.loadedCount
     }
-    const newJobs = [];
-    const updated = [];
-    for (const jobName in jobConfigsWithCron) {
-      const jobConfig = jobConfigsWithCron[jobName];
-      if (!jobConfig?.cron) continue;
-      if (existingSchedulerKeys.has(jobName)) {
-        updated.push(jobName);
-      } else {
-        newJobs.push(jobName);
+  });
+}
+
+// src/internal-jobs/stall-watcher/index.ts
+var StallWatcher = class {
+  constructor(jobConfigs, userQueues, defaultQueue, workerQueue) {
+    this.userQueues = userQueues;
+    this.workerQueue = workerQueue;
+    this.statefulEtlConfigs = /* @__PURE__ */ new Map();
+    for (const [jobName, config] of Object.entries(jobConfigs)) {
+      if (config.type === "stateful-etl") {
+        this.statefulEtlConfigs.set(jobName, {
+          config,
+          queue: config.queue ?? defaultQueue
+        });
       }
-      await queue.upsertJobScheduler(jobName, { pattern: jobConfig.cron });
     }
-    if (newJobs.length > 0) {
-      newJobs.forEach(
-        (name) => console.log(`  + ${name} \u2192 ${jobConfigsWithCron[name].cron}`)
-      );
+  }
+  canProcess(jobName) {
+    return jobName === STALL_CHECK || jobName === STALL_RESUME;
+  }
+  async start() {
+    if (this.statefulEtlConfigs.size === 0) return;
+    const logger2 = getLogger();
+    logger2.info(
+      { jobs: [...this.statefulEtlConfigs.keys()] },
+      "[stall-watcher] starting"
+    );
+    await this.workerQueue.upsertJobScheduler(STALL_CHECK, {
+      pattern: "* * * * *"
+    });
+  }
+  async stop() {
+    await this.workerQueue.removeJobScheduler(STALL_CHECK);
+  }
+  async process(job) {
+    switch (job.name) {
+      case STALL_CHECK:
+        return checkStalledRuns(
+          this.statefulEtlConfigs,
+          this.userQueues,
+          this.workerQueue
+        );
+      case STALL_RESUME:
+        return resumeStalledRun(
+          job.data,
+          this.userQueues
+        );
     }
-    if (updated.length > 0) {
-      updated.forEach(
-        (name) => console.log(`  \u21BB ${name} \u2192 ${jobConfigsWithCron[name].cron}`)
+  }
+};
+
+// src/app/app.ts
+var MONGO_REQUIRED_TYPES = /* @__PURE__ */ new Set(["stateful-etl", "token-bucket"]);
+function resolveMongoUri(mongo) {
+  if (!mongo) return null;
+  if (typeof mongo === "string") return mongo;
+  const { host, port = 27017, username, password, database } = mongo;
+  const auth = username && password ? `${encodeURIComponent(username)}:${encodeURIComponent(password)}@` : "";
+  const db2 = database ? `/${database}` : "";
+  return `mongodb://${auth}${host}:${port}${db2}`;
+}
+function validateConfig(config, mongoUri) {
+  const errors = [];
+  for (const [jobName, jobConfig] of Object.entries(config.jobs)) {
+    const jc = jobConfig;
+    if (MONGO_REQUIRED_TYPES.has(jc.type) && !mongoUri) {
+      errors.push(
+        `"${jobName}" (${jc.type}) requires a mongo connection. Pass { mongo: "mongodb://..." } to createRavenflowApp().`
       );
     }
-    if (removed.length > 0) {
-      removed.forEach((name) => console.log(`  - ${name}`));
+    if (jc.type === "token-bucket" && !jc.cron) {
+      errors.push(`"${jobName}" (token-bucket) requires a cron schedule.`);
     }
-    if (newJobs.length === 0 && updated.length === 0 && removed.length === 0) {
-      console.log(`  (no scheduled jobs)`);
+    if ("steps" in jc && jc.steps) {
+      for (const [stepName, stepConfig] of Object.entries(jc.steps)) {
+        if (stepConfig?.queue && stepConfig.queue !== config.defaultQueue) {
+          const knownQueues = /* @__PURE__ */ new Set([config.defaultQueue]);
+          for (const j of Object.values(config.jobs)) {
+            if (j.queue) knownQueues.add(j.queue);
+          }
+          if (!knownQueues.has(stepConfig.queue)) {
+            errors.push(
+              `"${jobName}" step "${stepName}" references queue "${stepConfig.queue}" which is not declared by any job or as defaultQueue.`
+            );
+          }
+        }
+      }
     }
-    console.log("");
   }
-  async processJob(job) {
-    const isEtlTransformJob = job.name.endsWith(":transform");
-    const jobName = isEtlTransformJob ? job.name.slice(0, -":transform".length) : job.name;
-    const jobConfig = this.jobConfigs[jobName];
-    if (!jobConfig) {
-      throw new Error(`Job handler not found: ${jobName}`);
+  if (errors.length > 0) {
+    throw new Error(
+      `[ravenflow] Config validation failed:
+  - ${errors.join("\n  - ")}`
+    );
+  }
+}
+var RavenflowApp = class {
+  constructor(config, options) {
+    this.defaultConcurrency = 5;
+    this.workers = [];
+    this.stallWatcher = null;
+    this.redisConnection = options.redis;
+    this.config = config;
+    this.mongoUri = resolveMongoUri(options.mongo);
+    initLogger(options.logger);
+    validateConfig(config, this.mongoUri);
+    this.queues = initQueues(config, options.redis, options.retention);
+    const { jobsProxy, jobConfigs } = initJobs(config, this.queues);
+    this.jobs = jobsProxy;
+    this.jobConfigs = jobConfigs;
+  }
+  async startWorker(queueName, options) {
+    if (this.mongoUri) {
+      await connectMongo(this.mongoUri);
     }
-    if (jobConfig.type === "job") {
-      return await jobConfig.handler(job, job.data);
+    const queueData = this.queues[queueName];
+    if (!queueData) {
+      throw new Error(`Queue ${queueName} not found in config.`);
     }
-    if (jobConfig.type === "etl" && !isEtlTransformJob) {
-      const transformQueueName = jobConfig.transformQueue || jobConfig.queueName;
-      const queueData = this.queues[transformQueueName];
-      if (!queueData) {
-        throw new Error(`Queue ${transformQueueName} not found in config.`);
-      }
-      return await processEtlJob(
-        job,
-        jobConfig,
-        queueData.queue,
-        queueData.events
+    const concurrency = options?.concurrency ?? this.defaultConcurrency;
+    printBanner(
+      queueName,
+      concurrency,
+      this.redisConnection,
+      this.jobConfigs,
+      this.mongoUri
+    );
+    await syncJobSchedulers(queueData.queue, this.jobConfigs);
+    const hasStatefulEtl = Object.values(this.jobConfigs).some(
+      (c) => c.type === "stateful-etl"
+    );
+    if (hasStatefulEtl && !this.stallWatcher) {
+      this.stallWatcher = new StallWatcher(
+        this.jobConfigs,
+        this.queues,
+        this.config.defaultQueue,
+        queueData.queue
       );
     }
-    if (jobConfig.type === "etl" && isEtlTransformJob) {
-      return await jobConfig.transform(job.data);
+    const worker = createWorker(
+      queueName,
+      this.redisConnection,
+      concurrency,
+      async (job) => {
+        if (this.stallWatcher?.canProcess(job.name)) {
+          return this.stallWatcher.process(job);
+        }
+        return processJob(job, this.jobConfigs, this.queues, this.config.defaultQueue);
+      }
+    );
+    setupWorkerEvents(worker, queueName);
+    this.workers.push(worker);
+    worker.run();
+    if (this.stallWatcher) {
+      await this.stallWatcher.start();
+    }
+  }
+  async close() {
+    const logger2 = getLogger();
+    logger2.info("Shutting down...");
+    if (this.stallWatcher) {
+      await this.stallWatcher.stop();
+      this.stallWatcher = null;
     }
-    throw new Error(`Unknown job type: ${jobConfig.type}`);
+    await closeWorkers(this.workers);
+    await closeQueues(this.queues);
+    await closeMongo();
+    logger2.info("Shutdown complete.");
   }
 };
 
 // src/index.ts
-function createRavenflowApp(config, connection, loggerOptions) {
-  return new RavenflowApp(connection, config, loggerOptions);
+function createRavenflowApp(config, options) {
+  return new RavenflowApp(config, options);
 }
 
 exports.RavenflowApp = RavenflowApp;
-exports.createLogger = createLogger;
 exports.createRavenflowApp = createRavenflowApp;
+exports.defineConfig = defineConfig;
+exports.defineDistributedEtlJob = defineDistributedEtlJob;
+exports.defineMemoryEtlJob = defineMemoryEtlJob;
+exports.defineSimpleJob = defineSimpleJob;
+exports.defineStatefulEtlJob = defineStatefulEtlJob;
+exports.defineTokenBucketJob = defineTokenBucketJob;
+exports.getLogger = getLogger;
+exports.initLogger = initLogger;
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map