ravenflow 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -0
- package/dist/index.d.mts +83 -0
- package/dist/index.d.ts +83 -0
- package/dist/index.js +302 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +294 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +58 -0
package/README.md
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Ravenflow
|
|
2
|
+
|
|
3
|
+
Type-safe job queue management for TypeScript.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npm install ravenflow ioredis
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Quick Start
|
|
12
|
+
|
|
13
|
+
```typescript
|
|
14
|
+
import { createRavenflowApp } from "ravenflow";
|
|
15
|
+
|
|
16
|
+
const app = createRavenflowApp(
|
|
17
|
+
{
|
|
18
|
+
defaultQueue: "default",
|
|
19
|
+
queues: ["default"],
|
|
20
|
+
jobs: {
|
|
21
|
+
sendEmail: {
|
|
22
|
+
type: "job",
|
|
23
|
+
queueName: "default",
|
|
24
|
+
handler: async (job, params: { to: string; subject: string }) => {
|
|
25
|
+
console.log(`Sending email to ${params.to}`);
|
|
26
|
+
return { sent: true };
|
|
27
|
+
},
|
|
28
|
+
},
|
|
29
|
+
},
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
host: "localhost",
|
|
33
|
+
port: 6379,
|
|
34
|
+
}
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
// Enqueue job
|
|
38
|
+
await app.jobs.sendEmail.send({
|
|
39
|
+
to: "user@example.com",
|
|
40
|
+
subject: "Hello",
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
// Start worker
|
|
44
|
+
app.startWorker("default");
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## License
|
|
48
|
+
|
|
49
|
+
MIT
|
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { Job, ConnectionOptions, Queue, QueueEvents } from 'bullmq';
|
|
2
|
+
import * as pino from 'pino';
|
|
3
|
+
|
|
4
|
+
interface RavenflowConfig<TQueues extends string = string, TJobs extends string = string> {
|
|
5
|
+
defaultQueue: TQueues;
|
|
6
|
+
queues: TQueues[];
|
|
7
|
+
jobs: Record<TJobs, JobConfig>;
|
|
8
|
+
}
|
|
9
|
+
type JobNames<T extends RavenflowConfig> = T extends RavenflowConfig<infer _, infer TJobs> ? TJobs : never;
|
|
10
|
+
type QueueNames<T extends RavenflowConfig> = T extends RavenflowConfig<infer TQueues, infer _> ? TQueues : never;
|
|
11
|
+
interface BaseJobConfig {
|
|
12
|
+
queueName: string;
|
|
13
|
+
cron?: string;
|
|
14
|
+
attempts?: number;
|
|
15
|
+
backoff?: {
|
|
16
|
+
type: "fixed" | "exponential";
|
|
17
|
+
delay: number;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
interface SimpleJobConfig<TData, TReturn> extends BaseJobConfig {
|
|
21
|
+
type: "job";
|
|
22
|
+
handler: (job: Job, params: TData) => Promise<TReturn>;
|
|
23
|
+
}
|
|
24
|
+
interface EtlJobConfig<TRaw, TProcessed> extends BaseJobConfig {
|
|
25
|
+
type: "etl";
|
|
26
|
+
extract: (page: number) => Promise<{
|
|
27
|
+
data: TRaw[];
|
|
28
|
+
hasNextPage?: boolean;
|
|
29
|
+
}>;
|
|
30
|
+
transform: (data: TRaw) => Promise<TProcessed>;
|
|
31
|
+
load: (data: TProcessed[]) => Promise<void>;
|
|
32
|
+
transformQueue?: string;
|
|
33
|
+
}
|
|
34
|
+
type JobConfig = SimpleJobConfig<any, any> | EtlJobConfig<any, any>;
|
|
35
|
+
type JobProxy<T extends JobConfig> = T extends SimpleJobConfig<infer TData, infer TReturn> ? {
|
|
36
|
+
send: (params: TData) => Promise<Job<TData, TReturn>>;
|
|
37
|
+
} : T extends EtlJobConfig<any, any> ? {
|
|
38
|
+
send: (params?: {
|
|
39
|
+
startPage?: number;
|
|
40
|
+
batchSize?: number;
|
|
41
|
+
}) => Promise<void>;
|
|
42
|
+
} : never;
|
|
43
|
+
type JobsProxy<TJobs extends Record<string, JobConfig>> = {
|
|
44
|
+
[K in keyof TJobs]: JobProxy<TJobs[K]>;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
interface LoggerOptions {
|
|
48
|
+
level?: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
|
|
49
|
+
pretty?: boolean;
|
|
50
|
+
}
|
|
51
|
+
declare function createLogger(options?: LoggerOptions): pino.Logger<never>;
|
|
52
|
+
type Logger = ReturnType<typeof createLogger>;
|
|
53
|
+
|
|
54
|
+
declare class RavenflowApp<TConfig extends RavenflowConfig = RavenflowConfig> {
|
|
55
|
+
protected connection: ConnectionOptions;
|
|
56
|
+
protected defaultConcurrency: number;
|
|
57
|
+
protected logger: Logger;
|
|
58
|
+
protected config: TConfig;
|
|
59
|
+
jobs: JobsProxy<TConfig["jobs"]>;
|
|
60
|
+
protected jobConfigs: Record<JobNames<TConfig>, JobConfig>;
|
|
61
|
+
protected queues: Record<QueueNames<TConfig>, {
|
|
62
|
+
queue: Queue;
|
|
63
|
+
events: QueueEvents;
|
|
64
|
+
}>;
|
|
65
|
+
constructor(connection: ConnectionOptions, config: TConfig, loggerOptions?: LoggerOptions);
|
|
66
|
+
private initQueues;
|
|
67
|
+
private initJobs;
|
|
68
|
+
private sendJob;
|
|
69
|
+
startWorker(queueName: QueueNames<TConfig>, options?: {
|
|
70
|
+
concurrency?: number;
|
|
71
|
+
}): void;
|
|
72
|
+
private printWorkerBanner;
|
|
73
|
+
private setupWorkerEvents;
|
|
74
|
+
private syncJobSchedulers;
|
|
75
|
+
processJob(job: Job): Promise<any>;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
declare function createRavenflowApp<TConfig extends RavenflowConfig>(config: TConfig, connection: ConnectionOptions, loggerOptions?: {
|
|
79
|
+
level?: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
|
|
80
|
+
pretty?: boolean;
|
|
81
|
+
}): RavenflowApp<TConfig>;
|
|
82
|
+
|
|
83
|
+
export { type EtlJobConfig, type JobConfig, type Logger, type LoggerOptions, RavenflowApp, type RavenflowConfig, type SimpleJobConfig, createLogger, createRavenflowApp };
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { Job, ConnectionOptions, Queue, QueueEvents } from 'bullmq';
|
|
2
|
+
import * as pino from 'pino';
|
|
3
|
+
|
|
4
|
+
interface RavenflowConfig<TQueues extends string = string, TJobs extends string = string> {
|
|
5
|
+
defaultQueue: TQueues;
|
|
6
|
+
queues: TQueues[];
|
|
7
|
+
jobs: Record<TJobs, JobConfig>;
|
|
8
|
+
}
|
|
9
|
+
type JobNames<T extends RavenflowConfig> = T extends RavenflowConfig<infer _, infer TJobs> ? TJobs : never;
|
|
10
|
+
type QueueNames<T extends RavenflowConfig> = T extends RavenflowConfig<infer TQueues, infer _> ? TQueues : never;
|
|
11
|
+
interface BaseJobConfig {
|
|
12
|
+
queueName: string;
|
|
13
|
+
cron?: string;
|
|
14
|
+
attempts?: number;
|
|
15
|
+
backoff?: {
|
|
16
|
+
type: "fixed" | "exponential";
|
|
17
|
+
delay: number;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
interface SimpleJobConfig<TData, TReturn> extends BaseJobConfig {
|
|
21
|
+
type: "job";
|
|
22
|
+
handler: (job: Job, params: TData) => Promise<TReturn>;
|
|
23
|
+
}
|
|
24
|
+
interface EtlJobConfig<TRaw, TProcessed> extends BaseJobConfig {
|
|
25
|
+
type: "etl";
|
|
26
|
+
extract: (page: number) => Promise<{
|
|
27
|
+
data: TRaw[];
|
|
28
|
+
hasNextPage?: boolean;
|
|
29
|
+
}>;
|
|
30
|
+
transform: (data: TRaw) => Promise<TProcessed>;
|
|
31
|
+
load: (data: TProcessed[]) => Promise<void>;
|
|
32
|
+
transformQueue?: string;
|
|
33
|
+
}
|
|
34
|
+
type JobConfig = SimpleJobConfig<any, any> | EtlJobConfig<any, any>;
|
|
35
|
+
type JobProxy<T extends JobConfig> = T extends SimpleJobConfig<infer TData, infer TReturn> ? {
|
|
36
|
+
send: (params: TData) => Promise<Job<TData, TReturn>>;
|
|
37
|
+
} : T extends EtlJobConfig<any, any> ? {
|
|
38
|
+
send: (params?: {
|
|
39
|
+
startPage?: number;
|
|
40
|
+
batchSize?: number;
|
|
41
|
+
}) => Promise<void>;
|
|
42
|
+
} : never;
|
|
43
|
+
type JobsProxy<TJobs extends Record<string, JobConfig>> = {
|
|
44
|
+
[K in keyof TJobs]: JobProxy<TJobs[K]>;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
interface LoggerOptions {
|
|
48
|
+
level?: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
|
|
49
|
+
pretty?: boolean;
|
|
50
|
+
}
|
|
51
|
+
declare function createLogger(options?: LoggerOptions): pino.Logger<never>;
|
|
52
|
+
type Logger = ReturnType<typeof createLogger>;
|
|
53
|
+
|
|
54
|
+
declare class RavenflowApp<TConfig extends RavenflowConfig = RavenflowConfig> {
|
|
55
|
+
protected connection: ConnectionOptions;
|
|
56
|
+
protected defaultConcurrency: number;
|
|
57
|
+
protected logger: Logger;
|
|
58
|
+
protected config: TConfig;
|
|
59
|
+
jobs: JobsProxy<TConfig["jobs"]>;
|
|
60
|
+
protected jobConfigs: Record<JobNames<TConfig>, JobConfig>;
|
|
61
|
+
protected queues: Record<QueueNames<TConfig>, {
|
|
62
|
+
queue: Queue;
|
|
63
|
+
events: QueueEvents;
|
|
64
|
+
}>;
|
|
65
|
+
constructor(connection: ConnectionOptions, config: TConfig, loggerOptions?: LoggerOptions);
|
|
66
|
+
private initQueues;
|
|
67
|
+
private initJobs;
|
|
68
|
+
private sendJob;
|
|
69
|
+
startWorker(queueName: QueueNames<TConfig>, options?: {
|
|
70
|
+
concurrency?: number;
|
|
71
|
+
}): void;
|
|
72
|
+
private printWorkerBanner;
|
|
73
|
+
private setupWorkerEvents;
|
|
74
|
+
private syncJobSchedulers;
|
|
75
|
+
processJob(job: Job): Promise<any>;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
declare function createRavenflowApp<TConfig extends RavenflowConfig>(config: TConfig, connection: ConnectionOptions, loggerOptions?: {
|
|
79
|
+
level?: "trace" | "debug" | "info" | "warn" | "error" | "fatal";
|
|
80
|
+
pretty?: boolean;
|
|
81
|
+
}): RavenflowApp<TConfig>;
|
|
82
|
+
|
|
83
|
+
export { type EtlJobConfig, type JobConfig, type Logger, type LoggerOptions, RavenflowApp, type RavenflowConfig, type SimpleJobConfig, createLogger, createRavenflowApp };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var bullmq = require('bullmq');
|
|
4
|
+
var pino = require('pino');
|
|
5
|
+
|
|
6
|
+
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
7
|
+
|
|
8
|
+
var pino__default = /*#__PURE__*/_interopDefault(pino);
|
|
9
|
+
|
|
10
|
+
// src/app.ts
|
|
11
|
+
|
|
12
|
+
// src/processor.ts
|
|
13
|
+
async function processEtlJob(job, etlConfig, transformQueue, transformQueueEvents) {
|
|
14
|
+
let currentPage = 1;
|
|
15
|
+
let run = true;
|
|
16
|
+
const metadata = {
|
|
17
|
+
count: 0,
|
|
18
|
+
pages: 0,
|
|
19
|
+
currentPage,
|
|
20
|
+
startedAt: /* @__PURE__ */ new Date(),
|
|
21
|
+
endedAt: void 0
|
|
22
|
+
};
|
|
23
|
+
const pagination = {
|
|
24
|
+
maxPages: Infinity};
|
|
25
|
+
while (run && currentPage <= (pagination.maxPages || Infinity)) {
|
|
26
|
+
metadata.currentPage = currentPage;
|
|
27
|
+
metadata.pages = currentPage;
|
|
28
|
+
const {
|
|
29
|
+
data: rawElements,
|
|
30
|
+
hasNextPage: nextPageExists = false
|
|
31
|
+
} = await etlConfig.extract(currentPage);
|
|
32
|
+
const processedElements = [];
|
|
33
|
+
for (const element of rawElements) {
|
|
34
|
+
const transformJobName = `${job.name}:transform`;
|
|
35
|
+
const transformJob = await transformQueue.add(transformJobName, element, {
|
|
36
|
+
attempts: etlConfig.attempts,
|
|
37
|
+
backoff: etlConfig.backoff
|
|
38
|
+
});
|
|
39
|
+
const processedElement = await transformJob.waitUntilFinished(transformQueueEvents);
|
|
40
|
+
processedElements.push(processedElement);
|
|
41
|
+
}
|
|
42
|
+
await etlConfig.load(processedElements);
|
|
43
|
+
metadata.count += processedElements.length;
|
|
44
|
+
run = nextPageExists;
|
|
45
|
+
if (run) {
|
|
46
|
+
currentPage++;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return metadata;
|
|
50
|
+
}
|
|
51
|
+
function createLogger(options = {}) {
|
|
52
|
+
const isDev = process.env.NODE_ENV !== "production";
|
|
53
|
+
const usePretty = options.pretty ?? isDev;
|
|
54
|
+
return pino__default.default({
|
|
55
|
+
level: options.level || (isDev ? "debug" : "info"),
|
|
56
|
+
transport: usePretty ? {
|
|
57
|
+
target: "pino-pretty",
|
|
58
|
+
options: {
|
|
59
|
+
colorize: true,
|
|
60
|
+
translateTime: "HH:MM:ss",
|
|
61
|
+
ignore: "pid,hostname"
|
|
62
|
+
}
|
|
63
|
+
} : void 0
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// src/banner.ts
|
|
68
|
+
var RAVENFLOW_ASCII = `
|
|
69
|
+
\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557 \u2588\u2588\u2557
|
|
70
|
+
\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551
|
|
71
|
+
\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2557 \u2588\u2588\u2551
|
|
72
|
+
\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u255A\u2588\u2588\u2557 \u2588\u2588\u2554\u255D\u2588\u2588\u2554\u2550\u2550\u255D \u2588\u2588\u2551\u255A\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255D \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551\u2588\u2588\u2588\u2557\u2588\u2588\u2551
|
|
73
|
+
\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551 \u255A\u2588\u2588\u2588\u2588\u2554\u255D \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u255A\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u255A\u2588\u2588\u2588\u2554\u2588\u2588\u2588\u2554\u255D
|
|
74
|
+
\u255A\u2550\u255D \u255A\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u255D \u255A\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u2550\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u255D\u255A\u2550\u2550\u255D
|
|
75
|
+
`;
|
|
76
|
+
function printWorkerBanner(config) {
|
|
77
|
+
const { queueName, concurrency, redis, jobs } = config;
|
|
78
|
+
console.log("\n" + RAVENFLOW_ASCII);
|
|
79
|
+
console.log("\u2550".repeat(85));
|
|
80
|
+
console.log("\n[Config]");
|
|
81
|
+
console.log(` -> queue: ${queueName}`);
|
|
82
|
+
console.log(` -> concurrency: ${concurrency}`);
|
|
83
|
+
console.log(` -> redis: ${redis.host}:${redis.port}/${redis.db}`);
|
|
84
|
+
console.log("\n[Registered Jobs]");
|
|
85
|
+
const regularJobs = [];
|
|
86
|
+
const cronJobs = [];
|
|
87
|
+
for (const [jobName, jobConfig] of Object.entries(jobs)) {
|
|
88
|
+
if (jobConfig.cron) {
|
|
89
|
+
cronJobs.push({ name: jobName, cron: jobConfig.cron });
|
|
90
|
+
} else {
|
|
91
|
+
regularJobs.push(jobName);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (regularJobs.length > 0) {
|
|
95
|
+
regularJobs.forEach((jobName) => {
|
|
96
|
+
console.log(` . ${jobName}`);
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
if (cronJobs.length > 0) {
|
|
100
|
+
cronJobs.forEach(({ name, cron }) => {
|
|
101
|
+
console.log(` . ${name} (\u21BB ${cron})`);
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
console.log("\n" + "\u2550".repeat(85));
|
|
105
|
+
console.log("Ready to process jobs...\n");
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// src/app.ts
|
|
109
|
+
var RavenflowApp = class {
|
|
110
|
+
constructor(connection, config, loggerOptions) {
|
|
111
|
+
this.defaultConcurrency = 5;
|
|
112
|
+
this.connection = connection;
|
|
113
|
+
this.config = config;
|
|
114
|
+
this.logger = createLogger(loggerOptions);
|
|
115
|
+
if (!config.queues.includes(config.defaultQueue)) {
|
|
116
|
+
throw new Error(
|
|
117
|
+
`Default queue ${config.defaultQueue} is not in the list of queues`
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
const { jobsProxy, jobConfigs } = this.initJobs(config);
|
|
121
|
+
this.jobs = jobsProxy;
|
|
122
|
+
this.jobConfigs = jobConfigs;
|
|
123
|
+
this.queues = this.initQueues(config);
|
|
124
|
+
}
|
|
125
|
+
initQueues(config) {
|
|
126
|
+
const queues = {};
|
|
127
|
+
for (const queueName of config.queues) {
|
|
128
|
+
queues[queueName] = {
|
|
129
|
+
queue: new bullmq.Queue(queueName, {
|
|
130
|
+
connection: this.connection
|
|
131
|
+
}),
|
|
132
|
+
events: new bullmq.QueueEvents(queueName, {
|
|
133
|
+
connection: this.connection
|
|
134
|
+
})
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
return queues;
|
|
138
|
+
}
|
|
139
|
+
initJobs(config) {
|
|
140
|
+
const jobsProxy = {};
|
|
141
|
+
const jobConfigs = {};
|
|
142
|
+
for (const [jobName, jobConfig] of Object.entries(config.jobs)) {
|
|
143
|
+
jobConfigs[jobName] = jobConfig;
|
|
144
|
+
jobsProxy[jobName] = {
|
|
145
|
+
send: async (params) => {
|
|
146
|
+
return this.sendJob(jobName, params);
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
return {
|
|
151
|
+
jobsProxy,
|
|
152
|
+
jobConfigs
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
async sendJob(jobName, params) {
|
|
156
|
+
const jobConfig = this.jobConfigs[jobName];
|
|
157
|
+
if (!jobConfig) {
|
|
158
|
+
throw new Error(`Job ${jobName} not found`);
|
|
159
|
+
}
|
|
160
|
+
const queueData = this.queues[jobConfig.queueName];
|
|
161
|
+
if (!queueData) {
|
|
162
|
+
throw new Error(`Queue ${jobConfig.queueName} not found`);
|
|
163
|
+
}
|
|
164
|
+
return await queueData.queue.add(jobName, params);
|
|
165
|
+
}
|
|
166
|
+
startWorker(queueName, options) {
|
|
167
|
+
const queueConfig = this.queues[queueName];
|
|
168
|
+
if (!queueConfig) {
|
|
169
|
+
throw new Error(`Queue ${queueName} not found in config.`);
|
|
170
|
+
}
|
|
171
|
+
const workerConcurrency = options?.concurrency || this.defaultConcurrency;
|
|
172
|
+
this.printWorkerBanner(queueName, workerConcurrency);
|
|
173
|
+
this.syncJobSchedulers(queueName);
|
|
174
|
+
const worker = new bullmq.Worker(queueName, (job) => this.processJob(job), {
|
|
175
|
+
connection: this.connection,
|
|
176
|
+
autorun: false,
|
|
177
|
+
concurrency: workerConcurrency
|
|
178
|
+
});
|
|
179
|
+
this.setupWorkerEvents(worker, queueName);
|
|
180
|
+
worker.run();
|
|
181
|
+
}
|
|
182
|
+
printWorkerBanner(queueName, concurrency) {
|
|
183
|
+
const redisInfo = {
|
|
184
|
+
host: typeof this.connection === "object" && "host" in this.connection ? this.connection.host : "localhost",
|
|
185
|
+
port: typeof this.connection === "object" && "port" in this.connection ? this.connection.port : 6379,
|
|
186
|
+
db: typeof this.connection === "object" && "db" in this.connection ? this.connection.db : 0
|
|
187
|
+
};
|
|
188
|
+
printWorkerBanner({
|
|
189
|
+
queueName,
|
|
190
|
+
concurrency,
|
|
191
|
+
redis: redisInfo,
|
|
192
|
+
jobs: this.jobConfigs
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
setupWorkerEvents(worker, queueName) {
|
|
196
|
+
worker.on("active", (job) => {
|
|
197
|
+
this.logger.info(`[${queueName}] ${job.name} - active`);
|
|
198
|
+
});
|
|
199
|
+
worker.on("completed", (job) => {
|
|
200
|
+
this.logger.info(`[${queueName}] ${job.name} - completed`);
|
|
201
|
+
});
|
|
202
|
+
worker.on("failed", (job, error) => {
|
|
203
|
+
this.logger.error(
|
|
204
|
+
`[${queueName}] ${job?.name} - failed: ${error.message}`
|
|
205
|
+
);
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
async syncJobSchedulers(queueName) {
|
|
209
|
+
const queueData = this.queues[queueName];
|
|
210
|
+
if (!queueData) {
|
|
211
|
+
throw new Error(`Queue ${queueName} not found in config.`);
|
|
212
|
+
}
|
|
213
|
+
const queue = queueData.queue;
|
|
214
|
+
console.log("[Scheduled Jobs]");
|
|
215
|
+
const jobConfigsWithCron = {};
|
|
216
|
+
for (const jobName in this.jobConfigs) {
|
|
217
|
+
const jobConfig = this.jobConfigs[jobName];
|
|
218
|
+
if (jobConfig.cron) {
|
|
219
|
+
jobConfigsWithCron[jobName] = jobConfig;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
const registeredJobSchedulers = await queue.getJobSchedulers();
|
|
223
|
+
const existingSchedulerKeys = new Set(
|
|
224
|
+
registeredJobSchedulers.map((s) => s.key)
|
|
225
|
+
);
|
|
226
|
+
const removed = [];
|
|
227
|
+
for (const jobScheduler of registeredJobSchedulers) {
|
|
228
|
+
if (!jobConfigsWithCron[jobScheduler.key]) {
|
|
229
|
+
removed.push(jobScheduler.key);
|
|
230
|
+
await queue.removeJobScheduler(jobScheduler.key);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
const newJobs = [];
|
|
234
|
+
const updated = [];
|
|
235
|
+
for (const jobName in jobConfigsWithCron) {
|
|
236
|
+
const jobConfig = jobConfigsWithCron[jobName];
|
|
237
|
+
if (!jobConfig?.cron) continue;
|
|
238
|
+
if (existingSchedulerKeys.has(jobName)) {
|
|
239
|
+
updated.push(jobName);
|
|
240
|
+
} else {
|
|
241
|
+
newJobs.push(jobName);
|
|
242
|
+
}
|
|
243
|
+
await queue.upsertJobScheduler(jobName, { pattern: jobConfig.cron });
|
|
244
|
+
}
|
|
245
|
+
if (newJobs.length > 0) {
|
|
246
|
+
newJobs.forEach(
|
|
247
|
+
(name) => console.log(` + ${name} \u2192 ${jobConfigsWithCron[name].cron}`)
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
if (updated.length > 0) {
|
|
251
|
+
updated.forEach(
|
|
252
|
+
(name) => console.log(` \u21BB ${name} \u2192 ${jobConfigsWithCron[name].cron}`)
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
if (removed.length > 0) {
|
|
256
|
+
removed.forEach((name) => console.log(` - ${name}`));
|
|
257
|
+
}
|
|
258
|
+
if (newJobs.length === 0 && updated.length === 0 && removed.length === 0) {
|
|
259
|
+
console.log(` (no scheduled jobs)`);
|
|
260
|
+
}
|
|
261
|
+
console.log("");
|
|
262
|
+
}
|
|
263
|
+
async processJob(job) {
|
|
264
|
+
const isEtlTransformJob = job.name.endsWith(":transform");
|
|
265
|
+
const jobName = isEtlTransformJob ? job.name.slice(0, -":transform".length) : job.name;
|
|
266
|
+
const jobConfig = this.jobConfigs[jobName];
|
|
267
|
+
if (!jobConfig) {
|
|
268
|
+
throw new Error(`Job handler not found: ${jobName}`);
|
|
269
|
+
}
|
|
270
|
+
if (jobConfig.type === "job") {
|
|
271
|
+
return await jobConfig.handler(job, job.data);
|
|
272
|
+
}
|
|
273
|
+
if (jobConfig.type === "etl" && !isEtlTransformJob) {
|
|
274
|
+
const transformQueueName = jobConfig.transformQueue || jobConfig.queueName;
|
|
275
|
+
const queueData = this.queues[transformQueueName];
|
|
276
|
+
if (!queueData) {
|
|
277
|
+
throw new Error(`Queue ${transformQueueName} not found in config.`);
|
|
278
|
+
}
|
|
279
|
+
return await processEtlJob(
|
|
280
|
+
job,
|
|
281
|
+
jobConfig,
|
|
282
|
+
queueData.queue,
|
|
283
|
+
queueData.events
|
|
284
|
+
);
|
|
285
|
+
}
|
|
286
|
+
if (jobConfig.type === "etl" && isEtlTransformJob) {
|
|
287
|
+
return await jobConfig.transform(job.data);
|
|
288
|
+
}
|
|
289
|
+
throw new Error(`Unknown job type: ${jobConfig.type}`);
|
|
290
|
+
}
|
|
291
|
+
};
|
|
292
|
+
|
|
293
|
+
// src/index.ts
|
|
294
|
+
function createRavenflowApp(config, connection, loggerOptions) {
|
|
295
|
+
return new RavenflowApp(connection, config, loggerOptions);
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
exports.RavenflowApp = RavenflowApp;
|
|
299
|
+
exports.createLogger = createLogger;
|
|
300
|
+
exports.createRavenflowApp = createRavenflowApp;
|
|
301
|
+
//# sourceMappingURL=index.js.map
|
|
302
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/processor.ts","../src/logger.ts","../src/banner.ts","../src/app.ts","../src/index.ts"],"names":["pino","Queue","QueueEvents","Worker"],"mappings":";;;;;;;;;;;;AAGA,eAAsB,aAAA,CACpB,GAAA,EACA,SAAA,EACA,cAAA,EACA,oBAAA,EACA;AACA,EAAA,IAAI,WAAA,GAAc,CAAA;AAClB,EAAA,IAAI,GAAA,GAAM,IAAA;AACV,EAAA,MAAM,QAAA,GAAW;AAAA,IACf,KAAA,EAAO,CAAA;AAAA,IACP,KAAA,EAAO,CAAA;AAAA,IACP,WAAA;AAAA,IACA,SAAA,sBAAe,IAAA,EAAK;AAAA,IACpB,OAAA,EAAS;AAAA,GACX;AACA,EAAA,MAAM,UAAA,GAAa;AAAA,IAEjB,QAAA,EAAU,QAEZ,CAAA;AAEA,EAAA,OAAO,GAAA,IAAO,WAAA,KAAgB,UAAA,CAAW,QAAA,IAAY,QAAA,CAAA,EAAW;AAC9D,IAAA,QAAA,CAAS,WAAA,GAAc,WAAA;AACvB,IAAA,QAAA,CAAS,KAAA,GAAQ,WAAA;AAEjB,IAAA,MAAM;AAAA,MACJ,IAAA,EAAM,WAAA;AAAA,MACN,aAAa,cAAA,GAAiB;AAAA,KAChC,GAGI,MAAM,SAAA,CAAU,OAAA,CAAQ,WAAW,CAAA;AAEvC,IAAA,MAAM,oBAAkC,EAAC;AAEzC,IAAA,KAAA,MAAW,WAAW,WAAA,EAAa;AACjC,MAAA,MAAM,gBAAA,GAAmB,CAAA,EAAG,GAAA,CAAI,IAAI,CAAA,UAAA,CAAA;AACpC,MAAA,MAAM,YAAA,GAAe,MAAM,cAAA,CAAe,GAAA,CAAI,kBAAkB,OAAA,EAAS;AAAA,QACvE,UAAU,SAAA,CAAU,QAAA;AAAA,QACpB,SAAS,SAAA,CAAU;AAAA,OACpB,CAAA;AACD,MAAA,MAAM,gBAAA,GACJ,MAAM,YAAA,CAAa,iBAAA,CAAkB,oBAAoB,CAAA;AAE3D,MAAA,iBAAA,CAAkB,KAAK,gBAA8B,CAAA;AAAA,IACvD;AAEA,IAAA,MAAM,SAAA,CAAU,KAAK,iBAAiB,CAAA;AACtC,IAAA,QAAA,CAAS,SAAS,iBAAA,CAAkB,MAAA;AAEpC,IAAA,GAAA,GAAM,cAAA;AACN,IAAA,IAAI,GAAA,EAAK;AACP,MAAA,WAAA,EAAA;AAAA,IACF;AAAA,EACF;AACA,EAAA,OAAO,QAAA;AACT;ACpDO,SAAS,YAAA,CAAa,OAAA,GAAyB,EAAC,EAAG;AACxD,EAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,GAAA,CAAI,QAAA,KAAa,YAAA;AACvC,EAAA,MAAM,SAAA,GAAY,QAAQ,MAAA,IAAU,KAAA;AAEpC,EAAA,OAAOA,qBAAA,CAAK;AAAA,IACV,KAAA,EAAO,OAAA,CAAQ,KAAA,KAAU,KAAA,GAAQ,OAAA,GAAU,MAAA,CAAA;AAAA,IAC3C,WAAW,SAAA,GACP;AAAA,MACE,MAAA,EAAQ,aAAA;AAAA,MACR,OAAA,EAAS;AAAA,QACP,QAAA,EAAU,IAAA;AAAA,QACV,aAAA,EAAe,UAAA;AAAA,QACf,MAAA,EAAQ;AAAA;AACV,KACF,GACA;AAAA,GACL,CAAA;AACH;;;ACtBA,IAAM,eAAA,GAAkB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,CAAA;AAoBjB,SAAS,kBAAkB,MAAA,EAA4B;AAC5D,EAAA,MAAM,EAAE,SAAA,EAAW,WAAA,EAAa,KAAA,EAAO,MAAK,GAAI,MAAA;AAEhD,EAAA,OAAA,CAAQ,GAAA,CAAI,OAAO,eAAe,CAAA;AAClC,EAAA,OAAA,CAAQ,GAAA,CAAI,QAAA,CAAI,MAAA,CAAO,EAAE,CAAC,CAAA;AAE1B,EAAA,OAAA,CAAQ,IAAI,YAAY,CAAA;AACxB,EAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,kBAAA,EAAqB,SAAS,CAAA,CAAE,CAAA;AAC5C,EAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,kBAAA,EAAqB,WAAW,CAAA,CAAE,CAAA;AAC9C,EAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,kBAAA,EAAqB,KAAA,CAAM,IAAI,CAAA,CAAA,EAAI,MAAM,IAAI,CAAA,CAAA,EAAI,KAAA,CAAM,EAAE,CAAA,CAAE,CAAA;AAEvE,EAAA,OAAA,CAAQ,IAAI,qBAAqB,CAAA;AAEjC,EAAA,MAAM,cAAwB,EAAC;AAC/B,EAAA,MAAM,WAA6C,EAAC;AAEpD,EAAA,KAAA,MAAW,CAAC,OAAA,EAAS,SAAS,KAAK,MAAA,CAAO,OAAA,CAAQ,IAAI,CAAA,EAAG;AACvD,IAAA,IAAI,UAAU,IAAA,EAAM;AAClB,MAAA,QAAA,CAAS,KAAK,EAAE,IAAA,EAAM,SAAS,IAAA,EAAM,SAAA,CAAU,MAAM,CAAA;AAAA,IACvD,CAAA,MAAO;AACL,MAAA,WAAA,CAAY,KAAK,OAAO,CAAA;AAAA,IAC1B;AAAA,EACF;AAEA,EAAA,IAAI,WAAA,CAAY,SAAS,CAAA,EAAG;AAC1B,IAAA,WAAA,CAAY,OAAA,CAAQ,CAAC,OAAA,KAAY;AAC/B,MAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,IAAA,EAAO,OAAO,CAAA,CAAE,CAAA;AAAA,IAC9B,CAAC,CAAA;AAAA,EACH;AAEA,EAAA,IAAI,QAAA,CAAS,SAAS,CAAA,EAAG;AACvB,IAAA,QAAA,CAAS,OAAA,CAAQ,CAAC,EAAE,IAAA,EAAM,MAAK,KAAM;AACnC,MAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,IAAA,EAAO,IAAI,CAAA,SAAA,EAAO,IAAI,CAAA,CAAA,CAAG,CAAA;AAAA,IACvC,CAAC,CAAA;AAAA,EACH;AAEA,EAAA,OAAA,CAAQ,GAAA,CAAI,IAAA,GAAO,QAAA,CAAI,MAAA,CAAO,EAAE,CAAC,CAAA;AACjC,EAAA,OAAA,CAAQ,IAAI,4BAA4B,CAAA;AAC1C;;;AChDO,IAAM,eAAN,MAAsE;AAAA,EAc3E,WAAA,CACE,UAAA,EACA,MAAA,EACA,aAAA,EACA;AAhBF,IAAA,IAAA,CAAU,kBAAA,GAA6B,CAAA;AAiBrC,IAAA,IAAA,CAAK,UAAA,GAAa,UAAA;AAClB,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,MAAA,GAAS,aAAa,aAAa,CAAA;AAExC,IAAA,IAAI,CAAC,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,MAAA,CAAO,YAAY,CAAA,EAAG;AAChD,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,cAAA,EAAiB,OAAO,YAAY,CAAA,6BAAA;AAAA,OACtC;AAAA,IACF;AAEA,IAAA,MAAM,EAAE,SAAA,EAAW,UAAA,EAAW,GAAI,IAAA,CAAK,SAAS,MAAM,CAAA;AACtD,IAAA,IAAA,CAAK,IAAA,GAAO,SAAA;AACZ,IAAA,IAAA,CAAK,UAAA,GAAa,UAAA;AAElB,IAAA,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,UAAA,CAAW,MAAM,CAAA;AAAA,EACtC;AAAA,EAEQ,WACN,MAAA,EACoE;AACpE,IAAA,MAAM,SAAgE,EAAC;AACvE,IAAA,KAAA,MAAW,SAAA,IAAa,OAAO,MAAA,EAAQ;AACrC,MAAA,MAAA,CAAO,SAAS,CAAA,GAAI;AAAA,QAClB,KAAA,EAAO,IAAIC,YAAA,CAAM,SAAA,EAAW;AAAA,UAC1B,YAAY,IAAA,CAAK;AAAA,SAClB,CAAA;AAAA,QACD,MAAA,EAAQ,IAAIC,kBAAA,CAAY,SAAA,EAAW;AAAA,UACjC,YAAY,IAAA,CAAK;AAAA,SAClB;AAAA,OACH;AAAA,IACF;AACA,IAAA,OAAO,MAAA;AAAA,EACT;AAAA,EAEQ,SAAS,MAAA,EAGf;AACA,IAAA,MAAM,YAA4D,EAAC;AACnE,IAAA,MAAM,aAAwC,EAAC;AAE/C,IAAA,KAAA,MAAW,CAAC,SAAS,SAAS,CAAA,IAAK,OAAO,OAAA,CAAQ,MAAA,CAAO,IAAI,CAAA,EAAG;AAC9D,MAAA,UAAA,CAAW,OAAO,CAAA,GAAI,SAAA;AAEtB,MAAA,SAAA,CAAU,OAAO,CAAA,GAAI;AAAA,QACnB,IAAA,EAAM,OAAO,MAAA,KAAgB;AAC3B,UAAA,OAAO,IAAA,CAAK,OAAA,CAAQ,OAAA,EAA8B,MAAM,CAAA;AAAA,QAC1D;AAAA,OACF;AAAA,IACF;AACA,IAAA,OAAO;AAAA,MACL,SAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAAA,EAEA,MAAc,OAAA,CAAQ,OAAA,EAA4B,MAAA,EAAa;AAC7D,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,UAAA,CAAW,OAAO,CAAA;AACzC,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,IAAA,EAAO,OAAO,CAAA,UAAA,CAAY,CAAA;AAAA,IAC5C;AACA,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,MAAA,CAAO,SAAA,CAAU,SAAgC,CAAA;AACxE,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,MAAA,EAAS,SAAA,CAAU,SAAS,CAAA,UAAA,CAAY,CAAA;AAAA,IAC1D;AACA,IAAA,OAAO,MAAM,SAAA,CAAU,KAAA,CAAM,GAAA,CAAI,SAAS,MAAM,CAAA;AAAA,EAClD;AAAA,EAEO,WAAA,CACL,WACA,OAAA,EAGA;AACA,IAAA,MAAM,WAAA,GAAc,IAAA,CAAK,MAAA,CAAO,SAAS,CAAA;AACzC,IAAA,IAAI,CAAC,WAAA,EAAa;AAChB,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,MAAA,EAAS,SAAS,CAAA,qBAAA,CAAuB,CAAA;AAAA,IAC3D;AAEA,IAAA,MAAM,iBAAA,GAAoB,OAAA,EAAS,WAAA,IAAe,IAAA,CAAK,kBAAA;AAEvD,IAAA,IAAA,CAAK,iBAAA,CAAkB,WAAW,iBAAiB,CAAA;AAEnD,IAAA,IAAA,CAAK,kBAAkB,SAAS,CAAA;AAEhC,IAAA,MAAM,MAAA,GAAS,IAAIC,aAAA,CAAO,SAAA,EAAW,CAAC,GAAA,KAAQ,IAAA,CAAK,UAAA,CAAW,GAAG,CAAA,EAAG;AAAA,MAClE,YAAY,IAAA,CAAK,UAAA;AAAA,MACjB,OAAA,EAAS,KAAA;AAAA,MACT,WAAA,EAAa;AAAA,KACd,CAAA;AACD,IAAA,IAAA,CAAK,iBAAA,CAAkB,QAAQ,SAAS,CAAA;AACxC,IAAA,MAAA,CAAO,GAAA,EAAI;AAAA,EACb;AAAA,EAEQ,iBAAA,CACN,WACA,WAAA,EACM;AACN,IAAA,MAAM,SAAA,GAAY;AAAA,MAChB,IAAA,EACE,OAAO,IAAA,CAAK,UAAA,KAAe,QAAA,IAAY,UAAU,IAAA,CAAK,UAAA,GACjD,IAAA,CAAK,UAAA,CAAW,IAAA,GACjB,WAAA;AAAA,MACN,IAAA,EACE,OAAO,IAAA,CAAK,UAAA,KAAe,QAAA,IAAY,UAAU,IAAA,CAAK,UAAA,GACjD,IAAA,CAAK,UAAA,CAAW,IAAA,GACjB,IAAA;AAAA,MACN,EAAA,EACE,OAAO,IAAA,CAAK,UAAA,KAAe,QAAA,IAAY,QAAQ,IAAA,CAAK,UAAA,GAC/C,IAAA,CAAK,UAAA,CAAW,EAAA,GACjB;AAAA,KACR;AAEA,IAAA,iBAAA,CAAkB;AAAA,MAChB,SAAA;AAAA,MACA,WAAA;AAAA,MACA,KAAA,EAAO,SAAA;AAAA,MACP,MAAM,IAAA,CAAK;AAAA,KACZ,CAAA;AAAA,EACH;AAAA,EAEQ,iBAAA,CAAkB,QAAgB,SAAA,EAAyB;AACjE,IAAA,MAAA,CAAO,EAAA,CAAG,QAAA,EAAU,CAAC,GAAA,KAAQ;AAC3B,MAAA,IAAA,CAAK,OAAO,IAAA,CAAK,CAAA,CAAA,EAAI,SAAS,CAAA,EAAA,EAAK,GAAA,CAAI,IAAI,CAAA,SAAA,CAAW,CAAA;AAAA,IACxD,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,EAAA,CAAG,WAAA,EAAa,CAAC,GAAA,KAAQ;AAC9B,MAAA,IAAA,CAAK,OAAO,IAAA,CAAK,CAAA,CAAA,EAAI,SAAS,CAAA,EAAA,EAAK,GAAA,CAAI,IAAI,CAAA,YAAA,CAAc,CAAA;AAAA,IAC3D,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,EAAA,CAAG,QAAA,EAAU,CAAC,GAAA,EAAK,KAAA,KAAU;AAClC,MAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,QACV,IAAI,SAAS,CAAA,EAAA,EAAK,KAAK,IAAI,CAAA,WAAA,EAAc,MAAM,OAAO,CAAA;AAAA,OACxD;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA,EAEA,MAAc,kBAAkB,SAAA,EAAgC;AAC9D,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,MAAA,CAAO,SAAS,CAAA;AACvC,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,MAAA,EAAS,SAAS,CAAA,qBAAA,CAAuB,CAAA;AAAA,IAC3D;AACA,IAAA,MAAM,QAAQ,SAAA,CAAU,KAAA;AAExB,IAAA,OAAA,CAAQ,IAAI,kBAAkB,CAAA;AAE9B,IAAA,MAAM,qBAAgD,EAAC;AACvD,IAAA,KAAA,MAAW,OAAA,IAAW,KAAK,UAAA,EAAY;AACrC,MAAA,MAAM,SAAA,GAAY,IAAA,CAAK,UAAA,CAAW,OAA4B,CAAA;AAC9D,MAAA,IAAI,UAAU,IAAA,EAAM;AAClB,QAAA,kBAAA,CAAmB,OAAO,CAAA,GAAI,SAAA;AAAA,MAChC;AAAA,IACF;AAEA,IAAA,MAAM,uBAAA,GAA0B,MAAM,KAAA,CAAM,gBAAA,EAAiB;AAC7D,IAAA,MAAM,wBAAwB,IAAI,GAAA;AAAA,MAChC,uBAAA,CAAwB,GAAA,CAAI,CAAC,CAAA,KAAM,EAAE,GAAG;AAAA,KAC1C;AAEA,IAAA,MAAM,UAAoB,EAAC;AAC3B,IAAA,KAAA,MAAW,gBAAgB,uBAAA,EAAyB;AAClD,MAAA,IAAI,CAAC,kBAAA,CAAmB,YAAA,CAAa,GAAG,CAAA,EAAG;AACzC,QAAA,OAAA,CAAQ,IAAA,CAAK,aAAa,GAAG,CAAA;AAC7B,QAAA,MAAM,KAAA,CAAM,kBAAA,CAAmB,YAAA,CAAa,GAAG,CAAA;AAAA,MACjD;AAAA,IACF;AAEA,IAAA,MAAM,UAAoB,EAAC;AAC3B,IAAA,MAAM,UAAoB,EAAC;AAE3B,IAAA,KAAA,MAAW,WAAW,kBAAA,EAAoB;AACxC,MAAA,MAAM,SAAA,GAAY,mBAAmB,OAAO,CAAA;AAC5C,MAAA,IAAI,CAAC,WAAW,IAAA,EAAM;AAEtB,MAAA,IAAI,qBAAA,CAAsB,GAAA,CAAI,OAAO,CAAA,EAAG;AACtC,QAAA,OAAA,CAAQ,KAAK,OAAO,CAAA;AAAA,MACtB,CAAA,MAAO;AACL,QAAA,OAAA,CAAQ,KAAK,OAAO,CAAA;AAAA,MACtB;AAEA,MAAA,MAAM,MAAM,kBAAA,CAAmB,OAAA,EAAS,EAAE,OAAA,EAAS,SAAA,CAAU,MAAM,CAAA;AAAA,IACrE;AAEA,IAAA,IAAI,OAAA,CAAQ,SAAS,CAAA,EAAG;AACtB,MAAA,OAAA,CAAQ,OAAA;AAAA,QAAQ,CAAC,IAAA,KACf,OAAA,CAAQ,GAAA,CAAI,CAAA,IAAA,EAAO,IAAI,CAAA,QAAA,EAAM,kBAAA,CAAmB,IAAI,CAAA,CAAE,IAAI,CAAA,CAAE;AAAA,OAC9D;AAAA,IACF;AACA,IAAA,IAAI,OAAA,CAAQ,SAAS,CAAA,EAAG;AACtB,MAAA,OAAA,CAAQ,OAAA;AAAA,QAAQ,CAAC,IAAA,KACf,OAAA,CAAQ,GAAA,CAAI,CAAA,SAAA,EAAO,IAAI,CAAA,QAAA,EAAM,kBAAA,CAAmB,IAAI,CAAA,CAAE,IAAI,CAAA,CAAE;AAAA,OAC9D;AAAA,IACF;AACA,IAAA,IAAI,OAAA,CAAQ,SAAS,CAAA,EAAG;AACtB,MAAA,OAAA,CAAQ,OAAA,CAAQ,CAAC,IAAA,KAAS,OAAA,CAAQ,IAAI,CAAA,IAAA,EAAO,IAAI,EAAE,CAAC,CAAA;AAAA,IACtD;AACA,IAAA,IAAI,OAAA,CAAQ,WAAW,CAAA,IAAK,OAAA,CAAQ,WAAW,CAAA,IAAK,OAAA,CAAQ,WAAW,CAAA,EAAG;AACxE,MAAA,OAAA,CAAQ,IAAI,CAAA,qBAAA,CAAuB,CAAA;AAAA,IACrC;AAEA,IAAA,OAAA,CAAQ,IAAI,EAAE,CAAA;AAAA,EAChB;AAAA,EAEA,MAAM,WAAW,GAAA,EAAU;AACzB,IAAA,MAAM,iBAAA,GAAoB,GAAA,CAAI,IAAA,CAAK,QAAA,CAAS,YAAY,CAAA;AACxD,IAAA,MAAM,OAAA,GAAU,iBAAA,GACZ,GAAA,CAAI,IAAA,CAAK,KAAA,CAAM,GAAG,CAAC,YAAA,CAAa,MAAM,CAAA,GACtC,GAAA,CAAI,IAAA;AAER,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,UAAA,CAAW,OAA4B,CAAA;AAC9D,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,uBAAA,EAA0B,OAAO,CAAA,CAAE,CAAA;AAAA,IACrD;AAEA,IAAA,IAAI,SAAA,CAAU,SAAS,KAAA,EAAO;AAC5B,MAAA,OAAO,MAAM,SAAA,CAAU,OAAA,CAAQ,GAAA,EAAK,IAAI,IAAI,CAAA;AAAA,IAC9C;AAEA,IAAA,IAAI,SAAA,CAAU,IAAA,KAAS,KAAA,IAAS,CAAC,iBAAA,EAAmB;AAClD,MAAA,MAAM,kBAAA,GACJ,SAAA,CAAU,cAAA,IAAkB,SAAA,CAAU,SAAA;AACxC,MAAA,MAAM,SAAA,GAAY,IAAA,CAAK,MAAA,CAAO,kBAAyC,CAAA;AAEvE,MAAA,IAAI,CAAC,SAAA,EAAW;AACd,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,MAAA,EAAS,kBAAkB,CAAA,qBAAA,CAAuB,CAAA;AAAA,MACpE;AACA,MAAA,OAAO,MAAM,aAAA;AAAA,QACX,GAAA;AAAA,QACA,SAAA;AAAA,QACA,SAAA,CAAU,KAAA;AAAA,QACV,SAAA,CAAU;AAAA,OACZ;AAAA,IACF;AAEA,IAAA,IAAI,SAAA,CAAU,IAAA,KAAS,KAAA,IAAS,iBAAA,EAAmB;AACjD,MAAA,OAAO,MAAM,SAAA,CAAU,SAAA,CAAU,GAAA,CAAI,IAAI,CAAA;AAAA,IAC3C;AAEA,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,kBAAA,EAAqB,SAAA,CAAU,IAAI,CAAA,CAAE,CAAA;AAAA,EACvD;AACF;;;AC/PO,SAAS,kBAAA,CACd,MAAA,EACA,UAAA,EACA,aAAA,EAIuB;AACvB,EAAA,OAAO,IAAI,YAAA,CAAa,UAAA,EAAY,MAAA,EAAQ,aAAa,CAAA;AAC3D","file":"index.js","sourcesContent":["import { Job, Queue, QueueEvents } from \"bullmq\";\nimport { EtlJobConfig } from \"./types\";\n\nexport async function processEtlJob<TRaw, TProcessed>(\n job: Job,\n etlConfig: EtlJobConfig<TRaw, TProcessed>,\n transformQueue: Queue,\n transformQueueEvents: QueueEvents\n) {\n let currentPage = 1;\n let run = true;\n const metadata = {\n count: 0,\n pages: 0,\n currentPage,\n startedAt: new Date(),\n endedAt: undefined,\n };\n const pagination = {\n pageSize: 100,\n maxPages: Infinity,\n pageParam: \"page\",\n };\n\n while (run && currentPage <= (pagination.maxPages || Infinity)) {\n metadata.currentPage = currentPage;\n metadata.pages = currentPage;\n\n const {\n data: rawElements,\n hasNextPage: nextPageExists = false,\n }: {\n data: TRaw[];\n hasNextPage?: boolean;\n } = await etlConfig.extract(currentPage);\n\n const processedElements: TProcessed[] = [];\n\n for (const element of rawElements) {\n const transformJobName = `${job.name}:transform`;\n const transformJob = await transformQueue.add(transformJobName, element, {\n attempts: etlConfig.attempts,\n backoff: etlConfig.backoff,\n });\n const processedElement =\n await transformJob.waitUntilFinished(transformQueueEvents);\n\n processedElements.push(processedElement as TProcessed);\n }\n\n await etlConfig.load(processedElements);\n metadata.count += processedElements.length;\n\n run = nextPageExists;\n if (run) {\n currentPage++;\n }\n }\n return metadata;\n}\n","import pino from \"pino\";\n\nexport interface LoggerOptions {\n level?: \"trace\" | \"debug\" | \"info\" | \"warn\" | \"error\" | \"fatal\";\n pretty?: boolean;\n}\n\nexport function createLogger(options: LoggerOptions = {}) {\n const isDev = process.env.NODE_ENV !== \"production\";\n const usePretty = options.pretty ?? isDev;\n\n return pino({\n level: options.level || (isDev ? \"debug\" : \"info\"),\n transport: usePretty\n ? {\n target: \"pino-pretty\",\n options: {\n colorize: true,\n translateTime: \"HH:MM:ss\",\n ignore: \"pid,hostname\",\n },\n }\n : undefined,\n });\n}\n\nexport type Logger = ReturnType<typeof createLogger>;\n","import { JobConfig } from \"./types\";\n\nconst RAVENFLOW_ASCII = `\n ██████╗ █████╗ ██╗ ██╗███████╗███╗ ██╗███████╗██╗ ██████╗ ██╗ ██╗\n ██╔══██╗██╔══██╗██║ ██║██╔════╝████╗ ██║██╔════╝██║ ██╔═══██╗██║ ██║\n ██████╔╝███████║██║ ██║█████╗ ██╔██╗ ██║█████╗ ██║ ██║ ██║██║ █╗ ██║\n ██╔══██╗██╔══██║╚██╗ ██╔╝██╔══╝ ██║╚██╗██║██╔══╝ ██║ ██║ ██║██║███╗██║\n ██║ ██║██║ ██║ ╚████╔╝ ███████╗██║ ╚████║██║ ███████╗╚██████╔╝╚███╔███╔╝\n ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═══╝ ╚══════╝╚═╝ ╚═══╝╚═╝ ╚══════╝ ╚═════╝ ╚══╝╚══╝ \n`;\n\ninterface BannerConfig {\n queueName: string;\n concurrency: number;\n redis: {\n host: string;\n port: number;\n db: number;\n };\n jobs: Record<string, JobConfig>;\n}\n\nexport function printWorkerBanner(config: BannerConfig): void {\n const { queueName, concurrency, redis, jobs } = config;\n\n console.log(\"\\n\" + RAVENFLOW_ASCII);\n console.log(\"═\".repeat(85));\n\n console.log(\"\\n[Config]\");\n console.log(` -> queue: ${queueName}`);\n console.log(` -> concurrency: ${concurrency}`);\n console.log(` -> redis: ${redis.host}:${redis.port}/${redis.db}`);\n\n console.log(\"\\n[Registered Jobs]\");\n\n const regularJobs: string[] = [];\n const cronJobs: { name: string; cron: string }[] = [];\n\n for (const [jobName, jobConfig] of Object.entries(jobs)) {\n if (jobConfig.cron) {\n cronJobs.push({ name: jobName, cron: jobConfig.cron });\n } else {\n regularJobs.push(jobName);\n }\n }\n\n if (regularJobs.length > 0) {\n regularJobs.forEach((jobName) => {\n console.log(` . ${jobName}`);\n });\n }\n\n if (cronJobs.length > 0) {\n cronJobs.forEach(({ name, cron }) => {\n console.log(` . ${name} (↻ ${cron})`);\n });\n }\n\n console.log(\"\\n\" + \"═\".repeat(85));\n console.log(\"Ready to process jobs...\\n\");\n}\n","import { ConnectionOptions, Queue, Worker, Job, QueueEvents } from \"bullmq\";\nimport {\n RavenflowConfig,\n QueueNames,\n JobConfig,\n JobNames,\n JobsProxy,\n} from \"./types\";\nimport { processEtlJob } from \"./processor\";\nimport { createLogger, Logger, LoggerOptions } from \"./logger\";\nimport { printWorkerBanner } from \"./banner\";\n\nexport class RavenflowApp<TConfig extends RavenflowConfig = RavenflowConfig> {\n protected connection: ConnectionOptions;\n protected defaultConcurrency: number = 5;\n protected logger: Logger;\n\n protected config: TConfig;\n\n public jobs: JobsProxy<TConfig[\"jobs\"]>;\n protected jobConfigs: Record<JobNames<TConfig>, JobConfig>;\n protected queues: Record<\n QueueNames<TConfig>,\n { queue: Queue; events: QueueEvents }\n >;\n\n constructor(\n connection: ConnectionOptions,\n config: TConfig,\n loggerOptions?: LoggerOptions\n ) {\n this.connection = connection;\n this.config = config;\n this.logger = createLogger(loggerOptions);\n\n if (!config.queues.includes(config.defaultQueue)) {\n throw new Error(\n `Default queue ${config.defaultQueue} is not in the list of queues`\n );\n }\n\n const { jobsProxy, jobConfigs } = this.initJobs(config);\n this.jobs = jobsProxy;\n this.jobConfigs = jobConfigs;\n\n this.queues = this.initQueues(config);\n }\n\n private initQueues(\n config: TConfig\n ): Record<QueueNames<TConfig>, { queue: Queue; events: QueueEvents }> {\n const queues: Record<string, { queue: Queue; events: QueueEvents }> = {};\n for (const queueName of config.queues) {\n queues[queueName] = {\n queue: new Queue(queueName, {\n connection: this.connection,\n }),\n events: new QueueEvents(queueName, {\n connection: this.connection,\n }),\n };\n }\n return queues;\n }\n\n private initJobs(config: TConfig): {\n jobsProxy: JobsProxy<TConfig[\"jobs\"]>;\n jobConfigs: Record<JobNames<TConfig>, JobConfig>;\n } {\n const jobsProxy: Record<string, { send: (params: any) => any }> = {};\n const jobConfigs: Record<string, JobConfig> = {};\n\n for (const [jobName, jobConfig] of Object.entries(config.jobs)) {\n jobConfigs[jobName] = jobConfig;\n\n jobsProxy[jobName] = {\n send: async (params: any) => {\n return this.sendJob(jobName as JobNames<TConfig>, params);\n },\n };\n }\n return {\n jobsProxy: jobsProxy as JobsProxy<TConfig[\"jobs\"]>,\n jobConfigs,\n };\n }\n\n private async sendJob(jobName: JobNames<TConfig>, params: any) {\n const jobConfig = this.jobConfigs[jobName];\n if (!jobConfig) {\n throw new Error(`Job ${jobName} not found`);\n }\n const queueData = this.queues[jobConfig.queueName as QueueNames<TConfig>];\n if (!queueData) {\n throw new Error(`Queue ${jobConfig.queueName} not found`);\n }\n return await queueData.queue.add(jobName, params);\n }\n\n public startWorker(\n queueName: QueueNames<TConfig>,\n options?: {\n concurrency?: number;\n }\n ) {\n const queueConfig = this.queues[queueName];\n if (!queueConfig) {\n throw new Error(`Queue ${queueName} not found in config.`);\n }\n\n const workerConcurrency = options?.concurrency || this.defaultConcurrency;\n\n this.printWorkerBanner(queueName, workerConcurrency);\n\n this.syncJobSchedulers(queueName);\n\n const worker = new Worker(queueName, (job) => this.processJob(job), {\n connection: this.connection,\n autorun: false,\n concurrency: workerConcurrency,\n });\n this.setupWorkerEvents(worker, queueName);\n worker.run();\n }\n\n private printWorkerBanner(\n queueName: QueueNames<TConfig>,\n concurrency: number\n ): void {\n const redisInfo = {\n host:\n typeof this.connection === \"object\" && \"host\" in this.connection\n ? (this.connection.host as string)\n : \"localhost\",\n port:\n typeof this.connection === \"object\" && \"port\" in this.connection\n ? (this.connection.port as number)\n : 6379,\n db:\n typeof this.connection === \"object\" && \"db\" in this.connection\n ? (this.connection.db as number)\n : 0,\n };\n\n printWorkerBanner({\n queueName: queueName as string,\n concurrency,\n redis: redisInfo,\n jobs: this.jobConfigs,\n });\n }\n\n private setupWorkerEvents(worker: Worker, queueName: string): void {\n worker.on(\"active\", (job) => {\n this.logger.info(`[${queueName}] ${job.name} - active`);\n });\n\n worker.on(\"completed\", (job) => {\n this.logger.info(`[${queueName}] ${job.name} - completed`);\n });\n\n worker.on(\"failed\", (job, error) => {\n this.logger.error(\n `[${queueName}] ${job?.name} - failed: ${error.message}`\n );\n });\n }\n\n private async syncJobSchedulers(queueName: QueueNames<TConfig>) {\n const queueData = this.queues[queueName];\n if (!queueData) {\n throw new Error(`Queue ${queueName} not found in config.`);\n }\n const queue = queueData.queue;\n\n console.log(\"[Scheduled Jobs]\");\n\n const jobConfigsWithCron: Record<string, JobConfig> = {};\n for (const jobName in this.jobConfigs) {\n const jobConfig = this.jobConfigs[jobName as JobNames<TConfig>];\n if (jobConfig.cron) {\n jobConfigsWithCron[jobName] = jobConfig;\n }\n }\n\n const registeredJobSchedulers = await queue.getJobSchedulers();\n const existingSchedulerKeys = new Set(\n registeredJobSchedulers.map((s) => s.key)\n );\n\n const removed: string[] = [];\n for (const jobScheduler of registeredJobSchedulers) {\n if (!jobConfigsWithCron[jobScheduler.key]) {\n removed.push(jobScheduler.key);\n await queue.removeJobScheduler(jobScheduler.key);\n }\n }\n\n const newJobs: string[] = [];\n const updated: string[] = [];\n\n for (const jobName in jobConfigsWithCron) {\n const jobConfig = jobConfigsWithCron[jobName];\n if (!jobConfig?.cron) continue;\n\n if (existingSchedulerKeys.has(jobName)) {\n updated.push(jobName);\n } else {\n newJobs.push(jobName);\n }\n\n await queue.upsertJobScheduler(jobName, { pattern: jobConfig.cron });\n }\n\n if (newJobs.length > 0) {\n newJobs.forEach((name) =>\n console.log(` + ${name} → ${jobConfigsWithCron[name].cron}`)\n );\n }\n if (updated.length > 0) {\n updated.forEach((name) =>\n console.log(` ↻ ${name} → ${jobConfigsWithCron[name].cron}`)\n );\n }\n if (removed.length > 0) {\n removed.forEach((name) => console.log(` - ${name}`));\n }\n if (newJobs.length === 0 && updated.length === 0 && removed.length === 0) {\n console.log(` (no scheduled jobs)`);\n }\n\n console.log(\"\");\n }\n\n async processJob(job: Job) {\n const isEtlTransformJob = job.name.endsWith(\":transform\");\n const jobName = isEtlTransformJob\n ? job.name.slice(0, -\":transform\".length)\n : job.name;\n\n const jobConfig = this.jobConfigs[jobName as JobNames<TConfig>];\n if (!jobConfig) {\n throw new Error(`Job handler not found: ${jobName}`);\n }\n\n if (jobConfig.type === \"job\") {\n return await jobConfig.handler(job, job.data);\n }\n\n if (jobConfig.type === \"etl\" && !isEtlTransformJob) {\n const transformQueueName =\n jobConfig.transformQueue || jobConfig.queueName;\n const queueData = this.queues[transformQueueName as QueueNames<TConfig>];\n\n if (!queueData) {\n throw new Error(`Queue ${transformQueueName} not found in config.`);\n }\n return await processEtlJob(\n job,\n jobConfig,\n queueData.queue,\n queueData.events\n );\n }\n\n if (jobConfig.type === \"etl\" && isEtlTransformJob) {\n return await jobConfig.transform(job.data);\n }\n\n throw new Error(`Unknown job type: ${jobConfig.type}`);\n }\n}\n","import { RavenflowApp } from \"./app\";\nimport { RavenflowConfig } from \"./types\";\nimport { ConnectionOptions } from \"bullmq\";\n\nexport { RavenflowApp } from \"./app\";\nexport { createLogger } from \"./logger\";\n\nexport type {\n JobConfig,\n RavenflowConfig,\n SimpleJobConfig,\n EtlJobConfig,\n} from \"./types\";\n\nexport type { Logger, LoggerOptions } from \"./logger\";\n\nexport function createRavenflowApp<TConfig extends RavenflowConfig>(\n config: TConfig,\n connection: ConnectionOptions,\n loggerOptions?: {\n level?: \"trace\" | \"debug\" | \"info\" | \"warn\" | \"error\" | \"fatal\";\n pretty?: boolean;\n }\n): RavenflowApp<TConfig> {\n return new RavenflowApp(connection, config, loggerOptions);\n}\n"]}
|
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
import { QueueEvents, Queue, Worker } from 'bullmq';
|
|
2
|
+
import pino from 'pino';
|
|
3
|
+
|
|
4
|
+
// src/app.ts
|
|
5
|
+
|
|
6
|
+
// src/processor.ts
|
|
7
|
+
async function processEtlJob(job, etlConfig, transformQueue, transformQueueEvents) {
|
|
8
|
+
let currentPage = 1;
|
|
9
|
+
let run = true;
|
|
10
|
+
const metadata = {
|
|
11
|
+
count: 0,
|
|
12
|
+
pages: 0,
|
|
13
|
+
currentPage,
|
|
14
|
+
startedAt: /* @__PURE__ */ new Date(),
|
|
15
|
+
endedAt: void 0
|
|
16
|
+
};
|
|
17
|
+
const pagination = {
|
|
18
|
+
maxPages: Infinity};
|
|
19
|
+
while (run && currentPage <= (pagination.maxPages || Infinity)) {
|
|
20
|
+
metadata.currentPage = currentPage;
|
|
21
|
+
metadata.pages = currentPage;
|
|
22
|
+
const {
|
|
23
|
+
data: rawElements,
|
|
24
|
+
hasNextPage: nextPageExists = false
|
|
25
|
+
} = await etlConfig.extract(currentPage);
|
|
26
|
+
const processedElements = [];
|
|
27
|
+
for (const element of rawElements) {
|
|
28
|
+
const transformJobName = `${job.name}:transform`;
|
|
29
|
+
const transformJob = await transformQueue.add(transformJobName, element, {
|
|
30
|
+
attempts: etlConfig.attempts,
|
|
31
|
+
backoff: etlConfig.backoff
|
|
32
|
+
});
|
|
33
|
+
const processedElement = await transformJob.waitUntilFinished(transformQueueEvents);
|
|
34
|
+
processedElements.push(processedElement);
|
|
35
|
+
}
|
|
36
|
+
await etlConfig.load(processedElements);
|
|
37
|
+
metadata.count += processedElements.length;
|
|
38
|
+
run = nextPageExists;
|
|
39
|
+
if (run) {
|
|
40
|
+
currentPage++;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return metadata;
|
|
44
|
+
}
|
|
45
|
+
function createLogger(options = {}) {
|
|
46
|
+
const isDev = process.env.NODE_ENV !== "production";
|
|
47
|
+
const usePretty = options.pretty ?? isDev;
|
|
48
|
+
return pino({
|
|
49
|
+
level: options.level || (isDev ? "debug" : "info"),
|
|
50
|
+
transport: usePretty ? {
|
|
51
|
+
target: "pino-pretty",
|
|
52
|
+
options: {
|
|
53
|
+
colorize: true,
|
|
54
|
+
translateTime: "HH:MM:ss",
|
|
55
|
+
ignore: "pid,hostname"
|
|
56
|
+
}
|
|
57
|
+
} : void 0
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// src/banner.ts
|
|
62
|
+
var RAVENFLOW_ASCII = `
|
|
63
|
+
\u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2588\u2557 \u2588\u2588\u2557\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557 \u2588\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2557 \u2588\u2588\u2557
|
|
64
|
+
\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u2588\u2588\u2551 \u2588\u2588\u2554\u2550\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2551 \u2588\u2588\u2551
|
|
65
|
+
\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551\u2588\u2588\u2588\u2588\u2588\u2557 \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2557 \u2588\u2588\u2551
|
|
66
|
+
\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2557\u2588\u2588\u2554\u2550\u2550\u2588\u2588\u2551\u255A\u2588\u2588\u2557 \u2588\u2588\u2554\u255D\u2588\u2588\u2554\u2550\u2550\u255D \u2588\u2588\u2551\u255A\u2588\u2588\u2557\u2588\u2588\u2551\u2588\u2588\u2554\u2550\u2550\u255D \u2588\u2588\u2551 \u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551\u2588\u2588\u2588\u2557\u2588\u2588\u2551
|
|
67
|
+
\u2588\u2588\u2551 \u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2551 \u255A\u2588\u2588\u2588\u2588\u2554\u255D \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2551 \u255A\u2588\u2588\u2588\u2588\u2551\u2588\u2588\u2551 \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u255A\u2588\u2588\u2588\u2554\u2588\u2588\u2588\u2554\u255D
|
|
68
|
+
\u255A\u2550\u255D \u255A\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u255D \u255A\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u2550\u2550\u255D\u255A\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u255D\u255A\u2550\u2550\u255D
|
|
69
|
+
`;
|
|
70
|
+
function printWorkerBanner(config) {
|
|
71
|
+
const { queueName, concurrency, redis, jobs } = config;
|
|
72
|
+
console.log("\n" + RAVENFLOW_ASCII);
|
|
73
|
+
console.log("\u2550".repeat(85));
|
|
74
|
+
console.log("\n[Config]");
|
|
75
|
+
console.log(` -> queue: ${queueName}`);
|
|
76
|
+
console.log(` -> concurrency: ${concurrency}`);
|
|
77
|
+
console.log(` -> redis: ${redis.host}:${redis.port}/${redis.db}`);
|
|
78
|
+
console.log("\n[Registered Jobs]");
|
|
79
|
+
const regularJobs = [];
|
|
80
|
+
const cronJobs = [];
|
|
81
|
+
for (const [jobName, jobConfig] of Object.entries(jobs)) {
|
|
82
|
+
if (jobConfig.cron) {
|
|
83
|
+
cronJobs.push({ name: jobName, cron: jobConfig.cron });
|
|
84
|
+
} else {
|
|
85
|
+
regularJobs.push(jobName);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
if (regularJobs.length > 0) {
|
|
89
|
+
regularJobs.forEach((jobName) => {
|
|
90
|
+
console.log(` . ${jobName}`);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
if (cronJobs.length > 0) {
|
|
94
|
+
cronJobs.forEach(({ name, cron }) => {
|
|
95
|
+
console.log(` . ${name} (\u21BB ${cron})`);
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
console.log("\n" + "\u2550".repeat(85));
|
|
99
|
+
console.log("Ready to process jobs...\n");
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// src/app.ts
|
|
103
|
+
var RavenflowApp = class {
|
|
104
|
+
constructor(connection, config, loggerOptions) {
|
|
105
|
+
this.defaultConcurrency = 5;
|
|
106
|
+
this.connection = connection;
|
|
107
|
+
this.config = config;
|
|
108
|
+
this.logger = createLogger(loggerOptions);
|
|
109
|
+
if (!config.queues.includes(config.defaultQueue)) {
|
|
110
|
+
throw new Error(
|
|
111
|
+
`Default queue ${config.defaultQueue} is not in the list of queues`
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
const { jobsProxy, jobConfigs } = this.initJobs(config);
|
|
115
|
+
this.jobs = jobsProxy;
|
|
116
|
+
this.jobConfigs = jobConfigs;
|
|
117
|
+
this.queues = this.initQueues(config);
|
|
118
|
+
}
|
|
119
|
+
initQueues(config) {
|
|
120
|
+
const queues = {};
|
|
121
|
+
for (const queueName of config.queues) {
|
|
122
|
+
queues[queueName] = {
|
|
123
|
+
queue: new Queue(queueName, {
|
|
124
|
+
connection: this.connection
|
|
125
|
+
}),
|
|
126
|
+
events: new QueueEvents(queueName, {
|
|
127
|
+
connection: this.connection
|
|
128
|
+
})
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
return queues;
|
|
132
|
+
}
|
|
133
|
+
initJobs(config) {
|
|
134
|
+
const jobsProxy = {};
|
|
135
|
+
const jobConfigs = {};
|
|
136
|
+
for (const [jobName, jobConfig] of Object.entries(config.jobs)) {
|
|
137
|
+
jobConfigs[jobName] = jobConfig;
|
|
138
|
+
jobsProxy[jobName] = {
|
|
139
|
+
send: async (params) => {
|
|
140
|
+
return this.sendJob(jobName, params);
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
return {
|
|
145
|
+
jobsProxy,
|
|
146
|
+
jobConfigs
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
async sendJob(jobName, params) {
|
|
150
|
+
const jobConfig = this.jobConfigs[jobName];
|
|
151
|
+
if (!jobConfig) {
|
|
152
|
+
throw new Error(`Job ${jobName} not found`);
|
|
153
|
+
}
|
|
154
|
+
const queueData = this.queues[jobConfig.queueName];
|
|
155
|
+
if (!queueData) {
|
|
156
|
+
throw new Error(`Queue ${jobConfig.queueName} not found`);
|
|
157
|
+
}
|
|
158
|
+
return await queueData.queue.add(jobName, params);
|
|
159
|
+
}
|
|
160
|
+
startWorker(queueName, options) {
|
|
161
|
+
const queueConfig = this.queues[queueName];
|
|
162
|
+
if (!queueConfig) {
|
|
163
|
+
throw new Error(`Queue ${queueName} not found in config.`);
|
|
164
|
+
}
|
|
165
|
+
const workerConcurrency = options?.concurrency || this.defaultConcurrency;
|
|
166
|
+
this.printWorkerBanner(queueName, workerConcurrency);
|
|
167
|
+
this.syncJobSchedulers(queueName);
|
|
168
|
+
const worker = new Worker(queueName, (job) => this.processJob(job), {
|
|
169
|
+
connection: this.connection,
|
|
170
|
+
autorun: false,
|
|
171
|
+
concurrency: workerConcurrency
|
|
172
|
+
});
|
|
173
|
+
this.setupWorkerEvents(worker, queueName);
|
|
174
|
+
worker.run();
|
|
175
|
+
}
|
|
176
|
+
printWorkerBanner(queueName, concurrency) {
|
|
177
|
+
const redisInfo = {
|
|
178
|
+
host: typeof this.connection === "object" && "host" in this.connection ? this.connection.host : "localhost",
|
|
179
|
+
port: typeof this.connection === "object" && "port" in this.connection ? this.connection.port : 6379,
|
|
180
|
+
db: typeof this.connection === "object" && "db" in this.connection ? this.connection.db : 0
|
|
181
|
+
};
|
|
182
|
+
printWorkerBanner({
|
|
183
|
+
queueName,
|
|
184
|
+
concurrency,
|
|
185
|
+
redis: redisInfo,
|
|
186
|
+
jobs: this.jobConfigs
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
setupWorkerEvents(worker, queueName) {
|
|
190
|
+
worker.on("active", (job) => {
|
|
191
|
+
this.logger.info(`[${queueName}] ${job.name} - active`);
|
|
192
|
+
});
|
|
193
|
+
worker.on("completed", (job) => {
|
|
194
|
+
this.logger.info(`[${queueName}] ${job.name} - completed`);
|
|
195
|
+
});
|
|
196
|
+
worker.on("failed", (job, error) => {
|
|
197
|
+
this.logger.error(
|
|
198
|
+
`[${queueName}] ${job?.name} - failed: ${error.message}`
|
|
199
|
+
);
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
async syncJobSchedulers(queueName) {
|
|
203
|
+
const queueData = this.queues[queueName];
|
|
204
|
+
if (!queueData) {
|
|
205
|
+
throw new Error(`Queue ${queueName} not found in config.`);
|
|
206
|
+
}
|
|
207
|
+
const queue = queueData.queue;
|
|
208
|
+
console.log("[Scheduled Jobs]");
|
|
209
|
+
const jobConfigsWithCron = {};
|
|
210
|
+
for (const jobName in this.jobConfigs) {
|
|
211
|
+
const jobConfig = this.jobConfigs[jobName];
|
|
212
|
+
if (jobConfig.cron) {
|
|
213
|
+
jobConfigsWithCron[jobName] = jobConfig;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
const registeredJobSchedulers = await queue.getJobSchedulers();
|
|
217
|
+
const existingSchedulerKeys = new Set(
|
|
218
|
+
registeredJobSchedulers.map((s) => s.key)
|
|
219
|
+
);
|
|
220
|
+
const removed = [];
|
|
221
|
+
for (const jobScheduler of registeredJobSchedulers) {
|
|
222
|
+
if (!jobConfigsWithCron[jobScheduler.key]) {
|
|
223
|
+
removed.push(jobScheduler.key);
|
|
224
|
+
await queue.removeJobScheduler(jobScheduler.key);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
const newJobs = [];
|
|
228
|
+
const updated = [];
|
|
229
|
+
for (const jobName in jobConfigsWithCron) {
|
|
230
|
+
const jobConfig = jobConfigsWithCron[jobName];
|
|
231
|
+
if (!jobConfig?.cron) continue;
|
|
232
|
+
if (existingSchedulerKeys.has(jobName)) {
|
|
233
|
+
updated.push(jobName);
|
|
234
|
+
} else {
|
|
235
|
+
newJobs.push(jobName);
|
|
236
|
+
}
|
|
237
|
+
await queue.upsertJobScheduler(jobName, { pattern: jobConfig.cron });
|
|
238
|
+
}
|
|
239
|
+
if (newJobs.length > 0) {
|
|
240
|
+
newJobs.forEach(
|
|
241
|
+
(name) => console.log(` + ${name} \u2192 ${jobConfigsWithCron[name].cron}`)
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
if (updated.length > 0) {
|
|
245
|
+
updated.forEach(
|
|
246
|
+
(name) => console.log(` \u21BB ${name} \u2192 ${jobConfigsWithCron[name].cron}`)
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
if (removed.length > 0) {
|
|
250
|
+
removed.forEach((name) => console.log(` - ${name}`));
|
|
251
|
+
}
|
|
252
|
+
if (newJobs.length === 0 && updated.length === 0 && removed.length === 0) {
|
|
253
|
+
console.log(` (no scheduled jobs)`);
|
|
254
|
+
}
|
|
255
|
+
console.log("");
|
|
256
|
+
}
|
|
257
|
+
async processJob(job) {
|
|
258
|
+
const isEtlTransformJob = job.name.endsWith(":transform");
|
|
259
|
+
const jobName = isEtlTransformJob ? job.name.slice(0, -":transform".length) : job.name;
|
|
260
|
+
const jobConfig = this.jobConfigs[jobName];
|
|
261
|
+
if (!jobConfig) {
|
|
262
|
+
throw new Error(`Job handler not found: ${jobName}`);
|
|
263
|
+
}
|
|
264
|
+
if (jobConfig.type === "job") {
|
|
265
|
+
return await jobConfig.handler(job, job.data);
|
|
266
|
+
}
|
|
267
|
+
if (jobConfig.type === "etl" && !isEtlTransformJob) {
|
|
268
|
+
const transformQueueName = jobConfig.transformQueue || jobConfig.queueName;
|
|
269
|
+
const queueData = this.queues[transformQueueName];
|
|
270
|
+
if (!queueData) {
|
|
271
|
+
throw new Error(`Queue ${transformQueueName} not found in config.`);
|
|
272
|
+
}
|
|
273
|
+
return await processEtlJob(
|
|
274
|
+
job,
|
|
275
|
+
jobConfig,
|
|
276
|
+
queueData.queue,
|
|
277
|
+
queueData.events
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
if (jobConfig.type === "etl" && isEtlTransformJob) {
|
|
281
|
+
return await jobConfig.transform(job.data);
|
|
282
|
+
}
|
|
283
|
+
throw new Error(`Unknown job type: ${jobConfig.type}`);
|
|
284
|
+
}
|
|
285
|
+
};
|
|
286
|
+
|
|
287
|
+
// src/index.ts
|
|
288
|
+
function createRavenflowApp(config, connection, loggerOptions) {
|
|
289
|
+
return new RavenflowApp(connection, config, loggerOptions);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
export { RavenflowApp, createLogger, createRavenflowApp };
|
|
293
|
+
//# sourceMappingURL=index.mjs.map
|
|
294
|
+
//# sourceMappingURL=index.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/processor.ts","../src/logger.ts","../src/banner.ts","../src/app.ts","../src/index.ts"],"names":[],"mappings":";;;;;;AAGA,eAAsB,aAAA,CACpB,GAAA,EACA,SAAA,EACA,cAAA,EACA,oBAAA,EACA;AACA,EAAA,IAAI,WAAA,GAAc,CAAA;AAClB,EAAA,IAAI,GAAA,GAAM,IAAA;AACV,EAAA,MAAM,QAAA,GAAW;AAAA,IACf,KAAA,EAAO,CAAA;AAAA,IACP,KAAA,EAAO,CAAA;AAAA,IACP,WAAA;AAAA,IACA,SAAA,sBAAe,IAAA,EAAK;AAAA,IACpB,OAAA,EAAS;AAAA,GACX;AACA,EAAA,MAAM,UAAA,GAAa;AAAA,IAEjB,QAAA,EAAU,QAEZ,CAAA;AAEA,EAAA,OAAO,GAAA,IAAO,WAAA,KAAgB,UAAA,CAAW,QAAA,IAAY,QAAA,CAAA,EAAW;AAC9D,IAAA,QAAA,CAAS,WAAA,GAAc,WAAA;AACvB,IAAA,QAAA,CAAS,KAAA,GAAQ,WAAA;AAEjB,IAAA,MAAM;AAAA,MACJ,IAAA,EAAM,WAAA;AAAA,MACN,aAAa,cAAA,GAAiB;AAAA,KAChC,GAGI,MAAM,SAAA,CAAU,OAAA,CAAQ,WAAW,CAAA;AAEvC,IAAA,MAAM,oBAAkC,EAAC;AAEzC,IAAA,KAAA,MAAW,WAAW,WAAA,EAAa;AACjC,MAAA,MAAM,gBAAA,GAAmB,CAAA,EAAG,GAAA,CAAI,IAAI,CAAA,UAAA,CAAA;AACpC,MAAA,MAAM,YAAA,GAAe,MAAM,cAAA,CAAe,GAAA,CAAI,kBAAkB,OAAA,EAAS;AAAA,QACvE,UAAU,SAAA,CAAU,QAAA;AAAA,QACpB,SAAS,SAAA,CAAU;AAAA,OACpB,CAAA;AACD,MAAA,MAAM,gBAAA,GACJ,MAAM,YAAA,CAAa,iBAAA,CAAkB,oBAAoB,CAAA;AAE3D,MAAA,iBAAA,CAAkB,KAAK,gBAA8B,CAAA;AAAA,IACvD;AAEA,IAAA,MAAM,SAAA,CAAU,KAAK,iBAAiB,CAAA;AACtC,IAAA,QAAA,CAAS,SAAS,iBAAA,CAAkB,MAAA;AAEpC,IAAA,GAAA,GAAM,cAAA;AACN,IAAA,IAAI,GAAA,EAAK;AACP,MAAA,WAAA,EAAA;AAAA,IACF;AAAA,EACF;AACA,EAAA,OAAO,QAAA;AACT;ACpDO,SAAS,YAAA,CAAa,OAAA,GAAyB,EAAC,EAAG;AACxD,EAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,GAAA,CAAI,QAAA,KAAa,YAAA;AACvC,EAAA,MAAM,SAAA,GAAY,QAAQ,MAAA,IAAU,KAAA;AAEpC,EAAA,OAAO,IAAA,CAAK;AAAA,IACV,KAAA,EAAO,OAAA,CAAQ,KAAA,KAAU,KAAA,GAAQ,OAAA,GAAU,MAAA,CAAA;AAAA,IAC3C,WAAW,SAAA,GACP;AAAA,MACE,MAAA,EAAQ,aAAA;AAAA,MACR,OAAA,EAAS;AAAA,QACP,QAAA,EAAU,IAAA;AAAA,QACV,aAAA,EAAe,UAAA;AAAA,QACf,MAAA,EAAQ;AAAA;AACV,KACF,GACA;AAAA,GACL,CAAA;AACH;;;ACtBA,IAAM,eAAA,GAAkB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,CAAA;AAoBjB,SAAS,kBAAkB,MAAA,EAA4B;AAC5D,EAAA,MAAM,EAAE,SAAA,EAAW,WAAA,EAAa,KAAA,EAAO,MAAK,GAAI,MAAA;AAEhD,EAAA,OAAA,CAAQ,GAAA,CAAI,OAAO,eAAe,CAAA;AAClC,EAAA,OAAA,CAAQ,GAAA,CAAI,QAAA,CAAI,MAAA,CAAO,EAAE,CAAC,CAAA;AAE1B,EAAA,OAAA,CAAQ,IAAI,YAAY,CAAA;AACxB,EAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,kBAAA,EAAqB,SAAS,CAAA,CAAE,CAAA;AAC5C,EAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,kBAAA,EAAqB,WAAW,CAAA,CAAE,CAAA;AAC9C,EAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,kBAAA,EAAqB,KAAA,CAAM,IAAI,CAAA,CAAA,EAAI,MAAM,IAAI,CAAA,CAAA,EAAI,KAAA,CAAM,EAAE,CAAA,CAAE,CAAA;AAEvE,EAAA,OAAA,CAAQ,IAAI,qBAAqB,CAAA;AAEjC,EAAA,MAAM,cAAwB,EAAC;AAC/B,EAAA,MAAM,WAA6C,EAAC;AAEpD,EAAA,KAAA,MAAW,CAAC,OAAA,EAAS,SAAS,KAAK,MAAA,CAAO,OAAA,CAAQ,IAAI,CAAA,EAAG;AACvD,IAAA,IAAI,UAAU,IAAA,EAAM;AAClB,MAAA,QAAA,CAAS,KAAK,EAAE,IAAA,EAAM,SAAS,IAAA,EAAM,SAAA,CAAU,MAAM,CAAA;AAAA,IACvD,CAAA,MAAO;AACL,MAAA,WAAA,CAAY,KAAK,OAAO,CAAA;AAAA,IAC1B;AAAA,EACF;AAEA,EAAA,IAAI,WAAA,CAAY,SAAS,CAAA,EAAG;AAC1B,IAAA,WAAA,CAAY,OAAA,CAAQ,CAAC,OAAA,KAAY;AAC/B,MAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,IAAA,EAAO,OAAO,CAAA,CAAE,CAAA;AAAA,IAC9B,CAAC,CAAA;AAAA,EACH;AAEA,EAAA,IAAI,QAAA,CAAS,SAAS,CAAA,EAAG;AACvB,IAAA,QAAA,CAAS,OAAA,CAAQ,CAAC,EAAE,IAAA,EAAM,MAAK,KAAM;AACnC,MAAA,OAAA,CAAQ,GAAA,CAAI,CAAA,IAAA,EAAO,IAAI,CAAA,SAAA,EAAO,IAAI,CAAA,CAAA,CAAG,CAAA;AAAA,IACvC,CAAC,CAAA;AAAA,EACH;AAEA,EAAA,OAAA,CAAQ,GAAA,CAAI,IAAA,GAAO,QAAA,CAAI,MAAA,CAAO,EAAE,CAAC,CAAA;AACjC,EAAA,OAAA,CAAQ,IAAI,4BAA4B,CAAA;AAC1C;;;AChDO,IAAM,eAAN,MAAsE;AAAA,EAc3E,WAAA,CACE,UAAA,EACA,MAAA,EACA,aAAA,EACA;AAhBF,IAAA,IAAA,CAAU,kBAAA,GAA6B,CAAA;AAiBrC,IAAA,IAAA,CAAK,UAAA,GAAa,UAAA;AAClB,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,MAAA,GAAS,aAAa,aAAa,CAAA;AAExC,IAAA,IAAI,CAAC,MAAA,CAAO,MAAA,CAAO,QAAA,CAAS,MAAA,CAAO,YAAY,CAAA,EAAG;AAChD,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,cAAA,EAAiB,OAAO,YAAY,CAAA,6BAAA;AAAA,OACtC;AAAA,IACF;AAEA,IAAA,MAAM,EAAE,SAAA,EAAW,UAAA,EAAW,GAAI,IAAA,CAAK,SAAS,MAAM,CAAA;AACtD,IAAA,IAAA,CAAK,IAAA,GAAO,SAAA;AACZ,IAAA,IAAA,CAAK,UAAA,GAAa,UAAA;AAElB,IAAA,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,UAAA,CAAW,MAAM,CAAA;AAAA,EACtC;AAAA,EAEQ,WACN,MAAA,EACoE;AACpE,IAAA,MAAM,SAAgE,EAAC;AACvE,IAAA,KAAA,MAAW,SAAA,IAAa,OAAO,MAAA,EAAQ;AACrC,MAAA,MAAA,CAAO,SAAS,CAAA,GAAI;AAAA,QAClB,KAAA,EAAO,IAAI,KAAA,CAAM,SAAA,EAAW;AAAA,UAC1B,YAAY,IAAA,CAAK;AAAA,SAClB,CAAA;AAAA,QACD,MAAA,EAAQ,IAAI,WAAA,CAAY,SAAA,EAAW;AAAA,UACjC,YAAY,IAAA,CAAK;AAAA,SAClB;AAAA,OACH;AAAA,IACF;AACA,IAAA,OAAO,MAAA;AAAA,EACT;AAAA,EAEQ,SAAS,MAAA,EAGf;AACA,IAAA,MAAM,YAA4D,EAAC;AACnE,IAAA,MAAM,aAAwC,EAAC;AAE/C,IAAA,KAAA,MAAW,CAAC,SAAS,SAAS,CAAA,IAAK,OAAO,OAAA,CAAQ,MAAA,CAAO,IAAI,CAAA,EAAG;AAC9D,MAAA,UAAA,CAAW,OAAO,CAAA,GAAI,SAAA;AAEtB,MAAA,SAAA,CAAU,OAAO,CAAA,GAAI;AAAA,QACnB,IAAA,EAAM,OAAO,MAAA,KAAgB;AAC3B,UAAA,OAAO,IAAA,CAAK,OAAA,CAAQ,OAAA,EAA8B,MAAM,CAAA;AAAA,QAC1D;AAAA,OACF;AAAA,IACF;AACA,IAAA,OAAO;AAAA,MACL,SAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAAA,EAEA,MAAc,OAAA,CAAQ,OAAA,EAA4B,MAAA,EAAa;AAC7D,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,UAAA,CAAW,OAAO,CAAA;AACzC,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,IAAA,EAAO,OAAO,CAAA,UAAA,CAAY,CAAA;AAAA,IAC5C;AACA,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,MAAA,CAAO,SAAA,CAAU,SAAgC,CAAA;AACxE,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,MAAA,EAAS,SAAA,CAAU,SAAS,CAAA,UAAA,CAAY,CAAA;AAAA,IAC1D;AACA,IAAA,OAAO,MAAM,SAAA,CAAU,KAAA,CAAM,GAAA,CAAI,SAAS,MAAM,CAAA;AAAA,EAClD;AAAA,EAEO,WAAA,CACL,WACA,OAAA,EAGA;AACA,IAAA,MAAM,WAAA,GAAc,IAAA,CAAK,MAAA,CAAO,SAAS,CAAA;AACzC,IAAA,IAAI,CAAC,WAAA,EAAa;AAChB,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,MAAA,EAAS,SAAS,CAAA,qBAAA,CAAuB,CAAA;AAAA,IAC3D;AAEA,IAAA,MAAM,iBAAA,GAAoB,OAAA,EAAS,WAAA,IAAe,IAAA,CAAK,kBAAA;AAEvD,IAAA,IAAA,CAAK,iBAAA,CAAkB,WAAW,iBAAiB,CAAA;AAEnD,IAAA,IAAA,CAAK,kBAAkB,SAAS,CAAA;AAEhC,IAAA,MAAM,MAAA,GAAS,IAAI,MAAA,CAAO,SAAA,EAAW,CAAC,GAAA,KAAQ,IAAA,CAAK,UAAA,CAAW,GAAG,CAAA,EAAG;AAAA,MAClE,YAAY,IAAA,CAAK,UAAA;AAAA,MACjB,OAAA,EAAS,KAAA;AAAA,MACT,WAAA,EAAa;AAAA,KACd,CAAA;AACD,IAAA,IAAA,CAAK,iBAAA,CAAkB,QAAQ,SAAS,CAAA;AACxC,IAAA,MAAA,CAAO,GAAA,EAAI;AAAA,EACb;AAAA,EAEQ,iBAAA,CACN,WACA,WAAA,EACM;AACN,IAAA,MAAM,SAAA,GAAY;AAAA,MAChB,IAAA,EACE,OAAO,IAAA,CAAK,UAAA,KAAe,QAAA,IAAY,UAAU,IAAA,CAAK,UAAA,GACjD,IAAA,CAAK,UAAA,CAAW,IAAA,GACjB,WAAA;AAAA,MACN,IAAA,EACE,OAAO,IAAA,CAAK,UAAA,KAAe,QAAA,IAAY,UAAU,IAAA,CAAK,UAAA,GACjD,IAAA,CAAK,UAAA,CAAW,IAAA,GACjB,IAAA;AAAA,MACN,EAAA,EACE,OAAO,IAAA,CAAK,UAAA,KAAe,QAAA,IAAY,QAAQ,IAAA,CAAK,UAAA,GAC/C,IAAA,CAAK,UAAA,CAAW,EAAA,GACjB;AAAA,KACR;AAEA,IAAA,iBAAA,CAAkB;AAAA,MAChB,SAAA;AAAA,MACA,WAAA;AAAA,MACA,KAAA,EAAO,SAAA;AAAA,MACP,MAAM,IAAA,CAAK;AAAA,KACZ,CAAA;AAAA,EACH;AAAA,EAEQ,iBAAA,CAAkB,QAAgB,SAAA,EAAyB;AACjE,IAAA,MAAA,CAAO,EAAA,CAAG,QAAA,EAAU,CAAC,GAAA,KAAQ;AAC3B,MAAA,IAAA,CAAK,OAAO,IAAA,CAAK,CAAA,CAAA,EAAI,SAAS,CAAA,EAAA,EAAK,GAAA,CAAI,IAAI,CAAA,SAAA,CAAW,CAAA;AAAA,IACxD,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,EAAA,CAAG,WAAA,EAAa,CAAC,GAAA,KAAQ;AAC9B,MAAA,IAAA,CAAK,OAAO,IAAA,CAAK,CAAA,CAAA,EAAI,SAAS,CAAA,EAAA,EAAK,GAAA,CAAI,IAAI,CAAA,YAAA,CAAc,CAAA;AAAA,IAC3D,CAAC,CAAA;AAED,IAAA,MAAA,CAAO,EAAA,CAAG,QAAA,EAAU,CAAC,GAAA,EAAK,KAAA,KAAU;AAClC,MAAA,IAAA,CAAK,MAAA,CAAO,KAAA;AAAA,QACV,IAAI,SAAS,CAAA,EAAA,EAAK,KAAK,IAAI,CAAA,WAAA,EAAc,MAAM,OAAO,CAAA;AAAA,OACxD;AAAA,IACF,CAAC,CAAA;AAAA,EACH;AAAA,EAEA,MAAc,kBAAkB,SAAA,EAAgC;AAC9D,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,MAAA,CAAO,SAAS,CAAA;AACvC,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,MAAA,EAAS,SAAS,CAAA,qBAAA,CAAuB,CAAA;AAAA,IAC3D;AACA,IAAA,MAAM,QAAQ,SAAA,CAAU,KAAA;AAExB,IAAA,OAAA,CAAQ,IAAI,kBAAkB,CAAA;AAE9B,IAAA,MAAM,qBAAgD,EAAC;AACvD,IAAA,KAAA,MAAW,OAAA,IAAW,KAAK,UAAA,EAAY;AACrC,MAAA,MAAM,SAAA,GAAY,IAAA,CAAK,UAAA,CAAW,OAA4B,CAAA;AAC9D,MAAA,IAAI,UAAU,IAAA,EAAM;AAClB,QAAA,kBAAA,CAAmB,OAAO,CAAA,GAAI,SAAA;AAAA,MAChC;AAAA,IACF;AAEA,IAAA,MAAM,uBAAA,GAA0B,MAAM,KAAA,CAAM,gBAAA,EAAiB;AAC7D,IAAA,MAAM,wBAAwB,IAAI,GAAA;AAAA,MAChC,uBAAA,CAAwB,GAAA,CAAI,CAAC,CAAA,KAAM,EAAE,GAAG;AAAA,KAC1C;AAEA,IAAA,MAAM,UAAoB,EAAC;AAC3B,IAAA,KAAA,MAAW,gBAAgB,uBAAA,EAAyB;AAClD,MAAA,IAAI,CAAC,kBAAA,CAAmB,YAAA,CAAa,GAAG,CAAA,EAAG;AACzC,QAAA,OAAA,CAAQ,IAAA,CAAK,aAAa,GAAG,CAAA;AAC7B,QAAA,MAAM,KAAA,CAAM,kBAAA,CAAmB,YAAA,CAAa,GAAG,CAAA;AAAA,MACjD;AAAA,IACF;AAEA,IAAA,MAAM,UAAoB,EAAC;AAC3B,IAAA,MAAM,UAAoB,EAAC;AAE3B,IAAA,KAAA,MAAW,WAAW,kBAAA,EAAoB;AACxC,MAAA,MAAM,SAAA,GAAY,mBAAmB,OAAO,CAAA;AAC5C,MAAA,IAAI,CAAC,WAAW,IAAA,EAAM;AAEtB,MAAA,IAAI,qBAAA,CAAsB,GAAA,CAAI,OAAO,CAAA,EAAG;AACtC,QAAA,OAAA,CAAQ,KAAK,OAAO,CAAA;AAAA,MACtB,CAAA,MAAO;AACL,QAAA,OAAA,CAAQ,KAAK,OAAO,CAAA;AAAA,MACtB;AAEA,MAAA,MAAM,MAAM,kBAAA,CAAmB,OAAA,EAAS,EAAE,OAAA,EAAS,SAAA,CAAU,MAAM,CAAA;AAAA,IACrE;AAEA,IAAA,IAAI,OAAA,CAAQ,SAAS,CAAA,EAAG;AACtB,MAAA,OAAA,CAAQ,OAAA;AAAA,QAAQ,CAAC,IAAA,KACf,OAAA,CAAQ,GAAA,CAAI,CAAA,IAAA,EAAO,IAAI,CAAA,QAAA,EAAM,kBAAA,CAAmB,IAAI,CAAA,CAAE,IAAI,CAAA,CAAE;AAAA,OAC9D;AAAA,IACF;AACA,IAAA,IAAI,OAAA,CAAQ,SAAS,CAAA,EAAG;AACtB,MAAA,OAAA,CAAQ,OAAA;AAAA,QAAQ,CAAC,IAAA,KACf,OAAA,CAAQ,GAAA,CAAI,CAAA,SAAA,EAAO,IAAI,CAAA,QAAA,EAAM,kBAAA,CAAmB,IAAI,CAAA,CAAE,IAAI,CAAA,CAAE;AAAA,OAC9D;AAAA,IACF;AACA,IAAA,IAAI,OAAA,CAAQ,SAAS,CAAA,EAAG;AACtB,MAAA,OAAA,CAAQ,OAAA,CAAQ,CAAC,IAAA,KAAS,OAAA,CAAQ,IAAI,CAAA,IAAA,EAAO,IAAI,EAAE,CAAC,CAAA;AAAA,IACtD;AACA,IAAA,IAAI,OAAA,CAAQ,WAAW,CAAA,IAAK,OAAA,CAAQ,WAAW,CAAA,IAAK,OAAA,CAAQ,WAAW,CAAA,EAAG;AACxE,MAAA,OAAA,CAAQ,IAAI,CAAA,qBAAA,CAAuB,CAAA;AAAA,IACrC;AAEA,IAAA,OAAA,CAAQ,IAAI,EAAE,CAAA;AAAA,EAChB;AAAA,EAEA,MAAM,WAAW,GAAA,EAAU;AACzB,IAAA,MAAM,iBAAA,GAAoB,GAAA,CAAI,IAAA,CAAK,QAAA,CAAS,YAAY,CAAA;AACxD,IAAA,MAAM,OAAA,GAAU,iBAAA,GACZ,GAAA,CAAI,IAAA,CAAK,KAAA,CAAM,GAAG,CAAC,YAAA,CAAa,MAAM,CAAA,GACtC,GAAA,CAAI,IAAA;AAER,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,UAAA,CAAW,OAA4B,CAAA;AAC9D,IAAA,IAAI,CAAC,SAAA,EAAW;AACd,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,uBAAA,EAA0B,OAAO,CAAA,CAAE,CAAA;AAAA,IACrD;AAEA,IAAA,IAAI,SAAA,CAAU,SAAS,KAAA,EAAO;AAC5B,MAAA,OAAO,MAAM,SAAA,CAAU,OAAA,CAAQ,GAAA,EAAK,IAAI,IAAI,CAAA;AAAA,IAC9C;AAEA,IAAA,IAAI,SAAA,CAAU,IAAA,KAAS,KAAA,IAAS,CAAC,iBAAA,EAAmB;AAClD,MAAA,MAAM,kBAAA,GACJ,SAAA,CAAU,cAAA,IAAkB,SAAA,CAAU,SAAA;AACxC,MAAA,MAAM,SAAA,GAAY,IAAA,CAAK,MAAA,CAAO,kBAAyC,CAAA;AAEvE,MAAA,IAAI,CAAC,SAAA,EAAW;AACd,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,MAAA,EAAS,kBAAkB,CAAA,qBAAA,CAAuB,CAAA;AAAA,MACpE;AACA,MAAA,OAAO,MAAM,aAAA;AAAA,QACX,GAAA;AAAA,QACA,SAAA;AAAA,QACA,SAAA,CAAU,KAAA;AAAA,QACV,SAAA,CAAU;AAAA,OACZ;AAAA,IACF;AAEA,IAAA,IAAI,SAAA,CAAU,IAAA,KAAS,KAAA,IAAS,iBAAA,EAAmB;AACjD,MAAA,OAAO,MAAM,SAAA,CAAU,SAAA,CAAU,GAAA,CAAI,IAAI,CAAA;AAAA,IAC3C;AAEA,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,kBAAA,EAAqB,SAAA,CAAU,IAAI,CAAA,CAAE,CAAA;AAAA,EACvD;AACF;;;AC/PO,SAAS,kBAAA,CACd,MAAA,EACA,UAAA,EACA,aAAA,EAIuB;AACvB,EAAA,OAAO,IAAI,YAAA,CAAa,UAAA,EAAY,MAAA,EAAQ,aAAa,CAAA;AAC3D","file":"index.mjs","sourcesContent":["import { Job, Queue, QueueEvents } from \"bullmq\";\nimport { EtlJobConfig } from \"./types\";\n\nexport async function processEtlJob<TRaw, TProcessed>(\n job: Job,\n etlConfig: EtlJobConfig<TRaw, TProcessed>,\n transformQueue: Queue,\n transformQueueEvents: QueueEvents\n) {\n let currentPage = 1;\n let run = true;\n const metadata = {\n count: 0,\n pages: 0,\n currentPage,\n startedAt: new Date(),\n endedAt: undefined,\n };\n const pagination = {\n pageSize: 100,\n maxPages: Infinity,\n pageParam: \"page\",\n };\n\n while (run && currentPage <= (pagination.maxPages || Infinity)) {\n metadata.currentPage = currentPage;\n metadata.pages = currentPage;\n\n const {\n data: rawElements,\n hasNextPage: nextPageExists = false,\n }: {\n data: TRaw[];\n hasNextPage?: boolean;\n } = await etlConfig.extract(currentPage);\n\n const processedElements: TProcessed[] = [];\n\n for (const element of rawElements) {\n const transformJobName = `${job.name}:transform`;\n const transformJob = await transformQueue.add(transformJobName, element, {\n attempts: etlConfig.attempts,\n backoff: etlConfig.backoff,\n });\n const processedElement =\n await transformJob.waitUntilFinished(transformQueueEvents);\n\n processedElements.push(processedElement as TProcessed);\n }\n\n await etlConfig.load(processedElements);\n metadata.count += processedElements.length;\n\n run = nextPageExists;\n if (run) {\n currentPage++;\n }\n }\n return metadata;\n}\n","import pino from \"pino\";\n\nexport interface LoggerOptions {\n level?: \"trace\" | \"debug\" | \"info\" | \"warn\" | \"error\" | \"fatal\";\n pretty?: boolean;\n}\n\nexport function createLogger(options: LoggerOptions = {}) {\n const isDev = process.env.NODE_ENV !== \"production\";\n const usePretty = options.pretty ?? isDev;\n\n return pino({\n level: options.level || (isDev ? \"debug\" : \"info\"),\n transport: usePretty\n ? {\n target: \"pino-pretty\",\n options: {\n colorize: true,\n translateTime: \"HH:MM:ss\",\n ignore: \"pid,hostname\",\n },\n }\n : undefined,\n });\n}\n\nexport type Logger = ReturnType<typeof createLogger>;\n","import { JobConfig } from \"./types\";\n\nconst RAVENFLOW_ASCII = `\n ██████╗ █████╗ ██╗ ██╗███████╗███╗ ██╗███████╗██╗ ██████╗ ██╗ ██╗\n ██╔══██╗██╔══██╗██║ ██║██╔════╝████╗ ██║██╔════╝██║ ██╔═══██╗██║ ██║\n ██████╔╝███████║██║ ██║█████╗ ██╔██╗ ██║█████╗ ██║ ██║ ██║██║ █╗ ██║\n ██╔══██╗██╔══██║╚██╗ ██╔╝██╔══╝ ██║╚██╗██║██╔══╝ ██║ ██║ ██║██║███╗██║\n ██║ ██║██║ ██║ ╚████╔╝ ███████╗██║ ╚████║██║ ███████╗╚██████╔╝╚███╔███╔╝\n ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═══╝ ╚══════╝╚═╝ ╚═══╝╚═╝ ╚══════╝ ╚═════╝ ╚══╝╚══╝ \n`;\n\ninterface BannerConfig {\n queueName: string;\n concurrency: number;\n redis: {\n host: string;\n port: number;\n db: number;\n };\n jobs: Record<string, JobConfig>;\n}\n\nexport function printWorkerBanner(config: BannerConfig): void {\n const { queueName, concurrency, redis, jobs } = config;\n\n console.log(\"\\n\" + RAVENFLOW_ASCII);\n console.log(\"═\".repeat(85));\n\n console.log(\"\\n[Config]\");\n console.log(` -> queue: ${queueName}`);\n console.log(` -> concurrency: ${concurrency}`);\n console.log(` -> redis: ${redis.host}:${redis.port}/${redis.db}`);\n\n console.log(\"\\n[Registered Jobs]\");\n\n const regularJobs: string[] = [];\n const cronJobs: { name: string; cron: string }[] = [];\n\n for (const [jobName, jobConfig] of Object.entries(jobs)) {\n if (jobConfig.cron) {\n cronJobs.push({ name: jobName, cron: jobConfig.cron });\n } else {\n regularJobs.push(jobName);\n }\n }\n\n if (regularJobs.length > 0) {\n regularJobs.forEach((jobName) => {\n console.log(` . ${jobName}`);\n });\n }\n\n if (cronJobs.length > 0) {\n cronJobs.forEach(({ name, cron }) => {\n console.log(` . ${name} (↻ ${cron})`);\n });\n }\n\n console.log(\"\\n\" + \"═\".repeat(85));\n console.log(\"Ready to process jobs...\\n\");\n}\n","import { ConnectionOptions, Queue, Worker, Job, QueueEvents } from \"bullmq\";\nimport {\n RavenflowConfig,\n QueueNames,\n JobConfig,\n JobNames,\n JobsProxy,\n} from \"./types\";\nimport { processEtlJob } from \"./processor\";\nimport { createLogger, Logger, LoggerOptions } from \"./logger\";\nimport { printWorkerBanner } from \"./banner\";\n\nexport class RavenflowApp<TConfig extends RavenflowConfig = RavenflowConfig> {\n protected connection: ConnectionOptions;\n protected defaultConcurrency: number = 5;\n protected logger: Logger;\n\n protected config: TConfig;\n\n public jobs: JobsProxy<TConfig[\"jobs\"]>;\n protected jobConfigs: Record<JobNames<TConfig>, JobConfig>;\n protected queues: Record<\n QueueNames<TConfig>,\n { queue: Queue; events: QueueEvents }\n >;\n\n constructor(\n connection: ConnectionOptions,\n config: TConfig,\n loggerOptions?: LoggerOptions\n ) {\n this.connection = connection;\n this.config = config;\n this.logger = createLogger(loggerOptions);\n\n if (!config.queues.includes(config.defaultQueue)) {\n throw new Error(\n `Default queue ${config.defaultQueue} is not in the list of queues`\n );\n }\n\n const { jobsProxy, jobConfigs } = this.initJobs(config);\n this.jobs = jobsProxy;\n this.jobConfigs = jobConfigs;\n\n this.queues = this.initQueues(config);\n }\n\n private initQueues(\n config: TConfig\n ): Record<QueueNames<TConfig>, { queue: Queue; events: QueueEvents }> {\n const queues: Record<string, { queue: Queue; events: QueueEvents }> = {};\n for (const queueName of config.queues) {\n queues[queueName] = {\n queue: new Queue(queueName, {\n connection: this.connection,\n }),\n events: new QueueEvents(queueName, {\n connection: this.connection,\n }),\n };\n }\n return queues;\n }\n\n private initJobs(config: TConfig): {\n jobsProxy: JobsProxy<TConfig[\"jobs\"]>;\n jobConfigs: Record<JobNames<TConfig>, JobConfig>;\n } {\n const jobsProxy: Record<string, { send: (params: any) => any }> = {};\n const jobConfigs: Record<string, JobConfig> = {};\n\n for (const [jobName, jobConfig] of Object.entries(config.jobs)) {\n jobConfigs[jobName] = jobConfig;\n\n jobsProxy[jobName] = {\n send: async (params: any) => {\n return this.sendJob(jobName as JobNames<TConfig>, params);\n },\n };\n }\n return {\n jobsProxy: jobsProxy as JobsProxy<TConfig[\"jobs\"]>,\n jobConfigs,\n };\n }\n\n private async sendJob(jobName: JobNames<TConfig>, params: any) {\n const jobConfig = this.jobConfigs[jobName];\n if (!jobConfig) {\n throw new Error(`Job ${jobName} not found`);\n }\n const queueData = this.queues[jobConfig.queueName as QueueNames<TConfig>];\n if (!queueData) {\n throw new Error(`Queue ${jobConfig.queueName} not found`);\n }\n return await queueData.queue.add(jobName, params);\n }\n\n public startWorker(\n queueName: QueueNames<TConfig>,\n options?: {\n concurrency?: number;\n }\n ) {\n const queueConfig = this.queues[queueName];\n if (!queueConfig) {\n throw new Error(`Queue ${queueName} not found in config.`);\n }\n\n const workerConcurrency = options?.concurrency || this.defaultConcurrency;\n\n this.printWorkerBanner(queueName, workerConcurrency);\n\n this.syncJobSchedulers(queueName);\n\n const worker = new Worker(queueName, (job) => this.processJob(job), {\n connection: this.connection,\n autorun: false,\n concurrency: workerConcurrency,\n });\n this.setupWorkerEvents(worker, queueName);\n worker.run();\n }\n\n private printWorkerBanner(\n queueName: QueueNames<TConfig>,\n concurrency: number\n ): void {\n const redisInfo = {\n host:\n typeof this.connection === \"object\" && \"host\" in this.connection\n ? (this.connection.host as string)\n : \"localhost\",\n port:\n typeof this.connection === \"object\" && \"port\" in this.connection\n ? (this.connection.port as number)\n : 6379,\n db:\n typeof this.connection === \"object\" && \"db\" in this.connection\n ? (this.connection.db as number)\n : 0,\n };\n\n printWorkerBanner({\n queueName: queueName as string,\n concurrency,\n redis: redisInfo,\n jobs: this.jobConfigs,\n });\n }\n\n private setupWorkerEvents(worker: Worker, queueName: string): void {\n worker.on(\"active\", (job) => {\n this.logger.info(`[${queueName}] ${job.name} - active`);\n });\n\n worker.on(\"completed\", (job) => {\n this.logger.info(`[${queueName}] ${job.name} - completed`);\n });\n\n worker.on(\"failed\", (job, error) => {\n this.logger.error(\n `[${queueName}] ${job?.name} - failed: ${error.message}`\n );\n });\n }\n\n private async syncJobSchedulers(queueName: QueueNames<TConfig>) {\n const queueData = this.queues[queueName];\n if (!queueData) {\n throw new Error(`Queue ${queueName} not found in config.`);\n }\n const queue = queueData.queue;\n\n console.log(\"[Scheduled Jobs]\");\n\n const jobConfigsWithCron: Record<string, JobConfig> = {};\n for (const jobName in this.jobConfigs) {\n const jobConfig = this.jobConfigs[jobName as JobNames<TConfig>];\n if (jobConfig.cron) {\n jobConfigsWithCron[jobName] = jobConfig;\n }\n }\n\n const registeredJobSchedulers = await queue.getJobSchedulers();\n const existingSchedulerKeys = new Set(\n registeredJobSchedulers.map((s) => s.key)\n );\n\n const removed: string[] = [];\n for (const jobScheduler of registeredJobSchedulers) {\n if (!jobConfigsWithCron[jobScheduler.key]) {\n removed.push(jobScheduler.key);\n await queue.removeJobScheduler(jobScheduler.key);\n }\n }\n\n const newJobs: string[] = [];\n const updated: string[] = [];\n\n for (const jobName in jobConfigsWithCron) {\n const jobConfig = jobConfigsWithCron[jobName];\n if (!jobConfig?.cron) continue;\n\n if (existingSchedulerKeys.has(jobName)) {\n updated.push(jobName);\n } else {\n newJobs.push(jobName);\n }\n\n await queue.upsertJobScheduler(jobName, { pattern: jobConfig.cron });\n }\n\n if (newJobs.length > 0) {\n newJobs.forEach((name) =>\n console.log(` + ${name} → ${jobConfigsWithCron[name].cron}`)\n );\n }\n if (updated.length > 0) {\n updated.forEach((name) =>\n console.log(` ↻ ${name} → ${jobConfigsWithCron[name].cron}`)\n );\n }\n if (removed.length > 0) {\n removed.forEach((name) => console.log(` - ${name}`));\n }\n if (newJobs.length === 0 && updated.length === 0 && removed.length === 0) {\n console.log(` (no scheduled jobs)`);\n }\n\n console.log(\"\");\n }\n\n async processJob(job: Job) {\n const isEtlTransformJob = job.name.endsWith(\":transform\");\n const jobName = isEtlTransformJob\n ? job.name.slice(0, -\":transform\".length)\n : job.name;\n\n const jobConfig = this.jobConfigs[jobName as JobNames<TConfig>];\n if (!jobConfig) {\n throw new Error(`Job handler not found: ${jobName}`);\n }\n\n if (jobConfig.type === \"job\") {\n return await jobConfig.handler(job, job.data);\n }\n\n if (jobConfig.type === \"etl\" && !isEtlTransformJob) {\n const transformQueueName =\n jobConfig.transformQueue || jobConfig.queueName;\n const queueData = this.queues[transformQueueName as QueueNames<TConfig>];\n\n if (!queueData) {\n throw new Error(`Queue ${transformQueueName} not found in config.`);\n }\n return await processEtlJob(\n job,\n jobConfig,\n queueData.queue,\n queueData.events\n );\n }\n\n if (jobConfig.type === \"etl\" && isEtlTransformJob) {\n return await jobConfig.transform(job.data);\n }\n\n throw new Error(`Unknown job type: ${jobConfig.type}`);\n }\n}\n","import { RavenflowApp } from \"./app\";\nimport { RavenflowConfig } from \"./types\";\nimport { ConnectionOptions } from \"bullmq\";\n\nexport { RavenflowApp } from \"./app\";\nexport { createLogger } from \"./logger\";\n\nexport type {\n JobConfig,\n RavenflowConfig,\n SimpleJobConfig,\n EtlJobConfig,\n} from \"./types\";\n\nexport type { Logger, LoggerOptions } from \"./logger\";\n\nexport function createRavenflowApp<TConfig extends RavenflowConfig>(\n config: TConfig,\n connection: ConnectionOptions,\n loggerOptions?: {\n level?: \"trace\" | \"debug\" | \"info\" | \"warn\" | \"error\" | \"fatal\";\n pretty?: boolean;\n }\n): RavenflowApp<TConfig> {\n return new RavenflowApp(connection, config, loggerOptions);\n}\n"]}
|
package/package.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "ravenflow",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Type-safe job queue management for TypeScript",
|
|
5
|
+
"author": "Jeremy LP",
|
|
6
|
+
"email": "ravenflow.io@pm.me",
|
|
7
|
+
"license": "MIT",
|
|
8
|
+
"keywords": [
|
|
9
|
+
"queue",
|
|
10
|
+
"job",
|
|
11
|
+
"task",
|
|
12
|
+
"bullmq",
|
|
13
|
+
"redis",
|
|
14
|
+
"typescript",
|
|
15
|
+
"type-safe",
|
|
16
|
+
"background-jobs"
|
|
17
|
+
],
|
|
18
|
+
"repository": {
|
|
19
|
+
"type": "git",
|
|
20
|
+
"url": "https://github.com/ravenflowio/ravenflow.git"
|
|
21
|
+
},
|
|
22
|
+
"homepage": "https://ravenflow.io",
|
|
23
|
+
"bugs": {
|
|
24
|
+
"url": "https://github.com/ravenflowio/ravenflow/issues"
|
|
25
|
+
},
|
|
26
|
+
"main": "./dist/index.js",
|
|
27
|
+
"types": "./dist/index.d.ts",
|
|
28
|
+
"module": "./dist/index.mjs",
|
|
29
|
+
"exports": {
|
|
30
|
+
".": {
|
|
31
|
+
"types": "./dist/index.d.ts",
|
|
32
|
+
"import": "./dist/index.mjs",
|
|
33
|
+
"require": "./dist/index.js"
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
"files": [
|
|
37
|
+
"dist",
|
|
38
|
+
"README.md"
|
|
39
|
+
],
|
|
40
|
+
"scripts": {
|
|
41
|
+
"build": "tsup",
|
|
42
|
+
"build:watch": "tsup --watch",
|
|
43
|
+
"dev": "tsup --watch",
|
|
44
|
+
"prepublishOnly": "pnpm build",
|
|
45
|
+
"check-types": "tsc --noEmit"
|
|
46
|
+
},
|
|
47
|
+
"devDependencies": {
|
|
48
|
+
"@types/node": "^22.18.4",
|
|
49
|
+
"tsup": "^8.5.1",
|
|
50
|
+
"typescript": "^5.8.2"
|
|
51
|
+
},
|
|
52
|
+
"dependencies": {
|
|
53
|
+
"bullmq": "^5.58.5",
|
|
54
|
+
"ioredis": "^5.7.0",
|
|
55
|
+
"pino": "^8.17.2",
|
|
56
|
+
"pino-pretty": "^10.3.1"
|
|
57
|
+
}
|
|
58
|
+
}
|