startx 1.0.4 → 1.0.8

package/packages/queue/src/adapter/bullmq-adapter.ts

@@ -1,18 +1,25 @@
 import { logger } from "@repo/logger";
-import { Queue, Worker, QueueEvents, type ConnectionOptions, type JobsOptions } from "bullmq";
-import type { IQueueProvider } from "../queue-interface.js";
+import { Queue, Worker, QueueEvents, type ConnectionOptions, type JobsOptions, type WorkerOptions } from "bullmq";
+
+import type { IQueueProvider, RegisterCronConfig } from "../queue-interface.js";
 import { JobSchemas, type JobRegistry } from "../registry.js";
-import type { JobOptions, JobHandler, JobContext } from "../types.js";
+import type { JobContext, JobHandler, JobOptions } from "../types.js";
 
 export class BullMQProvider implements IQueueProvider {
 	private queues: Map<string, Queue> = new Map();
-	private workers: Worker[] = [];
+	private workers: Map<string, Worker> = new Map();
 	private events: Map<string, QueueEvents> = new Map();
 
-	constructor(private connection: ConnectionOptions = { host: "localhost", port: 6379 }) {}
+	constructor(
+		private connection: ConnectionOptions = {
+			host: "localhost",
+			port: 6379,
+		}
+	) {}
 
 	private mapOptions(options?: JobOptions): JobsOptions {
 		if (!options) return {};
+
 		return {
 			jobId: options.jobId,
 			delay: options.delay,
@@ -21,10 +28,15 @@ export class BullMQProvider implements IQueueProvider {
 		};
 	}
 
-	private getQueue(queueName: string): Queue {
+	getQueue(queueName: string): Queue {
 		if (!this.queues.has(queueName)) {
-			this.queues.set(queueName, new Queue(queueName, { connection: this.connection }));
+			const queue = new Queue(queueName, {
+				connection: this.connection,
+			});
+
+			this.queues.set(queueName, queue);
 		}
+
 		return this.queues.get(queueName)!;
 	}
 
@@ -34,9 +46,10 @@ export class BullMQProvider implements IQueueProvider {
 		options?: JobOptions
 	): Promise<string> {
 		const validated = JobSchemas[queueName].params.parse(params);
-		const queue = this.getQueue(queueName as string);
+		const queue = this.getQueue(queueName);
 		const job = await queue.add("job", validated, this.mapOptions(options));
-		return job.id!;
+
+		return String(job.id);
 	}
 
 	async enqueueMany<K extends keyof JobRegistry>(
@@ -44,52 +57,115 @@ export class BullMQProvider implements IQueueProvider {
 		paramsList: Array<JobRegistry[K]["params"]>,
 		options?: JobOptions
 	): Promise<string[]> {
-		const queue = this.getQueue(queueName as string);
-		const bullOptions = this.mapOptions(options);
+		if (options?.jobId && paramsList.length > 1) {
+			throw new Error("`jobId` cannot be shared across multiple jobs in enqueueMany()");
+		}
+
+		const queue = this.getQueue(queueName);
+		const jobs = await queue.addBulk(
+			paramsList.map(params => ({
+				name: "job",
+				data: JobSchemas[queueName].params.parse(params),
+				opts: this.mapOptions(options),
+			}))
+		);
+
+		return jobs.map(job => String(job.id));
+	}
+
+	async registerCron<K extends keyof JobRegistry>(config: RegisterCronConfig<K>): Promise<void> {
+		const { queueName, schedulerId, cronExpression, params, handler, options } = config;
 
-		const jobsToPush = paramsList.map(params => ({
-			name: "job",
-			data: JobSchemas[queueName].params.parse(params) as JobRegistry[K]["params"],
-			opts: bullOptions,
-		}));
+		const validated = JobSchemas[queueName].params.parse(params);
+		const queue = this.getQueue(queueName);
+
+		await queue.upsertJobScheduler(
+			schedulerId,
+			{ pattern: cronExpression },
+			{
+				name: "cron-job",
+				data: validated,
+				opts: this.mapOptions(options),
+			}
+		);
+
+		this.registerWorker(queueName, handler, options);
+	}
 
-		const jobs = await queue.addBulk(jobsToPush);
-		return jobs.map(j => j.id!);
+	async removeCron<K extends keyof JobRegistry>(queueName: K, schedulerId: string): Promise<boolean> {
+		const queue = this.getQueue(queueName);
+		return await queue.removeJobScheduler(schedulerId);
 	}
 
 	registerWorker<K extends keyof JobRegistry>(
 		queueName: K,
 		handler: JobHandler<JobRegistry[K]["params"], JobRegistry[K]["result"]>,
-		options: {
-			concurrency?: number;
-		}
+		options?: Omit<WorkerOptions, "connection">
 	): void {
-		logger.info(
-			`Registering worker for queue ${queueName} with options ${Object.entries(options)
-				.map(([k, v]) => `${k}: ${v}`)
-				.join(", ")}`
-		);
-		const worker = new Worker(
-			queueName as string,
-			async job => {
-				const validData = JobSchemas[queueName].params.parse(job.data) as JobRegistry[K]["params"];
+		if (this.workers.has(queueName)) {
+			return;
+		}
 
-				const context: JobContext = {
-					jobId: job.id!,
-					attemptsMade: job.attemptsMade,
-				};
+		logger.info(`Registering worker`, { queue: queueName, options });
 
-				const result = await handler(validData, context);
-				return result;
+		const worker = new Worker(
+			queueName,
+			async job => {
+				try {
+					const validData = JobSchemas[queueName].params.parse(job.data) as JobRegistry[K]["params"];
+					const context: JobContext = {
+						jobId: String(job.id),
+						attemptsMade: job.attemptsMade,
+					};
+					return await handler(validData, context);
+				} catch (error) {
+					logger.error(`Job execution failed`, {
+						queue: queueName,
+						jobId: job.id,
+						attemptsMade: job.attemptsMade,
+						data: job.data,
+						error,
+					});
+					throw error;
+				}
 			},
-			{ connection: this.connection, concurrency: options.concurrency }
+			{
+				connection: this.connection,
+				...options,
+			}
 		);
-		this.workers.push(worker);
+
+		worker.on("ready", () => logger.info(`Worker ready`, { queue: queueName }));
+		worker.on("active", job => logger.debug(`Job started`, { queue: queueName, jobId: job.id }));
+		worker.on("completed", job =>
+			logger.info(`Job completed`, { queue: queueName, jobId: job.id, attemptsMade: job.attemptsMade })
+		);
+		worker.on("failed", (job, error) =>
+			logger.error(`Job failed`, { queue: queueName, jobId: job?.id, attemptsMade: job?.attemptsMade, error })
+		);
+		worker.on("stalled", jobId => logger.warn(`Job stalled`, { queue: queueName, jobId }));
+		worker.on("closing", () => logger.warn(`Worker closing`, { queue: queueName }));
+		worker.on("closed", () => logger.warn(`Worker closed`, { queue: queueName }));
+		worker.on("error", error => logger.error(`Worker error`, { queue: queueName, error }));
+
+		void worker.client.then(client => {
+			client.on("ready", () => logger.info(`Redis ready`, { queue: queueName }));
+			client.on("reconnecting", () => logger.warn(`Redis reconnecting`, { queue: queueName }));
+			client.on("end", () => logger.error(`Redis connection ended`, { queue: queueName }));
+			client.on("error", error => logger.error(`Redis connection error`, { queue: queueName, error }));
+		});
+
+		this.workers.set(queueName, worker);
 	}
 
 	private getQueueEvents(queueName: string): QueueEvents {
 		if (!this.events.has(queueName)) {
-			this.events.set(queueName, new QueueEvents(queueName, { connection: this.connection }));
+			const events = new QueueEvents(queueName, {
+				connection: this.connection,
+			});
+
+			events.on("error", error => logger.error(`QueueEvents error`, { queue: queueName, error }));
+			this.events.set(queueName, events);
 		}
 		return this.events.get(queueName)!;
 	}
@@ -98,24 +174,39 @@ export class BullMQProvider implements IQueueProvider {
 		queueName: K,
 		callback: (jobId: string, result: JobRegistry[K]["result"]) => void
 	): void {
-		const events = this.getQueueEvents(queueName as string);
+		const events = this.getQueueEvents(queueName);
 		events.on("completed", ({ jobId, returnvalue }) => {
-			// Cast the Redis returnvalue to our strongly typed result
-			callback(jobId, returnvalue as JobRegistry[K]["result"]);
+			try {
+				callback(jobId, returnvalue as JobRegistry[K]["result"]);
+			} catch (error) {
+				logger.error("Job completion callback failed", { queue: queueName, jobId, error });
+			}
 		});
 	}
 
 	onJobFailed<K extends keyof JobRegistry>(queueName: K, callback: (jobId: string, error: Error) => void): void {
-		const events = this.getQueueEvents(queueName as string);
-		events.on("failed", ({ jobId, failedReason }) => callback(jobId, new Error(failedReason)));
+		const events = this.getQueueEvents(queueName);
+		events.on("failed", ({ jobId, failedReason }) => {
+			try {
+				callback(jobId, new Error(failedReason));
+			} catch (error) {
+				logger.error("Job failure callback failed", { queue: queueName, jobId, error });
+			}
+		});
 	}
 
 	async close(): Promise<void> {
 		logger.warn("Shutting down queue system...");
 
-		await Promise.all(this.workers.map(w => w.close()));
-		await Promise.all(Array.from(this.events.values()).map(e => e.close()));
-		await Promise.all(Array.from(this.queues.values()).map(q => q.close()));
+		const workerResults = await Promise.allSettled(Array.from(this.workers.values()).map(worker => worker.close()));
+		const eventResults = await Promise.allSettled(Array.from(this.events.values()).map(event => event.close()));
+		const queueResults = await Promise.allSettled(Array.from(this.queues.values()).map(queue => queue.close()));
+
+		[...workerResults, ...eventResults, ...queueResults]
+			.filter((result): result is PromiseRejectedResult => result.status === "rejected")
+			.forEach(result => {
+				logger.error("Error during shutdown", result.reason);
+			});
 
 		logger.warn("Queue system shut down safely.");
 	}

package/packages/queue/src/index.ts

@@ -1,5 +1,8 @@
 import { getRedis } from "@repo/redis";
 import { BullMQProvider } from "./adapter/bullmq-adapter.js";
+import { JobSchemas } from "./registry.js";
 
 const redisConnection = getRedis({ db: 10 });
+export const queueList = Object.keys(JobSchemas) as Array<keyof typeof JobSchemas>;
+
 export const BullQueue = new BullMQProvider(redisConnection);

package/packages/queue/src/queue-interface.ts

@@ -1,6 +1,14 @@
+import type { WorkerOptions } from "bullmq";
 import type { JobRegistry } from "./registry.js";
 import type { JobOptions, JobHandler } from "./types.js";
-
+export interface RegisterCronConfig<K extends keyof JobRegistry> {
+	queueName: K;
+	schedulerId: string;
+	cronExpression: string;
+	params: JobRegistry[K]["params"];
+	handler: JobHandler<JobRegistry[K]["params"], JobRegistry[K]["result"]>;
+	options?: Omit<JobOptions, "delay" | "jobId"> & Omit<WorkerOptions, "connection">;
+}
 export interface IQueueProvider {
 	enqueue<K extends keyof JobRegistry>(
 		queueName: K,
@@ -17,11 +25,10 @@ export interface IQueueProvider {
 	registerWorker<K extends keyof JobRegistry>(
 		queueName: K,
 		handler: JobHandler<JobRegistry[K]["params"], JobRegistry[K]["result"]>,
-		options: {
-			concurrency?: number;
-		}
+		options?: unknown
 	): void;
-
+	registerCron<K extends keyof JobRegistry>(config: RegisterCronConfig<K>): Promise<void>;
+	removeCron<K extends keyof JobRegistry>(queueName: K, schedulerId: string): Promise<boolean>;
 	onJobComplete<K extends keyof JobRegistry>(
 		queueName: K,
 		callback: (jobId: string, result: JobRegistry[K]["result"]) => void

package/packages/queue/src/registry.ts

@@ -1,8 +1,8 @@
 import { z } from "zod";
-import { emailQueue } from "./common/email-queue.js";
+import { commonQueue } from "./common/index.js";
 
 export const JobSchemas = {
-	...emailQueue,
+	...commonQueue,
 } as const;
 
 export type JobRegistry = {

package/packages/queue/tsconfig.json

@@ -1,7 +1,7 @@
 {
 	"extends": "typescript-config/tsconfig.node.json",
 	"compilerOptions": {
-		"baseUrl": "./src"
+		// "baseUrl": "./src"
 	},
 	"include": ["src/**/*.ts"]
 }

package/packages/ui/src/api/use-api/react-query/types.ts

@@ -2,9 +2,9 @@ import type { UseQueryResult, UseMutationResult } from "@tanstack/react-query";
 import type { z } from "zod";
 import type { IPaginatedData, TimeString } from "../api-types";
 
-export type ExtractData<E> = "data" extends keyof E ? E["data"] : unknown;
-export type ExtractZQuery<E> = "zQuery" extends keyof E ? E["zQuery"] : never;
-export type ExtractZParams<E> = "zParams" extends keyof E ? E["zParams"] : never;
+export type ExtractData<E> = "data" extends keyof E ? Exclude<E["data"], undefined> : unknown;
+export type ExtractZQuery<E> = "zQuery" extends keyof E ? NonNullable<E["zQuery"]> : never;
+export type ExtractZParams<E> = "zParams" extends keyof E ? NonNullable<E["zParams"]> : never;
 export type ExtractZBody<E> = "zBody" extends keyof E ? NonNullable<E["zBody"]> : never;
 
 export type WithAbort<T> = T & { abort: () => void };

package/packages/ui/src/api/use-api/react-query/use-api.ts

@@ -126,7 +126,6 @@ function useFetchApi<ID, ZQ extends ZQuery, ZP extends ZParams>(
 		abort: () => queryClient.cancelQueries({ queryKey }),
 	};
 }
-
 function usePaginatedFetchApi<ID, IO, ZQ extends ZQuery, ZP extends ZParams>(
 	key: keyof RawSchema,
 	endpoint: IPaginatedFetchOptions<ID, IO, ZQ, ZP>,
@@ -139,7 +138,7 @@ function usePaginatedFetchApi<ID, IO, ZQ extends ZQuery, ZP extends ZParams>(
 		staleTime?: number;
 		enabled?: boolean;
 	}
-): UseQueryResult<{ data: IPaginatedData<ID, IO> }> & { abort: () => void } {
+): UseQueryResult<IPaginatedData<ID, IO>> & { abort: () => void } {
 	const queryClient = useQueryClient();
 	const mergedQuery = useMemo(
 		() =>
@@ -161,10 +160,13 @@ function usePaginatedFetchApi<ID, IO, ZQ extends ZQuery, ZP extends ZParams>(
 
 	const staleTime = ApiHelper.parseTime(ApiHelper.merge(options.staleTime, endpoint.staleTime));
 
-	const query = useQuery<{ data: IPaginatedData<ID, IO> }>({
+	const query = useQuery<IPaginatedData<ID, IO>>({
 		queryKey,
 		queryFn: async ({ signal }) => {
-			const config: AxiosRequestConfig = {
+			const resp = await axiosClient.request<{
+				message: string;
+				data: IPaginatedData<ID, IO>;
+			}>({
 				method: endpoint.method || "GET",
 				url: ApiHelper.buildUrl({
 					route: endpoint.route,
@@ -172,13 +174,9 @@ function usePaginatedFetchApi<ID, IO, ZQ extends ZQuery, ZP extends ZParams>(
 					searchParams: mergedQuery,
 				}),
 				signal,
-			};
+			});
 
-			const resp = await axiosClient.request<{
-				data: IPaginatedData<ID, IO>;
-			}>(config);
-
-			return resp.data;
+			return resp.data.data;
 		},
 		staleTime,
 		enabled: options.enabled ?? endpoint.enable?.isEnable ?? true,
@@ -188,13 +186,14 @@ function usePaginatedFetchApi<ID, IO, ZQ extends ZQuery, ZP extends ZParams>(
 		refetchIntervalInBackground: endpoint.refetch?.interval?.inBackground,
 		refetchOnMount: endpoint.refetch?.onMount ? "always" : false,
 		refetchOnWindowFocus: endpoint.refetch?.onFocus ? "always" : false,
-	} as UseQueryOptions<{ data: IPaginatedData<ID, IO> }>);
+	});
 
 	return {
 		...query,
 		abort: () => queryClient.cancelQueries({ queryKey }),
 	};
 }
+
 function useMutationApi<Schema extends RawSchema, K extends keyof Schema & string>(
 	endpoint: IFetchMutationOptions<K>,
 	axiosClient: AxiosInstance,

package/pnpm-workspace.yaml

@@ -75,8 +75,8 @@ catalog:
   "@types/morgan": "^1.9.10"
 
   # db
-  drizzle-orm: "^0.45.1"
-  drizzle-kit: "^0.31.10"
+  drizzle-orm: "^1.0.0-beta.22"
+  drizzle-kit: "^1.0.0-beta.22"
   pg: "^8.19.0"
   better-sqlite3: "^12.9.0"
 
@@ -102,6 +102,8 @@ catalog:
 
   # bullmq
   bullmq: "^5.76.4"
+  "@bull-board/express": "^7.1.5"
+  "@bull-board/api": "^7.1.5"
 
   # redis
   ioredis: "^5.10.1"

package/turbo.json

@@ -69,6 +69,26 @@
 			"cache": false,
 			"interactive": true
 		},
+		"db:migrate": {
+			"cache": false,
+			"interactive": true
+		},
+		"db:up": {
+			"cache": false,
+			"interactive": true
+		},
+		"db:check": {
+			"cache": false,
+			"interactive": true
+		},
+		"db:pull": {
+			"cache": false,
+			"interactive": true
+		},
+		"db:generate": {
+			"cache": false,
+			"interactive": true
+		},
 		"db:studio": {
 			"cache": false,
 			"interactive": false,