servcraft 0.1.0 → 0.1.3

package/src/modules/queue/types.ts

@@ -0,0 +1,222 @@
+export type JobStatus = 'waiting' | 'active' | 'completed' | 'failed' | 'delayed' | 'paused';
+
+export type JobPriority = 'low' | 'normal' | 'high' | 'critical';
+
+export interface JobOptions {
+  /** Job priority */
+  priority?: JobPriority;
+  /** Delay in milliseconds before job starts */
+  delay?: number;
+  /** Number of retry attempts */
+  attempts?: number;
+  /** Backoff strategy for retries */
+  backoff?: {
+    type: 'fixed' | 'exponential';
+    delay: number;
+  };
+  /** Remove job on completion */
+  removeOnComplete?: boolean | number;
+  /** Remove job on failure */
+  removeOnFail?: boolean | number;
+  /** Job timeout in milliseconds */
+  timeout?: number;
+  /** Repeat job on cron schedule */
+  repeat?: {
+    cron?: string;
+    every?: number;
+    limit?: number;
+    immediately?: boolean;
+  };
+}
+
+export interface Job<T = any> {
+  /** Unique job ID */
+  id: string;
+  /** Queue name */
+  queueName: string;
+  /** Job name/type */
+  name: string;
+  /** Job data */
+  data: T;
+  /** Job options */
+  options: JobOptions;
+  /** Job status */
+  status: JobStatus;
+  /** Progress (0-100) */
+  progress?: number;
+  /** Number of attempts made */
+  attemptsMade: number;
+  /** Job result if completed */
+  result?: any;
+  /** Error if failed */
+  error?: string;
+  /** Stack trace if failed */
+  stacktrace?: string[];
+  /** Creation timestamp */
+  createdAt: Date;
+  /** Processing start timestamp */
+  processedAt?: Date;
+  /** Completion timestamp */
+  completedAt?: Date;
+  /** Failure timestamp */
+  failedAt?: Date;
+  /** Delay until (for delayed jobs) */
+  delayedUntil?: Date;
+}
+
+export interface QueueConfig {
+  /** Redis connection options */
+  redis?: {
+    host?: string;
+    port?: number;
+    password?: string;
+    db?: number;
+  };
+  /** Default job options */
+  defaultJobOptions?: JobOptions;
+  /** Queue prefix */
+  prefix?: string;
+  /** Enable metrics */
+  metrics?: boolean;
+}
+
+export interface QueueStats {
+  /** Queue name */
+  name: string;
+  /** Waiting jobs count */
+  waiting: number;
+  /** Active jobs count */
+  active: number;
+  /** Completed jobs count */
+  completed: number;
+  /** Failed jobs count */
+  failed: number;
+  /** Delayed jobs count */
+  delayed: number;
+  /** Paused jobs count */
+  paused: number;
+}
+
+export interface Worker<T = any> {
+  /** Worker name/type */
+  name: string;
+  /** Worker function */
+  process: (job: Job<T>) => Promise<any>;
+  /** Concurrency (parallel jobs) */
+  concurrency?: number;
+}
+
+export interface CronJob {
+  /** Unique cron job ID */
+  id: string;
+  /** Cron job name */
+  name: string;
+  /** Cron expression */
+  cron: string;
+  /** Job data */
+  data?: any;
+  /** Queue to add job to */
+  queueName: string;
+  /** Job name/type */
+  jobName: string;
+  /** Whether the cron is enabled */
+  enabled: boolean;
+  /** Last run timestamp */
+  lastRun?: Date;
+  /** Next run timestamp */
+  nextRun?: Date;
+  /** Creation timestamp */
+  createdAt: Date;
+}
+
+export interface EmailJobData {
+  to: string | string[];
+  subject: string;
+  html?: string;
+  text?: string;
+  from?: string;
+  cc?: string | string[];
+  bcc?: string | string[];
+  attachments?: Array<{
+    filename: string;
+    path?: string;
+    content?: Buffer | string;
+  }>;
+  template?: {
+    name: string;
+    data: Record<string, any>;
+  };
+}
+
+export interface ImageProcessingJobData {
+  /** Source image path or URL */
+  source: string;
+  /** Operations to perform */
+  operations: Array<{
+    type: 'resize' | 'crop' | 'rotate' | 'watermark' | 'compress';
+    options: Record<string, any>;
+  }>;
+  /** Output path */
+  output: string;
+  /** Output format */
+  format?: 'jpg' | 'png' | 'webp' | 'avif';
+  /** Quality (1-100) */
+  quality?: number;
+}
+
+export interface NotificationJobData {
+  userId: string;
+  type: 'push' | 'sms' | 'email';
+  title?: string;
+  message: string;
+  data?: Record<string, any>;
+}
+
+export interface WebhookJobData {
+  url: string;
+  method?: 'POST' | 'PUT' | 'PATCH';
+  headers?: Record<string, string>;
+  body: Record<string, any>;
+  retries?: number;
+}
+
+export interface QueueMetrics {
+  /** Total jobs processed */
+  totalProcessed: number;
+  /** Total jobs failed */
+  totalFailed: number;
+  /** Average processing time (ms) */
+  avgProcessingTime: number;
+  /** Jobs per minute */
+  throughput: number;
+  /** Peak concurrency */
+  peakConcurrency: number;
+  /** Current concurrency */
+  currentConcurrency: number;
+  /** Success rate (0-100) */
+  successRate: number;
+}
+
+export interface BulkJobOptions {
+  jobs: Array<{
+    name: string;
+    data: any;
+    opts?: JobOptions;
+  }>;
+}
+
+export interface JobFilter {
+  status?: JobStatus | JobStatus[];
+  name?: string;
+  limit?: number;
+  offset?: number;
+  startDate?: Date;
+  endDate?: Date;
+}
+
+export interface JobEvent {
+  event: 'added' | 'active' | 'completed' | 'failed' | 'progress' | 'stalled' | 'removed';
+  jobId: string;
+  data?: any;
+  timestamp: Date;
+}

package/src/modules/queue/workers.ts

@@ -0,0 +1,366 @@
+import { logger } from '../../core/logger.js';
+import type {
+  Worker,
+  Job,
+  EmailJobData,
+  ImageProcessingJobData,
+  NotificationJobData,
+  WebhookJobData,
+} from './types.js';
+
+/**
+ * Email Worker
+ * Sends emails asynchronously
+ */
+export const emailWorker: Worker<EmailJobData> = {
+  name: 'send-email',
+  concurrency: 5,
+  async process(job: Job<EmailJobData>): Promise<{ messageId: string }> {
+    const { to, subject } = job.data;
+
+    logger.info({ jobId: job.id, to, subject }, 'Processing email job');
+
+    // Simulate email sending
+    // In production, integrate with your email service (nodemailer, SendGrid, etc.)
+    await new Promise((resolve) => setTimeout(resolve, 500));
+
+    const messageId = `<${Date.now()}@servcraft.local>`;
+
+    logger.info({ jobId: job.id, messageId, to }, 'Email sent successfully');
+
+    return { messageId };
+  },
+};
+
+/**
+ * Image Processing Worker
+ * Processes images (resize, crop, watermark, etc.)
+ */
+export const imageProcessingWorker: Worker<ImageProcessingJobData> = {
+  name: 'process-image',
+  concurrency: 3,
+  async process(job: Job<ImageProcessingJobData>): Promise<{ output: string; size: number }> {
+    const { source, operations, output } = job.data;
+
+    logger.info({ jobId: job.id, source, operations: operations.length }, 'Processing image job');
+
+    // Simulate image processing
+    // In production, use Sharp, Jimp, or similar library
+    for (const operation of operations) {
+      logger.debug(
+        { jobId: job.id, operation: operation.type },
+        `Applying ${operation.type} operation`
+      );
+      await new Promise((resolve) => setTimeout(resolve, 200));
+    }
+
+    const outputPath = output;
+    const fileSize = Math.floor(Math.random() * 1000000); // Mock file size
+
+    logger.info({ jobId: job.id, output: outputPath, size: fileSize }, 'Image processed');
+
+    return { output: outputPath, size: fileSize };
+  },
+};
+
+/**
+ * Notification Worker
+ * Sends push notifications, SMS, or emails
+ */
+export const notificationWorker: Worker<NotificationJobData> = {
+  name: 'send-notification',
+  concurrency: 10,
+  async process(job: Job<NotificationJobData>): Promise<{ sent: boolean; notificationId: string }> {
+    const { userId, type } = job.data;
+
+    logger.info({ jobId: job.id, userId, type }, 'Processing notification job');
+
+    // Simulate notification sending
+    // In production, integrate with FCM, SNS, Twilio, etc.
+    await new Promise((resolve) => setTimeout(resolve, 300));
+
+    const notificationId = `notif_${Date.now()}`;
+
+    logger.info({ jobId: job.id, userId, notificationId }, 'Notification sent');
+
+    return { sent: true, notificationId };
+  },
+};
+
+/**
+ * Webhook Worker
+ * Sends HTTP webhooks to external URLs
+ */
+export const webhookWorker: Worker<WebhookJobData> = {
+  name: 'send-webhook',
+  concurrency: 5,
+  async process(job: Job<WebhookJobData>): Promise<{ statusCode: number; response: unknown }> {
+    const { url, method = 'POST', headers = {}, body } = job.data;
+
+    logger.info({ jobId: job.id, url, method }, 'Processing webhook job');
+
+    try {
+      const response = await fetch(url, {
+        method,
+        headers: {
+          'Content-Type': 'application/json',
+          ...headers,
+        },
+        body: JSON.stringify(body),
+      });
+
+      const responseData = await response.text().catch(() => '');
+
+      logger.info({ jobId: job.id, url, statusCode: response.status }, 'Webhook sent successfully');
+
+      return {
+        statusCode: response.status,
+        response: responseData,
+      };
+    } catch (error) {
+      logger.error({ jobId: job.id, url, error }, 'Webhook failed');
+      throw error;
+    }
+  },
+};
+
+/**
+ * Data Export Worker
+ * Exports data to CSV, Excel, PDF, etc.
+ */
+export const dataExportWorker: Worker<{
+  type: 'csv' | 'excel' | 'pdf';
+  data: unknown[];
+  filename: string;
+  options?: Record<string, unknown>;
+}> = {
+  name: 'export-data',
+  concurrency: 2,
+  async process(job): Promise<{ filename: string; size: number; url: string }> {
+    const { type, data, filename } = job.data;
+
+    logger.info({ jobId: job.id, type, rows: data.length, filename }, 'Processing export job');
+
+    // Simulate data export
+    // In production, use libraries like csv-writer, exceljs, pdfkit
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+
+    const outputPath = `/exports/${filename}`;
+    const fileSize = data.length * 100; // Mock file size
+    const downloadUrl = `https://example.com${outputPath}`;
+
+    logger.info({ jobId: job.id, filename, size: fileSize }, 'Data exported');
+
+    return { filename: outputPath, size: fileSize, url: downloadUrl };
+  },
+};
+
+/**
+ * Report Generation Worker
+ * Generates reports (analytics, financial, etc.)
+ */
+export const reportGenerationWorker: Worker<{
+  reportType: string;
+  parameters: Record<string, unknown>;
+  format: 'pdf' | 'html' | 'excel';
+}> = {
+  name: 'generate-report',
+  concurrency: 2,
+  async process(job): Promise<{ reportUrl: string; generatedAt: Date }> {
+    const { reportType, format } = job.data;
+
+    logger.info({ jobId: job.id, reportType, format }, 'Processing report generation job');
+
+    // Simulate report generation
+    // In production, integrate with your reporting engine
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+
+    const reportUrl = `/reports/${reportType}_${Date.now()}.${format}`;
+
+    logger.info({ jobId: job.id, reportType, reportUrl }, 'Report generated');
+
+    return { reportUrl, generatedAt: new Date() };
+  },
+};
+
+/**
+ * Database Backup Worker
+ * Creates database backups
+ */
+export const databaseBackupWorker: Worker<{
+  databases: string[];
+  destination: string;
+  compress?: boolean;
+}> = {
+  name: 'database-backup',
+  concurrency: 1, // Only one backup at a time
+  async process(
+    job
+  ): Promise<{ backups: Array<{ database: string; size: number; path: string }> }> {
+    const { databases, destination, compress = true } = job.data;
+
+    logger.info(
+      { jobId: job.id, databases, destination, compress },
+      'Processing database backup job'
+    );
+
+    const backups = [];
+
+    for (const database of databases) {
+      logger.debug({ jobId: job.id, database }, `Backing up database: ${database}`);
+
+      // Simulate backup
+      await new Promise((resolve) => setTimeout(resolve, 3000));
+
+      const backupPath = `${destination}/${database}_${Date.now()}.sql${compress ? '.gz' : ''}`;
+      const size = Math.floor(Math.random() * 50000000); // Mock size
+
+      backups.push({ database, size, path: backupPath });
+
+      logger.info({ jobId: job.id, database, size, path: backupPath }, 'Database backed up');
+    }
+
+    return { backups };
+  },
+};
+
+/**
+ * Cache Warming Worker
+ * Warms up cache with frequently accessed data
+ */
+export const cacheWarmingWorker: Worker<{
+  keys: string[];
+  ttl?: number;
+}> = {
+  name: 'warm-cache',
+  concurrency: 5,
+  async process(job): Promise<{ warmed: number; failed: number }> {
+    const { keys, ttl = 3600 } = job.data;
+
+    logger.info({ jobId: job.id, keys: keys.length, ttl }, 'Processing cache warming job');
+
+    let warmed = 0;
+    let failed = 0;
+
+    for (const key of keys) {
+      try {
+        // Simulate cache warming
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        warmed++;
+      } catch (error) {
+        failed++;
+        logger.warn({ jobId: job.id, key, error }, 'Failed to warm cache key');
+      }
+    }
+
+    logger.info({ jobId: job.id, warmed, failed }, 'Cache warming completed');
+
+    return { warmed, failed };
+  },
+};
+
+/**
+ * Data Cleanup Worker
+ * Cleans up old data (logs, temp files, etc.)
+ */
+export const dataCleanupWorker: Worker<{
+  targets: Array<{
+    type: 'database' | 'files' | 'logs';
+    table?: string;
+    directory?: string;
+    olderThanDays: number;
+  }>;
+}> = {
+  name: 'cleanup-data',
+  concurrency: 1,
+  async process(job): Promise<{ cleaned: Array<{ type: string; count: number; size: number }> }> {
+    const { targets } = job.data;
+
+    logger.info({ jobId: job.id, targets: targets.length }, 'Processing data cleanup job');
+
+    const cleaned = [];
+
+    for (const target of targets) {
+      logger.debug({ jobId: job.id, target }, `Cleaning up ${target.type}`);
+
+      // Simulate cleanup
+      await new Promise((resolve) => setTimeout(resolve, 500));
+
+      const count = Math.floor(Math.random() * 1000);
+      const size = count * 1024; // Mock size
+
+      cleaned.push({
+        type: target.type,
+        count,
+        size,
+      });
+
+      logger.info({ jobId: job.id, target: target.type, count, size }, 'Cleanup completed');
+    }
+
+    return { cleaned };
+  },
+};
+
+/**
+ * Batch Processing Worker
+ * Processes items in batches
+ */
+export const batchProcessingWorker: Worker<{
+  items: unknown[];
+  batchSize: number;
+  processor: 'validate' | 'transform' | 'import';
+}> = {
+  name: 'batch-process',
+  concurrency: 3,
+  async process(job): Promise<{ processed: number; failed: number; errors: unknown[] }> {
+    const { items, batchSize, processor } = job.data;
+
+    logger.info(
+      { jobId: job.id, items: items.length, batchSize, processor },
+      'Processing batch job'
+    );
+
+    let processed = 0;
+    let failed = 0;
+    const errors = [];
+
+    for (let i = 0; i < items.length; i += batchSize) {
+      const batch = items.slice(i, i + batchSize);
+
+      try {
+        // Simulate batch processing
+        await new Promise((resolve) => setTimeout(resolve, 200));
+        processed += batch.length;
+
+        // Update progress
+        const progress = Math.floor(((i + batch.length) / items.length) * 100);
+        logger.debug({ jobId: job.id, progress }, `Batch progress: ${progress}%`);
+      } catch (error) {
+        failed += batch.length;
+        errors.push({ batch: i / batchSize, error });
+        logger.warn({ jobId: job.id, batch: i / batchSize, error }, 'Batch failed');
+      }
+    }
+
+    logger.info({ jobId: job.id, processed, failed }, 'Batch processing completed');
+
+    return { processed, failed, errors };
+  },
+};
+
+/**
+ * Export all default workers
+ */
+export const defaultWorkers = [
+  emailWorker,
+  imageProcessingWorker,
+  notificationWorker,
+  webhookWorker,
+  dataExportWorker,
+  reportGenerationWorker,
+  databaseBackupWorker,
+  cacheWarmingWorker,
+  dataCleanupWorker,
+  batchProcessingWorker,
+];

package/src/modules/rate-limit/index.ts

@@ -0,0 +1,59 @@
+/**
+ * Rate Limiting Module
+ *
+ * Provides advanced rate limiting capabilities with multiple algorithms:
+ * - Fixed Window: Simple, but can allow bursts at boundaries
+ * - Sliding Window: More accurate, prevents boundary bursts
+ * - Token Bucket: Allows controlled bursts while enforcing average rate
+ *
+ * Features:
+ * - IP-based, user-based, or custom key rate limiting
+ * - Whitelist/blacklist support
+ * - Custom limits per endpoint or user role
+ * - Memory or Redis storage
+ * - Admin routes for management
+ * - Standard rate limit headers
+ *
+ * @example
+ * ```typescript
+ * import { createRateLimiter, RateLimitService } from './modules/rate-limit';
+ *
+ * // Basic usage
+ * app.use(createRateLimiter({
+ *   max: 100,
+ *   windowMs: 60 * 1000, // 1 minute
+ *   algorithm: 'sliding-window'
+ * }));
+ *
+ * // Pre-configured limiters
+ * import { strictRateLimit, authRateLimit } from './modules/rate-limit';
+ * app.post('/login', authRateLimit, loginHandler);
+ * app.post('/sensitive', strictRateLimit, sensitiveHandler);
+ * ```
+ */
+
+// Types
+export * from './types.js';
+
+// Service
+export { RateLimitService } from './rate-limit.service.js';
+
+// Middleware
+export {
+  createRateLimiter,
+  rateLimitEndpoint,
+  strictRateLimit,
+  standardRateLimit,
+  relaxedRateLimit,
+  authRateLimit,
+  ipRateLimit,
+  userRateLimit,
+  apiKeyRateLimit,
+} from './rate-limit.middleware.js';
+
+// Routes
+export { createRateLimitRoutes } from './rate-limit.routes.js';
+
+// Stores
+export { MemoryStore } from './stores/memory.store.js';
+export { RedisStore } from './stores/redis.store.js';