te.js 2.1.6 → 2.2.1

package/radar/index.js

@@ -0,0 +1,382 @@
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createHash, randomUUID } from 'node:crypto';
+import { gzip } from 'node:zlib';
+import { promisify } from 'node:util';
+
+const gzipAsync = promisify(gzip);
+import { AsyncLocalStorage } from 'node:async_hooks';
+import TejLogger from 'tej-logger';
+
+const logger = new TejLogger('Tejas.Radar');
+
+/**
+ * AsyncLocalStorage instance for propagating trace context across async
+ * boundaries within a single request.  Middleware sets `{ traceId }` on
+ * entry; downstream code can read it via `traceStore.getStore()?.traceId`.
+ */
+export const traceStore = new AsyncLocalStorage();
+
+/**
+ * Attempt to read the `name` field from the nearest package.json at startup.
+ * Returns null if the file cannot be read or parsed.
+ * @returns {Promise<string|null>}
+ */
+async function readPackageJsonName() {
+  try {
+    const raw = await readFile(join(process.cwd(), 'package.json'), 'utf8');
+    return JSON.parse(raw).name ?? null;
+  } catch (err) {
+    logger.warn(`Could not read package.json name: ${err?.message ?? err}`);
+    return null;
+  }
+}
+
+/**
+ * Recursively walk a plain object/array and replace the value of any key whose
+ * lowercase name appears in `blocklist` with the string `"*"`.  Returns a new
+ * deep-cloned structure; the original is never mutated.
+ *
+ * Non-object values (strings, numbers, null, …) are returned as-is.
+ *
+ * @param {unknown} value
+ * @param {Set<string>} blocklist  Lower-cased field names to mask.
+ * @returns {unknown}
+ */
+function deepMask(value, blocklist) {
+  if (value === null || typeof value !== 'object') return value;
+
+  if (Array.isArray(value)) {
+    return value.map((item) => deepMask(item, blocklist));
+  }
+
+  const result = Object.create(null);
+  for (const [k, v] of Object.entries(value)) {
+    result[k] = blocklist.has(k.toLowerCase()) ? '*' : deepMask(v, blocklist);
+  }
+  return result;
+}
+
+/**
+ * Build the headers object to include in the metric record based on the
+ * `capture.headers` configuration value:
+ *   - `false`      → null (default; nothing sent)
+ *   - `true`       → shallow copy of all headers
+ *   - `string[]`   → object containing only the listed header names
+ *
+ * @param {Record<string, string>|undefined} rawHeaders
+ * @param {boolean|string[]} captureHeaders
+ * @returns {Record<string, string>|null}
+ */
+function buildHeaders(rawHeaders, captureHeaders) {
+  if (!captureHeaders || !rawHeaders) return null;
+  if (captureHeaders === true) return { ...rawHeaders };
+  return Object.fromEntries(
+    captureHeaders
+      .map((k) => [k, rawHeaders[k.toLowerCase()]])
+      .filter(([, v]) => v != null),
+  );
+}
+
+/**
+ * Attempt to parse a JSON string. Returns the parsed value on success, or
+ * `null` on failure.  Used for response bodies which may not always be JSON.
+ *
+ * @param {string|undefined|null} raw
+ * @returns {unknown}
+ */
+function parseJsonSafe(raw) {
+  if (!raw) return null;
+  try {
+    return JSON.parse(raw);
+  } catch (err) {
+    logger.warn(`parseJsonSafe: JSON parse failed — ${err?.message ?? err}`);
+    return null;
+  }
+}
+
+/**
+ * Factory that returns a te.js-compatible `(ammo, next)` middleware which
+ * captures HTTP request metrics and forwards them to the Tejas Radar collector.
+ *
+ * @param {Object} [config]
+ * @param {string} [config.apiKey]           Bearer token (rdr_xxx). Falls back to RADAR_API_KEY env. Required.
+ * @param {string} [config.projectName]      Project identifier. Falls back to RADAR_PROJECT_NAME env, then package.json `name`, then "tejas-app".
+ * @param {number} [config.flushInterval]    Milliseconds between periodic flushes (default 2000).
+ * @param {number} [config.batchSize]        Flush immediately when batch reaches this size (default 100).
+ * @param {number} [config.maxQueueSize]     Maximum events held in memory before oldest are dropped (default 10000).
+ * @param {Function} [config.transport]      Custom transport `(events) => Promise<{ok, status}>`.
+ *                                            Defaults to gzip-compressed HTTP POST to the collector.
+ * @param {string[]} [config.ignore]         Request paths to skip (default ['/health']).
+ * @param {Object}  [config.capture]         Controls what additional data is captured beyond metrics.
+ * @param {boolean} [config.capture.request]          Capture and send request body (default false).
+ * @param {boolean} [config.capture.response]         Capture and send response body (default false).
+ * @param {boolean|string[]} [config.capture.headers] Capture request headers. `true` sends all headers;
+ *                                                     a `string[]` sends only the named headers (allowlist);
+ *                                                     `false` (default) sends nothing.
+ * @param {Object}   [config.mask]           Client-side masking applied before data is sent.
+ * @param {string[]} [config.mask.fields]    Extra field names to mask in request/response bodies.
+ *                                            These are merged with the collector's server-side GDPR blocklist.
+ *                                            Note: the collector enforces its own non-bypassable masking
+ *                                            regardless of this setting.
+ * @returns {Promise<Function>} Middleware function `(ammo, next)`
+ */
+async function radarMiddleware(config = {}) {
+  // RADAR_COLLECTOR_URL is an undocumented internal escape hatch used only
+  // during local development. In production, telemetry always goes to the
+  // hosted collector and this env var should not be set.
+  const collectorUrl =
+    process.env.RADAR_COLLECTOR_URL ?? 'http://localhost:3100';
+
+  const apiKey = config.apiKey ?? process.env.RADAR_API_KEY ?? null;
+
+  const projectName =
+    config.projectName ??
+    process.env.RADAR_PROJECT_NAME ??
+    (await readPackageJsonName()) ??
+    'tejas-app';
+
+  const flushInterval = config.flushInterval ?? 2000;
+  const batchSize = config.batchSize ?? 100;
+  const maxQueueSize = config.maxQueueSize ?? 10_000;
+  const ignorePaths = new Set(config.ignore ?? ['/health']);
+
+  const capture = Object.freeze({
+    request: config.capture?.request === true,
+    response: config.capture?.response === true,
+    headers: config.capture?.headers ?? false,
+  });
+
+  // Build the client-side field blocklist from developer-supplied extra fields.
+  // The collector enforces its own non-bypassable GDPR blocklist server-side;
+  // this is an additional best-effort layer for application-specific fields.
+  const clientMaskBlocklist = new Set(
+    (config.mask?.fields ?? []).map((f) => f.toLowerCase()),
+  );
+
+  if (!apiKey) {
+    const mw = (_ammo, next) => next();
+    mw._radarStatus = {
+      feature: 'Radar',
+      ok: null,
+      detail: 'disabled (no API key)',
+    };
+    return mw;
+  }
+
+  const ingestUrl = `${collectorUrl}/ingest`;
+  const healthUrl = `${collectorUrl}/health`;
+  const authHeader = `Bearer ${apiKey}`;
+
+  /** @type {Array<Object>} */
+  let batch = [];
+  let connected = false;
+  let retryQueue = null;
+  let retryCount = 0;
+  const MAX_RETRIES = 3;
+
+  async function defaultHttpTransport(events) {
+    const json = JSON.stringify(events);
+    const compressed = await gzipAsync(Buffer.from(json));
+    return fetch(ingestUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Encoding': 'gzip',
+        Authorization: authHeader,
+      },
+      body: compressed,
+    });
+  }
+
+  const send = config.transport ?? defaultHttpTransport;
+
+  /** @type {{ feature: string, ok: boolean, detail: string }} */
+  let radarStatus;
+  try {
+    const healthRes = await fetch(healthUrl);
+    if (healthRes.ok) {
+      radarStatus = {
+        feature: 'Radar',
+        ok: true,
+        detail: `connected (${collectorUrl})`,
+      };
+    } else {
+      radarStatus = {
+        feature: 'Radar',
+        ok: false,
+        detail: `collector returned ${healthRes.status} (${collectorUrl})`,
+      };
+    }
+  } catch (err) {
+    radarStatus = {
+      feature: 'Radar',
+      ok: false,
+      detail: `unreachable (${collectorUrl})`,
+    };
+  }
+
+  async function sendPayload(payload) {
+    try {
+      const res = await send(payload);
+      if (res.ok) {
+        if (!connected) {
+          connected = true;
+          logger.info(
+            `Connected — project: "${projectName}", collector: ${collectorUrl}`,
+          );
+        }
+        return true;
+      }
+      if (res.status === 401) {
+        if (connected) connected = false;
+        logger.warn(
+          'Radar API key rejected by collector. Telemetry will not be recorded.',
+        );
+        return true;
+      }
+      return false;
+    } catch (err) {
+      logger.warn(`Radar flush failed: ${err.message}`);
+      return false;
+    }
+  }
+
+  async function flush() {
+    if (retryQueue) {
+      const ok = await sendPayload(retryQueue);
+      if (ok) {
+        retryQueue = null;
+        retryCount = 0;
+      } else {
+        retryCount++;
+        if (retryCount >= MAX_RETRIES) {
+          logger.warn(
+            `Radar dropping ${retryQueue.length} events after ${MAX_RETRIES} failed retries`,
+          );
+          retryQueue = null;
+          retryCount = 0;
+        }
+        return;
+      }
+    }
+
+    if (batch.length === 0) return;
+    const payload = batch;
+    batch = [];
+
+    const ok = await sendPayload(payload);
+    if (!ok) {
+      retryQueue = payload;
+      retryCount = 1;
+    }
+  }
+
+  const timer = setInterval(flush, flushInterval);
+  if (timer.unref) timer.unref();
+
+  function radarCapture(ammo, next) {
+    const startTime = Date.now();
+    const traceId = randomUUID().replace(/-/g, '');
+
+    ammo.res.on('finish', () => {
+      const path = ammo.endpoint ?? ammo.path ?? '/';
+
+      if (ammo.method === 'OPTIONS' || ignorePaths.has(path)) return;
+
+      const status = ammo.res.statusCode;
+      const endTimestamp = Date.now();
+      const duration = endTimestamp - startTime;
+      const payloadSize = Buffer.byteLength(
+        JSON.stringify(ammo.payload ?? {}),
+        'utf8',
+      );
+      const responseSize = Buffer.byteLength(ammo.dispatchedData ?? '', 'utf8');
+      const ip = ammo.ip ?? null;
+      const userAgent = ammo.headers?.['user-agent'] ?? null;
+      const headers = buildHeaders(ammo.headers, capture.headers);
+      const requestBody = capture.request
+        ? deepMask(ammo.payload ?? null, clientMaskBlocklist)
+        : null;
+      const responseBody = capture.response
+        ? deepMask(parseJsonSafe(ammo.dispatchedData), clientMaskBlocklist)
+        : null;
+
+      function pushEvents() {
+        const errorInfo = ammo._errorInfo ?? null;
+
+        let errorField = null;
+        if (status >= 400 && errorInfo) {
+          errorField = JSON.stringify({
+            message: errorInfo.message ?? null,
+            type: errorInfo.type ?? null,
+            devInsight: errorInfo.devInsight ?? null,
+          });
+        }
+
+        const incoming = status >= 400 ? 2 : 1;
+        if (batch.length + incoming > maxQueueSize) {
+          const overflow = batch.length + incoming - maxQueueSize;
+          batch.splice(0, overflow);
+        }
+
+        batch.push({
+          type: 'log',
+          projectName,
+          method: ammo.method,
+          path,
+          status,
+          duration_ms: duration,
+          payload_size: payloadSize,
+          response_size: responseSize,
+          timestamp: endTimestamp,
+          ip,
+          traceId,
+          user_agent: userAgent,
+          headers,
+          request_body: requestBody,
+          response_body: responseBody,
+          error: errorField,
+        });
+
+        if (status >= 400) {
+          const message = errorInfo?.message ?? `HTTP ${status}`;
+          const fingerprint = createHash('sha256')
+            .update(`${message}:${path}`)
+            .digest('hex');
+          batch.push({
+            type: 'error',
+            projectName,
+            fingerprint,
+            message,
+            stack: errorInfo?.stack ?? null,
+            endpoint: `${ammo.method} ${path}`,
+            traceId,
+            timestamp: endTimestamp,
+          });
+        }
+
+        if (batch.length >= batchSize) flush();
+      }
+
+      if (ammo._llmPromise) {
+        const timeout = new Promise((resolve) => {
+          const t = setTimeout(resolve, 30000);
+          if (t.unref) t.unref();
+        });
+        Promise.race([ammo._llmPromise, timeout])
+          .catch(() => {})
+          .then(pushEvents);
+      } else {
+        pushEvents();
+      }
+    });
+
+    traceStore.run({ traceId }, () => next());
+  }
+
+  radarCapture._radarStatus = radarStatus;
+  return radarCapture;
+}
+
+export default radarMiddleware;

package/rate-limit/base.js

@@ -1,7 +1,6 @@
 import TejError from '../server/error.js';
 import MemoryStorage from './storage/memory.js';
 import RedisStorage from './storage/redis.js';
-import dbManager from '../database/index.js';
 
 /**
  * Base rate limiter class implementing common functionality for rate limiting algorithms
@@ -15,11 +14,9 @@ import dbManager from '../database/index.js';
  * object is provided (tokenBucketConfig, slidingWindowConfig, or fixedWindowConfig).
  *
  * @example
- * // Using with Redis storage and token bucket algorithm
  * const limiter = new TokenBucketRateLimiter({
  *   maxRequests: 10,
  *   timeWindowSeconds: 60,
- *   store: 'redis',
  *   tokenBucketConfig: {
  *     refillRate: 0.5,
  *     burstSize: 15
@@ -40,7 +37,9 @@ class RateLimiter {
    *                                                 For token bucket, this affects the default refill rate calculation.
    * @param {string} [options.keyPrefix='rl:'] - Prefix for storage keys. Useful when implementing different rate limit
    *                                           rules with different prefixes (e.g., 'rl:api:', 'rl:web:').
-   * @param {string} [options.store='memory'] - Storage backend to use ('memory' or 'redis')
+   * @param {string|Object} [options.store='memory'] - Storage backend: 'memory' (default) or
+   *   { type: 'redis', url: 'redis://...', ...redisOptions }.
+   *   In-memory storage is not shared across processes; use Redis for distributed deployments.
    * @param {Object} [options.tokenBucketConfig] - Token bucket algorithm specific options
    * @param {Object} [options.slidingWindowConfig] - Sliding window algorithm specific options
    * @param {Object} [options.fixedWindowConfig] - Fixed window algorithm specific options
@@ -101,18 +100,16 @@ class RateLimiter {
         }
       : null;
 
-    // Initialize storage based on store type
-    if (this.options.store === 'redis') {
-      if (!dbManager.hasConnection('redis')) {
-        throw new TejError(
-          500,
-          'Redis store selected but no Redis connection available. Call withRedis() first.',
-        );
-      }
-      const redisClient = dbManager.getConnection('redis');
-      this.storage = new RedisStorage(redisClient);
-    } else {
+    const store = this.options.store;
+    if (!store || store === 'memory') {
       this.storage = new MemoryStorage();
+    } else if (typeof store === 'object' && store.type === 'redis') {
+      this.storage = new RedisStorage(store);
+    } else {
+      throw new TejError(
+        400,
+        `Invalid store config. Use 'memory' or { type: 'redis', url: '...' }.`,
+      );
     }
   }
 

package/rate-limit/index.js

@@ -2,7 +2,6 @@ import TejError from '../server/error.js';
 import FixedWindowRateLimiter from './algorithms/fixed-window.js';
 import SlidingWindowRateLimiter from './algorithms/sliding-window.js';
 import TokenBucketRateLimiter from './algorithms/token-bucket.js';
-import dbManager from '../database/index.js';
 
 /**
  * Creates a rate limiting middleware function with the specified algorithm and storage
@@ -14,9 +13,11 @@ import dbManager from '../database/index.js';
  *                                                     - 'token-bucket': Best for handling traffic bursts
  *                                                     - 'sliding-window': Best for smooth rate limiting
  *                                                     - 'fixed-window': Simplest approach
- * @param {string} [options.store='memory'] - Storage backend to use:
- *                                         - 'memory': In-memory storage (default)
- *                                         - 'redis': Redis-based storage (requires global Redis config)
+ * @param {string|Object} [options.store='memory'] - Storage backend to use:
+ *                                         - 'memory': In-memory storage (default, single-instance only)
+ *                                         - { type: 'redis', url: 'redis://...' }: Redis storage for distributed deployments.
+ *                                           The redis npm package is auto-installed on first use.
+ *                                           Any extra properties are forwarded to node-redis createClient.
  * @param {Object} [options.algorithmOptions] - Algorithm-specific options
  * @param {Function} [options.keyGenerator] - Optional function to generate unique identifiers
  * @param {Object} [options.headerFormat] - Rate limit header format configuration
@@ -39,36 +40,32 @@ function rateLimiter(options) {
     ...limiterOptions
   } = options;
 
-  // Check Redis connectivity if Redis store is selected
-  if (store === 'redis' && !dbManager.hasConnection('redis', {})) {
+  const configMap = Object.create(null);
+  configMap['token-bucket'] = 'tokenBucketConfig';
+  configMap['sliding-window'] = 'slidingWindowConfig';
+  configMap['fixed-window'] = 'fixedWindowConfig';
+
+  const configKey = configMap[algorithm];
+  if (!configKey) {
     throw new TejError(
       400,
-      'Redis store selected but no Redis connection found. Please use withRedis() before using withRateLimit()',
+      `Invalid algorithm: ${algorithm}. Must be one of: ${Object.keys(configMap).join(', ')}`,
     );
   }
 
-  // Map algorithm names to their config property names
-  const configMap = {
-    'token-bucket': 'tokenBucketConfig',
-    'sliding-window': 'slidingWindowConfig',
-    'fixed-window': 'fixedWindowConfig',
-  };
-
-  const configKey = configMap[algorithm];
-  if (!configKey) {
+  if (typeof store === 'object' && store.type === 'redis' && !store.url) {
     throw new TejError(
       400,
-      `Invalid algorithm: ${algorithm}. Must be one of: ${Object.keys(configMap).join(', ')}`,
+      `Redis store requires a url. Provide store: { type: "redis", url: "redis://..." }`,
     );
   }
 
-  // Create algorithm-specific config
-  const limiterConfig = {
+  const limiterConfig = Object.freeze({
     maxRequests: limiterOptions.maxRequests,
     timeWindowSeconds: limiterOptions.timeWindowSeconds,
     [configKey]: limiterOptions.algorithmOptions || {},
-    store, // Pass the store type to the limiter
-  };
+    store,
+  });
 
   // Create the appropriate limiter instance
   let limiter;
@@ -128,7 +125,7 @@ function rateLimiter(options) {
 
   // Return middleware function
   return async (ammo, next) => {
-    const key = keyGenerator(ammo);
+    const key = keyGenerator(ammo) ?? ammo.ip ?? 'unknown';
     const result = await limiter.consume(key);
 
     setRateLimitHeaders(ammo, result);

package/rate-limit/index.test.js

@@ -0,0 +1,93 @@
+/**
+ * @fileoverview Tests for the rate limiter middleware factory.
+ */
+import { describe, it, expect, vi } from 'vitest';
+import rateLimiter from './index.js';
+
+function makeAmmo(ip = '127.0.0.1') {
+  return {
+    ip,
+    res: {
+      setHeader: vi.fn(),
+    },
+    throw: vi.fn(),
+  };
+}
+
+describe('rateLimiter', () => {
+  it('should throw on invalid algorithm', () => {
+    expect(() =>
+      rateLimiter({
+        maxRequests: 10,
+        timeWindowSeconds: 60,
+        algorithm: 'invalid-algo',
+      }),
+    ).toThrow();
+  });
+
+  it('should return a middleware function', () => {
+    const mw = rateLimiter({ maxRequests: 100, timeWindowSeconds: 60 });
+    expect(typeof mw).toBe('function');
+  });
+
+  it('should call next when under limit', async () => {
+    const mw = rateLimiter({ maxRequests: 100, timeWindowSeconds: 60 });
+    const ammo = makeAmmo();
+    const next = vi.fn().mockResolvedValue(undefined);
+    await mw(ammo, next);
+    expect(next).toHaveBeenCalled();
+  });
+
+  it('should use fallback key "unknown" when ammo.ip is undefined', async () => {
+    const mw = rateLimiter({ maxRequests: 100, timeWindowSeconds: 60 });
+    const ammo = makeAmmo(undefined);
+    const next = vi.fn().mockResolvedValue(undefined);
+    await mw(ammo, next);
+    expect(next).toHaveBeenCalled();
+  });
+
+  it('should throw on invalid store config', () => {
+    expect(() =>
+      rateLimiter({
+        maxRequests: 10,
+        timeWindowSeconds: 60,
+        store: 'postgres',
+      }),
+    ).toThrow(/Invalid store config/);
+  });
+
+  it('should throw when redis store has no url', () => {
+    expect(() =>
+      rateLimiter({
+        maxRequests: 10,
+        timeWindowSeconds: 60,
+        store: { type: 'redis' },
+      }),
+    ).toThrow(/requires a url/);
+  });
+
+  it('should accept redis store config without throwing on creation', () => {
+    vi.mock('./storage/redis.js', () => ({
+      default: class MockRedisStorage {
+        async get() {
+          return null;
+        }
+        async set() {}
+        async increment() {
+          return null;
+        }
+        async delete() {}
+      },
+    }));
+
+    expect(() =>
+      rateLimiter({
+        maxRequests: 10,
+        timeWindowSeconds: 60,
+        store: { type: 'redis', url: 'redis://localhost:6379' },
+      }),
+    ).not.toThrow();
+
+    vi.restoreAllMocks();
+  });
+});