trickle-observe 0.1.0

package/src/proxy-tracker.ts

@@ -0,0 +1,208 @@
+import { TypeNode } from './types';
+import { inferType } from './type-inference';
+
+/**
+ * Creates a deep Proxy around a value to track property accesses.
+ * Returns the proxy and a function to retrieve all accessed paths with their inferred types.
+ *
+ * The proxy is designed to be fully transparent:
+ * - Array.isArray() returns true for proxied arrays
+ * - JSON.stringify works correctly
+ * - typeof, ===, iteration, spread, Object.keys all work identically
+ * - Symbol.toPrimitive, Symbol.iterator, Symbol.toStringTag are forwarded
+ */
+export function createTracker(value: unknown): {
+  proxy: unknown;
+  getAccessedPaths: () => Map<string, TypeNode>;
+} {
+  const accessedPaths = new Map<string, TypeNode>();
+  const proxyCache = new WeakMap<object, unknown>();
+
+  function wrap(val: unknown, path: string): unknown {
+    // Only proxy objects and arrays (not primitives or functions at top level)
+    if (val === null || val === undefined) return val;
+    if (typeof val !== 'object' && typeof val !== 'function') return val;
+
+    const obj = val as object;
+
+    // Check cache to prevent double-wrapping
+    if (proxyCache.has(obj)) {
+      return proxyCache.get(obj);
+    }
+
+    // For arrays, the proxy target must be an array so Array.isArray() works
+    const target = Array.isArray(obj) ? obj : obj;
+
+    const proxy = new Proxy(target, {
+      get(target: any, prop: string | symbol, receiver: any): any {
+        // Forward well-known symbols transparently
+        if (typeof prop === 'symbol') {
+          // Forward Symbol.toPrimitive, Symbol.iterator, Symbol.toStringTag, Symbol.hasInstance
+          const raw = Reflect.get(target, prop, target);
+          // For iterator, bind to target so iteration works on original
+          if (prop === Symbol.iterator && typeof raw === 'function') {
+            return raw.bind(target);
+          }
+          return raw;
+        }
+
+        const raw = Reflect.get(target, prop, target);
+
+        // Don't track internal/meta properties
+        if (prop === 'constructor' || prop === 'prototype' || prop === '__proto__') {
+          return raw;
+        }
+
+        // toJSON: return original value's toJSON or the target itself for JSON.stringify
+        if (prop === 'toJSON') {
+          if (typeof raw === 'function') {
+            return raw.bind(target);
+          }
+          return raw;
+        }
+
+        // For arrays, forward array methods transparently
+        if (Array.isArray(target)) {
+          if (prop === 'length') {
+            // Record that length was accessed
+            const fullPath = path ? `${path}.length` : 'length';
+            accessedPaths.set(fullPath, { kind: 'primitive', name: 'number' });
+            return raw;
+          }
+
+          // Array index access
+          if (isArrayIndex(prop)) {
+            const fullPath = path ? `${path}[${prop}]` : `[${prop}]`;
+            const val = raw;
+            if (val !== undefined) {
+              accessedPaths.set(fullPath, inferType(val, 3));
+            }
+            // Recursively wrap object elements
+            if (val !== null && val !== undefined && typeof val === 'object') {
+              return wrap(val, fullPath);
+            }
+            return val;
+          }
+
+          // Array methods that take callbacks - wrap to track callback args
+          if (typeof raw === 'function') {
+            if (isCallbackMethod(prop)) {
+              return createTrackedArrayMethod(target, prop, path, raw, accessedPaths, wrap);
+            }
+            // Other array methods: bind to original
+            return raw.bind(target);
+          }
+        }
+
+        // Record the access path and type
+        const fullPath = path ? `${path}.${prop}` : prop;
+        if (raw !== undefined) {
+          accessedPaths.set(fullPath, inferType(raw, 3));
+        }
+
+        // Recursively wrap sub-objects
+        if (raw !== null && raw !== undefined && typeof raw === 'object') {
+          return wrap(raw, fullPath);
+        }
+
+        // Bind functions to original target
+        if (typeof raw === 'function') {
+          return raw.bind(target);
+        }
+
+        return raw;
+      },
+
+      set(target: any, prop: string | symbol, value: any, receiver: any): boolean {
+        return Reflect.set(target, prop, value, target);
+      },
+
+      has(target: any, prop: string | symbol): boolean {
+        return Reflect.has(target, prop);
+      },
+
+      ownKeys(target: any): (string | symbol)[] {
+        return Reflect.ownKeys(target);
+      },
+
+      getOwnPropertyDescriptor(target: any, prop: string | symbol): PropertyDescriptor | undefined {
+        return Reflect.getOwnPropertyDescriptor(target, prop);
+      },
+
+      getPrototypeOf(target: any): object | null {
+        return Reflect.getPrototypeOf(target);
+      },
+
+      isExtensible(target: any): boolean {
+        return Reflect.isExtensible(target);
+      },
+
+      preventExtensions(target: any): boolean {
+        return Reflect.preventExtensions(target);
+      },
+
+      defineProperty(target: any, prop: string | symbol, descriptor: PropertyDescriptor): boolean {
+        return Reflect.defineProperty(target, prop, descriptor);
+      },
+
+      deleteProperty(target: any, prop: string | symbol): boolean {
+        return Reflect.deleteProperty(target, prop);
+      },
+    });
+
+    proxyCache.set(obj, proxy);
+    return proxy;
+  }
+
+  const proxy = wrap(value, '');
+  return {
+    proxy,
+    getAccessedPaths: () => new Map(accessedPaths),
+  };
+}
+
+function isArrayIndex(prop: string): boolean {
+  const num = Number(prop);
+  return Number.isInteger(num) && num >= 0 && String(num) === prop;
+}
+
+const CALLBACK_METHODS = new Set([
+  'map', 'filter', 'forEach', 'find', 'findIndex', 'some', 'every',
+  'reduce', 'reduceRight', 'flatMap', 'sort',
+]);
+
+function isCallbackMethod(prop: string): boolean {
+  return CALLBACK_METHODS.has(prop);
+}
+
+/**
+ * Creates a tracked version of an array method that takes a callback.
+ * Wraps callback arguments (the element) in proxies so property accesses within
+ * the callback are also tracked.
+ */
+function createTrackedArrayMethod(
+  target: any[],
+  method: string,
+  basePath: string,
+  rawFn: Function,
+  accessedPaths: Map<string, TypeNode>,
+  wrap: (val: unknown, path: string) => unknown,
+): Function {
+  return function (this: any, ...args: any[]) {
+    if (args.length > 0 && typeof args[0] === 'function') {
+      const originalCb = args[0];
+      args[0] = function (element: any, index: number, array: any[]) {
+        const elementPath = basePath ? `${basePath}[${index}]` : `[${index}]`;
+        // Wrap the element so accesses inside the callback are tracked
+        let wrappedElement = element;
+        if (element !== null && element !== undefined && typeof element === 'object') {
+          wrappedElement = wrap(element, elementPath);
+        } else if (element !== undefined) {
+          accessedPaths.set(elementPath, inferType(element, 3));
+        }
+        return originalCb.call(this, wrappedElement, index, array);
+      };
+    }
+    return rawFn.apply(target, args);
+  };
+}

package/src/register.ts

@@ -0,0 +1,110 @@
+/**
+ * Zero-code auto-instrumentation for Node.js applications.
+ *
+ * Usage — just add a flag to your start command:
+ *
+ *   node -r trickle/register app.js
+ *
+ * Or with environment variables:
+ *
+ *   TRICKLE_BACKEND_URL=http://localhost:4888 node -r trickle/register app.js
+ *
+ * This module patches Node's module loader to intercept `require('express')`
+ * and automatically instrument any Express app created — no code changes needed.
+ *
+ * Supported environment variables:
+ *   TRICKLE_BACKEND_URL  — Backend URL (default: http://localhost:4888)
+ *   TRICKLE_ENABLED      — Set to "0" or "false" to disable (default: enabled)
+ *   TRICKLE_DEBUG        — Set to "1" or "true" for debug logging
+ *   TRICKLE_ENV          — Override detected environment name
+ */
+
+import Module from 'module';
+import { configure } from './transport';
+import { instrumentExpress } from './express';
+import { detectEnvironment } from './env-detect';
+
+const M = Module as any;
+const originalLoad = M._load;
+const patched = new Set<string>();
+
+// Read config from environment
+const backendUrl = process.env.TRICKLE_BACKEND_URL || 'http://localhost:4888';
+const enabled = process.env.TRICKLE_ENABLED !== '0' && process.env.TRICKLE_ENABLED !== 'false';
+const debug = process.env.TRICKLE_DEBUG === '1' || process.env.TRICKLE_DEBUG === 'true';
+const envOverride = process.env.TRICKLE_ENV || undefined;
+
+if (enabled) {
+  // Configure the transport
+  configure({
+    backendUrl,
+    batchIntervalMs: 2000,
+    debug,
+    enabled: true,
+    environment: envOverride || detectEnvironment(),
+  });
+
+  if (debug) {
+    console.log(`[trickle] Auto-instrumentation enabled (backend: ${backendUrl})`);
+  }
+
+  // Patch Module._load to intercept framework requires
+  M._load = function hookedLoad(request: string, parent: any, isMain: boolean): any {
+    const exports = originalLoad.apply(this, arguments);
+
+    // Intercept require('express')
+    if (request === 'express' && !patched.has('express')) {
+      patched.add('express');
+      return patchExpress(exports, request, parent);
+    }
+
+    return exports;
+  };
+} else if (debug) {
+  console.log('[trickle] Auto-instrumentation disabled (TRICKLE_ENABLED=false)');
+}
+
+/**
+ * Wrap the Express factory function so every app created is auto-instrumented.
+ * Preserves all static properties (express.json, express.static, etc.).
+ */
+function patchExpress(originalExpress: any, request: string, parent: any): any {
+  function wrappedExpress(this: any, ...args: any[]): any {
+    const app = originalExpress.apply(this, args);
+    try {
+      instrumentExpress(app, {
+        environment: envOverride || detectEnvironment(),
+      });
+      if (debug) {
+        console.log('[trickle] Auto-instrumented Express app');
+      }
+    } catch (err: unknown) {
+      if (debug) {
+        const msg = err instanceof Error ? err.message : String(err);
+        console.warn(`[trickle] Failed to auto-instrument Express: ${msg}`);
+      }
+    }
+    return app;
+  }
+
+  // Copy all static properties (express.json, express.static, express.Router, etc.)
+  for (const key of Object.keys(originalExpress)) {
+    (wrappedExpress as any)[key] = originalExpress[key];
+  }
+
+  // Preserve prototype chain
+  Object.setPrototypeOf(wrappedExpress, Object.getPrototypeOf(originalExpress));
+
+  // Update require cache so subsequent require('express') returns the patched version
+  try {
+    const resolvedPath = M._resolveFilename(request, parent);
+    if (require.cache[resolvedPath]) {
+      require.cache[resolvedPath]!.exports = wrappedExpress;
+    }
+  } catch {
+    // Cache update failed — first require still returns patched, but subsequent
+    // requires from other modules may get the original. This is rare.
+  }
+
+  return wrappedExpress;
+}

package/src/transport.ts

@@ -0,0 +1,207 @@
+import * as fs from 'fs';
+import * as pathMod from 'path';
+import { IngestPayload, GlobalOpts } from './types';
+
+const MAX_RETRIES = 3;
+const INITIAL_RETRY_DELAY_MS = 1000;
+const DEFAULT_BATCH_INTERVAL_MS = 2000;
+const DEFAULT_MAX_BATCH_SIZE = 50;
+
+let backendUrl = 'http://localhost:4888';
+let batchIntervalMs = DEFAULT_BATCH_INTERVAL_MS;
+let maxBatchSize = DEFAULT_MAX_BATCH_SIZE;
+let enabled = true;
+let debug = false;
+let localMode = process.env.TRICKLE_LOCAL === '1';
+let localFilePath = '';
+
+let queue: IngestPayload[] = [];
+let flushTimer: ReturnType<typeof setInterval> | null = null;
+let isFlushing = false;
+
+/**
+ * Configure the transport layer with global options.
+ */
+export function configure(opts: GlobalOpts): void {
+  backendUrl = opts.backendUrl || backendUrl;
+  batchIntervalMs = opts.batchIntervalMs || DEFAULT_BATCH_INTERVAL_MS;
+  maxBatchSize = opts.maxBatchSize || DEFAULT_MAX_BATCH_SIZE;
+  enabled = opts.enabled !== false;
+  debug = opts.debug === true;
+
+  // Check for local/file-based mode
+  if (process.env.TRICKLE_LOCAL === '1') {
+    localMode = true;
+    const dir = process.env.TRICKLE_LOCAL_DIR || pathMod.join(process.cwd(), '.trickle');
+    try { fs.mkdirSync(dir, { recursive: true }); } catch {}
+    localFilePath = pathMod.join(dir, 'observations.jsonl');
+    if (debug) {
+      console.log(`[trickle] Local mode: writing to ${localFilePath}`);
+    }
+    return; // no timer needed for file mode
+  }
+
+  // Restart the flush timer with new interval
+  stopTimer();
+  startTimer();
+}
+
+/**
+ * Enqueue a payload for batched sending.
+ */
+export function enqueue(payload: IngestPayload): void {
+  if (!enabled) return;
+
+  // Local file mode: append directly to JSONL file
+  if (localMode && localFilePath) {
+    try {
+      fs.appendFileSync(localFilePath, JSON.stringify(payload) + '\n');
+    } catch {
+      // Never crash user's app
+    }
+    return;
+  }
+
+  queue.push(payload);
+
+  // Flush immediately if batch is full
+  if (queue.length >= maxBatchSize) {
+    flush().catch(silentError);
+  }
+
+  // Ensure timer is running
+  if (!flushTimer) {
+    startTimer();
+  }
+}
+
+/**
+ * Flush all queued payloads to the backend.
+ * Returns a promise that resolves when the flush completes.
+ */
+export async function flush(): Promise<void> {
+  if (queue.length === 0) return;
+  if (isFlushing) return;
+
+  isFlushing = true;
+  const batch = queue.splice(0);
+
+  try {
+    await sendBatch(batch);
+  } catch {
+    // Batch is already dropped after max retries — nothing more to do
+    if (debug) {
+      console.warn('[trickle] Failed to flush batch, data dropped');
+    }
+  } finally {
+    isFlushing = false;
+  }
+}
+
+/**
+ * Send a batch with exponential backoff retry.
+ */
+async function sendBatch(batch: IngestPayload[]): Promise<void> {
+  let delay = INITIAL_RETRY_DELAY_MS;
+
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      const response = await fetch(`${backendUrl}/api/ingest/batch`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ payloads: batch }),
+        signal: AbortSignal.timeout(10000), // 10 second timeout
+      });
+
+      if (response.ok) {
+        if (debug) {
+          console.log(`[trickle] Sent batch of ${batch.length} events`);
+        }
+        return;
+      }
+
+      // Server error — retry
+      if (response.status >= 500 && attempt < MAX_RETRIES) {
+        await sleep(delay);
+        delay *= 2;
+        continue;
+      }
+
+      // Client error (4xx) or final attempt — drop
+      if (debug) {
+        console.warn(`[trickle] Backend returned ${response.status}, dropping batch`);
+      }
+      return;
+    } catch (err: unknown) {
+      if (attempt < MAX_RETRIES) {
+        await sleep(delay);
+        delay *= 2;
+        continue;
+      }
+
+      // Final attempt failed
+      if (debug) {
+        const msg = err instanceof Error ? err.message : String(err);
+        console.warn(`[trickle] Could not reach backend after ${MAX_RETRIES + 1} attempts: ${msg}`);
+      }
+      return;
+    }
+  }
+}
+
+function startTimer(): void {
+  if (flushTimer) return;
+  flushTimer = setInterval(() => {
+    flush().catch(silentError);
+  }, batchIntervalMs);
+
+  // Don't keep the process alive just for trickle
+  if (flushTimer && typeof flushTimer === 'object' && 'unref' in flushTimer) {
+    flushTimer.unref();
+  }
+}
+
+function stopTimer(): void {
+  if (flushTimer) {
+    clearInterval(flushTimer);
+    flushTimer = null;
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function silentError(): void {
+  // Intentionally empty — never crash user's app
+}
+
+// Register process exit handler to flush remaining events
+if (typeof process !== 'undefined' && process.on) {
+  process.on('beforeExit', () => {
+    flush().catch(silentError);
+  });
+
+  // SIGTERM / SIGINT: attempt a sync-ish flush
+  const exitHandler = () => {
+    flush().catch(silentError);
+  };
+  process.on('SIGTERM', exitHandler);
+  process.on('SIGINT', exitHandler);
+}
+
+/**
+ * Get the current queue length (for testing/debugging).
+ */
+export function getQueueLength(): number {
+  return queue.length;
+}
+
+/**
+ * Reset the transport state (for testing).
+ */
+export function reset(): void {
+  queue = [];
+  stopTimer();
+  isFlushing = false;
+}

package/src/type-hash.ts

@@ -0,0 +1,71 @@
+import { createHash } from 'crypto';
+import { TypeNode } from './types';
+
+/**
+ * Canonicalize a TypeNode for deterministic hashing.
+ * - Object properties are sorted alphabetically by key at all levels.
+ * - Union members are sorted by their canonical string representation.
+ */
+function canonicalize(node: TypeNode): unknown {
+  switch (node.kind) {
+    case 'primitive':
+      return { kind: 'primitive', name: node.name };
+
+    case 'array':
+      return { kind: 'array', element: canonicalize(node.element) };
+
+    case 'object': {
+      const sortedKeys = Object.keys(node.properties).sort();
+      const properties: Record<string, unknown> = {};
+      for (const key of sortedKeys) {
+        properties[key] = canonicalize(node.properties[key]);
+      }
+      return { kind: 'object', properties };
+    }
+
+    case 'union': {
+      const members = node.members
+        .map(m => canonicalize(m))
+        .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)));
+      return { kind: 'union', members };
+    }
+
+    case 'function': {
+      return {
+        kind: 'function',
+        params: node.params.map(canonicalize),
+        returnType: canonicalize(node.returnType),
+      };
+    }
+
+    case 'promise':
+      return { kind: 'promise', resolved: canonicalize(node.resolved) };
+
+    case 'map':
+      return { kind: 'map', key: canonicalize(node.key), value: canonicalize(node.value) };
+
+    case 'set':
+      return { kind: 'set', element: canonicalize(node.element) };
+
+    case 'tuple':
+      return { kind: 'tuple', elements: node.elements.map(canonicalize) };
+
+    case 'unknown':
+      return { kind: 'unknown' };
+
+    default:
+      return { kind: 'unknown' };
+  }
+}
+
+/**
+ * Hash the combined args + return type into a deterministic 16-hex-char string.
+ */
+export function hashType(argsType: TypeNode, returnType: TypeNode): string {
+  const canonical = JSON.stringify({
+    args: canonicalize(argsType),
+    ret: canonicalize(returnType),
+  });
+
+  return createHash('sha256').update(canonical).digest('hex').substring(0, 16);
+}