trickle-backend 0.1.0

package/dist/types.d.ts

@@ -0,0 +1,56 @@
+export type TypeNode = {
+    kind: "primitive";
+    name: "string" | "number" | "boolean" | "null" | "undefined" | "bigint" | "symbol";
+} | {
+    kind: "array";
+    element: TypeNode;
+} | {
+    kind: "object";
+    properties: Record<string, TypeNode>;
+} | {
+    kind: "union";
+    members: TypeNode[];
+} | {
+    kind: "function";
+    params: TypeNode[];
+    returnType: TypeNode;
+} | {
+    kind: "promise";
+    resolved: TypeNode;
+} | {
+    kind: "map";
+    key: TypeNode;
+    value: TypeNode;
+} | {
+    kind: "set";
+    element: TypeNode;
+} | {
+    kind: "tuple";
+    elements: TypeNode[];
+} | {
+    kind: "unknown";
+};
+export interface IngestPayload {
+    functionName: string;
+    module: string;
+    language: "js" | "python";
+    environment: string;
+    typeHash: string;
+    argsType: TypeNode;
+    returnType: TypeNode;
+    sampleInput?: unknown;
+    sampleOutput?: unknown;
+    error?: {
+        type: string;
+        message: string;
+        stackTrace?: string;
+        argsSnapshot?: unknown;
+    };
+}
+export interface TypeDiff {
+    kind: "added" | "removed" | "changed";
+    path: string;
+    from?: TypeNode;
+    to?: TypeNode;
+    type?: TypeNode;
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "trickle-backend",
+  "version": "0.1.0",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "tsc",
+    "dev": "ts-node src/index.ts",
+    "start": "node dist/index.js"
+  },
+  "dependencies": {
+    "better-sqlite3": "^11.0.0",
+    "cors": "^2.8.5",
+    "express": "^4.21.0"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.0",
+    "@types/cors": "^2.8.0",
+    "@types/express": "^4.17.0",
+    "ts-node": "^10.9.0",
+    "typescript": "^5.4.0"
+  }
+}

package/src/db/connection.ts

@@ -0,0 +1,16 @@
+import path from "path";
+import fs from "fs";
+import Database, { Database as DatabaseType } from "better-sqlite3";
+
+const trickleDir = path.join(process.env.HOME || "~", ".trickle");
+
+fs.mkdirSync(trickleDir, { recursive: true });
+
+const dbPath = path.join(trickleDir, "trickle.db");
+
+const db: DatabaseType = new Database(dbPath);
+
+db.pragma("journal_mode = WAL");
+db.pragma("foreign_keys = ON");
+
+export { db };

package/src/db/migrations.ts

@@ -0,0 +1,50 @@
+import type Database from "better-sqlite3";
+
+export function runMigrations(db: Database.Database): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS functions (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      function_name TEXT NOT NULL,
+      module TEXT NOT NULL,
+      environment TEXT NOT NULL DEFAULT 'unknown',
+      language TEXT NOT NULL,
+      first_seen_at TEXT NOT NULL DEFAULT (datetime('now')),
+      last_seen_at TEXT NOT NULL DEFAULT (datetime('now')),
+      UNIQUE(function_name, module, language)
+    );
+
+    CREATE TABLE IF NOT EXISTS type_snapshots (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      function_id INTEGER NOT NULL REFERENCES functions(id),
+      type_hash TEXT NOT NULL,
+      args_type TEXT NOT NULL,
+      return_type TEXT NOT NULL,
+      variables_type TEXT,
+      sample_input TEXT,
+      sample_output TEXT,
+      env TEXT NOT NULL DEFAULT 'unknown',
+      observed_at TEXT NOT NULL DEFAULT (datetime('now')),
+      UNIQUE(function_id, type_hash, env)
+    );
+
+    CREATE TABLE IF NOT EXISTS errors (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      function_id INTEGER NOT NULL REFERENCES functions(id),
+      error_type TEXT NOT NULL,
+      error_message TEXT NOT NULL,
+      stack_trace TEXT,
+      args_type TEXT,
+      return_type TEXT,
+      variables_type TEXT,
+      args_snapshot TEXT,
+      type_hash TEXT,
+      env TEXT NOT NULL DEFAULT 'unknown',
+      occurred_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_snapshots_function_id ON type_snapshots(function_id);
+    CREATE INDEX IF NOT EXISTS idx_errors_function_id ON errors(function_id);
+    CREATE INDEX IF NOT EXISTS idx_errors_occurred_at ON errors(occurred_at);
+    CREATE INDEX IF NOT EXISTS idx_errors_env ON errors(env);
+  `);
+}

package/src/db/queries.ts

@@ -0,0 +1,260 @@
+import type Database from "better-sqlite3";
+
+// --- Functions ---
+
+export function upsertFunction(
+  db: Database.Database,
+  params: { functionName: string; module: string; environment: string; language: string }
+) {
+  const insertStmt = db.prepare(`
+    INSERT OR IGNORE INTO functions (function_name, module, environment, language)
+    VALUES (@functionName, @module, @environment, @language)
+  `);
+  const updateStmt = db.prepare(`
+    UPDATE functions
+    SET last_seen_at = datetime('now'), environment = @environment
+    WHERE function_name = @functionName AND module = @module AND language = @language
+  `);
+  const selectStmt = db.prepare(`
+    SELECT * FROM functions
+    WHERE function_name = @functionName AND module = @module AND language = @language
+  `);
+
+  insertStmt.run(params);
+  updateStmt.run(params);
+  return selectStmt.get(params) as Record<string, unknown>;
+}
+
+// --- Type Snapshots ---
+
+export function findSnapshotByHash(
+  db: Database.Database,
+  functionId: number,
+  typeHash: string,
+  env: string
+) {
+  const stmt = db.prepare(`
+    SELECT * FROM type_snapshots
+    WHERE function_id = ? AND type_hash = ? AND env = ?
+  `);
+  return stmt.get(functionId, typeHash, env) as Record<string, unknown> | undefined;
+}
+
+export function insertSnapshot(
+  db: Database.Database,
+  params: {
+    functionId: number;
+    typeHash: string;
+    argsType: string;
+    returnType: string;
+    variablesType?: string;
+    sampleInput?: string;
+    sampleOutput?: string;
+    env: string;
+  }
+) {
+  const stmt = db.prepare(`
+    INSERT INTO type_snapshots (function_id, type_hash, args_type, return_type, variables_type, sample_input, sample_output, env)
+    VALUES (@functionId, @typeHash, @argsType, @returnType, @variablesType, @sampleInput, @sampleOutput, @env)
+  `);
+  const result = stmt.run(params);
+  return { id: result.lastInsertRowid, ...params };
+}
+
+// --- Errors ---
+
+export function insertError(
+  db: Database.Database,
+  params: {
+    functionId: number;
+    errorType: string;
+    errorMessage: string;
+    stackTrace?: string;
+    argsType?: string;
+    returnType?: string;
+    variablesType?: string;
+    argsSnapshot?: string;
+    typeHash?: string;
+    env: string;
+  }
+) {
+  const stmt = db.prepare(`
+    INSERT INTO errors (function_id, error_type, error_message, stack_trace, args_type, return_type, variables_type, args_snapshot, type_hash, env)
+    VALUES (@functionId, @errorType, @errorMessage, @stackTrace, @argsType, @returnType, @variablesType, @argsSnapshot, @typeHash, @env)
+  `);
+  const result = stmt.run(params);
+
+  const selectStmt = db.prepare(`SELECT * FROM errors WHERE id = ?`);
+  return selectStmt.get(result.lastInsertRowid) as Record<string, unknown>;
+}
+
+// --- List / Get Functions ---
+
+export function listFunctions(
+  db: Database.Database,
+  params: { search?: string; env?: string; language?: string; limit?: number; offset?: number }
+) {
+  const conditions: string[] = [];
+  const bindings: unknown[] = [];
+
+  if (params.search) {
+    conditions.push("(function_name LIKE ? OR module LIKE ?)");
+    bindings.push(`%${params.search}%`, `%${params.search}%`);
+  }
+  if (params.env) {
+    conditions.push("environment = ?");
+    bindings.push(params.env);
+  }
+  if (params.language) {
+    conditions.push("language = ?");
+    bindings.push(params.language);
+  }
+
+  const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+  const limit = params.limit ?? 50;
+  const offset = params.offset ?? 0;
+
+  const stmt = db.prepare(`
+    SELECT * FROM functions ${where}
+    ORDER BY last_seen_at DESC
+    LIMIT ? OFFSET ?
+  `);
+  bindings.push(limit, offset);
+
+  const countStmt = db.prepare(`SELECT COUNT(*) as total FROM functions ${where}`);
+  const countBindings = bindings.slice(0, bindings.length - 2);
+
+  const rows = stmt.all(...bindings) as Record<string, unknown>[];
+  const total = (countStmt.get(...countBindings) as { total: number }).total;
+
+  return { rows, total };
+}
+
+export function getFunction(db: Database.Database, id: number) {
+  const stmt = db.prepare(`SELECT * FROM functions WHERE id = ?`);
+  return stmt.get(id) as Record<string, unknown> | undefined;
+}
+
+export function getFunctionByName(db: Database.Database, functionName: string) {
+  const stmt = db.prepare(`SELECT * FROM functions WHERE function_name LIKE ?`);
+  return stmt.all(`%${functionName}%`) as Record<string, unknown>[];
+}
+
+// --- List / Get Snapshots ---
+
+export function listSnapshots(
+  db: Database.Database,
+  params: { functionId: number; env?: string; limit?: number }
+) {
+  const conditions: string[] = ["function_id = ?"];
+  const bindings: unknown[] = [params.functionId];
+
+  if (params.env) {
+    conditions.push("env = ?");
+    bindings.push(params.env);
+  }
+
+  const limit = params.limit ?? 50;
+  const where = conditions.join(" AND ");
+
+  const stmt = db.prepare(`
+    SELECT * FROM type_snapshots
+    WHERE ${where}
+    ORDER BY observed_at DESC
+    LIMIT ?
+  `);
+  bindings.push(limit);
+
+  return stmt.all(...bindings) as Record<string, unknown>[];
+}
+
+export function getLatestSnapshot(db: Database.Database, functionId: number, env?: string) {
+  if (env) {
+    const stmt = db.prepare(`
+      SELECT * FROM type_snapshots
+      WHERE function_id = ? AND env = ?
+      ORDER BY observed_at DESC
+      LIMIT 1
+    `);
+    return stmt.get(functionId, env) as Record<string, unknown> | undefined;
+  }
+
+  const stmt = db.prepare(`
+    SELECT * FROM type_snapshots
+    WHERE function_id = ?
+    ORDER BY observed_at DESC
+    LIMIT 1
+  `);
+  return stmt.get(functionId) as Record<string, unknown> | undefined;
+}
+
+// --- List / Get Errors ---
+
+export function listErrors(
+  db: Database.Database,
+  params: {
+    functionId?: number;
+    functionName?: string;
+    env?: string;
+    since?: string;
+    limit?: number;
+    offset?: number;
+  }
+) {
+  const conditions: string[] = [];
+  const bindings: unknown[] = [];
+
+  if (params.functionId) {
+    conditions.push("e.function_id = ?");
+    bindings.push(params.functionId);
+  }
+  if (params.functionName) {
+    conditions.push("f.function_name LIKE ?");
+    bindings.push(`%${params.functionName}%`);
+  }
+  if (params.env) {
+    conditions.push("e.env = ?");
+    bindings.push(params.env);
+  }
+  if (params.since) {
+    conditions.push("e.occurred_at >= ?");
+    bindings.push(params.since);
+  }
+
+  const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+  const limit = params.limit ?? 50;
+  const offset = params.offset ?? 0;
+
+  const stmt = db.prepare(`
+    SELECT e.*, f.function_name, f.module, f.language
+    FROM errors e
+    JOIN functions f ON f.id = e.function_id
+    ${where}
+    ORDER BY e.occurred_at DESC
+    LIMIT ? OFFSET ?
+  `);
+  bindings.push(limit, offset);
+
+  const countStmt = db.prepare(`
+    SELECT COUNT(*) as total
+    FROM errors e
+    JOIN functions f ON f.id = e.function_id
+    ${where}
+  `);
+  const countBindings = bindings.slice(0, bindings.length - 2);
+
+  const rows = stmt.all(...bindings) as Record<string, unknown>[];
+  const total = (countStmt.get(...countBindings) as { total: number }).total;
+
+  return { rows, total };
+}
+
+export function getError(db: Database.Database, id: number) {
+  const stmt = db.prepare(`
+    SELECT e.*, f.function_name, f.module, f.language
+    FROM errors e
+    JOIN functions f ON f.id = e.function_id
+    WHERE e.id = ?
+  `);
+  return stmt.get(id) as Record<string, unknown> | undefined;
+}

package/src/index.ts

@@ -0,0 +1,11 @@
+import { app } from "./server";
+import { db } from "./db/connection";
+import { runMigrations } from "./db/migrations";
+
+runMigrations(db);
+
+const PORT = parseInt(process.env.PORT || "4888", 10);
+
+app.listen(PORT, () => {
+  console.log(`[trickle] Backend listening on http://localhost:${PORT}`);
+});

package/src/routes/audit.ts

@@ -0,0 +1,283 @@
+import { Router, Request, Response } from "express";
+import { db } from "../db/connection";
+import { listFunctions, getLatestSnapshot } from "../db/queries";
+import { TypeNode } from "../types";
+
+const router = Router();
+
+interface AuditIssue {
+  severity: "error" | "warning" | "info";
+  rule: string;
+  message: string;
+  route?: string;
+  field?: string;
+}
+
+const SENSITIVE_FIELD_NAMES = new Set([
+  "password", "passwd", "pass", "secret", "token", "apikey", "api_key",
+  "apiKey", "accesstoken", "access_token", "accessToken", "refreshtoken",
+  "refresh_token", "refreshToken", "privatekey", "private_key", "privateKey",
+  "ssn", "creditcard", "credit_card", "creditCard", "cvv", "pin",
+]);
+
+function tryParseJson(value: string): unknown {
+  try {
+    return JSON.parse(value);
+  } catch {
+    return value;
+  }
+}
+
+/** Count properties in a TypeNode recursively */
+function countProperties(node: TypeNode): number {
+  if (node.kind === "object") {
+    return Object.keys(node.properties).length;
+  }
+  return 0;
+}
+
+/** Get max nesting depth of a TypeNode */
+function maxDepth(node: TypeNode, current: number = 0): number {
+  switch (node.kind) {
+    case "object": {
+      if (Object.keys(node.properties).length === 0) return current;
+      let max = current;
+      for (const val of Object.values(node.properties)) {
+        max = Math.max(max, maxDepth(val, current + 1));
+      }
+      return max;
+    }
+    case "array":
+      return maxDepth(node.element, current + 1);
+    case "union":
+      return Math.max(...node.members.map((m) => maxDepth(m, current)), current);
+    case "tuple":
+      return Math.max(...node.elements.map((e) => maxDepth(e, current + 1)), current);
+    case "map":
+      return maxDepth(node.value, current + 1);
+    case "set":
+      return maxDepth(node.element, current + 1);
+    case "promise":
+      return maxDepth(node.resolved, current);
+    default:
+      return current;
+  }
+}
+
+/** Check if an object mixes camelCase and snake_case */
+function detectNamingInconsistency(node: TypeNode, path: string): string | null {
+  if (node.kind !== "object") return null;
+  const keys = Object.keys(node.properties);
+  if (keys.length < 2) return null;
+
+  let hasCamel = false;
+  let hasSnake = false;
+  for (const key of keys) {
+    if (key.includes("_") && key !== key.toUpperCase()) hasSnake = true;
+    if (/[a-z][A-Z]/.test(key)) hasCamel = true;
+  }
+  if (hasCamel && hasSnake) {
+    const camelKeys = keys.filter((k) => /[a-z][A-Z]/.test(k));
+    const snakeKeys = keys.filter((k) => k.includes("_") && k !== k.toUpperCase());
+    return `Mixed naming: camelCase (${camelKeys.slice(0, 3).join(", ")}) and snake_case (${snakeKeys.slice(0, 3).join(", ")})`;
+  }
+  return null;
+}
+
+/** Find sensitive fields in a TypeNode recursively */
+function findSensitiveFields(node: TypeNode, path: string, results: string[]): void {
+  if (node.kind === "object") {
+    for (const [key, val] of Object.entries(node.properties)) {
+      const fullPath = path ? `${path}.${key}` : key;
+      if (SENSITIVE_FIELD_NAMES.has(key.toLowerCase())) {
+        results.push(fullPath);
+      }
+      findSensitiveFields(val, fullPath, results);
+    }
+  } else if (node.kind === "array") {
+    findSensitiveFields(node.element, `${path}[]`, results);
+  } else if (node.kind === "union") {
+    for (const m of node.members) {
+      findSensitiveFields(m, path, results);
+    }
+  }
+}
+
+/** Collect all field names with their types across routes */
+function collectFieldTypes(
+  node: TypeNode,
+  prefix: string,
+  result: Map<string, Set<string>>,
+): void {
+  if (node.kind === "object") {
+    for (const [key, val] of Object.entries(node.properties)) {
+      const fullKey = prefix ? `${prefix}.${key}` : key;
+      if (!result.has(key)) result.set(key, new Set());
+      if (val.kind === "primitive") {
+        result.get(key)!.add(val.name);
+      } else {
+        result.get(key)!.add(val.kind);
+      }
+      collectFieldTypes(val, fullKey, result);
+    }
+  } else if (node.kind === "array") {
+    collectFieldTypes(node.element, `${prefix}[]`, result);
+  }
+}
+
+router.get("/", (req: Request, res: Response) => {
+  try {
+    const { env } = req.query;
+    const issues: AuditIssue[] = [];
+
+    // Collect all functions with their types
+    const listed = listFunctions(db, {
+      env: env as string | undefined,
+      limit: 500,
+    });
+
+    const functions: Array<{
+      name: string;
+      argsType: TypeNode;
+      returnType: TypeNode;
+    }> = [];
+
+    for (const fn of listed.rows) {
+      const functionId = fn.id as number;
+      const functionName = fn.function_name as string;
+
+      const snapshot = getLatestSnapshot(db, functionId, env as string | undefined);
+      if (!snapshot) {
+        // Untyped function
+        issues.push({
+          severity: "warning",
+          rule: "no-types",
+          message: `No type observations recorded`,
+          route: functionName,
+        });
+        continue;
+      }
+
+      const argsType = tryParseJson(snapshot.args_type as string) as TypeNode;
+      const returnType = tryParseJson(snapshot.return_type as string) as TypeNode;
+      if (!argsType || !returnType) continue;
+
+      functions.push({ name: functionName, argsType, returnType });
+    }
+
+    // Analyze each function
+    const globalFieldTypes = new Map<string, Set<string>>();
+
+    for (const fn of functions) {
+      const routeName = fn.name;
+
+      // 1. Sensitive data in responses
+      const sensitiveFields: string[] = [];
+      findSensitiveFields(fn.returnType, "", sensitiveFields);
+      for (const field of sensitiveFields) {
+        issues.push({
+          severity: "error",
+          rule: "sensitive-data",
+          message: `Response exposes potentially sensitive field "${field}"`,
+          route: routeName,
+          field,
+        });
+      }
+
+      // 2. Oversized responses (>15 top-level properties)
+      const propCount = countProperties(fn.returnType);
+      if (propCount > 15) {
+        issues.push({
+          severity: "warning",
+          rule: "oversized-response",
+          message: `Response has ${propCount} top-level fields — consider pagination or field selection`,
+          route: routeName,
+        });
+      }
+
+      // 3. Deeply nested types (>4 levels)
+      const depth = maxDepth(fn.returnType);
+      if (depth > 4) {
+        issues.push({
+          severity: "warning",
+          rule: "deep-nesting",
+          message: `Response is nested ${depth} levels deep — consider flattening`,
+          route: routeName,
+        });
+      }
+
+      // 4. Naming inconsistency
+      const namingIssue = detectNamingInconsistency(fn.returnType, "");
+      if (namingIssue) {
+        issues.push({
+          severity: "warning",
+          rule: "inconsistent-naming",
+          message: namingIssue,
+          route: routeName,
+        });
+      }
+
+      // Also check args for naming issues
+      const argsNaming = detectNamingInconsistency(fn.argsType, "");
+      if (argsNaming) {
+        issues.push({
+          severity: "warning",
+          rule: "inconsistent-naming",
+          message: `Request: ${argsNaming}`,
+          route: routeName,
+        });
+      }
+
+      // 5. Empty response type
+      if (fn.returnType.kind === "unknown" ||
+          (fn.returnType.kind === "object" && Object.keys(fn.returnType.properties).length === 0)) {
+        issues.push({
+          severity: "info",
+          rule: "empty-response",
+          message: `Response type is empty or unknown — may need more observations`,
+          route: routeName,
+        });
+      }
+
+      // Collect field types for cross-route consistency check
+      collectFieldTypes(fn.returnType, "", globalFieldTypes);
+    }
+
+    // 6. Cross-route type inconsistency (same field name, different types)
+    for (const [fieldName, types] of globalFieldTypes) {
+      if (types.size > 1 && !["kind", "type", "data", "value", "result", "status"].includes(fieldName)) {
+        const typeList = Array.from(types).sort().join(", ");
+        issues.push({
+          severity: "warning",
+          rule: "type-inconsistency",
+          message: `Field "${fieldName}" has different types across routes: ${typeList}`,
+        });
+      }
+    }
+
+    // Sort: errors first, then warnings, then info
+    const severityOrder = { error: 0, warning: 1, info: 2 };
+    issues.sort((a, b) => severityOrder[a.severity] - severityOrder[b.severity]);
+
+    // Summary
+    const errorCount = issues.filter((i) => i.severity === "error").length;
+    const warningCount = issues.filter((i) => i.severity === "warning").length;
+    const infoCount = issues.filter((i) => i.severity === "info").length;
+
+    res.json({
+      summary: {
+        total: issues.length,
+        errors: errorCount,
+        warnings: warningCount,
+        info: infoCount,
+        routesAnalyzed: functions.length,
+      },
+      issues,
+    });
+  } catch (err) {
+    console.error("Audit error:", err);
+    res.status(500).json({ error: "Internal server error" });
+  }
+});
+
+export default router;