syndes 0.1.0

package/collect/classify.mjs

@@ -0,0 +1,226 @@
+/**
+ * Raw hook payload → ledger record(s). Runs in the worker, never in the hook.
+ *
+ * Pure given its context: the same payload and the same session state always
+ * produce the same drafts, which is what makes a replay of the spool reproduce
+ * an identical chain. The session state it reads and updates is passed in, not
+ * reached for.
+ *
+ * Hook payloads carry session_id, transcript_path, cwd, hook_event_name, plus:
+ *   PreToolUse         tool_name, tool_input
+ *   PostToolUse        tool_name, tool_input, tool_response
+ *   PermissionRequest  tool_name, tool_input, permission_mode
+ *   UserPromptSubmit   prompt
+ *   Notification       message
+ *   Stop / SubagentStop stop_hook_active
+ *   PreCompact         trigger, custom_instructions
+ *   SessionStart       source: startup | resume | clear | compact
+ *   SessionEnd         reason: clear | logout | prompt_input_exit | other
+ */
+
+import { draft, KIND } from '../ledger/schema.mjs';
+import { redact, clamp } from './redact.mjs';
+import { familyOf, summarise, isWrite, isRead, isSearchThroughBash, isVerification } from './tools.mjs';
+import { markToolStart, takeToolDuration, markRead, wasRead } from './sessions.mjs';
+
+/**
+ * @param {{t: number, e: string, p: string}} spooled
+ * @param {{config: object, session: object, project: object|null}} ctx
+ * @returns {object[]} drafts, in order
+ */
+export function classify(spooled, ctx) {
+  // The worker has usually parsed this already; parsing a megabyte of
+  // tool_response twice is pure waste on the one path that handles every call.
+  let payload = spooled.parsed;
+  if (!payload) {
+    try {
+      payload = JSON.parse(spooled.p);
+    } catch {
+      return []; // an unparseable payload is not evidence of anything
+    }
+  }
+
+  const ts = spooled.t;
+  const session = payload.session_id ?? null;
+  const project = ctx.project?.id ?? null;
+  const base = { ts, session, project };
+  const mode = ctx.config.privacy;
+  const extra = ctx.config.redactPatterns;
+
+  switch (payload.hook_event_name) {
+    case 'SessionStart':
+      return [draft(KIND.SESSION_START, {
+        ...base,
+        data: {
+          source: payload.source ?? 'startup',
+          cwd: mode === 'metadata' ? null : payload.cwd ?? null,
+          projectName: ctx.project?.name ?? null,
+          resumed: payload.source === 'resume' || payload.source === 'compact',
+        },
+      })];
+
+    case 'SessionEnd':
+      return [draft(KIND.SESSION_END, {
+        ...base,
+        data: {
+          reason: payload.reason ?? 'other',
+          durationMs: ctx.session.startedAt ? ts - ctx.session.startedAt : null,
+          counts: { ...ctx.session.counts },
+        },
+      })];
+
+    case 'PreCompact':
+      return [draft(KIND.COMPACT, {
+        ...base,
+        // 'auto' means the context ran out; 'manual' means the user chose it.
+        // Only the first one is a problem, so the trigger is the whole point.
+        data: { trigger: payload.trigger ?? 'auto', custom: Boolean(payload.custom_instructions) },
+      })];
+
+    case 'UserPromptSubmit': {
+      const prompt = typeof payload.prompt === 'string' ? payload.prompt : '';
+      const scrubbed = redact(prompt, { mode, extra });
+      const stored = clamp(scrubbed.text);
+      return [draft(KIND.PROMPT, {
+        ...base,
+        data: {
+          chars: prompt.length,
+          words: prompt.trim() ? prompt.trim().split(/\s+/).length : 0,
+          lines: prompt.split('\n').length,
+          text: stored.text,
+          truncated: stored.truncated,
+          redactions: scrubbed.redactions,
+          // A correction is the evidence that the previous prompt under-specified.
+          correction: looksLikeCorrection(prompt),
+        },
+      })];
+    }
+
+    case 'PreToolUse': {
+      const tool = payload.tool_name ?? 'Unknown';
+      const { target, detail } = summarise(tool, payload.tool_input);
+      markToolStart(ctx.session, tool, target, ts);
+      if (isRead(tool)) markRead(ctx.session, target);
+
+      const command = payload.tool_input?.command;
+      return [draft(KIND.TOOL_PRE, {
+        ...base,
+        data: {
+          tool,
+          family: familyOf(tool),
+          target: mode === 'metadata' ? null : target,
+          detail: mode === 'metadata' ? null : redact(detail ?? '', { mode, extra }).text || null,
+          searchViaBash: tool === 'Bash' ? isSearchThroughBash(command) : false,
+          verification: tool === 'Bash' ? isVerification(command) : false,
+        },
+      })];
+    }
+
+    case 'PostToolUse': {
+      const tool = payload.tool_name ?? 'Unknown';
+      const { target } = summarise(tool, payload.tool_input);
+      const durationMs = takeToolDuration(ctx.session, tool, target, ts);
+      const drafts = [draft(KIND.TOOL_POST, {
+        ...base,
+        data: {
+          tool,
+          family: familyOf(tool),
+          target: mode === 'metadata' ? null : target,
+          durationMs,
+          // Heuristic only. The authoritative failure signal is the transcript's
+          // tool_result.is_error, which PostToolUse does not carry.
+          failed: looksFailed(payload.tool_response),
+          bytes: sizeOf(payload.tool_response),
+        },
+      })];
+
+      if (isWrite(tool) && target) {
+        drafts.push(draft(KIND.FILE_TOUCH, {
+          ...base,
+          data: {
+            path: mode === 'metadata' ? null : target,
+            ext: extensionOf(target),
+            op: tool === 'Write' ? 'write' : 'edit',
+            // Editing a file this session never read is the strongest single
+            // predictor of an edit that has to be redone.
+            readFirst: wasRead(ctx.session, target),
+          },
+        }));
+        markRead(ctx.session, target); // after a write, the content is known
+      }
+      return drafts;
+    }
+
+    case 'PermissionRequest': {
+      const tool = payload.tool_name ?? 'Unknown';
+      const { target } = summarise(tool, payload.tool_input);
+      return [draft(KIND.TOOL_BLOCKED, {
+        ...base,
+        data: {
+          tool,
+          family: familyOf(tool),
+          target: mode === 'metadata' ? null : target,
+          mode: payload.permission_mode ?? null,
+        },
+      })];
+    }
+
+    case 'Notification': {
+      const message = typeof payload.message === 'string' ? payload.message : '';
+      return [draft(KIND.NOTIFY, {
+        ...base,
+        data: {
+          kind: /permission/i.test(message) ? 'permission' : 'waiting',
+          message: mode === 'metadata' ? null : message.slice(0, 200),
+        },
+      })];
+    }
+
+    case 'Stop':
+      return [draft(KIND.STOP, { ...base, data: { chained: payload.stop_hook_active === true } })];
+
+    case 'SubagentStop':
+      return [draft(KIND.SUBAGENT_STOP, { ...base, data: { chained: payload.stop_hook_active === true } })];
+
+    default:
+      return [];
+  }
+}
+
+/** Phrases that mean the last turn missed. Deliberately narrow — a false positive here inflates the correction rate and slanders the user's prompting. */
+const CORRECTIONS = [
+  /^\s*(no|nope|nah)\b/i,
+  /\b(actually|instead)\b.{0,40}\b(i|we)\b/i,
+  /\bi (meant|said)\b/i,
+  /\bthat'?s (wrong|not right|not what)\b/i,
+  /\b(undo|revert) (that|it|this)\b/i,
+  /\bnot what i (asked|wanted|meant)\b/i,
+];
+
+function looksLikeCorrection(prompt) {
+  return CORRECTIONS.some((pattern) => pattern.test(prompt));
+}
+
+function looksFailed(response) {
+  if (!response) return false;
+  if (response.is_error === true || response.interrupted === true) return true;
+  if (typeof response === 'string') return /^(error|command failed)\b/i.test(response.trim());
+  if (typeof response.stderr === 'string' && response.stderr.trim() && !response.stdout) return true;
+  return false;
+}
+
+function sizeOf(response) {
+  if (response == null) return 0;
+  try {
+    return typeof response === 'string' ? response.length : JSON.stringify(response).length;
+  } catch {
+    return 0;
+  }
+}
+
+function extensionOf(path) {
+  if (typeof path !== 'string') return null;
+  const name = path.split(/[\\/]/).pop() ?? '';
+  const dot = name.lastIndexOf('.');
+  return dot > 0 ? name.slice(dot + 1).toLowerCase().slice(0, 12) : null;
+}

package/collect/git.mjs

@@ -0,0 +1,78 @@
+/**
+ * Optional git context: branch, and commits made during a session.
+ *
+ * Strictly best-effort and strictly in the worker. Every call is execFile with
+ * an argv array and a hard timeout, so a repo that is huge, locked or mid-rebase
+ * gets skipped rather than blocking the drain. Read-only commands only — nothing
+ * here may mutate a user's repository, ever.
+ */
+
+import { execFileSync } from 'node:child_process';
+import { which } from '../runtime/platform.mjs';
+import { debug } from '../runtime/log.mjs';
+
+const TIMEOUT_MS = 1500;
+
+function git(cwd, args) {
+  const binary = which('git');
+  if (!binary) return null;
+  try {
+    return execFileSync(binary, args, {
+      cwd,
+      encoding: 'utf8',
+      timeout: TIMEOUT_MS,
+      stdio: ['ignore', 'pipe', 'ignore'],
+      windowsHide: true,
+    }).trim();
+  } catch (error) {
+    debug('git failed', args.join(' '), error.message);
+    return null;
+  }
+}
+
+export function branchOf(cwd) {
+  return git(cwd, ['rev-parse', '--abbrev-ref', 'HEAD']);
+}
+
+export function headOf(cwd) {
+  return git(cwd, ['rev-parse', 'HEAD']);
+}
+
+/**
+ * Commits reachable from HEAD that were not there last time we looked.
+ *
+ * `since` is a sha, not a timestamp: a rebase or an amend changes history in
+ * ways a clock cannot describe, and counting commits by time would double-count
+ * every rewritten one.
+ *
+ * @returns {{sha, subject, files, insertions, deletions}[]}
+ */
+export function commitsSince(cwd, since) {
+  if (!since) {
+    const head = headOf(cwd);
+    return head ? [] : []; // first sighting establishes the baseline only
+  }
+
+  const range = git(cwd, ['rev-list', '--max-count=20', `${since}..HEAD`]);
+  if (!range) return [];
+
+  const out = [];
+  for (const sha of range.split('\n').filter(Boolean).reverse()) {
+    const stat = git(cwd, ['show', '--stat=200', '--format=%s', '--no-color', sha]);
+    if (!stat) continue;
+    const lines = stat.split('\n');
+    const summary = lines[lines.length - 1] ?? '';
+    out.push({
+      sha: sha.slice(0, 12),
+      subject: (lines[0] ?? '').slice(0, 120),
+      files: number(/(\d+) files? changed/.exec(summary)),
+      insertions: number(/(\d+) insertions?/.exec(summary)),
+      deletions: number(/(\d+) deletions?/.exec(summary)),
+    });
+  }
+  return out;
+}
+
+function number(match) {
+  return match ? Number(match[1]) : 0;
+}

package/collect/projects.mjs

@@ -0,0 +1,82 @@
+/**
+ * cwd → stable project identity.
+ *
+ * A path is not an identity. The same repo cloned twice, or moved, must not
+ * become two projects in the metrics; two unrelated ~/work/app directories on
+ * two machines must not merge into one.
+ *
+ * Keyed on the git toplevel where there is one, else the directory itself, and
+ * cached — resolving this costs a subprocess and the answer never changes for a
+ * given cwd within a run.
+ */
+
+import { execFileSync } from 'node:child_process';
+import { basename, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { projectsFile } from '../runtime/paths.mjs';
+import { which } from '../runtime/platform.mjs';
+
+const cache = new Map();
+
+/** @returns {{id: string, name: string, root: string}|null} */
+export function projectFor(cwd) {
+  if (!cwd) return null;
+  const key = resolve(cwd);
+  if (cache.has(key)) return cache.get(key);
+
+  const root = gitRoot(key) ?? key;
+  const project = {
+    id: createHash('sha256').update(root).digest('hex').slice(0, 12),
+    name: basename(root) || root,
+    root,
+  };
+  cache.set(key, project);
+  return project;
+}
+
+function gitRoot(cwd) {
+  const git = which('git');
+  if (!git) return null;
+  try {
+    return execFileSync(git, ['rev-parse', '--show-toplevel'], {
+      cwd,
+      encoding: 'utf8',
+      timeout: 1500,
+      stdio: ['ignore', 'pipe', 'ignore'],
+      windowsHide: true,
+    }).trim() || null;
+  } catch {
+    return null; // not a repo, git is broken, or the repo is mid-rebase
+  }
+}
+
+/**
+ * The id → name map, so 'metadata' privacy can still show per-project
+ * breakdowns without the ledger recording anyone's directory layout.
+ */
+export function rememberProject(project) {
+  if (!project) return;
+  try {
+    const map = loadProjects();
+    if (map[project.id]?.name === project.name) return;
+    map[project.id] = { name: project.name, root: project.root, seen: Date.now() };
+    mkdirSync(dirname(projectsFile), { recursive: true });
+    writeFileSync(projectsFile, `${JSON.stringify(map, null, 2)}\n`);
+  } catch {
+    // A missing name map degrades the UI to ids. It never blocks a write.
+  }
+}
+
+export function loadProjects() {
+  try {
+    return existsSync(projectsFile) ? JSON.parse(readFileSync(projectsFile, 'utf8')) : {};
+  } catch {
+    return {};
+  }
+}
+
+export function nameFor(projectId) {
+  return loadProjects()[projectId]?.name ?? projectId;
+}

package/collect/redact.mjs

@@ -0,0 +1,85 @@
+/**
+ * Secret scrubbing, run BEFORE anything is chained.
+ *
+ * Order matters and is not negotiable: an append-only ledger has no take-backs,
+ * so a secret must never be written and then regretted. This is the first stage
+ * of the worker pipeline, not a view-time filter.
+ *
+ * A redaction records that it happened. Silently altered evidence is worse than
+ * absent evidence.
+ *
+ * ── Pattern contract ────────────────────────────────────────────────────────
+ * Each entry is [regexp, label, secretGroup]. The capture groups must TILE the
+ * whole match — concatenating them must reproduce it exactly — because the
+ * replacement is built by swapping one group for the label and re-joining the
+ * rest. That is what keeps the surrounding shape (`KEY=`, `Bearer `, the `@` in
+ * a connection string) intact, and the shape is usually the useful part.
+ */
+
+const PATTERNS = [
+  [/(sk-ant-[A-Za-z0-9_-]{8,})/g, 'ANTHROPIC_KEY', 1],
+  [/(sk-[A-Za-z0-9]{20,})/g, 'API_KEY', 1],
+  [/(gh[pousr]_[A-Za-z0-9]{16,})/g, 'GITHUB_TOKEN', 1],
+  [/(github_pat_[A-Za-z0-9_]{20,})/g, 'GITHUB_TOKEN', 1],
+  [/(xox[baprs]-[A-Za-z0-9-]{10,})/g, 'SLACK_TOKEN', 1],
+  [/(AKIA[0-9A-Z]{16})/g, 'AWS_KEY_ID', 1],
+  [/(AIza[0-9A-Za-z_-]{35})/g, 'GOOGLE_KEY', 1],
+  [/(eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})/g, 'JWT', 1],
+  [/(-----BEGIN[ A-Z]*PRIVATE KEY-----[\s\S]*?-----END[ A-Z]*PRIVATE KEY-----)/g, 'PRIVATE_KEY', 1],
+  // scheme://user:SECRET@host — keep the scheme, the user and the @.
+  [/([A-Za-z][A-Za-z0-9+.-]*:\/\/[^\s:@/]+:)([^\s@/]+)(@)/g, 'URL_PASSWORD', 2],
+  [/(Bearer\s+)([A-Za-z0-9._~+/-]{20,}=*)/gi, 'BEARER', 2],
+  // NAME=value in shell or .env form. The name survives; only the value goes.
+  // The closing quote is its own group so the groups still tile the match.
+  [/([A-Z][A-Z0-9_]{2,}(?:KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|API)[A-Z0-9_]*\s*=\s*)(["']?)([^\s"'#]{6,})(\2)/g, 'ENV_SECRET', 3],
+];
+
+/**
+ * @param {string} text
+ * @param {{mode?: 'redacted'|'full'|'metadata', extra?: string[]}} options
+ * @returns {{text: string|null, redactions: number, kinds: string[]}}
+ */
+export function redact(text, { mode = 'redacted', extra = [] } = {}) {
+  if (typeof text !== 'string') return { text: null, redactions: 0, kinds: [] };
+  if (mode === 'metadata') return { text: null, redactions: 0, kinds: [] };
+  if (mode === 'full') return { text, redactions: 0, kinds: [] };
+
+  let out = text;
+  let redactions = 0;
+  const kinds = new Set();
+
+  for (const [pattern, label, secretGroup] of [...PATTERNS, ...compileExtra(extra)]) {
+    out = out.replace(pattern, (...args) => {
+      // args is [match, ...groups, offset, string]; no named groups are used.
+      const groups = args.slice(1, -2);
+      redactions += 1;
+      kinds.add(label);
+      if (!groups.length) return `[${label}]`;
+      return groups
+        .map((group, index) => (index === secretGroup - 1 ? `[${label}]` : group ?? ''))
+        .join('');
+    });
+  }
+
+  return { text: out, redactions, kinds: [...kinds] };
+}
+
+/** User patterns from config. A bad regex is skipped, never thrown into a hook. */
+function compileExtra(sources) {
+  const out = [];
+  for (const source of sources ?? []) {
+    try {
+      out.push([new RegExp(source, 'g'), 'CUSTOM', 1]);
+    } catch {
+      // An unusable pattern must not take the pipeline down with it.
+    }
+  }
+  return out;
+}
+
+/** Truncate for storage, flagging that it happened. Bounds the ledger's growth. */
+export function clamp(text, limit = 2000) {
+  if (typeof text !== 'string') return { text: null, truncated: false };
+  if (text.length <= limit) return { text, truncated: false };
+  return { text: text.slice(0, limit), truncated: true };
+}

package/collect/sessions.mjs

@@ -0,0 +1,119 @@
+/**
+ * Session lifecycle, held in state/sessions.json.
+ *
+ * Disposable: the ledger is truth. This file exists so the worker can answer
+ * questions a single hook payload cannot — how long a tool took, whether a file
+ * was read before it was edited, where the transcript was last read to.
+ *
+ * A four-hour session with eleven minutes of work is the single most useful
+ * thing this tool can show someone, and separating active from wall-clock time
+ * is what makes that possible.
+ */
+
+import { readFileSync, writeFileSync, mkdirSync, renameSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { sessionsFile } from '../runtime/paths.mjs';
+import { debug } from '../runtime/log.mjs';
+
+/** Bounded so a long session cannot grow the state file without limit. */
+const MAX_READ_PATHS = 400;
+const MAX_PENDING = 200;
+/** A session with no activity for this long ended without telling us. */
+const REAP_MS = 12 * 60 * 60 * 1000;
+
+let state = null;
+
+export function load() {
+  if (state) return state;
+  try {
+    state = JSON.parse(readFileSync(sessionsFile, 'utf8'));
+  } catch {
+    state = {};
+  }
+  return state;
+}
+
+export function save() {
+  if (!state) return;
+  try {
+    mkdirSync(dirname(sessionsFile), { recursive: true });
+    const staging = `${sessionsFile}.tmp`;
+    writeFileSync(staging, `${JSON.stringify(state)}\n`);
+    renameSync(staging, sessionsFile);
+  } catch (error) {
+    debug('session state save failed', error.message);
+  }
+}
+
+export function get(sessionId) {
+  const all = load();
+  if (!all[sessionId]) {
+    all[sessionId] = {
+      startedAt: null, lastAt: null, project: null, cwd: null,
+      transcript: null, cursor: 0,
+      pending: {}, reads: [],
+      counts: { prompts: 0, tools: 0, blocks: 0, errors: 0, compacts: 0, writes: 0 },
+    };
+  }
+  return all[sessionId];
+}
+
+export function forget(sessionId) {
+  const all = load();
+  delete all[sessionId];
+}
+
+/** Record that a tool started, so the matching post can be given a duration. */
+export function markToolStart(session, toolName, target, ts) {
+  const key = `${toolName}|${target ?? ''}`;
+  session.pending[key] = ts;
+
+  const keys = Object.keys(session.pending);
+  if (keys.length > MAX_PENDING) delete session.pending[keys[0]];
+}
+
+/**
+ * How long a tool took, or null.
+ *
+ * Null when the pre-hook was never wired or the pair did not match — an
+ * unmeasured call must read as unmeasured, never as zero, or every average is
+ * dragged toward a number nobody observed.
+ */
+export function takeToolDuration(session, toolName, target, ts) {
+  const key = `${toolName}|${target ?? ''}`;
+  const started = session.pending[key];
+  if (started === undefined) return null;
+  delete session.pending[key];
+  return Math.max(0, ts - started);
+}
+
+export function markRead(session, path) {
+  if (!path) return;
+  if (!session.reads.includes(path)) session.reads.push(path);
+  if (session.reads.length > MAX_READ_PATHS) session.reads.shift();
+}
+
+export function wasRead(session, path) {
+  return Boolean(path) && session.reads.includes(path);
+}
+
+/** Sessions that stopped reporting. Marked inferred, never as a clean close. */
+export function reapStale(now = Date.now()) {
+  const all = load();
+  const reaped = [];
+  for (const [id, session] of Object.entries(all)) {
+    if (session.lastAt && now - session.lastAt > REAP_MS) {
+      reaped.push({ id, session });
+      delete all[id];
+    }
+  }
+  return reaped;
+}
+
+export function all() {
+  return load();
+}
+
+export function resetCache() {
+  state = null;
+}