pipework 0.252.0 → 0.253.0

package/src/trace/check/coverage-check.ts

@@ -0,0 +1,188 @@
+// Tier-2 coverage check (Q9): every I/O call site is inside a `@flow_step`,
+// and a flow_step performs at most one I/O — at most one write, in particular.
+// Without this, the trace is gap-prone by construction: an unannotated read or
+// write goes silently untraced and the build doesn't notice.
+//
+// Ships as warning (Phase 6f): violations are surfaced but do not fail
+// `pipework check`. Once the consumer corpus has been migrated, Phase 7a
+// flips it to an error.
+//
+// Two rules:
+//   - flow/io-outside-flow-step — an I/O call appears outside any annotated
+//     flow_step function. The runtime can't open a `trace_step` for it.
+//   - flow/multi-io-per-flow-step — a flow_step body contains more than one
+//     I/O call. Split it so each I/O is its own step. The stricter variant,
+//     more than one *write* in the same step, is reported as
+//     flow/multi-write-per-flow-step — surfaces the Q8 one-write-per-step
+//     rule with its own diagnostic.
+//
+// Lexical: I/O inside a nested closure inside a flow_step body still counts
+// as "inside the flow_step" — otherwise writes hide inside callbacks. The
+// cardinality analyzer's "don't follow nested closures" convention is the
+// opposite default, by design (Q1 vs Q9).
+
+import { readFileSync, existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { parse } from '@typescript-eslint/parser'
+import { collectFlowStepSites, isNode, type AstNode, type FlowStepSite } from './flow-step-site.js'
+import { classifyIo, type IoKind } from './io-pattern.js'
+
+export interface CoverageViolation {
+  readonly rule: string
+  readonly file: string
+  readonly line: number
+  readonly message: string
+}
+
+export interface CoverageCheckResult {
+  readonly violations: readonly CoverageViolation[]
+  readonly scanned: number
+}
+
+interface IoSite {
+  readonly kind: IoKind
+  /** 1-based line. */
+  readonly line: number
+  /** Character offset of the call expression in the file (for in-range checks). */
+  readonly start: number
+}
+
+/**
+ * Tier-2 coverage check across `sourceFiles` (paths relative to `cwd`).
+ *
+ * `extras` extends the stock pipework I/O registry with consumer-declared
+ * patterns from `config.trace.io`. Defaults to empty — the stock registry
+ * (`pipe.*` writes, `pipe(...)`-chain reads) is always live.
+ */
+export function checkCoverage(
+  cwd: string,
+  sourceFiles: readonly string[],
+  extras: Readonly<Record<string, IoKind>> = {},
+): CoverageCheckResult {
+  const violations: CoverageViolation[] = []
+  let scanned = 0
+
+  for (const file of sourceFiles) {
+    const abs = join(cwd, file)
+    if (!existsSync(abs)) continue
+
+    let program: AstNode | null = null
+    let sites: readonly FlowStepSite[] = []
+    try {
+      // Read the file once — #322: it was being read (and parsed) twice here.
+      const source = readFileSync(abs, 'utf-8')
+      program = parseProgram(source, file)
+      sites = collectFlowStepSites(source, file)
+    } catch {
+      // Unparseable — leave syntax errors to the type checker.
+      continue
+    }
+    scanned++
+
+    const ioSites = collectIoSites(program, extras)
+    const flowStepRanges = collectFlowStepRanges(sites)
+
+    for (const io of ioSites) {
+      const enclosing = flowStepRanges.find(r => io.start >= r.bodyStart && io.start < r.bodyEnd)
+      if (enclosing === undefined) {
+        violations.push({
+          rule: 'flow/io-outside-flow-step',
+          file,
+          line: io.line,
+          message:
+            `I/O ${io.kind} is outside any @flow_step. Wrap it in a standalone function and ` +
+            `annotate that function with @flow_step so the runtime can open a trace_step for it.`,
+        })
+      }
+    }
+
+    for (const range of flowStepRanges) {
+      const inside = ioSites.filter(s => s.start >= range.bodyStart && s.start < range.bodyEnd)
+      if (inside.length <= 1) continue
+
+      const writes = inside.filter(s => s.kind === 'write')
+      if (writes.length > 1) {
+        violations.push({
+          rule: 'flow/multi-write-per-flow-step',
+          file,
+          line: range.line,
+          message:
+            `@flow_step "${range.name}" performs ${String(writes.length)} writes. A flow_step ` +
+            `may perform at most one write (Q8). Split each write into its own flow_step.`,
+        })
+        continue
+      }
+
+      violations.push({
+        rule: 'flow/multi-io-per-flow-step',
+        file,
+        line: range.line,
+        message:
+          `@flow_step "${range.name}" performs ${String(inside.length)} I/O calls. A flow_step ` +
+          `may perform at most one I/O call. Split each call into its own flow_step.`,
+      })
+    }
+  }
+
+  return { violations, scanned }
+}
+
+interface FlowStepRange {
+  readonly name: string
+  readonly line: number
+  readonly bodyStart: number
+  readonly bodyEnd: number
+}
+
+function collectFlowStepRanges(sites: readonly FlowStepSite[]): FlowStepRange[] {
+  const ranges: FlowStepRange[] = []
+  for (const site of sites) {
+    const body = site.subject.body
+    if (body === null) continue
+    ranges.push({
+      name: site.subject.name,
+      line: site.subject.line,
+      bodyStart: body.range[0],
+      bodyEnd: body.range[1],
+    })
+  }
+  return ranges
+}
+
+function collectIoSites(program: AstNode, extras: Readonly<Record<string, IoKind>>): IoSite[] {
+  const out: IoSite[] = []
+  walk(program, node => {
+    if (node.type !== 'CallExpression') return
+    const kind = classifyIo(node, extras)
+    if (kind === null) return
+    out.push({ kind, line: node.loc.start.line, start: node.range[0] })
+  })
+  return out
+}
+
+const SKIP_KEYS = new Set(['loc', 'range', 'parent'])
+
+function walk(node: AstNode, visit: (n: AstNode) => void): void {
+  visit(node)
+  for (const key of Object.keys(node)) {
+    if (SKIP_KEYS.has(key)) continue
+    const value = node[key]
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        if (isNode(item)) walk(item, visit)
+      }
+    } else if (isNode(value)) {
+      walk(value, visit)
+    }
+  }
+}
+
+function parseProgram(code: string, fileName: string): AstNode {
+  return parse(code, {
+    ...(fileName.endsWith('.tsx') ? { jsx: true } : {}),
+    range: true,
+    loc: true,
+    comment: true,
+    sourceType: 'module',
+  }) as unknown as AstNode
+}

package/src/trace/check/flow-check.ts

@@ -0,0 +1,103 @@
+// The `flow` check: validates `@flow_step` annotations across a set of source
+// files. This is the syntactic tier — it rejects what is locally wrong about an
+// annotation without consulting the call graph. Three rules:
+//
+//   - a `@flow_step` block must document a standalone function, not a method;
+//   - every `@downstream` must declare its cardinality (`step` | `branch`);
+//   - the annotation must parse.
+//
+// Cardinality is *correct* only relative to the call graph — that is the
+// cardinality analyzer's job, not this one. See PLAN.tracing.md, component 2.
+
+import { readFileSync, existsSync } from 'node:fs'
+import { join } from 'node:path'
+import { collectFlowStepSites, type FlowStepSite } from './flow-step-site.js'
+
+export interface FlowViolation {
+  readonly rule: string
+  readonly file: string
+  readonly line: number
+  readonly message: string
+}
+
+export interface FlowCheckResult {
+  readonly violations: readonly FlowViolation[]
+  readonly scanned: number
+}
+
+/** Validates every `@flow_step` annotation in `sourceFiles` (paths relative to `cwd`). */
+export function checkFlowAnnotations(cwd: string, sourceFiles: readonly string[]): FlowCheckResult {
+  const violations: FlowViolation[] = []
+  let scanned = 0
+
+  for (const file of sourceFiles) {
+    const abs = join(cwd, file)
+    if (!existsSync(abs)) continue
+
+    let sites: readonly FlowStepSite[]
+    try {
+      sites = collectFlowStepSites(readFileSync(abs, 'utf-8'), file)
+    } catch {
+      // Unparseable file — leave syntax errors to the type checker.
+      continue
+    }
+    scanned++
+
+    for (const site of sites) {
+      collectSiteViolations(file, site, violations)
+    }
+  }
+
+  return { violations, scanned }
+}
+
+function collectSiteViolations(file: string, site: FlowStepSite, out: FlowViolation[]): void {
+  const { subject, annotation, commentLine } = site
+
+  if (subject.shape === 'orphan') {
+    out.push({
+      rule: 'flow/orphan-annotation',
+      file,
+      line: commentLine,
+      message:
+        '@flow_step JSDoc is not attached to a function. A flow step must document a '
+        + 'standalone function declaration so it has a stable path:symbol identity.',
+    })
+  } else if (subject.shape === 'method') {
+    out.push({
+      rule: 'flow/no-method-flow-step',
+      file,
+      line: subject.line,
+      message:
+        `@flow_step on method "${subject.name}" — a method cannot be a flow step. It carries `
+        + `hidden "this" state, so the trace boundary is untrustworthy. Extract "${subject.name}" `
+        + 'into a standalone function.',
+    })
+  }
+
+  if (annotation.kind === 'invalid') {
+    for (const err of annotation.errors) {
+      out.push({
+        rule: 'flow/invalid-annotation',
+        file,
+        line: commentLine + err.line - 1,
+        message: err.message,
+      })
+    }
+    return
+  }
+
+  for (const edge of annotation.annotation.downstream) {
+    if (edge.cardinality === null) {
+      out.push({
+        rule: 'flow/missing-cardinality',
+        file,
+        line: commentLine,
+        message:
+          `@downstream ${edge.path}:${edge.symbol} is missing a cardinality. Add "step" or `
+          + `"branch": @downstream ${edge.path}:${edge.symbol} branch. A branch is an edge `
+          + 'traversed more than once (loop, batch, recursion, fan-out).',
+      })
+    }
+  }
+}

package/src/trace/check/flow-step-site.ts

@@ -0,0 +1,234 @@
+// Locates `@flow_step` annotation sites in a TypeScript source file: each site
+// pairs a parsed annotation with the declaration it documents and how that
+// declaration is shaped — a standalone function, a method, or nothing at all.
+// The `flow` check and the cardinality analyzer both build on this; it is the
+// one place that answers "where are the flow steps in this file".
+
+import { parse } from '@typescript-eslint/parser'
+import { parseFlowStepAnnotation, type FlowStepParseResult } from 'pipework-flow-parser'
+
+interface Position {
+  readonly line: number
+  readonly column: number
+}
+
+interface SourceLocation {
+  readonly start: Position
+  readonly end: Position
+}
+
+/** A structurally-typed ESTree node — enough to walk without a parser type dependency. */
+export interface AstNode {
+  readonly type: string
+  readonly range: readonly [number, number]
+  readonly loc: SourceLocation
+  readonly [key: string]: unknown
+}
+
+interface CommentNode {
+  readonly type: 'Block' | 'Line'
+  readonly value: string
+  readonly range: readonly [number, number]
+  readonly loc: SourceLocation
+}
+
+interface Program extends AstNode {
+  readonly comments: readonly CommentNode[]
+}
+
+/**
+ * How a `@flow_step`-annotated declaration is shaped.
+ *
+ * - `function` — a standalone function (declaration or a `const` bound to an
+ *   arrow / function expression). The only legal shape.
+ * - `method` — a class method, class field bound to a function, or object
+ *   literal method. Banned: a method carries hidden `this` state, so the trace
+ *   boundary is untrustworthy. See PLAN.tracing.md, Q3.
+ * - `orphan` — the JSDoc block documents no declaration the analyzer can see.
+ */
+export type FlowStepShape = 'function' | 'method' | 'orphan'
+
+export interface FlowStepSubject {
+  readonly shape: FlowStepShape
+  /** Declared name, or `<anonymous>` / `<computed>` when none can be read. */
+  readonly name: string
+  /** 1-based line of the declaration, or of the JSDoc block when orphaned. */
+  readonly line: number
+  /** The declaration node, or `null` when orphaned. */
+  readonly node: AstNode | null
+  /**
+   * The documented function's executable body — what the cardinality analyzer
+   * walks. `null` exactly when `node` is `null` (an orphan): every `function`
+   * and `method` subject resolves to a function with a body.
+   */
+  readonly body: AstNode | null
+}
+
+export interface FlowStepSite {
+  /** Parsed content of the `@flow_step` JSDoc block — never `not-a-flow-step`. */
+  readonly annotation: Exclude<FlowStepParseResult, { kind: 'not-a-flow-step' }>
+  /** The declaration the block documents, and how it is shaped. */
+  readonly subject: FlowStepSubject
+  /** 1-based line where the `@flow_step` JSDoc block opens. */
+  readonly commentLine: number
+}
+
+interface Candidate {
+  readonly node: AstNode
+  readonly shape: Exclude<FlowStepShape, 'orphan'>
+  readonly name: string
+  readonly body: AstNode | null
+}
+
+const flowStepSiteCache = new Map<string, FlowStepSite[]>()
+
+/**
+ * Parse `code` and return every `@flow_step` site it contains, in source order.
+ *
+ * Memoized by source text: the flow, cardinality, and coverage checks each scan
+ * the same file, and the parse-and-walk produces a bit-identical result every
+ * time — the cache collapses it to one parse per file. Issue #322.
+ *
+ * Throws if `code` is not parseable — callers scanning many files should catch
+ * and skip, leaving syntax errors to the type checker. A `fileName` ending in
+ * `.tsx` parses with JSX enabled.
+ */
+export function collectFlowStepSites(code: string, fileName?: string): FlowStepSite[] {
+  const jsx = fileName?.endsWith('.tsx') === true
+  const key = jsx ? 'jsx\0' + code : code
+  const cached = flowStepSiteCache.get(key)
+  if (cached !== undefined) return cached
+  const sites = parseFlowStepSites(code, jsx)
+  flowStepSiteCache.set(key, sites)
+  return sites
+}
+
+function parseFlowStepSites(code: string, jsx: boolean): FlowStepSite[] {
+  const program = parse(code, {
+    ...(jsx ? { jsx: true } : {}),
+    range: true,
+    loc: true,
+    comment: true,
+    sourceType: 'module',
+  }) as unknown as Program
+
+  const candidates: Candidate[] = []
+  walk(program, node => {
+    const candidate = classify(node)
+    if (candidate !== null) candidates.push(candidate)
+  })
+  candidates.sort((a, b) => a.node.range[0] - b.node.range[0])
+
+  const sites: FlowStepSite[] = []
+  for (const comment of program.comments) {
+    if (comment.type !== 'Block') continue
+    const annotation = parseFlowStepAnnotation(`/*${comment.value}*/`)
+    if (annotation.kind === 'not-a-flow-step') continue
+    sites.push({
+      annotation,
+      subject: subjectFor(comment, candidates),
+      commentLine: comment.loc.start.line,
+    })
+  }
+  return sites
+}
+
+/** The declaration a `@flow_step` block documents: the nearest one that follows it. */
+function subjectFor(comment: CommentNode, candidates: readonly Candidate[]): FlowStepSubject {
+  const documented = candidates.find(c => c.node.range[0] >= comment.range[1])
+  if (documented === undefined) {
+    return { shape: 'orphan', name: '<unknown>', line: comment.loc.start.line, node: null, body: null }
+  }
+  return {
+    shape: documented.shape,
+    name: documented.name,
+    line: documented.node.loc.start.line,
+    node: documented.node,
+    body: documented.body,
+  }
+}
+
+/** Classifies a node as a flow-step-able declaration, or `null` if it is not one. */
+function classify(node: AstNode): Candidate | null {
+  switch (node.type) {
+    case 'FunctionDeclaration':
+      return { node, shape: 'function', name: readName(node['id']), body: bodyOf(node) }
+    case 'VariableDeclaration': {
+      const bound = functionBoundDeclarator(node)
+      if (bound === null) return null
+      return { node, shape: 'function', name: readName(bound['id']), body: bodyOf(bound['init']) }
+    }
+    case 'MethodDefinition':
+    case 'TSAbstractMethodDefinition':
+      return { node, shape: 'method', name: readName(node['key']), body: bodyOf(node['value']) }
+    case 'PropertyDefinition':
+    case 'TSAbstractPropertyDefinition':
+    case 'Property':
+      return isFunctionExpression(node['value'])
+        ? { node, shape: 'method', name: readName(node['key']), body: bodyOf(node['value']) }
+        : null
+    default:
+      return null
+  }
+}
+
+/** The executable `body` of a function node — a `BlockStatement` or, for a
+ * concise arrow, the returned expression. `null` when `fn` is not a function. */
+function bodyOf(fn: unknown): AstNode | null {
+  if (!isNode(fn)) return null
+  const body = fn['body']
+  return isNode(body) ? body : null
+}
+
+/** The first declarator in a `VariableDeclaration` bound to a function, or `null`. */
+function functionBoundDeclarator(node: AstNode): AstNode | null {
+  const declarations = node['declarations']
+  if (!Array.isArray(declarations)) return null
+  for (const declarator of declarations) {
+    if (isNode(declarator) && isFunctionExpression(declarator['init'])) return declarator
+  }
+  return null
+}
+
+function isFunctionExpression(value: unknown): boolean {
+  return isNode(value)
+    && (value.type === 'ArrowFunctionExpression' || value.type === 'FunctionExpression')
+}
+
+function readName(keyOrId: unknown): string {
+  if (!isNode(keyOrId)) return '<anonymous>'
+  switch (keyOrId.type) {
+    case 'Identifier':
+      return String(keyOrId['name'])
+    case 'PrivateIdentifier':
+      return `#${String(keyOrId['name'])}`
+    case 'Literal':
+      return String(keyOrId['value'])
+    default:
+      return '<computed>'
+  }
+}
+
+const SKIP_KEYS = new Set(['loc', 'range', 'parent'])
+
+/** Depth-first visit of every node reachable from `node`, parents before children. */
+function walk(node: AstNode, visit: (n: AstNode) => void): void {
+  visit(node)
+  for (const key of Object.keys(node)) {
+    if (SKIP_KEYS.has(key)) continue
+    const value = node[key]
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        if (isNode(item)) walk(item, visit)
+      }
+    } else if (isNode(value)) {
+      walk(value, visit)
+    }
+  }
+}
+
+export function isNode(value: unknown): value is AstNode {
+  return typeof value === 'object'
+    && value !== null
+    && typeof (value as { type?: unknown }).type === 'string'
+}

package/src/trace/check/io-pattern.ts

@@ -0,0 +1,116 @@
+// The I/O call-target registry consumed by the Tier-2 coverage check. Given a
+// `CallExpression`, classifies it as a read, a write, or not-I/O.
+//
+// Stock list (pipework-owned, lexical):
+//   - Writes: `pipe.insert / pipe.update / pipe.delete / pipe.upsert /
+//     pipe.bulkSet`. These are the locked write entrypoints from Q8; the
+//     raw-write lint rule (component 13) backs them up.
+//   - Reads: any call whose callee chains off `pipe(...)` — e.g.
+//     `pipe('db').execute(...)`, `pipe(name).select(...)`,
+//     `pipe('db').query.users.findMany(...)`. The `pipe(name)` accessor only
+//     exists to perform I/O, so any method called on its result counts.
+//
+// Consumer extension (Q9): cache, external HTTP, queue publishes — anything
+// outside the stock list — is registered through `pipework.config.ts` under
+// `trace.io`. The registry is a `Record<string, 'read' | 'write'>` where the
+// key is a lexical call pattern: a bare identifier (`"fetch"`) matches an
+// identifier callee; an `"object.method"` pattern matches a single-level
+// member-access callee. The schema validator rejects anything more elaborate —
+// type resolution is out of scope, lexical matching is the contract.
+
+import { isNode, type AstNode } from './flow-step-site.js'
+
+export type IoKind = 'read' | 'write'
+
+const PIPE_WRITE_METHODS = new Set(['insert', 'update', 'delete', 'upsert', 'bulkSet'])
+
+/**
+ * Classify `call` against the stock I/O registry and the optional consumer
+ * `extras` registry, or return `null` if the call is not I/O. `extras` is the
+ * `Record` shape `config.trace.io` resolves to — keys are lexical patterns
+ * (`"fetch"` or `"cache.get"`), values are `'read' | 'write'`.
+ *
+ * `call` must be a `CallExpression`; callers walk the AST and hand each one
+ * over. This function is pure over the node — no parent lookups, no type
+ * resolution. Lexical match only.
+ */
+export function classifyIo(
+  call: AstNode,
+  extras: Readonly<Record<string, IoKind>> = {},
+): IoKind | null {
+  if (call.type !== 'CallExpression') return null
+  const callee = call['callee']
+  if (!isNode(callee)) return null
+
+  if (isPipeWriteCallee(callee)) return 'write'
+  if (chainsOffPipeAccessor(callee)) return 'read'
+
+  return matchExtras(callee, extras)
+}
+
+/** `pipe.<verb>` where `<verb>` is one of the locked write entrypoints. */
+function isPipeWriteCallee(callee: AstNode): boolean {
+  if (callee.type !== 'MemberExpression') return false
+  if (callee['computed'] === true) return false
+
+  const object = callee['object']
+  const property = callee['property']
+  if (!isNode(object) || object.type !== 'Identifier' || object['name'] !== 'pipe') return false
+  if (!isNode(property) || property.type !== 'Identifier') return false
+
+  const name = property['name']
+  return typeof name === 'string' && PIPE_WRITE_METHODS.has(name)
+}
+
+/**
+ * The callee is a chain off `pipe(...)` — anywhere up the member-expression
+ * spine. `pipe('db').execute(sql)` matches; `pipe('db').query.users.findMany()`
+ * matches; `pipe.update(t)` (no call on `pipe`) does not.
+ */
+function chainsOffPipeAccessor(callee: AstNode): boolean {
+  let cursor: AstNode = callee
+  while (cursor.type === 'MemberExpression') {
+    const object = cursor['object']
+    if (!isNode(object)) return false
+    if (isPipeInvocation(object)) return true
+    cursor = object
+  }
+  return false
+}
+
+/** `pipe(...)` — a call expression whose callee is the identifier `pipe`. */
+function isPipeInvocation(node: AstNode): boolean {
+  if (node.type !== 'CallExpression') return false
+  const callee = node['callee']
+  return isNode(callee) && callee.type === 'Identifier' && callee['name'] === 'pipe'
+}
+
+/**
+ * Looks up `callee` against the consumer registry. Two shapes match:
+ *   - Identifier `foo` → key `"foo"`
+ *   - Non-computed MemberExpression `obj.method` (Identifier object,
+ *     Identifier property) → key `"obj.method"`
+ * Anything else falls through (returns `null`) — the stock registry has
+ * already exited above, so this is the not-I/O bucket.
+ */
+function matchExtras(
+  callee: AstNode,
+  extras: Readonly<Record<string, IoKind>>,
+): IoKind | null {
+  if (callee.type === 'Identifier') {
+    const name = callee['name']
+    if (typeof name !== 'string') return null
+    return extras[name] ?? null
+  }
+  if (callee.type === 'MemberExpression' && callee['computed'] !== true) {
+    const object = callee['object']
+    const property = callee['property']
+    if (!isNode(object) || object.type !== 'Identifier') return null
+    if (!isNode(property) || property.type !== 'Identifier') return null
+    const objectName = object['name']
+    const propertyName = property['name']
+    if (typeof objectName !== 'string' || typeof propertyName !== 'string') return null
+    return extras[`${objectName}.${propertyName}`] ?? null
+  }
+  return null
+}