tubeless 0.1.0

package/examples/rendering.ts

@@ -0,0 +1,34 @@
+import { createSteps, definePipeline, type PipelineError } from "tubeless";
+import { renderPipelineError, renderPipelinePlan } from "tubeless/render";
+
+const step = createSteps();
+
+const load = step("load", {
+  description: "Load the release inputs.",
+  run: () => "artifact",
+});
+
+const publish = step("publish", {
+  dependsOn: [load],
+  description: "Publish the built artifact.",
+  run: () => "published",
+});
+
+export const ReleasePipeline = definePipeline({
+  id: "release",
+  steps: [load, publish],
+  targets: [publish],
+  finalize: (outputs) => outputs.publish,
+});
+
+const plan = ReleasePipeline.plan({ targets: ["publish"] });
+
+// The same structured plan backs terminal output and machine-readable tooling.
+export const humanPlan = renderPipelinePlan(plan);
+export const jsonPlan = renderPipelinePlan(plan, { format: "json", pretty: true });
+
+export function renderDiagnostic(error: PipelineError, format: "human" | "json"): string {
+  return format === "json"
+    ? renderPipelineError(error, { format: "json" })
+    : renderPipelineError(error);
+}

package/examples/resumable-enrichment.ts

@@ -0,0 +1,75 @@
+import { resolve } from "node:path";
+import { createSteps, definePipeline } from "tubeless";
+import { runBatched } from "tubeless/batch";
+import { openCheckpoint, withCheckpointedBatch } from "tubeless/node";
+import { RateLimiter } from "tubeless/rate-limit";
+import { withRetry } from "tubeless/retry";
+
+interface EnrichmentOptions {
+  checkpointPath: string;
+  items: readonly string[];
+}
+
+interface EnrichedItem {
+  id: string;
+  summary: string;
+}
+
+const step = createSteps<EnrichmentOptions>();
+
+const loadPendingItems = step("load-pending-items", {
+  description: "Skip work that was already checkpointed by a previous run.",
+  run: (_inputs, context) => {
+    const checkpoint = openCheckpoint(resolve(context.cwd, context.options.checkpointPath));
+    return context.options.items.filter((item) => !checkpoint.has(item));
+  },
+});
+
+const enrichItems = step("enrich-items", {
+  dependsOn: [loadPendingItems],
+  description: "Process API work in small batches with retry and rate limiting.",
+  dryRun: "skip",
+  run: async ({ "load-pending-items": items }, context) => {
+    const checkpoint = openCheckpoint(resolve(context.cwd, context.options.checkpointPath));
+    const limiter = new RateLimiter(100);
+
+    return runBatched(items, { size: 5, concurrency: 2, signal: context.signal }, async (batch) => {
+      const enriched = await Promise.all(
+        batch.map((id) =>
+          withRetry(
+            async (): Promise<EnrichedItem> => {
+              await limiter.wait(context.signal);
+              return { id, summary: `summary for ${id}` };
+            },
+            {
+              baseDelayMs: 250,
+              maxAttempts: 3,
+              signal: context.signal,
+              sleep: context.sleep,
+            }
+          )
+        )
+      );
+
+      await withCheckpointedBatch(
+        checkpoint,
+        batch,
+        (id) => id,
+        async () => {
+          // Persist `enriched` here before recording the checkpoint.
+          void enriched;
+        }
+      );
+
+      return enriched;
+    });
+  },
+});
+
+export const EnrichmentPipeline = definePipeline({
+  id: "enrichment",
+  steps: [loadPendingItems, enrichItems],
+  finalize: (outputs) => ({
+    enrichedCount: outputs["enrich-items"]?.flat().length ?? 0,
+  }),
+});

package/examples/tracing.ts

@@ -0,0 +1,35 @@
+import { createSteps, defaultPipelineContext, definePipeline } from "tubeless";
+import { withRetry } from "tubeless/retry";
+import { createJsonTraceExporter } from "tubeless/tracing/json";
+
+interface TracingExampleOptions {
+  rows: readonly string[];
+}
+
+const step = createSteps<TracingExampleOptions>();
+const normalize = step("normalize", {
+  run: async (_inputs, context) =>
+    withRetry(
+      async ({ attempt }) => {
+        context.reportAttempt(attempt, { operation: "normalize" });
+        return context.options.rows.map((row) => row.trim().toLowerCase());
+      },
+      { baseDelayMs: 0, maxAttempts: 1 }
+    ),
+});
+
+export const TracingExamplePipeline = definePipeline({
+  id: "tracing-example",
+  steps: [normalize],
+  finalize: (outputs) => outputs.normalize ?? [],
+});
+
+export async function runTracingExample(rows: readonly string[]): Promise<readonly string[]> {
+  return TracingExamplePipeline.runOrThrow(
+    { rows },
+    {
+      ...defaultPipelineContext(),
+      tracing: { exporter: createJsonTraceExporter({ log: console }) },
+    }
+  );
+}

package/examples/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "esModuleInterop": true,
+    "lib": ["ES2022", "DOM"],
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "noEmit": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "target": "ES2022",
+    "types": ["node"],
+    "verbatimModuleSyntax": true
+  },
+  "include": ["*.ts"]
+}

package/examples/typed-import.ts

@@ -0,0 +1,54 @@
+import { createSteps, definePipeline, requireOutputs } from "tubeless";
+
+interface ImportOptions {
+  lines: readonly string[];
+}
+
+const step = createSteps<ImportOptions>();
+
+const loadRows = step("load-rows", {
+  description: "Read raw input records from the caller.",
+  run: (_inputs, context) => context.options.lines,
+});
+
+const normalizeRows = step("normalize-rows", {
+  name: "Normalize Rows",
+  dependsOn: [loadRows],
+  description: "Normalize records after the raw rows are available.",
+  run: ({ "load-rows": rows }) =>
+    rows.map((row) => row.trim().toLowerCase()).filter((row) => row.length > 0),
+});
+
+export const ImportPipeline = definePipeline({
+  id: "import",
+  steps: [loadRows, normalizeRows],
+  targets: [normalizeRows],
+  finalize: requireOutputs([normalizeRows], (outputs) => ({
+    count: outputs["normalize-rows"].length,
+    rows: outputs["normalize-rows"],
+  })),
+});
+
+export async function runImportExample() {
+  return ImportPipeline.runOrThrow({
+    lines: [" Alpha ", "", "Beta"],
+  });
+}
+
+export function renderImportDiagram() {
+  return ImportPipeline.toMermaid({ direction: "LR" });
+}
+
+export function planNormalizeTarget(lines: readonly string[]) {
+  void lines;
+  return ImportPipeline.plan({ targets: ["normalize-rows"] });
+}
+
+if (false) {
+  // @ts-expect-error Pipeline step IDs stay literal, so typos fail typecheck.
+  ImportPipeline.plan({ stepIds: ["normalise-rows"] });
+  // @ts-expect-error Pipeline targets use only declared literal target IDs.
+  ImportPipeline.plan({ targets: ["normalise-rows"] });
+  // @ts-expect-error Existing internal step IDs are not automatically public targets.
+  ImportPipeline.plan({ targets: ["load-rows"] });
+}

package/examples/validated-boundaries.ts

@@ -0,0 +1,75 @@
+import { createSteps, definePipeline, requireOutputs, type StandardSchemaV1 } from "tubeless";
+
+function standardSchema<TInput, TOutput>(
+  validate: StandardSchemaV1<TInput, TOutput>["~standard"]["validate"]
+): StandardSchemaV1<TInput, TOutput> {
+  return {
+    "~standard": {
+      validate,
+      vendor: "example",
+      version: 1,
+    },
+  };
+}
+
+const optionsSchema = standardSchema<{ source: string }, { limit: number; source: string }>(
+  (value) => {
+    // SAFETY: value is unvalidated external input; the optional `source` field is only
+    // read so the typeof check below can validate it before any use.
+    const source = (value as { source?: unknown }).source;
+    return typeof source === "string"
+      ? { value: { limit: 100, source } }
+      : { issues: [{ message: "Expected a source path", path: ["source"] }] };
+  }
+);
+
+const rowsSchema = standardSchema<readonly string[], { readonly rows: readonly string[] }>(
+  (value) =>
+    Array.isArray(value)
+      ? // SAFETY: Array.isArray proved an array; this minimal example trusts element
+        // types instead of checking every item.
+        { value: { rows: value as readonly string[] } }
+      : { issues: [{ message: "Expected rows" }] }
+);
+
+const resultSchema = standardSchema<
+  { count: number; source: string },
+  { count: number; source: string; validated: true }
+>((value) => {
+  // SAFETY: the result schema validates this pipeline's own finalize output, which is
+  // constructed as { count, source } by the finalize below — internal data, not external input.
+  const result = value as { count: number; source: string };
+  return { value: { ...result, validated: true } };
+});
+
+const step = createSteps(optionsSchema);
+
+const load = step("load", {
+  outputSchema: rowsSchema,
+  run: (_inputs, context) => [context.options.source].slice(0, context.options.limit),
+});
+
+export const ValidatedPipeline = definePipeline({
+  id: "validated-import",
+  steps: [load],
+  resultSchema,
+  finalize: requireOutputs([load], ({ load }, context) => ({
+    count: load.rows.length,
+    source: context.options.source,
+  })),
+});
+
+export async function runValidatedExample() {
+  const result = await ValidatedPipeline.runOrThrow({ source: "rows.txt" });
+  return result.validated;
+}
+
+if (false) {
+  // @ts-expect-error Options are inferred from the schema input.
+  ValidatedPipeline.runOrThrow({});
+
+  // SAFETY: an empty array literal has no elements that could violate readonly string[].
+  const duplicate = step("load", { run: () => [] as readonly string[] });
+  // @ts-expect-error Literal duplicate step IDs are rejected at definition time.
+  definePipeline({ id: "duplicate", steps: [load, duplicate], finalize: () => undefined });
+}

package/package.json

@@ -0,0 +1,156 @@
+{
+  "name": "tubeless",
+  "version": "0.1.0",
+  "description": "Typed pipeline primitives for composable data-processing workflows.",
+  "author": {
+    "name": "Chet Mancini",
+    "url": "https://github.com/chetmancini"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/chetmancini/tubeless.git"
+  },
+  "homepage": "https://github.com/chetmancini/tubeless#readme",
+  "bugs": {
+    "url": "https://github.com/chetmancini/tubeless/issues"
+  },
+  "keywords": [
+    "pipeline",
+    "workflow",
+    "typescript",
+    "cli",
+    "esm"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "packageManager": "bun@1.3.14",
+  "engines": {
+    "node": ">=22",
+    "bun": ">=1.3.14"
+  },
+  "sideEffects": false,
+  "type": "module",
+  "bin": {
+    "tubeless": "./dist/workbench-bin.js"
+  },
+  "files": [
+    "dist",
+    "docs",
+    "examples",
+    "LICENSE",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "api:check": "node scripts/generate-api-reference.mjs --check",
+    "api:generate": "bun run build && node scripts/generate-api-reference.mjs",
+    "check": "bun run build && bun run lint:run && bun run format:check && bun run typecheck:run && bun run docs:check && bun run api:check && bun run test:run && bun run eval:verify && bun run pack:verify",
+    "docs:check": "node scripts/validate-learning-surface.mjs",
+    "eval:agent": "node scripts/evaluate-agent-submission.mjs",
+    "eval:verify": "node scripts/verify-agent-evaluation-runner.mjs",
+    "format": "oxfmt --write .",
+    "format:check": "oxfmt --check .",
+    "lint": "bun run build && bun run lint:run",
+    "lint:run": "oxlint .",
+    "pack:dry-run": "bun pm pack --dry-run --ignore-scripts",
+    "tubeless": "bun run --silent build && bun dist/workbench-bin.js",
+    "prepack": "bun run build",
+    "test": "bun run build && bun run test:run",
+    "test:run": "vitest run --config vitest.config.ts",
+    "typecheck": "bun run build && bun run typecheck:run",
+    "typecheck:run": "tsc -p tsconfig.json --noEmit && tsc -p examples/tsconfig.json --noEmit",
+    "pack:verify": "node scripts/verify-packed-artifact.mjs"
+  },
+  "devDependencies": {
+    "@oxlint/plugins": "1.80.0",
+    "@types/node": "26.4.0",
+    "oxfmt": "0.65.0",
+    "oxlint": "1.80.0",
+    "typescript": "7.0.2",
+    "vitest": "4.1.11"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/pipeline.d.ts",
+      "import": "./dist/pipeline.js",
+      "default": "./dist/pipeline.js"
+    },
+    "./batch": {
+      "types": "./dist/batch.d.ts",
+      "import": "./dist/batch.js",
+      "default": "./dist/batch.js"
+    },
+    "./cli": {
+      "types": "./dist/cli.d.ts",
+      "import": "./dist/cli.js",
+      "default": "./dist/cli.js"
+    },
+    "./node": {
+      "types": "./dist/node.d.ts",
+      "import": "./dist/node.js",
+      "default": "./dist/node.js"
+    },
+    "./rate-limit": {
+      "types": "./dist/rate-limit.d.ts",
+      "import": "./dist/rate-limit.js",
+      "default": "./dist/rate-limit.js"
+    },
+    "./render": {
+      "types": "./dist/render.d.ts",
+      "import": "./dist/render.js",
+      "default": "./dist/render.js"
+    },
+    "./reporter": {
+      "types": "./dist/reporter-entry.d.ts",
+      "import": "./dist/reporter-entry.js",
+      "default": "./dist/reporter-entry.js"
+    },
+    "./retry": {
+      "types": "./dist/retry.d.ts",
+      "import": "./dist/retry.js",
+      "default": "./dist/retry.js"
+    },
+    "./run-store": {
+      "types": "./dist/run-store.d.ts",
+      "import": "./dist/run-store.js",
+      "default": "./dist/run-store.js"
+    },
+    "./run-store/sqlite": {
+      "types": "./dist/run-store-sqlite.d.ts",
+      "import": "./dist/run-store-sqlite.js",
+      "default": "./dist/run-store-sqlite.js"
+    },
+    "./workbench/studio": {
+      "types": "./dist/workbench-studio.d.ts",
+      "import": "./dist/workbench-studio.js",
+      "default": "./dist/workbench-studio.js"
+    },
+    "./run-store/ui": {
+      "types": "./dist/run-store-ui.d.ts",
+      "import": "./dist/run-store-ui.js",
+      "default": "./dist/run-store-ui.js"
+    },
+    "./testing": {
+      "types": "./dist/testing.d.ts",
+      "import": "./dist/testing.js",
+      "default": "./dist/testing.js"
+    },
+    "./tracing": {
+      "types": "./dist/tracing.d.ts",
+      "import": "./dist/tracing.js",
+      "default": "./dist/tracing.js"
+    },
+    "./tracing/json": {
+      "types": "./dist/tracing-json.d.ts",
+      "import": "./dist/tracing-json.js",
+      "default": "./dist/tracing-json.js"
+    },
+    "./tracing/otel": {
+      "types": "./dist/tracing-otel.d.ts",
+      "import": "./dist/tracing-otel.js",
+      "default": "./dist/tracing-otel.js"
+    }
+  }
+}