tubeless 0.2.12 → 0.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/dist/core/child-execution.d.ts +0 -23
  2. package/dist/core/child-execution.js +10 -12
  3. package/dist/core/child-execution.js.map +1 -1
  4. package/dist/core/child-progress.js +7 -2
  5. package/dist/core/child-progress.js.map +1 -1
  6. package/dist/core/iteration.d.ts +27 -0
  7. package/dist/core/iteration.js +111 -0
  8. package/dist/core/iteration.js.map +1 -0
  9. package/dist/core/lifecycle.d.ts +7 -5
  10. package/dist/core/lifecycle.js +36 -4
  11. package/dist/core/lifecycle.js.map +1 -1
  12. package/dist/core/pipeline-artifacts.d.ts +11 -0
  13. package/dist/core/pipeline-artifacts.js +2 -0
  14. package/dist/core/pipeline-artifacts.js.map +1 -0
  15. package/dist/core/pipeline-compiler.d.ts +1 -0
  16. package/dist/core/pipeline-compiler.js +1 -0
  17. package/dist/core/pipeline-compiler.js.map +1 -1
  18. package/dist/core/pipeline-definition-identity.d.ts +2 -2
  19. package/dist/core/pipeline-definition-identity.js +28 -5
  20. package/dist/core/pipeline-definition-identity.js.map +1 -1
  21. package/dist/core/pipeline-execute.d.ts +1 -6
  22. package/dist/core/pipeline-execute.js +28 -215
  23. package/dist/core/pipeline-execute.js.map +1 -1
  24. package/dist/core/pipeline-execution-error.d.ts +35 -0
  25. package/dist/core/pipeline-execution-error.js +195 -0
  26. package/dist/core/pipeline-execution-error.js.map +1 -0
  27. package/dist/core/pipeline-identity.d.ts +1 -0
  28. package/dist/core/pipeline-identity.js +1 -0
  29. package/dist/core/pipeline-identity.js.map +1 -1
  30. package/dist/core/pipeline-mermaid.js +4 -1
  31. package/dist/core/pipeline-mermaid.js.map +1 -1
  32. package/dist/core/pipeline-run-state.d.ts +4 -4
  33. package/dist/core/pipeline-run-state.js +10 -16
  34. package/dist/core/pipeline-run-state.js.map +1 -1
  35. package/dist/core/pipeline-step-executor.d.ts +2 -0
  36. package/dist/core/pipeline-step-executor.js +11 -3
  37. package/dist/core/pipeline-step-executor.js.map +1 -1
  38. package/dist/core/pipeline-steps.d.ts +20 -0
  39. package/dist/core/pipeline-steps.js +62 -0
  40. package/dist/core/pipeline-steps.js.map +1 -1
  41. package/dist/core/pipeline-types.d.ts +15 -2
  42. package/dist/core/pipeline-types.js.map +1 -1
  43. package/dist/core/pipeline.d.ts +4 -1
  44. package/dist/core/pipeline.js +18 -3
  45. package/dist/core/pipeline.js.map +1 -1
  46. package/dist/project/project-compiler.d.ts +3 -31
  47. package/dist/project/project-compiler.js +11 -180
  48. package/dist/project/project-compiler.js.map +1 -1
  49. package/dist/project/project-registry.d.ts +44 -0
  50. package/dist/project/project-registry.js +107 -0
  51. package/dist/project/project-registry.js.map +1 -0
  52. package/dist/project/project-step-graph.d.ts +9 -0
  53. package/dist/project/project-step-graph.js +99 -0
  54. package/dist/project/project-step-graph.js.map +1 -0
  55. package/dist/render/render.js +1 -1
  56. package/dist/render/render.js.map +1 -1
  57. package/dist/reporter/interactive-reporter.js +6 -3
  58. package/dist/reporter/interactive-reporter.js.map +1 -1
  59. package/dist/reporter/live-ticker-frame.d.ts +21 -0
  60. package/dist/reporter/live-ticker-frame.js +196 -0
  61. package/dist/reporter/live-ticker-frame.js.map +1 -0
  62. package/dist/reporter/live-ticker-protocol.d.ts +41 -0
  63. package/dist/reporter/live-ticker-protocol.js +59 -0
  64. package/dist/reporter/live-ticker-protocol.js.map +1 -0
  65. package/dist/reporter/live-ticker-worker.js +11 -25
  66. package/dist/reporter/live-ticker-worker.js.map +1 -1
  67. package/dist/reporter/live-ticker.d.ts +0 -15
  68. package/dist/reporter/live-ticker.js +11 -216
  69. package/dist/reporter/live-ticker.js.map +1 -1
  70. package/dist/reporter/reporter.js +7 -6
  71. package/dist/reporter/reporter.js.map +1 -1
  72. package/dist/run-store/definition-projection.d.ts +8 -0
  73. package/dist/run-store/definition-projection.js +89 -0
  74. package/dist/run-store/definition-projection.js.map +1 -0
  75. package/dist/run-store/run-projection.d.ts +15 -0
  76. package/dist/run-store/run-projection.js +203 -0
  77. package/dist/run-store/run-projection.js.map +1 -0
  78. package/dist/run-store/run-store-codec.d.ts +2 -0
  79. package/dist/run-store/run-store-codec.js +11 -0
  80. package/dist/run-store/run-store-codec.js.map +1 -1
  81. package/dist/run-store/run-store-event-decoder.js +3 -0
  82. package/dist/run-store/run-store-event-decoder.js.map +1 -1
  83. package/dist/run-store/run-store-reader.d.ts +7 -0
  84. package/dist/run-store/run-store-reader.js +31 -0
  85. package/dist/run-store/run-store-reader.js.map +1 -0
  86. package/dist/run-store/run-store-sqlite.js +23 -7
  87. package/dist/run-store/run-store-sqlite.js.map +1 -1
  88. package/dist/run-store/run-store.d.ts +16 -4
  89. package/dist/run-store/run-store.js +14 -263
  90. package/dist/run-store/run-store.js.map +1 -1
  91. package/dist/studio/run-store-ui-api.d.ts +13 -0
  92. package/dist/studio/run-store-ui-api.js +163 -0
  93. package/dist/studio/run-store-ui-api.js.map +1 -0
  94. package/dist/studio/run-store-ui-client-browser.js +2 -2
  95. package/dist/studio/run-store-ui-client-transport.js +28 -3
  96. package/dist/studio/run-store-ui-client-transport.js.map +1 -1
  97. package/dist/studio/run-store-ui-client.d.ts +2 -1
  98. package/dist/studio/run-store-ui-client.js +15 -9
  99. package/dist/studio/run-store-ui-client.js.map +1 -1
  100. package/dist/studio/run-store-ui-http.d.ts +11 -0
  101. package/dist/studio/run-store-ui-http.js +52 -0
  102. package/dist/studio/run-store-ui-http.js.map +1 -1
  103. package/dist/studio/run-store-ui-state.d.ts +1 -1
  104. package/dist/studio/run-store-ui-state.js +10 -28
  105. package/dist/studio/run-store-ui-state.js.map +1 -1
  106. package/dist/studio/run-store-ui.d.ts +5 -11
  107. package/dist/studio/run-store-ui.js +5 -203
  108. package/dist/studio/run-store-ui.js.map +1 -1
  109. package/dist/testing/testing.d.ts +18 -2
  110. package/dist/testing/testing.js +35 -2
  111. package/dist/testing/testing.js.map +1 -1
  112. package/dist/tracing/artifact-metadata.d.ts +28 -0
  113. package/dist/tracing/artifact-metadata.js +105 -0
  114. package/dist/tracing/artifact-metadata.js.map +1 -0
  115. package/dist/tracing/tracing-codec.d.ts +1 -1
  116. package/dist/tracing/tracing-codec.js +1 -1
  117. package/dist/tracing/tracing-codec.js.map +1 -1
  118. package/dist/tracing/tracing-constants.d.ts +1 -1
  119. package/dist/tracing/tracing-constants.js +1 -1
  120. package/dist/tracing/tracing-constants.js.map +1 -1
  121. package/dist/tracing/tracing-contracts.d.ts +3 -1
  122. package/dist/tracing/tracing-contracts.js.map +1 -1
  123. package/dist/tracing/tracing-internal.d.ts +2 -0
  124. package/dist/tracing/tracing-internal.js +36 -13
  125. package/dist/tracing/tracing-internal.js.map +1 -1
  126. package/dist/tracing/tracing-schema.d.ts +330 -49
  127. package/dist/tracing/tracing-schema.js +102 -10
  128. package/dist/tracing/tracing-schema.js.map +1 -1
  129. package/dist/workbench/workbench-command-execution.d.ts +6 -0
  130. package/dist/workbench/workbench-command-execution.js +48 -0
  131. package/dist/workbench/workbench-command-execution.js.map +1 -0
  132. package/dist/workbench/workbench-history.js +11 -18
  133. package/dist/workbench/workbench-history.js.map +1 -1
  134. package/dist/workbench/workbench-launch-session.d.ts +2 -2
  135. package/dist/workbench/workbench-launch-session.js.map +1 -1
  136. package/dist/workbench/workbench-run.d.ts +0 -4
  137. package/dist/workbench/workbench-run.js +3 -47
  138. package/dist/workbench/workbench-run.js.map +1 -1
  139. package/dist/workbench/workbench-studio-launcher.d.ts +27 -0
  140. package/dist/workbench/workbench-studio-launcher.js +95 -0
  141. package/dist/workbench/workbench-studio-launcher.js.map +1 -0
  142. package/dist/workbench/workbench-ui.js +10 -83
  143. package/dist/workbench/workbench-ui.js.map +1 -1
  144. package/docs/README.md +13 -7
  145. package/docs/agent-guide.md +69 -6
  146. package/docs/airflow.md +187 -0
  147. package/docs/api-reference.md +87 -76
  148. package/docs/api-report.json +17 -6
  149. package/docs/artifacts.md +173 -0
  150. package/docs/child-pipeline-composition.md +65 -2
  151. package/docs/comparison.md +20 -0
  152. package/docs/concepts.md +70 -2
  153. package/docs/dagster.md +161 -0
  154. package/docs/inngest.md +163 -0
  155. package/docs/llms.txt +30 -1
  156. package/docs/recipes.md +82 -33
  157. package/docs/remote-step-composition.md +6 -0
  158. package/docs/step-functions.md +190 -0
  159. package/docs/studio.md +31 -12
  160. package/docs/temporal.md +183 -0
  161. package/examples/airflow/contract.ts +46 -0
  162. package/examples/airflow/hosted.ts +70 -0
  163. package/examples/airflow/remote.ts +175 -0
  164. package/examples/airflow/tubeless_example.py +49 -0
  165. package/examples/airflow/worker.ts +24 -0
  166. package/examples/artifact-lineage.ts +70 -0
  167. package/examples/dagster/definitions.py +59 -0
  168. package/examples/dagster/hosted.ts +78 -0
  169. package/examples/dagster/pipeline.ts +62 -0
  170. package/examples/dagster/test_definitions.py +47 -0
  171. package/examples/dagster/tsconfig.build.json +10 -0
  172. package/examples/dagster/worker.ts +22 -0
  173. package/examples/inngest/client.ts +4 -0
  174. package/examples/inngest/functions.ts +65 -0
  175. package/examples/inngest/pipeline.ts +41 -0
  176. package/examples/inngest/send.ts +11 -0
  177. package/examples/inngest/server.ts +17 -0
  178. package/examples/inngest/tsconfig.build.json +10 -0
  179. package/examples/iteration.ts +52 -0
  180. package/examples/parameterized-steps.ts +96 -0
  181. package/examples/project/tubeless.project.ts +46 -0
  182. package/examples/step-functions/handler.ts +110 -0
  183. package/examples/step-functions/input.json +1 -0
  184. package/examples/step-functions/pipeline.ts +41 -0
  185. package/examples/step-functions/state-machine.asl.json +41 -0
  186. package/examples/step-functions/template.json +42 -0
  187. package/examples/step-output-overrides.ts +41 -0
  188. package/examples/temporal/activities.ts +94 -0
  189. package/examples/temporal/client.ts +19 -0
  190. package/examples/temporal/pipeline.ts +41 -0
  191. package/examples/temporal/tsconfig.build.json +10 -0
  192. package/examples/temporal/worker.ts +21 -0
  193. package/examples/temporal/workflows.ts +16 -0
  194. package/package.json +13 -4
  195. package/skills/tubeless/SKILL.md +30 -3
@@ -54,3 +54,23 @@ one of those tools in either of two ways:
54
54
  The host owns persistence and acknowledgement. A rejected run tells it that
55
55
  processing failed. The parent Tubeless process does not restore itself after
56
56
  exit; if that is required, the host must provide it.
57
+
58
+ The [Tubeless ↔ Airflow guide](./airflow.md) connects both patterns in one example:
59
+ submit a DAG through `fromRemote`, run a Tubeless pipeline inside its task, and
60
+ validate the returned XCom before continuing locally.
61
+
62
+ The [Temporal guide](./temporal.md) hosts a pipeline inside an Activity and keeps
63
+ its execution outside replayed Workflow code, with Activity-owned retries,
64
+ heartbeats, and cancellation.
65
+
66
+ The [Inngest guide](./inngest.md) hosts a pipeline inside `step.run`. Inngest
67
+ saves the successful result and retries failed attempts; each attempt executes
68
+ the whole Tubeless pipeline. The example includes an event sender and local server.
69
+
70
+ The [Dagster guide](./dagster.md) materializes an asset through Dagster Pipes,
71
+ with progress logs, content data versions, and a trace for Tubeless Studio.
72
+ Dagster owns retries of the asset; Tubeless owns execution of its internal steps.
73
+
74
+ The [Step Functions guide](./step-functions.md) invokes a pipeline inside a Lambda
75
+ Task and includes a SAM deployment. Step Functions owns workflow recovery;
76
+ Tubeless supplies typed steps and progress within each Lambda invocation.
package/docs/concepts.md CHANGED
@@ -399,14 +399,15 @@ result.steps.find((step) => step.attemptId)?.attemptId;
399
399
  result.finishedAtMs - result.startedAtMs;
400
400
  ```
401
401
 
402
- Trace events use the version 2 discriminated payload model. Every event carries
402
+ New trace events use the version 3 discriminated payload model. Every event carries
403
403
  a unique execution `runId` and keeps reusable external correlation separate.
404
404
  `tubeless/tracing` intentionally exposes only that typed event union, the exporter
405
405
  interface, and exporter composition. Concrete JSON and telemetry SDK adapters
406
406
  belong in the workbench or application integration; see the
407
407
  [tracing recipe](../examples/tracing.ts). This narrow API does not weaken the
408
408
  recording contract: existing version 2 NDJSON traces remain readable by
409
- `tubeless history` and `tubeless ui`.
409
+ `tubeless history` and `tubeless ui`. Version 3 adds iteration relationships for
410
+ repeated child runs; their execution IDs remain unique.
410
411
 
411
412
  ### Local event store and studio
412
413
 
@@ -624,6 +625,73 @@ interleave, or advance time differently; call `test.abort()` to exercise the
624
625
  normal cancellation path. Use your test framework to assert on the results. Replace external I/O yourself;
625
626
  the test runtime does not mock network or filesystem operations.
626
627
 
628
+ ### Supplying step outputs
629
+
630
+ Use `overrideStep` from `tubeless/testing` to supply a typed intermediate value
631
+ without modifying the pipeline definition:
632
+
633
+ ```ts
634
+ const test = createPipelineTestRuntime();
635
+ const run = await test.run(pipeline, options, {
636
+ stepIds: ["load", "summarize"],
637
+ overrides: [overrideStep(load, ["alice", "bob"])],
638
+ });
639
+ ```
640
+
641
+ Each override pairs an actual step reference with a resolved handler output.
642
+ For schema-backed steps, supply the **schema input** type: `outputSchema` still
643
+ validates and transforms it, and dependents receive the transformed output.
644
+ Invalid values fail with `TUBELESS_STEP_OUTPUT_VALIDATION_FAILED`. An explicit
645
+ `undefined` is a published value when the step's type and schema allow it.
646
+ Functions can be domain values; they are never called as lazy factories.
647
+ Await asynchronous fixture setup before constructing overrides.
648
+
649
+ A selected overridden step never calls its `run`, `skip`, or custom `dryRun`
650
+ handler. It bypasses required-input and failure-gate checks and `dryRun: "skip"`,
651
+ since the supplied value replaces its entire computation. Overriding a child,
652
+ fan-out, or remote wrapper also bypasses item enumeration, mappings, adapter
653
+ calls, and child launches. Overrides belong to this run only and do not flow
654
+ into children. Duplicate overrides and foreign references (including same-named
655
+ steps from another graph) reject before execution. Custom pipeline wrappers
656
+ are unsupported; use the object returned by `definePipeline`.
657
+
658
+ Selection and scheduling remain explicit:
659
+
660
+ - Overrides do not select steps, prune ancestors, or change target closure.
661
+ A filtered step stays filtered and does not validate or publish its override.
662
+ To isolate a downstream branch, use exact `stepIds` including the supplied
663
+ intermediate and its dependents, excluding upstream I/O. Plans describe the
664
+ ordinary graph and do not apply overrides.
665
+ - Scheduling still waits for all prerequisite steps to reach terminal states.
666
+ Override validation occupies a concurrency slot. Fail-fast can prevent an
667
+ override from starting; `continueOnError` allows it to publish after upstream
668
+ failure, while the run retains that failure.
669
+ - Cancellation prevents unstarted overrides and is checked again after async
670
+ output validation. Successfully overridden steps unlock required dependents
671
+ and failure gates and supply optional inputs normally.
672
+ - Finalization and `resultSchema` use the supplied outputs through the usual
673
+ boundaries, including required finalizers and the default last-step result.
674
+
675
+ An override uses the normal lifecycle: `running`, then `completed`, `failed`,
676
+ or `cancelled`. From the start of its validation attempt, reports and hooks
677
+ carry `outputSource: "override"`. The same metadata survives validation failure
678
+ or cancellation. Successful overrides emit `onStepComplete`, so tests exercise
679
+ the same completion callbacks and dependency rules as handler-produced outputs.
680
+ There is no separate override status or hook.
681
+
682
+ Traces use the existing `step.running`, `step.complete`, `step.failed`, and
683
+ `step.cancelled` events with `payload.outputSource: "override"`; they do not
684
+ record the supplied value. Both CLI reporters and Studio show the ordinary
685
+ status with an override annotation, such as **completed (overridden)** or
686
+ **failed (overridden)**. Existing recordings remain readable. Ordinary runs omit
687
+ this metadata, as do filtered overrides and overrides cancelled before their
688
+ validation attempt starts; absence does not prove a handler ran.
689
+
690
+ Overrides are available only through the test runtime's `run` and `runOrThrow`.
691
+ Ordinary pipeline controls, CLI flags, and Studio launch forms do not expose
692
+ them, and trace recordings cannot automatically be replayed as override values.
693
+ See the [executable override recipe](../examples/step-output-overrides.ts).
694
+
627
695
  ## Implementation guidelines
628
696
 
629
697
  - Keep step IDs stable, use `name` only for a friendlier display label, and keep
@@ -0,0 +1,161 @@
1
+ # Tubeless on Dagster
2
+
3
+ Run a Tubeless pipeline as a Dagster asset using Dagster Pipes. Dagster owns
4
+ asset scheduling, dependencies, and retries; Tubeless owns the typed steps,
5
+ progress, and execution trace inside the asset's subprocess.
6
+
7
+ ```text
8
+ Dagster asset → PipesSubprocessClient → Node.js → Tubeless pipeline
9
+ ← logs, traces, and materialization metadata ←
10
+ ```
11
+
12
+ The example normalizes rows, validates the dataset, and writes a JSON artifact.
13
+ After successful execution, it reports a materialization with the row count,
14
+ artifact path, content data version, Tubeless run ID, and trace path. Operators
15
+ can follow progress in Dagster and inspect individual pipeline steps in Tubeless
16
+ Studio. See Dagster's [TypeScript Pipes integration](https://docs.dagster.io/integrations/libraries/typescript).
17
+
18
+ ## The example
19
+
20
+ | File | Responsibility |
21
+ | ---------------------------------------------------------------- | ------------------------------------------------------------------------- |
22
+ | [`pipeline.ts`](../examples/dagster/pipeline.ts) | Normalize, validate, and publish through public Tubeless imports |
23
+ | [`hosted.ts`](../examples/dagster/hosted.ts) | Bridge Pipes context, logs, traces, and materialization reporting |
24
+ | [`worker.ts`](../examples/dagster/worker.ts) | Open and close Pipes, forward process termination to Tubeless |
25
+ | [`definitions.py`](../examples/dagster/definitions.py) | Define the asset, launch Node, save the trace, and yield materializations |
26
+ | [`test_definitions.py`](../examples/dagster/test_definitions.py) | Exercise the real Python host and Node worker |
27
+
28
+ The pipeline has no Dagster imports and also runs from the local project.
29
+ The bridge uses `@dagster-io/dagster-pipes` 0.1.0, installed as a repository
30
+ development dependency. Applications adopting this bridge install the SDK in
31
+ their own project. Tubeless itself retains a dependency-free runtime.
32
+ The Python example is tested with Dagster 1.13.24.
33
+
34
+ ## Run locally
35
+
36
+ Use Node.js 22.6 or later, Bun, Python 3.12, and
37
+ [uv](https://docs.astral.sh/uv/getting-started/installation/). From the repository root:
38
+
39
+ ```sh
40
+ bun ci
41
+ bun run build
42
+ bunx tsc -p examples/dagster/tsconfig.build.json
43
+ uv venv --python 3.12 .context/dagster-venv
44
+ uv pip install --python .context/dagster-venv/bin/python \
45
+ 'dagster==1.13.24' 'dagster-webserver==1.13.24'
46
+ .context/dagster-venv/bin/dagster dev -f examples/dagster/definitions.py
47
+ ```
48
+
49
+ The compiler writes JavaScript to `.context/dagster/`. Open the local Dagster
50
+ UI at `http://localhost:3000`, select `normalized_rows`, and materialize it.
51
+ The execution host must have Node, the compiled worker, and its installed
52
+ dependencies available. Keep the dev server's process running while you inspect
53
+ the asset and run history.
54
+
55
+ The default input produces `{"rows":["alpha","beta"]}` in
56
+ `.context/dagster-artifacts/<dagster-run-id>/rows.json`. Open the materialization's
57
+ metadata to find the artifact, row count, data version, and Tubeless trace.
58
+ These paths refer to the execution host's filesystem. For distributed workers,
59
+ publish artifacts and traces to shared storage and report accessible locations.
60
+
61
+ To change the input, open the asset's launch configuration and supply:
62
+
63
+ ```yaml
64
+ ops:
65
+ normalized_rows:
66
+ config:
67
+ lines: [" Alpha ", "Beta", "ALPHA"]
68
+ ```
69
+
70
+ Use the local Tubeless command below for previews. The Dagster asset always
71
+ publishes: Pipes requires asset outputs even when an asset declares its output
72
+ optional. The host disables implicit materializations and requires exactly one
73
+ explicit materialization from the worker, so a missing report fails the attempt.
74
+ The worker rejects `dryRun: true` instead of accidentally publishing a preview.
75
+
76
+ ## Logs, traces, and asset identity
77
+
78
+ The bridge forwards pipeline logs and progress counts through the Pipes logger.
79
+ It sends raw Tubeless trace events as Pipes custom messages. After a successful
80
+ subprocess exit, the Python host writes these events to `trace.ndjson`.
81
+ The sample does not retain a separate trace file for failed or
82
+ killed subprocesses; their Pipes logs remain in Dagster. For large or long-running
83
+ jobs, stream traces to your own durable exporter rather than accumulating custom
84
+ messages in the host.
85
+
86
+ Copy the recorded trace path into these commands to inspect a finished run:
87
+
88
+ ```sh
89
+ bun run tubeless -- history --trace .context/dagster-artifacts/<dagster-run-id>/trace.ndjson
90
+ bun run tubeless -- ui --trace .context/dagster-artifacts/<dagster-run-id>/trace.ndjson
91
+ ```
92
+
93
+ Replace `<dagster-run-id>` with the actual ID before running the commands.
94
+ Trace files contain recorded logs and event payloads; apply your application's
95
+ redaction policy at the exporter.
96
+
97
+ Each Tubeless attempt has a fresh `runId`. Its correlation ID combines the
98
+ Dagster run ID and asset key and stays stable across step retries within that
99
+ run. The optional `parentRunId` in the bridge is reserved for a known Tubeless
100
+ execution; a Dagster run ID is a separate identity.
101
+
102
+ The materialization's `dataVersion` hashes the normalized row array, so repeated
103
+ attempts with the same rows have the same version. It describes logical content,
104
+ not the file's formatting or the implementation release. Tubeless's
105
+ `implementationVersion` separately identifies the example's handler release.
106
+ This recipe handles one selected asset per invocation.
107
+
108
+ ## Retries and cancellation
109
+
110
+ Dagster retries the asset at most twice after its initial attempt. An exception
111
+ from `runOrThrow` causes a nonzero worker exit, which fails the asset attempt.
112
+ Validation failure prevents publication and materialization. Successful
113
+ publication is reported only after the pipeline finishes.
114
+
115
+ A retry runs the whole Tubeless pipeline again. The example replaces the same
116
+ run-specific JSON dataset; a new Dagster run gets a separate directory. This
117
+ does not make arbitrary side effects exactly-once. If publication succeeds and
118
+ reporting fails, Dagster may retry. Use idempotent writes or business-level keys
119
+ when adapting the pipeline to external services. Split independently retryable
120
+ datasets into separate Dagster assets when you need recovery at those boundaries.
121
+
122
+ The default `PipesSubprocessClient` forwards host termination to its subprocess.
123
+ The worker turns `SIGTERM` and `SIGINT` into an abort signal and closes Pipes
124
+ when execution settles. Normalization cooperates through `context.sleep`.
125
+ Cancellation does not roll back an artifact already written, and blocking I/O
126
+ can delay it; Dagster may forcibly stop a subprocess that exceeds its termination
127
+ timeout. See [Dagster Pipes](https://docs.dagster.io/integrations/external-pipelines/using-dagster-pipes)
128
+ for other execution environments.
129
+
130
+ ## Run the pipeline directly
131
+
132
+ Use the same pipeline through the example project for local iteration:
133
+
134
+ ```sh
135
+ bun run tubeless -- plan --project examples/project/tubeless.project.ts dagster-dataset
136
+ bun run tubeless -- run --project examples/project/tubeless.project.ts dagster-dataset -- \
137
+ --lines " Alpha " --lines "Beta" --output-path ./rows.json --dry-run
138
+ ```
139
+
140
+ Local execution writes no Dagster metadata. Relative output paths resolve from
141
+ the project execution directory. The Pipes worker requires a real Pipes context
142
+ and rejects standalone launches without one.
143
+
144
+ ## Verification
145
+
146
+ `make check` compiles the TypeScript examples and tests the real Pipes SDK in
147
+ subprocesses without a Dagster service. Tests cover metadata and content versions,
148
+ retry identity, raw traces, malformed inputs, validation failure, dry runs, and
149
+ cooperative termination.
150
+
151
+ After the local setup and worker compilation, run the Python integration tests:
152
+
153
+ ```sh
154
+ .context/dagster-venv/bin/python -m unittest discover -s examples/dagster -p 'test_*.py'
155
+ ```
156
+
157
+ These launch the compiled Node worker from the real Dagster execution engine.
158
+ They verify a successful materialization and trace, and a failed dataset that
159
+ exhausts Dagster's retry policy without materializing an asset.
160
+ The Python tests are optional and are not part of the dependency-free package's
161
+ `make check` environment.
@@ -0,0 +1,163 @@
1
+ # Tubeless on Inngest
2
+
3
+ Run a Tubeless pipeline inside an Inngest `step.run`. Inngest owns event delivery,
4
+ retries, and durable step results; Tubeless owns the typed graph, progress, and
5
+ execution report within each attempt.
6
+
7
+ ```text
8
+ Event sender → Inngest function → step.run → Tubeless pipeline
9
+ ```
10
+
11
+ Keep the pipeline invocation inside the durable step callback. Inngest may
12
+ execute the function handler several times as it restores saved step results.
13
+ Code outside steps can execute again. See
14
+ [how Inngest functions execute](https://www.inngest.com/docs/learn/how-functions-are-executed).
15
+
16
+ ## The example
17
+
18
+ | File | Responsibility |
19
+ | -------------------------------------------------- | ------------------------------------------------------------------------------- |
20
+ | [`pipeline.ts`](../examples/inngest/pipeline.ts) | Normalize and deduplicate rows using public Tubeless imports |
21
+ | [`functions.ts`](../examples/inngest/functions.ts) | Validate event data and run the pipeline in a durable step with correlated logs |
22
+ | [`client.ts`](../examples/inngest/client.ts) | Configure the shared Inngest client |
23
+ | [`server.ts`](../examples/inngest/server.ts) | Serve the function at `/api/inngest` using Node's HTTP server |
24
+ | [`send.ts`](../examples/inngest/send.ts) | Send a sample event and print its accepted event IDs |
25
+
26
+ The pipeline has no Inngest imports and also runs locally. The example uses
27
+ Inngest TypeScript SDK 4.21.0 and `@inngest/test` 1.0.0, installed as development
28
+ dependencies here. Consumer applications install `inngest` themselves. Tubeless
29
+ keeps its dependency-free runtime and adds no Inngest public API.
30
+
31
+ SDK v4 puts `triggers` in the `createFunction` options. Older v3 examples use a
32
+ separate trigger argument; see the
33
+ [v4 migration guide](https://www.inngest.com/docs/reference/typescript/v4/migrations/v3-to-v4).
34
+
35
+ ## Run locally
36
+
37
+ Use Node.js 22.6 or later and Bun 1.3.14 or later. From the repository root:
38
+
39
+ ```sh
40
+ bun ci
41
+ bun run build
42
+ bunx tsc -p examples/inngest/tsconfig.build.json
43
+ INNGEST_DEV=1 node .context/inngest/server.js
44
+ ```
45
+
46
+ The compiler writes JavaScript to `.context/inngest/`. The server listens on
47
+ `127.0.0.1:3000`. In a second terminal, start the
48
+ [Inngest Dev Server](https://www.inngest.com/docs/local-development):
49
+
50
+ ```sh
51
+ npx inngest-cli@latest dev -u http://127.0.0.1:3000/api/inngest
52
+ ```
53
+
54
+ Open `http://localhost:8288` and confirm that `tubeless-example` has synced and
55
+ lists `normalize-rows`. In a third terminal, send the sample event:
56
+
57
+ ```sh
58
+ INNGEST_DEV=1 node .context/inngest/send.js
59
+ ```
60
+
61
+ The sender prints accepted event IDs. It does not wait for the pipeline result.
62
+ Open the resulting run in the Dev Server UI: `run-pipeline` returns
63
+ `rows: ["alpha", "beta"]`, `count: 2`, a Tubeless `runId`, and `preview: false`.
64
+ Sending again creates another event and function run. This example does not
65
+ deduplicate separate submissions.
66
+
67
+ `INNGEST_DEV=1` selects local development for both server and sender. For a
68
+ deployment, host the endpoint where Inngest can reach it, remove that setting,
69
+ configure `INNGEST_SIGNING_KEY` and `INNGEST_EVENT_KEY` through your host's secret
70
+ configuration, and sync the app. The example server always binds to
71
+ `127.0.0.1:3000`. To reuse it in production, run a reverse proxy on the same host
72
+ and forward your public HTTPS `/api/inngest` endpoint to
73
+ `http://127.0.0.1:3000/api/inngest`, preserving the request method, body, and
74
+ headers. Sync the public URL with Inngest. If your deployment cannot proxy to
75
+ that loopback listener, register `normalizeFunction` using your application's
76
+ Inngest framework adapter instead of running `server.ts`.
77
+ See [serving functions](https://www.inngest.com/docs/reference/typescript/serve)
78
+ and [client configuration](https://www.inngest.com/docs/reference/typescript/client/create).
79
+
80
+ ## Validation, results, and observation
81
+
82
+ The function validates `event.data` before calling `runOrThrow`. Invalid rows or
83
+ control metadata throw Inngest's `NonRetriableError` directly. A TypeScript type
84
+ on the sender cannot validate incoming events. Keep this validation outside the
85
+ Tubeless invocation so its execution-error wrapper cannot hide the host's
86
+ non-retryable error type. See
87
+ [Inngest errors](https://www.inngest.com/docs/reference/typescript/errors).
88
+
89
+ The correlation ID combines the Inngest run ID and durable step ID. It stays
90
+ stable across retries of that step. Each pipeline attempt gets a fresh Tubeless
91
+ `runId`. Only pass `parentRunId` when it identifies a known Tubeless execution;
92
+ an Inngest run ID belongs in correlation metadata.
93
+
94
+ The pipeline returns plain JSON data. Inngest serializes step results, so avoid
95
+ returning contexts, errors, signals, class instances, or a whole execution report.
96
+ Large datasets should live in external storage with a small reference returned
97
+ from the step. See [step result serialization](https://www.inngest.com/docs/reference/typescript/functions/step-run).
98
+
99
+ Pipeline logs, progress hooks, and trace events go through the function logger.
100
+ The Inngest UI records one durable step for the pipeline. To inspect individual
101
+ Tubeless steps in Studio, retain raw trace events using an application-owned
102
+ exporter. Logger envelopes are not importable Tubeless NDJSON traces. Apply
103
+ redaction before retaining or forwarding events.
104
+
105
+ ## Retries and durable boundaries
106
+
107
+ The function sets `retries: 2`: up to three attempts for the durable step.
108
+ `runOrThrow` lets execution failures reach Inngest. A failed attempt reruns the
109
+ entire pipeline, including any successful internal steps. Once Inngest saves
110
+ the durable step's successful result, subsequent handler executions reuse that
111
+ result, including the original Tubeless `runId`, without invoking the pipeline.
112
+
113
+ Normalization and deduplication are safe to repeat. If you add writes, use
114
+ business-level idempotency keys: a process can fail after a write but before its
115
+ result is saved. Correlation IDs do not deduplicate side effects. See
116
+ [Inngest steps](https://www.inngest.com/docs/learn/inngest-steps).
117
+
118
+ For independently retryable phases, define multiple Inngest steps at the
119
+ function level, each calling a smaller Tubeless pipeline and passing JSON
120
+ results to the next phase. Do not call Inngest step tools from inside Tubeless
121
+ handlers or hooks nested in `step.run`. Place durable sleeps and event waits
122
+ between these phases. Tubeless `context.sleep` is an ordinary process-local
123
+ wait, and internal progress is not a durable checkpoint.
124
+
125
+ ## Cancellation and dry runs
126
+
127
+ Inngest cancellation prevents subsequent steps from running; it does not stop
128
+ an already executing step. This adapter therefore does not supply a host
129
+ cancellation signal. Cancelling a run in the dashboard can leave its active
130
+ Tubeless pipeline running to completion. Closing the sender does not cancel
131
+ the function. See [Inngest cancellation](https://www.inngest.com/docs/features/inngest-functions/cancellation).
132
+
133
+ The pipeline cooperates with signals supplied by local callers. Applications
134
+ that need an execution deadline can supply their own signal and forward it to
135
+ I/O, but a local abort is separate from Inngest cancellation. Keep work within
136
+ the execution limits of your deployment and make side effects safe to retry.
137
+
138
+ Set `dryRun: true` in the sender's job to forward Tubeless dry-run controls.
139
+ This still sends a real event and runs an Inngest function. The example performs
140
+ only pure transformations. Mark any added writes with `dryRun: "skip"` or a
141
+ side-effect-free preview handler.
142
+
143
+ To plan or preview without contacting Inngest, use the project registration:
144
+
145
+ ```sh
146
+ bun run tubeless -- plan --project examples/project/tubeless.project.ts inngest-normalize
147
+ bun run tubeless -- run --project examples/project/tubeless.project.ts inngest-normalize -- \
148
+ --lines " Alpha " --lines "Beta" --dry-run
149
+ ```
150
+
151
+ ## Verification
152
+
153
+ `make check` compiles the examples against the real SDK and uses
154
+ [`InngestTestEngine`](https://www.inngest.com/docs/reference/typescript/v4/testing)
155
+ to exercise the durable step with the real Tubeless pipeline. Tests cover JSON
156
+ results, progress and traces, invalid input, dry runs, and mocked durable step
157
+ output. The mocked-output test checks that the function returns the supplied
158
+ value without calling the pipeline; it does not test persisted-state recovery.
159
+ A separate test substitutes a real two-step Tubeless pipeline with fake storage
160
+ I/O: its save handler fails once, then a fresh attempt reruns both load and save
161
+ with stable correlation. These tests need no service or credentials and do not
162
+ test server-side persistence or retry scheduling. Use the local server flow above
163
+ to verify event delivery and inspect the run; test deployment authentication and side-effect idempotency separately.
package/docs/llms.txt CHANGED
@@ -12,6 +12,7 @@ Start:
12
12
  - First pipeline: ./getting-started.md
13
13
  - CLI: ./cli.md
14
14
  - Local studio: ./studio.md
15
+ - Artifacts and lineage: ./artifacts.md
15
16
  - YAML and JSON pipelines: ./declarative-pipelines.md
16
17
  - Pipeline document JSON Schema v1: ./pipeline-document.schema.json
17
18
  - Choose a pattern: ./recipes.md
@@ -26,6 +27,7 @@ Agent instructions:
26
27
  - Minimal project: ../examples/tubeless.project.ts
27
28
  - Project with custom adapters: ../examples/project/tubeless.project.ts
28
29
  - Maintain stable step IDs.
30
+ - For static parameterized step families, keep expansion synchronous and preserve literal IDs with the recipe-local tuple helper in ../examples/parameterized-steps.ts
29
31
  - Use optional step names only as display overrides; dependencies and selection use IDs.
30
32
  - Use pipeline.toMermaid() to generate static graph documentation.
31
33
  - Use tubeless list to enumerate pipelines from defineProject. Use stable pipeline IDs with inspect, plan, graph, and run; no command modules are discovered from the filesystem or history.
@@ -36,7 +38,7 @@ Agent instructions:
36
38
  - Use tubeless history to list or show recorded runs from SQLite, or pass --trace to inspect a finished NDJSON artifact without importing it. Studio also accepts --trace as a strictly read-only source. Default print is the projection; --events emits raw NDJSON. Trace files may contain sensitive logs and event payloads.
37
39
  - Set dryRun: "skip" on external side effects, or provide a side-effect-free dryRun handler.
38
40
  - Use context.log, context.signal, context.sleep, and progress APIs.
39
- - Use createPipelineTestRuntime from tubeless/testing for deterministic tests with captured logs, statuses, progress, clock, and cancellation.
41
+ - Use createPipelineTestRuntime from tubeless/testing for deterministic tests with captured logs, statuses, progress, clock, and cancellation. Pass overrideStep(step, value) through test-run overrides to supply typed outputs; output schemas still validate and normal step statuses carry outputSource: "override" provenance.
40
42
  - Import compilePipelineDocument from tubeless/project to compile parsed YAML/JSON with an explicit registry; use compiled.get(id) directly or pass selected pipelines to defineProject. Reuse compiled.metadata for project presentation. Import definePipelineCommand from tubeless/cli for explicit adapters; register each pipeline exactly once, directly or through its command in the same project entry list for CLI and Studio.
41
43
  - Omit mapOptions only when validated CLI flags already satisfy same-name pipeline options; map renamed, loaded, or derived values explicitly. --step and --target are argv flags; mapOptions and hooks receive stepIds and targets.
42
44
  - Expose --resume only through managed checkpoint configuration or an explicit resume: true command that handles values.resume. Unsupported commands reject it and omit it from Studio.
@@ -58,6 +60,11 @@ Advanced:
58
60
 
59
61
  - Child pipelines: ./child-pipeline-composition.md
60
62
  - Remote steps: ./remote-step-composition.md
63
+ - Tubeless and Airflow: ./airflow.md
64
+ - Tubeless on Temporal: ./temporal.md
65
+ - Tubeless on Inngest: ./inngest.md
66
+ - Tubeless on Dagster: ./dagster.md
67
+ - Tubeless on AWS Step Functions: ./step-functions.md
61
68
  - Public exports: ./api-reference.md
62
69
  - Machine-readable surface: ./api-report.json
63
70
  Executable examples:
@@ -74,10 +81,32 @@ Executable examples:
74
81
  - ../examples/best-effort.ts
75
82
  - ../examples/child-pipeline.ts
76
83
  - ../examples/remote-steps.ts
84
+ - ../examples/airflow/remote.ts
85
+ - ../examples/airflow/hosted.ts
86
+ - ../examples/airflow/tubeless_example.py
87
+ - ../examples/inngest/pipeline.ts
88
+ - ../examples/inngest/functions.ts
89
+ - ../examples/inngest/client.ts
90
+ - ../examples/inngest/server.ts
91
+ - ../examples/inngest/send.ts
92
+ - ../examples/temporal/pipeline.ts
93
+ - ../examples/temporal/activities.ts
94
+ - ../examples/temporal/workflows.ts
95
+ - ../examples/temporal/worker.ts
96
+ - ../examples/temporal/client.ts
97
+ - ../examples/dagster/pipeline.ts
98
+ - ../examples/dagster/hosted.ts
99
+ - ../examples/dagster/worker.ts
100
+ - ../examples/dagster/definitions.py
101
+ - ../examples/step-functions/pipeline.ts
102
+ - ../examples/step-functions/handler.ts
103
+ - ../examples/step-functions/state-machine.asl.json
104
+ - ../examples/step-functions/template.json
77
105
  - ../examples/fan-out-progress.ts
78
106
  - ../examples/resumable-enrichment.ts
79
107
  - ../examples/cli-job.ts
80
108
  - ../examples/cancellation-and-testing.ts
109
+ - ../examples/step-output-overrides.ts
81
110
  - ../examples/live-tui.ts
82
111
  - ../examples/peloton.ts
83
112
  - ../examples/tracing.ts