tubeless 0.2.12 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/child-execution.d.ts +0 -23
- package/dist/core/child-execution.js +10 -12
- package/dist/core/child-execution.js.map +1 -1
- package/dist/core/child-progress.js +7 -2
- package/dist/core/child-progress.js.map +1 -1
- package/dist/core/iteration.d.ts +27 -0
- package/dist/core/iteration.js +111 -0
- package/dist/core/iteration.js.map +1 -0
- package/dist/core/lifecycle.d.ts +7 -5
- package/dist/core/lifecycle.js +36 -4
- package/dist/core/lifecycle.js.map +1 -1
- package/dist/core/pipeline-artifacts.d.ts +11 -0
- package/dist/core/pipeline-artifacts.js +2 -0
- package/dist/core/pipeline-artifacts.js.map +1 -0
- package/dist/core/pipeline-compiler.d.ts +1 -0
- package/dist/core/pipeline-compiler.js +1 -0
- package/dist/core/pipeline-compiler.js.map +1 -1
- package/dist/core/pipeline-definition-identity.d.ts +2 -2
- package/dist/core/pipeline-definition-identity.js +28 -5
- package/dist/core/pipeline-definition-identity.js.map +1 -1
- package/dist/core/pipeline-execute.d.ts +1 -6
- package/dist/core/pipeline-execute.js +28 -215
- package/dist/core/pipeline-execute.js.map +1 -1
- package/dist/core/pipeline-execution-error.d.ts +35 -0
- package/dist/core/pipeline-execution-error.js +195 -0
- package/dist/core/pipeline-execution-error.js.map +1 -0
- package/dist/core/pipeline-identity.d.ts +1 -0
- package/dist/core/pipeline-identity.js +1 -0
- package/dist/core/pipeline-identity.js.map +1 -1
- package/dist/core/pipeline-mermaid.js +4 -1
- package/dist/core/pipeline-mermaid.js.map +1 -1
- package/dist/core/pipeline-run-state.d.ts +4 -4
- package/dist/core/pipeline-run-state.js +10 -16
- package/dist/core/pipeline-run-state.js.map +1 -1
- package/dist/core/pipeline-step-executor.d.ts +2 -0
- package/dist/core/pipeline-step-executor.js +11 -3
- package/dist/core/pipeline-step-executor.js.map +1 -1
- package/dist/core/pipeline-steps.d.ts +20 -0
- package/dist/core/pipeline-steps.js +62 -0
- package/dist/core/pipeline-steps.js.map +1 -1
- package/dist/core/pipeline-types.d.ts +15 -2
- package/dist/core/pipeline-types.js.map +1 -1
- package/dist/core/pipeline.d.ts +4 -1
- package/dist/core/pipeline.js +18 -3
- package/dist/core/pipeline.js.map +1 -1
- package/dist/project/project-compiler.d.ts +3 -31
- package/dist/project/project-compiler.js +11 -180
- package/dist/project/project-compiler.js.map +1 -1
- package/dist/project/project-registry.d.ts +44 -0
- package/dist/project/project-registry.js +107 -0
- package/dist/project/project-registry.js.map +1 -0
- package/dist/project/project-step-graph.d.ts +9 -0
- package/dist/project/project-step-graph.js +99 -0
- package/dist/project/project-step-graph.js.map +1 -0
- package/dist/render/render.js +1 -1
- package/dist/render/render.js.map +1 -1
- package/dist/reporter/interactive-reporter.js +6 -3
- package/dist/reporter/interactive-reporter.js.map +1 -1
- package/dist/reporter/live-ticker-frame.d.ts +21 -0
- package/dist/reporter/live-ticker-frame.js +196 -0
- package/dist/reporter/live-ticker-frame.js.map +1 -0
- package/dist/reporter/live-ticker-protocol.d.ts +41 -0
- package/dist/reporter/live-ticker-protocol.js +59 -0
- package/dist/reporter/live-ticker-protocol.js.map +1 -0
- package/dist/reporter/live-ticker-worker.js +11 -25
- package/dist/reporter/live-ticker-worker.js.map +1 -1
- package/dist/reporter/live-ticker.d.ts +0 -15
- package/dist/reporter/live-ticker.js +11 -216
- package/dist/reporter/live-ticker.js.map +1 -1
- package/dist/reporter/reporter.js +7 -6
- package/dist/reporter/reporter.js.map +1 -1
- package/dist/run-store/definition-projection.d.ts +8 -0
- package/dist/run-store/definition-projection.js +89 -0
- package/dist/run-store/definition-projection.js.map +1 -0
- package/dist/run-store/run-projection.d.ts +15 -0
- package/dist/run-store/run-projection.js +203 -0
- package/dist/run-store/run-projection.js.map +1 -0
- package/dist/run-store/run-store-codec.d.ts +2 -0
- package/dist/run-store/run-store-codec.js +11 -0
- package/dist/run-store/run-store-codec.js.map +1 -1
- package/dist/run-store/run-store-event-decoder.js +3 -0
- package/dist/run-store/run-store-event-decoder.js.map +1 -1
- package/dist/run-store/run-store-reader.d.ts +7 -0
- package/dist/run-store/run-store-reader.js +31 -0
- package/dist/run-store/run-store-reader.js.map +1 -0
- package/dist/run-store/run-store-sqlite.js +23 -7
- package/dist/run-store/run-store-sqlite.js.map +1 -1
- package/dist/run-store/run-store.d.ts +16 -4
- package/dist/run-store/run-store.js +14 -263
- package/dist/run-store/run-store.js.map +1 -1
- package/dist/studio/run-store-ui-api.d.ts +13 -0
- package/dist/studio/run-store-ui-api.js +163 -0
- package/dist/studio/run-store-ui-api.js.map +1 -0
- package/dist/studio/run-store-ui-client-browser.js +2 -2
- package/dist/studio/run-store-ui-client-transport.js +28 -3
- package/dist/studio/run-store-ui-client-transport.js.map +1 -1
- package/dist/studio/run-store-ui-client.d.ts +2 -1
- package/dist/studio/run-store-ui-client.js +15 -9
- package/dist/studio/run-store-ui-client.js.map +1 -1
- package/dist/studio/run-store-ui-http.d.ts +11 -0
- package/dist/studio/run-store-ui-http.js +52 -0
- package/dist/studio/run-store-ui-http.js.map +1 -1
- package/dist/studio/run-store-ui-state.d.ts +1 -1
- package/dist/studio/run-store-ui-state.js +10 -28
- package/dist/studio/run-store-ui-state.js.map +1 -1
- package/dist/studio/run-store-ui.d.ts +5 -11
- package/dist/studio/run-store-ui.js +5 -203
- package/dist/studio/run-store-ui.js.map +1 -1
- package/dist/testing/testing.d.ts +18 -2
- package/dist/testing/testing.js +35 -2
- package/dist/testing/testing.js.map +1 -1
- package/dist/tracing/artifact-metadata.d.ts +28 -0
- package/dist/tracing/artifact-metadata.js +105 -0
- package/dist/tracing/artifact-metadata.js.map +1 -0
- package/dist/tracing/tracing-codec.d.ts +1 -1
- package/dist/tracing/tracing-codec.js +1 -1
- package/dist/tracing/tracing-codec.js.map +1 -1
- package/dist/tracing/tracing-constants.d.ts +1 -1
- package/dist/tracing/tracing-constants.js +1 -1
- package/dist/tracing/tracing-constants.js.map +1 -1
- package/dist/tracing/tracing-contracts.d.ts +3 -1
- package/dist/tracing/tracing-contracts.js.map +1 -1
- package/dist/tracing/tracing-internal.d.ts +2 -0
- package/dist/tracing/tracing-internal.js +36 -13
- package/dist/tracing/tracing-internal.js.map +1 -1
- package/dist/tracing/tracing-schema.d.ts +330 -49
- package/dist/tracing/tracing-schema.js +102 -10
- package/dist/tracing/tracing-schema.js.map +1 -1
- package/dist/workbench/workbench-command-execution.d.ts +6 -0
- package/dist/workbench/workbench-command-execution.js +48 -0
- package/dist/workbench/workbench-command-execution.js.map +1 -0
- package/dist/workbench/workbench-history.js +11 -18
- package/dist/workbench/workbench-history.js.map +1 -1
- package/dist/workbench/workbench-launch-session.d.ts +2 -2
- package/dist/workbench/workbench-launch-session.js.map +1 -1
- package/dist/workbench/workbench-run.d.ts +0 -4
- package/dist/workbench/workbench-run.js +3 -47
- package/dist/workbench/workbench-run.js.map +1 -1
- package/dist/workbench/workbench-studio-launcher.d.ts +27 -0
- package/dist/workbench/workbench-studio-launcher.js +95 -0
- package/dist/workbench/workbench-studio-launcher.js.map +1 -0
- package/dist/workbench/workbench-ui.js +10 -83
- package/dist/workbench/workbench-ui.js.map +1 -1
- package/docs/README.md +13 -7
- package/docs/agent-guide.md +69 -6
- package/docs/airflow.md +187 -0
- package/docs/api-reference.md +87 -76
- package/docs/api-report.json +17 -6
- package/docs/artifacts.md +173 -0
- package/docs/child-pipeline-composition.md +65 -2
- package/docs/comparison.md +20 -0
- package/docs/concepts.md +70 -2
- package/docs/dagster.md +161 -0
- package/docs/inngest.md +163 -0
- package/docs/llms.txt +30 -1
- package/docs/recipes.md +82 -33
- package/docs/remote-step-composition.md +6 -0
- package/docs/step-functions.md +190 -0
- package/docs/studio.md +31 -12
- package/docs/temporal.md +183 -0
- package/examples/airflow/contract.ts +46 -0
- package/examples/airflow/hosted.ts +70 -0
- package/examples/airflow/remote.ts +175 -0
- package/examples/airflow/tubeless_example.py +49 -0
- package/examples/airflow/worker.ts +24 -0
- package/examples/artifact-lineage.ts +70 -0
- package/examples/dagster/definitions.py +59 -0
- package/examples/dagster/hosted.ts +78 -0
- package/examples/dagster/pipeline.ts +62 -0
- package/examples/dagster/test_definitions.py +47 -0
- package/examples/dagster/tsconfig.build.json +10 -0
- package/examples/dagster/worker.ts +22 -0
- package/examples/inngest/client.ts +4 -0
- package/examples/inngest/functions.ts +65 -0
- package/examples/inngest/pipeline.ts +41 -0
- package/examples/inngest/send.ts +11 -0
- package/examples/inngest/server.ts +17 -0
- package/examples/inngest/tsconfig.build.json +10 -0
- package/examples/iteration.ts +52 -0
- package/examples/parameterized-steps.ts +96 -0
- package/examples/project/tubeless.project.ts +46 -0
- package/examples/step-functions/handler.ts +110 -0
- package/examples/step-functions/input.json +1 -0
- package/examples/step-functions/pipeline.ts +41 -0
- package/examples/step-functions/state-machine.asl.json +41 -0
- package/examples/step-functions/template.json +42 -0
- package/examples/step-output-overrides.ts +41 -0
- package/examples/temporal/activities.ts +94 -0
- package/examples/temporal/client.ts +19 -0
- package/examples/temporal/pipeline.ts +41 -0
- package/examples/temporal/tsconfig.build.json +10 -0
- package/examples/temporal/worker.ts +21 -0
- package/examples/temporal/workflows.ts +16 -0
- package/package.json +13 -4
- package/skills/tubeless/SKILL.md +30 -3
package/docs/comparison.md
CHANGED
|
@@ -54,3 +54,23 @@ one of those tools in either of two ways:
|
|
|
54
54
|
The host owns persistence and acknowledgement. A rejected run tells it that
|
|
55
55
|
processing failed. The parent Tubeless process does not restore itself after
|
|
56
56
|
exit; if that is required, the host must provide it.
|
|
57
|
+
|
|
58
|
+
The [Tubeless ↔ Airflow guide](./airflow.md) connects both patterns in one example:
|
|
59
|
+
submit a DAG through `fromRemote`, run a Tubeless pipeline inside its task, and
|
|
60
|
+
validate the returned XCom before continuing locally.
|
|
61
|
+
|
|
62
|
+
The [Temporal guide](./temporal.md) hosts a pipeline inside an Activity and keeps
|
|
63
|
+
its execution outside replayed Workflow code, with Activity-owned retries,
|
|
64
|
+
heartbeats, and cancellation.
|
|
65
|
+
|
|
66
|
+
The [Inngest guide](./inngest.md) hosts a pipeline inside `step.run`. Inngest
|
|
67
|
+
saves the successful result and retries failed attempts; each attempt executes
|
|
68
|
+
the whole Tubeless pipeline. The example includes an event sender and local server.
|
|
69
|
+
|
|
70
|
+
The [Dagster guide](./dagster.md) materializes an asset through Dagster Pipes,
|
|
71
|
+
with progress logs, content data versions, and a trace for Tubeless Studio.
|
|
72
|
+
Dagster owns retries of the asset; Tubeless owns execution of its internal steps.
|
|
73
|
+
|
|
74
|
+
The [Step Functions guide](./step-functions.md) invokes a pipeline inside a Lambda
|
|
75
|
+
Task and includes a SAM deployment. Step Functions owns workflow recovery;
|
|
76
|
+
Tubeless supplies typed steps and progress within each Lambda invocation.
|
package/docs/concepts.md
CHANGED
|
@@ -399,14 +399,15 @@ result.steps.find((step) => step.attemptId)?.attemptId;
|
|
|
399
399
|
result.finishedAtMs - result.startedAtMs;
|
|
400
400
|
```
|
|
401
401
|
|
|
402
|
-
|
|
402
|
+
New trace events use the version 3 discriminated payload model. Every event carries
|
|
403
403
|
a unique execution `runId` and keeps reusable external correlation separate.
|
|
404
404
|
`tubeless/tracing` intentionally exposes only that typed event union, the exporter
|
|
405
405
|
interface, and exporter composition. Concrete JSON and telemetry SDK adapters
|
|
406
406
|
belong in the workbench or application integration; see the
|
|
407
407
|
[tracing recipe](../examples/tracing.ts). This narrow API does not weaken the
|
|
408
408
|
recording contract: existing version 2 NDJSON traces remain readable by
|
|
409
|
-
`tubeless history` and `tubeless ui`.
|
|
409
|
+
`tubeless history` and `tubeless ui`. Version 3 adds iteration relationships for
|
|
410
|
+
repeated child runs; their execution IDs remain unique.
|
|
410
411
|
|
|
411
412
|
### Local event store and studio
|
|
412
413
|
|
|
@@ -624,6 +625,73 @@ interleave, or advance time differently; call `test.abort()` to exercise the
|
|
|
624
625
|
normal cancellation path. Use your test framework to assert on the results. Replace external I/O yourself;
|
|
625
626
|
the test runtime does not mock network or filesystem operations.
|
|
626
627
|
|
|
628
|
+
### Supplying step outputs
|
|
629
|
+
|
|
630
|
+
Use `overrideStep` from `tubeless/testing` to supply a typed intermediate value
|
|
631
|
+
without modifying the pipeline definition:
|
|
632
|
+
|
|
633
|
+
```ts
|
|
634
|
+
const test = createPipelineTestRuntime();
|
|
635
|
+
const run = await test.run(pipeline, options, {
|
|
636
|
+
stepIds: ["load", "summarize"],
|
|
637
|
+
overrides: [overrideStep(load, ["alice", "bob"])],
|
|
638
|
+
});
|
|
639
|
+
```
|
|
640
|
+
|
|
641
|
+
Each override pairs an actual step reference with a resolved handler output.
|
|
642
|
+
For schema-backed steps, supply the **schema input** type: `outputSchema` still
|
|
643
|
+
validates and transforms it, and dependents receive the transformed output.
|
|
644
|
+
Invalid values fail with `TUBELESS_STEP_OUTPUT_VALIDATION_FAILED`. An explicit
|
|
645
|
+
`undefined` is a published value when the step's type and schema allow it.
|
|
646
|
+
Functions can be domain values; they are never called as lazy factories.
|
|
647
|
+
Await asynchronous fixture setup before constructing overrides.
|
|
648
|
+
|
|
649
|
+
A selected overridden step never calls its `run`, `skip`, or custom `dryRun`
|
|
650
|
+
handler. It bypasses required-input and failure-gate checks and `dryRun: "skip"`,
|
|
651
|
+
since the supplied value replaces its entire computation. Overriding a child,
|
|
652
|
+
fan-out, or remote wrapper also bypasses item enumeration, mappings, adapter
|
|
653
|
+
calls, and child launches. Overrides belong to this run only and do not flow
|
|
654
|
+
into children. Duplicate overrides and foreign references (including same-named
|
|
655
|
+
steps from another graph) reject before execution. Custom pipeline wrappers
|
|
656
|
+
are unsupported; use the object returned by `definePipeline`.
|
|
657
|
+
|
|
658
|
+
Selection and scheduling remain explicit:
|
|
659
|
+
|
|
660
|
+
- Overrides do not select steps, prune ancestors, or change target closure.
|
|
661
|
+
A filtered step stays filtered and does not validate or publish its override.
|
|
662
|
+
To isolate a downstream branch, use exact `stepIds` including the supplied
|
|
663
|
+
intermediate and its dependents, excluding upstream I/O. Plans describe the
|
|
664
|
+
ordinary graph and do not apply overrides.
|
|
665
|
+
- Scheduling still waits for all prerequisite steps to reach terminal states.
|
|
666
|
+
Override validation occupies a concurrency slot. Fail-fast can prevent an
|
|
667
|
+
override from starting; `continueOnError` allows it to publish after upstream
|
|
668
|
+
failure, while the run retains that failure.
|
|
669
|
+
- Cancellation prevents unstarted overrides and is checked again after async
|
|
670
|
+
output validation. Successfully overridden steps unlock required dependents
|
|
671
|
+
and failure gates and supply optional inputs normally.
|
|
672
|
+
- Finalization and `resultSchema` use the supplied outputs through the usual
|
|
673
|
+
boundaries, including required finalizers and the default last-step result.
|
|
674
|
+
|
|
675
|
+
An override uses the normal lifecycle: `running`, then `completed`, `failed`,
|
|
676
|
+
or `cancelled`. From the start of its validation attempt, reports and hooks
|
|
677
|
+
carry `outputSource: "override"`. The same metadata survives validation failure
|
|
678
|
+
or cancellation. Successful overrides emit `onStepComplete`, so tests exercise
|
|
679
|
+
the same completion callbacks and dependency rules as handler-produced outputs.
|
|
680
|
+
There is no separate override status or hook.
|
|
681
|
+
|
|
682
|
+
Traces use the existing `step.running`, `step.complete`, `step.failed`, and
|
|
683
|
+
`step.cancelled` events with `payload.outputSource: "override"`; they do not
|
|
684
|
+
record the supplied value. Both CLI reporters and Studio show the ordinary
|
|
685
|
+
status with an override annotation, such as **completed (overridden)** or
|
|
686
|
+
**failed (overridden)**. Existing recordings remain readable. Ordinary runs omit
|
|
687
|
+
this metadata, as do filtered overrides and overrides cancelled before their
|
|
688
|
+
validation attempt starts; absence does not prove a handler ran.
|
|
689
|
+
|
|
690
|
+
Overrides are available only through the test runtime's `run` and `runOrThrow`.
|
|
691
|
+
Ordinary pipeline controls, CLI flags, and Studio launch forms do not expose
|
|
692
|
+
them, and trace recordings cannot automatically be replayed as override values.
|
|
693
|
+
See the [executable override recipe](../examples/step-output-overrides.ts).
|
|
694
|
+
|
|
627
695
|
## Implementation guidelines
|
|
628
696
|
|
|
629
697
|
- Keep step IDs stable, use `name` only for a friendlier display label, and keep
|
package/docs/dagster.md
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# Tubeless on Dagster
|
|
2
|
+
|
|
3
|
+
Run a Tubeless pipeline as a Dagster asset using Dagster Pipes. Dagster owns
|
|
4
|
+
asset scheduling, dependencies, and retries; Tubeless owns the typed steps,
|
|
5
|
+
progress, and execution trace inside the asset's subprocess.
|
|
6
|
+
|
|
7
|
+
```text
|
|
8
|
+
Dagster asset → PipesSubprocessClient → Node.js → Tubeless pipeline
|
|
9
|
+
← logs, traces, and materialization metadata ←
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
The example normalizes rows, validates the dataset, and writes a JSON artifact.
|
|
13
|
+
After successful execution, it reports a materialization with the row count,
|
|
14
|
+
artifact path, content data version, Tubeless run ID, and trace path. Operators
|
|
15
|
+
can follow progress in Dagster and inspect individual pipeline steps in Tubeless
|
|
16
|
+
Studio. See Dagster's [TypeScript Pipes integration](https://docs.dagster.io/integrations/libraries/typescript).
|
|
17
|
+
|
|
18
|
+
## The example
|
|
19
|
+
|
|
20
|
+
| File | Responsibility |
|
|
21
|
+
| ---------------------------------------------------------------- | ------------------------------------------------------------------------- |
|
|
22
|
+
| [`pipeline.ts`](../examples/dagster/pipeline.ts) | Normalize, validate, and publish through public Tubeless imports |
|
|
23
|
+
| [`hosted.ts`](../examples/dagster/hosted.ts) | Bridge Pipes context, logs, traces, and materialization reporting |
|
|
24
|
+
| [`worker.ts`](../examples/dagster/worker.ts) | Open and close Pipes, forward process termination to Tubeless |
|
|
25
|
+
| [`definitions.py`](../examples/dagster/definitions.py) | Define the asset, launch Node, save the trace, and yield materializations |
|
|
26
|
+
| [`test_definitions.py`](../examples/dagster/test_definitions.py) | Exercise the real Python host and Node worker |
|
|
27
|
+
|
|
28
|
+
The pipeline has no Dagster imports and also runs from the local project.
|
|
29
|
+
The bridge uses `@dagster-io/dagster-pipes` 0.1.0, installed as a repository
|
|
30
|
+
development dependency. Applications adopting this bridge install the SDK in
|
|
31
|
+
their own project. Tubeless itself retains a dependency-free runtime.
|
|
32
|
+
The Python example is tested with Dagster 1.13.24.
|
|
33
|
+
|
|
34
|
+
## Run locally
|
|
35
|
+
|
|
36
|
+
Use Node.js 22.6 or later, Bun, Python 3.12, and
|
|
37
|
+
[uv](https://docs.astral.sh/uv/getting-started/installation/). From the repository root:
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
bun ci
|
|
41
|
+
bun run build
|
|
42
|
+
bunx tsc -p examples/dagster/tsconfig.build.json
|
|
43
|
+
uv venv --python 3.12 .context/dagster-venv
|
|
44
|
+
uv pip install --python .context/dagster-venv/bin/python \
|
|
45
|
+
'dagster==1.13.24' 'dagster-webserver==1.13.24'
|
|
46
|
+
.context/dagster-venv/bin/dagster dev -f examples/dagster/definitions.py
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
The compiler writes JavaScript to `.context/dagster/`. Open the local Dagster
|
|
50
|
+
UI at `http://localhost:3000`, select `normalized_rows`, and materialize it.
|
|
51
|
+
The execution host must have Node, the compiled worker, and its installed
|
|
52
|
+
dependencies available. Keep the dev server's process running while you inspect
|
|
53
|
+
the asset and run history.
|
|
54
|
+
|
|
55
|
+
The default input produces `{"rows":["alpha","beta"]}` in
|
|
56
|
+
`.context/dagster-artifacts/<dagster-run-id>/rows.json`. Open the materialization's
|
|
57
|
+
metadata to find the artifact, row count, data version, and Tubeless trace.
|
|
58
|
+
These paths refer to the execution host's filesystem. For distributed workers,
|
|
59
|
+
publish artifacts and traces to shared storage and report accessible locations.
|
|
60
|
+
|
|
61
|
+
To change the input, open the asset's launch configuration and supply:
|
|
62
|
+
|
|
63
|
+
```yaml
|
|
64
|
+
ops:
|
|
65
|
+
normalized_rows:
|
|
66
|
+
config:
|
|
67
|
+
lines: [" Alpha ", "Beta", "ALPHA"]
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Use the local Tubeless command below for previews. The Dagster asset always
|
|
71
|
+
publishes: Pipes requires asset outputs even when an asset declares its output
|
|
72
|
+
optional. The host disables implicit materializations and requires exactly one
|
|
73
|
+
explicit materialization from the worker, so a missing report fails the attempt.
|
|
74
|
+
The worker rejects `dryRun: true` instead of accidentally publishing a preview.
|
|
75
|
+
|
|
76
|
+
## Logs, traces, and asset identity
|
|
77
|
+
|
|
78
|
+
The bridge forwards pipeline logs and progress counts through the Pipes logger.
|
|
79
|
+
It sends raw Tubeless trace events as Pipes custom messages. After a successful
|
|
80
|
+
subprocess exit, the Python host writes these events to `trace.ndjson`.
|
|
81
|
+
The sample does not retain a separate trace file for failed or
|
|
82
|
+
killed subprocesses; their Pipes logs remain in Dagster. For large or long-running
|
|
83
|
+
jobs, stream traces to your own durable exporter rather than accumulating custom
|
|
84
|
+
messages in the host.
|
|
85
|
+
|
|
86
|
+
Copy the recorded trace path into these commands to inspect a finished run:
|
|
87
|
+
|
|
88
|
+
```sh
|
|
89
|
+
bun run tubeless -- history --trace .context/dagster-artifacts/<dagster-run-id>/trace.ndjson
|
|
90
|
+
bun run tubeless -- ui --trace .context/dagster-artifacts/<dagster-run-id>/trace.ndjson
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Replace `<dagster-run-id>` with the actual ID before running the commands.
|
|
94
|
+
Trace files contain recorded logs and event payloads; apply your application's
|
|
95
|
+
redaction policy at the exporter.
|
|
96
|
+
|
|
97
|
+
Each Tubeless attempt has a fresh `runId`. Its correlation ID combines the
|
|
98
|
+
Dagster run ID and asset key and stays stable across step retries within that
|
|
99
|
+
run. The optional `parentRunId` in the bridge is reserved for a known Tubeless
|
|
100
|
+
execution; a Dagster run ID is a separate identity.
|
|
101
|
+
|
|
102
|
+
The materialization's `dataVersion` hashes the normalized row array, so repeated
|
|
103
|
+
attempts with the same rows have the same version. It describes logical content,
|
|
104
|
+
not the file's formatting or the implementation release. Tubeless's
|
|
105
|
+
`implementationVersion` separately identifies the example's handler release.
|
|
106
|
+
This recipe handles one selected asset per invocation.
|
|
107
|
+
|
|
108
|
+
## Retries and cancellation
|
|
109
|
+
|
|
110
|
+
Dagster retries the asset at most twice after its initial attempt. An exception
|
|
111
|
+
from `runOrThrow` causes a nonzero worker exit, which fails the asset attempt.
|
|
112
|
+
Validation failure prevents publication and materialization. Successful
|
|
113
|
+
publication is reported only after the pipeline finishes.
|
|
114
|
+
|
|
115
|
+
A retry runs the whole Tubeless pipeline again. The example replaces the same
|
|
116
|
+
run-specific JSON dataset; a new Dagster run gets a separate directory. This
|
|
117
|
+
does not make arbitrary side effects exactly-once. If publication succeeds and
|
|
118
|
+
reporting fails, Dagster may retry. Use idempotent writes or business-level keys
|
|
119
|
+
when adapting the pipeline to external services. Split independently retryable
|
|
120
|
+
datasets into separate Dagster assets when you need recovery at those boundaries.
|
|
121
|
+
|
|
122
|
+
The default `PipesSubprocessClient` forwards host termination to its subprocess.
|
|
123
|
+
The worker turns `SIGTERM` and `SIGINT` into an abort signal and closes Pipes
|
|
124
|
+
when execution settles. Normalization cooperates through `context.sleep`.
|
|
125
|
+
Cancellation does not roll back an artifact already written, and blocking I/O
|
|
126
|
+
can delay it; Dagster may forcibly stop a subprocess that exceeds its termination
|
|
127
|
+
timeout. See [Dagster Pipes](https://docs.dagster.io/integrations/external-pipelines/using-dagster-pipes)
|
|
128
|
+
for other execution environments.
|
|
129
|
+
|
|
130
|
+
## Run the pipeline directly
|
|
131
|
+
|
|
132
|
+
Use the same pipeline through the example project for local iteration:
|
|
133
|
+
|
|
134
|
+
```sh
|
|
135
|
+
bun run tubeless -- plan --project examples/project/tubeless.project.ts dagster-dataset
|
|
136
|
+
bun run tubeless -- run --project examples/project/tubeless.project.ts dagster-dataset -- \
|
|
137
|
+
--lines " Alpha " --lines "Beta" --output-path ./rows.json --dry-run
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Local execution writes no Dagster metadata. Relative output paths resolve from
|
|
141
|
+
the project execution directory. The Pipes worker requires a real Pipes context
|
|
142
|
+
and rejects standalone launches without one.
|
|
143
|
+
|
|
144
|
+
## Verification
|
|
145
|
+
|
|
146
|
+
`make check` compiles the TypeScript examples and tests the real Pipes SDK in
|
|
147
|
+
subprocesses without a Dagster service. Tests cover metadata and content versions,
|
|
148
|
+
retry identity, raw traces, malformed inputs, validation failure, dry runs, and
|
|
149
|
+
cooperative termination.
|
|
150
|
+
|
|
151
|
+
After the local setup and worker compilation, run the Python integration tests:
|
|
152
|
+
|
|
153
|
+
```sh
|
|
154
|
+
.context/dagster-venv/bin/python -m unittest discover -s examples/dagster -p 'test_*.py'
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
These launch the compiled Node worker from the real Dagster execution engine.
|
|
158
|
+
They verify a successful materialization and trace, and a failed dataset that
|
|
159
|
+
exhausts Dagster's retry policy without materializing an asset.
|
|
160
|
+
The Python tests are optional and are not part of the dependency-free package's
|
|
161
|
+
`make check` environment.
|
package/docs/inngest.md
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# Tubeless on Inngest
|
|
2
|
+
|
|
3
|
+
Run a Tubeless pipeline inside an Inngest `step.run`. Inngest owns event delivery,
|
|
4
|
+
retries, and durable step results; Tubeless owns the typed graph, progress, and
|
|
5
|
+
execution report within each attempt.
|
|
6
|
+
|
|
7
|
+
```text
|
|
8
|
+
Event sender → Inngest function → step.run → Tubeless pipeline
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Keep the pipeline invocation inside the durable step callback. Inngest may
|
|
12
|
+
execute the function handler several times as it restores saved step results.
|
|
13
|
+
Code outside steps can execute again. See
|
|
14
|
+
[how Inngest functions execute](https://www.inngest.com/docs/learn/how-functions-are-executed).
|
|
15
|
+
|
|
16
|
+
## The example
|
|
17
|
+
|
|
18
|
+
| File | Responsibility |
|
|
19
|
+
| -------------------------------------------------- | ------------------------------------------------------------------------------- |
|
|
20
|
+
| [`pipeline.ts`](../examples/inngest/pipeline.ts) | Normalize and deduplicate rows using public Tubeless imports |
|
|
21
|
+
| [`functions.ts`](../examples/inngest/functions.ts) | Validate event data and run the pipeline in a durable step with correlated logs |
|
|
22
|
+
| [`client.ts`](../examples/inngest/client.ts) | Configure the shared Inngest client |
|
|
23
|
+
| [`server.ts`](../examples/inngest/server.ts) | Serve the function at `/api/inngest` using Node's HTTP server |
|
|
24
|
+
| [`send.ts`](../examples/inngest/send.ts) | Send a sample event and print its accepted event IDs |
|
|
25
|
+
|
|
26
|
+
The pipeline has no Inngest imports and also runs locally. The example uses
|
|
27
|
+
Inngest TypeScript SDK 4.21.0 and `@inngest/test` 1.0.0, installed as development
|
|
28
|
+
dependencies here. Consumer applications install `inngest` themselves. Tubeless
|
|
29
|
+
keeps its dependency-free runtime and adds no Inngest public API.
|
|
30
|
+
|
|
31
|
+
SDK v4 puts `triggers` in the `createFunction` options. Older v3 examples use a
|
|
32
|
+
separate trigger argument; see the
|
|
33
|
+
[v4 migration guide](https://www.inngest.com/docs/reference/typescript/v4/migrations/v3-to-v4).
|
|
34
|
+
|
|
35
|
+
## Run locally
|
|
36
|
+
|
|
37
|
+
Use Node.js 22.6 or later and Bun 1.3.14 or later. From the repository root:
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
bun ci
|
|
41
|
+
bun run build
|
|
42
|
+
bunx tsc -p examples/inngest/tsconfig.build.json
|
|
43
|
+
INNGEST_DEV=1 node .context/inngest/server.js
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
The compiler writes JavaScript to `.context/inngest/`. The server listens on
|
|
47
|
+
`127.0.0.1:3000`. In a second terminal, start the
|
|
48
|
+
[Inngest Dev Server](https://www.inngest.com/docs/local-development):
|
|
49
|
+
|
|
50
|
+
```sh
|
|
51
|
+
npx inngest-cli@latest dev -u http://127.0.0.1:3000/api/inngest
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Open `http://localhost:8288` and confirm that `tubeless-example` has synced and
|
|
55
|
+
lists `normalize-rows`. In a third terminal, send the sample event:
|
|
56
|
+
|
|
57
|
+
```sh
|
|
58
|
+
INNGEST_DEV=1 node .context/inngest/send.js
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The sender prints accepted event IDs. It does not wait for the pipeline result.
|
|
62
|
+
Open the resulting run in the Dev Server UI: `run-pipeline` returns
|
|
63
|
+
`rows: ["alpha", "beta"]`, `count: 2`, a Tubeless `runId`, and `preview: false`.
|
|
64
|
+
Sending again creates another event and function run. This example does not
|
|
65
|
+
deduplicate separate submissions.
|
|
66
|
+
|
|
67
|
+
`INNGEST_DEV=1` selects local development for both server and sender. For a
|
|
68
|
+
deployment, host the endpoint where Inngest can reach it, remove that setting,
|
|
69
|
+
configure `INNGEST_SIGNING_KEY` and `INNGEST_EVENT_KEY` through your host's secret
|
|
70
|
+
configuration, and sync the app. The example server always binds to
|
|
71
|
+
`127.0.0.1:3000`. To reuse it in production, run a reverse proxy on the same host
|
|
72
|
+
and forward your public HTTPS `/api/inngest` endpoint to
|
|
73
|
+
`http://127.0.0.1:3000/api/inngest`, preserving the request method, body, and
|
|
74
|
+
headers. Sync the public URL with Inngest. If your deployment cannot proxy to
|
|
75
|
+
that loopback listener, register `normalizeFunction` using your application's
|
|
76
|
+
Inngest framework adapter instead of running `server.ts`.
|
|
77
|
+
See [serving functions](https://www.inngest.com/docs/reference/typescript/serve)
|
|
78
|
+
and [client configuration](https://www.inngest.com/docs/reference/typescript/client/create).
|
|
79
|
+
|
|
80
|
+
## Validation, results, and observation
|
|
81
|
+
|
|
82
|
+
The function validates `event.data` before calling `runOrThrow`. Invalid rows or
|
|
83
|
+
control metadata throw Inngest's `NonRetriableError` directly. A TypeScript type
|
|
84
|
+
on the sender cannot validate incoming events. Keep this validation outside the
|
|
85
|
+
Tubeless invocation so its execution-error wrapper cannot hide the host's
|
|
86
|
+
non-retryable error type. See
|
|
87
|
+
[Inngest errors](https://www.inngest.com/docs/reference/typescript/errors).
|
|
88
|
+
|
|
89
|
+
The correlation ID combines the Inngest run ID and durable step ID. It stays
|
|
90
|
+
stable across retries of that step. Each pipeline attempt gets a fresh Tubeless
|
|
91
|
+
`runId`. Only pass `parentRunId` when it identifies a known Tubeless execution;
|
|
92
|
+
an Inngest run ID belongs in correlation metadata.
|
|
93
|
+
|
|
94
|
+
The pipeline returns plain JSON data. Inngest serializes step results, so avoid
|
|
95
|
+
returning contexts, errors, signals, class instances, or a whole execution report.
|
|
96
|
+
Large datasets should live in external storage with a small reference returned
|
|
97
|
+
from the step. See [step result serialization](https://www.inngest.com/docs/reference/typescript/functions/step-run).
|
|
98
|
+
|
|
99
|
+
Pipeline logs, progress hooks, and trace events go through the function logger.
|
|
100
|
+
The Inngest UI records one durable step for the pipeline. To inspect individual
|
|
101
|
+
Tubeless steps in Studio, retain raw trace events using an application-owned
|
|
102
|
+
exporter. Logger envelopes are not importable Tubeless NDJSON traces. Apply
|
|
103
|
+
redaction before retaining or forwarding events.
|
|
104
|
+
|
|
105
|
+
## Retries and durable boundaries
|
|
106
|
+
|
|
107
|
+
The function sets `retries: 2`: up to three attempts for the durable step.
|
|
108
|
+
`runOrThrow` lets execution failures reach Inngest. A failed attempt reruns the
|
|
109
|
+
entire pipeline, including any successful internal steps. Once Inngest saves
|
|
110
|
+
the durable step's successful result, subsequent handler executions reuse that
|
|
111
|
+
result, including the original Tubeless `runId`, without invoking the pipeline.
|
|
112
|
+
|
|
113
|
+
Normalization and deduplication are safe to repeat. If you add writes, use
|
|
114
|
+
business-level idempotency keys: a process can fail after a write but before its
|
|
115
|
+
result is saved. Correlation IDs do not deduplicate side effects. See
|
|
116
|
+
[Inngest steps](https://www.inngest.com/docs/learn/inngest-steps).
|
|
117
|
+
|
|
118
|
+
For independently retryable phases, define multiple Inngest steps at the
|
|
119
|
+
function level, each calling a smaller Tubeless pipeline and passing JSON
|
|
120
|
+
results to the next phase. Do not call Inngest step tools from inside Tubeless
|
|
121
|
+
handlers or hooks nested in `step.run`. Place durable sleeps and event waits
|
|
122
|
+
between these phases. Tubeless `context.sleep` is an ordinary process-local
|
|
123
|
+
wait, and internal progress is not a durable checkpoint.
|
|
124
|
+
|
|
125
|
+
## Cancellation and dry runs
|
|
126
|
+
|
|
127
|
+
Inngest cancellation prevents subsequent steps from running; it does not stop
|
|
128
|
+
an already executing step. This adapter therefore does not supply a host
|
|
129
|
+
cancellation signal. Cancelling a run in the dashboard can leave its active
|
|
130
|
+
Tubeless pipeline running to completion. Closing the sender does not cancel
|
|
131
|
+
the function. See [Inngest cancellation](https://www.inngest.com/docs/features/inngest-functions/cancellation).
|
|
132
|
+
|
|
133
|
+
The pipeline cooperates with signals supplied by local callers. Applications
|
|
134
|
+
that need an execution deadline can supply their own signal and forward it to
|
|
135
|
+
I/O, but a local abort is separate from Inngest cancellation. Keep work within
|
|
136
|
+
the execution limits of your deployment and make side effects safe to retry.
|
|
137
|
+
|
|
138
|
+
Set `dryRun: true` in the sender's job to forward Tubeless dry-run controls.
|
|
139
|
+
This still sends a real event and runs an Inngest function. The example performs
|
|
140
|
+
only pure transformations. Mark any added writes with `dryRun: "skip"` or a
|
|
141
|
+
side-effect-free preview handler.
|
|
142
|
+
|
|
143
|
+
To plan or preview without contacting Inngest, use the project registration:
|
|
144
|
+
|
|
145
|
+
```sh
|
|
146
|
+
bun run tubeless -- plan --project examples/project/tubeless.project.ts inngest-normalize
|
|
147
|
+
bun run tubeless -- run --project examples/project/tubeless.project.ts inngest-normalize -- \
|
|
148
|
+
--lines " Alpha " --lines "Beta" --dry-run
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
## Verification
|
|
152
|
+
|
|
153
|
+
`make check` compiles the examples against the real SDK and uses
|
|
154
|
+
[`InngestTestEngine`](https://www.inngest.com/docs/reference/typescript/v4/testing)
|
|
155
|
+
to exercise the durable step with the real Tubeless pipeline. Tests cover JSON
|
|
156
|
+
results, progress and traces, invalid input, dry runs, and mocked durable step
|
|
157
|
+
output. The mocked-output test checks that the function returns the supplied
|
|
158
|
+
value without calling the pipeline; it does not test persisted-state recovery.
|
|
159
|
+
A separate test substitutes a real two-step Tubeless pipeline with fake storage
|
|
160
|
+
I/O: its save handler fails once, then a fresh attempt reruns both load and save
|
|
161
|
+
with stable correlation. These tests need no service or credentials and do not
|
|
162
|
+
test server-side persistence or retry scheduling. Use the local server flow above
|
|
163
|
+
to verify event delivery and inspect the run; test deployment authentication and side-effect idempotency separately.
|
package/docs/llms.txt
CHANGED
|
@@ -12,6 +12,7 @@ Start:
|
|
|
12
12
|
- First pipeline: ./getting-started.md
|
|
13
13
|
- CLI: ./cli.md
|
|
14
14
|
- Local studio: ./studio.md
|
|
15
|
+
- Artifacts and lineage: ./artifacts.md
|
|
15
16
|
- YAML and JSON pipelines: ./declarative-pipelines.md
|
|
16
17
|
- Pipeline document JSON Schema v1: ./pipeline-document.schema.json
|
|
17
18
|
- Choose a pattern: ./recipes.md
|
|
@@ -26,6 +27,7 @@ Agent instructions:
|
|
|
26
27
|
- Minimal project: ../examples/tubeless.project.ts
|
|
27
28
|
- Project with custom adapters: ../examples/project/tubeless.project.ts
|
|
28
29
|
- Maintain stable step IDs.
|
|
30
|
+
- For static parameterized step families, keep expansion synchronous and preserve literal IDs with the recipe-local tuple helper in ../examples/parameterized-steps.ts
|
|
29
31
|
- Use optional step names only as display overrides; dependencies and selection use IDs.
|
|
30
32
|
- Use pipeline.toMermaid() to generate static graph documentation.
|
|
31
33
|
- Use tubeless list to enumerate pipelines from defineProject. Use stable pipeline IDs with inspect, plan, graph, and run; no command modules are discovered from the filesystem or history.
|
|
@@ -36,7 +38,7 @@ Agent instructions:
|
|
|
36
38
|
- Use tubeless history to list or show recorded runs from SQLite, or pass --trace to inspect a finished NDJSON artifact without importing it. Studio also accepts --trace as a strictly read-only source. Default print is the projection; --events emits raw NDJSON. Trace files may contain sensitive logs and event payloads.
|
|
37
39
|
- Set dryRun: "skip" on external side effects, or provide a side-effect-free dryRun handler.
|
|
38
40
|
- Use context.log, context.signal, context.sleep, and progress APIs.
|
|
39
|
-
- Use createPipelineTestRuntime from tubeless/testing for deterministic tests with captured logs, statuses, progress, clock, and cancellation.
|
|
41
|
+
- Use createPipelineTestRuntime from tubeless/testing for deterministic tests with captured logs, statuses, progress, clock, and cancellation. Pass overrideStep(step, value) through test-run overrides to supply typed outputs; output schemas still validate and normal step statuses carry outputSource: "override" provenance.
|
|
40
42
|
- Import compilePipelineDocument from tubeless/project to compile parsed YAML/JSON with an explicit registry; use compiled.get(id) directly or pass selected pipelines to defineProject. Reuse compiled.metadata for project presentation. Import definePipelineCommand from tubeless/cli for explicit adapters; register each pipeline exactly once, directly or through its command in the same project entry list for CLI and Studio.
|
|
41
43
|
- Omit mapOptions only when validated CLI flags already satisfy same-name pipeline options; map renamed, loaded, or derived values explicitly. --step and --target are argv flags; mapOptions and hooks receive stepIds and targets.
|
|
42
44
|
- Expose --resume only through managed checkpoint configuration or an explicit resume: true command that handles values.resume. Unsupported commands reject it and omit it from Studio.
|
|
@@ -58,6 +60,11 @@ Advanced:
|
|
|
58
60
|
|
|
59
61
|
- Child pipelines: ./child-pipeline-composition.md
|
|
60
62
|
- Remote steps: ./remote-step-composition.md
|
|
63
|
+
- Tubeless and Airflow: ./airflow.md
|
|
64
|
+
- Tubeless on Temporal: ./temporal.md
|
|
65
|
+
- Tubeless on Inngest: ./inngest.md
|
|
66
|
+
- Tubeless on Dagster: ./dagster.md
|
|
67
|
+
- Tubeless on AWS Step Functions: ./step-functions.md
|
|
61
68
|
- Public exports: ./api-reference.md
|
|
62
69
|
- Machine-readable surface: ./api-report.json
|
|
63
70
|
Executable examples:
|
|
@@ -74,10 +81,32 @@ Executable examples:
|
|
|
74
81
|
- ../examples/best-effort.ts
|
|
75
82
|
- ../examples/child-pipeline.ts
|
|
76
83
|
- ../examples/remote-steps.ts
|
|
84
|
+
- ../examples/airflow/remote.ts
|
|
85
|
+
- ../examples/airflow/hosted.ts
|
|
86
|
+
- ../examples/airflow/tubeless_example.py
|
|
87
|
+
- ../examples/inngest/pipeline.ts
|
|
88
|
+
- ../examples/inngest/functions.ts
|
|
89
|
+
- ../examples/inngest/client.ts
|
|
90
|
+
- ../examples/inngest/server.ts
|
|
91
|
+
- ../examples/inngest/send.ts
|
|
92
|
+
- ../examples/temporal/pipeline.ts
|
|
93
|
+
- ../examples/temporal/activities.ts
|
|
94
|
+
- ../examples/temporal/workflows.ts
|
|
95
|
+
- ../examples/temporal/worker.ts
|
|
96
|
+
- ../examples/temporal/client.ts
|
|
97
|
+
- ../examples/dagster/pipeline.ts
|
|
98
|
+
- ../examples/dagster/hosted.ts
|
|
99
|
+
- ../examples/dagster/worker.ts
|
|
100
|
+
- ../examples/dagster/definitions.py
|
|
101
|
+
- ../examples/step-functions/pipeline.ts
|
|
102
|
+
- ../examples/step-functions/handler.ts
|
|
103
|
+
- ../examples/step-functions/state-machine.asl.json
|
|
104
|
+
- ../examples/step-functions/template.json
|
|
77
105
|
- ../examples/fan-out-progress.ts
|
|
78
106
|
- ../examples/resumable-enrichment.ts
|
|
79
107
|
- ../examples/cli-job.ts
|
|
80
108
|
- ../examples/cancellation-and-testing.ts
|
|
109
|
+
- ../examples/step-output-overrides.ts
|
|
81
110
|
- ../examples/live-tui.ts
|
|
82
111
|
- ../examples/peloton.ts
|
|
83
112
|
- ../examples/tracing.ts
|