tangle-cli 0.0.1a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tangle_cli/__init__.py +19 -0
- tangle_cli/api_cli.py +787 -0
- tangle_cli/api_schema.py +633 -0
- tangle_cli/api_transport.py +461 -0
- tangle_cli/args_container.py +244 -0
- tangle_cli/artifacts.py +293 -0
- tangle_cli/artifacts_cli.py +108 -0
- tangle_cli/cli.py +57 -0
- tangle_cli/cli_helpers.py +116 -0
- tangle_cli/cli_options.py +52 -0
- tangle_cli/client.py +677 -0
- tangle_cli/component_from_func.py +1856 -0
- tangle_cli/component_generator.py +298 -0
- tangle_cli/component_inspector.py +494 -0
- tangle_cli/component_publisher.py +921 -0
- tangle_cli/components_cli.py +269 -0
- tangle_cli/dynamic_discovery_client.py +296 -0
- tangle_cli/generated_model_extensions.py +405 -0
- tangle_cli/generated_runtime.py +43 -0
- tangle_cli/handler.py +96 -0
- tangle_cli/hydration_trust.py +222 -0
- tangle_cli/logger.py +166 -0
- tangle_cli/models.py +407 -0
- tangle_cli/module_bundler.py +662 -0
- tangle_cli/openapi/__init__.py +0 -0
- tangle_cli/openapi/codegen.py +1090 -0
- tangle_cli/openapi/parser.py +77 -0
- tangle_cli/pipeline_dehydrator.py +720 -0
- tangle_cli/pipeline_hydrator.py +1785 -0
- tangle_cli/pipeline_run_annotations.py +41 -0
- tangle_cli/pipeline_run_details.py +203 -0
- tangle_cli/pipeline_run_manager.py +1994 -0
- tangle_cli/pipeline_run_search.py +712 -0
- tangle_cli/pipeline_runner.py +620 -0
- tangle_cli/pipeline_runs_cli.py +584 -0
- tangle_cli/pipelines.py +581 -0
- tangle_cli/pipelines_cli.py +271 -0
- tangle_cli/published_components_cli.py +373 -0
- tangle_cli/py.typed +0 -0
- tangle_cli/quickstart.py +110 -0
- tangle_cli/secrets.py +156 -0
- tangle_cli/secrets_cli.py +269 -0
- tangle_cli/utils.py +942 -0
- tangle_cli/version_manager.py +470 -0
- tangle_cli-0.0.1a1.dist-info/METADATA +561 -0
- tangle_cli-0.0.1a1.dist-info/RECORD +48 -0
- tangle_cli-0.0.1a1.dist-info/WHEEL +4 -0
- tangle_cli-0.0.1a1.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,561 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: tangle-cli
|
|
3
|
+
Version: 0.0.1a1
|
|
4
|
+
Summary: CLI for Tangle, the open-source ML pipeline orchestration platform
|
|
5
|
+
Author: Alexey Volkov, Tangle authors
|
|
6
|
+
Author-email: Alexey Volkov <alexey.volkov@ark-kun.com>
|
|
7
|
+
Requires-Dist: cloud-pipelines>=0.26.3.12
|
|
8
|
+
Requires-Dist: cyclopts>=4.16.1
|
|
9
|
+
Requires-Dist: docstring-parser>=0.16
|
|
10
|
+
Requires-Dist: httpx>=0.28.1
|
|
11
|
+
Requires-Dist: jinja2>=3.1
|
|
12
|
+
Requires-Dist: platformdirs>=4.10.0
|
|
13
|
+
Requires-Dist: pydantic>=2.0
|
|
14
|
+
Requires-Dist: pyyaml>=6.0
|
|
15
|
+
Requires-Dist: requests>=2.32.0
|
|
16
|
+
Requires-Dist: tomli>=2.0 ; python_full_version < '3.11'
|
|
17
|
+
Requires-Dist: tangle-api==0.1.0 ; extra == 'native'
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Project-URL: Homepage, https://tangleml.com
|
|
20
|
+
Project-URL: Documentation, https://tangleml.com/docs/
|
|
21
|
+
Project-URL: Repository, https://github.com/TangleML/tangle-cli
|
|
22
|
+
Project-URL: Issues, https://github.com/TangleML/tangle-cli/issues
|
|
23
|
+
Provides-Extra: native
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# tangle-cli
|
|
27
|
+
|
|
28
|
+
CLI for Tangle, the open-source ML pipeline orchestration platform.
|
|
29
|
+
|
|
30
|
+
This repository contains the public Tangle CLI package. The CLI is built with [Cyclopts](https://cyclopts.readthedocs.io/) and is intentionally split into two command families:
|
|
31
|
+
|
|
32
|
+
- `tangle api ...` — pure OpenAPI wrappers around Tangle backend endpoints.
|
|
33
|
+
- `tangle sdk ...` — hand-written SDK, local, and compound commands that may call the API or may run entirely locally.
|
|
34
|
+
|
|
35
|
+
Start here:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
uv run tangle quickstart
|
|
39
|
+
uv run tangle --help
|
|
40
|
+
uv run tangle api --help
|
|
41
|
+
uv run tangle sdk --help
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Command families
|
|
45
|
+
|
|
46
|
+
### `tangle api ...`: direct OpenAPI wrappers
|
|
47
|
+
|
|
48
|
+
`tangle api` commands are generated/dynamic wrappers for backend HTTP endpoints. They are useful when you want to call the API directly with minimal CLI behavior layered on top.
|
|
49
|
+
|
|
50
|
+
API command sources are:
|
|
51
|
+
|
|
52
|
+
- **Official static schema**: the checked-in OpenAPI snapshot packaged in `tangle_api.schema` and generated into `tangle_api.generated`.
|
|
53
|
+
- **Dynamic cache**: live schemas fetched with `tangle api refresh` and merged in by default as cached-only extension commands.
|
|
54
|
+
|
|
55
|
+
By default `tangle api` uses `--schema-source auto`, which means official static operations plus cached live-backend extensions when a cache exists. Official operations win if a cached schema has the same method/path.
|
|
56
|
+
|
|
57
|
+
### `tangle sdk ...`: hand-written SDK commands
|
|
58
|
+
|
|
59
|
+
`tangle sdk` commands are hand-written workflows. They can be:
|
|
60
|
+
|
|
61
|
+
- **local-only**: no generated/native API bindings required, e.g. pipeline validation/layout and component generation;
|
|
62
|
+
- **API-backed**: use the generated client but add domain behavior, e.g. pipeline-run submit payload construction, hydration, artifact lookup, publishing/version checks, or config batching.
|
|
63
|
+
|
|
64
|
+
Current SDK groups include:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
uv run tangle sdk artifacts --help
|
|
68
|
+
uv run tangle sdk components --help
|
|
69
|
+
uv run tangle sdk pipelines --help
|
|
70
|
+
uv run tangle sdk pipeline-runs --help
|
|
71
|
+
uv run tangle sdk published-components --help
|
|
72
|
+
uv run tangle sdk secrets --help
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Common parameters and environment
|
|
76
|
+
|
|
77
|
+
API-backed commands commonly accept these options. Explicit CLI options win over config-file values, and config-file values win over environment defaults.
|
|
78
|
+
|
|
79
|
+
| Option / env | Purpose |
|
|
80
|
+
| --- | --- |
|
|
81
|
+
| `--base-url`, `TANGLE_API_URL` | API origin. Defaults to local development API URL when omitted. |
|
|
82
|
+
| `--token`, `TANGLE_API_TOKEN` | Bearer token shorthand. |
|
|
83
|
+
| `--auth-header`, `TANGLE_API_AUTH_HEADER`, `TANGLE_AUTH_HEADER` | Full `Authorization` value such as `Bearer ...` or `Basic ...`. |
|
|
84
|
+
| `-H`, `--header`, `TANGLE_API_HEADERS` | Extra headers. Repeatable as CLI flags; env accepts a JSON object or newline-separated `Name: value` entries. |
|
|
85
|
+
| `--config` | YAML/JSON defaults. Many commands accept a single object, a list of objects, or `_defaults` + `configs`. |
|
|
86
|
+
| `--log-type` | SDK progress logs: `console`, `none`, or `file`. Logs go to stderr or a temp log file so structured stdout stays parseable. |
|
|
87
|
+
| `TANGLE_VERBOSE=1` | Redacted HTTP request/response diagnostics only. This is separate from normal progress logging. |
|
|
88
|
+
|
|
89
|
+
Examples for protected APIs:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
uv run tangle api refresh --base-url https://api.example \
|
|
93
|
+
--auth-header 'Bearer ...' \
|
|
94
|
+
-H 'X-Gateway-Auth: ...'
|
|
95
|
+
|
|
96
|
+
uv run tangle api pipeline-runs list --base-url https://api.example \
|
|
97
|
+
--auth-header 'Basic ...' \
|
|
98
|
+
-H 'X-Api-Key: ...'
|
|
99
|
+
|
|
100
|
+
uv run tangle sdk pipeline-runs submit pipeline.yaml \
|
|
101
|
+
--base-url https://api.example \
|
|
102
|
+
--auth-header 'Bearer ...' \
|
|
103
|
+
-H 'X-Gateway-Auth: ...' \
|
|
104
|
+
--log-type console
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Use `--log-type none` for quiet machine-readable runs, and `--log-type file` to capture progress logs in a temporary file while keeping stdout clean.
|
|
108
|
+
|
|
109
|
+
## Installation and package split
|
|
110
|
+
|
|
111
|
+
The repository contains two Python import packages with different responsibilities:
|
|
112
|
+
|
|
113
|
+
- `tangle_cli` is hand-written. It contains CLI wiring, SDK/business helpers, local pipeline/component workflows, dynamic API discovery, codegen, shared runtime classes, logging, and extension classes.
|
|
114
|
+
- `tangle_api` is generated/native. It contains checked-in generated Pydantic models, generated endpoint operation methods, and the official OpenAPI snapshot.
|
|
115
|
+
|
|
116
|
+
The default `tangle-cli` package keeps the top-level import and local-only SDK commands native-free. Install the native extra when you want static API-backed commands and the handwritten `TangleApiClient` wrapper to use the checked-in generated bindings:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
pip install 'tangle-cli[native]'
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
In this workspace, `uv` installs the workspace `tangle-api` package for development and tests:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
uv run tangle api --help
|
|
126
|
+
uv run tangle sdk pipelines validate pipeline.yaml
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
If you are embedding `tangle_cli` in a downstream project, you can provide your own local `tangle_api.generated` package produced from your backend schema instead of using this repo's official generated package.
|
|
130
|
+
|
|
131
|
+
## Quick command examples
|
|
132
|
+
|
|
133
|
+
Local-only SDK commands:
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
uv run tangle sdk pipelines validate pipeline.yaml
|
|
137
|
+
uv run tangle sdk pipelines diagram pipeline.yaml
|
|
138
|
+
uv run tangle sdk pipelines layout pipeline.yaml --recursive
|
|
139
|
+
uv run tangle sdk pipelines hydrate pipeline.yaml --output hydrated.yaml
|
|
140
|
+
uv run tangle sdk components generate from-python path/to/component.py --image python:3.12
|
|
141
|
+
uv run tangle sdk components bump-version path/to/component.yaml
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
API-backed SDK commands:
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
uv run tangle sdk published-components search transformer --base-url https://api.example
|
|
148
|
+
uv run tangle sdk published-components inspect transformer --base-url https://api.example
|
|
149
|
+
uv run tangle sdk published-components publish components/my-component.yaml --dry-run
|
|
150
|
+
uv run tangle sdk pipeline-runs submit pipeline.yaml --dry-run --log-type none
|
|
151
|
+
uv run tangle sdk pipeline-runs submit pipeline.yaml --base-url https://api.example --log-type console
|
|
152
|
+
uv run tangle sdk pipeline-runs status RUN_ID --base-url https://api.example
|
|
153
|
+
uv run tangle sdk artifacts get --run-id RUN_ID --query '{"artifact_ids":["artifact-id"]}'
|
|
154
|
+
uv run tangle sdk secrets list --base-url https://api.example
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
Direct API commands:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
uv run tangle api refresh --base-url https://api.example
|
|
161
|
+
uv run tangle api pipeline-runs list --base-url https://api.example
|
|
162
|
+
uv run tangle api pipeline-runs get RUN_ID --base-url https://api.example
|
|
163
|
+
uv run tangle api components get DIGEST --base-url https://api.example
|
|
164
|
+
uv run tangle api published-components list --base-url https://api.example
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Path parameters are positional arguments and query parameters become options. Check generated help for the exact options exposed by the active schema source:
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
uv run tangle api pipeline-runs list --help
|
|
171
|
+
uv run tangle api pipeline-runs list --include-execution-stats
|
|
172
|
+
uv run tangle api pipeline-runs create --body @pipeline-run.json
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Responses are printed as JSON when the backend returns JSON.
|
|
176
|
+
|
|
177
|
+
## Config files
|
|
178
|
+
|
|
179
|
+
Implemented API-backed commands and many SDK commands accept `--config path/to/config.yaml` (or JSON). Config files may contain a single object, a list of objects, or a `_defaults` + `configs` object; with multiple config entries, the command runs once per entry.
|
|
180
|
+
|
|
181
|
+
```yaml
|
|
182
|
+
_defaults:
|
|
183
|
+
base_url: https://api.example
|
|
184
|
+
auth_header: Bearer ...
|
|
185
|
+
header:
|
|
186
|
+
- "X-Gateway-Auth: ..."
|
|
187
|
+
log_type: none
|
|
188
|
+
|
|
189
|
+
configs:
|
|
190
|
+
- filter: active
|
|
191
|
+
limit: 10
|
|
192
|
+
- filter: finished
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
```bash
|
|
196
|
+
uv run tangle api pipeline-runs list --config api-config.yaml --limit 5
|
|
197
|
+
uv run tangle sdk published-components search --config components.yaml
|
|
198
|
+
uv run tangle sdk pipeline-runs submit --config submit.yaml
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
For generated `tangle api` commands, config keys use generated CLI parameter names such as `base_url`, `schema_source`, `body`, and endpoint parameters like `limit`, `filter`, or `id`.
|
|
202
|
+
|
|
203
|
+
## API schema cache and dynamic commands
|
|
204
|
+
|
|
205
|
+
Refresh the local schema cache for a live backend with:
|
|
206
|
+
|
|
207
|
+
```bash
|
|
208
|
+
uv run tangle api refresh --base-url http://localhost:8000
|
|
209
|
+
uv run tangle api refresh --base-url https://api.example --auth-header 'Bearer ...'
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
`refresh` fetches:
|
|
213
|
+
|
|
214
|
+
```text
|
|
215
|
+
<base-url>/openapi.json
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
Schemas are cached under the OS-specific user cache directory via `platformdirs`, with an `openapi` subdirectory. Override that directory with:
|
|
219
|
+
|
|
220
|
+
```bash
|
|
221
|
+
export TANGLE_CLI_CACHE_DIR=/path/to/openapi-schema-cache
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
Delete a cached live schema without touching the checked-in official snapshot:
|
|
225
|
+
|
|
226
|
+
```bash
|
|
227
|
+
uv run tangle api reset-cache --base-url https://api.example
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
Schema source modes are:
|
|
231
|
+
|
|
232
|
+
- `--schema-source auto` (default): official static operations plus cached-only backend extensions when a cache exists. Requires the native `tangle-api` package for official operations.
|
|
233
|
+
- `--schema-source official`: only the checked-in official static schema. Requires the native `tangle-api` package.
|
|
234
|
+
- `--schema-source cache`: only the schema previously written by `tangle api refresh` for the selected base URL. Does not require the native package.
|
|
235
|
+
|
|
236
|
+
For resource help, put `--schema-source` on the resource group:
|
|
237
|
+
|
|
238
|
+
```bash
|
|
239
|
+
uv run tangle api published-components --schema-source official --help
|
|
240
|
+
uv run tangle api published-components --schema-source cache --help
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
For endpoint calls, put it on the endpoint command:
|
|
244
|
+
|
|
245
|
+
```bash
|
|
246
|
+
uv run tangle api published-components experimental-search \
|
|
247
|
+
--schema-source cache \
|
|
248
|
+
--base-url https://api.example \
|
|
249
|
+
--body @query.json
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
## SDK command details
|
|
253
|
+
|
|
254
|
+
### Local components
|
|
255
|
+
|
|
256
|
+
`generate from-python` converts a local Python function into a component YAML using inline source by default, or `--mode bundle` to embed local dependency modules. Common options include `--function`, `--output`, `--name`, `--image`, `--dependencies-from`, `--strip-code`, `--use-legacy-naming`, and `--resolve-root`.
|
|
257
|
+
|
|
258
|
+
`bump-version` increments or sets component version metadata in YAML and updates/regenerates a referenced Python source when the component contains `python_original_code_path` annotations.
|
|
259
|
+
|
|
260
|
+
Generation and version-bump commands accept `--config` YAML/JSON files via `tangle_cli.args_container`. Use keys such as `python_file`, `image`, `function`, `mode`, `resolve_root`, `yaml_file`, `set_version`, and `update_timestamp`; explicit CLI values take precedence.
|
|
261
|
+
|
|
262
|
+
### Published components
|
|
263
|
+
|
|
264
|
+
Published/registry component operations live under `sdk published-components` so local component authoring and registry calls do not share a command group.
|
|
265
|
+
|
|
266
|
+
```bash
|
|
267
|
+
uv run tangle sdk published-components publish components/my-component.yaml \
|
|
268
|
+
--base-url https://api.example \
|
|
269
|
+
--image python:3.12 \
|
|
270
|
+
--name "My component"
|
|
271
|
+
|
|
272
|
+
uv run tangle sdk published-components publish components/my-component.yaml --dry-run
|
|
273
|
+
uv run tangle sdk published-components deprecate sha256:old --superseded-by sha256:new
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
`publish` accepts `--image`, `--name`, `--description`, `--annotations` (JSON), `--dry-run`, `--published-by`, generic git metadata fields, generic API auth fields, `--log-type`, and `--config`. By default it scopes version checks and automatic old-version deprecation to the current authenticated user via `users_me()`; use `--published-by` to supply an explicit owner/publisher filter. Publishing fails closed if no owner can be determined.
|
|
277
|
+
|
|
278
|
+
There is no separate OSS `publish-all` command. To publish multiple components, pass a YAML/JSON config list, or `_defaults` + `configs`, to the same `published-components publish` command; the command aggregates results and exits nonzero if any component errors.
|
|
279
|
+
|
|
280
|
+
```yaml
|
|
281
|
+
_defaults:
|
|
282
|
+
base_url: https://api.example
|
|
283
|
+
image: python:3.12
|
|
284
|
+
configs:
|
|
285
|
+
- component_path: components/first.yaml
|
|
286
|
+
name: First component
|
|
287
|
+
- component_path: components/second.yaml
|
|
288
|
+
name: Second component
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
Batch `publish-all`, notification integrations, dbt generation, from-container generation, and backend-specific advanced search workflows remain out of this OSS CLI package.
|
|
292
|
+
|
|
293
|
+
### Pipelines and pipeline runs
|
|
294
|
+
|
|
295
|
+
Local pipeline commands live under `sdk pipelines`:
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
uv run tangle sdk pipelines validate pipeline.yaml
|
|
299
|
+
uv run tangle sdk pipelines hydrate pipeline.yaml --output hydrated.yaml
|
|
300
|
+
uv run tangle sdk pipelines diagram pipeline.yaml
|
|
301
|
+
uv run tangle sdk pipelines layout pipeline.yaml --recursive
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
Pipeline run API/submit commands live under `sdk pipeline-runs`:
|
|
305
|
+
|
|
306
|
+
```bash
|
|
307
|
+
uv run tangle sdk pipeline-runs submit pipeline.yaml --dry-run
|
|
308
|
+
uv run tangle sdk pipeline-runs submit pipeline.yaml --arg key=value --annotation owner=team
|
|
309
|
+
uv run tangle sdk pipeline-runs wait RUN_ID --max-wait 600 --poll-interval 10
|
|
310
|
+
uv run tangle sdk pipeline-runs logs EXECUTION_ID
|
|
311
|
+
uv run tangle sdk pipeline-runs annotations set RUN_ID key value
|
|
312
|
+
uv run tangle sdk pipeline-runs export RUN_ID --output pipeline.yaml
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
`submit` hydrates refs by default and builds an API submit payload with `root_task.componentRef.spec`. Use `--no-hydrate` to submit the local YAML structure as-is. Use `--dry-run` to print the payload without creating a run.
|
|
316
|
+
|
|
317
|
+
## Programmatic client
|
|
318
|
+
|
|
319
|
+
The stable public wrapper for downstream Python tools is:
|
|
320
|
+
|
|
321
|
+
```python
|
|
322
|
+
from tangle_cli.client import TangleApiClient
|
|
323
|
+
|
|
324
|
+
client = TangleApiClient("http://localhost:8000")
|
|
325
|
+
run = client.pipeline_runs_get("run-id")
|
|
326
|
+
existing = client.find_existing_components(
|
|
327
|
+
["component-name"],
|
|
328
|
+
published_by_substring="alice@example.com",
|
|
329
|
+
)
|
|
330
|
+
```
|
|
331
|
+
|
|
332
|
+
`TangleApiClient` is handwritten in `tangle_cli.client` and inherits generated endpoint methods from `tangle_api.generated.operations.GeneratedTangleApiOperations`. The generated endpoint methods call the handwritten transport/request logic. Handwritten semantic helpers such as `find_existing_components(...)` return domain models and normalize common compatibility cases.
|
|
333
|
+
|
|
334
|
+
The top-level `import tangle_cli` is lightweight and does not import native static bindings. Install the native extra or otherwise provide a local `tangle_api.generated` package before importing `tangle_cli.client`.
|
|
335
|
+
|
|
336
|
+
## Codegen/autogen from OpenAPI
|
|
337
|
+
|
|
338
|
+
Use codegen when you want to update the checked-in official generated package or generate bindings for your own Tangle-compatible API instance.
|
|
339
|
+
|
|
340
|
+
Official backend/submodule flow:
|
|
341
|
+
|
|
342
|
+
```bash
|
|
343
|
+
git submodule update --init --recursive
|
|
344
|
+
uv sync --group codegen
|
|
345
|
+
uv run --group codegen python -m tangle_cli.openapi.codegen
|
|
346
|
+
uv run pytest
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
With no source flags, codegen loads OpenAPI from the default official backend submodule at `third_party/tangle`, writes `packages/tangle-api/src/tangle_api/schema/openapi.json`, and regenerates `packages/tangle-api/src/tangle_api/generated`. The backend import creates a database engine at import time; codegen points it at a temporary SQLite database unless `--backend-database-uri` is provided.
|
|
350
|
+
|
|
351
|
+
Regenerate from the checked-in API-package snapshot:
|
|
352
|
+
|
|
353
|
+
```bash
|
|
354
|
+
uv run python -m tangle_cli.openapi.codegen --from-snapshot
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
Fetch a remote OpenAPI JSON document directly:
|
|
358
|
+
|
|
359
|
+
```bash
|
|
360
|
+
uv run python -m tangle_cli.openapi.codegen \
|
|
361
|
+
--openapi-url https://api.example/openapi.json \
|
|
362
|
+
--out src/tangle_api/generated
|
|
363
|
+
```
|
|
364
|
+
|
|
365
|
+
Generate from a backend checkout explicitly:
|
|
366
|
+
|
|
367
|
+
```bash
|
|
368
|
+
uv run --group codegen python -m tangle_cli.openapi.codegen \
|
|
369
|
+
--backend-path /path/to/tangle/backend \
|
|
370
|
+
--backend-database-uri sqlite:////tmp/tangle-openapi.sqlite
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
Important codegen options:
|
|
374
|
+
|
|
375
|
+
- `--out`: directory that receives `__init__.py`, `models.py`, and `operations.py`. Defaults to `packages/tangle-api/src/tangle_api/generated`.
|
|
376
|
+
- `--operations-class-name`: generated operations mixin class name. Defaults to `GeneratedTangleApiOperations`.
|
|
377
|
+
- `--model-extension-module`: importable module with `MODEL_EXTENSIONS`; repeat to compose modules.
|
|
378
|
+
- `--model-alias`: expose a stable public model name from one or more source schema names, e.g. `ComponentSpec=ComponentSpecOutput,ComponentSpecInput`.
|
|
379
|
+
- `--request-body-schema` / `--request-body-schema-file`: override a specific operation's JSON request-body schema without mutating the fetched OpenAPI document.
|
|
380
|
+
|
|
381
|
+
At runtime, more `tangle api ...` commands become available in two ways:
|
|
382
|
+
|
|
383
|
+
1. Static codegen: regenerate and install/provide a `tangle_api.generated` package for the schema.
|
|
384
|
+
2. Dynamic cache: run `tangle api refresh --base-url ...` and use `--schema-source auto` or `--schema-source cache` to expose cached-only operations through the dynamic CLI.
|
|
385
|
+
|
|
386
|
+
## Generated model extension pattern
|
|
387
|
+
|
|
388
|
+
Generated models use a generated implementation base plus a stable public subclass. For example, codegen emits this shape for a model with a handwritten extension:
|
|
389
|
+
|
|
390
|
+
```python
|
|
391
|
+
class _ComponentSpecGenerated(TangleGeneratedModel):
|
|
392
|
+
name: Any = None
|
|
393
|
+
# generated OpenAPI fields...
|
|
394
|
+
|
|
395
|
+
class ComponentSpec(ComponentSpecExtensions, _ComponentSpecGenerated):
|
|
396
|
+
pass
|
|
397
|
+
```
|
|
398
|
+
|
|
399
|
+
The public class is a subclass rather than an alias because the public class name is the stable contract while the generated base can be regenerated. Subclassing lets the public class keep the OpenAPI/Pydantic fields from `_ComponentSpecGenerated` and add or override behavior through normal Python MRO.
|
|
400
|
+
|
|
401
|
+
Extension bases are placed to the **left** of the generated base:
|
|
402
|
+
|
|
403
|
+
```python
|
|
404
|
+
class ComponentSpec(ComponentSpecExtensions, _ComponentSpecGenerated):
|
|
405
|
+
pass
|
|
406
|
+
```
|
|
407
|
+
|
|
408
|
+
That means extension methods/properties override generated-base behavior when names overlap, while generated fields and `TangleGeneratedModel` runtime helpers such as `to_dict()` remain available.
|
|
409
|
+
|
|
410
|
+
The built-in default extension module is:
|
|
411
|
+
|
|
412
|
+
```text
|
|
413
|
+
tangle_cli.generated_model_extensions
|
|
414
|
+
```
|
|
415
|
+
|
|
416
|
+
It defines:
|
|
417
|
+
|
|
418
|
+
```python
|
|
419
|
+
MODEL_EXTENSIONS = {
|
|
420
|
+
"ComponentSpec": "ComponentSpecExtensions",
|
|
421
|
+
"GetExecutionInfoResponse": "GetExecutionInfoResponseExtensions",
|
|
422
|
+
"GetGraphExecutionStateResponse": "GetGraphExecutionStateResponseExtensions",
|
|
423
|
+
}
|
|
424
|
+
```
|
|
425
|
+
|
|
426
|
+
During codegen, `tangle_api.generated.models` imports those extension classes from `tangle_cli.generated_model_extensions`. This preserves the package boundary: `tangle_api` remains generated bindings, while `tangle_cli` owns handwritten runtime and extension behavior.
|
|
427
|
+
|
|
428
|
+
Downstream projects can layer their own extensions:
|
|
429
|
+
|
|
430
|
+
```python
|
|
431
|
+
# my_project/tangle_model_extensions.py
|
|
432
|
+
class MyComponentSpecExtensions:
|
|
433
|
+
@property
|
|
434
|
+
def owning_team(self) -> str | None:
|
|
435
|
+
return (self.metadata or {}).get("annotations", {}).get("team")
|
|
436
|
+
|
|
437
|
+
MODEL_EXTENSIONS = {
|
|
438
|
+
"ComponentSpec": "MyComponentSpecExtensions",
|
|
439
|
+
}
|
|
440
|
+
```
|
|
441
|
+
|
|
442
|
+
```bash
|
|
443
|
+
uv run python -m tangle_cli.openapi.codegen \
|
|
444
|
+
--openapi-url https://api.example/openapi.json \
|
|
445
|
+
--out src/tangle_api/generated \
|
|
446
|
+
--model-extension-module my_project.tangle_model_extensions
|
|
447
|
+
```
|
|
448
|
+
|
|
449
|
+
The default module is applied first. Repeated `--model-extension-module` values are applied in order, and later/downstream modules become leftmost in the generated public class MRO, so they override earlier/default extensions. If two modules export the same extension class name, codegen imports them with deterministic aliases.
|
|
450
|
+
|
|
451
|
+
Pass an empty string to disable built-in default extensions:
|
|
452
|
+
|
|
453
|
+
```bash
|
|
454
|
+
uv run python -m tangle_cli.openapi.codegen \
|
|
455
|
+
--from-snapshot \
|
|
456
|
+
--model-extension-module ""
|
|
457
|
+
```
|
|
458
|
+
|
|
459
|
+
The same empty-string sentinel can disable built-in `--model-alias` defaults. Built-in aliases keep stable public model names such as `ComponentSpec` even when a backend schema uses names like `ComponentSpecOutput` or `ComponentSpecInput`.
|
|
460
|
+
|
|
461
|
+
Extension classes should be importable from their modules and should not import generated model classes. They should be mixins over generated data, not replacements for generated schemas.
|
|
462
|
+
|
|
463
|
+
## Extending SDK behavior
|
|
464
|
+
|
|
465
|
+
The CLI exposes small explicit seams rather than requiring downstream forks.
|
|
466
|
+
|
|
467
|
+
### Hydrator resolvers
|
|
468
|
+
|
|
469
|
+
`packages/tangle-cli/src/tangle_cli/pipeline_hydrator.py` exposes a resolver registry:
|
|
470
|
+
|
|
471
|
+
```python
|
|
472
|
+
from tangle_cli.pipeline_hydrator import PipelineHydrator, register_component_resolver
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def resolve_from_catalog(hydrator: PipelineHydrator, value, path: str, base_dir):
|
|
476
|
+
# return (digest, component_spec_dict) or None
|
|
477
|
+
return "sha256:...", {"name": "Resolved", "implementation": {"container": {"image": "python:3.12"}}}
|
|
478
|
+
|
|
479
|
+
register_component_resolver("catalog", resolve_from_catalog)
|
|
480
|
+
```
|
|
481
|
+
|
|
482
|
+
Resolvers receive the hydrator instance, the reference value, a display path, and the current base directory. They can use `hydrator._api_client()` for API-backed lookups, `hydrator.log` for progress logs, and `hydrator.resolution_overrides` for template/config variables. There is also an instance method `hydrator.register_component_resolver(...)` for per-hydrator overrides. Built-in kinds include `digest`, `name`, `url`, `file`, `resolve`, `http`, `https`, `local`, and `local_from_python`.
|
|
483
|
+
|
|
484
|
+
Downstream-only features such as Docker/from-container materialization or cloud storage can be added by registering new resolvers while the OSS default remains explicit about unsupported kinds.
|
|
485
|
+
|
|
486
|
+
### Pipeline run hooks
|
|
487
|
+
|
|
488
|
+
`packages/tangle-cli/src/tangle_cli/pipeline_runs.py` defines `PipelineRunHooks`, passed into `PipelineRunManager`. Subclass it to customize submit/load/wait/log behavior:
|
|
489
|
+
|
|
490
|
+
```python
|
|
491
|
+
from tangle_cli.pipeline_runs import PipelineRunHooks, PipelineRunManager
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
class MyRunHooks(PipelineRunHooks):
|
|
495
|
+
def read_pipeline_yaml(self, pipeline_path):
|
|
496
|
+
if str(pipeline_path).startswith("s3://"):
|
|
497
|
+
return load_from_s3(pipeline_path)
|
|
498
|
+
return super().read_pipeline_yaml(pipeline_path)
|
|
499
|
+
|
|
500
|
+
def extra_submit_annotations(self, *, pipeline_spec, pipeline_path, run_as=None):
|
|
501
|
+
annotations = super().extra_submit_annotations(
|
|
502
|
+
pipeline_spec=pipeline_spec,
|
|
503
|
+
pipeline_path=pipeline_path,
|
|
504
|
+
run_as=run_as,
|
|
505
|
+
)
|
|
506
|
+
annotations["submitted_by"] = "my-tool"
|
|
507
|
+
return annotations
|
|
508
|
+
|
|
509
|
+
def fetch_logs(self, client, execution_id):
|
|
510
|
+
return client.executions_container_log(execution_id)
|
|
511
|
+
|
|
512
|
+
manager = PipelineRunManager(client=my_client, hooks=MyRunHooks())
|
|
513
|
+
```
|
|
514
|
+
|
|
515
|
+
Available hooks include:
|
|
516
|
+
|
|
517
|
+
- `read_pipeline_yaml(...)`
|
|
518
|
+
- `hydrate_pipeline(...)`
|
|
519
|
+
- `prepare_run_arguments(...)`
|
|
520
|
+
- `extra_submit_annotations(...)`
|
|
521
|
+
- `before_submit(...)`
|
|
522
|
+
- `after_submit(...)`
|
|
523
|
+
- `after_wait(...)`
|
|
524
|
+
- `fetch_logs(...)`
|
|
525
|
+
|
|
526
|
+
Use these for generic downstream behavior such as alternate storage, extra annotations, scheduling/time input defaults, mutex checks, notifications, or alternate log providers. The OSS defaults intentionally exclude provider-specific cloud, notification, and scheduler behavior.
|
|
527
|
+
|
|
528
|
+
### Component publish hooks
|
|
529
|
+
|
|
530
|
+
`packages/tangle-cli/src/tangle_cli/component_publisher.py` defines `ComponentPublishHook` with:
|
|
531
|
+
|
|
532
|
+
- `before_batch(components_config)`
|
|
533
|
+
- `after_component(component_path, result)`
|
|
534
|
+
- `after_batch(results)`
|
|
535
|
+
|
|
536
|
+
`ComponentPublisher(..., hooks=[...])` calls these around publish batches. Use them for downstream summaries, audit records, or notifications while keeping OSS publishing generic.
|
|
537
|
+
|
|
538
|
+
### Shared CLI helpers and logging
|
|
539
|
+
|
|
540
|
+
`cli_options.py` centralizes shared Cyclopts annotations such as `BaseUrlOption`, `TokenOption`, `AuthHeaderOption`, `HeaderOption`, `ConfigOption`, and `LogTypeOption`. `cli_helpers.py` centralizes config loading, JSON printing, credential-isolation helpers, and the native-safe `LazyTangleApiClient` proxy. `logger.py` provides `ConsoleLogger`, `NullLogger`, `CaptureLogger`, `logger_for_log_type(...)`, and `run_with_logging(...)`.
|
|
541
|
+
|
|
542
|
+
Use these helpers for new SDK commands so top-level imports remain native-free, `--config` behavior stays consistent, credentials from config do not accidentally mix with ambient environment auth, and progress logs stay off structured stdout.
|
|
543
|
+
|
|
544
|
+
## Development checks
|
|
545
|
+
|
|
546
|
+
Common validation commands:
|
|
547
|
+
|
|
548
|
+
```bash
|
|
549
|
+
uv run --frozen pytest -q
|
|
550
|
+
uv build --sdist --wheel
|
|
551
|
+
uv build --sdist --wheel --package tangle-api
|
|
552
|
+
git diff --check
|
|
553
|
+
```
|
|
554
|
+
|
|
555
|
+
Targeted CLI smoke:
|
|
556
|
+
|
|
557
|
+
```bash
|
|
558
|
+
uv run tangle quickstart
|
|
559
|
+
uv run tangle api --help
|
|
560
|
+
uv run tangle sdk --help
|
|
561
|
+
```
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
tangle_cli/__init__.py,sha256=KpIzluxyPdTkIAPzHCL5q2PyjqhNDpvJCLQx4LtIozk,649
|
|
2
|
+
tangle_cli/api_cli.py,sha256=04x8gxlKnoTZ2WA5hMtZ6K6BRfWnRRfVSVeqxyz8rPo,27327
|
|
3
|
+
tangle_cli/api_schema.py,sha256=0FBdk4WXpyeeUSmYzo7ELyeo5HhLSnHgsh3OPCDyCBw,21763
|
|
4
|
+
tangle_cli/api_transport.py,sha256=v3Q-3UmYLPweir_n6tCVEDVm-i9XHP-54wpX3Em2DgE,15735
|
|
5
|
+
tangle_cli/args_container.py,sha256=Ft4TbTVKAy3tDPkyqf0vYLaOLbCBuL-EnSd5hQqaQio,9602
|
|
6
|
+
tangle_cli/artifacts.py,sha256=UfVKrwJKSa3o-oTkVntsuslu5V2OanpkWYF-j65MEkM,11159
|
|
7
|
+
tangle_cli/artifacts_cli.py,sha256=V0BOkgutm4xSAgOenLNBrDe4s6c-fYkhVIdsAxYRhro,3079
|
|
8
|
+
tangle_cli/cli.py,sha256=sgZt9pR1Wccz3KwsJ45HBnusBtqrXEe3cXNI1QnX8eU,1244
|
|
9
|
+
tangle_cli/cli_helpers.py,sha256=vy1HymkDokBZY5E-n81qOvY292iYrNuA-5OSij_DJgQ,3978
|
|
10
|
+
tangle_cli/cli_options.py,sha256=489WJiHbwelQ3ZrlgZSGiSYDLi8kODy_IvcKMLlijzg,1271
|
|
11
|
+
tangle_cli/client.py,sha256=MMmvKSjLP7qtocCGcLLQDviGM3ICd03n3lfkQSPqQXE,25134
|
|
12
|
+
tangle_cli/component_from_func.py,sha256=hlCcVOH-s_gZ4egHx-xodvthh8aa5W7pd6XfBBRVUZA,77791
|
|
13
|
+
tangle_cli/component_generator.py,sha256=rX_dBzEA35fCrIc8WpHybsePFPAuM5b46HbJkURhPxo,10493
|
|
14
|
+
tangle_cli/component_inspector.py,sha256=P3XOhyqL69qv_s2yYczEQiQZwbEPDnCkSC_21Cgb_6I,17719
|
|
15
|
+
tangle_cli/component_publisher.py,sha256=IL3RSbUtZzE_1wt_jWlqwFh9P0OxOCoKXKhcUHrOees,34751
|
|
16
|
+
tangle_cli/components_cli.py,sha256=38EY2j1KhsU98QtMzww7xtA6QhLFwKgjjtUxNjbRwME,8433
|
|
17
|
+
tangle_cli/dynamic_discovery_client.py,sha256=JMzwziySkIoSBGszsfhFtkda-5CXHYAvD60cVHXlPXk,10099
|
|
18
|
+
tangle_cli/generated_model_extensions.py,sha256=kOqJqO8TyxUbCQ6c70p7_F50-3aQzAOF6B7CuRaNBZk,14063
|
|
19
|
+
tangle_cli/generated_runtime.py,sha256=JBv-GjSq4EwQNYQCq6p9tyFZCympQdKu7mkLWcBMTZ8,1276
|
|
20
|
+
tangle_cli/handler.py,sha256=rsP_gZDvJyPJEkrwU-FFR2ZGvzJJoWASmc0Xbb4uyeU,3355
|
|
21
|
+
tangle_cli/hydration_trust.py,sha256=_aCeRTYg1Cad_O_vXhpW32-J4x20mmcR5P6m2aC_m4M,7257
|
|
22
|
+
tangle_cli/logger.py,sha256=hnyHoIKGOajsGgKiW6Wq9Mnj1xFVuPxsAKDujGBK1UA,5213
|
|
23
|
+
tangle_cli/models.py,sha256=XTcHb5rkjzo6HdPURG3HzVH9-g6vMDlYsTC2yLpXiKI,13521
|
|
24
|
+
tangle_cli/module_bundler.py,sha256=RXY0hcsFnoRbKlZObQjWYUiGygkBEXVQwbkH4bHevKA,29501
|
|
25
|
+
tangle_cli/openapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
26
|
+
tangle_cli/openapi/codegen.py,sha256=4zAv8CHeqMJSlBIY1relnisv0RVOBn0FRpz6R2i5zSI,41783
|
|
27
|
+
tangle_cli/openapi/parser.py,sha256=XxxhiMR5oJg9wWvPVuZxY1XX0KCHjcBMQfedKM7g3XI,2852
|
|
28
|
+
tangle_cli/pipeline_dehydrator.py,sha256=CcdBdKMBECSS3IBF-RcGEDX_FTxaf4FUvZn-ulVVwY8,30935
|
|
29
|
+
tangle_cli/pipeline_hydrator.py,sha256=2S_i4urN-jLNUiFAgUF3kZ1-W11JNyO5oos0beZWFKY,68992
|
|
30
|
+
tangle_cli/pipeline_run_annotations.py,sha256=cWFmqk5xFsyTlosZ2NNDRqiTRqC-oKFdfP2cq9DUvmo,1416
|
|
31
|
+
tangle_cli/pipeline_run_details.py,sha256=Y5yAoUXQApqo4eRlECoJOm83FfS7o7UNE3LYdEhMAXI,7819
|
|
32
|
+
tangle_cli/pipeline_run_manager.py,sha256=CvfX4RW5X-5Wde8kLPlHI9sT7QIkvHe7nurBKaWW5Ts,81734
|
|
33
|
+
tangle_cli/pipeline_run_search.py,sha256=RTcOFrFzN_vHreQ9Xb_VNXW6Aj_DVI7b6Bo-AHasQi0,24709
|
|
34
|
+
tangle_cli/pipeline_runner.py,sha256=95TJhkh4yUOF40KPRTcDHKWNhNqYBY8IGPOyeDGRovo,23648
|
|
35
|
+
tangle_cli/pipeline_runs_cli.py,sha256=2JKnfyaFHjBjoeCUDoYcwY4PWfh6LujvS1FRsnJuqjE,20733
|
|
36
|
+
tangle_cli/pipelines.py,sha256=NAd-P9c8iSZcwPkF0HJmqFpt6OYLdaltL4IcV52sGxM,20272
|
|
37
|
+
tangle_cli/pipelines_cli.py,sha256=ry2uHiCjPG_t2s6J-AfKoi_e7RBl8TqlnoEPOzd0Lfc,8694
|
|
38
|
+
tangle_cli/published_components_cli.py,sha256=6fEpDGXU9PHfSq9PQMPsEoOhIgcgaSK6BnexyzKH-eY,12735
|
|
39
|
+
tangle_cli/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
40
|
+
tangle_cli/quickstart.py,sha256=9_wP-aY66xtJFOic_oFVXgL59zLRmoGEplNwj6h2MUc,4448
|
|
41
|
+
tangle_cli/secrets.py,sha256=yQ5276A2GbS2NX7FBkLsbVlidnBv1t2ALW5wMoZRY58,5162
|
|
42
|
+
tangle_cli/secrets_cli.py,sha256=vN9ukNA5H3U6DfGDeXikaWAMceYd0S4DDxPONMHwR0g,8150
|
|
43
|
+
tangle_cli/utils.py,sha256=54Ou1qy9UiDxJefJ33W4WHeMnmk_19KEZSrUApkTbRg,35708
|
|
44
|
+
tangle_cli/version_manager.py,sha256=FG-ezTdEy5Q1aN3j8Pc1QjyFwm8qgzv3ZkRmIxftTZ8,18402
|
|
45
|
+
tangle_cli-0.0.1a1.dist-info/WHEEL,sha256=jK0lbM7sVtq70msNoYotEXYS3OJMDdns2CRgyjhimnE,81
|
|
46
|
+
tangle_cli-0.0.1a1.dist-info/entry_points.txt,sha256=s-RfG6xRFLUdecYo4xNMN_0TuFbc74Lkbiz85PBpU-c,48
|
|
47
|
+
tangle_cli-0.0.1a1.dist-info/METADATA,sha256=2O9_vEgVnhfF2-MUObONIqDbkHSaeHZ7C6eqB49_Ey0,24539
|
|
48
|
+
tangle_cli-0.0.1a1.dist-info/RECORD,,
|