eastworlds-forge 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eastworlds_forge-0.2.0/.gitignore +16 -0
- eastworlds_forge-0.2.0/PKG-INFO +93 -0
- eastworlds_forge-0.2.0/README.md +66 -0
- eastworlds_forge-0.2.0/pyproject.toml +138 -0
- eastworlds_forge-0.2.0/src/forge_cli/__init__.py +3 -0
- eastworlds_forge-0.2.0/src/forge_cli/__main__.py +3 -0
- eastworlds_forge-0.2.0/src/forge_cli/api/__init__.py +0 -0
- eastworlds_forge-0.2.0/src/forge_cli/api/backend.py +131 -0
- eastworlds_forge-0.2.0/src/forge_cli/api/http.py +207 -0
- eastworlds_forge-0.2.0/src/forge_cli/api/uploads.py +79 -0
- eastworlds_forge-0.2.0/src/forge_cli/cli/__init__.py +199 -0
- eastworlds_forge-0.2.0/src/forge_cli/cli/auth.py +276 -0
- eastworlds_forge-0.2.0/src/forge_cli/cli/catalog.py +185 -0
- eastworlds_forge-0.2.0/src/forge_cli/cli/context.py +89 -0
- eastworlds_forge-0.2.0/src/forge_cli/cli/convert.py +127 -0
- eastworlds_forge-0.2.0/src/forge_cli/cli/output.py +85 -0
- eastworlds_forge-0.2.0/src/forge_cli/cli/self_cmd.py +261 -0
- eastworlds_forge-0.2.0/src/forge_cli/cli/transfer.py +308 -0
- eastworlds_forge-0.2.0/src/forge_cli/convert/__init__.py +0 -0
- eastworlds_forge-0.2.0/src/forge_cli/convert/dataset.py +539 -0
- eastworlds_forge-0.2.0/src/forge_cli/convert/handheld.py +470 -0
- eastworlds_forge-0.2.0/src/forge_cli/convert/msgdefs/ffmpeg_image_transport_msgs__msg__FFMPEGPacket.msg +50 -0
- eastworlds_forge-0.2.0/src/forge_cli/convert/msgdefs/sensor_msgs__msg__JointState.msg +51 -0
- eastworlds_forge-0.2.0/src/forge_cli/convert/msgdefs/tf2_msgs__msg__TFMessage.msg +77 -0
- eastworlds_forge-0.2.0/src/forge_cli/convert/to_mcap.py +270 -0
- eastworlds_forge-0.2.0/src/forge_cli/core/__init__.py +0 -0
- eastworlds_forge-0.2.0/src/forge_cli/core/config.py +161 -0
- eastworlds_forge-0.2.0/src/forge_cli/core/credentials.py +174 -0
- eastworlds_forge-0.2.0/src/forge_cli/core/errors.py +49 -0
- eastworlds_forge-0.2.0/src/forge_cli/core/hashing.py +71 -0
- eastworlds_forge-0.2.0/src/forge_cli/core/util.py +51 -0
- eastworlds_forge-0.2.0/src/forge_cli/media/__init__.py +0 -0
- eastworlds_forge-0.2.0/src/forge_cli/media/h264.py +114 -0
- eastworlds_forge-0.2.0/src/forge_cli/media/lerobot_resample.py +179 -0
- eastworlds_forge-0.2.0/src/forge_cli/media/resample.py +220 -0
- eastworlds_forge-0.2.0/src/forge_cli/selfmgmt/__init__.py +0 -0
- eastworlds_forge-0.2.0/src/forge_cli/selfmgmt/check.py +74 -0
- eastworlds_forge-0.2.0/src/forge_cli/selfmgmt/install.py +298 -0
- eastworlds_forge-0.2.0/src/forge_cli/selfmgmt/layout.py +111 -0
- eastworlds_forge-0.2.0/src/forge_cli/selfmgmt/release.py +119 -0
- eastworlds_forge-0.2.0/src/forge_cli/services/__init__.py +0 -0
- eastworlds_forge-0.2.0/src/forge_cli/services/download.py +502 -0
- eastworlds_forge-0.2.0/src/forge_cli/services/events.py +34 -0
- eastworlds_forge-0.2.0/src/forge_cli/services/postprocess.py +83 -0
- eastworlds_forge-0.2.0/src/forge_cli/services/scan.py +110 -0
- eastworlds_forge-0.2.0/src/forge_cli/services/selection.py +82 -0
- eastworlds_forge-0.2.0/src/forge_cli/services/upload.py +607 -0
- eastworlds_forge-0.2.0/src/forge_cli/services/validate.py +107 -0
- eastworlds_forge-0.2.0/src/forge_cli/transfer/__init__.py +37 -0
- eastworlds_forge-0.2.0/src/forge_cli/transfer/get.py +83 -0
- eastworlds_forge-0.2.0/src/forge_cli/transfer/put.py +78 -0
- eastworlds_forge-0.2.0/src/forge_cli/vendor/__init__.py +0 -0
- eastworlds_forge-0.2.0/src/forge_cli/vendor/forge_ingest.py +738 -0
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: eastworlds-forge
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: The Forge command-line interface: sign in with a Forge API key, download and upload dataset episodes, convert MCAP <-> LeRobot v2.1 locally.
|
|
5
|
+
Project-URL: Documentation, https://forge.eastworlds.io/how-to/cli
|
|
6
|
+
Project-URL: Source, https://github.com/Eastworld-Labs/forge_cli
|
|
7
|
+
Project-URL: Releases, https://github.com/Eastworld-Labs/forge_cli/releases
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Requires-Dist: av>=15
|
|
10
|
+
Requires-Dist: httpx>=0.27
|
|
11
|
+
Requires-Dist: mcap-ros2-support>=0.5
|
|
12
|
+
Requires-Dist: mcap>=1.2
|
|
13
|
+
Requires-Dist: numpy>=1.26
|
|
14
|
+
Requires-Dist: platformdirs>=4
|
|
15
|
+
Requires-Dist: pyarrow>=15
|
|
16
|
+
Requires-Dist: rich>=13
|
|
17
|
+
Requires-Dist: tomli-w>=1
|
|
18
|
+
Requires-Dist: tomli>=2; python_version < '3.11'
|
|
19
|
+
Requires-Dist: typer>=0.12
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
22
|
+
Requires-Dist: respx>=0.21; extra == 'dev'
|
|
23
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
24
|
+
Provides-Extra: keyring
|
|
25
|
+
Requires-Dist: keyring>=25; extra == 'keyring'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# forge
|
|
29
|
+
|
|
30
|
+
The Forge command-line interface. Sign in once with a Forge API key, then download and upload dataset episodes, convert MCAP ↔ LeRobot v2.1, and lower the video frame rate. **Conversion and frame-rate changes run on your computer only**; Forge stores and serves episodes exactly as they were uploaded.
|
|
31
|
+
|
|
32
|
+
The full guide is on Forge: **[Documentation → Command-Line Interface (CLI)](https://forge.eastworlds.io/how-to/cli)**.
|
|
33
|
+
|
|
34
|
+
## Install
|
|
35
|
+
|
|
36
|
+
**Linux only:** x86_64 or arm64, glibc 2.28+ (e.g. Ubuntu 20.04+, Debian 12+, RHEL/Rocky 8+). On Windows, run it inside WSL2 (Ubuntu). macOS is not supported. No root, Python or ffmpeg needed.
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
curl -fsSL https://forge.eastworlds.io/install.sh | bash
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
This installs forge with its own Python into `~/.forge` (about 310 MB) and links `forge` and `eforge` (the same program) into `~/.local/bin`. A specific version: `… | bash -s -- 0.2.0`.
|
|
43
|
+
|
|
44
|
+
Prefer a Python tool manager (Linux)? `uv tool install eastworlds-forge` or `pipx install eastworlds-forge`.
|
|
45
|
+
|
|
46
|
+
## Sign in
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
forge
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The first run opens **Forge → API keys** with a CLI key filled in. Press **Create**, copy the command the page shows, and paste it at the prompt. In CI, set `FORGE_API_KEY` instead.
|
|
53
|
+
|
|
54
|
+
## Use
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
forge datasets list
|
|
58
|
+
forge download eastworlds/yam-fold-handkerchief -o ./data --verdict valid
|
|
59
|
+
forge download eastworlds/yam-fold-handkerchief -o ./lerobot --format lerobot-v21 --fps 10
|
|
60
|
+
forge upload ./recordings/2026-09-29 --to eastworlds/yam-fold-handkerchief
|
|
61
|
+
forge convert mcap-to-lerobot ./data/eastworlds/yam-fold-handkerchief -o ./lerobot/yam_fold
|
|
62
|
+
forge convert lerobot-to-mcap ./lerobot/yam_fold -o ./mcap_again
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Global options such as `--json`, `--profile` and `-y` go before the command: `forge --json datasets list`.
|
|
66
|
+
|
|
67
|
+
`forge update` updates, `forge doctor` checks the setup, `forge uninstall` removes everything (your keys stay unless you confirm).
|
|
68
|
+
|
|
69
|
+
| Exit code | Meaning |
|
|
70
|
+
|---|---|
|
|
71
|
+
| 0 | OK |
|
|
72
|
+
| 1 | Failure |
|
|
73
|
+
| 2 | Usage or validation error |
|
|
74
|
+
| 3 | Bad or expired key |
|
|
75
|
+
| 4 | Missing scope |
|
|
76
|
+
| 5 | Not found |
|
|
77
|
+
| 6 | Conflict |
|
|
78
|
+
| 7 | Server or network error |
|
|
79
|
+
| 130 | Interrupted |
|
|
80
|
+
|
|
81
|
+
## Develop
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
python3.11 -m venv .venv && .venv/bin/pip install -e '.[dev]'
|
|
85
|
+
.venv/bin/pytest
|
|
86
|
+
.venv/bin/ruff check src tests
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
- `src/forge_cli/convert/` is a port of the production-verified MCAP → LeRobot v2.1 converter. `tests/test_convert_golden.py` compares the two on real recordings when `FORGE_VERIFIED_SAMPLE` and `FORGE_GOLDEN_EPISODES` are set.
|
|
90
|
+
- `install/install.sh` is the installer; `tests/test_install.py` runs it against a local fake release.
|
|
91
|
+
- `scripts/build_release.py --out dist/release` builds what the download host serves. On a `vX.Y.Z` tag, `.github/workflows/release.yml` publishes it to Tencent COS behind EdgeOne (`https://cli-forge.eastworlds.io`); setup in [docs/RELEASE_INFRA.md](docs/RELEASE_INFRA.md).
|
|
92
|
+
|
|
93
|
+
Design: [docs/INITIAL_PLAN_V2.md](docs/INITIAL_PLAN_V2.md). What was built: [docs/INITIAL_DONE_V2.md](docs/INITIAL_DONE_V2.md).
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# forge
|
|
2
|
+
|
|
3
|
+
The Forge command-line interface. Sign in once with a Forge API key, then download and upload dataset episodes, convert MCAP ↔ LeRobot v2.1, and lower the video frame rate. **Conversion and frame-rate changes run on your computer only**; Forge stores and serves episodes exactly as they were uploaded.
|
|
4
|
+
|
|
5
|
+
The full guide is on Forge: **[Documentation → Command-Line Interface (CLI)](https://forge.eastworlds.io/how-to/cli)**.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
**Linux only:** x86_64 or arm64, glibc 2.28+ (e.g. Ubuntu 20.04+, Debian 12+, RHEL/Rocky 8+). On Windows, run it inside WSL2 (Ubuntu). macOS is not supported. No root, Python or ffmpeg needed.
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
curl -fsSL https://forge.eastworlds.io/install.sh | bash
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
This installs forge with its own Python into `~/.forge` (about 310 MB) and links `forge` and `eforge` (the same program) into `~/.local/bin`. A specific version: `… | bash -s -- 0.2.0`.
|
|
16
|
+
|
|
17
|
+
Prefer a Python tool manager (Linux)? `uv tool install eastworlds-forge` or `pipx install eastworlds-forge`.
|
|
18
|
+
|
|
19
|
+
## Sign in
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
forge
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
The first run opens **Forge → API keys** with a CLI key filled in. Press **Create**, copy the command the page shows, and paste it at the prompt. In CI, set `FORGE_API_KEY` instead.
|
|
26
|
+
|
|
27
|
+
## Use
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
forge datasets list
|
|
31
|
+
forge download eastworlds/yam-fold-handkerchief -o ./data --verdict valid
|
|
32
|
+
forge download eastworlds/yam-fold-handkerchief -o ./lerobot --format lerobot-v21 --fps 10
|
|
33
|
+
forge upload ./recordings/2026-09-29 --to eastworlds/yam-fold-handkerchief
|
|
34
|
+
forge convert mcap-to-lerobot ./data/eastworlds/yam-fold-handkerchief -o ./lerobot/yam_fold
|
|
35
|
+
forge convert lerobot-to-mcap ./lerobot/yam_fold -o ./mcap_again
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Global options such as `--json`, `--profile` and `-y` go before the command: `forge --json datasets list`.
|
|
39
|
+
|
|
40
|
+
`forge update` updates, `forge doctor` checks the setup, `forge uninstall` removes everything (your keys stay unless you confirm).
|
|
41
|
+
|
|
42
|
+
| Exit code | Meaning |
|
|
43
|
+
|---|---|
|
|
44
|
+
| 0 | OK |
|
|
45
|
+
| 1 | Failure |
|
|
46
|
+
| 2 | Usage or validation error |
|
|
47
|
+
| 3 | Bad or expired key |
|
|
48
|
+
| 4 | Missing scope |
|
|
49
|
+
| 5 | Not found |
|
|
50
|
+
| 6 | Conflict |
|
|
51
|
+
| 7 | Server or network error |
|
|
52
|
+
| 130 | Interrupted |
|
|
53
|
+
|
|
54
|
+
## Develop
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
python3.11 -m venv .venv && .venv/bin/pip install -e '.[dev]'
|
|
58
|
+
.venv/bin/pytest
|
|
59
|
+
.venv/bin/ruff check src tests
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
- `src/forge_cli/convert/` is a port of the production-verified MCAP → LeRobot v2.1 converter. `tests/test_convert_golden.py` compares the two on real recordings when `FORGE_VERIFIED_SAMPLE` and `FORGE_GOLDEN_EPISODES` are set.
|
|
63
|
+
- `install/install.sh` is the installer; `tests/test_install.py` runs it against a local fake release.
|
|
64
|
+
- `scripts/build_release.py --out dist/release` builds what the download host serves. On a `vX.Y.Z` tag, `.github/workflows/release.yml` publishes it to Tencent COS behind EdgeOne (`https://cli-forge.eastworlds.io`); setup in [docs/RELEASE_INFRA.md](docs/RELEASE_INFRA.md).
|
|
65
|
+
|
|
66
|
+
Design: [docs/INITIAL_PLAN_V2.md](docs/INITIAL_PLAN_V2.md). What was built: [docs/INITIAL_DONE_V2.md](docs/INITIAL_DONE_V2.md).
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.24"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
[project]
|
|
7
|
+
name = "eastworlds-forge"
|
|
8
|
+
dynamic = ["version"]
|
|
9
|
+
|
|
10
|
+
description = "The Forge command-line interface: sign in with a Forge API key, download and upload dataset episodes, convert MCAP <-> LeRobot v2.1 locally."
|
|
11
|
+
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
|
|
14
|
+
requires-python = ">=3.10"
|
|
15
|
+
|
|
16
|
+
dependencies = [
|
|
17
|
+
"httpx>=0.27",
|
|
18
|
+
"typer>=0.12",
|
|
19
|
+
"rich>=13",
|
|
20
|
+
"platformdirs>=4",
|
|
21
|
+
|
|
22
|
+
# Python < 3.11 does not include tomllib.
|
|
23
|
+
"tomli>=2; python_version < '3.11'",
|
|
24
|
+
|
|
25
|
+
"tomli-w>=1",
|
|
26
|
+
|
|
27
|
+
# Conversion and frame-rate changes run locally and are core CLI features.
|
|
28
|
+
"av>=15",
|
|
29
|
+
"mcap>=1.2",
|
|
30
|
+
"mcap-ros2-support>=0.5",
|
|
31
|
+
"numpy>=1.26",
|
|
32
|
+
"pyarrow>=15",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
|
|
38
|
+
# Optional OS keychain/keyring integration.
|
|
39
|
+
keyring = [
|
|
40
|
+
"keyring>=25",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
# Local development/test dependencies.
|
|
44
|
+
dev = [
|
|
45
|
+
"pytest>=8",
|
|
46
|
+
"respx>=0.21",
|
|
47
|
+
"ruff>=0.5",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
[project.scripts]
|
|
52
|
+
|
|
53
|
+
# Primary CLI command.
|
|
54
|
+
forge = "forge_cli.cli:main"
|
|
55
|
+
|
|
56
|
+
# Alias that avoids conflicts with other tools named `forge`.
|
|
57
|
+
eforge = "forge_cli.cli:main"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
[project.urls]
|
|
61
|
+
Documentation = "https://forge.eastworlds.io/how-to/cli"
|
|
62
|
+
Source = "https://github.com/Eastworld-Labs/forge_cli"
|
|
63
|
+
Releases = "https://github.com/Eastworld-Labs/forge_cli/releases"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# ---------------------------------------------------------------------------
|
|
67
|
+
# Versioning
|
|
68
|
+
# ---------------------------------------------------------------------------
|
|
69
|
+
|
|
70
|
+
# Keep the package version in one location:
|
|
71
|
+
#
|
|
72
|
+
# src/forge_cli/__init__.py
|
|
73
|
+
#
|
|
74
|
+
# Example:
|
|
75
|
+
#
|
|
76
|
+
# __version__ = "0.2.0"
|
|
77
|
+
#
|
|
78
|
+
# Hatch reads that value when building the wheel/sdist.
|
|
79
|
+
[tool.hatch.version]
|
|
80
|
+
path = "src/forge_cli/__init__.py"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# ---------------------------------------------------------------------------
|
|
84
|
+
# Build configuration
|
|
85
|
+
# ---------------------------------------------------------------------------
|
|
86
|
+
|
|
87
|
+
[tool.hatch.build.targets.wheel]
|
|
88
|
+
packages = ["src/forge_cli"]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
[tool.hatch.build.targets.sdist]
|
|
92
|
+
include = [
|
|
93
|
+
"/src",
|
|
94
|
+
"/README.md",
|
|
95
|
+
"/pyproject.toml",
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
# ---------------------------------------------------------------------------
|
|
100
|
+
# Pytest
|
|
101
|
+
# ---------------------------------------------------------------------------
|
|
102
|
+
|
|
103
|
+
[tool.pytest.ini_options]
|
|
104
|
+
testpaths = ["tests"]
|
|
105
|
+
addopts = "-q"
|
|
106
|
+
|
|
107
|
+
filterwarnings = [
|
|
108
|
+
"ignore::DeprecationWarning:mcap_ros2.*",
|
|
109
|
+
]
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# ---------------------------------------------------------------------------
|
|
113
|
+
# Ruff
|
|
114
|
+
# ---------------------------------------------------------------------------
|
|
115
|
+
|
|
116
|
+
[tool.ruff]
|
|
117
|
+
line-length = 120
|
|
118
|
+
src = ["src", "tests"]
|
|
119
|
+
|
|
120
|
+
extend-exclude = [
|
|
121
|
+
"src/forge_cli/vendor",
|
|
122
|
+
"scripts",
|
|
123
|
+
]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
[tool.ruff.lint]
|
|
127
|
+
select = [
|
|
128
|
+
"E",
|
|
129
|
+
"F",
|
|
130
|
+
"W",
|
|
131
|
+
"I",
|
|
132
|
+
"B",
|
|
133
|
+
"UP",
|
|
134
|
+
]
|
|
135
|
+
|
|
136
|
+
# typer.Option(...) / typer.Argument(...) in function defaults is the
|
|
137
|
+
# conventional Typer API style.
|
|
138
|
+
ignore = ["B008"]
|
|
File without changes
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""The Forge API (`api-forge.*`): identity, catalog, downloads.
|
|
2
|
+
|
|
3
|
+
Only this module knows these paths and payload shapes. Responses stay plain
|
|
4
|
+
dicts: the server adds fields freely and an old CLI must keep working.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Iterator
|
|
10
|
+
from typing import Any
|
|
11
|
+
from urllib.parse import quote
|
|
12
|
+
|
|
13
|
+
from .http import ApiClient, _flatten
|
|
14
|
+
|
|
15
|
+
TERMINAL_DOWNLOAD_STATES = {"ready", "failed", "expired", "needs_resolution"}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def split_dataset(full_name: str) -> tuple[str, str]:
|
|
19
|
+
namespace, sep, slug = full_name.strip().strip("/").partition("/")
|
|
20
|
+
if not sep or not namespace or not slug or "/" in slug:
|
|
21
|
+
from ..core.errors import UsageError
|
|
22
|
+
|
|
23
|
+
raise UsageError(f"expected NAMESPACE/DATASET, got {full_name!r}")
|
|
24
|
+
return namespace, slug
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _ds(namespace: str, slug: str) -> str:
|
|
28
|
+
return f"/api/v1/{quote(namespace, safe='')}/{quote(slug, safe='')}"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Backend:
|
|
32
|
+
def __init__(self, client: ApiClient) -> None:
|
|
33
|
+
self.c = client
|
|
34
|
+
|
|
35
|
+
# ------------------------------------------------------------ identity
|
|
36
|
+
def me(self) -> dict[str, Any]:
|
|
37
|
+
return self.c.get("/api/v1/me", what="check key")
|
|
38
|
+
|
|
39
|
+
def viewer_config(self) -> dict[str, Any]:
|
|
40
|
+
return self.c.get("/api/v1/viewer-config", what="read viewer-config")
|
|
41
|
+
|
|
42
|
+
# ------------------------------------------------------------ datasets
|
|
43
|
+
def list_datasets(self, *, limit: int | None = None, **filters: Any) -> Iterator[dict[str, Any]]:
|
|
44
|
+
return self.c.paginate("/api/v1/datasets", filters, limit=limit)
|
|
45
|
+
|
|
46
|
+
def get_dataset(self, namespace: str, slug: str) -> dict[str, Any]:
|
|
47
|
+
return self.c.get(_ds(namespace, slug), what=f"dataset {namespace}/{slug}")
|
|
48
|
+
|
|
49
|
+
def create_dataset(self, namespace: str, slug: str, **fields: Any) -> dict[str, Any]:
|
|
50
|
+
body = {"namespace": namespace, "slug": slug, **{k: v for k, v in fields.items() if v is not None}}
|
|
51
|
+
return self.c.post("/api/v1/datasets", body, what=f"create dataset {namespace}/{slug}")
|
|
52
|
+
|
|
53
|
+
# ------------------------------------------------------------ episodes
|
|
54
|
+
def list_episodes(
|
|
55
|
+
self, namespace: str, slug: str, *, limit: int | None = None, **filters: Any
|
|
56
|
+
) -> Iterator[dict[str, Any]]:
|
|
57
|
+
return self.c.paginate(f"{_ds(namespace, slug)}/episodes", filters, limit=limit)
|
|
58
|
+
|
|
59
|
+
def get_episode(self, namespace: str, slug: str, name: str) -> dict[str, Any]:
|
|
60
|
+
return self.c.get(
|
|
61
|
+
f"{_ds(namespace, slug)}/episodes/{quote(name, safe='')}", what=f"episode {name}"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
def signed_file_url(self, namespace: str, slug: str, name: str, relpath: str) -> str | None:
|
|
65
|
+
"""A short-lived presigned GET for one manifest file, or None when the
|
|
66
|
+
server has per-file links switched off (the media gateway is on)."""
|
|
67
|
+
from ..core.errors import ForgeError
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
body = self.c.get(
|
|
71
|
+
f"{_ds(namespace, slug)}/episodes/{quote(name, safe='')}/files/{quote(relpath)}",
|
|
72
|
+
params={"signed": "1"}, what=f"link for {name}/{relpath}",
|
|
73
|
+
)
|
|
74
|
+
except ForgeError as exc:
|
|
75
|
+
if exc.exit_code in (4, 5): # forbidden / not found: no per-file links here
|
|
76
|
+
return None
|
|
77
|
+
raise
|
|
78
|
+
return body.get("url") if isinstance(body, dict) else None
|
|
79
|
+
|
|
80
|
+
# ----------------------------------------------------------- downloads
|
|
81
|
+
def downloads_health(self) -> dict[str, Any]:
|
|
82
|
+
return self.c.get("/api/v1/downloads/health", what="downloads health")
|
|
83
|
+
|
|
84
|
+
def file_types(self, dataset: str, episodes: list[str] | None) -> dict[str, Any]:
|
|
85
|
+
# GET with names in the query string until it gets long, then POST --
|
|
86
|
+
# the same switch the web app makes (80 names).
|
|
87
|
+
if episodes is not None and len(episodes) > 80:
|
|
88
|
+
return self.c.post(
|
|
89
|
+
"/api/v1/downloads/file-types",
|
|
90
|
+
{"dataset": dataset, "episodes": episodes},
|
|
91
|
+
retry=True,
|
|
92
|
+
what="list file types",
|
|
93
|
+
)
|
|
94
|
+
params = _flatten({"dataset": dataset, "episodes": episodes})
|
|
95
|
+
return self.c.get("/api/v1/downloads/file-types", params=params, what="list file types")
|
|
96
|
+
|
|
97
|
+
def create_download(self, selections: list[dict[str, Any]], mode: str) -> dict[str, Any]:
|
|
98
|
+
return self.c.post(
|
|
99
|
+
"/api/v1/downloads", {"selections": selections, "mode": mode}, what="create download"
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
def get_download(self, sid: str) -> dict[str, Any]:
|
|
103
|
+
return self.c.get(f"/api/v1/downloads/{quote(sid, safe='')}", what=f"download {sid}")
|
|
104
|
+
|
|
105
|
+
def repackage(self, sid: str) -> dict[str, Any]:
|
|
106
|
+
return self.c.post(f"/api/v1/downloads/{quote(sid, safe='')}/repackage", what="repackage")
|
|
107
|
+
|
|
108
|
+
def resolve_meta(self, sid: str, info: dict[str, Any], tasks: list[str] | None) -> dict[str, Any]:
|
|
109
|
+
body: dict[str, Any] = {"info": info}
|
|
110
|
+
if tasks is not None:
|
|
111
|
+
body["tasks"] = tasks
|
|
112
|
+
return self.c.post(f"/api/v1/downloads/{quote(sid, safe='')}/meta", body, what="resolve info.json")
|
|
113
|
+
|
|
114
|
+
def archive_location(self, sid: str, part: int | None) -> str | None:
|
|
115
|
+
"""Where the archive bytes are.
|
|
116
|
+
|
|
117
|
+
Redirect mode (the default): the 307's presigned COS URL, valid ~300 s,
|
|
118
|
+
so callers ask again when a resumed transfer needs a fresh one. Proxy
|
|
119
|
+
mode: None, meaning "stream it from this API" (`archive_stream`).
|
|
120
|
+
"""
|
|
121
|
+
params = {"part": part} if part else None
|
|
122
|
+
response = self.c.send(
|
|
123
|
+
"GET", f"/api/v1/downloads/{quote(sid, safe='')}/archive", params=params, what="archive"
|
|
124
|
+
)
|
|
125
|
+
if response.status_code in (301, 302, 303, 307, 308):
|
|
126
|
+
return response.headers["location"]
|
|
127
|
+
response.close()
|
|
128
|
+
return None
|
|
129
|
+
|
|
130
|
+
def archive_path(self, sid: str) -> str:
|
|
131
|
+
return f"/api/v1/downloads/{quote(sid, safe='')}/archive"
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""The one HTTP client both Forge services are called through.
|
|
2
|
+
|
|
3
|
+
- `Authorization: Bearer` goes ONLY to the host it was built for. Presigned
|
|
4
|
+
COS URLs are fetched with a separate, credential-free client (transfer/).
|
|
5
|
+
- A real `User-Agent`: Cloudflare in front of api-forge/uploads-forge answers
|
|
6
|
+
the default Python agents with a 403 (error 1010).
|
|
7
|
+
- Retries with exponential backoff + jitter on connection errors, 429 (honouring
|
|
8
|
+
Retry-After) and 502/503/504. Non-idempotent POSTs are retried only when the
|
|
9
|
+
caller says the endpoint is safe to repeat.
|
|
10
|
+
- FastAPI's `detail` becomes a ForgeError carrying the CLI's exit code.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import platform
|
|
16
|
+
import random
|
|
17
|
+
import time
|
|
18
|
+
from collections.abc import Callable
|
|
19
|
+
from typing import Any
|
|
20
|
+
from urllib.parse import urlsplit
|
|
21
|
+
|
|
22
|
+
import httpx
|
|
23
|
+
|
|
24
|
+
from .. import __version__
|
|
25
|
+
from ..core.errors import (
|
|
26
|
+
AuthError,
|
|
27
|
+
Conflict,
|
|
28
|
+
ForgeError,
|
|
29
|
+
NotFound,
|
|
30
|
+
PermissionDenied,
|
|
31
|
+
ServerError,
|
|
32
|
+
UsageError,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
USER_AGENT = f"forge-cli/{__version__} (python-httpx/{httpx.__version__}; {platform.system().lower()})"
|
|
36
|
+
RETRY_STATUSES = {429, 502, 503, 504}
|
|
37
|
+
IDEMPOTENT = {"GET", "HEAD", "PUT", "DELETE", "OPTIONS"}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def detail_text(response: httpx.Response) -> tuple[str, Any]:
|
|
41
|
+
try:
|
|
42
|
+
body = response.json()
|
|
43
|
+
except ValueError:
|
|
44
|
+
return (response.text or response.reason_phrase or "").strip()[:500], None
|
|
45
|
+
detail = body.get("detail", body) if isinstance(body, dict) else body
|
|
46
|
+
if isinstance(detail, str):
|
|
47
|
+
return detail, body
|
|
48
|
+
if isinstance(detail, dict) and isinstance(detail.get("message"), str):
|
|
49
|
+
return detail["message"], detail
|
|
50
|
+
if isinstance(detail, list): # FastAPI 422 validation errors
|
|
51
|
+
parts = []
|
|
52
|
+
for item in detail[:5]:
|
|
53
|
+
if isinstance(item, dict):
|
|
54
|
+
loc = ".".join(str(p) for p in item.get("loc", []) if p != "body")
|
|
55
|
+
parts.append(f"{loc}: {item.get('msg')}" if loc else str(item.get("msg")))
|
|
56
|
+
return "; ".join(parts) or "invalid request", detail
|
|
57
|
+
return str(detail)[:500], detail
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def error_for(response: httpx.Response, *, what: str) -> ForgeError:
|
|
61
|
+
message, detail = detail_text(response)
|
|
62
|
+
status = response.status_code
|
|
63
|
+
text = f"{what}: {message}" if message else f"{what}: HTTP {status}"
|
|
64
|
+
if status == 401:
|
|
65
|
+
return AuthError(
|
|
66
|
+
text,
|
|
67
|
+
hint="the key is not valid, revoked or expired. Run: forge auth login",
|
|
68
|
+
detail=detail,
|
|
69
|
+
)
|
|
70
|
+
if status == 403:
|
|
71
|
+
hint = None
|
|
72
|
+
if "scope" in message:
|
|
73
|
+
hint = "create a key with that scope on Forge → API keys, then run: forge auth login"
|
|
74
|
+
elif "1010" in message or "cloudflare" in message.lower():
|
|
75
|
+
hint = "blocked by Cloudflare; check your network/proxy"
|
|
76
|
+
return PermissionDenied(text, hint=hint, detail=detail)
|
|
77
|
+
if status == 404:
|
|
78
|
+
return NotFound(text, detail=detail)
|
|
79
|
+
if status == 409:
|
|
80
|
+
return Conflict(text, detail=detail)
|
|
81
|
+
if status in (400, 413, 422):
|
|
82
|
+
return UsageError(text, detail=detail)
|
|
83
|
+
if status >= 500:
|
|
84
|
+
return ServerError(text, detail=detail)
|
|
85
|
+
return ForgeError(text, detail=detail)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class ApiClient:
|
|
89
|
+
def __init__(
|
|
90
|
+
self,
|
|
91
|
+
base_url: str,
|
|
92
|
+
api_key: str | None,
|
|
93
|
+
*,
|
|
94
|
+
timeout: float = 60.0,
|
|
95
|
+
retries: int = 5,
|
|
96
|
+
backoff: float = 0.5,
|
|
97
|
+
transport: httpx.BaseTransport | None = None,
|
|
98
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
99
|
+
) -> None:
|
|
100
|
+
self.base_url = base_url.rstrip("/")
|
|
101
|
+
self.host = urlsplit(self.base_url).netloc
|
|
102
|
+
self.retries = retries
|
|
103
|
+
self.backoff = backoff
|
|
104
|
+
self._sleep = sleep
|
|
105
|
+
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
|
106
|
+
if api_key:
|
|
107
|
+
headers["Authorization"] = f"Bearer {api_key}"
|
|
108
|
+
self.http = httpx.Client(
|
|
109
|
+
base_url=self.base_url,
|
|
110
|
+
headers=headers,
|
|
111
|
+
timeout=httpx.Timeout(timeout, connect=10.0),
|
|
112
|
+
follow_redirects=False, # a 307 to COS must not carry the Bearer header
|
|
113
|
+
transport=transport,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
def close(self) -> None:
|
|
117
|
+
self.http.close()
|
|
118
|
+
|
|
119
|
+
def __enter__(self) -> ApiClient:
|
|
120
|
+
return self
|
|
121
|
+
|
|
122
|
+
def __exit__(self, *exc: object) -> None:
|
|
123
|
+
self.close()
|
|
124
|
+
|
|
125
|
+
def _delay(self, attempt: int, response: httpx.Response | None) -> float:
|
|
126
|
+
if response is not None:
|
|
127
|
+
retry_after = response.headers.get("retry-after")
|
|
128
|
+
if retry_after and retry_after.isdigit():
|
|
129
|
+
return min(60.0, float(retry_after))
|
|
130
|
+
return min(30.0, self.backoff * (2**attempt)) * (0.5 + random.random() / 2)
|
|
131
|
+
|
|
132
|
+
def send(
|
|
133
|
+
self,
|
|
134
|
+
method: str,
|
|
135
|
+
path: str,
|
|
136
|
+
*,
|
|
137
|
+
params: Any = None,
|
|
138
|
+
json: Any = None,
|
|
139
|
+
retry: bool | None = None,
|
|
140
|
+
ok: tuple[int, ...] = (),
|
|
141
|
+
what: str | None = None,
|
|
142
|
+
) -> httpx.Response:
|
|
143
|
+
"""One request with the retry policy. Returns the response for 2xx/3xx
|
|
144
|
+
(and any status in `ok`); raises ForgeError otherwise."""
|
|
145
|
+
method = method.upper()
|
|
146
|
+
may_retry = (method in IDEMPOTENT) if retry is None else retry
|
|
147
|
+
attempt = 0
|
|
148
|
+
label = what or f"{method} {path}"
|
|
149
|
+
while True:
|
|
150
|
+
response: httpx.Response | None = None
|
|
151
|
+
try:
|
|
152
|
+
response = self.http.request(method, path, params=params, json=json)
|
|
153
|
+
except httpx.TransportError as exc:
|
|
154
|
+
if not may_retry or attempt >= self.retries:
|
|
155
|
+
raise ServerError(
|
|
156
|
+
f"{label}: cannot reach {self.host} ({type(exc).__name__}: {exc})",
|
|
157
|
+
hint="check the network, or the profile's URL (forge auth status)",
|
|
158
|
+
) from exc
|
|
159
|
+
else:
|
|
160
|
+
if response.status_code < 400 or response.status_code in ok:
|
|
161
|
+
return response
|
|
162
|
+
if response.status_code not in RETRY_STATUSES or not may_retry or attempt >= self.retries:
|
|
163
|
+
raise error_for(response, what=label)
|
|
164
|
+
self._sleep(self._delay(attempt, response))
|
|
165
|
+
attempt += 1
|
|
166
|
+
|
|
167
|
+
def get(self, path: str, **kw: Any) -> Any:
|
|
168
|
+
return self.send("GET", path, **kw).json()
|
|
169
|
+
|
|
170
|
+
def post(self, path: str, body: Any = None, **kw: Any) -> Any:
|
|
171
|
+
return self.send("POST", path, json=body if body is not None else {}, **kw).json()
|
|
172
|
+
|
|
173
|
+
def delete(self, path: str, **kw: Any) -> Any:
|
|
174
|
+
return self.send("DELETE", path, **kw).json()
|
|
175
|
+
|
|
176
|
+
def paginate(self, path: str, params: dict | None = None, *, page: int = 200, limit: int | None = None):
|
|
177
|
+
"""Yield items across `{items,total,limit,offset}` pages."""
|
|
178
|
+
offset, seen = 0, 0
|
|
179
|
+
while True:
|
|
180
|
+
query = dict(params or {})
|
|
181
|
+
query.update({"limit": page, "offset": offset})
|
|
182
|
+
data = self.get(path, params=list(_flatten(query)))
|
|
183
|
+
items = data.get("items") or []
|
|
184
|
+
for item in items:
|
|
185
|
+
yield item
|
|
186
|
+
seen += 1
|
|
187
|
+
if limit is not None and seen >= limit:
|
|
188
|
+
return
|
|
189
|
+
offset += len(items)
|
|
190
|
+
total = data.get("total")
|
|
191
|
+
if not items or (total is not None and offset >= total):
|
|
192
|
+
return
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _flatten(params: dict) -> list[tuple[str, Any]]:
|
|
196
|
+
"""`{"task": ["a","b"]}` -> repeated query params, dropping None."""
|
|
197
|
+
out: list[tuple[str, Any]] = []
|
|
198
|
+
for key, value in params.items():
|
|
199
|
+
if value is None:
|
|
200
|
+
continue
|
|
201
|
+
if isinstance(value, (list, tuple)):
|
|
202
|
+
out.extend((key, v) for v in value)
|
|
203
|
+
elif isinstance(value, bool):
|
|
204
|
+
out.append((key, "true" if value else "false"))
|
|
205
|
+
else:
|
|
206
|
+
out.append((key, value))
|
|
207
|
+
return out
|