omicslab 2.2.0__tar.gz → 2.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {omicslab-2.2.0 → omicslab-2.5.1}/PKG-INFO +14 -5
- {omicslab-2.2.0 → omicslab-2.5.1}/README.developer.md +43 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/README.md +12 -4
- {omicslab-2.2.0 → omicslab-2.5.1}/pyproject.toml +1 -0
- omicslab-2.5.1/src/omicslab/_version.py +1 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/main.py +78 -11
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/client.py +10 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/config.py +16 -1
- omicslab-2.5.1/src/omicslab/job/_run/_broker.py +235 -0
- omicslab-2.5.1/src/omicslab/job/_run/_git_proxy.py +196 -0
- omicslab-2.5.1/src/omicslab/job/_run/_s3_proxy.py +242 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_storage.py +19 -5
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/analysis.py +6 -0
- omicslab-2.5.1/src/omicslab/job/_run/studio.py +1189 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/storage.py +16 -9
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/conftest.py +2 -1
- omicslab-2.5.1/tests/job/_run/test__broker.py +146 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__engine_helpers.py +26 -2
- omicslab-2.5.1/tests/job/_run/test__git_proxy.py +105 -0
- omicslab-2.5.1/tests/job/_run/test__s3_proxy.py +175 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__storage.py +88 -0
- omicslab-2.5.1/tests/job/_run/test__studio_docker.py +349 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_cli.py +2 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_cli_storage.py +170 -0
- omicslab-2.5.1/tests/test_client_broker.py +51 -0
- omicslab-2.5.1/tests/test_config.py +20 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_resources.py +6 -2
- omicslab-2.2.0/src/omicslab/_version.py +0 -1
- omicslab-2.2.0/src/omicslab/job/_run/studio.py +0 -613
- omicslab-2.2.0/tests/job/_run/test__studio_docker.py +0 -107
- {omicslab-2.2.0 → omicslab-2.5.1}/.gitignore +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/Makefile +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/.gitignore +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/hcm.pixi.lock +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/hcm.pixi.toml +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/pixi.lock +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/pixi.toml +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/us-east-1.pixi.lock +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/us-east-1.pixi.toml +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/pixi.toml +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/prod/hcm.pixi.lock +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/prod/hcm.pixi.toml +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/prod/pixi.lock +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/prod/pixi.toml +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/envs/regions.json +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/pixi.config.toml +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/scripts/bootstrap-env.sh +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/scripts/check_publish.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/scripts/write_version_manifest.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/__init__.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/__init__.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/analysis_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/auth_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/context_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/credential_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/fuse_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/job_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/org_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/runner_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/setup_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/studio_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/token_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/utils.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/workspace_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/ws_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/exceptions.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/__init__.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/__init__.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_auth.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_config_download.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_consts.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_data_prep.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_env.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_exec.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_git.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_images.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_log.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_monitor.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_nextflow.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_params.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_proc.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_proxy.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_spec.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_studio_prep.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/client.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/console.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/executor.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/paths.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/system.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/__init__.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/analysis.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/base.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/credentials.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/jobs.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/orgs.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/runners.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/studio.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/tokens.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/workspaces.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/selfupdate.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/__init__.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/e2e/__init__.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/e2e/conftest.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/e2e/test_cli_e2e.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/e2e/test_tokens.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/fixtures/nfcore-demo-custom.config +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/fixtures/studio_specs/jupyter.json +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/fixtures/studio_specs/rstudio.json +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__auth.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__data_prep.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__env.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__get_images_cli.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__git.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__images.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__log.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__monitor.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__nextflow.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__params.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__proxy.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__proxy_extra.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__spec.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test_studio_proxy_start.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_cli.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_client.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_client_extra.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_executor.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_executor_extra.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_fuse.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_resource.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_system.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_snapshot_request.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_studio_image_download.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_studio_validate_cli.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_ws_helpers.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_cli_catalog.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_cli_context.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_cli_utils.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_client.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_fuse.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_job_exec.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_job_heartbeat.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_misc.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_runner_helpers.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_runner_self_update.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_sandbox_report.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_selfupdate.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_studio_snapshot.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_system_monitor_interval.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_ws_commands.py +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-nextflow/input.txt +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-nextflow/main.nf +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-nextflow/omicslab/main.sh +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-nextflow/pixi.toml +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-nextflow/schema.json +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-simple/omicslab/main.sh +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-simple/pixi.toml +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-simple/schema.json +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-snakemake/Snakefile +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-snakemake/input.txt +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-snakemake/omicslab/main.sh +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-snakemake/pixi.toml +0 -0
- {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-snakemake/schema.json +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: omicslab
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.5.1
|
|
4
4
|
Summary: Python SDK and CLI for the OmicsLab Platform
|
|
5
5
|
Project-URL: Homepage, https://platform.omicslab.io
|
|
6
6
|
Project-URL: Repository, https://github.com/vieomics/omicslab-platform
|
|
@@ -14,6 +14,7 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
15
15
|
Requires-Python: >=3.12
|
|
16
16
|
Requires-Dist: aiohttp>=3.10
|
|
17
|
+
Requires-Dist: botocore<2,>=1.35
|
|
17
18
|
Requires-Dist: httpx>=0.27
|
|
18
19
|
Requires-Dist: psutil>=6
|
|
19
20
|
Requires-Dist: pydantic>=2
|
|
@@ -93,7 +94,7 @@ omicslab upload -y ./data.csv s3://my-bucket/data/
|
|
|
93
94
|
|
|
94
95
|
### Download
|
|
95
96
|
|
|
96
|
-
Download files or directories from workspace storage.
|
|
97
|
+
Download files or directories from workspace storage. A folder is detected automatically, so a trailing slash and `-r` are optional; recursive downloads preserve subdirectory structure and use parallel threads.
|
|
97
98
|
|
|
98
99
|
```bash
|
|
99
100
|
# Single file (defaults to filename in current directory)
|
|
@@ -105,7 +106,13 @@ omicslab download results/run1/data.csv ./local_copy.csv
|
|
|
105
106
|
# Download from s3:// path (auto-resolves workspace)
|
|
106
107
|
omicslab download s3://my-bucket/data/results/report.html
|
|
107
108
|
|
|
108
|
-
#
|
|
109
|
+
# Folder download (auto-detected; defaults to ./run1)
|
|
110
|
+
omicslab download results/run1
|
|
111
|
+
|
|
112
|
+
# Folder download with explicit destination
|
|
113
|
+
omicslab download results/run1 ./local_results/
|
|
114
|
+
|
|
115
|
+
# Same thing, explicit recursive (preserves subdirectory structure)
|
|
109
116
|
omicslab download -r s3://my-bucket/data/results/ ./local_results/
|
|
110
117
|
|
|
111
118
|
# Recursive download with custom thread count
|
|
@@ -119,12 +126,13 @@ omicslab download -y -r s3://my-bucket/data/results/ ./local_results/
|
|
|
119
126
|
|
|
120
127
|
| Flag | Description |
|
|
121
128
|
|------|-------------|
|
|
122
|
-
| `--recursive`, `-r` |
|
|
129
|
+
| `--recursive`, `-r` | Force recursive download (folders are detected automatically) |
|
|
130
|
+
| `--zip` | Download a folder as a server-built zip archive |
|
|
123
131
|
| `--threads` | Concurrent download threads (default: 4) |
|
|
124
132
|
| `--workspace-id`, `-w` | Workspace ID (inferred from s3:// path if provided) |
|
|
125
133
|
| `--yes`, `-y` | Skip confirmation prompt for s3:// path resolution |
|
|
126
134
|
|
|
127
|
-
**Recursive download** lists all objects under the prefix, then downloads them in parallel while preserving the original subdirectory structure under the destination directory.
|
|
135
|
+
**Recursive download** lists all objects under the prefix, then downloads them in parallel while preserving the original subdirectory structure under the destination directory. A folder key without a destination downloads into a local directory named after its last segment (`omicslab download snapshots` → `./snapshots/`). An exact object always wins over a same-named folder prefix, and a key that matches neither reports "File or folder not found" instead of a provider 403/404.
|
|
128
136
|
|
|
129
137
|
## Environment Variables
|
|
130
138
|
|
|
@@ -132,6 +140,7 @@ omicslab download -y -r s3://my-bucket/data/results/ ./local_results/
|
|
|
132
140
|
|---|---|
|
|
133
141
|
| `OMICSLAB_TOKEN` | API token (overrides config file) |
|
|
134
142
|
| `OMICSLAB_BASE_URL` | API base URL (default: `https://platform.omicslab.io/api`) |
|
|
143
|
+
| `OMICSLAB_CONFIG_DIR` | Directory holding `config.json` (default: `~/.omicslab`); used by studio sessions to read a container-private config without changing `$HOME` |
|
|
135
144
|
|
|
136
145
|
## Jobs API
|
|
137
146
|
|
|
@@ -463,6 +463,49 @@ Studio jobs are unchanged: they run as background sandbox services via
|
|
|
463
463
|
`executor.submit(...)` (spec/bash path), as do jobs dispatched to a runner
|
|
464
464
|
daemon over WebSocket. Only `job run --work-dir` analysis is native/synchronous.
|
|
465
465
|
|
|
466
|
+
Studio session mounts (no token exposure, snapshot-safe):
|
|
467
|
+
|
|
468
|
+
- The SDK environment is bind-mounted **read-only at its own absolute path**
|
|
469
|
+
(pixi/conda envs are path-sensitive), and `omicslab`/`omx` are exposed on
|
|
470
|
+
`PATH` via a dedicated `/opt/omicslab/bin` directory.
|
|
471
|
+
- `pixi` itself is mounted there too, together with the SDK project's
|
|
472
|
+
`pixi.toml`/`pixi.lock`, so `pixi run ...` works inside the session. A
|
|
473
|
+
writable copy of the env's small `conda-meta` directory is mounted over the
|
|
474
|
+
read-only one because pixi takes its install lock there and appends to
|
|
475
|
+
`history`; the copy lives in the job dir and is never snapshotted.
|
|
476
|
+
- Users can create their own project envs in the writable workspace
|
|
477
|
+
(`pixi init` + `pixi add`); those envs are in the writable layer and **are**
|
|
478
|
+
included in snapshots. The pixi package cache goes to a RAM-backed tmpfs
|
|
479
|
+
(`PIXI_CACHE_DIR=/run/omicslab-cache`) so downloads never bloat snapshots.
|
|
480
|
+
- The staged CLI config is mounted from the host into
|
|
481
|
+
`OMICSLAB_CONFIG_DIR=/run/omicslab` before the container starts. The token
|
|
482
|
+
is replaced with a placeholder first; the SDK talks to a host-side
|
|
483
|
+
credential broker over `OMICSLAB_BROKER_SOCKET`, and the broker injects the
|
|
484
|
+
real `Authorization` header upstream. The mount is writable so context
|
|
485
|
+
commands such as `omx cd` can persist, but it contains only the placeholder
|
|
486
|
+
token — the real token never appears in the container env, `docker inspect`,
|
|
487
|
+
the writable layer, or a snapshot.
|
|
488
|
+
Presigned S3 URLs are still fetched directly by the container. Commands
|
|
489
|
+
that bypass `Client` (for example the websocket log stream) are not proxied
|
|
490
|
+
yet.
|
|
491
|
+
- A selected **personal** GitHub credential is injected by a host-side Git
|
|
492
|
+
HTTPS proxy. Git inside the session rewrites `https://github.com/` to the
|
|
493
|
+
proxy URL, so no token or credential helper is present in the container.
|
|
494
|
+
Studio launches reject organization and marketplace credentials. The token
|
|
495
|
+
stays in the agent process and is never logged or returned to the session.
|
|
496
|
+
- When `STUDIO_EXPORT_AWS_CREDENTIALS` is enabled (Settings → Studio), the
|
|
497
|
+
backend mints short-lived STS credentials for the workspace storage and the
|
|
498
|
+
agent starts a host-side S3 signing proxy. The session receives only
|
|
499
|
+
`AWS_ACCESS_KEY_ID=proxy`, `AWS_SECRET_ACCESS_KEY=proxy`, and
|
|
500
|
+
`AWS_ENDPOINT_URL=<host proxy>`; the real STS never enters the container.
|
|
501
|
+
Only credentials carrying a session token are proxied; long-lived provider
|
|
502
|
+
keys are never exported by this path.
|
|
503
|
+
- The old implicit `/home/omicslab` bind is gone (opt-in via
|
|
504
|
+
`runtime.home_mount` only): user files stay in the container's writable layer
|
|
505
|
+
and are therefore included in `docker commit` snapshots.
|
|
506
|
+
- Bind mounts and tmpfs are never committed, so neither the SDK nor the token
|
|
507
|
+
can leak into a snapshot.
|
|
508
|
+
|
|
466
509
|
The SDK bootstrap URL is configurable:
|
|
467
510
|
|
|
468
511
|
- Env var `SDK_TOOLS_BASE` (backend `settings.py`), default
|
|
@@ -66,7 +66,7 @@ omicslab upload -y ./data.csv s3://my-bucket/data/
|
|
|
66
66
|
|
|
67
67
|
### Download
|
|
68
68
|
|
|
69
|
-
Download files or directories from workspace storage.
|
|
69
|
+
Download files or directories from workspace storage. A folder is detected automatically, so a trailing slash and `-r` are optional; recursive downloads preserve subdirectory structure and use parallel threads.
|
|
70
70
|
|
|
71
71
|
```bash
|
|
72
72
|
# Single file (defaults to filename in current directory)
|
|
@@ -78,7 +78,13 @@ omicslab download results/run1/data.csv ./local_copy.csv
|
|
|
78
78
|
# Download from s3:// path (auto-resolves workspace)
|
|
79
79
|
omicslab download s3://my-bucket/data/results/report.html
|
|
80
80
|
|
|
81
|
-
#
|
|
81
|
+
# Folder download (auto-detected; defaults to ./run1)
|
|
82
|
+
omicslab download results/run1
|
|
83
|
+
|
|
84
|
+
# Folder download with explicit destination
|
|
85
|
+
omicslab download results/run1 ./local_results/
|
|
86
|
+
|
|
87
|
+
# Same thing, explicit recursive (preserves subdirectory structure)
|
|
82
88
|
omicslab download -r s3://my-bucket/data/results/ ./local_results/
|
|
83
89
|
|
|
84
90
|
# Recursive download with custom thread count
|
|
@@ -92,12 +98,13 @@ omicslab download -y -r s3://my-bucket/data/results/ ./local_results/
|
|
|
92
98
|
|
|
93
99
|
| Flag | Description |
|
|
94
100
|
|------|-------------|
|
|
95
|
-
| `--recursive`, `-r` |
|
|
101
|
+
| `--recursive`, `-r` | Force recursive download (folders are detected automatically) |
|
|
102
|
+
| `--zip` | Download a folder as a server-built zip archive |
|
|
96
103
|
| `--threads` | Concurrent download threads (default: 4) |
|
|
97
104
|
| `--workspace-id`, `-w` | Workspace ID (inferred from s3:// path if provided) |
|
|
98
105
|
| `--yes`, `-y` | Skip confirmation prompt for s3:// path resolution |
|
|
99
106
|
|
|
100
|
-
**Recursive download** lists all objects under the prefix, then downloads them in parallel while preserving the original subdirectory structure under the destination directory.
|
|
107
|
+
**Recursive download** lists all objects under the prefix, then downloads them in parallel while preserving the original subdirectory structure under the destination directory. A folder key without a destination downloads into a local directory named after its last segment (`omicslab download snapshots` → `./snapshots/`). An exact object always wins over a same-named folder prefix, and a key that matches neither reports "File or folder not found" instead of a provider 403/404.
|
|
101
108
|
|
|
102
109
|
## Environment Variables
|
|
103
110
|
|
|
@@ -105,6 +112,7 @@ omicslab download -y -r s3://my-bucket/data/results/ ./local_results/
|
|
|
105
112
|
|---|---|
|
|
106
113
|
| `OMICSLAB_TOKEN` | API token (overrides config file) |
|
|
107
114
|
| `OMICSLAB_BASE_URL` | API base URL (default: `https://platform.omicslab.io/api`) |
|
|
115
|
+
| `OMICSLAB_CONFIG_DIR` | Directory holding `config.json` (default: `~/.omicslab`); used by studio sessions to read a container-private config without changing `$HOME` |
|
|
108
116
|
|
|
109
117
|
## Jobs API
|
|
110
118
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "2.5.1"
|
|
@@ -563,6 +563,11 @@ def upload(
|
|
|
563
563
|
error(f"Cannot use --recursive with a file: {source}")
|
|
564
564
|
raise typer.Exit(1)
|
|
565
565
|
|
|
566
|
+
# Studio/sandbox sessions export the job id; use it when the caller did
|
|
567
|
+
# not pass --job-id so uploads are attributed to the running job.
|
|
568
|
+
if job_id is None:
|
|
569
|
+
job_id = os.environ.get("OMICSLAB_JOB_ID") or os.environ.get("JOB_ID") or None
|
|
570
|
+
|
|
566
571
|
config = Config()
|
|
567
572
|
client = get_client()
|
|
568
573
|
|
|
@@ -819,19 +824,31 @@ def upload(
|
|
|
819
824
|
_BATCH_SIZE = 500
|
|
820
825
|
|
|
821
826
|
def _get_upload_urls_batch(batch: list[tuple[str, str]]) -> dict[str, str]:
|
|
822
|
-
"""Get presigned URLs for a batch of files.
|
|
827
|
+
"""Get presigned URLs for a batch of files.
|
|
828
|
+
|
|
829
|
+
Keys are sent relative to the folder ``prefix`` so the upload is
|
|
830
|
+
audited against the folder, with the files listed as affected keys.
|
|
831
|
+
Presigned responses echo those relative keys, so map them back to the
|
|
832
|
+
full remote key used by the uploader.
|
|
833
|
+
"""
|
|
823
834
|
files_payload = []
|
|
824
835
|
for local_path, remote_key in batch:
|
|
825
836
|
try:
|
|
826
837
|
sz = os.path.getsize(local_path)
|
|
827
838
|
except OSError:
|
|
828
839
|
sz = 0
|
|
829
|
-
files_payload.append(
|
|
840
|
+
files_payload.append(
|
|
841
|
+
{"key": remote_key.removeprefix(remote_prefix), "size": sz}
|
|
842
|
+
)
|
|
830
843
|
try:
|
|
831
|
-
result = client.storage.batch_upload(
|
|
844
|
+
result = client.storage.batch_upload(
|
|
845
|
+
ws, remote_prefix, files_payload, job_id=job_id
|
|
846
|
+
)
|
|
832
847
|
url_list = result.get("urls", []) if isinstance(result, dict) else result
|
|
833
848
|
return {
|
|
834
|
-
entry[
|
|
849
|
+
f"{remote_prefix}{entry['key']}": entry["url"]
|
|
850
|
+
for entry in url_list
|
|
851
|
+
if entry.get("url")
|
|
835
852
|
}
|
|
836
853
|
except Exception:
|
|
837
854
|
return {}
|
|
@@ -966,10 +983,15 @@ def _download_folder_as_zip(
|
|
|
966
983
|
@app.command()
|
|
967
984
|
def download(
|
|
968
985
|
key: str = typer.Argument(
|
|
969
|
-
...,
|
|
986
|
+
...,
|
|
987
|
+
help=(
|
|
988
|
+
"Remote file or folder key to download, "
|
|
989
|
+
"or s3://bucket/key or s3://bucket/dir/"
|
|
990
|
+
),
|
|
970
991
|
),
|
|
971
992
|
destination: str | None = typer.Argument(
|
|
972
|
-
None,
|
|
993
|
+
None,
|
|
994
|
+
help="Local path (defaults to the filename, or the folder name for a folder)",
|
|
973
995
|
),
|
|
974
996
|
workspace_id: str | None = typer.Option(
|
|
975
997
|
None,
|
|
@@ -981,7 +1003,7 @@ def download(
|
|
|
981
1003
|
False,
|
|
982
1004
|
"--recursive",
|
|
983
1005
|
"-r",
|
|
984
|
-
help="
|
|
1006
|
+
help="Force recursive download (folders are detected automatically)",
|
|
985
1007
|
),
|
|
986
1008
|
zip_archive: bool = typer.Option(
|
|
987
1009
|
False,
|
|
@@ -998,7 +1020,11 @@ def download(
|
|
|
998
1020
|
"""Download a file or directory from workspace storage.
|
|
999
1021
|
|
|
1000
1022
|
Accepts an s3://bucket/key to resolve the workspace automatically.
|
|
1001
|
-
|
|
1023
|
+
|
|
1024
|
+
Folders are detected automatically: ``omicslab download snapshots`` and
|
|
1025
|
+
``omicslab download snapshots/ ./snapshots -r`` do the same thing. A
|
|
1026
|
+
folder downloads into a local directory named after its last key segment
|
|
1027
|
+
unless a destination is given.
|
|
1002
1028
|
"""
|
|
1003
1029
|
config = Config()
|
|
1004
1030
|
client = get_client()
|
|
@@ -1012,8 +1038,6 @@ def download(
|
|
|
1012
1038
|
if s3_info:
|
|
1013
1039
|
ws = s3_info["workspace_id"]
|
|
1014
1040
|
full_key = s3_info["key_prefix"]
|
|
1015
|
-
if recursive:
|
|
1016
|
-
full_key = full_key.rstrip("/") + "/" if full_key else ""
|
|
1017
1041
|
print_panel(
|
|
1018
1042
|
"S3 Path Resolution",
|
|
1019
1043
|
{
|
|
@@ -1029,6 +1053,45 @@ def download(
|
|
|
1029
1053
|
if prefix:
|
|
1030
1054
|
full_key = prefix + key
|
|
1031
1055
|
|
|
1056
|
+
# A trailing slash always means a folder. Without one, ask the listing
|
|
1057
|
+
# whether the key is an object or a folder instead of blindly presigning
|
|
1058
|
+
# it: a folder key would fail with a provider 403/404 and look like a
|
|
1059
|
+
# broken URL.
|
|
1060
|
+
folder = bool(full_key) and full_key.endswith("/")
|
|
1061
|
+
if full_key and not folder:
|
|
1062
|
+
try:
|
|
1063
|
+
listing = client.storage.list(ws, prefix=full_key)
|
|
1064
|
+
except Exception as e:
|
|
1065
|
+
error(f"Failed to inspect {full_key!r}: {e}")
|
|
1066
|
+
raise typer.Exit(1)
|
|
1067
|
+
objects = (
|
|
1068
|
+
listing
|
|
1069
|
+
if isinstance(listing, list)
|
|
1070
|
+
else listing.get("objects", [])
|
|
1071
|
+
if isinstance(listing, dict)
|
|
1072
|
+
else None
|
|
1073
|
+
)
|
|
1074
|
+
# ``objects is None`` only happens with an unexpected client shape;
|
|
1075
|
+
# fall back to the object download rather than guessing.
|
|
1076
|
+
if objects is not None:
|
|
1077
|
+
if any(obj.get("is_file") and not obj.get("name") for obj in objects):
|
|
1078
|
+
folder = False
|
|
1079
|
+
elif any(
|
|
1080
|
+
not obj.get("is_file") and obj.get("name") == "/" for obj in objects
|
|
1081
|
+
):
|
|
1082
|
+
folder = True
|
|
1083
|
+
else:
|
|
1084
|
+
error(f"File or folder not found: {full_key}")
|
|
1085
|
+
raise typer.Exit(1)
|
|
1086
|
+
|
|
1087
|
+
if zip_archive and not folder:
|
|
1088
|
+
error("--zip requires a folder key.")
|
|
1089
|
+
raise typer.Exit(1)
|
|
1090
|
+
|
|
1091
|
+
if folder:
|
|
1092
|
+
full_key = full_key.rstrip("/") + "/"
|
|
1093
|
+
recursive = True
|
|
1094
|
+
|
|
1032
1095
|
# Recursive directory download
|
|
1033
1096
|
if recursive and (full_key.endswith("/") or not full_key):
|
|
1034
1097
|
if zip_archive:
|
|
@@ -1076,7 +1139,11 @@ def download(
|
|
|
1076
1139
|
info("No files found in this path.")
|
|
1077
1140
|
return
|
|
1078
1141
|
|
|
1079
|
-
|
|
1142
|
+
# A folder downloads into a directory named after its last key
|
|
1143
|
+
# segment, so `download snapshots` is equivalent to
|
|
1144
|
+
# `download snapshots/ ./snapshots -r`.
|
|
1145
|
+
folder_name = full_key.rstrip("/").rsplit("/", 1)[-1]
|
|
1146
|
+
dest_dir = Path(destination) if destination else Path(folder_name or ".")
|
|
1080
1147
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
1081
1148
|
|
|
1082
1149
|
from rich.progress import (
|
|
@@ -46,6 +46,16 @@ class Client:
|
|
|
46
46
|
).rstrip("/")
|
|
47
47
|
self._base_url = resolved_base_url
|
|
48
48
|
|
|
49
|
+
# Inside a studio session the real token stays in the host-side
|
|
50
|
+
# credential broker. The session config contains a placeholder token
|
|
51
|
+
# and this socket; every API call goes through the broker, which
|
|
52
|
+
# injects the real Authorization header upstream.
|
|
53
|
+
broker_socket = os.environ.get("OMICSLAB_BROKER_SOCKET")
|
|
54
|
+
if broker_socket and transport is None:
|
|
55
|
+
resolved_base_url = "http://broker"
|
|
56
|
+
self._base_url = resolved_base_url
|
|
57
|
+
transport = httpx.HTTPTransport(uds=broker_socket)
|
|
58
|
+
|
|
49
59
|
if token:
|
|
50
60
|
self._config.set_token(token)
|
|
51
61
|
if base_url:
|
|
@@ -9,11 +9,26 @@ class Config:
|
|
|
9
9
|
DEFAULT_PROFILE = "default"
|
|
10
10
|
|
|
11
11
|
def __init__(self, config_path: Path | None = None, profile: str | None = None):
|
|
12
|
-
self._config_path = config_path or self.
|
|
12
|
+
self._config_path = config_path or self._resolve_config_path()
|
|
13
13
|
self._profile_name = profile or os.environ.get("OMICSLAB_PROFILE")
|
|
14
14
|
self._data: dict = {}
|
|
15
15
|
self._load()
|
|
16
16
|
|
|
17
|
+
@classmethod
|
|
18
|
+
def _resolve_config_path(cls) -> Path:
|
|
19
|
+
"""Resolve the config file path for a new instance.
|
|
20
|
+
|
|
21
|
+
``OMICSLAB_CONFIG_DIR`` lets a caller point the CLI at a dedicated
|
|
22
|
+
config directory (e.g. a container-private mount) without changing
|
|
23
|
+
``$HOME`` — the token is then read from that directory while the
|
|
24
|
+
user's real profile stays untouched. ``DEFAULT_CONFIG_FILE`` is kept
|
|
25
|
+
as the fallback (and as the monkeypatch hook used by tests).
|
|
26
|
+
"""
|
|
27
|
+
config_dir = os.environ.get("OMICSLAB_CONFIG_DIR")
|
|
28
|
+
if config_dir:
|
|
29
|
+
return Path(config_dir).expanduser() / "config.json"
|
|
30
|
+
return cls.DEFAULT_CONFIG_FILE
|
|
31
|
+
|
|
17
32
|
def _load(self):
|
|
18
33
|
if self._config_path.exists():
|
|
19
34
|
try:
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""Host-side credential broker for studio sessions.
|
|
2
|
+
|
|
3
|
+
The session container must be able to call the platform API (``omicslab
|
|
4
|
+
download``/``upload`` and the SDK resources) without ever receiving the real
|
|
5
|
+
job token. This module runs a tiny HTTP proxy on a Unix socket in the agent
|
|
6
|
+
process, injects the real ``Authorization`` header, and forwards the request
|
|
7
|
+
to the API. The container's config contains only a placeholder token.
|
|
8
|
+
|
|
9
|
+
Security properties:
|
|
10
|
+
|
|
11
|
+
- The real token never leaves the host process. It is not in the container's
|
|
12
|
+
environment, argv, files, tmpfs, or snapshot.
|
|
13
|
+
- The upstream base URL is fixed at construction time; request paths are
|
|
14
|
+
appended to it, so the container cannot turn the broker into an open proxy
|
|
15
|
+
(absolute and scheme-relative URLs are rejected).
|
|
16
|
+
- Redirects are not followed; the response is streamed back as-is.
|
|
17
|
+
- The socket is only mounted into the session container. Its host directory
|
|
18
|
+
is private to the agent (``mkdtemp``).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import asyncio
|
|
24
|
+
import os
|
|
25
|
+
import threading
|
|
26
|
+
from collections.abc import Callable
|
|
27
|
+
|
|
28
|
+
from aiohttp import ClientError, ClientSession, ClientTimeout, web
|
|
29
|
+
|
|
30
|
+
# Hop-by-hop headers must not be forwarded (RFC 7230 §6.1).
|
|
31
|
+
_HOP_BY_HOP = frozenset(
|
|
32
|
+
{
|
|
33
|
+
"connection",
|
|
34
|
+
"keep-alive",
|
|
35
|
+
"proxy-authenticate",
|
|
36
|
+
"proxy-authorization",
|
|
37
|
+
"te",
|
|
38
|
+
"trailer",
|
|
39
|
+
"trailers",
|
|
40
|
+
"transfer-encoding",
|
|
41
|
+
"upgrade",
|
|
42
|
+
}
|
|
43
|
+
)
|
|
44
|
+
_SKIP_REQUEST = _HOP_BY_HOP | {
|
|
45
|
+
"host",
|
|
46
|
+
"authorization",
|
|
47
|
+
# The backend trusts forwarding headers from its private-network peers;
|
|
48
|
+
# the broker is such a peer, so a session must not be able to spoof its
|
|
49
|
+
# audit IP through them.
|
|
50
|
+
"forwarded",
|
|
51
|
+
"x-forwarded-for",
|
|
52
|
+
"x-forwarded-host",
|
|
53
|
+
"x-forwarded-proto",
|
|
54
|
+
"x-real-ip",
|
|
55
|
+
}
|
|
56
|
+
_SKIP_RESPONSE = _HOP_BY_HOP | {"content-length"}
|
|
57
|
+
|
|
58
|
+
_CHUNK_SIZE = 64 * 1024
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class CredentialBroker:
|
|
62
|
+
"""Inject the real API token into requests made by the session container.
|
|
63
|
+
|
|
64
|
+
``start()`` binds a Unix socket and blocks until it is accepting requests;
|
|
65
|
+
``stop()`` shuts the listener down and removes the socket. The broker is
|
|
66
|
+
designed to live for one studio session.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
def __init__(
|
|
70
|
+
self,
|
|
71
|
+
upstream_base: str,
|
|
72
|
+
token: str,
|
|
73
|
+
socket_path: str,
|
|
74
|
+
log: Callable[[str], None] | None = None,
|
|
75
|
+
):
|
|
76
|
+
self._upstream_base = upstream_base.rstrip("/")
|
|
77
|
+
self._token = token
|
|
78
|
+
self._socket_path = socket_path
|
|
79
|
+
self._log = log or (lambda _message: None)
|
|
80
|
+
self._ready = threading.Event()
|
|
81
|
+
self._error: BaseException | None = None
|
|
82
|
+
self._loop: asyncio.AbstractEventLoop | None = None
|
|
83
|
+
self._stopped: asyncio.Event | None = None
|
|
84
|
+
self._session: ClientSession | None = None
|
|
85
|
+
self._thread: threading.Thread | None = None
|
|
86
|
+
self._started = False
|
|
87
|
+
|
|
88
|
+
# ── lifecycle ──────────────────────────────────────────────────────
|
|
89
|
+
|
|
90
|
+
def start(self, timeout: float = 10.0) -> None:
|
|
91
|
+
"""Start the broker and wait until the socket is accepting requests."""
|
|
92
|
+
if self._thread is not None:
|
|
93
|
+
return
|
|
94
|
+
parent = os.path.dirname(self._socket_path)
|
|
95
|
+
if parent:
|
|
96
|
+
os.makedirs(parent, exist_ok=True)
|
|
97
|
+
try:
|
|
98
|
+
os.unlink(self._socket_path)
|
|
99
|
+
except FileNotFoundError:
|
|
100
|
+
pass
|
|
101
|
+
|
|
102
|
+
self._thread = threading.Thread(
|
|
103
|
+
target=self._run,
|
|
104
|
+
name="omicslab-credential-broker",
|
|
105
|
+
daemon=True,
|
|
106
|
+
)
|
|
107
|
+
self._thread.start()
|
|
108
|
+
if not self._ready.wait(timeout):
|
|
109
|
+
self.stop()
|
|
110
|
+
raise RuntimeError("credential broker did not start in time")
|
|
111
|
+
if self._error is not None:
|
|
112
|
+
self.stop()
|
|
113
|
+
raise RuntimeError(f"credential broker failed to start: {self._error}")
|
|
114
|
+
self._started = True
|
|
115
|
+
self._log(f"[BROKER] in-session API broker listening on {self._socket_path}")
|
|
116
|
+
|
|
117
|
+
def stop(self, timeout: float = 5.0) -> None:
|
|
118
|
+
"""Stop the listener and remove the Unix socket."""
|
|
119
|
+
if self._loop is not None and self._stopped is not None:
|
|
120
|
+
try:
|
|
121
|
+
self._loop.call_soon_threadsafe(self._stopped.set)
|
|
122
|
+
except RuntimeError:
|
|
123
|
+
# The loop already closed (e.g. startup failed).
|
|
124
|
+
pass
|
|
125
|
+
if self._thread is not None:
|
|
126
|
+
self._thread.join(timeout=timeout)
|
|
127
|
+
self._thread = None
|
|
128
|
+
try:
|
|
129
|
+
os.unlink(self._socket_path)
|
|
130
|
+
except FileNotFoundError:
|
|
131
|
+
pass
|
|
132
|
+
if self._started:
|
|
133
|
+
self._started = False
|
|
134
|
+
self._log("[BROKER] in-session API broker stopped")
|
|
135
|
+
|
|
136
|
+
def _run(self) -> None:
|
|
137
|
+
try:
|
|
138
|
+
asyncio.run(self._serve())
|
|
139
|
+
except BaseException as exc: # noqa: BLE001 - reported by start()
|
|
140
|
+
self._error = exc
|
|
141
|
+
self._ready.set()
|
|
142
|
+
|
|
143
|
+
async def _serve(self) -> None:
|
|
144
|
+
self._loop = asyncio.get_running_loop()
|
|
145
|
+
self._stopped = asyncio.Event()
|
|
146
|
+
self._session = ClientSession(
|
|
147
|
+
auto_decompress=False,
|
|
148
|
+
trust_env=False,
|
|
149
|
+
timeout=ClientTimeout(total=None, connect=30, sock_read=600),
|
|
150
|
+
)
|
|
151
|
+
app = web.Application()
|
|
152
|
+
app.router.add_route("*", "/{tail:.*}", self._handle)
|
|
153
|
+
runner = web.AppRunner(app, access_log=None)
|
|
154
|
+
await runner.setup()
|
|
155
|
+
try:
|
|
156
|
+
site = web.UnixSite(runner, self._socket_path)
|
|
157
|
+
await site.start()
|
|
158
|
+
try:
|
|
159
|
+
os.chmod(self._socket_path, 0o666)
|
|
160
|
+
except OSError:
|
|
161
|
+
pass
|
|
162
|
+
self._ready.set()
|
|
163
|
+
await self._stopped.wait()
|
|
164
|
+
finally:
|
|
165
|
+
await self._session.close()
|
|
166
|
+
await runner.cleanup()
|
|
167
|
+
|
|
168
|
+
# ── proxy ──────────────────────────────────────────────────────────
|
|
169
|
+
|
|
170
|
+
def _upstream_url(self, path: str, query: str) -> str:
|
|
171
|
+
"""Build the fixed-upstream URL for a request path.
|
|
172
|
+
|
|
173
|
+
Only relative paths are accepted. ``//host`` and absolute URLs would
|
|
174
|
+
let a session redirect the broker (and the token) elsewhere.
|
|
175
|
+
"""
|
|
176
|
+
if not path.startswith("/") or path.startswith("//") or "://" in path:
|
|
177
|
+
raise ValueError("invalid request path")
|
|
178
|
+
url = f"{self._upstream_base}{path}"
|
|
179
|
+
return f"{url}?{query}" if query else url
|
|
180
|
+
|
|
181
|
+
async def _handle(self, request: web.Request) -> web.StreamResponse:
|
|
182
|
+
try:
|
|
183
|
+
url = self._upstream_url(request.path, request.query_string)
|
|
184
|
+
except ValueError:
|
|
185
|
+
return web.json_response({"message": "invalid request path"}, status=400)
|
|
186
|
+
|
|
187
|
+
headers = {
|
|
188
|
+
key: value
|
|
189
|
+
for key, value in request.headers.items()
|
|
190
|
+
if key.lower() not in _SKIP_REQUEST
|
|
191
|
+
}
|
|
192
|
+
# Always replace any client-supplied credential.
|
|
193
|
+
headers["Authorization"] = f"Bearer {self._token}"
|
|
194
|
+
|
|
195
|
+
data = self._body(request)
|
|
196
|
+
session = self._session
|
|
197
|
+
if session is None:
|
|
198
|
+
return web.json_response({"message": "broker not ready"}, status=503)
|
|
199
|
+
|
|
200
|
+
try:
|
|
201
|
+
async with session.request(
|
|
202
|
+
request.method,
|
|
203
|
+
url,
|
|
204
|
+
headers=headers,
|
|
205
|
+
data=data,
|
|
206
|
+
allow_redirects=False,
|
|
207
|
+
) as upstream:
|
|
208
|
+
response = web.StreamResponse(status=upstream.status)
|
|
209
|
+
for key, value in upstream.headers.items():
|
|
210
|
+
if key.lower() not in _SKIP_RESPONSE:
|
|
211
|
+
response.headers.add(key, value)
|
|
212
|
+
if request.method == "HEAD" or upstream.status in (204, 304):
|
|
213
|
+
await response.prepare(request)
|
|
214
|
+
await response.write_eof()
|
|
215
|
+
return response
|
|
216
|
+
await response.prepare(request)
|
|
217
|
+
async for chunk in upstream.content.iter_chunked(_CHUNK_SIZE):
|
|
218
|
+
await response.write(chunk)
|
|
219
|
+
await response.write_eof()
|
|
220
|
+
return response
|
|
221
|
+
except asyncio.TimeoutError:
|
|
222
|
+
return web.json_response({"message": "upstream timeout"}, status=504)
|
|
223
|
+
except ClientError:
|
|
224
|
+
return web.json_response({"message": "upstream unavailable"}, status=502)
|
|
225
|
+
|
|
226
|
+
def _body(self, request: web.Request):
|
|
227
|
+
"""Stream a request body without buffering it in memory."""
|
|
228
|
+
if not request.can_read_body:
|
|
229
|
+
return None
|
|
230
|
+
|
|
231
|
+
async def _iter():
|
|
232
|
+
async for chunk in request.content.iter_chunked(_CHUNK_SIZE):
|
|
233
|
+
yield chunk
|
|
234
|
+
|
|
235
|
+
return _iter()
|