omicslab 2.2.0__tar.gz → 2.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. {omicslab-2.2.0 → omicslab-2.5.1}/PKG-INFO +14 -5
  2. {omicslab-2.2.0 → omicslab-2.5.1}/README.developer.md +43 -0
  3. {omicslab-2.2.0 → omicslab-2.5.1}/README.md +12 -4
  4. {omicslab-2.2.0 → omicslab-2.5.1}/pyproject.toml +1 -0
  5. omicslab-2.5.1/src/omicslab/_version.py +1 -0
  6. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/main.py +78 -11
  7. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/client.py +10 -0
  8. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/config.py +16 -1
  9. omicslab-2.5.1/src/omicslab/job/_run/_broker.py +235 -0
  10. omicslab-2.5.1/src/omicslab/job/_run/_git_proxy.py +196 -0
  11. omicslab-2.5.1/src/omicslab/job/_run/_s3_proxy.py +242 -0
  12. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_storage.py +19 -5
  13. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/analysis.py +6 -0
  14. omicslab-2.5.1/src/omicslab/job/_run/studio.py +1189 -0
  15. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/storage.py +16 -9
  16. {omicslab-2.2.0 → omicslab-2.5.1}/tests/conftest.py +2 -1
  17. omicslab-2.5.1/tests/job/_run/test__broker.py +146 -0
  18. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__engine_helpers.py +26 -2
  19. omicslab-2.5.1/tests/job/_run/test__git_proxy.py +105 -0
  20. omicslab-2.5.1/tests/job/_run/test__s3_proxy.py +175 -0
  21. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__storage.py +88 -0
  22. omicslab-2.5.1/tests/job/_run/test__studio_docker.py +349 -0
  23. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_cli.py +2 -0
  24. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_cli_storage.py +170 -0
  25. omicslab-2.5.1/tests/test_client_broker.py +51 -0
  26. omicslab-2.5.1/tests/test_config.py +20 -0
  27. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_resources.py +6 -2
  28. omicslab-2.2.0/src/omicslab/_version.py +0 -1
  29. omicslab-2.2.0/src/omicslab/job/_run/studio.py +0 -613
  30. omicslab-2.2.0/tests/job/_run/test__studio_docker.py +0 -107
  31. {omicslab-2.2.0 → omicslab-2.5.1}/.gitignore +0 -0
  32. {omicslab-2.2.0 → omicslab-2.5.1}/Makefile +0 -0
  33. {omicslab-2.2.0 → omicslab-2.5.1}/envs/.gitignore +0 -0
  34. {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/hcm.pixi.lock +0 -0
  35. {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/hcm.pixi.toml +0 -0
  36. {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/pixi.lock +0 -0
  37. {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/pixi.toml +0 -0
  38. {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/us-east-1.pixi.lock +0 -0
  39. {omicslab-2.2.0 → omicslab-2.5.1}/envs/dev/us-east-1.pixi.toml +0 -0
  40. {omicslab-2.2.0 → omicslab-2.5.1}/envs/pixi.toml +0 -0
  41. {omicslab-2.2.0 → omicslab-2.5.1}/envs/prod/hcm.pixi.lock +0 -0
  42. {omicslab-2.2.0 → omicslab-2.5.1}/envs/prod/hcm.pixi.toml +0 -0
  43. {omicslab-2.2.0 → omicslab-2.5.1}/envs/prod/pixi.lock +0 -0
  44. {omicslab-2.2.0 → omicslab-2.5.1}/envs/prod/pixi.toml +0 -0
  45. {omicslab-2.2.0 → omicslab-2.5.1}/envs/regions.json +0 -0
  46. {omicslab-2.2.0 → omicslab-2.5.1}/pixi.config.toml +0 -0
  47. {omicslab-2.2.0 → omicslab-2.5.1}/scripts/bootstrap-env.sh +0 -0
  48. {omicslab-2.2.0 → omicslab-2.5.1}/scripts/check_publish.py +0 -0
  49. {omicslab-2.2.0 → omicslab-2.5.1}/scripts/write_version_manifest.py +0 -0
  50. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/__init__.py +0 -0
  51. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/__init__.py +0 -0
  52. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/analysis_commands.py +0 -0
  53. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/auth_commands.py +0 -0
  54. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/context_commands.py +0 -0
  55. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/credential_commands.py +0 -0
  56. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/fuse_commands.py +0 -0
  57. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/job_commands.py +0 -0
  58. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/org_commands.py +0 -0
  59. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/runner_commands.py +0 -0
  60. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/setup_commands.py +0 -0
  61. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/studio_commands.py +0 -0
  62. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/token_commands.py +0 -0
  63. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/utils.py +0 -0
  64. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/workspace_commands.py +0 -0
  65. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/cli/ws_commands.py +0 -0
  66. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/exceptions.py +0 -0
  67. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/__init__.py +0 -0
  68. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/__init__.py +0 -0
  69. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_auth.py +0 -0
  70. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_config_download.py +0 -0
  71. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_consts.py +0 -0
  72. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_data_prep.py +0 -0
  73. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_env.py +0 -0
  74. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_exec.py +0 -0
  75. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_git.py +0 -0
  76. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_images.py +0 -0
  77. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_log.py +0 -0
  78. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_monitor.py +0 -0
  79. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_nextflow.py +0 -0
  80. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_params.py +0 -0
  81. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_proc.py +0 -0
  82. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_proxy.py +0 -0
  83. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_spec.py +0 -0
  84. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/_run/_studio_prep.py +0 -0
  85. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/client.py +0 -0
  86. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/console.py +0 -0
  87. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/executor.py +0 -0
  88. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/paths.py +0 -0
  89. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/job/system.py +0 -0
  90. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/__init__.py +0 -0
  91. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/analysis.py +0 -0
  92. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/base.py +0 -0
  93. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/credentials.py +0 -0
  94. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/jobs.py +0 -0
  95. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/orgs.py +0 -0
  96. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/runners.py +0 -0
  97. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/studio.py +0 -0
  98. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/tokens.py +0 -0
  99. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/resources/workspaces.py +0 -0
  100. {omicslab-2.2.0 → omicslab-2.5.1}/src/omicslab/selfupdate.py +0 -0
  101. {omicslab-2.2.0 → omicslab-2.5.1}/tests/__init__.py +0 -0
  102. {omicslab-2.2.0 → omicslab-2.5.1}/tests/e2e/__init__.py +0 -0
  103. {omicslab-2.2.0 → omicslab-2.5.1}/tests/e2e/conftest.py +0 -0
  104. {omicslab-2.2.0 → omicslab-2.5.1}/tests/e2e/test_cli_e2e.py +0 -0
  105. {omicslab-2.2.0 → omicslab-2.5.1}/tests/e2e/test_tokens.py +0 -0
  106. {omicslab-2.2.0 → omicslab-2.5.1}/tests/fixtures/nfcore-demo-custom.config +0 -0
  107. {omicslab-2.2.0 → omicslab-2.5.1}/tests/fixtures/studio_specs/jupyter.json +0 -0
  108. {omicslab-2.2.0 → omicslab-2.5.1}/tests/fixtures/studio_specs/rstudio.json +0 -0
  109. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__auth.py +0 -0
  110. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__data_prep.py +0 -0
  111. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__env.py +0 -0
  112. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__get_images_cli.py +0 -0
  113. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__git.py +0 -0
  114. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__images.py +0 -0
  115. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__log.py +0 -0
  116. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__monitor.py +0 -0
  117. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__nextflow.py +0 -0
  118. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__params.py +0 -0
  119. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__proxy.py +0 -0
  120. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__proxy_extra.py +0 -0
  121. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test__spec.py +0 -0
  122. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/_run/test_studio_proxy_start.py +0 -0
  123. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_cli.py +0 -0
  124. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_client.py +0 -0
  125. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_client_extra.py +0 -0
  126. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_executor.py +0 -0
  127. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_executor_extra.py +0 -0
  128. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_fuse.py +0 -0
  129. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_resource.py +0 -0
  130. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_job_system.py +0 -0
  131. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_snapshot_request.py +0 -0
  132. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_studio_image_download.py +0 -0
  133. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_studio_validate_cli.py +0 -0
  134. {omicslab-2.2.0 → omicslab-2.5.1}/tests/job/test_ws_helpers.py +0 -0
  135. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_cli_catalog.py +0 -0
  136. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_cli_context.py +0 -0
  137. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_cli_utils.py +0 -0
  138. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_client.py +0 -0
  139. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_fuse.py +0 -0
  140. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_job_exec.py +0 -0
  141. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_job_heartbeat.py +0 -0
  142. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_misc.py +0 -0
  143. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_runner_helpers.py +0 -0
  144. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_runner_self_update.py +0 -0
  145. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_sandbox_report.py +0 -0
  146. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_selfupdate.py +0 -0
  147. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_studio_snapshot.py +0 -0
  148. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_system_monitor_interval.py +0 -0
  149. {omicslab-2.2.0 → omicslab-2.5.1}/tests/test_ws_commands.py +0 -0
  150. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-nextflow/input.txt +0 -0
  151. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-nextflow/main.nf +0 -0
  152. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-nextflow/omicslab/main.sh +0 -0
  153. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-nextflow/pixi.toml +0 -0
  154. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-nextflow/schema.json +0 -0
  155. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-simple/omicslab/main.sh +0 -0
  156. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-simple/pixi.toml +0 -0
  157. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-simple/schema.json +0 -0
  158. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-snakemake/Snakefile +0 -0
  159. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-snakemake/input.txt +0 -0
  160. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-snakemake/omicslab/main.sh +0 -0
  161. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-snakemake/pixi.toml +0 -0
  162. {omicslab-2.2.0 → omicslab-2.5.1}/tests/tools/hello-snakemake/schema.json +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: omicslab
3
- Version: 2.2.0
3
+ Version: 2.5.1
4
4
  Summary: Python SDK and CLI for the OmicsLab Platform
5
5
  Project-URL: Homepage, https://platform.omicslab.io
6
6
  Project-URL: Repository, https://github.com/vieomics/omicslab-platform
@@ -14,6 +14,7 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
14
14
  Classifier: Programming Language :: Python :: 3.12
15
15
  Requires-Python: >=3.12
16
16
  Requires-Dist: aiohttp>=3.10
17
+ Requires-Dist: botocore<2,>=1.35
17
18
  Requires-Dist: httpx>=0.27
18
19
  Requires-Dist: psutil>=6
19
20
  Requires-Dist: pydantic>=2
@@ -93,7 +94,7 @@ omicslab upload -y ./data.csv s3://my-bucket/data/
93
94
 
94
95
  ### Download
95
96
 
96
- Download files or directories from workspace storage. Recursive downloads preserve subdirectory structure and use parallel threads.
97
+ Download files or directories from workspace storage. A folder is detected automatically, so a trailing slash and `-r` are optional; recursive downloads preserve subdirectory structure and use parallel threads.
97
98
 
98
99
  ```bash
99
100
  # Single file (defaults to filename in current directory)
@@ -105,7 +106,13 @@ omicslab download results/run1/data.csv ./local_copy.csv
105
106
  # Download from s3:// path (auto-resolves workspace)
106
107
  omicslab download s3://my-bucket/data/results/report.html
107
108
 
108
- # Recursive directory download (preserves subdirectory structure)
109
+ # Folder download (auto-detected; defaults to ./run1)
110
+ omicslab download results/run1
111
+
112
+ # Folder download with explicit destination
113
+ omicslab download results/run1 ./local_results/
114
+
115
+ # Same thing, explicit recursive (preserves subdirectory structure)
109
116
  omicslab download -r s3://my-bucket/data/results/ ./local_results/
110
117
 
111
118
  # Recursive download with custom thread count
@@ -119,12 +126,13 @@ omicslab download -y -r s3://my-bucket/data/results/ ./local_results/
119
126
 
120
127
  | Flag | Description |
121
128
  |------|-------------|
122
- | `--recursive`, `-r` | Download all files under an s3:// directory prefix |
129
+ | `--recursive`, `-r` | Force recursive download (folders are detected automatically) |
130
+ | `--zip` | Download a folder as a server-built zip archive |
123
131
  | `--threads` | Concurrent download threads (default: 4) |
124
132
  | `--workspace-id`, `-w` | Workspace ID (inferred from s3:// path if provided) |
125
133
  | `--yes`, `-y` | Skip confirmation prompt for s3:// path resolution |
126
134
 
127
- **Recursive download** lists all objects under the prefix, then downloads them in parallel while preserving the original subdirectory structure under the destination directory.
135
+ **Recursive download** lists all objects under the prefix, then downloads them in parallel while preserving the original subdirectory structure under the destination directory. A folder key without a destination downloads into a local directory named after its last segment (`omicslab download snapshots` → `./snapshots/`). An exact object always wins over a same-named folder prefix, and a key that matches neither reports "File or folder not found" instead of a provider 403/404.
128
136
 
129
137
  ## Environment Variables
130
138
 
@@ -132,6 +140,7 @@ omicslab download -y -r s3://my-bucket/data/results/ ./local_results/
132
140
  |---|---|
133
141
  | `OMICSLAB_TOKEN` | API token (overrides config file) |
134
142
  | `OMICSLAB_BASE_URL` | API base URL (default: `https://platform.omicslab.io/api`) |
143
+ | `OMICSLAB_CONFIG_DIR` | Directory holding `config.json` (default: `~/.omicslab`); used by studio sessions to read a container-private config without changing `$HOME` |
135
144
 
136
145
  ## Jobs API
137
146
 
@@ -463,6 +463,49 @@ Studio jobs are unchanged: they run as background sandbox services via
463
463
  `executor.submit(...)` (spec/bash path), as do jobs dispatched to a runner
464
464
  daemon over WebSocket. Only `job run --work-dir` analysis is native/synchronous.
465
465
 
466
+ Studio session mounts (no token exposure, snapshot-safe):
467
+
468
+ - The SDK environment is bind-mounted **read-only at its own absolute path**
469
+ (pixi/conda envs are path-sensitive), and `omicslab`/`omx` are exposed on
470
+ `PATH` via a dedicated `/opt/omicslab/bin` directory.
471
+ - `pixi` itself is mounted there too, together with the SDK project's
472
+ `pixi.toml`/`pixi.lock`, so `pixi run ...` works inside the session. A
473
+ writable copy of the env's small `conda-meta` directory is mounted over the
474
+ read-only one because pixi takes its install lock there and appends to
475
+ `history`; the copy lives in the job dir and is never snapshotted.
476
+ - Users can create their own project envs in the writable workspace
477
+ (`pixi init` + `pixi add`); those envs are in the writable layer and **are**
478
+ included in snapshots. The pixi package cache goes to a RAM-backed tmpfs
479
+ (`PIXI_CACHE_DIR=/run/omicslab-cache`) so downloads never bloat snapshots.
480
+ - The staged CLI config is mounted from the host into
481
+ `OMICSLAB_CONFIG_DIR=/run/omicslab` before the container starts. The token
482
+ is replaced with a placeholder first; the SDK talks to a host-side
483
+ credential broker over `OMICSLAB_BROKER_SOCKET`, and the broker injects the
484
+ real `Authorization` header upstream. The mount is writable so context
485
+ commands such as `omx cd` can persist, but it contains only the placeholder
486
+ token — the real token never appears in the container env, `docker inspect`,
487
+ the writable layer, or a snapshot.
488
+ Presigned S3 URLs are still fetched directly by the container. Commands
489
+ that bypass `Client` (for example the websocket log stream) are not proxied
490
+ yet.
491
+ - A selected **personal** GitHub credential is injected by a host-side Git
492
+ HTTPS proxy. Git inside the session rewrites `https://github.com/` to the
493
+ proxy URL, so no token or credential helper is present in the container.
494
+ Studio launches reject organization and marketplace credentials. The token
495
+ stays in the agent process and is never logged or returned to the session.
496
+ - When `STUDIO_EXPORT_AWS_CREDENTIALS` is enabled (Settings → Studio), the
497
+ backend mints short-lived STS credentials for the workspace storage and the
498
+ agent starts a host-side S3 signing proxy. The session receives only
499
+ `AWS_ACCESS_KEY_ID=proxy`, `AWS_SECRET_ACCESS_KEY=proxy`, and
500
+ `AWS_ENDPOINT_URL=<host proxy>`; the real STS never enters the container.
501
+ Only credentials carrying a session token are proxied; long-lived provider
502
+ keys are never exported by this path.
503
+ - The old implicit `/home/omicslab` bind is gone (opt-in via
504
+ `runtime.home_mount` only): user files stay in the container's writable layer
505
+ and are therefore included in `docker commit` snapshots.
506
+ - Bind mounts and tmpfs are never committed, so neither the SDK nor the token
507
+ can leak into a snapshot.
508
+
466
509
  The SDK bootstrap URL is configurable:
467
510
 
468
511
  - Env var `SDK_TOOLS_BASE` (backend `settings.py`), default
@@ -66,7 +66,7 @@ omicslab upload -y ./data.csv s3://my-bucket/data/
66
66
 
67
67
  ### Download
68
68
 
69
- Download files or directories from workspace storage. Recursive downloads preserve subdirectory structure and use parallel threads.
69
+ Download files or directories from workspace storage. A folder is detected automatically, so a trailing slash and `-r` are optional; recursive downloads preserve subdirectory structure and use parallel threads.
70
70
 
71
71
  ```bash
72
72
  # Single file (defaults to filename in current directory)
@@ -78,7 +78,13 @@ omicslab download results/run1/data.csv ./local_copy.csv
78
78
  # Download from s3:// path (auto-resolves workspace)
79
79
  omicslab download s3://my-bucket/data/results/report.html
80
80
 
81
- # Recursive directory download (preserves subdirectory structure)
81
+ # Folder download (auto-detected; defaults to ./run1)
82
+ omicslab download results/run1
83
+
84
+ # Folder download with explicit destination
85
+ omicslab download results/run1 ./local_results/
86
+
87
+ # Same thing, explicit recursive (preserves subdirectory structure)
82
88
  omicslab download -r s3://my-bucket/data/results/ ./local_results/
83
89
 
84
90
  # Recursive download with custom thread count
@@ -92,12 +98,13 @@ omicslab download -y -r s3://my-bucket/data/results/ ./local_results/
92
98
 
93
99
  | Flag | Description |
94
100
  |------|-------------|
95
- | `--recursive`, `-r` | Download all files under an s3:// directory prefix |
101
+ | `--recursive`, `-r` | Force recursive download (folders are detected automatically) |
102
+ | `--zip` | Download a folder as a server-built zip archive |
96
103
  | `--threads` | Concurrent download threads (default: 4) |
97
104
  | `--workspace-id`, `-w` | Workspace ID (inferred from s3:// path if provided) |
98
105
  | `--yes`, `-y` | Skip confirmation prompt for s3:// path resolution |
99
106
 
100
- **Recursive download** lists all objects under the prefix, then downloads them in parallel while preserving the original subdirectory structure under the destination directory.
107
+ **Recursive download** lists all objects under the prefix, then downloads them in parallel while preserving the original subdirectory structure under the destination directory. A folder key without a destination downloads into a local directory named after its last segment (`omicslab download snapshots` → `./snapshots/`). An exact object always wins over a same-named folder prefix, and a key that matches neither reports "File or folder not found" instead of a provider 403/404.
101
108
 
102
109
  ## Environment Variables
103
110
 
@@ -105,6 +112,7 @@ omicslab download -y -r s3://my-bucket/data/results/ ./local_results/
105
112
  |---|---|
106
113
  | `OMICSLAB_TOKEN` | API token (overrides config file) |
107
114
  | `OMICSLAB_BASE_URL` | API base URL (default: `https://platform.omicslab.io/api`) |
115
+ | `OMICSLAB_CONFIG_DIR` | Directory holding `config.json` (default: `~/.omicslab`); used by studio sessions to read a container-private config without changing `$HOME` |
108
116
 
109
117
  ## Jobs API
110
118
 
@@ -32,6 +32,7 @@ dependencies = [
32
32
  "websockets>=13",
33
33
  "psutil>=6",
34
34
  "aiohttp>=3.10",
35
+ "botocore>=1.35,<2",
35
36
  ]
36
37
 
37
38
  [project.scripts]
@@ -0,0 +1 @@
1
+ __version__ = "2.5.1"
@@ -563,6 +563,11 @@ def upload(
563
563
  error(f"Cannot use --recursive with a file: {source}")
564
564
  raise typer.Exit(1)
565
565
 
566
+ # Studio/sandbox sessions export the job id; use it when the caller did
567
+ # not pass --job-id so uploads are attributed to the running job.
568
+ if job_id is None:
569
+ job_id = os.environ.get("OMICSLAB_JOB_ID") or os.environ.get("JOB_ID") or None
570
+
566
571
  config = Config()
567
572
  client = get_client()
568
573
 
@@ -819,19 +824,31 @@ def upload(
819
824
  _BATCH_SIZE = 500
820
825
 
821
826
  def _get_upload_urls_batch(batch: list[tuple[str, str]]) -> dict[str, str]:
822
- """Get presigned URLs for a batch of files."""
827
+ """Get presigned URLs for a batch of files.
828
+
829
+ Keys are sent relative to the folder ``prefix`` so the upload is
830
+ audited against the folder, with the files listed as affected keys.
831
+ Presigned responses echo those relative keys, so map them back to the
832
+ full remote key used by the uploader.
833
+ """
823
834
  files_payload = []
824
835
  for local_path, remote_key in batch:
825
836
  try:
826
837
  sz = os.path.getsize(local_path)
827
838
  except OSError:
828
839
  sz = 0
829
- files_payload.append({"key": remote_key, "size": sz})
840
+ files_payload.append(
841
+ {"key": remote_key.removeprefix(remote_prefix), "size": sz}
842
+ )
830
843
  try:
831
- result = client.storage.batch_upload(ws, "", files_payload)
844
+ result = client.storage.batch_upload(
845
+ ws, remote_prefix, files_payload, job_id=job_id
846
+ )
832
847
  url_list = result.get("urls", []) if isinstance(result, dict) else result
833
848
  return {
834
- entry["key"]: entry["url"] for entry in url_list if entry.get("url")
849
+ f"{remote_prefix}{entry['key']}": entry["url"]
850
+ for entry in url_list
851
+ if entry.get("url")
835
852
  }
836
853
  except Exception:
837
854
  return {}
@@ -966,10 +983,15 @@ def _download_folder_as_zip(
966
983
  @app.command()
967
984
  def download(
968
985
  key: str = typer.Argument(
969
- ..., help="Remote file key to download, or s3://bucket/key or s3://bucket/dir/"
986
+ ...,
987
+ help=(
988
+ "Remote file or folder key to download, "
989
+ "or s3://bucket/key or s3://bucket/dir/"
990
+ ),
970
991
  ),
971
992
  destination: str | None = typer.Argument(
972
- None, help="Local path (defaults to filename)"
993
+ None,
994
+ help="Local path (defaults to the filename, or the folder name for a folder)",
973
995
  ),
974
996
  workspace_id: str | None = typer.Option(
975
997
  None,
@@ -981,7 +1003,7 @@ def download(
981
1003
  False,
982
1004
  "--recursive",
983
1005
  "-r",
984
- help="Download all files under an s3:// directory prefix",
1006
+ help="Force recursive download (folders are detected automatically)",
985
1007
  ),
986
1008
  zip_archive: bool = typer.Option(
987
1009
  False,
@@ -998,7 +1020,11 @@ def download(
998
1020
  """Download a file or directory from workspace storage.
999
1021
 
1000
1022
  Accepts an s3://bucket/key to resolve the workspace automatically.
1001
- Use --recursive (-r) to download all files under an s3:// directory prefix.
1023
+
1024
+ Folders are detected automatically: ``omicslab download snapshots`` and
1025
+ ``omicslab download snapshots/ ./snapshots -r`` do the same thing. A
1026
+ folder downloads into a local directory named after its last key segment
1027
+ unless a destination is given.
1002
1028
  """
1003
1029
  config = Config()
1004
1030
  client = get_client()
@@ -1012,8 +1038,6 @@ def download(
1012
1038
  if s3_info:
1013
1039
  ws = s3_info["workspace_id"]
1014
1040
  full_key = s3_info["key_prefix"]
1015
- if recursive:
1016
- full_key = full_key.rstrip("/") + "/" if full_key else ""
1017
1041
  print_panel(
1018
1042
  "S3 Path Resolution",
1019
1043
  {
@@ -1029,6 +1053,45 @@ def download(
1029
1053
  if prefix:
1030
1054
  full_key = prefix + key
1031
1055
 
1056
+ # A trailing slash always means a folder. Without one, ask the listing
1057
+ # whether the key is an object or a folder instead of blindly presigning
1058
+ # it: a folder key would fail with a provider 403/404 and look like a
1059
+ # broken URL.
1060
+ folder = bool(full_key) and full_key.endswith("/")
1061
+ if full_key and not folder:
1062
+ try:
1063
+ listing = client.storage.list(ws, prefix=full_key)
1064
+ except Exception as e:
1065
+ error(f"Failed to inspect {full_key!r}: {e}")
1066
+ raise typer.Exit(1)
1067
+ objects = (
1068
+ listing
1069
+ if isinstance(listing, list)
1070
+ else listing.get("objects", [])
1071
+ if isinstance(listing, dict)
1072
+ else None
1073
+ )
1074
+ # ``objects is None`` only happens with an unexpected client shape;
1075
+ # fall back to the object download rather than guessing.
1076
+ if objects is not None:
1077
+ if any(obj.get("is_file") and not obj.get("name") for obj in objects):
1078
+ folder = False
1079
+ elif any(
1080
+ not obj.get("is_file") and obj.get("name") == "/" for obj in objects
1081
+ ):
1082
+ folder = True
1083
+ else:
1084
+ error(f"File or folder not found: {full_key}")
1085
+ raise typer.Exit(1)
1086
+
1087
+ if zip_archive and not folder:
1088
+ error("--zip requires a folder key.")
1089
+ raise typer.Exit(1)
1090
+
1091
+ if folder:
1092
+ full_key = full_key.rstrip("/") + "/"
1093
+ recursive = True
1094
+
1032
1095
  # Recursive directory download
1033
1096
  if recursive and (full_key.endswith("/") or not full_key):
1034
1097
  if zip_archive:
@@ -1076,7 +1139,11 @@ def download(
1076
1139
  info("No files found in this path.")
1077
1140
  return
1078
1141
 
1079
- dest_dir = Path(destination or ".") if destination else Path(".")
1142
+ # A folder downloads into a directory named after its last key
1143
+ # segment, so `download snapshots` is equivalent to
1144
+ # `download snapshots/ ./snapshots -r`.
1145
+ folder_name = full_key.rstrip("/").rsplit("/", 1)[-1]
1146
+ dest_dir = Path(destination) if destination else Path(folder_name or ".")
1080
1147
  dest_dir.mkdir(parents=True, exist_ok=True)
1081
1148
 
1082
1149
  from rich.progress import (
@@ -46,6 +46,16 @@ class Client:
46
46
  ).rstrip("/")
47
47
  self._base_url = resolved_base_url
48
48
 
49
+ # Inside a studio session the real token stays in the host-side
50
+ # credential broker. The session config contains a placeholder token
51
+ # and this socket; every API call goes through the broker, which
52
+ # injects the real Authorization header upstream.
53
+ broker_socket = os.environ.get("OMICSLAB_BROKER_SOCKET")
54
+ if broker_socket and transport is None:
55
+ resolved_base_url = "http://broker"
56
+ self._base_url = resolved_base_url
57
+ transport = httpx.HTTPTransport(uds=broker_socket)
58
+
49
59
  if token:
50
60
  self._config.set_token(token)
51
61
  if base_url:
@@ -9,11 +9,26 @@ class Config:
9
9
  DEFAULT_PROFILE = "default"
10
10
 
11
11
  def __init__(self, config_path: Path | None = None, profile: str | None = None):
12
- self._config_path = config_path or self.DEFAULT_CONFIG_FILE
12
+ self._config_path = config_path or self._resolve_config_path()
13
13
  self._profile_name = profile or os.environ.get("OMICSLAB_PROFILE")
14
14
  self._data: dict = {}
15
15
  self._load()
16
16
 
17
+ @classmethod
18
+ def _resolve_config_path(cls) -> Path:
19
+ """Resolve the config file path for a new instance.
20
+
21
+ ``OMICSLAB_CONFIG_DIR`` lets a caller point the CLI at a dedicated
22
+ config directory (e.g. a container-private mount) without changing
23
+ ``$HOME`` — the token is then read from that directory while the
24
+ user's real profile stays untouched. ``DEFAULT_CONFIG_FILE`` is kept
25
+ as the fallback (and as the monkeypatch hook used by tests).
26
+ """
27
+ config_dir = os.environ.get("OMICSLAB_CONFIG_DIR")
28
+ if config_dir:
29
+ return Path(config_dir).expanduser() / "config.json"
30
+ return cls.DEFAULT_CONFIG_FILE
31
+
17
32
  def _load(self):
18
33
  if self._config_path.exists():
19
34
  try:
@@ -0,0 +1,235 @@
1
+ """Host-side credential broker for studio sessions.
2
+
3
+ The session container must be able to call the platform API (``omicslab
4
+ download``/``upload`` and the SDK resources) without ever receiving the real
5
+ job token. This module runs a tiny HTTP proxy on a Unix socket in the agent
6
+ process, injects the real ``Authorization`` header, and forwards the request
7
+ to the API. The container's config contains only a placeholder token.
8
+
9
+ Security properties:
10
+
11
+ - The real token never leaves the host process. It is not in the container's
12
+ environment, argv, files, tmpfs, or snapshot.
13
+ - The upstream base URL is fixed at construction time; request paths are
14
+ appended to it, so the container cannot turn the broker into an open proxy
15
+ (absolute and scheme-relative URLs are rejected).
16
+ - Redirects are not followed; the response is streamed back as-is.
17
+ - The socket is only mounted into the session container. Its host directory
18
+ is private to the agent (``mkdtemp``).
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import asyncio
24
+ import os
25
+ import threading
26
+ from collections.abc import Callable
27
+
28
+ from aiohttp import ClientError, ClientSession, ClientTimeout, web
29
+
30
+ # Hop-by-hop headers must not be forwarded (RFC 7230 §6.1).
31
+ _HOP_BY_HOP = frozenset(
32
+ {
33
+ "connection",
34
+ "keep-alive",
35
+ "proxy-authenticate",
36
+ "proxy-authorization",
37
+ "te",
38
+ "trailer",
39
+ "trailers",
40
+ "transfer-encoding",
41
+ "upgrade",
42
+ }
43
+ )
44
+ _SKIP_REQUEST = _HOP_BY_HOP | {
45
+ "host",
46
+ "authorization",
47
+ # The backend trusts forwarding headers from its private-network peers;
48
+ # the broker is such a peer, so a session must not be able to spoof its
49
+ # audit IP through them.
50
+ "forwarded",
51
+ "x-forwarded-for",
52
+ "x-forwarded-host",
53
+ "x-forwarded-proto",
54
+ "x-real-ip",
55
+ }
56
+ _SKIP_RESPONSE = _HOP_BY_HOP | {"content-length"}
57
+
58
+ _CHUNK_SIZE = 64 * 1024
59
+
60
+
61
+ class CredentialBroker:
62
+ """Inject the real API token into requests made by the session container.
63
+
64
+ ``start()`` binds a Unix socket and blocks until it is accepting requests;
65
+ ``stop()`` shuts the listener down and removes the socket. The broker is
66
+ designed to live for one studio session.
67
+ """
68
+
69
+ def __init__(
70
+ self,
71
+ upstream_base: str,
72
+ token: str,
73
+ socket_path: str,
74
+ log: Callable[[str], None] | None = None,
75
+ ):
76
+ self._upstream_base = upstream_base.rstrip("/")
77
+ self._token = token
78
+ self._socket_path = socket_path
79
+ self._log = log or (lambda _message: None)
80
+ self._ready = threading.Event()
81
+ self._error: BaseException | None = None
82
+ self._loop: asyncio.AbstractEventLoop | None = None
83
+ self._stopped: asyncio.Event | None = None
84
+ self._session: ClientSession | None = None
85
+ self._thread: threading.Thread | None = None
86
+ self._started = False
87
+
88
+ # ── lifecycle ──────────────────────────────────────────────────────
89
+
90
+ def start(self, timeout: float = 10.0) -> None:
91
+ """Start the broker and wait until the socket is accepting requests."""
92
+ if self._thread is not None:
93
+ return
94
+ parent = os.path.dirname(self._socket_path)
95
+ if parent:
96
+ os.makedirs(parent, exist_ok=True)
97
+ try:
98
+ os.unlink(self._socket_path)
99
+ except FileNotFoundError:
100
+ pass
101
+
102
+ self._thread = threading.Thread(
103
+ target=self._run,
104
+ name="omicslab-credential-broker",
105
+ daemon=True,
106
+ )
107
+ self._thread.start()
108
+ if not self._ready.wait(timeout):
109
+ self.stop()
110
+ raise RuntimeError("credential broker did not start in time")
111
+ if self._error is not None:
112
+ self.stop()
113
+ raise RuntimeError(f"credential broker failed to start: {self._error}")
114
+ self._started = True
115
+ self._log(f"[BROKER] in-session API broker listening on {self._socket_path}")
116
+
117
+ def stop(self, timeout: float = 5.0) -> None:
118
+ """Stop the listener and remove the Unix socket."""
119
+ if self._loop is not None and self._stopped is not None:
120
+ try:
121
+ self._loop.call_soon_threadsafe(self._stopped.set)
122
+ except RuntimeError:
123
+ # The loop already closed (e.g. startup failed).
124
+ pass
125
+ if self._thread is not None:
126
+ self._thread.join(timeout=timeout)
127
+ self._thread = None
128
+ try:
129
+ os.unlink(self._socket_path)
130
+ except FileNotFoundError:
131
+ pass
132
+ if self._started:
133
+ self._started = False
134
+ self._log("[BROKER] in-session API broker stopped")
135
+
136
+ def _run(self) -> None:
137
+ try:
138
+ asyncio.run(self._serve())
139
+ except BaseException as exc: # noqa: BLE001 - reported by start()
140
+ self._error = exc
141
+ self._ready.set()
142
+
143
+ async def _serve(self) -> None:
144
+ self._loop = asyncio.get_running_loop()
145
+ self._stopped = asyncio.Event()
146
+ self._session = ClientSession(
147
+ auto_decompress=False,
148
+ trust_env=False,
149
+ timeout=ClientTimeout(total=None, connect=30, sock_read=600),
150
+ )
151
+ app = web.Application()
152
+ app.router.add_route("*", "/{tail:.*}", self._handle)
153
+ runner = web.AppRunner(app, access_log=None)
154
+ await runner.setup()
155
+ try:
156
+ site = web.UnixSite(runner, self._socket_path)
157
+ await site.start()
158
+ try:
159
+ os.chmod(self._socket_path, 0o666)
160
+ except OSError:
161
+ pass
162
+ self._ready.set()
163
+ await self._stopped.wait()
164
+ finally:
165
+ await self._session.close()
166
+ await runner.cleanup()
167
+
168
+ # ── proxy ──────────────────────────────────────────────────────────
169
+
170
+ def _upstream_url(self, path: str, query: str) -> str:
171
+ """Build the fixed-upstream URL for a request path.
172
+
173
+ Only relative paths are accepted. ``//host`` and absolute URLs would
174
+ let a session redirect the broker (and the token) elsewhere.
175
+ """
176
+ if not path.startswith("/") or path.startswith("//") or "://" in path:
177
+ raise ValueError("invalid request path")
178
+ url = f"{self._upstream_base}{path}"
179
+ return f"{url}?{query}" if query else url
180
+
181
+ async def _handle(self, request: web.Request) -> web.StreamResponse:
182
+ try:
183
+ url = self._upstream_url(request.path, request.query_string)
184
+ except ValueError:
185
+ return web.json_response({"message": "invalid request path"}, status=400)
186
+
187
+ headers = {
188
+ key: value
189
+ for key, value in request.headers.items()
190
+ if key.lower() not in _SKIP_REQUEST
191
+ }
192
+ # Always replace any client-supplied credential.
193
+ headers["Authorization"] = f"Bearer {self._token}"
194
+
195
+ data = self._body(request)
196
+ session = self._session
197
+ if session is None:
198
+ return web.json_response({"message": "broker not ready"}, status=503)
199
+
200
+ try:
201
+ async with session.request(
202
+ request.method,
203
+ url,
204
+ headers=headers,
205
+ data=data,
206
+ allow_redirects=False,
207
+ ) as upstream:
208
+ response = web.StreamResponse(status=upstream.status)
209
+ for key, value in upstream.headers.items():
210
+ if key.lower() not in _SKIP_RESPONSE:
211
+ response.headers.add(key, value)
212
+ if request.method == "HEAD" or upstream.status in (204, 304):
213
+ await response.prepare(request)
214
+ await response.write_eof()
215
+ return response
216
+ await response.prepare(request)
217
+ async for chunk in upstream.content.iter_chunked(_CHUNK_SIZE):
218
+ await response.write(chunk)
219
+ await response.write_eof()
220
+ return response
221
+ except asyncio.TimeoutError:
222
+ return web.json_response({"message": "upstream timeout"}, status=504)
223
+ except ClientError:
224
+ return web.json_response({"message": "upstream unavailable"}, status=502)
225
+
226
+ def _body(self, request: web.Request):
227
+ """Stream a request body without buffering it in memory."""
228
+ if not request.can_read_body:
229
+ return None
230
+
231
+ async def _iter():
232
+ async for chunk in request.content.iter_chunked(_CHUNK_SIZE):
233
+ yield chunk
234
+
235
+ return _iter()