envloop-cli 1.27.0__tar.gz → 1.29.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/PKG-INFO +4 -4
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/README.pypi.md +3 -3
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/_optional.py +4 -4
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/auth_cli.py +13 -11
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/cli_errors.py +1 -1
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/cli_output.py +2 -1
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/lifecycle_cli.py +4 -4
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/resource_cli.py +28 -28
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/terminal.py +8 -5
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/cli.py +14 -14
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/cli_output.py +5 -4
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/client.py +2 -2
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/data.py +35 -35
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/data_cli.py +18 -18
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/data_publications.py +6 -6
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/data_upload.py +1 -1
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/http.py +2 -2
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/navigation_resources.py +13 -11
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/resource_cli.py +21 -21
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/cli.py +54 -22
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/client.py +11 -11
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/decision.py +1 -1
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/http.py +9 -9
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/job_cost.py +12 -9
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/job_spec_cli.py +36 -5
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/resource_cli.py +5 -5
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/resources.py +2 -2
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/train.py +17 -12
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/train_log_cli.py +1 -1
- envloop_cli-1.29.0/packages/experiment/envloop_experiment/train_metrics_cli.py +145 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/train_progress.py +1 -1
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/train_runner_cli.py +1 -1
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/rsi-loop/envloop_rsi_loop/agent_runs.py +14 -19
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/rsi-loop/envloop_rsi_loop/cli.py +1 -1
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/rsi-loop/envloop_rsi_loop/cli_output.py +8 -8
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/pyproject.toml +1 -1
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/.gitignore +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/__init__.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/auth_login.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/auth_workspace.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/cli.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/client.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/config.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/downloads.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/http.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/navigation_resources.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/profile_secrets.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/resources.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/utils/__init__.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/envloop/utils/logging.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/__init__.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/profile_updates.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/recipes.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/data/envloop_data/runner_packages.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/__init__.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/cli_output.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/inference.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/job_logs.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/navigation_resources.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/runtime_params.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/sandbox.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/experiment/envloop_experiment/training/__init__.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/rsi-loop/envloop_rsi_loop/__init__.py +0 -0
- {envloop_cli-1.27.0 → envloop_cli-1.29.0}/packages/rsi-loop/envloop_rsi_loop/rsi_cli.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: envloop-cli
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.29.0
|
|
4
4
|
Summary: EnvLoop CLI and Python SDK for jobs, training, checkpoints and inference
|
|
5
5
|
Project-URL: Homepage, https://envloop.ai
|
|
6
6
|
Project-URL: Documentation, https://docs.envloop.ai
|
|
@@ -226,8 +226,8 @@ Choose a published Runner readable in your project. Save this request as `gpu-ru
|
|
|
226
226
|
{
|
|
227
227
|
"request_id": "gpu-run-001",
|
|
228
228
|
"trials": [
|
|
229
|
-
{"name": "first", "spec": {"runner": "envloop/bash@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "
|
|
230
|
-
{"name": "second", "spec": {"runner": "envloop/bash@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "
|
|
229
|
+
{"name": "first", "spec": {"runner": "envloop/bash@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_seconds": 600}},
|
|
230
|
+
{"name": "second", "spec": {"runner": "envloop/bash@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_seconds": 600}}
|
|
231
231
|
]
|
|
232
232
|
}
|
|
233
233
|
```
|
|
@@ -272,7 +272,7 @@ with Client.remote() as client:
|
|
|
272
272
|
```
|
|
273
273
|
|
|
274
274
|
Use the exact model returned by the Job. The SDK forwards the payload to the
|
|
275
|
-
Job's `/
|
|
275
|
+
Job's `/inference/decision` API; the approved plugin validates its business fields.
|
|
276
276
|
Availability requires matching platform support and approved content in your project.
|
|
277
277
|
|
|
278
278
|
## ANS data and verified downloads
|
|
@@ -209,8 +209,8 @@ Choose a published Runner readable in your project. Save this request as `gpu-ru
|
|
|
209
209
|
{
|
|
210
210
|
"request_id": "gpu-run-001",
|
|
211
211
|
"trials": [
|
|
212
|
-
{"name": "first", "spec": {"runner": "envloop/bash@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "
|
|
213
|
-
{"name": "second", "spec": {"runner": "envloop/bash@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "
|
|
212
|
+
{"name": "first", "spec": {"runner": "envloop/bash@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_seconds": 600}},
|
|
213
|
+
{"name": "second", "spec": {"runner": "envloop/bash@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_seconds": 600}}
|
|
214
214
|
]
|
|
215
215
|
}
|
|
216
216
|
```
|
|
@@ -255,7 +255,7 @@ with Client.remote() as client:
|
|
|
255
255
|
```
|
|
256
256
|
|
|
257
257
|
Use the exact model returned by the Job. The SDK forwards the payload to the
|
|
258
|
-
Job's `/
|
|
258
|
+
Job's `/inference/decision` API; the approved plugin validates its business fields.
|
|
259
259
|
Availability requires matching platform support and approved content in your project.
|
|
260
260
|
|
|
261
261
|
## ANS data and verified downloads
|
|
@@ -16,11 +16,11 @@ from typing import Any, override
|
|
|
16
16
|
FEATURES = {'data': 'envloop_data', 'experiment': 'envloop_experiment', 'rsi-loop': 'envloop_rsi_loop'}
|
|
17
17
|
FEATURE_COMMANDS = {
|
|
18
18
|
'data': {
|
|
19
|
-
'dataset': 'Discover
|
|
19
|
+
'dataset': 'Discover Datasets, revisions and tasks',
|
|
20
20
|
'data': 'Manage versioned Data resources and verified file downloads',
|
|
21
21
|
'runner': 'Discover available Runner revisions',
|
|
22
|
-
'checkpoint': 'List and download saved training
|
|
23
|
-
'profile': 'Manage platform runtime
|
|
22
|
+
'checkpoint': 'List and download saved training Checkpoints',
|
|
23
|
+
'profile': 'Manage platform runtime Profiles',
|
|
24
24
|
},
|
|
25
25
|
'experiment': {
|
|
26
26
|
'job': 'Submit and manage Agent, Sandbox, Train and Inference Jobs',
|
|
@@ -30,7 +30,7 @@ FEATURE_COMMANDS = {
|
|
|
30
30
|
|
|
31
31
|
MODULES: dict[str, str] = {
|
|
32
32
|
**dict.fromkeys('data data_publications data_upload data_cli profile_updates recipes'.split(), 'data'),
|
|
33
|
-
**dict.fromkeys('train train_progress train_runner_cli train_log_cli job_logs job_cost inference decision sandbox runtime_params job_spec_cli training'.split(), 'experiment'),
|
|
33
|
+
**dict.fromkeys('train train_progress train_metrics_cli train_runner_cli train_log_cli job_logs job_cost inference decision sandbox runtime_params job_spec_cli training'.split(), 'experiment'),
|
|
34
34
|
**dict.fromkeys('agent_runs rsi_cli'.split(), 'rsi-loop'),
|
|
35
35
|
}
|
|
36
36
|
CLIENTS = {
|
|
@@ -8,6 +8,8 @@ from typing import Any, cast
|
|
|
8
8
|
from .config import ConfigError, ConfigStore, normalize_url, validate_key
|
|
9
9
|
|
|
10
10
|
DEFAULT_BASE_URL = "https://api.envloop.ai"
|
|
11
|
+
CONFIG_FIELDS = ["base-url", "api-key", "workspace-id", "project-id"]
|
|
12
|
+
FIELD_HELP = "One of " + ", ".join(CONFIG_FIELDS)
|
|
11
13
|
|
|
12
14
|
|
|
13
15
|
def add_auth_commands(groups: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
|
|
@@ -23,29 +25,29 @@ def add_auth_commands(groups: "argparse._SubParsersAction[argparse.ArgumentParse
|
|
|
23
25
|
actions = configure.add_subparsers(dest="config_action")
|
|
24
26
|
init = actions.add_parser("init", help="Configure interactively with masked API key input")
|
|
25
27
|
init.add_argument("--no-check", action="store_true", default=argparse.SUPPRESS, help="Save without API verification (offline)")
|
|
26
|
-
project = actions.add_parser("project", help="Select and verify a
|
|
27
|
-
project.add_argument("project_id", nargs="?", help="Project ID; omit for an interactive list with names and IDs")
|
|
28
|
-
project.add_argument("--stdin", action="store_true", help="Read the selected
|
|
28
|
+
project = actions.add_parser("project", help="Select and verify a Project with its own API key")
|
|
29
|
+
project.add_argument("project_id", metavar="PROJECT_ID", nargs="?", help="Project ID; omit for an interactive list with names and IDs")
|
|
30
|
+
project.add_argument("--stdin", action="store_true", help="Read the selected Project's API key from stdin")
|
|
29
31
|
setting = actions.add_parser("set", help="Save one field (API key is masked or read from stdin)")
|
|
30
|
-
setting.add_argument("field",
|
|
31
|
-
setting.add_argument("value", nargs="?", help=f"Base URL (default: {DEFAULT_BASE_URL}) or
|
|
32
|
+
setting.add_argument("field", metavar="FIELD", choices=CONFIG_FIELDS, help=FIELD_HELP)
|
|
33
|
+
setting.add_argument("value", metavar="VALUE", nargs="?", help=f"Base URL (default: {DEFAULT_BASE_URL}) or Workspace ID; API keys use masked input or --stdin")
|
|
32
34
|
setting.add_argument("--stdin", action="store_true", help="Read a single API key from stdin")
|
|
33
35
|
setting.add_argument("--no-check", action="store_true", default=argparse.SUPPRESS, help="Save API key without verification (offline)")
|
|
34
36
|
getting = actions.add_parser("get", help="Read saved configuration; never export an API key")
|
|
35
|
-
getting.add_argument("field",
|
|
37
|
+
getting.add_argument("field", metavar="FIELD", choices=CONFIG_FIELDS, help=FIELD_HELP)
|
|
36
38
|
actions.add_parser("path", help="Print configuration and credential file paths")
|
|
37
39
|
show = actions.add_parser("show", help="Show saved and effective settings without revealing keys")
|
|
38
40
|
show.add_argument("--json", action="store_true")
|
|
39
41
|
remove = actions.add_parser("unset", help="Delete a saved field; does not revoke the server key")
|
|
40
|
-
remove.add_argument("field",
|
|
42
|
+
remove.add_argument("field", metavar="FIELD", choices=CONFIG_FIELDS, help=FIELD_HELP)
|
|
41
43
|
status = commands.add_parser("status", help="Inspect local credentials; --check verifies read access")
|
|
42
44
|
status.add_argument("--check", action="store_true", help="Make a read-only request to the effective Base URL")
|
|
43
45
|
status.add_argument("--json", action="store_true")
|
|
44
46
|
commands.add_parser("logout", help="Remove the saved key for the stored default Base URL (local only)")
|
|
45
|
-
workspace = commands.add_parser("workspace", help="Select and verify the active
|
|
46
|
-
workspace.add_argument("workspace_id", nargs="?", help="Workspace ID; omit for an interactive list")
|
|
47
|
-
workspace.add_argument("--project-id", help="Project within the
|
|
48
|
-
workspace.add_argument("--stdin", action="store_true", help="Read the selected
|
|
47
|
+
workspace = commands.add_parser("workspace", help="Select and verify the active Workspace")
|
|
48
|
+
workspace.add_argument("workspace_id", metavar="WORKSPACE_ID", nargs="?", help="Workspace ID; omit for an interactive list")
|
|
49
|
+
workspace.add_argument("--project-id", help="Project within the Workspace; its own API key is required")
|
|
50
|
+
workspace.add_argument("--stdin", action="store_true", help="Read the selected Workspace's API key from stdin")
|
|
49
51
|
|
|
50
52
|
|
|
51
53
|
def _key_from_input(stdin: bool = False) -> str:
|
|
@@ -182,7 +182,7 @@ TASK_REQUEST_CONFLICT TEMPORARY_DEPENDENCY_REQUIRES_RETAIN TEMPORARY_RESOURCE_RE
|
|
|
182
182
|
TEMPORARY_SCOPE_CONFLICT TEMPORARY_SCOPE_EXPIRED TRIAL_IDEMPOTENCY_CONFLICT TRIAL_VERSION_CONFLICT USAGE_EVENT_CONFLICT
|
|
183
183
|
PROFILE_CORRUPT ADAPTER_STORAGE_UNAVAILABLE INSUFFICIENT_BALANCE METHOD_NOT_ALLOWED REQUEST_TOO_LARGE
|
|
184
184
|
BATCH_POLICY_NOT_APPROVED BATCH_PROVIDER_UNAVAILABLE CHECKPOINT_ACCESS_DENIED CHECKPOINT_DOWNLOAD_UNAVAILABLE
|
|
185
|
-
CHECKPOINT_MANAGEMENT_UNAVAILABLE CHECKPOINT_MANAGEMENT_UNSUPPORTED CHECKPOINT_NOT_DURABLE
|
|
185
|
+
CHECKPOINT_MANAGEMENT_UNAVAILABLE CHECKPOINT_MANAGEMENT_UNSUPPORTED CHECKPOINT_NOT_DURABLE CHECKPOINT_TRIAL_REQUIRED
|
|
186
186
|
CHECKPOINT_SIGNER_UNAVAILABLE INVALID_CHECKPOINT_CURSOR INVALID_CHECKPOINT_TTL INVALID_CHECKPOINT_VISIBILITY
|
|
187
187
|
INVALID_EXECUTION_CONFIG INVALID_METRICS_RANGE INVALID_TRAIN_JOB_CURSOR INVALID_TRAIN_JOB_LIST_LIMIT
|
|
188
188
|
INVALID_TRAIN_JOB_METADATA INVALID_TRAIN_JOB_SPEC INVALID_TRAIN_JOB_STATUS LOG_RETENTION_EXPIRED
|
|
@@ -38,7 +38,8 @@ def table(headers: Sequence[object], rows: Sequence[Sequence[object]]) -> str:
|
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
def timestamp(value: object) -> str:
|
|
41
|
-
|
|
41
|
+
"""Table and summary times: UTC to seconds, labelled; --json keeps the original value."""
|
|
42
|
+
return compact_time(value, utc=True)
|
|
42
43
|
|
|
43
44
|
|
|
44
45
|
def next_page(args: argparse.Namespace, cursor: object, program: str) -> str:
|
|
@@ -6,7 +6,7 @@ from typing import Any
|
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
def wait_options(parser: argparse.ArgumentParser):
|
|
9
|
-
parser.add_argument('--wait', action='store_true', help='Wait for the persisted cleanup
|
|
9
|
+
parser.add_argument('--wait', action='store_true', help='Wait for the persisted cleanup Operation')
|
|
10
10
|
parser.add_argument('--timeout-seconds', type=float, default=600)
|
|
11
11
|
|
|
12
12
|
|
|
@@ -14,10 +14,10 @@ def add_management_commands(settings: "argparse._SubParsersAction[argparse.Argum
|
|
|
14
14
|
operations = settings.add_parser('operation', help='Inspect or retry persisted cleanup').add_subparsers(dest='operation_action', required=True)
|
|
15
15
|
for action in ('get', 'retry', 'wait'):
|
|
16
16
|
command = operations.add_parser(action)
|
|
17
|
-
command.add_argument('identifier')
|
|
17
|
+
command.add_argument('identifier', metavar='OPERATION_ID')
|
|
18
18
|
if action == 'wait':
|
|
19
19
|
command.add_argument('--timeout-seconds', type=float, default=600)
|
|
20
|
-
cleanup = settings.add_parser('cleanup', help='Clean an explicit set of
|
|
20
|
+
cleanup = settings.add_parser('cleanup', help='Clean an explicit set of Project resource IDs')
|
|
21
21
|
cleanup.add_argument('--file', type=Path, required=True, help='JSON array of resource kind and ID objects')
|
|
22
22
|
cleanup.add_argument('--request-id', required=True)
|
|
23
23
|
wait_options(cleanup)
|
|
@@ -27,7 +27,7 @@ def add_management_commands(settings: "argparse._SubParsersAction[argparse.Argum
|
|
|
27
27
|
create.add_argument('--request-id', required=True)
|
|
28
28
|
for action in ('get', 'cleanup', 'retain'):
|
|
29
29
|
command = scopes.add_parser(action)
|
|
30
|
-
command.add_argument('identifier')
|
|
30
|
+
command.add_argument('identifier', metavar='RESOURCE_SCOPE_ID')
|
|
31
31
|
if action == 'cleanup':
|
|
32
32
|
wait_options(command)
|
|
33
33
|
if action == 'retain':
|
|
@@ -17,10 +17,10 @@ def listing(commands: "argparse._SubParsersAction[argparse.ArgumentParser]", *,
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
def add_resource_commands(groups: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
|
|
20
|
-
dashboard = groups.add_parser('dashboard', help='Inspect the current
|
|
21
|
-
dashboard.add_parser('get', help='Show
|
|
20
|
+
dashboard = groups.add_parser('dashboard', help='Inspect the current Project overview').add_subparsers(dest='action', required=True)
|
|
21
|
+
dashboard.add_parser('get', help='Show Project details and recent experiment pages')
|
|
22
22
|
|
|
23
|
-
billing = groups.add_parser('billing', help='Inspect
|
|
23
|
+
billing = groups.add_parser('billing', help='Inspect Project billing and recorded usage').add_subparsers(dest='resource', required=True)
|
|
24
24
|
for resource in ('account', 'usage', 'ledger', 'price'):
|
|
25
25
|
commands = billing.add_parser(resource, help=f'Inspect billing {resource}').add_subparsers(dest='action', required=True)
|
|
26
26
|
command = commands.add_parser('get', help='Show the billing account') if resource == 'account' else listing(commands, paged=False)
|
|
@@ -31,27 +31,27 @@ def add_resource_commands(groups: "argparse._SubParsersAction[argparse.ArgumentP
|
|
|
31
31
|
if resource == 'ledger':
|
|
32
32
|
command.add_argument('--limit', type=int, default=25)
|
|
33
33
|
|
|
34
|
-
limits = groups.add_parser('limit', help='Inspect
|
|
34
|
+
limits = groups.add_parser('limit', help='Inspect Project resource limits').add_subparsers(dest='action', required=True)
|
|
35
35
|
limits.add_parser('get', help='Read effective execution limits, compose Runner state and the registry credential '
|
|
36
36
|
'count (read-only)')
|
|
37
|
-
members = groups.add_parser('member', help='Manage
|
|
37
|
+
members = groups.add_parser('member', help='Manage Project members subject to server authorization').add_subparsers(dest='action', required=True)
|
|
38
38
|
listing(members, paged=False)
|
|
39
|
-
candidates = members.add_parser('candidate', help='Find eligible
|
|
40
|
-
listing(candidates, paged=False, help='List eligible
|
|
39
|
+
candidates = members.add_parser('candidate', help='Find eligible Project members').add_subparsers(dest='candidate_action', required=True)
|
|
40
|
+
listing(candidates, paged=False, help='List eligible Project members matching a query').add_argument('--query', required=True)
|
|
41
41
|
for action in ('add', 'update', 'remove'):
|
|
42
|
-
command = members.add_parser(action, help=f'{action.capitalize()} a
|
|
43
|
-
command.add_argument('user_id')
|
|
42
|
+
command = members.add_parser(action, help=f'{action.capitalize()} a Project membership (human-session authorization required)')
|
|
43
|
+
command.add_argument('user_id', metavar='USER_ID')
|
|
44
44
|
if action != 'remove':
|
|
45
45
|
command.add_argument('--role', choices=['viewer', 'developer', 'admin'], default='developer')
|
|
46
46
|
command.add_argument('--expected-version', required=True, type=int, help='Current membership version; use 0 for a new grant')
|
|
47
47
|
|
|
48
|
-
settings = groups.add_parser('setting', help='Inspect and update the current
|
|
49
|
-
settings.add_parser('get', help='Show current
|
|
50
|
-
update = settings.add_parser('update', help='Change
|
|
48
|
+
settings = groups.add_parser('setting', help='Inspect and update the current Project settings').add_subparsers(dest='action', required=True)
|
|
49
|
+
settings.add_parser('get', help='Show current Project settings')
|
|
50
|
+
update = settings.add_parser('update', help='Change Project fields (human-session authorization required)')
|
|
51
51
|
for field in ('name', 'slug', 'region'):
|
|
52
52
|
update.add_argument('--' + field)
|
|
53
53
|
update.add_argument('--expected-version', required=True, type=int)
|
|
54
|
-
archive = settings.add_parser('archive', help='Archive the
|
|
54
|
+
archive = settings.add_parser('archive', help='Archive the Project (human-session authorization required)')
|
|
55
55
|
archive.add_argument('--expected-version', required=True, type=int)
|
|
56
56
|
from .lifecycle_cli import add_management_commands
|
|
57
57
|
add_management_commands(settings)
|
|
@@ -89,40 +89,40 @@ def _revision(value: str) -> str:
|
|
|
89
89
|
|
|
90
90
|
def add_self_service_commands(settings: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
|
|
91
91
|
"""Project Admin settings that also work with an API key whose ceiling includes them."""
|
|
92
|
-
registry = settings.add_parser('registry', help='Manage private registry credentials for this
|
|
92
|
+
registry = settings.add_parser('registry', help='Manage private registry credentials for this Project\'s images'
|
|
93
93
|
).add_subparsers(dest='registry_action', required=True)
|
|
94
94
|
listing(registry, paged=False, help='List registry credentials (metadata only; secrets are write-only)')
|
|
95
|
-
add = registry.add_parser('add', help='Register a write-only registry credential (
|
|
95
|
+
add = registry.add_parser('add', help='Register a write-only registry credential (Project Admin)')
|
|
96
96
|
add.add_argument('--registry', required=True, help='Registry host, for example ghcr.io')
|
|
97
97
|
add.add_argument('--repository-prefix', required=True, help='Repository prefix the credential covers, for example my-org')
|
|
98
98
|
add.add_argument('--username', required=True)
|
|
99
99
|
add.add_argument('--password-stdin', action='store_true',
|
|
100
100
|
help='Read the token or password from stdin; without it you are prompted (no echo)')
|
|
101
101
|
add.add_argument('--password', '--token', nargs='?', action=_RejectSecretArgument, help=argparse.SUPPRESS)
|
|
102
|
-
remove = registry.add_parser('remove', help='Remove a credential; later image pulls no longer receive it (
|
|
103
|
-
remove.add_argument('credential_id')
|
|
104
|
-
compose = settings.add_parser('compose', help='Compose Runner (multi-container Task) access for this
|
|
102
|
+
remove = registry.add_parser('remove', help='Remove a credential; later image pulls no longer receive it (Project Admin)')
|
|
103
|
+
remove.add_argument('credential_id', metavar='CREDENTIAL_ID')
|
|
104
|
+
compose = settings.add_parser('compose', help='Compose Runner (multi-container Task) access for this Project'
|
|
105
105
|
).add_subparsers(dest='compose_action', required=True)
|
|
106
106
|
compose.add_parser('get', help='Show the effective state, the reason and the platform bounds')
|
|
107
|
-
compose.add_parser('enable', help='Request compose Runners; the platform policy may need operator approval (
|
|
108
|
-
compose.add_parser('disable', help='Withdraw this
|
|
109
|
-
concurrency = settings.add_parser('concurrency', help='Concurrent Trial limit of this
|
|
107
|
+
compose.add_parser('enable', help='Request compose Runners; the platform policy may need operator approval (Project Admin)')
|
|
108
|
+
compose.add_parser('disable', help='Withdraw this Project\'s compose Runner request (Project Admin)')
|
|
109
|
+
concurrency = settings.add_parser('concurrency', help='Concurrent Trial limit of this Project'
|
|
110
110
|
).add_subparsers(dest='concurrency_action', required=True)
|
|
111
|
-
command = concurrency.add_parser('set', help='Lower the
|
|
112
|
-
'(
|
|
111
|
+
command = concurrency.add_parser('set', help='Lower the Project limit below the operator ceiling, or none to clear it '
|
|
112
|
+
'(Project Admin)')
|
|
113
113
|
command.add_argument('value', type=_concurrency, metavar='VALUE', help='1..10000 or none')
|
|
114
|
-
model = settings.add_parser('model-access', help='Metered model access for published Runners of this
|
|
114
|
+
model = settings.add_parser('model-access', help='Metered model access for published Runners of this Project'
|
|
115
115
|
).add_subparsers(dest='model_access_action', required=True)
|
|
116
116
|
listing(model, paged=False, help='List grants and whether the platform can deliver them now')
|
|
117
|
-
grant = model.add_parser('grant', help='Let one published Runner receive the
|
|
118
|
-
'Trials (
|
|
117
|
+
grant = model.add_parser('grant', help='Let one published Runner receive the Project\'s metered model key on Fly '
|
|
118
|
+
'Trials (Project Admin)')
|
|
119
119
|
grant.add_argument('runner', type=_runner_name, metavar='NAMESPACE/NAME', help='Published Runner name')
|
|
120
120
|
grant.add_argument('--protocol', dest='protocols', action='append', choices=['anthropic', 'openai'], required=True,
|
|
121
121
|
help='Model protocol to grant; repeat for both')
|
|
122
122
|
grant.add_argument('--revision', type=_revision, help='Limit the grant to one full commit Revision')
|
|
123
123
|
revoke = model.add_parser('revoke', help='Revoke a grant; Trials that have not started no longer get the key '
|
|
124
|
-
'(
|
|
125
|
-
revoke.add_argument('grant_id')
|
|
124
|
+
'(Project Admin)')
|
|
125
|
+
revoke.add_argument('grant_id', metavar='GRANT_ID')
|
|
126
126
|
|
|
127
127
|
|
|
128
128
|
def _read_secret(args: argparse.Namespace) -> str:
|
|
@@ -6,7 +6,7 @@ import sys
|
|
|
6
6
|
import unicodedata
|
|
7
7
|
from contextlib import contextmanager
|
|
8
8
|
from contextvars import ContextVar
|
|
9
|
-
from datetime import datetime
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
10
|
from typing import Any, TextIO
|
|
11
11
|
|
|
12
12
|
_NO_COLOR = ContextVar("envloop_cli_no_color", default=False)
|
|
@@ -47,15 +47,18 @@ def status_text(text: str) -> str:
|
|
|
47
47
|
return style(text, code) if code else text
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
def compact_time(value: Any, *,
|
|
51
|
-
"""Render aware times to seconds; preserve unknown or timezone-free values.
|
|
50
|
+
def compact_time(value: Any, *, utc: bool = False) -> Any:
|
|
51
|
+
"""Render aware times to seconds; preserve unknown or timezone-free values.
|
|
52
|
+
|
|
53
|
+
``utc`` converts to UTC and labels it, so a table read on any machine means the same instant.
|
|
54
|
+
"""
|
|
52
55
|
if not isinstance(value, str):
|
|
53
56
|
return value
|
|
54
57
|
try:
|
|
55
58
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
56
59
|
if parsed.tzinfo is not None:
|
|
57
|
-
if
|
|
58
|
-
|
|
60
|
+
if utc:
|
|
61
|
+
return parsed.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
|
59
62
|
return parsed.strftime("%Y-%m-%d %H:%M:%S")
|
|
60
63
|
except ValueError:
|
|
61
64
|
pass
|
|
@@ -79,10 +79,10 @@ def secret_option(command: argparse.ArgumentParser) -> None:
|
|
|
79
79
|
|
|
80
80
|
|
|
81
81
|
def add_commands(groups: "argparse._SubParsersAction[argparse.ArgumentParser]") -> None:
|
|
82
|
-
profiles = groups.add_parser("profile", help="Manage platform runtime
|
|
82
|
+
profiles = groups.add_parser("profile", help="Manage platform runtime Profiles").add_subparsers(dest="action", required=True)
|
|
83
83
|
put = profiles.add_parser("create", help="Create an immutable Profile version from TOML or JSON")
|
|
84
|
-
put.add_argument("name")
|
|
85
|
-
put.add_argument("version")
|
|
84
|
+
put.add_argument("name", metavar="NAME")
|
|
85
|
+
put.add_argument("version", metavar="VERSION")
|
|
86
86
|
put.add_argument("--file", type=Path, required=True, help="Profile TOML or JSON file")
|
|
87
87
|
put.add_argument("--runner", metavar="RUNNER", help="Bind a Runner revision to this version")
|
|
88
88
|
secret_input = put.add_mutually_exclusive_group()
|
|
@@ -90,8 +90,8 @@ def add_commands(groups: "argparse._SubParsersAction[argparse.ArgumentParser]")
|
|
|
90
90
|
secret_input.add_argument("--model-api-key-stdin", action="store_true", help="Read the model API key from one line of standard input")
|
|
91
91
|
secret_option(put)
|
|
92
92
|
update = profiles.add_parser("update", help="Edit an existing Profile by publishing a new immutable version")
|
|
93
|
-
update.add_argument("name")
|
|
94
|
-
update.add_argument("version", help="New version; existing versions cannot be overwritten")
|
|
93
|
+
update.add_argument("name", metavar="NAME")
|
|
94
|
+
update.add_argument("version", metavar="VERSION", help="New version; existing versions cannot be overwritten")
|
|
95
95
|
update.add_argument("--from-version", help="Source version; omit to use default, then newest unarchived creation")
|
|
96
96
|
update.add_argument("--file", type=Path, required=True, help="TOML or JSON patch: merge params keys, replace supplied metadata; preserve omitted fields")
|
|
97
97
|
update.add_argument("--runner", metavar="RUNNER", help="Bind a Runner revision; v1 requires a full commit; omit to preserve the source binding")
|
|
@@ -100,13 +100,13 @@ def add_commands(groups: "argparse._SubParsersAction[argparse.ArgumentParser]")
|
|
|
100
100
|
update_secret.add_argument("--model-api-key-stdin", action="store_true", help="Read an encrypted model connection key from one line of standard input")
|
|
101
101
|
secret_option(update)
|
|
102
102
|
fork = profiles.add_parser('fork', help='Fork an existing Profile into a new immutable identity')
|
|
103
|
-
fork.add_argument('source_name')
|
|
104
|
-
fork.add_argument('source_version')
|
|
105
|
-
fork.add_argument('name')
|
|
106
|
-
fork.add_argument('version')
|
|
103
|
+
fork.add_argument('source_name', metavar='SOURCE_NAME')
|
|
104
|
+
fork.add_argument('source_version', metavar='SOURCE_VERSION')
|
|
105
|
+
fork.add_argument('name', metavar='NAME')
|
|
106
|
+
fork.add_argument('version', metavar='VERSION')
|
|
107
107
|
secret_option(fork)
|
|
108
108
|
get = profiles.add_parser("get")
|
|
109
|
-
get.add_argument("name")
|
|
109
|
+
get.add_argument("name", metavar="NAME")
|
|
110
110
|
get.add_argument("--version")
|
|
111
111
|
listing = profiles.add_parser("list")
|
|
112
112
|
listing.add_argument("--name")
|
|
@@ -115,13 +115,13 @@ def add_commands(groups: "argparse._SubParsersAction[argparse.ArgumentParser]")
|
|
|
115
115
|
listing.add_argument("--cursor")
|
|
116
116
|
listing.add_argument("--json", action="store_true", help="Output the complete response as JSON (default: table)")
|
|
117
117
|
default = profiles.add_parser("set-default")
|
|
118
|
-
default.add_argument("name")
|
|
119
|
-
default.add_argument("version")
|
|
118
|
+
default.add_argument("name", metavar="NAME")
|
|
119
|
+
default.add_argument("version", metavar="VERSION")
|
|
120
120
|
default.add_argument("--expected-version", help="Omit only when no default exists")
|
|
121
121
|
for command in ("archive", "export"):
|
|
122
122
|
item = profiles.add_parser(command)
|
|
123
|
-
item.add_argument("name")
|
|
124
|
-
item.add_argument("version")
|
|
123
|
+
item.add_argument("name", metavar="NAME")
|
|
124
|
+
item.add_argument("version", metavar="VERSION")
|
|
125
125
|
from envloop_data.resource_cli import add_resource_commands
|
|
126
126
|
add_resource_commands(groups)
|
|
127
127
|
|
|
@@ -38,9 +38,10 @@ def list_layout(items: list[dict[str, Any]], args: argparse.Namespace) -> tuple[
|
|
|
38
38
|
render = dataset_row
|
|
39
39
|
empty = 'No datasets found.'
|
|
40
40
|
elif args.group == 'checkpoint':
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
# The checkpoint ID is the resource_id (cp_…); the save name (adapter-final) repeats across Jobs.
|
|
42
|
+
headers = ('CHECKPOINT ID', 'NAME', 'TYPE', 'MODEL', 'JOB ID', 'URI')
|
|
43
|
+
render = lambda item: (item.get('resource_id'), item.get('checkpoint_id'), item.get('checkpoint_type'),
|
|
44
|
+
item.get('base_model'), item.get('job_id'), item.get('uri'))
|
|
44
45
|
empty = 'No checkpoints found.'
|
|
45
46
|
elif args.group == "profile" and args.name is None:
|
|
46
47
|
headers = ("NAME", "VERSIONS", "DEFAULT", "UPDATED AT")
|
|
@@ -64,7 +65,7 @@ def pagination(command: list[str], args: argparse.Namespace, cursor: object) ->
|
|
|
64
65
|
command.extend(['--limit', str(args.limit), '--cursor=' + str(cursor)])
|
|
65
66
|
fields = {'dataset': ('type', 'query'),
|
|
66
67
|
'data': ('type', 'ref', 'job_id', 'trial_id'),
|
|
67
|
-
'checkpoint': ('job_id', '
|
|
68
|
+
'checkpoint': ('job_id', 'trial_id'), 'profile': ('name',)}
|
|
68
69
|
for field in fields.get(args.group, ()):
|
|
69
70
|
if getattr(args, field, None) is not None:
|
|
70
71
|
if field == 'query':
|
|
@@ -194,7 +194,7 @@ class Runners:
|
|
|
194
194
|
source_ref_kind: str = 'branch', dry_run: bool = False):
|
|
195
195
|
"""Publish a Runner package directory as the immutable revision NAME@REVISION.
|
|
196
196
|
|
|
197
|
-
This is `client.data.create(
|
|
197
|
+
This is `client.data.create(type='runner', ...)` with the native manifest built
|
|
198
198
|
from `directory` (runner.toml plus its main.py or main.sh entrypoint): the
|
|
199
199
|
platform stores the bytes, derives the package and index SHA-256, creates the
|
|
200
200
|
Runner asset on first use and registers the revision. `revision` is the full
|
|
@@ -219,7 +219,7 @@ class Runners:
|
|
|
219
219
|
raise RemoteError('RUNNER_SOURCE_REF_INVALID')
|
|
220
220
|
name, manifest, files, executables = runner_package(directory, name=name, executables=executables)
|
|
221
221
|
ref = f'envloop://runners/{name}@{revision}'
|
|
222
|
-
publication = Data(self._server).create(
|
|
222
|
+
publication = Data(self._server).create(type='runner', ref=ref, manifest=manifest, files=files,
|
|
223
223
|
executables=executables, dry_run=dry_run)
|
|
224
224
|
if dry_run:
|
|
225
225
|
return publication
|
|
@@ -114,10 +114,10 @@ class Data:
|
|
|
114
114
|
raise ValueError('limit must be between 1 and 100')
|
|
115
115
|
return path + '?' + urlencode({key: value for key, value in params.items() if value is not None})
|
|
116
116
|
|
|
117
|
-
def list(self, *,
|
|
118
|
-
if
|
|
119
|
-
raise ValueError('select a Data
|
|
120
|
-
return self.transport.request('GET', self._query(self._path(),
|
|
117
|
+
def list(self, *, type: Literal['dataset', 'artifact', 'checkpoint', 'runner', 'recipe'], ref: str | None = None, job_id: str | None = None, trial_id: str | None = None, limit: int = 25, cursor: str | None = None) -> dict[str, Any]:
|
|
118
|
+
if type not in ('dataset', 'artifact', 'checkpoint', 'runner', 'recipe') or (type != 'artifact' and (job_id or trial_id)):
|
|
119
|
+
raise ValueError('select a Data type and appropriate owner filters')
|
|
120
|
+
return self.transport.request('GET', self._query(self._path(), type=type, ref=ref, job_id=job_id, trial_id=trial_id, limit=limit, cursor=cursor))
|
|
121
121
|
|
|
122
122
|
def get(self, data_id: str) -> dict[str, Any]:
|
|
123
123
|
return self.transport.request('GET', self._path(data_id))
|
|
@@ -131,16 +131,16 @@ class Data:
|
|
|
131
131
|
from envhub.ans.proto import ANSError, DatasetRef, InvalidManifest, parse_uri
|
|
132
132
|
|
|
133
133
|
reference = parse_uri(uri)
|
|
134
|
-
|
|
134
|
+
data_types = {'datasets':'dataset', 'artifacts':'artifact', 'model':'checkpoint',
|
|
135
135
|
'runners':'runner', 'recipes':'recipe'}
|
|
136
|
-
if reference.kind not in
|
|
136
|
+
if reference.kind not in data_types or reference.uri != uri:
|
|
137
137
|
raise ValueError('select a complete resource ANS URI')
|
|
138
138
|
if isinstance(reference, DatasetRef) and (not isinstance(split, str) or not split):
|
|
139
139
|
raise ValueError('Dataset download requires a split')
|
|
140
140
|
if not isinstance(reference, DatasetRef) and split is not None:
|
|
141
141
|
raise ValueError('split is only supported for Dataset')
|
|
142
142
|
endpoint = self._path() + '/download'
|
|
143
|
-
parameters = {'
|
|
143
|
+
parameters = {'type': data_types[reference.kind], 'ref': uri, 'split': split}
|
|
144
144
|
plan = self.transport.request('GET', self._query(endpoint, **parameters))
|
|
145
145
|
from envhub.ans.client import validate_download_plan
|
|
146
146
|
try:
|
|
@@ -179,36 +179,36 @@ class Data:
|
|
|
179
179
|
return {'ref': uri, 'split': split, 'content_digest': plan['content_digest'],
|
|
180
180
|
'directory': str(target), 'verified': True}
|
|
181
181
|
|
|
182
|
-
def create(self, *,
|
|
182
|
+
def create(self, *, type: str, ref: str | None = None, manifest: dict[str, Any] | None = None, files: dict[str, Any] | None = None, model: str | None = None, revision: str | None = None,
|
|
183
183
|
executables: Sequence[str] = (), dry_run: bool = False, idempotency_key: str | None = None):
|
|
184
184
|
"""Register native content or frozen public-HF metadata in this project."""
|
|
185
185
|
from envloop_data.data_publications import hf_source
|
|
186
186
|
|
|
187
|
-
source = hf_source(
|
|
187
|
+
source = hf_source(type, model, revision, ref=ref, manifest=manifest, files=files,
|
|
188
188
|
executables=executables, dry_run=dry_run)
|
|
189
189
|
if source is not None:
|
|
190
190
|
result = self.transport.request('POST', self._path(), source)
|
|
191
|
-
return result if dry_run else self._native_result(result,
|
|
192
|
-
return self._publish(
|
|
191
|
+
return result if dry_run else self._native_result(result, type)
|
|
192
|
+
return self._publish(type, ref, manifest, files, dry_run,
|
|
193
193
|
executables=executables, idempotency_key=idempotency_key)
|
|
194
194
|
|
|
195
195
|
def update(self, data_id: str, *, ref: str | None = None, manifest: dict[str, Any] | None = None, files: dict[str, Any] | None = None, model: str | None = None, revision: str | None = None,
|
|
196
|
-
executables: Sequence[str] = (),
|
|
196
|
+
executables: Sequence[str] = (), type: str | None = None, dry_run: bool = False, idempotency_key: str | None = None):
|
|
197
197
|
"""Publish a new revision of the same asset, preserving its previous revision."""
|
|
198
198
|
from envloop_data.data_publications import hf_source
|
|
199
199
|
|
|
200
|
-
source = hf_source('checkpoint' if
|
|
200
|
+
source = hf_source('checkpoint' if type is None else type, model, revision, ref=ref, manifest=manifest, files=files,
|
|
201
201
|
executables=executables, dry_run=dry_run)
|
|
202
202
|
resource = self.get(data_id)
|
|
203
|
-
if
|
|
204
|
-
raise ValueError('update must preserve the publication
|
|
203
|
+
if type is not None and type != resource['type']:
|
|
204
|
+
raise ValueError('update must preserve the publication type')
|
|
205
205
|
if source is not None:
|
|
206
|
-
if (resource['
|
|
206
|
+
if (resource['type'] != 'checkpoint' or resource.get('metadata_only') is not True
|
|
207
207
|
or resource.get('name') != model or not resource.get('model_uri')):
|
|
208
208
|
raise ValueError('HF update must preserve the native model identity')
|
|
209
209
|
result = self.transport.request('POST', self._path(data_id) + '/revisions', source)
|
|
210
|
-
return result if dry_run else self._native_result(result, resource['
|
|
211
|
-
return self._publish(resource['
|
|
210
|
+
return result if dry_run else self._native_result(result, resource['type'])
|
|
211
|
+
return self._publish(resource['type'], ref, manifest, files, dry_run,
|
|
212
212
|
parent_id=data_id, executables=executables, idempotency_key=idempotency_key)
|
|
213
213
|
|
|
214
214
|
def delete(self, data_id: str, *, dry_run: bool = False):
|
|
@@ -217,9 +217,9 @@ class Data:
|
|
|
217
217
|
return self.transport.request('DELETE', self._path(data_id) + ('?dry_run=true' if dry_run else ''))
|
|
218
218
|
|
|
219
219
|
@staticmethod
|
|
220
|
-
def _native_result(result: Any,
|
|
220
|
+
def _native_result(result: Any, data_type: str):
|
|
221
221
|
resource = result.get('resource', result) if isinstance(result, dict) else None
|
|
222
|
-
if (isinstance(resource, dict) and resource.get('
|
|
222
|
+
if (isinstance(resource, dict) and resource.get('type') == data_type
|
|
223
223
|
and resource.get('native_registered') is True and resource.get('native_ready') is True
|
|
224
224
|
and isinstance(resource.get('data_id', resource.get('id')), str)
|
|
225
225
|
and re.fullmatch(r'ans_[A-Za-z0-9_-]+', resource.get('data_id', resource.get('id')))):
|
|
@@ -231,37 +231,37 @@ class Data:
|
|
|
231
231
|
error.staging_id = staging
|
|
232
232
|
raise error
|
|
233
233
|
|
|
234
|
-
def _publish(self,
|
|
234
|
+
def _publish(self, data_type: str, ref: str | None, manifest: dict[str, Any] | None, files: dict[str, Any] | None, dry_run: bool, *, parent_id: str | None = None, executables: Sequence[str] = (), idempotency_key: str | None = None):
|
|
235
235
|
from envloop_data.data_publications import (
|
|
236
236
|
INLINE_BYTES,
|
|
237
|
-
|
|
237
|
+
TYPES,
|
|
238
238
|
executable_files,
|
|
239
239
|
prepare,
|
|
240
240
|
)
|
|
241
241
|
|
|
242
|
-
if
|
|
243
|
-
raise ValueError('select a publication
|
|
244
|
-
executables = executable_files(
|
|
242
|
+
if data_type not in TYPES or not isinstance(ref, str) or not ref or type(dry_run) is not bool:
|
|
243
|
+
raise ValueError('select a publication type, immutable ANS ref and boolean dry_run')
|
|
244
|
+
executables = executable_files(data_type, manifest, executables)
|
|
245
245
|
with prepare(manifest, files) as (manifest, entries, payload, total, digest):
|
|
246
246
|
if total <= INLINE_BYTES and len(entries) <= 100:
|
|
247
247
|
encoded = {entry['path']: base64.b64encode(payload.read(cast(int, entry['size']))).decode() for entry in entries}
|
|
248
|
-
body: dict[str, Any] = {'
|
|
248
|
+
body: dict[str, Any] = {'type': data_type, 'ref': ref, 'manifest': manifest, 'files': encoded, 'dry_run': dry_run}
|
|
249
249
|
if executables:
|
|
250
250
|
body['executables'] = executables
|
|
251
251
|
path = self._path(parent_id) + '/revisions' if parent_id else self._path()
|
|
252
252
|
result = self.transport.request('POST', path, body)
|
|
253
|
-
return result if dry_run else self._native_result(result,
|
|
253
|
+
return result if dry_run else self._native_result(result, data_type)
|
|
254
254
|
request_key = uuid4().hex if idempotency_key is None else idempotency_key
|
|
255
255
|
identifier = None
|
|
256
256
|
try:
|
|
257
|
-
upload = self.create_upload(
|
|
257
|
+
upload = self.create_upload(type=data_type, ref=ref, manifest=manifest, payload_sha256=digest,
|
|
258
258
|
idempotency_key=request_key, parent_id=parent_id,
|
|
259
259
|
executables=executables, dry_run=dry_run)
|
|
260
260
|
if dry_run:
|
|
261
261
|
return upload
|
|
262
262
|
identifier = upload.get('upload_id')
|
|
263
263
|
if upload.get('state') == 'committed':
|
|
264
|
-
return self._native_result(upload,
|
|
264
|
+
return self._native_result(upload, data_type)
|
|
265
265
|
identifier = upload['upload_id']
|
|
266
266
|
part_bytes = upload['limits']['part_bytes']
|
|
267
267
|
if type(part_bytes) is not int or not 0 < part_bytes <= INLINE_BYTES:
|
|
@@ -278,7 +278,7 @@ class Data:
|
|
|
278
278
|
self.upload_part(identifier, number, chunk)
|
|
279
279
|
if existing:
|
|
280
280
|
raise RemoteError('DATA_UPLOAD_CONFLICT')
|
|
281
|
-
return self._native_result(self.complete_upload(identifier),
|
|
281
|
+
return self._native_result(self.complete_upload(identifier), data_type)
|
|
282
282
|
except BaseException as error:
|
|
283
283
|
# Keep server state recoverable; do not silently abort another
|
|
284
284
|
# process's resumable upload or lose its reservation identifier.
|
|
@@ -289,16 +289,16 @@ class Data:
|
|
|
289
289
|
cast(Any, error).upload_id = identifier
|
|
290
290
|
raise
|
|
291
291
|
|
|
292
|
-
def create_upload(self, *,
|
|
292
|
+
def create_upload(self, *, type: str, ref: str | None, manifest: dict[str, Any] | None, payload_sha256: str, idempotency_key: str, parent_id: str | None = None,
|
|
293
293
|
executables: Sequence[str] = (), dry_run: bool = False):
|
|
294
|
-
from envloop_data.data_publications import
|
|
294
|
+
from envloop_data.data_publications import TYPES, executable_files
|
|
295
295
|
|
|
296
|
-
if (
|
|
296
|
+
if (type not in TYPES or not isinstance(ref, str) or not ref or not isinstance(dry_run, bool)
|
|
297
297
|
or not isinstance(payload_sha256, str) or not re.fullmatch(r'[a-f0-9]{64}', payload_sha256)
|
|
298
298
|
or not isinstance(idempotency_key, str) or not re.fullmatch(r'[A-Za-z0-9_.:-]{1,128}', idempotency_key)):
|
|
299
299
|
raise ValueError('invalid upload metadata or idempotency key')
|
|
300
|
-
executables = executable_files(
|
|
301
|
-
body: dict[str, Any] = dict(
|
|
300
|
+
executables = executable_files(type, manifest, executables)
|
|
301
|
+
body: dict[str, Any] = dict(type=type, ref=ref, manifest=manifest, payload_sha256=payload_sha256, dry_run=dry_run)
|
|
302
302
|
if executables:
|
|
303
303
|
body['executables'] = executables
|
|
304
304
|
if parent_id is not None:
|