envloop-cli 1.23.0__py3-none-any.whl → 1.27.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- envloop/_optional.py +32 -14
- envloop/auth_cli.py +1 -1
- envloop/auth_login.py +2 -2
- envloop/cli.py +14 -7
- envloop/cli_errors.py +19 -4
- envloop/cli_output.py +21 -5
- envloop/client.py +2 -1
- envloop/http.py +18 -4
- envloop/lifecycle_cli.py +4 -4
- envloop/navigation_resources.py +49 -1
- envloop/profile_secrets.py +109 -4
- envloop/resource_cli.py +43 -4
- envloop/resources.py +2 -1
- {envloop_cli-1.23.0.dist-info → envloop_cli-1.27.0.dist-info}/METADATA +24 -18
- envloop_cli-1.27.0.dist-info/RECORD +63 -0
- envloop_data/__init__.py +1 -0
- envloop_data/cli.py +174 -0
- envloop_data/cli_output.py +81 -0
- envloop_data/client.py +239 -0
- envloop_data/data.py +483 -0
- envloop_data/data_cli.py +233 -0
- envloop_data/data_publications.py +133 -0
- envloop_data/data_upload.py +280 -0
- envloop_data/http.py +150 -0
- envloop_data/navigation_resources.py +257 -0
- envloop_data/profile_updates.py +77 -0
- envloop_data/recipes.py +25 -0
- envloop_data/resource_cli.py +112 -0
- envloop_data/runner_packages.py +71 -0
- envloop_experiment/__init__.py +1 -0
- envloop_experiment/cli.py +467 -0
- envloop_experiment/cli_output.py +50 -0
- envloop_experiment/client.py +473 -0
- envloop_experiment/decision.py +20 -0
- envloop_experiment/http.py +147 -0
- envloop_experiment/inference.py +36 -0
- envloop_experiment/job_cost.py +41 -0
- envloop_experiment/job_logs.py +21 -0
- envloop_experiment/job_spec_cli.py +146 -0
- envloop_experiment/navigation_resources.py +43 -0
- envloop_experiment/resource_cli.py +38 -0
- envloop_experiment/resources.py +153 -0
- envloop_experiment/runtime_params.py +83 -0
- envloop_experiment/sandbox.py +28 -0
- envloop_experiment/train.py +96 -0
- envloop_experiment/train_log_cli.py +138 -0
- envloop_experiment/train_progress.py +52 -0
- envloop_experiment/train_runner_cli.py +66 -0
- envloop_experiment/training/__init__.py +4 -0
- envloop_rsi_loop/__init__.py +1 -0
- envloop_rsi_loop/agent_runs.py +254 -0
- envloop_rsi_loop/cli.py +33 -0
- envloop_rsi_loop/cli_output.py +73 -0
- envloop_rsi_loop/rsi_cli.py +65 -0
- envloop_cli-1.23.0.dist-info/RECORD +0 -24
- {envloop_cli-1.23.0.dist-info → envloop_cli-1.27.0.dist-info}/WHEEL +0 -0
- {envloop_cli-1.23.0.dist-info → envloop_cli-1.27.0.dist-info}/entry_points.txt +0 -0
envloop/_optional.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Lazy, static adapters for
|
|
1
|
+
"""Lazy, static adapters for features bundled in the single client distribution.
|
|
2
2
|
|
|
3
3
|
No feature code is imported when the base client is imported or constructed.
|
|
4
4
|
Legacy module names resolve to their single feature owner on explicit import.
|
|
@@ -24,7 +24,7 @@ FEATURE_COMMANDS = {
|
|
|
24
24
|
},
|
|
25
25
|
'experiment': {
|
|
26
26
|
'job': 'Submit and manage Agent, Sandbox, Train and Inference Jobs',
|
|
27
|
-
'
|
|
27
|
+
'trial': 'Inspect and control existing Trials',
|
|
28
28
|
},
|
|
29
29
|
}
|
|
30
30
|
|
|
@@ -35,10 +35,10 @@ MODULES: dict[str, str] = {
|
|
|
35
35
|
}
|
|
36
36
|
CLIENTS = {
|
|
37
37
|
**{name: ('data', 'client') for name in ('ProfileService', 'RunnerService', 'Profiles', 'Runners')},
|
|
38
|
-
**{name: ('experiment', 'client') for name in ('
|
|
38
|
+
**{name: ('experiment', 'client') for name in ('TrialService', 'RpcService', 'load_task', 'validate_command', 'dataset_selector', 'Trials', 'runner_selector', 'job_budget', 'Jobs')},
|
|
39
39
|
**{name: ('base', 'downloads') for name in ('ArtifactError', 'artifact_path')},
|
|
40
40
|
}
|
|
41
|
-
DATA_ENDPOINTS = set('open_download resolve_dataset list_runners profile_path put_profile list_profiles get_profile resolve_profile set_default_profile archive_profile revoke_profile_secret export_profile import_profile open_checkpoint_archive'.split())
|
|
41
|
+
DATA_ENDPOINTS = set('open_download resolve_dataset list_runners get_runner_revision register_runner_revision runner_path profile_path put_profile list_profiles get_profile resolve_profile set_default_profile archive_profile revoke_profile_secret export_profile import_profile open_checkpoint_archive'.split())
|
|
42
42
|
EXPERIMENT_ENDPOINTS = set('submit_job get_job get_job_result get_job_logs list_jobs job_action job_events stop_job list_trials get_task_run get_trial_detail get_trial_training list_trial_metrics list_trial_checkpoints get_task_result get_task_snapshot stop_task list_trial_artifacts open_trial_artifact'.split())
|
|
43
43
|
|
|
44
44
|
|
|
@@ -48,7 +48,23 @@ class MissingFeatureError(ImportError):
|
|
|
48
48
|
|
|
49
49
|
def __init__(self, feature: str):
|
|
50
50
|
self.feature = feature
|
|
51
|
-
super().__init__(
|
|
51
|
+
super().__init__(self.instruction(feature))
|
|
52
|
+
|
|
53
|
+
def instruction(self, feature: str) -> str:
|
|
54
|
+
return f'Install optional functionality with: pip install "envloop-cli[{feature}]"'
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class IncompleteInstallationError(MissingFeatureError):
|
|
58
|
+
"""Feature code ships inside envloop-cli, so a missing module means removed files.
|
|
59
|
+
|
|
60
|
+
Uninstalling the separate envloop-cli-experiment package of 1.23.0/1.24.0 after an
|
|
61
|
+
upgrade deletes files it shared with the single distribution; an extra cannot restore them.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
@override
|
|
65
|
+
def instruction(self, feature: str) -> str:
|
|
66
|
+
return ('Client files are missing. Reinstall with: '
|
|
67
|
+
'pip uninstall -y envloop-cli envloop-cli-experiment && pip install envloop-cli')
|
|
52
68
|
|
|
53
69
|
|
|
54
70
|
class CommandExit(Exception):
|
|
@@ -60,9 +76,15 @@ def available(feature: str):
|
|
|
60
76
|
return importlib.util.find_spec(FEATURES[feature]) is not None
|
|
61
77
|
|
|
62
78
|
|
|
79
|
+
def require_data_dependencies():
|
|
80
|
+
"""Only ANS operations need the public protocol/download dependency."""
|
|
81
|
+
if importlib.util.find_spec('envhub') is None:
|
|
82
|
+
raise MissingFeatureError('data')
|
|
83
|
+
|
|
84
|
+
|
|
63
85
|
def feature_module(feature: str, module: str):
|
|
64
86
|
if not available(feature):
|
|
65
|
-
raise
|
|
87
|
+
raise IncompleteInstallationError(feature)
|
|
66
88
|
return importlib.import_module(f'{FEATURES[feature]}.{module}')
|
|
67
89
|
|
|
68
90
|
|
|
@@ -107,7 +129,7 @@ def client_resource(client: Any, name: str):
|
|
|
107
129
|
return getattr(importlib.import_module(f'envloop.{module}'), cls)(server)
|
|
108
130
|
if name == 'dashboard':
|
|
109
131
|
if available('experiment'):
|
|
110
|
-
return feature_attribute('experiment', 'navigation_resources', 'Dashboard')(client.project, client.jobs, client.
|
|
132
|
+
return feature_attribute('experiment', 'navigation_resources', 'Dashboard')(client.project, client.jobs, client.training_jobs)
|
|
111
133
|
from .navigation_resources import Dashboard
|
|
112
134
|
return Dashboard(client.project)
|
|
113
135
|
if name in {'profiles', 'runners'}:
|
|
@@ -121,17 +143,13 @@ def client_resource(client: Any, name: str):
|
|
|
121
143
|
if name == 'jobs':
|
|
122
144
|
return feature_attribute('experiment', 'client', 'Jobs')(server)
|
|
123
145
|
if name == 'trials':
|
|
124
|
-
return client
|
|
125
|
-
if name == 'tasks':
|
|
126
|
-
return feature_attribute('experiment', 'client', 'Tasks')(server)
|
|
146
|
+
return feature_attribute('experiment', 'client', 'Trials')(server)
|
|
127
147
|
if name == 'training_jobs':
|
|
128
|
-
return client.train
|
|
129
|
-
if name == 'train':
|
|
130
148
|
return feature_attribute('experiment', 'train', 'TrainJobs')(server)
|
|
131
149
|
if name == 'decision':
|
|
132
150
|
return feature_attribute('experiment', 'decision', 'DecisionClient')(server)
|
|
133
151
|
if name == 'agent_runs':
|
|
134
|
-
return feature_attribute('rsi-loop', 'agent_runs', 'AgentRuns')(client.
|
|
152
|
+
return feature_attribute('rsi-loop', 'agent_runs', 'AgentRuns')(client.training_jobs)
|
|
135
153
|
if name in {'endpoints', 'adapters'}:
|
|
136
154
|
return feature_attribute('experiment', 'resources', name.capitalize())(server)
|
|
137
155
|
if name == 'inference':
|
|
@@ -140,7 +158,7 @@ def client_resource(client: Any, name: str):
|
|
|
140
158
|
|
|
141
159
|
|
|
142
160
|
def execute_command(args: argparse.Namespace, client: Any, cli: Any):
|
|
143
|
-
if args.group in {'job', '
|
|
161
|
+
if args.group in {'job', 'trial'}:
|
|
144
162
|
return getattr(feature_module('experiment', 'cli'), 'execute_' + args.group)(args, client, cli)
|
|
145
163
|
if args.group == 'profile':
|
|
146
164
|
return feature_module('data', 'cli').execute_profile(args, client, cli)
|
envloop/auth_cli.py
CHANGED
|
@@ -16,7 +16,7 @@ def add_auth_commands(groups: "argparse._SubParsersAction[argparse.ArgumentParse
|
|
|
16
16
|
login = commands.add_parser("login", help="Open the Clerk sign-in page and authorize CLI/SDK access")
|
|
17
17
|
login.add_argument("--base-url", help="API Base URL; defaults to the effective connection or official API")
|
|
18
18
|
login.add_argument("--no-browser", action="store_true", help="Print the login link without opening a browser")
|
|
19
|
-
login.add_argument("--timeout", type=int, choices=range(1, 601), metavar="SECONDS", default=300,
|
|
19
|
+
login.add_argument("--timeout-seconds", type=int, choices=range(1, 601), metavar="SECONDS", default=300,
|
|
20
20
|
help="Wait up to 1–600 seconds (default: 300)")
|
|
21
21
|
configure = commands.add_parser("config", help="Save Base URL and API key for CLI and SDK")
|
|
22
22
|
configure.add_argument("--no-check", action="store_true", help="Save without API verification (offline; key remains unverified)")
|
envloop/auth_login.py
CHANGED
|
@@ -70,13 +70,13 @@ def login(args: argparse.Namespace, store: Any) -> int:
|
|
|
70
70
|
|
|
71
71
|
saved, credential = store.workspace_snapshot()
|
|
72
72
|
url = normalize_url(args.base_url or resolve_cli_connection().api_url or DEFAULT_BASE_URL)
|
|
73
|
-
deadline = time.monotonic() + args.
|
|
73
|
+
deadline = time.monotonic() + args.timeout_seconds
|
|
74
74
|
key, poll_token = 'epk_' + secrets.token_hex(32), secrets.token_urlsafe(32)
|
|
75
75
|
code = ''.join(secrets.choice('ABCDEFGHJKLMNPQRSTUVWXYZ23456789') for _ in range(8))
|
|
76
76
|
def digest(value: str) -> str:
|
|
77
77
|
return hashlib.sha256(value.encode()).hexdigest()
|
|
78
78
|
response = _request(url, 'start', {'key_hash': digest(key), 'poll_hash': digest(poll_token),
|
|
79
|
-
'code_hash': digest(code)}, min(5, args.
|
|
79
|
+
'code_hash': digest(code)}, min(5, args.timeout_seconds))
|
|
80
80
|
identifier, uri = _verification_url(url, response)
|
|
81
81
|
interval, expires = response.get('interval'), response.get('expires_in')
|
|
82
82
|
if (isinstance(interval, bool) or not isinstance(interval, (int, float)) or not 0 < interval <= 10 or
|
envloop/cli.py
CHANGED
|
@@ -14,7 +14,7 @@ from .http import CLI_VERSION
|
|
|
14
14
|
|
|
15
15
|
SECTIONS = {
|
|
16
16
|
'OVERVIEW': ('dashboard',),
|
|
17
|
-
'EXPERIMENT': ('job', '
|
|
17
|
+
'EXPERIMENT': ('job', 'trial'),
|
|
18
18
|
'DATA': ('dataset', 'data', 'runner', 'checkpoint', 'profile'),
|
|
19
19
|
'MANAGE': ('billing', 'limit', 'member', 'setting'),
|
|
20
20
|
'TOOLS': ('auth', 'help'),
|
|
@@ -43,6 +43,8 @@ class HelpArgumentParser(argparse.ArgumentParser):
|
|
|
43
43
|
|
|
44
44
|
def __init__(self, *args: Any, **kwargs: Any):
|
|
45
45
|
kwargs.setdefault('formatter_class', CommandHelpFormatter)
|
|
46
|
+
# Prefixes of an option would be unlisted aliases; only full option names are accepted.
|
|
47
|
+
kwargs.setdefault('allow_abbrev', False)
|
|
46
48
|
super().__init__(*args, **kwargs)
|
|
47
49
|
self.add_argument("--no-color", action="store_true", default=argparse.SUPPRESS,
|
|
48
50
|
help="Disable terminal highlighting")
|
|
@@ -140,7 +142,7 @@ def parser():
|
|
|
140
142
|
cli = HelpArgumentParser(
|
|
141
143
|
prog=Path(sys.argv[0]).name if Path(sys.argv[0]).name in {"envloop", "el"} else "el",
|
|
142
144
|
usage="%(prog)s [options] [command]", formatter_class=CommandHelpFormatter,
|
|
143
|
-
description="Manage Agent, Sandbox, Train and Inference Jobs and
|
|
145
|
+
description="Manage Agent, Sandbox, Train and Inference Jobs and their Trials with EnvLoop.\nConfigure your connection with 'auth config'.",
|
|
144
146
|
)
|
|
145
147
|
cli._optionals.title = "Options"
|
|
146
148
|
cli.set_defaults(api_url=None, workspace_id=None, no_color=False)
|
|
@@ -222,18 +224,18 @@ def _main(argv: list[str] | None = None):
|
|
|
222
224
|
disable_color()
|
|
223
225
|
if any(value == "--api-key" or value.startswith("--api-key=") for value in argv):
|
|
224
226
|
cli.error("API keys must be entered with auth config or supplied through ENVLOOP_API_KEY")
|
|
225
|
-
from ._optional import
|
|
227
|
+
from ._optional import IncompleteInstallationError, available
|
|
226
228
|
requested = next((value for value in argv if not value.startswith('-')), None)
|
|
227
|
-
feature = cast(dict[str | None, str], {'job': 'experiment', '
|
|
229
|
+
feature = cast(dict[str | None, str], {'job': 'experiment', 'trial': 'experiment', 'data': 'data', 'dataset': 'data',
|
|
228
230
|
'runner': 'data', 'checkpoint': 'data', 'profile': 'data'}).get(requested)
|
|
229
231
|
if feature is not None and not available(feature):
|
|
230
232
|
from .cli_errors import format_error
|
|
231
|
-
print(format_error(
|
|
233
|
+
print(format_error(IncompleteInstallationError(feature)), file=sys.stderr)
|
|
232
234
|
return 1
|
|
233
235
|
job_action = next((value for value in argv[argv.index('job') + 1:] if not value.startswith('-')), None) if requested == 'job' else None
|
|
234
236
|
if job_action == 'run' and not available('rsi-loop'):
|
|
235
237
|
from .cli_errors import format_error
|
|
236
|
-
print(format_error(
|
|
238
|
+
print(format_error(IncompleteInstallationError('rsi-loop')), file=sys.stderr)
|
|
237
239
|
return 1
|
|
238
240
|
args = cli.parse_args(argv)
|
|
239
241
|
if args.group == "help":
|
|
@@ -254,7 +256,7 @@ def _main(argv: list[str] | None = None):
|
|
|
254
256
|
|
|
255
257
|
print(format_error(exc), file=sys.stderr)
|
|
256
258
|
return 1
|
|
257
|
-
if args.group in {'job', '
|
|
259
|
+
if args.group in {'job', 'trial'}:
|
|
258
260
|
from envloop_experiment.cli import validate
|
|
259
261
|
status = validate(args, cli)
|
|
260
262
|
if status is not None:
|
|
@@ -308,6 +310,11 @@ def _main(argv: list[str] | None = None):
|
|
|
308
310
|
print(json.dumps(operation_output(exc.operation), ensure_ascii=False))
|
|
309
311
|
print(format_error(exc), file=sys.stderr)
|
|
310
312
|
return 1
|
|
313
|
+
from .http import WaitTimeoutError
|
|
314
|
+
if isinstance(exc, WaitTimeoutError) and isinstance(exc.last, dict):
|
|
315
|
+
# The work continues on the server; keep its ID so it can be queried or waited on again.
|
|
316
|
+
last = operation_output(exc.last) if 'operation_id' in exc.last else exc.last
|
|
317
|
+
print(json.dumps(last, ensure_ascii=False))
|
|
311
318
|
# Do not emit arbitrary exception messages, file content or credentials.
|
|
312
319
|
print(format_error(exc), file=sys.stderr)
|
|
313
320
|
return 1
|
envloop/cli_errors.py
CHANGED
|
@@ -17,7 +17,7 @@ from collections.abc import Iterable
|
|
|
17
17
|
from datetime import datetime
|
|
18
18
|
from typing import Any, cast
|
|
19
19
|
|
|
20
|
-
from .http import RemoteError
|
|
20
|
+
from .http import GATEWAY_TIMEOUT_STATUSES, RemoteError
|
|
21
21
|
|
|
22
22
|
_MESSAGES: dict[str, str] = {}
|
|
23
23
|
|
|
@@ -45,6 +45,13 @@ _register("EDGE_REJECTED", "The network edge (Cloudflare) rejected the request b
|
|
|
45
45
|
_register("HUMAN_SESSION_REQUIRED WORKSPACE_LIST_REQUIRES_SESSION PROJECT_LIST_REQUIRES_SESSION", "This operation requires a browser session authorized by the server.")
|
|
46
46
|
_register("CONFLICT VERSION_CONFLICT RESOURCE_VERSION_CONFLICT PRECONDITION_FAILED IDEMPOTENCY_CONFLICT", "The request conflicts with the current resource version or an earlier request.")
|
|
47
47
|
_register("BAD_REQUEST VALIDATION_ERROR VALIDATION_FAILED INVALID_REQUEST INVALID_ARGUMENT INVALID_CURSOR INVALID_PROFILE INVALID_PROFILE_SECRETS", "The request or input is invalid.")
|
|
48
|
+
_register("PROFILE_SECRET_NAME_INVALID", "A Secret parameter name must look like a credential, as the platform decides: it ends with API_KEY or _TOKEN, or contains PASSWORD, SECRET, CREDENTIAL, AUTHORIZATION, ACCESS_TOKEN, AUTH_TOKEN, BEARER_TOKEN or PRIVATE_KEY. Otherwise list the name in the Profile's secret_params. Names use letters, digits and underscores.")
|
|
49
|
+
_register("PROFILE_SECRET_VALUE_INVALID", "A Secret parameter value is missing or unusable: it must be non-empty UTF-8 text of at most 64 KiB from a prompt, stdin, an environment variable or a file.")
|
|
50
|
+
_register("PROFILE_SECRET_CONFLICT", "Give each Secret parameter once, by --secret only (not also in the Profile file or patch), and read at most one value from stdin.")
|
|
51
|
+
_register("RUNNER_PACKAGE_INVALID", "The Runner package directory is invalid. It needs runner.toml whose [runner] name matches the Runner and has no version, the [execution] entrypoint main.py or main.sh, at most 100 regular files, no symlinks, and path parts of letters, digits, '_', '.' or '-' that do not start with '.'.")
|
|
52
|
+
_register("RUNNER_REVISION_INVALID", "Use the full lowercase 40- or 64-character source commit as --revision and a NAMESPACE/NAME Runner name.")
|
|
53
|
+
_register("RUNNER_SOURCE_REF_INVALID", "Use a branch or tag name such as main or v1 with --source-ref-kind branch, tag or commit; HEAD, latest, default, release, '..', spaces and ~^:?*[\\ are not allowed.")
|
|
54
|
+
_register("RUNNER_REVISION_CONFLICT", "The platform catalog holds different package or index hashes for this Runner revision. Publish a new revision instead of changing an existing one.")
|
|
48
55
|
_register("DATASET_REQUIRED", "Missing Dataset input (--dataset or --source) for this Agent Job.")
|
|
49
56
|
_register("DATASET_CONFIG_REQUIRED", "An explicit Dataset configuration (--config) is required.")
|
|
50
57
|
_register("RUNNER_REQUIRED", "Missing Runner input (--runner) for this Job.")
|
|
@@ -54,6 +61,7 @@ _register("RATE_LIMITED RATE_LIMIT_EXCEEDED TOO_MANY_REQUESTS", "The API rate li
|
|
|
54
61
|
_register("SERVICE_UNAVAILABLE INTERNAL_SERVER_ERROR EXECUTOR_UNAVAILABLE PROVIDER_UNAVAILABLE LOG_STORAGE_UNAVAILABLE", "The service is unavailable or encountered an internal error.")
|
|
55
62
|
_register("INTERNAL_ERROR", "The command encountered an unexpected error.")
|
|
56
63
|
_register("NETWORK_ERROR", "A connection to the API or download service could not be completed.")
|
|
64
|
+
_register("SOURCE_UNAVAILABLE", "The API gateway did not get a valid answer from the service, for example because it stopped waiting. A write may still have been processed: read the Job or resource before retrying it.")
|
|
57
65
|
_register("REQUEST_TIMEOUT", "The API did not answer before the client timeout. The request may still have been processed: read the Job or resource before retrying a write.")
|
|
58
66
|
_register("RESPONSE_INTERRUPTED", "The connection closed before the API finished answering; the service may have failed or restarted. The request may have been processed: read the Job or resource before retrying a write, and report the time if it repeats.")
|
|
59
67
|
_register("INVALID_RESPONSE RESPONSE_TOO_LARGE INVALID_LOG_PAGE INVALID_PAGINATION TRAIN_JOB_CURSOR_REPEATED", "The service returned an invalid or unsupported response.")
|
|
@@ -114,6 +122,8 @@ _register("INVALID_RSI_REQUEST INVALID_STEER INVALID_DIRECTIVE", "The RSI servic
|
|
|
114
122
|
_register("RSI_CONFLICT RSI_RUN_NOT_ACTIVE", "The run id exists, the run is not in a state that allows this action, or no loop machine is free.")
|
|
115
123
|
_register("RSI_CONTROL_UNCONFIGURED RSI_CONTROL_UNAVAILABLE RSI_CONTROL_TIMEOUT", "The RSI loop service is not available through this platform right now.")
|
|
116
124
|
|
|
125
|
+
_register("UNSUPPORTED_ADAPTER_SOURCE", 'Register a training checkpoint with {"serving_profile": "NAME", "source": {"type": "train_checkpoint", "uri": "envloop://..."}}. '
|
|
126
|
+
"List serving profiles with `el job serving-profile list` and checkpoint paths with `el checkpoint list`.")
|
|
117
127
|
_register("JOB_ACTION_UNSUPPORTED", "This Job type does not support the requested action.")
|
|
118
128
|
_register("INFERENCE_ROUTE_RETIRED", "The global inference route has been retired.")
|
|
119
129
|
|
|
@@ -139,14 +149,14 @@ for _code, _resource in {
|
|
|
139
149
|
_KNOWN_REMOTE_CODES = set("""
|
|
140
150
|
ADAPTER_FILES_MISSING ADAPTER_FORMAT_UNSUPPORTED ADAPTER_RANK_EXCEEDS_PROFILE ADAPTER_TARGET_UNSUPPORTED
|
|
141
151
|
DATASET_SELECTION_UNSUPPORTED INVALID_ADAPTERS INVALID_ADAPTER_ALIAS INVALID_AMOUNT INVALID_ATTEMPTS
|
|
142
|
-
INVALID_BILLING_WINDOW INVALID_CHECKPOINT_PATH INVALID_CLEANUP_RESOURCES INVALID_COMMAND INVALID_DATASET_DELIVERY
|
|
152
|
+
INVALID_BILLING_WINDOW INVALID_CHECKPOINT_PATH INVALID_CHECKPOINT_URI INVALID_CLEANUP_RESOURCES INVALID_COMMAND INVALID_DATASET_DELIVERY
|
|
143
153
|
INVALID_DATASET_INPUT INVALID_DATASET_SELECTOR INVALID_ENDPOINT_NAME INVALID_ENFORCEMENT INVALID_ENTRY_ID
|
|
144
154
|
INVALID_ENTRY_KIND INVALID_ENVHUB_REQUEST INVALID_GROUP_BY INVALID_JOB INVALID_JOB_ID INVALID_JOB_MEMBER
|
|
145
155
|
INVALID_JOB_MODE INVALID_JOB_SIZE INVALID_JOB_STATUS INVALID_JOB_TYPE INVALID_LIMIT INVALID_LOG_CURSOR
|
|
146
156
|
INVALID_LOG_PROCESS INVALID_LOG_QUERY INVALID_LOG_SOURCE INVALID_LOG_STREAM INVALID_PRIORITY
|
|
147
157
|
INVALID_PROFILE_COMBINATION INVALID_PROJECT INVALID_QUOTE INVALID_REPOSITORY_COMMIT INVALID_REPOSITORY_ROOT
|
|
148
158
|
INVALID_REQUEST_ID INVALID_RESOLVED_JOB INVALID_RUNNER INVALID_RUNNER_ENVIRONMENT INVALID_SANDBOX_RUNNER
|
|
149
|
-
INVALID_SCALING INVALID_SCOPE_TTL INVALID_TAG
|
|
159
|
+
INVALID_SCALING INVALID_SCOPE_TTL INVALID_TAG INVALID_TRAINING_LINEAGE INVALID_TRIALS
|
|
150
160
|
INVALID_TRIAL_STATUS INVALID_WORKSPACE JOB_REQUIRED LEADERBOARD_PROVENANCE_REQUIRED LEADERBOARD_SCORE_REQUIRED
|
|
151
161
|
PROFILE_IDENTITY_MISMATCH QUEUE_NOT_FOUND REPOSITORY_BINDING_REQUIRED RUNNER_GPU_UNAVAILABLE RUNNER_HASH_MISMATCH
|
|
152
162
|
TOO_MANY_ADAPTERS UNSUPPORTED_ADAPTER_SOURCE UNSUPPORTED_CAPABILITY UNSUPPORTED_COMMAND UNSUPPORTED_HARDWARE
|
|
@@ -209,6 +219,9 @@ def _http_message(status: int | None) -> str:
|
|
|
209
219
|
return _MESSAGES["INVALID_REQUEST"]
|
|
210
220
|
if status == 429:
|
|
211
221
|
return _MESSAGES["RATE_LIMITED"]
|
|
222
|
+
if status in GATEWAY_TIMEOUT_STATUSES:
|
|
223
|
+
return ("The API gateway stopped waiting for the service. The request may still have been processed: "
|
|
224
|
+
"read the Job or resource before retrying a write.")
|
|
212
225
|
if status is not None and status >= 500:
|
|
213
226
|
return _MESSAGES["SERVICE_UNAVAILABLE"]
|
|
214
227
|
return _MESSAGES["HTTP_ERROR"]
|
|
@@ -294,7 +307,9 @@ def format_error(error: BaseException) -> str:
|
|
|
294
307
|
extra: dict[str, Any] = {"hint": remote.hint, "details": remote.details, "request_id": remote.request_id} if remote else {}
|
|
295
308
|
if remote and remote.message:
|
|
296
309
|
# Local text only adds guidance for codes the CLI documents.
|
|
297
|
-
|
|
310
|
+
# A gateway timeout always adds that the request may have been processed.
|
|
311
|
+
documented = (code in _MESSAGES and code != "HTTP_ERROR") or status in GATEWAY_TIMEOUT_STATUSES
|
|
312
|
+
guidance = local if documented and local != remote.message else None
|
|
298
313
|
rendered = error_block(code, remote.message, status=status, guidance=guidance, **extra)
|
|
299
314
|
else:
|
|
300
315
|
rendered = error_block(code, local, status=status, **extra)
|
envloop/cli_output.py
CHANGED
|
@@ -49,7 +49,7 @@ def next_page(args: argparse.Namespace, cursor: object, program: str) -> str:
|
|
|
49
49
|
if getattr(args, 'no_color', False):
|
|
50
50
|
command.append('--no-color')
|
|
51
51
|
from ._optional import feature_module
|
|
52
|
-
feature = 'data' if args.group in {'data', 'dataset', 'runner', 'checkpoint', 'profile'} else 'experiment' if args.group in {'job', '
|
|
52
|
+
feature = 'data' if args.group in {'data', 'dataset', 'runner', 'checkpoint', 'profile'} else 'experiment' if args.group in {'job', 'trial'} else None
|
|
53
53
|
if feature is not None:
|
|
54
54
|
feature_module(feature, 'cli_output').pagination(command, args, cursor)
|
|
55
55
|
else:
|
|
@@ -59,15 +59,31 @@ def next_page(args: argparse.Namespace, cursor: object, program: str) -> str:
|
|
|
59
59
|
return "Next page: " + shlex.join(command)
|
|
60
60
|
|
|
61
61
|
|
|
62
|
+
# Empty-list text names the listed resource, not the command group (`job adapter list` lists adapters).
|
|
63
|
+
EMPTY_RESOURCES = {
|
|
64
|
+
('job', 'adapter'): 'adapters', ('job', 'serving-profile'): 'serving profiles',
|
|
65
|
+
('trial',): 'trials', ('trial', 'artifact'): 'artifacts',
|
|
66
|
+
('dataset',): 'datasets', ('dataset', 'revision'): 'dataset revisions', ('dataset', 'task'): 'dataset tasks',
|
|
67
|
+
('data',): 'resources', ('data', 'files'): 'files',
|
|
68
|
+
('billing', 'usage'): 'usage records', ('billing', 'ledger'): 'ledger entries', ('billing', 'price'): 'prices',
|
|
69
|
+
('member',): 'members', ('member', 'candidate'): 'member candidates',
|
|
70
|
+
('setting', 'registry'): 'registry credentials',
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def empty_message(args: argparse.Namespace) -> str:
|
|
75
|
+
path = getattr(args, '_command_path', [args.group, 'list'])
|
|
76
|
+
resource = EMPTY_RESOURCES.get(tuple(path[:-1] if path[-1:] == ['list'] else path))
|
|
77
|
+
return f'No {resource or args.group} found.'
|
|
78
|
+
|
|
79
|
+
|
|
62
80
|
def generic_layout(items: list[dict[str, Any]], args: argparse.Namespace, layouts: dict[str, tuple[str, ...]]) -> tuple[tuple[str, ...], Callable[[dict[str, Any]], tuple[object, ...]], str]:
|
|
63
81
|
fields = tuple(field for field in layouts.get(args.group, ()) if any(field in item for item in items))
|
|
64
82
|
fields = fields or tuple(dict.fromkeys(key for item in items for key in item))[:6]
|
|
65
83
|
headers = tuple(field.replace('_', ' ').upper() for field in fields)
|
|
66
84
|
render: Callable[[dict[str, Any]], tuple[object, ...]] = lambda item: tuple(timestamp(item.get(field)) if field.endswith('_at') or field == 'at'
|
|
67
85
|
else item.get(field) for field in fields)
|
|
68
|
-
|
|
69
|
-
else f'No {getattr(args, "resource", args.group)} found.')
|
|
70
|
-
return headers, render, empty
|
|
86
|
+
return headers, render, empty_message(args)
|
|
71
87
|
|
|
72
88
|
|
|
73
89
|
def list_output(result: Any, args: argparse.Namespace, program: str) -> str:
|
|
@@ -79,7 +95,7 @@ def list_output(result: Any, args: argparse.Namespace, program: str) -> str:
|
|
|
79
95
|
items = [{'name': key, 'value': value} for key, value in items.items()]
|
|
80
96
|
from ._optional import feature_module
|
|
81
97
|
feature = ('rsi-loop' if args.group == 'job' and args.action == 'run' else
|
|
82
|
-
'experiment' if args.group in {'job', '
|
|
98
|
+
'experiment' if args.group in {'job', 'trial'} else
|
|
83
99
|
'data' if args.group in {'data', 'dataset', 'runner', 'checkpoint', 'profile'} else None)
|
|
84
100
|
if feature is not None:
|
|
85
101
|
headers, render, empty = feature_module(feature, 'cli_output').list_layout(items, args)
|
envloop/client.py
CHANGED
|
@@ -33,9 +33,10 @@ class Client:
|
|
|
33
33
|
raise RuntimeError(INTERACTIVE_TRAINING_REMOVED)
|
|
34
34
|
|
|
35
35
|
@classmethod
|
|
36
|
-
def remote(cls, api_url: str | None = None, *, timeout: float =
|
|
36
|
+
def remote(cls, api_url: str | None = None, *, timeout: float = 90, api_key: str | None = None, workspace_id: str | None = None, project_id: str | None = None, config: bool = True):
|
|
37
37
|
from .http import HttpService
|
|
38
38
|
|
|
39
|
+
# The default matches http.DEFAULT_TIMEOUT_SEC (longer than the hosted proxy's 60 s wait).
|
|
39
40
|
return cls(HttpService(api_url, timeout, api_key=api_key, workspace_id=workspace_id, project_id=project_id, config=config))
|
|
40
41
|
|
|
41
42
|
@classmethod
|
envloop/http.py
CHANGED
|
@@ -27,6 +27,14 @@ def validate_public_execution(value: Any) -> None:
|
|
|
27
27
|
validate_public_execution(value.get(key))
|
|
28
28
|
|
|
29
29
|
|
|
30
|
+
class WaitTimeoutError(TimeoutError):
|
|
31
|
+
"""A wait ended before the terminal state; ``last`` is the last resource or Operation read."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, message: str, last: dict[str, Any] | None = None):
|
|
34
|
+
super().__init__(message)
|
|
35
|
+
self.last = last
|
|
36
|
+
|
|
37
|
+
|
|
30
38
|
class RemoteError(RuntimeError):
|
|
31
39
|
"""Platform failure. Server-supplied text lives in bounded attributes, never in str()."""
|
|
32
40
|
|
|
@@ -63,6 +71,12 @@ def retry_after_seconds(value: str | None) -> float | None:
|
|
|
63
71
|
return None
|
|
64
72
|
|
|
65
73
|
|
|
74
|
+
# The hosted API proxy waits up to 60 s for the platform. Wait longer by default so a slow answer, or the
|
|
75
|
+
# proxy's own gateway timeout, reaches the caller instead of a client-side timeout. Explicit values are kept.
|
|
76
|
+
DEFAULT_TIMEOUT_SEC = 90.0
|
|
77
|
+
# The proxy or edge stopped waiting while the platform was still working: the request may have been processed.
|
|
78
|
+
GATEWAY_TIMEOUT_STATUSES = frozenset({504, 524})
|
|
79
|
+
|
|
66
80
|
# Transport failures that are not an HTTP answer. A request that was sent and then timed out or lost its
|
|
67
81
|
# response may still have been processed, unlike one that never connected.
|
|
68
82
|
TRANSIENT_CODES = frozenset({"NETWORK_ERROR", "REQUEST_TIMEOUT", "RESPONSE_INTERRUPTED"})
|
|
@@ -213,7 +227,7 @@ def encode_json(body: Any) -> bytes:
|
|
|
213
227
|
|
|
214
228
|
|
|
215
229
|
class HttpService:
|
|
216
|
-
def __init__(self, base_url: str | None = None, timeout: float =
|
|
230
|
+
def __init__(self, base_url: str | None = None, timeout: float = DEFAULT_TIMEOUT_SEC, *, api_key: str | None = None, workspace_id: str | None = None, project_id: str | None = None, config: bool = True, environment: bool = True):
|
|
217
231
|
from .config import require_remote, resolve_connection
|
|
218
232
|
|
|
219
233
|
connection = resolve_connection(base_url, api_key, workspace_id, project_id, config=config, environment=environment)
|
|
@@ -233,9 +247,9 @@ class HttpService:
|
|
|
233
247
|
self.opener = build_client_opener()
|
|
234
248
|
|
|
235
249
|
def remote_error(self, error: HTTPError, body: Any = None, **options: Any) -> RemoteError:
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
fields = error_fields(error, (self.secret, *
|
|
250
|
+
from .profile_secrets import submitted_secrets
|
|
251
|
+
|
|
252
|
+
fields = error_fields(error, (self.secret, *submitted_secrets(body)))
|
|
239
253
|
return RemoteError(fields.pop("code"), error.code, **fields, **options)
|
|
240
254
|
|
|
241
255
|
def request(self, method: str, path: str, body: Any = None, *, headers: dict[str, str] | None = None) -> Any:
|
envloop/lifecycle_cli.py
CHANGED
|
@@ -7,7 +7,7 @@ from typing import Any
|
|
|
7
7
|
|
|
8
8
|
def wait_options(parser: argparse.ArgumentParser):
|
|
9
9
|
parser.add_argument('--wait', action='store_true', help='Wait for the persisted cleanup operation')
|
|
10
|
-
parser.add_argument('--timeout-
|
|
10
|
+
parser.add_argument('--timeout-seconds', type=float, default=600)
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
def add_management_commands(settings: "argparse._SubParsersAction[argparse.ArgumentParser]"):
|
|
@@ -16,7 +16,7 @@ def add_management_commands(settings: "argparse._SubParsersAction[argparse.Argum
|
|
|
16
16
|
command = operations.add_parser(action)
|
|
17
17
|
command.add_argument('identifier')
|
|
18
18
|
if action == 'wait':
|
|
19
|
-
command.add_argument('--timeout-
|
|
19
|
+
command.add_argument('--timeout-seconds', type=float, default=600)
|
|
20
20
|
cleanup = settings.add_parser('cleanup', help='Clean an explicit set of project resource IDs')
|
|
21
21
|
cleanup.add_argument('--file', type=Path, required=True, help='JSON array of resource kind and ID objects')
|
|
22
22
|
cleanup.add_argument('--request-id', required=True)
|
|
@@ -38,7 +38,7 @@ def run_management(client: Any, args: argparse.Namespace):
|
|
|
38
38
|
from .resources import _id
|
|
39
39
|
|
|
40
40
|
if args.action == 'operation':
|
|
41
|
-
return (client.operations.wait(args.identifier, timeout=args.
|
|
41
|
+
return (client.operations.wait(args.identifier, timeout=args.timeout_seconds) if args.operation_action == 'wait'
|
|
42
42
|
else getattr(client.operations, args.operation_action)(args.identifier))
|
|
43
43
|
if args.action == 'resource-scope':
|
|
44
44
|
if args.scope_action == 'create':
|
|
@@ -52,4 +52,4 @@ def run_management(client: Any, args: argparse.Namespace):
|
|
|
52
52
|
if args.scope_action == 'retain' else client._transport.request('DELETE' if args.scope_action == 'cleanup' else 'GET', path))
|
|
53
53
|
else:
|
|
54
54
|
result = client.operations.cleanup(json.loads(args.file.read_text()), request_id=args.request_id)
|
|
55
|
-
return client.operations.wait(result['operation_id'], timeout=args.
|
|
55
|
+
return client.operations.wait(result['operation_id'], timeout=args.timeout_seconds) if getattr(args, 'wait', False) else result
|
envloop/navigation_resources.py
CHANGED
|
@@ -33,7 +33,14 @@ class Billing:
|
|
|
33
33
|
start, end = billing_timestamp(start), billing_timestamp(end)
|
|
34
34
|
if datetime.fromisoformat(end) <= datetime.fromisoformat(start):
|
|
35
35
|
raise ValueError('billing end must be after start')
|
|
36
|
-
|
|
36
|
+
# Workspace usage needs a human billing grant. An API key reads its own project's usage on the
|
|
37
|
+
# project route, which takes the same window and grouping.
|
|
38
|
+
identity = self.transport.request('GET', '/v1/me')
|
|
39
|
+
identity = identity if isinstance(identity, dict) else {}
|
|
40
|
+
project = identity.get('project_id') or (identity.get('project') or {}).get('project_id')
|
|
41
|
+
path = (f'/v1/projects/{quote(project, safe="")}/billing/usage'
|
|
42
|
+
if identity.get('auth_method') == 'api_key' and isinstance(project, str) and project else '/v1/billing/usage')
|
|
43
|
+
return self.transport.request('GET', query(path, start=start, end=end, group_by=group_by))
|
|
37
44
|
|
|
38
45
|
def ledger(self, *, limit: int = 25):
|
|
39
46
|
return self.transport.request('GET', query('/v1/billing/ledger', limit=limit))
|
|
@@ -119,6 +126,30 @@ class Project:
|
|
|
119
126
|
raise ValueError('enabled must be a boolean')
|
|
120
127
|
return self._settings_write('/compose', {'enabled': enabled}, expected_version)
|
|
121
128
|
|
|
129
|
+
def model_access(self):
|
|
130
|
+
"""Model access grants of this project and whether the platform can deliver them now."""
|
|
131
|
+
return self.execution_settings()['model_access']
|
|
132
|
+
|
|
133
|
+
def grant_model_access(self, runner: str, protocols: list[str], *, revision: str | None = None,
|
|
134
|
+
expected_version: int | str | None = None):
|
|
135
|
+
"""Let one published Runner (optionally one full commit Revision) receive the project's metered model
|
|
136
|
+
key for ``protocols`` (``anthropic``, ``openai``). Returns the settings view with the new ``grant``."""
|
|
137
|
+
validate_model_access_grant(runner, protocols, revision)
|
|
138
|
+
return self._settings_send('POST', '/model-access', {'runner': runner, 'revision': revision,
|
|
139
|
+
'protocols': list(protocols)}, expected_version)
|
|
140
|
+
|
|
141
|
+
def revoke_model_access(self, grant_id: str, *, expected_version: int | str | None = None):
|
|
142
|
+
"""Revoke a grant; Trials that have not started no longer receive the key."""
|
|
143
|
+
if not isinstance(grant_id, str) or not grant_id:
|
|
144
|
+
raise ValueError('grant_id is required')
|
|
145
|
+
return self._settings_send('DELETE', '/model-access/' + quote(grant_id, safe=''), None, expected_version)
|
|
146
|
+
|
|
147
|
+
def _settings_send(self, method: str, suffix: str, body: dict[str, Any] | None, expected_version: int | str | None):
|
|
148
|
+
path = self.path() + '/execution-settings'
|
|
149
|
+
if expected_version is None:
|
|
150
|
+
expected_version = self.transport.request('GET', path)['version']
|
|
151
|
+
return self.transport.request(method, path + suffix, body, headers={'If-Match': f'"{expected_version}"'})
|
|
152
|
+
|
|
122
153
|
def set_max_concurrent_trials(self, value: int | None, *, expected_version: int | str | None = None):
|
|
123
154
|
"""Lower the project's concurrent Trial limit (None clears it); the operator ceiling still applies."""
|
|
124
155
|
if value is not None and (type(value) is not int or not 1 <= value <= 10000):
|
|
@@ -126,6 +157,23 @@ class Project:
|
|
|
126
157
|
return self._settings_write('/concurrency', {'max_concurrent_trials': value}, expected_version)
|
|
127
158
|
|
|
128
159
|
|
|
160
|
+
MODEL_ACCESS_PROTOCOLS = ('anthropic', 'openai')
|
|
161
|
+
# Copied from EnvPlatform env_platform/project_settings.py RUNNER / REVISION (keep in sync).
|
|
162
|
+
_RUNNER_NAME = re.compile(r'[A-Za-z0-9][A-Za-z0-9._-]{0,127}/[A-Za-z0-9][A-Za-z0-9._-]{0,127}')
|
|
163
|
+
_REVISION = re.compile(r'[0-9a-f]{40}|[0-9a-f]{64}')
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def validate_model_access_grant(runner: str, protocols: Any, revision: str | None) -> None:
|
|
167
|
+
"""The same shape the platform accepts (EnvPlatform project_settings): checked before any request."""
|
|
168
|
+
if not isinstance(runner, str) or not _RUNNER_NAME.fullmatch(runner):
|
|
169
|
+
raise ValueError('runner must be namespace/name')
|
|
170
|
+
if (not isinstance(protocols, (list, tuple)) or not protocols or len(set(protocols)) != len(protocols)
|
|
171
|
+
or any(protocol not in MODEL_ACCESS_PROTOCOLS for protocol in protocols)):
|
|
172
|
+
raise ValueError('protocols must be anthropic, openai or both')
|
|
173
|
+
if revision is not None and (not isinstance(revision, str) or not _REVISION.fullmatch(revision)):
|
|
174
|
+
raise ValueError('revision must be a full 40- or 64-character lowercase commit')
|
|
175
|
+
|
|
176
|
+
|
|
129
177
|
class Dashboard:
|
|
130
178
|
def __init__(self, project: Any):
|
|
131
179
|
self.project = project
|
envloop/profile_secrets.py
CHANGED
|
@@ -1,13 +1,29 @@
|
|
|
1
|
-
"""Client validation for write-only Profile
|
|
2
|
-
|
|
1
|
+
"""Client validation for write-only Profile secrets.
|
|
2
|
+
|
|
3
|
+
Two kinds exist: the encrypted model connection key (`secrets.model_api_key`) and
|
|
4
|
+
Secret parameters, which the platform stores with the version but redacts on every
|
|
5
|
+
read and restores at dispatch.
|
|
6
|
+
"""
|
|
7
|
+
import copy
|
|
8
|
+
import re
|
|
9
|
+
from collections.abc import Callable, Iterable, Mapping
|
|
3
10
|
from typing import Any
|
|
4
11
|
|
|
12
|
+
PROFILE_V2 = "envplatform.profile/v2"
|
|
13
|
+
# Names that always hold credentials. Mirrors EnvPlatform SENSITIVE_PROFILE_KEY in
|
|
14
|
+
# env_platform/profile_secrets.py and frontend/shared/profile-secrets.ts; keep all three identical.
|
|
15
|
+
SENSITIVE_PROFILE_KEY = re.compile(
|
|
16
|
+
r'api[_-]?key$|authorization|auth[_-]?token|access[_-]?token|bearer[_-]?token|password|secret|credential|'
|
|
17
|
+
r'private[_-]?key|(?:^|_)token$', re.I)
|
|
18
|
+
PARAMETER_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
19
|
+
|
|
5
20
|
|
|
6
21
|
class InvalidProfileSecrets(ValueError):
|
|
7
22
|
code = "INVALID_PROFILE_SECRETS"
|
|
8
23
|
|
|
9
|
-
def __init__(self):
|
|
10
|
-
|
|
24
|
+
def __init__(self, code: str = "INVALID_PROFILE_SECRETS"):
|
|
25
|
+
self.code = code
|
|
26
|
+
super().__init__(code)
|
|
11
27
|
|
|
12
28
|
|
|
13
29
|
def profile_payload(spec: Mapping[str, Any], secrets: Mapping[str, Any] | None = None) -> dict[str, Any]:
|
|
@@ -26,3 +42,92 @@ def profile_payload(spec: Mapping[str, Any], secrets: Mapping[str, Any] | None =
|
|
|
26
42
|
raise InvalidProfileSecrets()
|
|
27
43
|
result["secrets"] = dict(secrets)
|
|
28
44
|
return result
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _params(spec: Mapping[str, Any]) -> Any:
|
|
48
|
+
return spec.get("spec", {}).get("params") if spec.get("schema_version") == PROFILE_V2 else spec.get("params")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _literal(key: str) -> str:
|
|
52
|
+
return key
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _fold(spec: Mapping[str, Any]) -> Callable[[str], str]:
|
|
56
|
+
# Profile v2 parameter names are case insensitive; v1 keys are literal.
|
|
57
|
+
return str.upper if spec.get("schema_version") == PROFILE_V2 else _literal
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def is_secret_param(spec: Mapping[str, Any], key: str) -> bool:
|
|
61
|
+
"""The server's rule for params: a credential-shaped name or a name listed in secret_params."""
|
|
62
|
+
return bool(SENSITIVE_PROFILE_KEY.search(key)) or key in (spec.get("secret_params") or ())
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def check_secret_values(values: Any) -> dict[str, str]:
|
|
66
|
+
if not isinstance(values, Mapping) or not values:
|
|
67
|
+
raise InvalidProfileSecrets("PROFILE_SECRET_VALUE_INVALID")
|
|
68
|
+
for name, value in values.items():
|
|
69
|
+
if not isinstance(name, str) or not PARAMETER_NAME.fullmatch(name):
|
|
70
|
+
raise InvalidProfileSecrets("PROFILE_SECRET_NAME_INVALID")
|
|
71
|
+
if not isinstance(value, str) or not value or "\x00" in value or len(value.encode("utf-8")) > 65536:
|
|
72
|
+
raise InvalidProfileSecrets("PROFILE_SECRET_VALUE_INVALID")
|
|
73
|
+
return dict(values)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def reject_supplied(spec: Mapping[str, Any], names: Iterable[str]):
|
|
77
|
+
"""A Secret parameter is given once: by --secret / secret_values, not also by the file or patch."""
|
|
78
|
+
params = _params(spec)
|
|
79
|
+
fold = _fold(spec)
|
|
80
|
+
present = {fold(key) for key in params} if isinstance(params, Mapping) else set()
|
|
81
|
+
if any(fold(name) in present for name in names):
|
|
82
|
+
raise InvalidProfileSecrets("PROFILE_SECRET_CONFLICT")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def with_secret_values(spec: Mapping[str, Any], values: Mapping[str, str]) -> dict[str, Any]:
|
|
86
|
+
"""Return a copy with each value set as a marked Secret parameter; existing values of that name are replaced.
|
|
87
|
+
|
|
88
|
+
A name qualifies under the server's rule (`is_secret_param`): credential-shaped,
|
|
89
|
+
or already listed in secret_params. Qualifying names are added to secret_params,
|
|
90
|
+
which the public API requires for credential-named parameters.
|
|
91
|
+
"""
|
|
92
|
+
values = check_secret_values(values)
|
|
93
|
+
result = copy.deepcopy(dict(spec))
|
|
94
|
+
fold = _fold(result)
|
|
95
|
+
if len({fold(name) for name in values}) != len(values):
|
|
96
|
+
raise InvalidProfileSecrets("PROFILE_SECRET_CONFLICT")
|
|
97
|
+
if result.get("schema_version") == PROFILE_V2:
|
|
98
|
+
body = result.setdefault("spec", {})
|
|
99
|
+
params = body.setdefault("params", {}) if isinstance(body, dict) else None
|
|
100
|
+
else:
|
|
101
|
+
params = result.setdefault("params", {})
|
|
102
|
+
if not isinstance(params, dict):
|
|
103
|
+
raise InvalidProfileSecrets()
|
|
104
|
+
markers = result.get("secret_params", [])
|
|
105
|
+
if not isinstance(markers, list):
|
|
106
|
+
raise InvalidProfileSecrets()
|
|
107
|
+
for name, value in values.items():
|
|
108
|
+
# Keep the spelling the Profile marks as Secret, as the server merge does.
|
|
109
|
+
key = next((marker for marker in markers if isinstance(marker, str) and fold(marker) == fold(name)), name)
|
|
110
|
+
if not is_secret_param(result, key):
|
|
111
|
+
raise InvalidProfileSecrets("PROFILE_SECRET_NAME_INVALID")
|
|
112
|
+
for existing in [item for item in params if fold(item) == fold(name)]:
|
|
113
|
+
del params[existing]
|
|
114
|
+
params[key] = value
|
|
115
|
+
# Mark it too: the public API rejects an unmarked credential-named parameter.
|
|
116
|
+
if key not in markers:
|
|
117
|
+
markers.append(key)
|
|
118
|
+
result["secret_params"] = markers
|
|
119
|
+
return result
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def submitted_secrets(body: Any) -> list[str]:
|
|
123
|
+
"""Secret strings in a Profile request body, so error text echoing them can be redacted."""
|
|
124
|
+
if not isinstance(body, dict):
|
|
125
|
+
return []
|
|
126
|
+
values = [value for value in (body.get("secrets") or {}).values()] if isinstance(body.get("secrets"), dict) else []
|
|
127
|
+
sections = ([body["spec"].get(name) for name in ("params", "env")] if isinstance(body.get("spec"), dict)
|
|
128
|
+
and body.get("schema_version") == PROFILE_V2 else [body.get("params")])
|
|
129
|
+
for index, section in enumerate(sections):
|
|
130
|
+
if isinstance(section, dict):
|
|
131
|
+
values += [value for key, value in section.items()
|
|
132
|
+
if isinstance(key, str) and (is_secret_param(body, key) if index == 0 else SENSITIVE_PROFILE_KEY.search(key))]
|
|
133
|
+
return [value for value in values if isinstance(value, str) and value]
|