dataspring-cli 0.5.1__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/PKG-INFO +1 -1
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/bundled_manifest.py +2 -1
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/contract.py +126 -4
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/generated.py +68 -8
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/main.py +282 -3
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/refusals.py +28 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/upgrade.py +6 -3
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/pyproject.toml +11 -1
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/settings.py +20 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/skills/dataspring-author/SKILL.md +10 -1
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/skills/dataspring-consume/SKILL.md +2 -2
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/.gitignore +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/README.md +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/__init__.py +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/auth.py +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/forspring.py +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/output.py +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/runtime.py +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/skills_commands.py +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/skilltree.py +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/transport.py +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/version.py +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/hatch_build.py +0 -0
- {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/skills/dataspring-correct/SKILL.md +0 -0
|
@@ -6,7 +6,7 @@ the server's to say whether the server has operations this CLI does not
|
|
|
6
6
|
know. tests/test_skills_surface.py asserts it is current.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
|
-
MANIFEST_HASH = "
|
|
9
|
+
MANIFEST_HASH = "a58c0bb7f14250cf69aae9f9459806f8d9369a289b1760910de915a209e9aad1"
|
|
10
10
|
TOOL_KEYS = (
|
|
11
11
|
"business_context_edit",
|
|
12
12
|
"semantic_model_edit",
|
|
@@ -31,4 +31,5 @@ TOOL_KEYS = (
|
|
|
31
31
|
"board_edit",
|
|
32
32
|
"board_fetch",
|
|
33
33
|
"watch_edit",
|
|
34
|
+
"python_run",
|
|
34
35
|
)
|
|
@@ -24,7 +24,7 @@ from __future__ import annotations
|
|
|
24
24
|
|
|
25
25
|
from typing import Annotated, Any, Literal
|
|
26
26
|
|
|
27
|
-
from pydantic import BaseModel, ConfigDict, Field
|
|
27
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
class DispatchParams(BaseModel):
|
|
@@ -528,7 +528,7 @@ class AddExternalWarehouseAction(DispatchParams):
|
|
|
528
528
|
data project and is only named here (``key_secret``)."""
|
|
529
529
|
|
|
530
530
|
action: Literal["add_external"]
|
|
531
|
-
label: str = Field(description="Display name, e.g. \"
|
|
531
|
+
label: str = Field(description="Display name, e.g. \"Acme's BigQuery\"")
|
|
532
532
|
project: str = Field(description="The customer's GCP project id")
|
|
533
533
|
dataset: str = Field(description="The dataset the semantic manifest points at")
|
|
534
534
|
location: str = Field(description="BigQuery location of that dataset, e.g. 'europe-north2'")
|
|
@@ -1172,7 +1172,7 @@ class SetSecretAction(DispatchParams):
|
|
|
1172
1172
|
name: str = Field(
|
|
1173
1173
|
description=(
|
|
1174
1174
|
"The secret to set or rotate: t-<org>-<connection>-<field> for your "
|
|
1175
|
-
"organization, lower-case (t-acme-kanpla-
|
|
1175
|
+
"organization, lower-case (t-acme-kanpla-api-key, "
|
|
1176
1176
|
"t-acme-external-bigquery-key). A connection file names it."
|
|
1177
1177
|
)
|
|
1178
1178
|
)
|
|
@@ -1225,7 +1225,12 @@ class RunSqlParams(DispatchParams):
|
|
|
1225
1225
|
"organization id with every hyphen as an underscore (acme-dk is "
|
|
1226
1226
|
"acme_dk_marts); the identity it runs as can read those five datasets of "
|
|
1227
1227
|
"its own org and nothing else, so a write to a dataset or a reach outside "
|
|
1228
|
-
"them is refused by the warehouse."
|
|
1228
|
+
"them is refused by the warehouse. On an external warehouse (the "
|
|
1229
|
+
"customer's own key) only ONE read statement runs: SELECT, WITH ... "
|
|
1230
|
+
"SELECT, EXPLAIN, DESCRIBE/SHOW or an INFORMATION_SCHEMA select, in "
|
|
1231
|
+
"that warehouse's dialect; a script, DML, DDL, MERGE, CALL, EXPORT "
|
|
1232
|
+
"DATA, LOAD DATA, GRANT, SET, SELECT ... INTO or unparseable SQL is "
|
|
1233
|
+
"refused as statement_not_allowed."
|
|
1229
1234
|
)
|
|
1230
1235
|
)
|
|
1231
1236
|
warehouse: str | None = Field(
|
|
@@ -1248,6 +1253,123 @@ class RunSqlParams(DispatchParams):
|
|
|
1248
1253
|
)
|
|
1249
1254
|
|
|
1250
1255
|
|
|
1256
|
+
# ---------------------------------------------------------------------------
|
|
1257
|
+
# python_run: hosted Python over the org's own data (epic kfh, tick r2v)
|
|
1258
|
+
# ---------------------------------------------------------------------------
|
|
1259
|
+
|
|
1260
|
+
#: The caps every surface states and the service enforces (epic kfh).
|
|
1261
|
+
PYTHON_MAX_CODE_BYTES = 256 * 1024
|
|
1262
|
+
PYTHON_MAX_INPUTS = 8
|
|
1263
|
+
PYTHON_DEFAULT_TIMEOUT_SECONDS = 120
|
|
1264
|
+
PYTHON_MAX_TIMEOUT_SECONDS = 600
|
|
1265
|
+
PYTHON_INPUT_NAME_PATTERN = r"^[a-z_][a-z0-9_]{0,63}$"
|
|
1266
|
+
|
|
1267
|
+
|
|
1268
|
+
class PythonInput(DispatchParams):
|
|
1269
|
+
"""One input of a run: a table or a read-only SQL, fetched as the org's
|
|
1270
|
+
read-only identity before the code runs and handed to it as Parquet."""
|
|
1271
|
+
|
|
1272
|
+
name: str = Field(
|
|
1273
|
+
pattern=PYTHON_INPUT_NAME_PATTERN,
|
|
1274
|
+
description=(
|
|
1275
|
+
"What the code reads it as: /work/in/<name>.parquet (the directory is also "
|
|
1276
|
+
"os.environ['DATASPRING_INPUTS']). Lower-case letters, digits and '_'."
|
|
1277
|
+
),
|
|
1278
|
+
)
|
|
1279
|
+
table: str | None = Field(
|
|
1280
|
+
default=None,
|
|
1281
|
+
description=(
|
|
1282
|
+
"A table or view of your organization, as <org>_<layer>.<table> (acme_marts.fct_orders), "
|
|
1283
|
+
"<layer>.<table> (marts.fct_orders; layers marts, core, staging, raw, dev) or a bare table "
|
|
1284
|
+
"name in marts. Exactly one of table and sql."
|
|
1285
|
+
),
|
|
1286
|
+
)
|
|
1287
|
+
sql: str | None = Field(
|
|
1288
|
+
default=None,
|
|
1289
|
+
description=(
|
|
1290
|
+
"A read-only SQL statement whose rows are the input (the same rules as run_sql: it runs "
|
|
1291
|
+
"as the reader identity, dry-run first and byte-capped). Exactly one of table and sql."
|
|
1292
|
+
),
|
|
1293
|
+
)
|
|
1294
|
+
|
|
1295
|
+
@model_validator(mode="after")
|
|
1296
|
+
def _one_source(self) -> "PythonInput":
|
|
1297
|
+
if bool(self.table and self.table.strip()) == bool(self.sql and self.sql.strip()):
|
|
1298
|
+
raise ValueError(f"input {self.name}: give exactly one of table and sql")
|
|
1299
|
+
return self
|
|
1300
|
+
|
|
1301
|
+
|
|
1302
|
+
class RunPythonAction(DispatchParams):
|
|
1303
|
+
"""Run Python over the org's data in an isolated sandbox (member)."""
|
|
1304
|
+
|
|
1305
|
+
action: Literal["run"]
|
|
1306
|
+
code: str = Field(
|
|
1307
|
+
max_length=PYTHON_MAX_CODE_BYTES,
|
|
1308
|
+
description=(
|
|
1309
|
+
"The Python program, run as /work/code/main.py with /work as the working directory. "
|
|
1310
|
+
"Read inputs from /work/in/<name>.parquet (pandas.read_parquet); write files to "
|
|
1311
|
+
"/work/out/ (png, csv, json, parquet, txt; at most 20, 10 MB each); print what the "
|
|
1312
|
+
"answer needs (stdout and stderr come back, 1 MB each). No network, no credentials."
|
|
1313
|
+
),
|
|
1314
|
+
)
|
|
1315
|
+
inputs: list[PythonInput] = Field(
|
|
1316
|
+
default_factory=list,
|
|
1317
|
+
max_length=PYTHON_MAX_INPUTS,
|
|
1318
|
+
description=(
|
|
1319
|
+
f"Up to {PYTHON_MAX_INPUTS} inputs, each {{name, table}} or {{name, sql}}. All of them "
|
|
1320
|
+
"together at most 2,000,000 rows and 500 MB, checked before anything runs; aggregate "
|
|
1321
|
+
"in SQL when the raw rows are more than the analysis needs."
|
|
1322
|
+
),
|
|
1323
|
+
)
|
|
1324
|
+
timeout_seconds: int = Field(
|
|
1325
|
+
default=PYTHON_DEFAULT_TIMEOUT_SECONDS,
|
|
1326
|
+
ge=1,
|
|
1327
|
+
le=PYTHON_MAX_TIMEOUT_SECONDS,
|
|
1328
|
+
description=(
|
|
1329
|
+
f"Wall-clock limit for the code (default {PYTHON_DEFAULT_TIMEOUT_SECONDS}, at most "
|
|
1330
|
+
f"{PYTHON_MAX_TIMEOUT_SECONDS}). For now at most 45: longer runs are refused until they "
|
|
1331
|
+
"run as their own job. A longer run will answer a run_id to poll with status."
|
|
1332
|
+
),
|
|
1333
|
+
)
|
|
1334
|
+
warehouse: str | None = Field(
|
|
1335
|
+
default=None,
|
|
1336
|
+
description="One of the org's warehouses (see dataspring://warehouses); omit for the active one.",
|
|
1337
|
+
)
|
|
1338
|
+
|
|
1339
|
+
|
|
1340
|
+
class PythonRunStatusAction(DispatchParams):
|
|
1341
|
+
"""Where a run is: its status, exit code, CPU and wall seconds, files."""
|
|
1342
|
+
|
|
1343
|
+
action: Literal["status"]
|
|
1344
|
+
run_id: str = Field(description="The run, from run's answer or list")
|
|
1345
|
+
|
|
1346
|
+
|
|
1347
|
+
class PythonRunResultAction(DispatchParams):
|
|
1348
|
+
"""A finished run's whole answer: stdout, stderr and the file links."""
|
|
1349
|
+
|
|
1350
|
+
action: Literal["result"]
|
|
1351
|
+
run_id: str = Field(description="The run, from run's answer or list")
|
|
1352
|
+
|
|
1353
|
+
|
|
1354
|
+
class ListPythonRunsAction(DispatchParams):
|
|
1355
|
+
"""The org's recent runs, newest first, and its allowance this month."""
|
|
1356
|
+
|
|
1357
|
+
action: Literal["list"]
|
|
1358
|
+
limit: int = Field(default=20, ge=1, le=100, description="Runs to list (newest first; default 20)")
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
PythonRunAction = Annotated[
|
|
1362
|
+
RunPythonAction | PythonRunStatusAction | PythonRunResultAction | ListPythonRunsAction,
|
|
1363
|
+
Field(discriminator="action"),
|
|
1364
|
+
]
|
|
1365
|
+
|
|
1366
|
+
|
|
1367
|
+
class PythonRunParams(DispatchParams):
|
|
1368
|
+
"""The ``python_run`` call: one ``action`` from the family's union."""
|
|
1369
|
+
|
|
1370
|
+
action: PythonRunAction
|
|
1371
|
+
|
|
1372
|
+
|
|
1251
1373
|
# ---------------------------------------------------------------------------
|
|
1252
1374
|
# Tenant lifecycle action types (D24)
|
|
1253
1375
|
# ---------------------------------------------------------------------------
|
|
@@ -55,6 +55,7 @@ MODELS: dict[str, type[contract.DispatchParams]] = {
|
|
|
55
55
|
'board_edit': contract.BoardEditParams,
|
|
56
56
|
'board_fetch': contract.BoardFetchParams,
|
|
57
57
|
'watch_edit': contract.WatchEditParams,
|
|
58
|
+
'python_run': contract.PythonRunParams,
|
|
58
59
|
}
|
|
59
60
|
|
|
60
61
|
#: The verbs of every edit family, in the order the action union declares them.
|
|
@@ -73,6 +74,7 @@ VERBS: dict[str, tuple[str, ...]] = {
|
|
|
73
74
|
'tenant_edit': ('create', 'suspend', 'resume', 'offboard'),
|
|
74
75
|
'board_edit': ('apply', 'patch', 'delete', 'convert_dashboard'),
|
|
75
76
|
'watch_edit': ('apply', 'patch', 'delete', 'rotate_secret'),
|
|
77
|
+
'python_run': ('run', 'status', 'result', 'list'),
|
|
76
78
|
}
|
|
77
79
|
|
|
78
80
|
|
|
@@ -482,7 +484,7 @@ def warehouse_edit__activate(
|
|
|
482
484
|
|
|
483
485
|
@warehouse_edit.command('add_external', short_help='Register a customer-owned BigQuery project as an external warehouse.')
|
|
484
486
|
def warehouse_edit__add_external(
|
|
485
|
-
label: Annotated[str, typer.Option('--label', help='Display name, e.g. "
|
|
487
|
+
label: Annotated[str, typer.Option('--label', help='Display name, e.g. "Acme\'s BigQuery"')],
|
|
486
488
|
project: Annotated[str, typer.Option('--project', help="The customer's GCP project id")],
|
|
487
489
|
dataset: Annotated[str, typer.Option('--dataset', help='The dataset the semantic manifest points at')],
|
|
488
490
|
location: Annotated[str, typer.Option('--location', help="BigQuery location of that dataset, e.g. 'europe-north2'")],
|
|
@@ -749,7 +751,7 @@ secret_edit = typer.Typer(name='secret_edit', help='Customer secrets, both direc
|
|
|
749
751
|
|
|
750
752
|
@secret_edit.command('set', short_help='Get a one-time link where a PERSON pastes a credential. The value never comes through here; the same name rotates a credential.')
|
|
751
753
|
def secret_edit__set(
|
|
752
|
-
name: Annotated[str, typer.Option('--name', help='The secret to set or rotate: t-<org>-<connection>-<field> for your organization, lower-case (t-acme-kanpla-
|
|
754
|
+
name: Annotated[str, typer.Option('--name', help='The secret to set or rotate: t-<org>-<connection>-<field> for your organization, lower-case (t-acme-kanpla-api-key, t-acme-external-bigquery-key). A connection file names it.')],
|
|
753
755
|
from_file: FromFile = None,
|
|
754
756
|
output: Output = None,
|
|
755
757
|
) -> None:
|
|
@@ -777,7 +779,7 @@ def secret_edit__reveal(
|
|
|
777
779
|
|
|
778
780
|
|
|
779
781
|
def run_sql(
|
|
780
|
-
sql: Annotated[str, typer.Option('--sql', help="One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse.")],
|
|
782
|
+
sql: Annotated[str, typer.Option('--sql', help="One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse. On an external warehouse (the customer's own key) only ONE read statement runs: SELECT, WITH ... SELECT, EXPLAIN, DESCRIBE/SHOW or an INFORMATION_SCHEMA select, in that warehouse's dialect; a script, DML, DDL, MERGE, CALL, EXPORT DATA, LOAD DATA, GRANT, SET, SELECT ... INTO or unparseable SQL is refused as statement_not_allowed.")],
|
|
781
783
|
warehouse: Annotated[Optional[str], typer.Option('--warehouse', help="One of the org's warehouses (see dataspring://warehouses); omit for the active one.")] = None,
|
|
782
784
|
max_rows: Annotated[Optional[int], typer.Option('--max-rows', help='Rows to return at most (default 1000, cap 10000). Aggregate in SQL rather than paging. (default: 1000)')] = None,
|
|
783
785
|
format: Annotated[Optional[str], typer.Option('--format', help="'json' answers rows in `data`; 'csv' or 'markdown' answer one document in `content`. (one of: json, csv, markdown) (default: json)")] = None,
|
|
@@ -785,8 +787,9 @@ def run_sql(
|
|
|
785
787
|
from_file: FromFile = None,
|
|
786
788
|
output: Output = None,
|
|
787
789
|
) -> None:
|
|
788
|
-
"""Run read-only SQL (one statement, or
|
|
789
|
-
answers) against the org's warehouse and get the
|
|
790
|
+
"""Run read-only SQL (one statement, or on the managed warehouse a script
|
|
791
|
+
whose last statement answers) against the org's warehouse and get the
|
|
792
|
+
rows back, ungoverned.
|
|
790
793
|
|
|
791
794
|
Metrics first: ``list_metrics`` / ``query_metrics`` answer with the
|
|
792
795
|
org's own definitions (units, filters, caveats) and are what a number
|
|
@@ -807,8 +810,13 @@ def run_sql(
|
|
|
807
810
|
in it is scratch space that ends with the script. It is dry-run first
|
|
808
811
|
and refused over the byte cap;
|
|
809
812
|
the timeout and ``max_rows`` bound it further. On an external
|
|
810
|
-
warehouse it runs with the customer's own key
|
|
811
|
-
|
|
813
|
+
warehouse it runs with the customer's own key, so there only ONE read
|
|
814
|
+
statement runs: SELECT (WITH ... SELECT), EXPLAIN, DESCRIBE / SHOW and
|
|
815
|
+
INFORMATION_SCHEMA selects. A script, DML, DDL, MERGE, CALL, EXPORT
|
|
816
|
+
DATA, LOAD DATA, GRANT, SET, SELECT ... INTO or anything that cannot be
|
|
817
|
+
read as SQL is refused (``statement_not_allowed``); change data in the
|
|
818
|
+
warehouse with its own tools. Every call is logged with the statement
|
|
819
|
+
and the caller.
|
|
812
820
|
"""
|
|
813
821
|
run_tool('run_sql', contract.RunSqlParams, None, {'sql': sql, 'warehouse': warehouse, 'max_rows': max_rows, 'format': format, 'dry_run': dry_run}, from_file=from_file, output=output)
|
|
814
822
|
|
|
@@ -1168,6 +1176,57 @@ def watch_edit__rotate_secret(
|
|
|
1168
1176
|
run_tool('watch_edit', contract.WatchEditParams, 'rotate_secret', {'id': id}, from_file=from_file, output=output)
|
|
1169
1177
|
|
|
1170
1178
|
|
|
1179
|
+
# ---------------------------------------------------------------------------
|
|
1180
|
+
# python_run
|
|
1181
|
+
# ---------------------------------------------------------------------------
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
python_run = typer.Typer(name='python_run', help="Run Python (pandas, numpy, scipy, statsmodels, scikit-learn, matplotlib, pyarrow) over the org's own data, for what SQL cannot do: a regression, a forecast, a clustering, a statistical test, a chart. Member. Numbers from it are ungoverned, like run_sql's: prefer query_metrics for anything a metric defines, and say which is which.", no_args_is_help=True)
|
|
1185
|
+
|
|
1186
|
+
|
|
1187
|
+
@python_run.command('run', short_help="Run Python over the org's data in an isolated sandbox (member).")
|
|
1188
|
+
def python_run__run(
|
|
1189
|
+
code: Annotated[str, typer.Option('--code', help='The Python program, run as /work/code/main.py with /work as the working directory. Read inputs from /work/in/<name>.parquet (pandas.read_parquet); write files to /work/out/ (png, csv, json, parquet, txt; at most 20, 10 MB each); print what the answer needs (stdout and stderr come back, 1 MB each). No network, no credentials.')],
|
|
1190
|
+
inputs: Annotated[Optional[str], typer.Option('--inputs-json', help='Up to 8 inputs, each {name, table} or {name, sql}. All of them together at most 2,000,000 rows and 500 MB, checked before anything runs; aggregate in SQL when the raw rows are more than the analysis needs. (JSON, or @path to a .json/.yaml file) (default: [])')] = None,
|
|
1191
|
+
timeout_seconds: Annotated[Optional[int], typer.Option('--timeout-seconds', help='Wall-clock limit for the code (default 120, at most 600). For now at most 45: longer runs are refused until they run as their own job. A longer run will answer a run_id to poll with status. (default: 120)')] = None,
|
|
1192
|
+
warehouse: Annotated[Optional[str], typer.Option('--warehouse', help="One of the org's warehouses (see dataspring://warehouses); omit for the active one.")] = None,
|
|
1193
|
+
from_file: FromFile = None,
|
|
1194
|
+
output: Output = None,
|
|
1195
|
+
) -> None:
|
|
1196
|
+
"""Run Python over the org's data in an isolated sandbox (member)."""
|
|
1197
|
+
run_tool('python_run', contract.PythonRunParams, 'run', {'code': code, 'inputs': J(inputs), 'timeout_seconds': timeout_seconds, 'warehouse': warehouse}, from_file=from_file, output=output)
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
@python_run.command('status', short_help='Where a run is: its status, exit code, CPU and wall seconds, files.')
|
|
1201
|
+
def python_run__status(
|
|
1202
|
+
run_id: Annotated[str, typer.Option('--run-id', help="The run, from run's answer or list")],
|
|
1203
|
+
from_file: FromFile = None,
|
|
1204
|
+
output: Output = None,
|
|
1205
|
+
) -> None:
|
|
1206
|
+
"""Where a run is: its status, exit code, CPU and wall seconds, files."""
|
|
1207
|
+
run_tool('python_run', contract.PythonRunParams, 'status', {'run_id': run_id}, from_file=from_file, output=output)
|
|
1208
|
+
|
|
1209
|
+
|
|
1210
|
+
@python_run.command('result', short_help="A finished run's whole answer: stdout, stderr and the file links.")
|
|
1211
|
+
def python_run__result(
|
|
1212
|
+
run_id: Annotated[str, typer.Option('--run-id', help="The run, from run's answer or list")],
|
|
1213
|
+
from_file: FromFile = None,
|
|
1214
|
+
output: Output = None,
|
|
1215
|
+
) -> None:
|
|
1216
|
+
"""A finished run's whole answer: stdout, stderr and the file links."""
|
|
1217
|
+
run_tool('python_run', contract.PythonRunParams, 'result', {'run_id': run_id}, from_file=from_file, output=output)
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
@python_run.command('list', short_help="The org's recent runs, newest first, and its allowance this month.")
|
|
1221
|
+
def python_run__list(
|
|
1222
|
+
limit: Annotated[Optional[int], typer.Option('--limit', help='Runs to list (newest first; default 20) (default: 20)')] = None,
|
|
1223
|
+
from_file: FromFile = None,
|
|
1224
|
+
output: Output = None,
|
|
1225
|
+
) -> None:
|
|
1226
|
+
"""The org's recent runs, newest first, and its allowance this month."""
|
|
1227
|
+
run_tool('python_run', contract.PythonRunParams, 'list', {'limit': limit}, from_file=from_file, output=output)
|
|
1228
|
+
|
|
1229
|
+
|
|
1171
1230
|
def mount(app: typer.Typer) -> None:
|
|
1172
1231
|
"""Add every generated command to ``app``: flat tools at the root, families as groups."""
|
|
1173
1232
|
app.add_typer(business_context_edit, name='business_context_edit')
|
|
@@ -1186,10 +1245,11 @@ def mount(app: typer.Typer) -> None:
|
|
|
1186
1245
|
app.command('import_manifest', short_help='Import a semantic manifest using replace semantics.')(import_manifest)
|
|
1187
1246
|
app.command('submit_error_report', short_help='Submit an error report to help improve DataSpring.')(submit_error_report)
|
|
1188
1247
|
app.add_typer(secret_edit, name='secret_edit')
|
|
1189
|
-
app.command('run_sql', short_help="Run read-only SQL (one statement, or a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.")(run_sql)
|
|
1248
|
+
app.command('run_sql', short_help="Run read-only SQL (one statement, or on the managed warehouse a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.")(run_sql)
|
|
1190
1249
|
app.add_typer(datacore_edit, name='datacore_edit')
|
|
1191
1250
|
app.add_typer(datacore_run, name='datacore_run')
|
|
1192
1251
|
app.add_typer(tenant_edit, name='tenant_edit')
|
|
1193
1252
|
app.add_typer(board_edit, name='board_edit')
|
|
1194
1253
|
app.command('board_fetch', short_help='Run every row of a board in one call and answer salience first.')(board_fetch)
|
|
1195
1254
|
app.add_typer(watch_edit, name='watch_edit')
|
|
1255
|
+
app.add_typer(python_run, name='python_run')
|
|
@@ -234,7 +234,7 @@ def _request(method: str, path: str, token: str, *, body: dict | None = None, ti
|
|
|
234
234
|
import httpx
|
|
235
235
|
|
|
236
236
|
url = f"{get_api_base()}{path}"
|
|
237
|
-
headers = {"Authorization": f"Bearer {token}"}
|
|
237
|
+
headers = {"Authorization": f"Bearer {token}", "User-Agent": _user_agent()}
|
|
238
238
|
attempts = 2 if retry else 1
|
|
239
239
|
with transport.ColdStartIndicator() as indicator, httpx.Client(timeout=timeout) as client:
|
|
240
240
|
for attempt in range(attempts):
|
|
@@ -252,6 +252,37 @@ def _request(method: str, path: str, token: str, *, body: dict | None = None, ti
|
|
|
252
252
|
return response.json()
|
|
253
253
|
|
|
254
254
|
|
|
255
|
+
def _user_agent() -> str:
|
|
256
|
+
"""``dataspring-cli/<version>``: how the server tells the CLI from other
|
|
257
|
+
callers (python_run records which client ran a program)."""
|
|
258
|
+
from cli.version import cli_version
|
|
259
|
+
|
|
260
|
+
return f"dataspring-cli/{cli_version()}"
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def api_get_bytes(path: str, token: str, timeout: float = API_TIMEOUT) -> bytes:
|
|
264
|
+
"""An authenticated GET whose answer is a file, not JSON (a python run's
|
|
265
|
+
outputs). Retried once like ``api_get``; errors as ``_request``'s."""
|
|
266
|
+
import httpx
|
|
267
|
+
|
|
268
|
+
url = f"{get_api_base()}{path}"
|
|
269
|
+
headers = {"Authorization": f"Bearer {token}", "User-Agent": _user_agent()}
|
|
270
|
+
with transport.ColdStartIndicator() as indicator, httpx.Client(timeout=timeout) as client:
|
|
271
|
+
for attempt in range(2):
|
|
272
|
+
try:
|
|
273
|
+
response = client.get(url, headers=headers)
|
|
274
|
+
break
|
|
275
|
+
except transport.RETRYABLE as e:
|
|
276
|
+
if attempt == 0:
|
|
277
|
+
indicator.mark_retrying()
|
|
278
|
+
continue
|
|
279
|
+
raise transport.unreachable(e, get_api_base(), timeout) from e
|
|
280
|
+
except httpx.TransportError as e:
|
|
281
|
+
raise transport.unreachable(e, get_api_base(), timeout) from e
|
|
282
|
+
response.raise_for_status()
|
|
283
|
+
return response.content
|
|
284
|
+
|
|
285
|
+
|
|
255
286
|
def api_get(path: str, token: str, timeout: float = API_TIMEOUT) -> dict:
|
|
256
287
|
"""Make an authenticated GET request to the backend API (idempotent:
|
|
257
288
|
retried once on a timeout or a failed connection)."""
|
|
@@ -2616,8 +2647,8 @@ def secret_set_cmd(
|
|
|
2616
2647
|
"""Set or rotate a customer secret from stdin, never from an argument.
|
|
2617
2648
|
|
|
2618
2649
|
Examples:
|
|
2619
|
-
printf '%s' "$KEY" | dataspring secret set t-acme-kanpla-
|
|
2620
|
-
dataspring secret set t-acme-kanpla-
|
|
2650
|
+
printf '%s' "$KEY" | dataspring secret set t-acme-kanpla-api-key --stdin
|
|
2651
|
+
dataspring secret set t-acme-kanpla-api-key --stdin < key.txt
|
|
2621
2652
|
"""
|
|
2622
2653
|
require_auth()
|
|
2623
2654
|
token = auth.get_access_token()
|
|
@@ -2711,6 +2742,14 @@ def sql_cmd(
|
|
|
2711
2742
|
ends with the script. Dry-run first, refused over the byte cap; at most
|
|
2712
2743
|
--max-rows rows (cap 10000), so aggregate in SQL.
|
|
2713
2744
|
|
|
2745
|
+
On an external warehouse (--warehouse external: the customer's own key,
|
|
2746
|
+
which can write) only ONE read statement runs: SELECT, WITH ... SELECT,
|
|
2747
|
+
EXPLAIN, DESCRIBE / SHOW, INFORMATION_SCHEMA selects. A script, INSERT,
|
|
2748
|
+
UPDATE, DELETE, MERGE, CREATE, DROP, ALTER, TRUNCATE, GRANT, CALL,
|
|
2749
|
+
EXPORT DATA, LOAD DATA, SET, SELECT ... INTO or SQL that cannot be read
|
|
2750
|
+
is refused with statement_not_allowed (exit 4); change data there with
|
|
2751
|
+
your own tools.
|
|
2752
|
+
|
|
2714
2753
|
Examples:
|
|
2715
2754
|
dataspring sql "SELECT venue, SUM(revenue) FROM acme_marts.fct_orders GROUP BY 1"
|
|
2716
2755
|
dataspring sql "SELECT * FROM acme_raw.kanpla_orders LIMIT 20"
|
|
@@ -3746,6 +3785,246 @@ def skill_instructions(
|
|
|
3746
3785
|
typer.echo(_fetch_skill_markdown(f"/api/v1/skill/{persona}/instructions"))
|
|
3747
3786
|
|
|
3748
3787
|
|
|
3788
|
+
# ============================================================================
|
|
3789
|
+
# python: hosted Python over the org's data (epic kfh, tick r2v)
|
|
3790
|
+
# ============================================================================
|
|
3791
|
+
#
|
|
3792
|
+
# `dataspring python run <file.py> --input name=<table> --input name='sql:...'`
|
|
3793
|
+
# is `python_run run` with the file as the code; `runs` is `list`; `result`
|
|
3794
|
+
# is `result` (with --wait, `status` polled first). Files come down through
|
|
3795
|
+
# GET /api/python/runs/<id>/files/<name> with the CLI's own credential.
|
|
3796
|
+
|
|
3797
|
+
python_app = typer.Typer(
|
|
3798
|
+
help="Hosted Python over the org's data: run a file in a sandbox with your tables as inputs, list runs, "
|
|
3799
|
+
"fetch a result and its files (python_run)",
|
|
3800
|
+
no_args_is_help=True,
|
|
3801
|
+
)
|
|
3802
|
+
app.add_typer(python_app, name="python")
|
|
3803
|
+
|
|
3804
|
+
#: How often --wait asks, and for how long at most.
|
|
3805
|
+
PYTHON_POLL_SECONDS = 3.0
|
|
3806
|
+
PYTHON_WAIT_SECONDS = 1800.0
|
|
3807
|
+
|
|
3808
|
+
|
|
3809
|
+
def _python_input(spec: str) -> dict:
|
|
3810
|
+
"""``name=<table>`` or ``name=sql:<statement>`` (``sql:@file.sql`` reads a file)."""
|
|
3811
|
+
name, sep, source = spec.partition("=")
|
|
3812
|
+
if not sep or not name or not source:
|
|
3813
|
+
print_error(f"--input {spec!r}: expected name=<table> or name=sql:<SELECT ...>")
|
|
3814
|
+
raise typer.Exit(2)
|
|
3815
|
+
if source.startswith("sql:"):
|
|
3816
|
+
sql = source[4:]
|
|
3817
|
+
if sql.startswith("@"):
|
|
3818
|
+
sql = Path(sql[1:]).read_text()
|
|
3819
|
+
return {"name": name, "sql": sql}
|
|
3820
|
+
return {"name": name, "table": source}
|
|
3821
|
+
|
|
3822
|
+
|
|
3823
|
+
def _python_save(result: dict, directory: Path, token: str) -> list[Path]:
|
|
3824
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
3825
|
+
saved = []
|
|
3826
|
+
for f in result.get("files") or []:
|
|
3827
|
+
name = f["name"]
|
|
3828
|
+
data = api_get_bytes(f"/api/python/runs/{result['run_id']}/files/{name}", token)
|
|
3829
|
+
target = directory / Path(name).name
|
|
3830
|
+
target.write_bytes(data)
|
|
3831
|
+
saved.append(target)
|
|
3832
|
+
return saved
|
|
3833
|
+
|
|
3834
|
+
|
|
3835
|
+
def _python_report(result: dict, *, output_dir: Optional[Path], token: str, as_json: bool) -> None:
|
|
3836
|
+
"""Print a finished run: its stdout on stdout, its stderr and a summary
|
|
3837
|
+
on stderr (so `> out.txt` captures exactly what the program printed),
|
|
3838
|
+
then exit as the program did (0, or 1 when it failed or timed out)."""
|
|
3839
|
+
saved = _python_save(result, output_dir, token) if output_dir and result.get("files") else []
|
|
3840
|
+
if as_json:
|
|
3841
|
+
typer.echo(json.dumps(result, indent=2, default=str))
|
|
3842
|
+
else:
|
|
3843
|
+
if result.get("stdout"):
|
|
3844
|
+
sys.stdout.write(result["stdout"])
|
|
3845
|
+
if not result["stdout"].endswith("\n"):
|
|
3846
|
+
sys.stdout.write("\n")
|
|
3847
|
+
if result.get("stderr"):
|
|
3848
|
+
sys.stderr.write(result["stderr"])
|
|
3849
|
+
if not result["stderr"].endswith("\n"):
|
|
3850
|
+
sys.stderr.write("\n")
|
|
3851
|
+
usage = result.get("usage") or {}
|
|
3852
|
+
cpu = result.get("cpu_seconds", usage.get("cpu_seconds"))
|
|
3853
|
+
wall = result.get("wall_seconds", usage.get("wall_seconds"))
|
|
3854
|
+
line = f"python run {result.get('run_id')}: {result.get('status')}"
|
|
3855
|
+
if result.get("exit_code") is not None:
|
|
3856
|
+
line += f", exit {result['exit_code']}"
|
|
3857
|
+
if result.get("exit_reason"):
|
|
3858
|
+
line += f" ({result['exit_reason']})"
|
|
3859
|
+
if cpu is not None:
|
|
3860
|
+
line += f", {cpu} CPU-s"
|
|
3861
|
+
if wall is not None:
|
|
3862
|
+
line += f", {wall} s"
|
|
3863
|
+
typer.echo(line, err=True)
|
|
3864
|
+
if result.get("error"):
|
|
3865
|
+
typer.echo(f" error: {result['error']}", err=True)
|
|
3866
|
+
if result.get("stdout_truncated") or result.get("stderr_truncated"):
|
|
3867
|
+
typer.echo(" output truncated at 1 MB", err=True)
|
|
3868
|
+
for target in saved:
|
|
3869
|
+
typer.echo(f" saved {target}", err=True)
|
|
3870
|
+
if not saved:
|
|
3871
|
+
for f in result.get("files") or []:
|
|
3872
|
+
typer.echo(f" file {f['name']} ({f.get('bytes')} bytes): {f.get('url')}", err=True)
|
|
3873
|
+
for d in result.get("dropped") or []:
|
|
3874
|
+
typer.echo(f" dropped {d.get('name')}: {d.get('reason')}", err=True)
|
|
3875
|
+
for w in result.get("warnings") or []:
|
|
3876
|
+
typer.echo(f" warning: {w}", err=True)
|
|
3877
|
+
if result.get("status") != "succeeded" or result.get("error"):
|
|
3878
|
+
raise typer.Exit(1)
|
|
3879
|
+
|
|
3880
|
+
|
|
3881
|
+
def _python_wait(run_id: str, token: str) -> dict:
|
|
3882
|
+
"""Poll ``status`` until the run has finished, then fetch ``result``."""
|
|
3883
|
+
import time
|
|
3884
|
+
|
|
3885
|
+
deadline = time.monotonic() + PYTHON_WAIT_SECONDS
|
|
3886
|
+
last = None
|
|
3887
|
+
while True:
|
|
3888
|
+
state = call_tool("python_run", "status", {"run_id": run_id}, token=token, timeout=API_TIMEOUT)
|
|
3889
|
+
if state.get("status") != last:
|
|
3890
|
+
last = state.get("status")
|
|
3891
|
+
print_info(f"python run {run_id}: {last}")
|
|
3892
|
+
if state.get("finished"):
|
|
3893
|
+
return call_tool("python_run", "result", {"run_id": run_id}, token=token, timeout=API_TIMEOUT)
|
|
3894
|
+
if time.monotonic() > deadline:
|
|
3895
|
+
print_error(f"python run {run_id} still {last} after {int(PYTHON_WAIT_SECONDS)} s; it carries on",
|
|
3896
|
+
hint=f"dataspring python result {run_id} --wait")
|
|
3897
|
+
raise typer.Exit(1)
|
|
3898
|
+
time.sleep(PYTHON_POLL_SECONDS)
|
|
3899
|
+
|
|
3900
|
+
|
|
3901
|
+
@python_app.command("run")
|
|
3902
|
+
def python_run_cmd(
|
|
3903
|
+
file: Annotated[str, typer.Argument(help="The Python file to run; '-' reads it from stdin")],
|
|
3904
|
+
input: Annotated[
|
|
3905
|
+
Optional[list[str]],
|
|
3906
|
+
typer.Option("--input", "-i", help="An input: name=<table> (marts.fct_orders, acme_raw.orders) or "
|
|
3907
|
+
"name=sql:<SELECT ...> (sql:@query.sql reads a file); repeatable, at most 8. The code reads "
|
|
3908
|
+
"it as /work/in/<name>.parquet"),
|
|
3909
|
+
] = None,
|
|
3910
|
+
timeout: Annotated[int, typer.Option("--timeout", help="Wall-clock limit in seconds (at most 600)")] = 120,
|
|
3911
|
+
warehouse: Annotated[Optional[str], typer.Option("--warehouse", help="Which of the org's warehouses; default the active one")] = None,
|
|
3912
|
+
wait: Annotated[bool, typer.Option("--wait", help="If the run outlives the inline answer (about 45 s), poll it to its end")] = False,
|
|
3913
|
+
output_dir: Annotated[Optional[Path], typer.Option("--output-dir", "-o", help="Download the files the run kept into this directory")] = None,
|
|
3914
|
+
as_json: Annotated[bool, typer.Option("--json", help="Print the whole answer as JSON")] = False,
|
|
3915
|
+
):
|
|
3916
|
+
"""Run a Python file over the org's data in an isolated sandbox (python_run).
|
|
3917
|
+
|
|
3918
|
+
Each --input is read as the org's read-only identity before the code
|
|
3919
|
+
runs (all of them at most 2,000,000 rows and 500 MB; aggregate in SQL
|
|
3920
|
+
first) and is there as /work/in/<name>.parquet. The sandbox has pandas,
|
|
3921
|
+
numpy, scipy, statsmodels, scikit-learn, matplotlib and pyarrow, and no
|
|
3922
|
+
network and no credentials: everything the code needs goes in as an
|
|
3923
|
+
input. Write charts and tables to /work/out/ (png, csv, json, parquet,
|
|
3924
|
+
txt; at most 20 files of 10 MB), kept 30 days; -o downloads them.
|
|
3925
|
+
|
|
3926
|
+
The program's stdout is printed on stdout, its stderr and a summary on
|
|
3927
|
+
stderr; the command exits 0 when the program did, else 1. A run still
|
|
3928
|
+
going after about 45 s prints its run id; --wait polls it to its end
|
|
3929
|
+
(or later: dataspring python result <id> --wait). A run is never retried.
|
|
3930
|
+
The org's allowance (runs per day, CPU-seconds per month) refuses with
|
|
3931
|
+
allowance_exhausted (exit 7) when it is used up.
|
|
3932
|
+
|
|
3933
|
+
Examples:
|
|
3934
|
+
dataspring python run analysis.py --input orders=marts.fct_orders -o out/
|
|
3935
|
+
dataspring python run forecast.py -i daily='sql:SELECT day, SUM(revenue) r FROM acme_marts.fct_orders GROUP BY 1' --wait
|
|
3936
|
+
dataspring python runs
|
|
3937
|
+
dataspring python result py-01jb... -o out/
|
|
3938
|
+
"""
|
|
3939
|
+
require_auth()
|
|
3940
|
+
token = auth.get_access_token()
|
|
3941
|
+
try:
|
|
3942
|
+
code = sys.stdin.read() if file == "-" else Path(file).read_text()
|
|
3943
|
+
except OSError as e:
|
|
3944
|
+
print_error(f"Could not read {file}: {e}")
|
|
3945
|
+
raise typer.Exit(2)
|
|
3946
|
+
body: dict = {"code": code, "timeout_seconds": timeout}
|
|
3947
|
+
inputs = [_python_input(spec) for spec in input or []]
|
|
3948
|
+
if inputs:
|
|
3949
|
+
body["inputs"] = inputs
|
|
3950
|
+
if warehouse:
|
|
3951
|
+
body["warehouse"] = warehouse
|
|
3952
|
+
try:
|
|
3953
|
+
result = call_tool("python_run", "run", body, token=token, timeout=API_TIMEOUT + 30)
|
|
3954
|
+
if result.get("status") in ("queued", "running"):
|
|
3955
|
+
if not wait:
|
|
3956
|
+
if as_json:
|
|
3957
|
+
typer.echo(json.dumps(result, indent=2, default=str))
|
|
3958
|
+
else:
|
|
3959
|
+
print_info(f"python run {result['run_id']} is still running; it carries on and is never retried")
|
|
3960
|
+
typer.echo(f" dataspring python result {result['run_id']} --wait", err=True)
|
|
3961
|
+
return
|
|
3962
|
+
result = _python_wait(result["run_id"], token)
|
|
3963
|
+
_python_report(result, output_dir=output_dir, token=token, as_json=as_json)
|
|
3964
|
+
except Exception as e:
|
|
3965
|
+
fail(e, role_action="run Python", prefix="python run failed")
|
|
3966
|
+
|
|
3967
|
+
|
|
3968
|
+
@python_app.command("runs")
|
|
3969
|
+
def python_runs_cmd(
|
|
3970
|
+
limit: Annotated[int, typer.Option("--limit", help="How many, newest first (at most 100)")] = 20,
|
|
3971
|
+
as_json: Annotated[bool, typer.Option("--json", help="Print the whole answer as JSON")] = False,
|
|
3972
|
+
):
|
|
3973
|
+
"""The org's recent python runs and its allowance this month.
|
|
3974
|
+
|
|
3975
|
+
Example:
|
|
3976
|
+
dataspring python runs --limit 5
|
|
3977
|
+
"""
|
|
3978
|
+
require_auth()
|
|
3979
|
+
token = auth.get_access_token()
|
|
3980
|
+
try:
|
|
3981
|
+
result = call_tool("python_run", "list", {"limit": limit}, token=token, timeout=API_TIMEOUT)
|
|
3982
|
+
except Exception as e:
|
|
3983
|
+
fail(e, role_action="list python runs")
|
|
3984
|
+
if as_json:
|
|
3985
|
+
typer.echo(json.dumps(result, indent=2, default=str))
|
|
3986
|
+
return
|
|
3987
|
+
rows = [{"run": r.get("run_id"), "status": r.get("status"), "exit": r.get("exit_code"),
|
|
3988
|
+
"cpu_s": r.get("cpu_seconds"), "files": r.get("files"), "by": r.get("person"),
|
|
3989
|
+
"created": r.get("created_at")} for r in result.get("runs") or []]
|
|
3990
|
+
format_output(rows, format="table", columns=["run", "status", "exit", "cpu_s", "files", "by", "created"],
|
|
3991
|
+
title=f"Python runs ({len(rows)})")
|
|
3992
|
+
allow = result.get("allowance") or {}
|
|
3993
|
+
if allow:
|
|
3994
|
+
typer.echo(
|
|
3995
|
+
f"allowance {allow.get('month')}: {allow.get('cpu_seconds')} of {allow.get('cpu_seconds_per_month')} "
|
|
3996
|
+
f"CPU-s; {allow.get('runs_today')} of {allow.get('runs_per_day')} runs today", err=True)
|
|
3997
|
+
|
|
3998
|
+
|
|
3999
|
+
@python_app.command("result")
|
|
4000
|
+
def python_result_cmd(
|
|
4001
|
+
run_id: Annotated[str, typer.Argument(help="The run (py-...), from `python run` or `python runs`")],
|
|
4002
|
+
wait: Annotated[bool, typer.Option("--wait", help="Poll a run still going to its end")] = False,
|
|
4003
|
+
output_dir: Annotated[Optional[Path], typer.Option("--output-dir", "-o", help="Download the files the run kept into this directory")] = None,
|
|
4004
|
+
as_json: Annotated[bool, typer.Option("--json", help="Print the whole answer as JSON")] = False,
|
|
4005
|
+
):
|
|
4006
|
+
"""A python run's result: its stdout, stderr and files (-o downloads them).
|
|
4007
|
+
|
|
4008
|
+
Exits 0 when the program succeeded, 1 when it failed or is still
|
|
4009
|
+
running (without --wait).
|
|
4010
|
+
|
|
4011
|
+
Example:
|
|
4012
|
+
dataspring python result py-01jb... --wait -o out/
|
|
4013
|
+
"""
|
|
4014
|
+
require_auth()
|
|
4015
|
+
token = auth.get_access_token()
|
|
4016
|
+
try:
|
|
4017
|
+
result = _python_wait(run_id, token) if wait else call_tool(
|
|
4018
|
+
"python_run", "result", {"run_id": run_id}, token=token, timeout=API_TIMEOUT)
|
|
4019
|
+
if not result.get("finished", True) and result.get("status") in ("queued", "running"):
|
|
4020
|
+
print_info(f"python run {run_id} is still {result.get('status')}")
|
|
4021
|
+
typer.echo(f" dataspring python result {run_id} --wait", err=True)
|
|
4022
|
+
raise typer.Exit(1)
|
|
4023
|
+
_python_report(result, output_dir=output_dir, token=token, as_json=as_json)
|
|
4024
|
+
except Exception as e:
|
|
4025
|
+
fail(e, role_action="read python runs")
|
|
4026
|
+
|
|
4027
|
+
|
|
3749
4028
|
# ============================================================================
|
|
3750
4029
|
# The generated tree: every registry operation, by its key
|
|
3751
4030
|
# ============================================================================
|
|
@@ -8,6 +8,18 @@ codes are suite standards §6's (0 ok, 1 failure, 2 invalid input, 3 not
|
|
|
8
8
|
found, 4 forbidden, 5 invalid state, 6 conflict, 7 rate limited, 64 usage).
|
|
9
9
|
``EN`` is the package's English text, reused where a message needs it.
|
|
10
10
|
|
|
11
|
+
``allowance_exhausted`` is the ninth (hub D-105, 2026-10-23: the suite's
|
|
12
|
+
metered products refuse with it when an org's allowance is used up, the fix
|
|
13
|
+
being the account's billing page). It is not in ``@forspring/ui`` yet (its
|
|
14
|
+
latest tag, v0.10.0, has the eight), so its status and exit code are the
|
|
15
|
+
suite standards' for rate limiting, 429 and exit 7, until the package names
|
|
16
|
+
them; DataSpring's ``python_run`` is its first user (tick r2v).
|
|
17
|
+
|
|
18
|
+
``statement_not_allowed`` is DataSpring's own beside them (tick w3c):
|
|
19
|
+
``run_sql`` on an external warehouse runs only a single read statement and
|
|
20
|
+
refuses anything else with it. It is not an access refusal and not in
|
|
21
|
+
``@forspring/ui``; it takes the suite standards' forbidden, 403 and exit 4.
|
|
22
|
+
|
|
11
23
|
This module ships in the CLI wheel (it imports nothing but the standard
|
|
12
24
|
library) and the server imports it too (``mcp_caller``), so the MCP's
|
|
13
25
|
``ToolError`` text and the CLI's error line spell a refusal the same way:
|
|
@@ -43,6 +55,10 @@ EXIT_CODES: dict[str, int] = {
|
|
|
43
55
|
"role_too_low": EXIT_FORBIDDEN,
|
|
44
56
|
"org_mismatch": EXIT_FORBIDDEN,
|
|
45
57
|
"account_unavailable": EXIT_FAILURE,
|
|
58
|
+
# Hub D-105; not in @forspring/ui v0.10.0 yet: suite standards §6's 7.
|
|
59
|
+
"allowance_exhausted": EXIT_RATE_LIMITED,
|
|
60
|
+
# DataSpring's own (tick w3c), not in @forspring/ui: forbidden, exit 4.
|
|
61
|
+
"statement_not_allowed": EXIT_FORBIDDEN,
|
|
46
62
|
}
|
|
47
63
|
|
|
48
64
|
#: The HTTP status for each (``REFUSALS[reason].status``).
|
|
@@ -55,6 +71,8 @@ HTTP_STATUS: dict[str, int] = {
|
|
|
55
71
|
"role_too_low": 403,
|
|
56
72
|
"org_mismatch": 403,
|
|
57
73
|
"account_unavailable": 503,
|
|
74
|
+
"allowance_exhausted": 429,
|
|
75
|
+
"statement_not_allowed": 403,
|
|
58
76
|
}
|
|
59
77
|
|
|
60
78
|
#: ``REFUSALS[reason].en``: the package's English title, text and fix.
|
|
@@ -99,6 +117,16 @@ EN: dict[str, dict[str, str]] = {
|
|
|
99
117
|
"text": "Forspring's account is not answering, so we cannot check your access.",
|
|
100
118
|
"fix": "Nothing is lost; try again in a minute.",
|
|
101
119
|
},
|
|
120
|
+
"allowance_exhausted": {
|
|
121
|
+
"title": "The organization's allowance is used up",
|
|
122
|
+
"text": "This would use more than the organization has left this period.",
|
|
123
|
+
"fix": "An owner can raise the allowance on the billing page.",
|
|
124
|
+
},
|
|
125
|
+
"statement_not_allowed": {
|
|
126
|
+
"title": "Only read statements run here",
|
|
127
|
+
"text": "On an external warehouse run_sql runs one read statement at a time, and this is not one.",
|
|
128
|
+
"fix": "Change data in your warehouse with your own tools.",
|
|
129
|
+
},
|
|
102
130
|
}
|
|
103
131
|
|
|
104
132
|
REASONS = tuple(EXIT_CODES)
|
|
@@ -9,7 +9,10 @@ taken for pip, which uv environments do not have (tick ghl). A uv tool
|
|
|
9
9
|
environment holds ``uv-receipt.toml`` (and lives under ``.../uv/tools/
|
|
10
10
|
dataspring-cli/``), pipx's lives under ``.../pipx/venvs/dataspring-cli/``,
|
|
11
11
|
and anything else is a pip install when pip is there, else unknown: then
|
|
12
|
-
the CLI prints the commands to run instead of failing.
|
|
12
|
+
the CLI prints the commands to run instead of failing. A uv tool upgrades
|
|
13
|
+
with ``uv tool install dataspring-cli@latest``, not ``uv tool upgrade``:
|
|
14
|
+
the latter keeps an exact version pin and leaves a pinned tool where it is
|
|
15
|
+
with exit 0 (tick 250).
|
|
13
16
|
``--check`` only compares the installed version with PyPI's latest.
|
|
14
17
|
"""
|
|
15
18
|
|
|
@@ -29,7 +32,7 @@ PYPI_JSON = f"https://pypi.org/pypi/{DIST}/json"
|
|
|
29
32
|
|
|
30
33
|
|
|
31
34
|
MANUAL_COMMANDS = (
|
|
32
|
-
f"uv tool
|
|
35
|
+
f"uv tool install {DIST}@latest",
|
|
33
36
|
f"pipx upgrade {DIST}",
|
|
34
37
|
f"python -m pip install --upgrade {DIST}",
|
|
35
38
|
)
|
|
@@ -51,7 +54,7 @@ def detect_install_method(prefix: str | None = None, *, has_pip: bool | None = N
|
|
|
51
54
|
def upgrade_command(method: str) -> list[str] | None:
|
|
52
55
|
"""The command for ``method``; None when there is none to run (``unknown``)."""
|
|
53
56
|
if method == "uv":
|
|
54
|
-
return ["uv", "tool", "
|
|
57
|
+
return ["uv", "tool", "install", f"{DIST}@latest"]
|
|
55
58
|
if method == "pipx":
|
|
56
59
|
return ["pipx", "upgrade", DIST]
|
|
57
60
|
if method == "pip":
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "dataspring-cli"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.6.0"
|
|
4
4
|
description = "DataSpring CLI - Query metrics and manage dashboards from the terminal"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.12"
|
|
@@ -93,6 +93,16 @@ server = [
|
|
|
93
93
|
# schemas (backend/generated/datacore_catalog.json)
|
|
94
94
|
"jsonschema>=4.23",
|
|
95
95
|
"ruamel-yaml>=0.18",
|
|
96
|
+
# python_run (tick r2v, services/python_run.py): inputs fetched as Arrow
|
|
97
|
+
# (through the Storage Read API, effectively free, rather than paging
|
|
98
|
+
# tabledata.list) and handed to the executor as Parquet, over HTTP/2
|
|
99
|
+
# (Cloud Run caps an HTTP/1 request at 32 MiB; inputs go to 500 MB).
|
|
100
|
+
"pyarrow>=21",
|
|
101
|
+
"google-cloud-bigquery-storage>=2.30",
|
|
102
|
+
"h2>=4.1",
|
|
103
|
+
# run_sql parses every statement on an external warehouse before it runs
|
|
104
|
+
# (services/sql_guard.py, tick w3c); was only MetricFlow's dependency.
|
|
105
|
+
"sqlglot>=30.18.0",
|
|
96
106
|
]
|
|
97
107
|
|
|
98
108
|
[tool.uv]
|
|
@@ -79,6 +79,26 @@ class Settings(BaseSettings):
|
|
|
79
79
|
# warehouse (BigQuery via `result(max_results=)`, DuckDB via `fetchmany`).
|
|
80
80
|
query_max_rows: int = 10_000
|
|
81
81
|
|
|
82
|
+
# python_run (epic kfh, tick r2v; services/python_run.py). The executor
|
|
83
|
+
# is the Cloud Run service python-executor (scripts/setup-python-executor.sh).
|
|
84
|
+
# Unset, its deterministic URL is derived on Cloud Run from the project
|
|
85
|
+
# number (https://python-executor-<number>.europe-west1.run.app), so a
|
|
86
|
+
# deploy needs no setting; set it to point a backend at another one.
|
|
87
|
+
python_executor_url: str | None = None
|
|
88
|
+
# The local per-org allowance until the account's meter exists (hub
|
|
89
|
+
# D-105): runs per UTC day, sandbox CPU-seconds and stored output bytes
|
|
90
|
+
# per calendar month. An org's own values live in
|
|
91
|
+
# organizations/<org>/config/python ({runs_per_day, cpu_seconds_per_month,
|
|
92
|
+
# cpu_seconds_hard_cap, output_bytes_per_month}).
|
|
93
|
+
python_runs_per_day: int = 200
|
|
94
|
+
python_cpu_seconds_per_month: int = 36_000
|
|
95
|
+
python_output_bytes_per_month: int = 5 * 1024**3
|
|
96
|
+
# Runs that outlive the inline wait (over 45 s) carry on as a task on the
|
|
97
|
+
# backend instance, which request-based billing throttles between
|
|
98
|
+
# requests, so they can stall or be lost with the instance. Off until
|
|
99
|
+
# they run as their own Cloud Run job (tick filed on epic kfh).
|
|
100
|
+
python_long_runs_enabled: bool = False
|
|
101
|
+
|
|
82
102
|
# Feature flags
|
|
83
103
|
schedules_enabled: bool = False # Scheduled reports (email delivery not implemented yet)
|
|
84
104
|
|
|
@@ -308,6 +308,15 @@ over raw), `<org>_core`, `<org>_marts` and `<org>_dev` (what the last
|
|
|
308
308
|
- A script of several statements runs as one job and answers the last
|
|
309
309
|
one's rows; `CREATE TEMP TABLE` in it is scratch space for an analysis
|
|
310
310
|
that ends with the script.
|
|
311
|
+
- On an **external** warehouse there is no read identity: the statement
|
|
312
|
+
runs with the customer's own key, so `run_sql` runs ONE read statement
|
|
313
|
+
at a time there (`SELECT`, `WITH ... SELECT`, `EXPLAIN`, `DESCRIBE` /
|
|
314
|
+
`SHOW`, `INFORMATION_SCHEMA` selects, in that warehouse's dialect) and
|
|
315
|
+
refuses a script, temp tables, any DML or DDL, `MERGE`, `CALL`, `EXPORT
|
|
316
|
+
DATA`, `LOAD DATA`, `GRANT`, `SET`, `SELECT ... INTO` and SQL it cannot
|
|
317
|
+
read with `statement_not_allowed`. Do not retry it another way: tell the
|
|
318
|
+
person to change data in their warehouse with their own tools, and use a
|
|
319
|
+
CTE where a temp table would have gone.
|
|
311
320
|
|
|
312
321
|
## 8.4 Workspace files keep the customer's text
|
|
313
322
|
|
|
@@ -632,7 +641,7 @@ MCP tool `warehouse_edit`; `POST /api/dispatch/warehouse_edit`. Manage the org's
|
|
|
632
641
|
|
|
633
642
|
| Parameter | Type | Required | Description |
|
|
634
643
|
|---|---|---|---|
|
|
635
|
-
| `label` | string | yes | Display name, e.g. "
|
|
644
|
+
| `label` | string | yes | Display name, e.g. "Acme's BigQuery" |
|
|
636
645
|
| `project` | string | yes | The customer's GCP project id |
|
|
637
646
|
| `dataset` | string | yes | The dataset the semantic manifest points at |
|
|
638
647
|
| `location` | string | yes | BigQuery location of that dataset, e.g. 'europe-north2' |
|
|
@@ -804,11 +804,11 @@ MCP tool `explain_query`; `POST /api/dispatch/explain_query`. Show what a query
|
|
|
804
804
|
|
|
805
805
|
#### `run_sql`
|
|
806
806
|
|
|
807
|
-
MCP tool `run_sql`; `POST /api/dispatch/run_sql`. Run read-only SQL (one statement, or a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.
|
|
807
|
+
MCP tool `run_sql`; `POST /api/dispatch/run_sql`. Run read-only SQL (one statement, or on the managed warehouse a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.
|
|
808
808
|
|
|
809
809
|
| Parameter | Type | Required | Description |
|
|
810
810
|
|---|---|---|---|
|
|
811
|
-
| `sql` | string | yes | One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse. |
|
|
811
|
+
| `sql` | string | yes | One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse. On an external warehouse (the customer's own key) only ONE read statement runs: SELECT, WITH ... SELECT, EXPLAIN, DESCRIBE/SHOW or an INFORMATION_SCHEMA select, in that warehouse's dialect; a script, DML, DDL, MERGE, CALL, EXPORT DATA, LOAD DATA, GRANT, SET, SELECT ... INTO or unparseable SQL is refused as statement_not_allowed. |
|
|
812
812
|
| `warehouse` | string | no | One of the org's warehouses (see dataspring://warehouses); omit for the active one. |
|
|
813
813
|
| `max_rows` | integer | no, default `1000` | Rows to return at most (default 1000, cap 10000). Aggregate in SQL rather than paging. |
|
|
814
814
|
| `format` | one of `json`, `csv`, `markdown` | no, default `json` | 'json' answers rows in `data`; 'csv' or 'markdown' answer one document in `content`. |
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|