dataspring-cli 0.5.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/PKG-INFO +1 -1
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/auth.py +31 -4
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/bundled_manifest.py +2 -1
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/contract.py +126 -4
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/generated.py +68 -8
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/main.py +384 -65
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/output.py +5 -0
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/refusals.py +28 -0
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/runtime.py +8 -0
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/skills_commands.py +2 -1
- dataspring_cli-0.6.0/cli/transport.py +143 -0
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/upgrade.py +6 -3
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/version.py +16 -7
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/pyproject.toml +11 -1
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/settings.py +20 -0
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/skills/dataspring-author/SKILL.md +10 -1
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/skills/dataspring-consume/SKILL.md +2 -2
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/.gitignore +0 -0
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/README.md +0 -0
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/__init__.py +0 -0
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/forspring.py +0 -0
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/cli/skilltree.py +0 -0
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/hatch_build.py +0 -0
- {dataspring_cli-0.5.0 → dataspring_cli-0.6.0}/skills/dataspring-correct/SKILL.md +0 -0
|
@@ -52,6 +52,8 @@ import httpx
|
|
|
52
52
|
|
|
53
53
|
from cli import forspring
|
|
54
54
|
from cli.forspring import CredentialStore, OAuthError
|
|
55
|
+
from cli import transport
|
|
56
|
+
from cli.runtime import API_TIMEOUT
|
|
55
57
|
from cli.refusals import (
|
|
56
58
|
EN,
|
|
57
59
|
EXIT_FAILURE,
|
|
@@ -517,7 +519,18 @@ class CLIAuthManager:
|
|
|
517
519
|
def _key_org(self, key: str) -> str | None:
|
|
518
520
|
"""The org a key not stored here belongs to, as the server says.
|
|
519
521
|
Synchronous: :meth:`resolve` runs inside the async commands too."""
|
|
520
|
-
|
|
522
|
+
with transport.ColdStartIndicator() as indicator:
|
|
523
|
+
for attempt in range(2): # a GET: one retry on what a cold start looks like
|
|
524
|
+
try:
|
|
525
|
+
r = httpx.get(f"{_host()}/api/me", headers={"Authorization": f"Bearer {key}"}, timeout=API_TIMEOUT)
|
|
526
|
+
break
|
|
527
|
+
except transport.RETRYABLE as e:
|
|
528
|
+
if attempt == 0:
|
|
529
|
+
indicator.mark_retrying()
|
|
530
|
+
continue
|
|
531
|
+
raise transport.unreachable(e, _host(), API_TIMEOUT) from e
|
|
532
|
+
except httpx.TransportError as e:
|
|
533
|
+
raise transport.unreachable(e, _host(), API_TIMEOUT) from e
|
|
521
534
|
return self._me_answer(r).get("org_id")
|
|
522
535
|
|
|
523
536
|
def resolved_org(self) -> str | None:
|
|
@@ -653,8 +666,22 @@ class CLIAuthManager:
|
|
|
653
666
|
return self.resolve().key
|
|
654
667
|
|
|
655
668
|
async def _me(self, key: str) -> dict:
|
|
656
|
-
|
|
657
|
-
|
|
669
|
+
"""``GET /api/me``: the first request of nearly every command, so the
|
|
670
|
+
one a cold backend meets first (`cli.transport`): the "waking up"
|
|
671
|
+
status while it waits, one retry, `ApiUnreachable` if no answer."""
|
|
672
|
+
with transport.ColdStartIndicator() as indicator:
|
|
673
|
+
async with httpx.AsyncClient(timeout=API_TIMEOUT) as client:
|
|
674
|
+
for attempt in range(2):
|
|
675
|
+
try:
|
|
676
|
+
r = await client.get(f"{_host()}/api/me", headers={"Authorization": f"Bearer {key}"})
|
|
677
|
+
break
|
|
678
|
+
except transport.RETRYABLE as e:
|
|
679
|
+
if attempt == 0:
|
|
680
|
+
indicator.mark_retrying()
|
|
681
|
+
continue
|
|
682
|
+
raise transport.unreachable(e, _host(), API_TIMEOUT) from e
|
|
683
|
+
except httpx.TransportError as e:
|
|
684
|
+
raise transport.unreachable(e, _host(), API_TIMEOUT) from e
|
|
658
685
|
return self._me_answer(r)
|
|
659
686
|
|
|
660
687
|
@staticmethod
|
|
@@ -810,7 +837,7 @@ class CLIAuthManager:
|
|
|
810
837
|
detail = ""
|
|
811
838
|
try:
|
|
812
839
|
r = httpx.delete(f"{_host()}/api/keys/{p['key_id']}",
|
|
813
|
-
headers={"Authorization": f"Bearer {key}"}, timeout=
|
|
840
|
+
headers={"Authorization": f"Bearer {key}"}, timeout=API_TIMEOUT)
|
|
814
841
|
except httpx.HTTPError as e:
|
|
815
842
|
r = None
|
|
816
843
|
reason, exit_code, detail = "server_unreachable", EXIT_FAILURE, f"DataSpring did not answer ({type(e).__name__})"
|
|
@@ -6,7 +6,7 @@ the server's to say whether the server has operations this CLI does not
|
|
|
6
6
|
know. tests/test_skills_surface.py asserts it is current.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
|
-
MANIFEST_HASH = "
|
|
9
|
+
MANIFEST_HASH = "a58c0bb7f14250cf69aae9f9459806f8d9369a289b1760910de915a209e9aad1"
|
|
10
10
|
TOOL_KEYS = (
|
|
11
11
|
"business_context_edit",
|
|
12
12
|
"semantic_model_edit",
|
|
@@ -31,4 +31,5 @@ TOOL_KEYS = (
|
|
|
31
31
|
"board_edit",
|
|
32
32
|
"board_fetch",
|
|
33
33
|
"watch_edit",
|
|
34
|
+
"python_run",
|
|
34
35
|
)
|
|
@@ -24,7 +24,7 @@ from __future__ import annotations
|
|
|
24
24
|
|
|
25
25
|
from typing import Annotated, Any, Literal
|
|
26
26
|
|
|
27
|
-
from pydantic import BaseModel, ConfigDict, Field
|
|
27
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
class DispatchParams(BaseModel):
|
|
@@ -528,7 +528,7 @@ class AddExternalWarehouseAction(DispatchParams):
|
|
|
528
528
|
data project and is only named here (``key_secret``)."""
|
|
529
529
|
|
|
530
530
|
action: Literal["add_external"]
|
|
531
|
-
label: str = Field(description="Display name, e.g. \"
|
|
531
|
+
label: str = Field(description="Display name, e.g. \"Acme's BigQuery\"")
|
|
532
532
|
project: str = Field(description="The customer's GCP project id")
|
|
533
533
|
dataset: str = Field(description="The dataset the semantic manifest points at")
|
|
534
534
|
location: str = Field(description="BigQuery location of that dataset, e.g. 'europe-north2'")
|
|
@@ -1172,7 +1172,7 @@ class SetSecretAction(DispatchParams):
|
|
|
1172
1172
|
name: str = Field(
|
|
1173
1173
|
description=(
|
|
1174
1174
|
"The secret to set or rotate: t-<org>-<connection>-<field> for your "
|
|
1175
|
-
"organization, lower-case (t-acme-kanpla-
|
|
1175
|
+
"organization, lower-case (t-acme-kanpla-api-key, "
|
|
1176
1176
|
"t-acme-external-bigquery-key). A connection file names it."
|
|
1177
1177
|
)
|
|
1178
1178
|
)
|
|
@@ -1225,7 +1225,12 @@ class RunSqlParams(DispatchParams):
|
|
|
1225
1225
|
"organization id with every hyphen as an underscore (acme-dk is "
|
|
1226
1226
|
"acme_dk_marts); the identity it runs as can read those five datasets of "
|
|
1227
1227
|
"its own org and nothing else, so a write to a dataset or a reach outside "
|
|
1228
|
-
"them is refused by the warehouse."
|
|
1228
|
+
"them is refused by the warehouse. On an external warehouse (the "
|
|
1229
|
+
"customer's own key) only ONE read statement runs: SELECT, WITH ... "
|
|
1230
|
+
"SELECT, EXPLAIN, DESCRIBE/SHOW or an INFORMATION_SCHEMA select, in "
|
|
1231
|
+
"that warehouse's dialect; a script, DML, DDL, MERGE, CALL, EXPORT "
|
|
1232
|
+
"DATA, LOAD DATA, GRANT, SET, SELECT ... INTO or unparseable SQL is "
|
|
1233
|
+
"refused as statement_not_allowed."
|
|
1229
1234
|
)
|
|
1230
1235
|
)
|
|
1231
1236
|
warehouse: str | None = Field(
|
|
@@ -1248,6 +1253,123 @@ class RunSqlParams(DispatchParams):
|
|
|
1248
1253
|
)
|
|
1249
1254
|
|
|
1250
1255
|
|
|
1256
|
+
# ---------------------------------------------------------------------------
|
|
1257
|
+
# python_run: hosted Python over the org's own data (epic kfh, tick r2v)
|
|
1258
|
+
# ---------------------------------------------------------------------------
|
|
1259
|
+
|
|
1260
|
+
#: The caps every surface states and the service enforces (epic kfh).
|
|
1261
|
+
PYTHON_MAX_CODE_BYTES = 256 * 1024
|
|
1262
|
+
PYTHON_MAX_INPUTS = 8
|
|
1263
|
+
PYTHON_DEFAULT_TIMEOUT_SECONDS = 120
|
|
1264
|
+
PYTHON_MAX_TIMEOUT_SECONDS = 600
|
|
1265
|
+
PYTHON_INPUT_NAME_PATTERN = r"^[a-z_][a-z0-9_]{0,63}$"
|
|
1266
|
+
|
|
1267
|
+
|
|
1268
|
+
class PythonInput(DispatchParams):
|
|
1269
|
+
"""One input of a run: a table or a read-only SQL, fetched as the org's
|
|
1270
|
+
read-only identity before the code runs and handed to it as Parquet."""
|
|
1271
|
+
|
|
1272
|
+
name: str = Field(
|
|
1273
|
+
pattern=PYTHON_INPUT_NAME_PATTERN,
|
|
1274
|
+
description=(
|
|
1275
|
+
"What the code reads it as: /work/in/<name>.parquet (the directory is also "
|
|
1276
|
+
"os.environ['DATASPRING_INPUTS']). Lower-case letters, digits and '_'."
|
|
1277
|
+
),
|
|
1278
|
+
)
|
|
1279
|
+
table: str | None = Field(
|
|
1280
|
+
default=None,
|
|
1281
|
+
description=(
|
|
1282
|
+
"A table or view of your organization, as <org>_<layer>.<table> (acme_marts.fct_orders), "
|
|
1283
|
+
"<layer>.<table> (marts.fct_orders; layers marts, core, staging, raw, dev) or a bare table "
|
|
1284
|
+
"name in marts. Exactly one of table and sql."
|
|
1285
|
+
),
|
|
1286
|
+
)
|
|
1287
|
+
sql: str | None = Field(
|
|
1288
|
+
default=None,
|
|
1289
|
+
description=(
|
|
1290
|
+
"A read-only SQL statement whose rows are the input (the same rules as run_sql: it runs "
|
|
1291
|
+
"as the reader identity, dry-run first and byte-capped). Exactly one of table and sql."
|
|
1292
|
+
),
|
|
1293
|
+
)
|
|
1294
|
+
|
|
1295
|
+
@model_validator(mode="after")
|
|
1296
|
+
def _one_source(self) -> "PythonInput":
|
|
1297
|
+
if bool(self.table and self.table.strip()) == bool(self.sql and self.sql.strip()):
|
|
1298
|
+
raise ValueError(f"input {self.name}: give exactly one of table and sql")
|
|
1299
|
+
return self
|
|
1300
|
+
|
|
1301
|
+
|
|
1302
|
+
class RunPythonAction(DispatchParams):
|
|
1303
|
+
"""Run Python over the org's data in an isolated sandbox (member)."""
|
|
1304
|
+
|
|
1305
|
+
action: Literal["run"]
|
|
1306
|
+
code: str = Field(
|
|
1307
|
+
max_length=PYTHON_MAX_CODE_BYTES,
|
|
1308
|
+
description=(
|
|
1309
|
+
"The Python program, run as /work/code/main.py with /work as the working directory. "
|
|
1310
|
+
"Read inputs from /work/in/<name>.parquet (pandas.read_parquet); write files to "
|
|
1311
|
+
"/work/out/ (png, csv, json, parquet, txt; at most 20, 10 MB each); print what the "
|
|
1312
|
+
"answer needs (stdout and stderr come back, 1 MB each). No network, no credentials."
|
|
1313
|
+
),
|
|
1314
|
+
)
|
|
1315
|
+
inputs: list[PythonInput] = Field(
|
|
1316
|
+
default_factory=list,
|
|
1317
|
+
max_length=PYTHON_MAX_INPUTS,
|
|
1318
|
+
description=(
|
|
1319
|
+
f"Up to {PYTHON_MAX_INPUTS} inputs, each {{name, table}} or {{name, sql}}. All of them "
|
|
1320
|
+
"together at most 2,000,000 rows and 500 MB, checked before anything runs; aggregate "
|
|
1321
|
+
"in SQL when the raw rows are more than the analysis needs."
|
|
1322
|
+
),
|
|
1323
|
+
)
|
|
1324
|
+
timeout_seconds: int = Field(
|
|
1325
|
+
default=PYTHON_DEFAULT_TIMEOUT_SECONDS,
|
|
1326
|
+
ge=1,
|
|
1327
|
+
le=PYTHON_MAX_TIMEOUT_SECONDS,
|
|
1328
|
+
description=(
|
|
1329
|
+
f"Wall-clock limit for the code (default {PYTHON_DEFAULT_TIMEOUT_SECONDS}, at most "
|
|
1330
|
+
f"{PYTHON_MAX_TIMEOUT_SECONDS}). For now at most 45: longer runs are refused until they "
|
|
1331
|
+
"run as their own job. A longer run will answer a run_id to poll with status."
|
|
1332
|
+
),
|
|
1333
|
+
)
|
|
1334
|
+
warehouse: str | None = Field(
|
|
1335
|
+
default=None,
|
|
1336
|
+
description="One of the org's warehouses (see dataspring://warehouses); omit for the active one.",
|
|
1337
|
+
)
|
|
1338
|
+
|
|
1339
|
+
|
|
1340
|
+
class PythonRunStatusAction(DispatchParams):
|
|
1341
|
+
"""Where a run is: its status, exit code, CPU and wall seconds, files."""
|
|
1342
|
+
|
|
1343
|
+
action: Literal["status"]
|
|
1344
|
+
run_id: str = Field(description="The run, from run's answer or list")
|
|
1345
|
+
|
|
1346
|
+
|
|
1347
|
+
class PythonRunResultAction(DispatchParams):
|
|
1348
|
+
"""A finished run's whole answer: stdout, stderr and the file links."""
|
|
1349
|
+
|
|
1350
|
+
action: Literal["result"]
|
|
1351
|
+
run_id: str = Field(description="The run, from run's answer or list")
|
|
1352
|
+
|
|
1353
|
+
|
|
1354
|
+
class ListPythonRunsAction(DispatchParams):
|
|
1355
|
+
"""The org's recent runs, newest first, and its allowance this month."""
|
|
1356
|
+
|
|
1357
|
+
action: Literal["list"]
|
|
1358
|
+
limit: int = Field(default=20, ge=1, le=100, description="Runs to list (newest first; default 20)")
|
|
1359
|
+
|
|
1360
|
+
|
|
1361
|
+
PythonRunAction = Annotated[
|
|
1362
|
+
RunPythonAction | PythonRunStatusAction | PythonRunResultAction | ListPythonRunsAction,
|
|
1363
|
+
Field(discriminator="action"),
|
|
1364
|
+
]
|
|
1365
|
+
|
|
1366
|
+
|
|
1367
|
+
class PythonRunParams(DispatchParams):
|
|
1368
|
+
"""The ``python_run`` call: one ``action`` from the family's union."""
|
|
1369
|
+
|
|
1370
|
+
action: PythonRunAction
|
|
1371
|
+
|
|
1372
|
+
|
|
1251
1373
|
# ---------------------------------------------------------------------------
|
|
1252
1374
|
# Tenant lifecycle action types (D24)
|
|
1253
1375
|
# ---------------------------------------------------------------------------
|
|
@@ -55,6 +55,7 @@ MODELS: dict[str, type[contract.DispatchParams]] = {
|
|
|
55
55
|
'board_edit': contract.BoardEditParams,
|
|
56
56
|
'board_fetch': contract.BoardFetchParams,
|
|
57
57
|
'watch_edit': contract.WatchEditParams,
|
|
58
|
+
'python_run': contract.PythonRunParams,
|
|
58
59
|
}
|
|
59
60
|
|
|
60
61
|
#: The verbs of every edit family, in the order the action union declares them.
|
|
@@ -73,6 +74,7 @@ VERBS: dict[str, tuple[str, ...]] = {
|
|
|
73
74
|
'tenant_edit': ('create', 'suspend', 'resume', 'offboard'),
|
|
74
75
|
'board_edit': ('apply', 'patch', 'delete', 'convert_dashboard'),
|
|
75
76
|
'watch_edit': ('apply', 'patch', 'delete', 'rotate_secret'),
|
|
77
|
+
'python_run': ('run', 'status', 'result', 'list'),
|
|
76
78
|
}
|
|
77
79
|
|
|
78
80
|
|
|
@@ -482,7 +484,7 @@ def warehouse_edit__activate(
|
|
|
482
484
|
|
|
483
485
|
@warehouse_edit.command('add_external', short_help='Register a customer-owned BigQuery project as an external warehouse.')
|
|
484
486
|
def warehouse_edit__add_external(
|
|
485
|
-
label: Annotated[str, typer.Option('--label', help='Display name, e.g. "
|
|
487
|
+
label: Annotated[str, typer.Option('--label', help='Display name, e.g. "Acme\'s BigQuery"')],
|
|
486
488
|
project: Annotated[str, typer.Option('--project', help="The customer's GCP project id")],
|
|
487
489
|
dataset: Annotated[str, typer.Option('--dataset', help='The dataset the semantic manifest points at')],
|
|
488
490
|
location: Annotated[str, typer.Option('--location', help="BigQuery location of that dataset, e.g. 'europe-north2'")],
|
|
@@ -749,7 +751,7 @@ secret_edit = typer.Typer(name='secret_edit', help='Customer secrets, both direc
|
|
|
749
751
|
|
|
750
752
|
@secret_edit.command('set', short_help='Get a one-time link where a PERSON pastes a credential. The value never comes through here; the same name rotates a credential.')
|
|
751
753
|
def secret_edit__set(
|
|
752
|
-
name: Annotated[str, typer.Option('--name', help='The secret to set or rotate: t-<org>-<connection>-<field> for your organization, lower-case (t-acme-kanpla-
|
|
754
|
+
name: Annotated[str, typer.Option('--name', help='The secret to set or rotate: t-<org>-<connection>-<field> for your organization, lower-case (t-acme-kanpla-api-key, t-acme-external-bigquery-key). A connection file names it.')],
|
|
753
755
|
from_file: FromFile = None,
|
|
754
756
|
output: Output = None,
|
|
755
757
|
) -> None:
|
|
@@ -777,7 +779,7 @@ def secret_edit__reveal(
|
|
|
777
779
|
|
|
778
780
|
|
|
779
781
|
def run_sql(
|
|
780
|
-
sql: Annotated[str, typer.Option('--sql', help="One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse.")],
|
|
782
|
+
sql: Annotated[str, typer.Option('--sql', help="One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse. On an external warehouse (the customer's own key) only ONE read statement runs: SELECT, WITH ... SELECT, EXPLAIN, DESCRIBE/SHOW or an INFORMATION_SCHEMA select, in that warehouse's dialect; a script, DML, DDL, MERGE, CALL, EXPORT DATA, LOAD DATA, GRANT, SET, SELECT ... INTO or unparseable SQL is refused as statement_not_allowed.")],
|
|
781
783
|
warehouse: Annotated[Optional[str], typer.Option('--warehouse', help="One of the org's warehouses (see dataspring://warehouses); omit for the active one.")] = None,
|
|
782
784
|
max_rows: Annotated[Optional[int], typer.Option('--max-rows', help='Rows to return at most (default 1000, cap 10000). Aggregate in SQL rather than paging. (default: 1000)')] = None,
|
|
783
785
|
format: Annotated[Optional[str], typer.Option('--format', help="'json' answers rows in `data`; 'csv' or 'markdown' answer one document in `content`. (one of: json, csv, markdown) (default: json)")] = None,
|
|
@@ -785,8 +787,9 @@ def run_sql(
|
|
|
785
787
|
from_file: FromFile = None,
|
|
786
788
|
output: Output = None,
|
|
787
789
|
) -> None:
|
|
788
|
-
"""Run read-only SQL (one statement, or
|
|
789
|
-
answers) against the org's warehouse and get the
|
|
790
|
+
"""Run read-only SQL (one statement, or on the managed warehouse a script
|
|
791
|
+
whose last statement answers) against the org's warehouse and get the
|
|
792
|
+
rows back, ungoverned.
|
|
790
793
|
|
|
791
794
|
Metrics first: ``list_metrics`` / ``query_metrics`` answer with the
|
|
792
795
|
org's own definitions (units, filters, caveats) and are what a number
|
|
@@ -807,8 +810,13 @@ def run_sql(
|
|
|
807
810
|
in it is scratch space that ends with the script. It is dry-run first
|
|
808
811
|
and refused over the byte cap;
|
|
809
812
|
the timeout and ``max_rows`` bound it further. On an external
|
|
810
|
-
warehouse it runs with the customer's own key
|
|
811
|
-
|
|
813
|
+
warehouse it runs with the customer's own key, so there only ONE read
|
|
814
|
+
statement runs: SELECT (WITH ... SELECT), EXPLAIN, DESCRIBE / SHOW and
|
|
815
|
+
INFORMATION_SCHEMA selects. A script, DML, DDL, MERGE, CALL, EXPORT
|
|
816
|
+
DATA, LOAD DATA, GRANT, SET, SELECT ... INTO or anything that cannot be
|
|
817
|
+
read as SQL is refused (``statement_not_allowed``); change data in the
|
|
818
|
+
warehouse with its own tools. Every call is logged with the statement
|
|
819
|
+
and the caller.
|
|
812
820
|
"""
|
|
813
821
|
run_tool('run_sql', contract.RunSqlParams, None, {'sql': sql, 'warehouse': warehouse, 'max_rows': max_rows, 'format': format, 'dry_run': dry_run}, from_file=from_file, output=output)
|
|
814
822
|
|
|
@@ -1168,6 +1176,57 @@ def watch_edit__rotate_secret(
|
|
|
1168
1176
|
run_tool('watch_edit', contract.WatchEditParams, 'rotate_secret', {'id': id}, from_file=from_file, output=output)
|
|
1169
1177
|
|
|
1170
1178
|
|
|
1179
|
+
# ---------------------------------------------------------------------------
|
|
1180
|
+
# python_run
|
|
1181
|
+
# ---------------------------------------------------------------------------
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
python_run = typer.Typer(name='python_run', help="Run Python (pandas, numpy, scipy, statsmodels, scikit-learn, matplotlib, pyarrow) over the org's own data, for what SQL cannot do: a regression, a forecast, a clustering, a statistical test, a chart. Member. Numbers from it are ungoverned, like run_sql's: prefer query_metrics for anything a metric defines, and say which is which.", no_args_is_help=True)
|
|
1185
|
+
|
|
1186
|
+
|
|
1187
|
+
@python_run.command('run', short_help="Run Python over the org's data in an isolated sandbox (member).")
|
|
1188
|
+
def python_run__run(
|
|
1189
|
+
code: Annotated[str, typer.Option('--code', help='The Python program, run as /work/code/main.py with /work as the working directory. Read inputs from /work/in/<name>.parquet (pandas.read_parquet); write files to /work/out/ (png, csv, json, parquet, txt; at most 20, 10 MB each); print what the answer needs (stdout and stderr come back, 1 MB each). No network, no credentials.')],
|
|
1190
|
+
inputs: Annotated[Optional[str], typer.Option('--inputs-json', help='Up to 8 inputs, each {name, table} or {name, sql}. All of them together at most 2,000,000 rows and 500 MB, checked before anything runs; aggregate in SQL when the raw rows are more than the analysis needs. (JSON, or @path to a .json/.yaml file) (default: [])')] = None,
|
|
1191
|
+
timeout_seconds: Annotated[Optional[int], typer.Option('--timeout-seconds', help='Wall-clock limit for the code (default 120, at most 600). For now at most 45: longer runs are refused until they run as their own job. A longer run will answer a run_id to poll with status. (default: 120)')] = None,
|
|
1192
|
+
warehouse: Annotated[Optional[str], typer.Option('--warehouse', help="One of the org's warehouses (see dataspring://warehouses); omit for the active one.")] = None,
|
|
1193
|
+
from_file: FromFile = None,
|
|
1194
|
+
output: Output = None,
|
|
1195
|
+
) -> None:
|
|
1196
|
+
"""Run Python over the org's data in an isolated sandbox (member)."""
|
|
1197
|
+
run_tool('python_run', contract.PythonRunParams, 'run', {'code': code, 'inputs': J(inputs), 'timeout_seconds': timeout_seconds, 'warehouse': warehouse}, from_file=from_file, output=output)
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
@python_run.command('status', short_help='Where a run is: its status, exit code, CPU and wall seconds, files.')
|
|
1201
|
+
def python_run__status(
|
|
1202
|
+
run_id: Annotated[str, typer.Option('--run-id', help="The run, from run's answer or list")],
|
|
1203
|
+
from_file: FromFile = None,
|
|
1204
|
+
output: Output = None,
|
|
1205
|
+
) -> None:
|
|
1206
|
+
"""Where a run is: its status, exit code, CPU and wall seconds, files."""
|
|
1207
|
+
run_tool('python_run', contract.PythonRunParams, 'status', {'run_id': run_id}, from_file=from_file, output=output)
|
|
1208
|
+
|
|
1209
|
+
|
|
1210
|
+
@python_run.command('result', short_help="A finished run's whole answer: stdout, stderr and the file links.")
|
|
1211
|
+
def python_run__result(
|
|
1212
|
+
run_id: Annotated[str, typer.Option('--run-id', help="The run, from run's answer or list")],
|
|
1213
|
+
from_file: FromFile = None,
|
|
1214
|
+
output: Output = None,
|
|
1215
|
+
) -> None:
|
|
1216
|
+
"""A finished run's whole answer: stdout, stderr and the file links."""
|
|
1217
|
+
run_tool('python_run', contract.PythonRunParams, 'result', {'run_id': run_id}, from_file=from_file, output=output)
|
|
1218
|
+
|
|
1219
|
+
|
|
1220
|
+
@python_run.command('list', short_help="The org's recent runs, newest first, and its allowance this month.")
|
|
1221
|
+
def python_run__list(
|
|
1222
|
+
limit: Annotated[Optional[int], typer.Option('--limit', help='Runs to list (newest first; default 20) (default: 20)')] = None,
|
|
1223
|
+
from_file: FromFile = None,
|
|
1224
|
+
output: Output = None,
|
|
1225
|
+
) -> None:
|
|
1226
|
+
"""The org's recent runs, newest first, and its allowance this month."""
|
|
1227
|
+
run_tool('python_run', contract.PythonRunParams, 'list', {'limit': limit}, from_file=from_file, output=output)
|
|
1228
|
+
|
|
1229
|
+
|
|
1171
1230
|
def mount(app: typer.Typer) -> None:
|
|
1172
1231
|
"""Add every generated command to ``app``: flat tools at the root, families as groups."""
|
|
1173
1232
|
app.add_typer(business_context_edit, name='business_context_edit')
|
|
@@ -1186,10 +1245,11 @@ def mount(app: typer.Typer) -> None:
|
|
|
1186
1245
|
app.command('import_manifest', short_help='Import a semantic manifest using replace semantics.')(import_manifest)
|
|
1187
1246
|
app.command('submit_error_report', short_help='Submit an error report to help improve DataSpring.')(submit_error_report)
|
|
1188
1247
|
app.add_typer(secret_edit, name='secret_edit')
|
|
1189
|
-
app.command('run_sql', short_help="Run read-only SQL (one statement, or a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.")(run_sql)
|
|
1248
|
+
app.command('run_sql', short_help="Run read-only SQL (one statement, or on the managed warehouse a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.")(run_sql)
|
|
1190
1249
|
app.add_typer(datacore_edit, name='datacore_edit')
|
|
1191
1250
|
app.add_typer(datacore_run, name='datacore_run')
|
|
1192
1251
|
app.add_typer(tenant_edit, name='tenant_edit')
|
|
1193
1252
|
app.add_typer(board_edit, name='board_edit')
|
|
1194
1253
|
app.command('board_fetch', short_help='Run every row of a board in one call and answer salience first.')(board_fetch)
|
|
1195
1254
|
app.add_typer(watch_edit, name='watch_edit')
|
|
1255
|
+
app.add_typer(python_run, name='python_run')
|