dataspring-cli 0.5.1__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/PKG-INFO +1 -1
  2. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/bundled_manifest.py +2 -1
  3. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/contract.py +126 -4
  4. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/generated.py +68 -8
  5. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/main.py +282 -3
  6. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/refusals.py +28 -0
  7. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/upgrade.py +6 -3
  8. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/pyproject.toml +11 -1
  9. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/settings.py +20 -0
  10. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/skills/dataspring-author/SKILL.md +10 -1
  11. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/skills/dataspring-consume/SKILL.md +2 -2
  12. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/.gitignore +0 -0
  13. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/README.md +0 -0
  14. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/__init__.py +0 -0
  15. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/auth.py +0 -0
  16. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/forspring.py +0 -0
  17. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/output.py +0 -0
  18. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/runtime.py +0 -0
  19. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/skills_commands.py +0 -0
  20. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/skilltree.py +0 -0
  21. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/transport.py +0 -0
  22. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/cli/version.py +0 -0
  23. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/hatch_build.py +0 -0
  24. {dataspring_cli-0.5.1 → dataspring_cli-0.6.0}/skills/dataspring-correct/SKILL.md +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: dataspring-cli
3
- Version: 0.5.1
3
+ Version: 0.6.0
4
4
  Summary: DataSpring CLI - Query metrics and manage dashboards from the terminal
5
5
  Project-URL: Homepage, https://dataspring.app
6
6
  Project-URL: Documentation, https://dataspring.app/docs
@@ -6,7 +6,7 @@ the server's to say whether the server has operations this CLI does not
6
6
  know. tests/test_skills_surface.py asserts it is current.
7
7
  """
8
8
 
9
- MANIFEST_HASH = "54af1f0a9d07ef692b63e0648d835ce5efc020bd9588db6d75897b36af4e0232"
9
+ MANIFEST_HASH = "a58c0bb7f14250cf69aae9f9459806f8d9369a289b1760910de915a209e9aad1"
10
10
  TOOL_KEYS = (
11
11
  "business_context_edit",
12
12
  "semantic_model_edit",
@@ -31,4 +31,5 @@ TOOL_KEYS = (
31
31
  "board_edit",
32
32
  "board_fetch",
33
33
  "watch_edit",
34
+ "python_run",
34
35
  )
@@ -24,7 +24,7 @@ from __future__ import annotations
24
24
 
25
25
  from typing import Annotated, Any, Literal
26
26
 
27
- from pydantic import BaseModel, ConfigDict, Field
27
+ from pydantic import BaseModel, ConfigDict, Field, model_validator
28
28
 
29
29
 
30
30
  class DispatchParams(BaseModel):
@@ -528,7 +528,7 @@ class AddExternalWarehouseAction(DispatchParams):
528
528
  data project and is only named here (``key_secret``)."""
529
529
 
530
530
  action: Literal["add_external"]
531
- label: str = Field(description="Display name, e.g. \"Noon's BigQuery\"")
531
+ label: str = Field(description="Display name, e.g. \"Acme's BigQuery\"")
532
532
  project: str = Field(description="The customer's GCP project id")
533
533
  dataset: str = Field(description="The dataset the semantic manifest points at")
534
534
  location: str = Field(description="BigQuery location of that dataset, e.g. 'europe-north2'")
@@ -1172,7 +1172,7 @@ class SetSecretAction(DispatchParams):
1172
1172
  name: str = Field(
1173
1173
  description=(
1174
1174
  "The secret to set or rotate: t-<org>-<connection>-<field> for your "
1175
- "organization, lower-case (t-acme-kanpla-api_key, "
1175
+ "organization, lower-case (t-acme-kanpla-api-key, "
1176
1176
  "t-acme-external-bigquery-key). A connection file names it."
1177
1177
  )
1178
1178
  )
@@ -1225,7 +1225,12 @@ class RunSqlParams(DispatchParams):
1225
1225
  "organization id with every hyphen as an underscore (acme-dk is "
1226
1226
  "acme_dk_marts); the identity it runs as can read those five datasets of "
1227
1227
  "its own org and nothing else, so a write to a dataset or a reach outside "
1228
- "them is refused by the warehouse."
1228
+ "them is refused by the warehouse. On an external warehouse (the "
1229
+ "customer's own key) only ONE read statement runs: SELECT, WITH ... "
1230
+ "SELECT, EXPLAIN, DESCRIBE/SHOW or an INFORMATION_SCHEMA select, in "
1231
+ "that warehouse's dialect; a script, DML, DDL, MERGE, CALL, EXPORT "
1232
+ "DATA, LOAD DATA, GRANT, SET, SELECT ... INTO or unparseable SQL is "
1233
+ "refused as statement_not_allowed."
1229
1234
  )
1230
1235
  )
1231
1236
  warehouse: str | None = Field(
@@ -1248,6 +1253,123 @@ class RunSqlParams(DispatchParams):
1248
1253
  )
1249
1254
 
1250
1255
 
1256
+ # ---------------------------------------------------------------------------
1257
+ # python_run: hosted Python over the org's own data (epic kfh, tick r2v)
1258
+ # ---------------------------------------------------------------------------
1259
+
1260
+ #: The caps every surface states and the service enforces (epic kfh).
1261
+ PYTHON_MAX_CODE_BYTES = 256 * 1024
1262
+ PYTHON_MAX_INPUTS = 8
1263
+ PYTHON_DEFAULT_TIMEOUT_SECONDS = 120
1264
+ PYTHON_MAX_TIMEOUT_SECONDS = 600
1265
+ PYTHON_INPUT_NAME_PATTERN = r"^[a-z_][a-z0-9_]{0,63}$"
1266
+
1267
+
1268
+ class PythonInput(DispatchParams):
1269
+ """One input of a run: a table or a read-only SQL, fetched as the org's
1270
+ read-only identity before the code runs and handed to it as Parquet."""
1271
+
1272
+ name: str = Field(
1273
+ pattern=PYTHON_INPUT_NAME_PATTERN,
1274
+ description=(
1275
+ "What the code reads it as: /work/in/<name>.parquet (the directory is also "
1276
+ "os.environ['DATASPRING_INPUTS']). Lower-case letters, digits and '_'."
1277
+ ),
1278
+ )
1279
+ table: str | None = Field(
1280
+ default=None,
1281
+ description=(
1282
+ "A table or view of your organization, as <org>_<layer>.<table> (acme_marts.fct_orders), "
1283
+ "<layer>.<table> (marts.fct_orders; layers marts, core, staging, raw, dev) or a bare table "
1284
+ "name in marts. Exactly one of table and sql."
1285
+ ),
1286
+ )
1287
+ sql: str | None = Field(
1288
+ default=None,
1289
+ description=(
1290
+ "A read-only SQL statement whose rows are the input (the same rules as run_sql: it runs "
1291
+ "as the reader identity, dry-run first and byte-capped). Exactly one of table and sql."
1292
+ ),
1293
+ )
1294
+
1295
+ @model_validator(mode="after")
1296
+ def _one_source(self) -> "PythonInput":
1297
+ if bool(self.table and self.table.strip()) == bool(self.sql and self.sql.strip()):
1298
+ raise ValueError(f"input {self.name}: give exactly one of table and sql")
1299
+ return self
1300
+
1301
+
1302
+ class RunPythonAction(DispatchParams):
1303
+ """Run Python over the org's data in an isolated sandbox (member)."""
1304
+
1305
+ action: Literal["run"]
1306
+ code: str = Field(
1307
+ max_length=PYTHON_MAX_CODE_BYTES,
1308
+ description=(
1309
+ "The Python program, run as /work/code/main.py with /work as the working directory. "
1310
+ "Read inputs from /work/in/<name>.parquet (pandas.read_parquet); write files to "
1311
+ "/work/out/ (png, csv, json, parquet, txt; at most 20, 10 MB each); print what the "
1312
+ "answer needs (stdout and stderr come back, 1 MB each). No network, no credentials."
1313
+ ),
1314
+ )
1315
+ inputs: list[PythonInput] = Field(
1316
+ default_factory=list,
1317
+ max_length=PYTHON_MAX_INPUTS,
1318
+ description=(
1319
+ f"Up to {PYTHON_MAX_INPUTS} inputs, each {{name, table}} or {{name, sql}}. All of them "
1320
+ "together at most 2,000,000 rows and 500 MB, checked before anything runs; aggregate "
1321
+ "in SQL when the raw rows are more than the analysis needs."
1322
+ ),
1323
+ )
1324
+ timeout_seconds: int = Field(
1325
+ default=PYTHON_DEFAULT_TIMEOUT_SECONDS,
1326
+ ge=1,
1327
+ le=PYTHON_MAX_TIMEOUT_SECONDS,
1328
+ description=(
1329
+ f"Wall-clock limit for the code (default {PYTHON_DEFAULT_TIMEOUT_SECONDS}, at most "
1330
+ f"{PYTHON_MAX_TIMEOUT_SECONDS}). For now at most 45: longer runs are refused until they "
1331
+ "run as their own job. A longer run will answer a run_id to poll with status."
1332
+ ),
1333
+ )
1334
+ warehouse: str | None = Field(
1335
+ default=None,
1336
+ description="One of the org's warehouses (see dataspring://warehouses); omit for the active one.",
1337
+ )
1338
+
1339
+
1340
+ class PythonRunStatusAction(DispatchParams):
1341
+ """Where a run is: its status, exit code, CPU and wall seconds, files."""
1342
+
1343
+ action: Literal["status"]
1344
+ run_id: str = Field(description="The run, from run's answer or list")
1345
+
1346
+
1347
+ class PythonRunResultAction(DispatchParams):
1348
+ """A finished run's whole answer: stdout, stderr and the file links."""
1349
+
1350
+ action: Literal["result"]
1351
+ run_id: str = Field(description="The run, from run's answer or list")
1352
+
1353
+
1354
+ class ListPythonRunsAction(DispatchParams):
1355
+ """The org's recent runs, newest first, and its allowance this month."""
1356
+
1357
+ action: Literal["list"]
1358
+ limit: int = Field(default=20, ge=1, le=100, description="Runs to list (newest first; default 20)")
1359
+
1360
+
1361
+ PythonRunAction = Annotated[
1362
+ RunPythonAction | PythonRunStatusAction | PythonRunResultAction | ListPythonRunsAction,
1363
+ Field(discriminator="action"),
1364
+ ]
1365
+
1366
+
1367
+ class PythonRunParams(DispatchParams):
1368
+ """The ``python_run`` call: one ``action`` from the family's union."""
1369
+
1370
+ action: PythonRunAction
1371
+
1372
+
1251
1373
  # ---------------------------------------------------------------------------
1252
1374
  # Tenant lifecycle action types (D24)
1253
1375
  # ---------------------------------------------------------------------------
@@ -55,6 +55,7 @@ MODELS: dict[str, type[contract.DispatchParams]] = {
55
55
  'board_edit': contract.BoardEditParams,
56
56
  'board_fetch': contract.BoardFetchParams,
57
57
  'watch_edit': contract.WatchEditParams,
58
+ 'python_run': contract.PythonRunParams,
58
59
  }
59
60
 
60
61
  #: The verbs of every edit family, in the order the action union declares them.
@@ -73,6 +74,7 @@ VERBS: dict[str, tuple[str, ...]] = {
73
74
  'tenant_edit': ('create', 'suspend', 'resume', 'offboard'),
74
75
  'board_edit': ('apply', 'patch', 'delete', 'convert_dashboard'),
75
76
  'watch_edit': ('apply', 'patch', 'delete', 'rotate_secret'),
77
+ 'python_run': ('run', 'status', 'result', 'list'),
76
78
  }
77
79
 
78
80
 
@@ -482,7 +484,7 @@ def warehouse_edit__activate(
482
484
 
483
485
  @warehouse_edit.command('add_external', short_help='Register a customer-owned BigQuery project as an external warehouse.')
484
486
  def warehouse_edit__add_external(
485
- label: Annotated[str, typer.Option('--label', help='Display name, e.g. "Noon\'s BigQuery"')],
487
+ label: Annotated[str, typer.Option('--label', help='Display name, e.g. "Acme\'s BigQuery"')],
486
488
  project: Annotated[str, typer.Option('--project', help="The customer's GCP project id")],
487
489
  dataset: Annotated[str, typer.Option('--dataset', help='The dataset the semantic manifest points at')],
488
490
  location: Annotated[str, typer.Option('--location', help="BigQuery location of that dataset, e.g. 'europe-north2'")],
@@ -749,7 +751,7 @@ secret_edit = typer.Typer(name='secret_edit', help='Customer secrets, both direc
749
751
 
750
752
  @secret_edit.command('set', short_help='Get a one-time link where a PERSON pastes a credential. The value never comes through here; the same name rotates a credential.')
751
753
  def secret_edit__set(
752
- name: Annotated[str, typer.Option('--name', help='The secret to set or rotate: t-<org>-<connection>-<field> for your organization, lower-case (t-acme-kanpla-api_key, t-acme-external-bigquery-key). A connection file names it.')],
754
+ name: Annotated[str, typer.Option('--name', help='The secret to set or rotate: t-<org>-<connection>-<field> for your organization, lower-case (t-acme-kanpla-api-key, t-acme-external-bigquery-key). A connection file names it.')],
753
755
  from_file: FromFile = None,
754
756
  output: Output = None,
755
757
  ) -> None:
@@ -777,7 +779,7 @@ def secret_edit__reveal(
777
779
 
778
780
 
779
781
  def run_sql(
780
- sql: Annotated[str, typer.Option('--sql', help="One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse.")],
782
+ sql: Annotated[str, typer.Option('--sql', help="One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse. On an external warehouse (the customer's own key) only ONE read statement runs: SELECT, WITH ... SELECT, EXPLAIN, DESCRIBE/SHOW or an INFORMATION_SCHEMA select, in that warehouse's dialect; a script, DML, DDL, MERGE, CALL, EXPORT DATA, LOAD DATA, GRANT, SET, SELECT ... INTO or unparseable SQL is refused as statement_not_allowed.")],
781
783
  warehouse: Annotated[Optional[str], typer.Option('--warehouse', help="One of the org's warehouses (see dataspring://warehouses); omit for the active one.")] = None,
782
784
  max_rows: Annotated[Optional[int], typer.Option('--max-rows', help='Rows to return at most (default 1000, cap 10000). Aggregate in SQL rather than paging. (default: 1000)')] = None,
783
785
  format: Annotated[Optional[str], typer.Option('--format', help="'json' answers rows in `data`; 'csv' or 'markdown' answer one document in `content`. (one of: json, csv, markdown) (default: json)")] = None,
@@ -785,8 +787,9 @@ def run_sql(
785
787
  from_file: FromFile = None,
786
788
  output: Output = None,
787
789
  ) -> None:
788
- """Run read-only SQL (one statement, or a script whose last statement
789
- answers) against the org's warehouse and get the rows back, ungoverned.
790
+ """Run read-only SQL (one statement, or on the managed warehouse a script
791
+ whose last statement answers) against the org's warehouse and get the
792
+ rows back, ungoverned.
790
793
 
791
794
  Metrics first: ``list_metrics`` / ``query_metrics`` answer with the
792
795
  org's own definitions (units, filters, caveats) and are what a number
@@ -807,8 +810,13 @@ def run_sql(
807
810
  in it is scratch space that ends with the script. It is dry-run first
808
811
  and refused over the byte cap;
809
812
  the timeout and ``max_rows`` bound it further. On an external
810
- warehouse it runs with the customer's own key. Every call is logged
811
- with the statement and the caller.
813
+ warehouse it runs with the customer's own key, so there only ONE read
814
+ statement runs: SELECT (WITH ... SELECT), EXPLAIN, DESCRIBE / SHOW and
815
+ INFORMATION_SCHEMA selects. A script, DML, DDL, MERGE, CALL, EXPORT
816
+ DATA, LOAD DATA, GRANT, SET, SELECT ... INTO or anything that cannot be
817
+ read as SQL is refused (``statement_not_allowed``); change data in the
818
+ warehouse with its own tools. Every call is logged with the statement
819
+ and the caller.
812
820
  """
813
821
  run_tool('run_sql', contract.RunSqlParams, None, {'sql': sql, 'warehouse': warehouse, 'max_rows': max_rows, 'format': format, 'dry_run': dry_run}, from_file=from_file, output=output)
814
822
 
@@ -1168,6 +1176,57 @@ def watch_edit__rotate_secret(
1168
1176
  run_tool('watch_edit', contract.WatchEditParams, 'rotate_secret', {'id': id}, from_file=from_file, output=output)
1169
1177
 
1170
1178
 
1179
+ # ---------------------------------------------------------------------------
1180
+ # python_run
1181
+ # ---------------------------------------------------------------------------
1182
+
1183
+
1184
+ python_run = typer.Typer(name='python_run', help="Run Python (pandas, numpy, scipy, statsmodels, scikit-learn, matplotlib, pyarrow) over the org's own data, for what SQL cannot do: a regression, a forecast, a clustering, a statistical test, a chart. Member. Numbers from it are ungoverned, like run_sql's: prefer query_metrics for anything a metric defines, and say which is which.", no_args_is_help=True)
1185
+
1186
+
1187
+ @python_run.command('run', short_help="Run Python over the org's data in an isolated sandbox (member).")
1188
+ def python_run__run(
1189
+ code: Annotated[str, typer.Option('--code', help='The Python program, run as /work/code/main.py with /work as the working directory. Read inputs from /work/in/<name>.parquet (pandas.read_parquet); write files to /work/out/ (png, csv, json, parquet, txt; at most 20, 10 MB each); print what the answer needs (stdout and stderr come back, 1 MB each). No network, no credentials.')],
1190
+ inputs: Annotated[Optional[str], typer.Option('--inputs-json', help='Up to 8 inputs, each {name, table} or {name, sql}. All of them together at most 2,000,000 rows and 500 MB, checked before anything runs; aggregate in SQL when the raw rows are more than the analysis needs. (JSON, or @path to a .json/.yaml file) (default: [])')] = None,
1191
+ timeout_seconds: Annotated[Optional[int], typer.Option('--timeout-seconds', help='Wall-clock limit for the code (default 120, at most 600). For now at most 45: longer runs are refused until they run as their own job. A longer run will answer a run_id to poll with status. (default: 120)')] = None,
1192
+ warehouse: Annotated[Optional[str], typer.Option('--warehouse', help="One of the org's warehouses (see dataspring://warehouses); omit for the active one.")] = None,
1193
+ from_file: FromFile = None,
1194
+ output: Output = None,
1195
+ ) -> None:
1196
+ """Run Python over the org's data in an isolated sandbox (member)."""
1197
+ run_tool('python_run', contract.PythonRunParams, 'run', {'code': code, 'inputs': J(inputs), 'timeout_seconds': timeout_seconds, 'warehouse': warehouse}, from_file=from_file, output=output)
1198
+
1199
+
1200
+ @python_run.command('status', short_help='Where a run is: its status, exit code, CPU and wall seconds, files.')
1201
+ def python_run__status(
1202
+ run_id: Annotated[str, typer.Option('--run-id', help="The run, from run's answer or list")],
1203
+ from_file: FromFile = None,
1204
+ output: Output = None,
1205
+ ) -> None:
1206
+ """Where a run is: its status, exit code, CPU and wall seconds, files."""
1207
+ run_tool('python_run', contract.PythonRunParams, 'status', {'run_id': run_id}, from_file=from_file, output=output)
1208
+
1209
+
1210
+ @python_run.command('result', short_help="A finished run's whole answer: stdout, stderr and the file links.")
1211
+ def python_run__result(
1212
+ run_id: Annotated[str, typer.Option('--run-id', help="The run, from run's answer or list")],
1213
+ from_file: FromFile = None,
1214
+ output: Output = None,
1215
+ ) -> None:
1216
+ """A finished run's whole answer: stdout, stderr and the file links."""
1217
+ run_tool('python_run', contract.PythonRunParams, 'result', {'run_id': run_id}, from_file=from_file, output=output)
1218
+
1219
+
1220
+ @python_run.command('list', short_help="The org's recent runs, newest first, and its allowance this month.")
1221
+ def python_run__list(
1222
+ limit: Annotated[Optional[int], typer.Option('--limit', help='Runs to list (newest first; default 20) (default: 20)')] = None,
1223
+ from_file: FromFile = None,
1224
+ output: Output = None,
1225
+ ) -> None:
1226
+ """The org's recent runs, newest first, and its allowance this month."""
1227
+ run_tool('python_run', contract.PythonRunParams, 'list', {'limit': limit}, from_file=from_file, output=output)
1228
+
1229
+
1171
1230
  def mount(app: typer.Typer) -> None:
1172
1231
  """Add every generated command to ``app``: flat tools at the root, families as groups."""
1173
1232
  app.add_typer(business_context_edit, name='business_context_edit')
@@ -1186,10 +1245,11 @@ def mount(app: typer.Typer) -> None:
1186
1245
  app.command('import_manifest', short_help='Import a semantic manifest using replace semantics.')(import_manifest)
1187
1246
  app.command('submit_error_report', short_help='Submit an error report to help improve DataSpring.')(submit_error_report)
1188
1247
  app.add_typer(secret_edit, name='secret_edit')
1189
- app.command('run_sql', short_help="Run read-only SQL (one statement, or a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.")(run_sql)
1248
+ app.command('run_sql', short_help="Run read-only SQL (one statement, or on the managed warehouse a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.")(run_sql)
1190
1249
  app.add_typer(datacore_edit, name='datacore_edit')
1191
1250
  app.add_typer(datacore_run, name='datacore_run')
1192
1251
  app.add_typer(tenant_edit, name='tenant_edit')
1193
1252
  app.add_typer(board_edit, name='board_edit')
1194
1253
  app.command('board_fetch', short_help='Run every row of a board in one call and answer salience first.')(board_fetch)
1195
1254
  app.add_typer(watch_edit, name='watch_edit')
1255
+ app.add_typer(python_run, name='python_run')
@@ -234,7 +234,7 @@ def _request(method: str, path: str, token: str, *, body: dict | None = None, ti
234
234
  import httpx
235
235
 
236
236
  url = f"{get_api_base()}{path}"
237
- headers = {"Authorization": f"Bearer {token}"}
237
+ headers = {"Authorization": f"Bearer {token}", "User-Agent": _user_agent()}
238
238
  attempts = 2 if retry else 1
239
239
  with transport.ColdStartIndicator() as indicator, httpx.Client(timeout=timeout) as client:
240
240
  for attempt in range(attempts):
@@ -252,6 +252,37 @@ def _request(method: str, path: str, token: str, *, body: dict | None = None, ti
252
252
  return response.json()
253
253
 
254
254
 
255
+ def _user_agent() -> str:
256
+ """``dataspring-cli/<version>``: how the server tells the CLI from other
257
+ callers (python_run records which client ran a program)."""
258
+ from cli.version import cli_version
259
+
260
+ return f"dataspring-cli/{cli_version()}"
261
+
262
+
263
+ def api_get_bytes(path: str, token: str, timeout: float = API_TIMEOUT) -> bytes:
264
+ """An authenticated GET whose answer is a file, not JSON (a python run's
265
+ outputs). Retried once like ``api_get``; errors as ``_request``'s."""
266
+ import httpx
267
+
268
+ url = f"{get_api_base()}{path}"
269
+ headers = {"Authorization": f"Bearer {token}", "User-Agent": _user_agent()}
270
+ with transport.ColdStartIndicator() as indicator, httpx.Client(timeout=timeout) as client:
271
+ for attempt in range(2):
272
+ try:
273
+ response = client.get(url, headers=headers)
274
+ break
275
+ except transport.RETRYABLE as e:
276
+ if attempt == 0:
277
+ indicator.mark_retrying()
278
+ continue
279
+ raise transport.unreachable(e, get_api_base(), timeout) from e
280
+ except httpx.TransportError as e:
281
+ raise transport.unreachable(e, get_api_base(), timeout) from e
282
+ response.raise_for_status()
283
+ return response.content
284
+
285
+
255
286
  def api_get(path: str, token: str, timeout: float = API_TIMEOUT) -> dict:
256
287
  """Make an authenticated GET request to the backend API (idempotent:
257
288
  retried once on a timeout or a failed connection)."""
@@ -2616,8 +2647,8 @@ def secret_set_cmd(
2616
2647
  """Set or rotate a customer secret from stdin, never from an argument.
2617
2648
 
2618
2649
  Examples:
2619
- printf '%s' "$KEY" | dataspring secret set t-acme-kanpla-api_key --stdin
2620
- dataspring secret set t-acme-kanpla-api_key --stdin < key.txt
2650
+ printf '%s' "$KEY" | dataspring secret set t-acme-kanpla-api-key --stdin
2651
+ dataspring secret set t-acme-kanpla-api-key --stdin < key.txt
2621
2652
  """
2622
2653
  require_auth()
2623
2654
  token = auth.get_access_token()
@@ -2711,6 +2742,14 @@ def sql_cmd(
2711
2742
  ends with the script. Dry-run first, refused over the byte cap; at most
2712
2743
  --max-rows rows (cap 10000), so aggregate in SQL.
2713
2744
 
2745
+ On an external warehouse (--warehouse external: the customer's own key,
2746
+ which can write) only ONE read statement runs: SELECT, WITH ... SELECT,
2747
+ EXPLAIN, DESCRIBE / SHOW, INFORMATION_SCHEMA selects. A script, INSERT,
2748
+ UPDATE, DELETE, MERGE, CREATE, DROP, ALTER, TRUNCATE, GRANT, CALL,
2749
+ EXPORT DATA, LOAD DATA, SET, SELECT ... INTO or SQL that cannot be read
2750
+ is refused with statement_not_allowed (exit 4); change data there with
2751
+ your own tools.
2752
+
2714
2753
  Examples:
2715
2754
  dataspring sql "SELECT venue, SUM(revenue) FROM acme_marts.fct_orders GROUP BY 1"
2716
2755
  dataspring sql "SELECT * FROM acme_raw.kanpla_orders LIMIT 20"
@@ -3746,6 +3785,246 @@ def skill_instructions(
3746
3785
  typer.echo(_fetch_skill_markdown(f"/api/v1/skill/{persona}/instructions"))
3747
3786
 
3748
3787
 
3788
+ # ============================================================================
3789
+ # python: hosted Python over the org's data (epic kfh, tick r2v)
3790
+ # ============================================================================
3791
+ #
3792
+ # `dataspring python run <file.py> --input name=<table> --input name='sql:...'`
3793
+ # is `python_run run` with the file as the code; `runs` is `list`; `result`
3794
+ # is `result` (with --wait, `status` polled first). Files come down through
3795
+ # GET /api/python/runs/<id>/files/<name> with the CLI's own credential.
3796
+
3797
+ python_app = typer.Typer(
3798
+ help="Hosted Python over the org's data: run a file in a sandbox with your tables as inputs, list runs, "
3799
+ "fetch a result and its files (python_run)",
3800
+ no_args_is_help=True,
3801
+ )
3802
+ app.add_typer(python_app, name="python")
3803
+
3804
+ #: How often --wait asks, and for how long at most.
3805
+ PYTHON_POLL_SECONDS = 3.0
3806
+ PYTHON_WAIT_SECONDS = 1800.0
3807
+
3808
+
3809
+ def _python_input(spec: str) -> dict:
3810
+ """``name=<table>`` or ``name=sql:<statement>`` (``sql:@file.sql`` reads a file)."""
3811
+ name, sep, source = spec.partition("=")
3812
+ if not sep or not name or not source:
3813
+ print_error(f"--input {spec!r}: expected name=<table> or name=sql:<SELECT ...>")
3814
+ raise typer.Exit(2)
3815
+ if source.startswith("sql:"):
3816
+ sql = source[4:]
3817
+ if sql.startswith("@"):
3818
+ sql = Path(sql[1:]).read_text()
3819
+ return {"name": name, "sql": sql}
3820
+ return {"name": name, "table": source}
3821
+
3822
+
3823
+ def _python_save(result: dict, directory: Path, token: str) -> list[Path]:
3824
+ directory.mkdir(parents=True, exist_ok=True)
3825
+ saved = []
3826
+ for f in result.get("files") or []:
3827
+ name = f["name"]
3828
+ data = api_get_bytes(f"/api/python/runs/{result['run_id']}/files/{name}", token)
3829
+ target = directory / Path(name).name
3830
+ target.write_bytes(data)
3831
+ saved.append(target)
3832
+ return saved
3833
+
3834
+
3835
+ def _python_report(result: dict, *, output_dir: Optional[Path], token: str, as_json: bool) -> None:
3836
+ """Print a finished run: its stdout on stdout, its stderr and a summary
3837
+ on stderr (so `> out.txt` captures exactly what the program printed),
3838
+ then exit as the program did (0, or 1 when it failed or timed out)."""
3839
+ saved = _python_save(result, output_dir, token) if output_dir and result.get("files") else []
3840
+ if as_json:
3841
+ typer.echo(json.dumps(result, indent=2, default=str))
3842
+ else:
3843
+ if result.get("stdout"):
3844
+ sys.stdout.write(result["stdout"])
3845
+ if not result["stdout"].endswith("\n"):
3846
+ sys.stdout.write("\n")
3847
+ if result.get("stderr"):
3848
+ sys.stderr.write(result["stderr"])
3849
+ if not result["stderr"].endswith("\n"):
3850
+ sys.stderr.write("\n")
3851
+ usage = result.get("usage") or {}
3852
+ cpu = result.get("cpu_seconds", usage.get("cpu_seconds"))
3853
+ wall = result.get("wall_seconds", usage.get("wall_seconds"))
3854
+ line = f"python run {result.get('run_id')}: {result.get('status')}"
3855
+ if result.get("exit_code") is not None:
3856
+ line += f", exit {result['exit_code']}"
3857
+ if result.get("exit_reason"):
3858
+ line += f" ({result['exit_reason']})"
3859
+ if cpu is not None:
3860
+ line += f", {cpu} CPU-s"
3861
+ if wall is not None:
3862
+ line += f", {wall} s"
3863
+ typer.echo(line, err=True)
3864
+ if result.get("error"):
3865
+ typer.echo(f" error: {result['error']}", err=True)
3866
+ if result.get("stdout_truncated") or result.get("stderr_truncated"):
3867
+ typer.echo(" output truncated at 1 MB", err=True)
3868
+ for target in saved:
3869
+ typer.echo(f" saved {target}", err=True)
3870
+ if not saved:
3871
+ for f in result.get("files") or []:
3872
+ typer.echo(f" file {f['name']} ({f.get('bytes')} bytes): {f.get('url')}", err=True)
3873
+ for d in result.get("dropped") or []:
3874
+ typer.echo(f" dropped {d.get('name')}: {d.get('reason')}", err=True)
3875
+ for w in result.get("warnings") or []:
3876
+ typer.echo(f" warning: {w}", err=True)
3877
+ if result.get("status") != "succeeded" or result.get("error"):
3878
+ raise typer.Exit(1)
3879
+
3880
+
3881
+ def _python_wait(run_id: str, token: str) -> dict:
3882
+ """Poll ``status`` until the run has finished, then fetch ``result``."""
3883
+ import time
3884
+
3885
+ deadline = time.monotonic() + PYTHON_WAIT_SECONDS
3886
+ last = None
3887
+ while True:
3888
+ state = call_tool("python_run", "status", {"run_id": run_id}, token=token, timeout=API_TIMEOUT)
3889
+ if state.get("status") != last:
3890
+ last = state.get("status")
3891
+ print_info(f"python run {run_id}: {last}")
3892
+ if state.get("finished"):
3893
+ return call_tool("python_run", "result", {"run_id": run_id}, token=token, timeout=API_TIMEOUT)
3894
+ if time.monotonic() > deadline:
3895
+ print_error(f"python run {run_id} still {last} after {int(PYTHON_WAIT_SECONDS)} s; it carries on",
3896
+ hint=f"dataspring python result {run_id} --wait")
3897
+ raise typer.Exit(1)
3898
+ time.sleep(PYTHON_POLL_SECONDS)
3899
+
3900
+
3901
+ @python_app.command("run")
3902
+ def python_run_cmd(
3903
+ file: Annotated[str, typer.Argument(help="The Python file to run; '-' reads it from stdin")],
3904
+ input: Annotated[
3905
+ Optional[list[str]],
3906
+ typer.Option("--input", "-i", help="An input: name=<table> (marts.fct_orders, acme_raw.orders) or "
3907
+ "name=sql:<SELECT ...> (sql:@query.sql reads a file); repeatable, at most 8. The code reads "
3908
+ "it as /work/in/<name>.parquet"),
3909
+ ] = None,
3910
+ timeout: Annotated[int, typer.Option("--timeout", help="Wall-clock limit in seconds (at most 600)")] = 120,
3911
+ warehouse: Annotated[Optional[str], typer.Option("--warehouse", help="Which of the org's warehouses; default the active one")] = None,
3912
+ wait: Annotated[bool, typer.Option("--wait", help="If the run outlives the inline answer (about 45 s), poll it to its end")] = False,
3913
+ output_dir: Annotated[Optional[Path], typer.Option("--output-dir", "-o", help="Download the files the run kept into this directory")] = None,
3914
+ as_json: Annotated[bool, typer.Option("--json", help="Print the whole answer as JSON")] = False,
3915
+ ):
3916
+ """Run a Python file over the org's data in an isolated sandbox (python_run).
3917
+
3918
+ Each --input is read as the org's read-only identity before the code
3919
+ runs (all of them at most 2,000,000 rows and 500 MB; aggregate in SQL
3920
+ first) and is there as /work/in/<name>.parquet. The sandbox has pandas,
3921
+ numpy, scipy, statsmodels, scikit-learn, matplotlib and pyarrow, and no
3922
+ network and no credentials: everything the code needs goes in as an
3923
+ input. Write charts and tables to /work/out/ (png, csv, json, parquet,
3924
+ txt; at most 20 files of 10 MB), kept 30 days; -o downloads them.
3925
+
3926
+ The program's stdout is printed on stdout, its stderr and a summary on
3927
+ stderr; the command exits 0 when the program did, else 1. A run still
3928
+ going after about 45 s prints its run id; --wait polls it to its end
3929
+ (or later: dataspring python result <id> --wait). A run is never retried.
3930
+ The org's allowance (runs per day, CPU-seconds per month) refuses with
3931
+ allowance_exhausted (exit 7) when it is used up.
3932
+
3933
+ Examples:
3934
+ dataspring python run analysis.py --input orders=marts.fct_orders -o out/
3935
+ dataspring python run forecast.py -i daily='sql:SELECT day, SUM(revenue) r FROM acme_marts.fct_orders GROUP BY 1' --wait
3936
+ dataspring python runs
3937
+ dataspring python result py-01jb... -o out/
3938
+ """
3939
+ require_auth()
3940
+ token = auth.get_access_token()
3941
+ try:
3942
+ code = sys.stdin.read() if file == "-" else Path(file).read_text()
3943
+ except OSError as e:
3944
+ print_error(f"Could not read {file}: {e}")
3945
+ raise typer.Exit(2)
3946
+ body: dict = {"code": code, "timeout_seconds": timeout}
3947
+ inputs = [_python_input(spec) for spec in input or []]
3948
+ if inputs:
3949
+ body["inputs"] = inputs
3950
+ if warehouse:
3951
+ body["warehouse"] = warehouse
3952
+ try:
3953
+ result = call_tool("python_run", "run", body, token=token, timeout=API_TIMEOUT + 30)
3954
+ if result.get("status") in ("queued", "running"):
3955
+ if not wait:
3956
+ if as_json:
3957
+ typer.echo(json.dumps(result, indent=2, default=str))
3958
+ else:
3959
+ print_info(f"python run {result['run_id']} is still running; it carries on and is never retried")
3960
+ typer.echo(f" dataspring python result {result['run_id']} --wait", err=True)
3961
+ return
3962
+ result = _python_wait(result["run_id"], token)
3963
+ _python_report(result, output_dir=output_dir, token=token, as_json=as_json)
3964
+ except Exception as e:
3965
+ fail(e, role_action="run Python", prefix="python run failed")
3966
+
3967
+
3968
+ @python_app.command("runs")
3969
+ def python_runs_cmd(
3970
+ limit: Annotated[int, typer.Option("--limit", help="How many, newest first (at most 100)")] = 20,
3971
+ as_json: Annotated[bool, typer.Option("--json", help="Print the whole answer as JSON")] = False,
3972
+ ):
3973
+ """The org's recent python runs and its allowance this month.
3974
+
3975
+ Example:
3976
+ dataspring python runs --limit 5
3977
+ """
3978
+ require_auth()
3979
+ token = auth.get_access_token()
3980
+ try:
3981
+ result = call_tool("python_run", "list", {"limit": limit}, token=token, timeout=API_TIMEOUT)
3982
+ except Exception as e:
3983
+ fail(e, role_action="list python runs")
3984
+ if as_json:
3985
+ typer.echo(json.dumps(result, indent=2, default=str))
3986
+ return
3987
+ rows = [{"run": r.get("run_id"), "status": r.get("status"), "exit": r.get("exit_code"),
3988
+ "cpu_s": r.get("cpu_seconds"), "files": r.get("files"), "by": r.get("person"),
3989
+ "created": r.get("created_at")} for r in result.get("runs") or []]
3990
+ format_output(rows, format="table", columns=["run", "status", "exit", "cpu_s", "files", "by", "created"],
3991
+ title=f"Python runs ({len(rows)})")
3992
+ allow = result.get("allowance") or {}
3993
+ if allow:
3994
+ typer.echo(
3995
+ f"allowance {allow.get('month')}: {allow.get('cpu_seconds')} of {allow.get('cpu_seconds_per_month')} "
3996
+ f"CPU-s; {allow.get('runs_today')} of {allow.get('runs_per_day')} runs today", err=True)
3997
+
3998
+
3999
+ @python_app.command("result")
4000
+ def python_result_cmd(
4001
+ run_id: Annotated[str, typer.Argument(help="The run (py-...), from `python run` or `python runs`")],
4002
+ wait: Annotated[bool, typer.Option("--wait", help="Poll a run still going to its end")] = False,
4003
+ output_dir: Annotated[Optional[Path], typer.Option("--output-dir", "-o", help="Download the files the run kept into this directory")] = None,
4004
+ as_json: Annotated[bool, typer.Option("--json", help="Print the whole answer as JSON")] = False,
4005
+ ):
4006
+ """A python run's result: its stdout, stderr and files (-o downloads them).
4007
+
4008
+ Exits 0 when the program succeeded, 1 when it failed or is still
4009
+ running (without --wait).
4010
+
4011
+ Example:
4012
+ dataspring python result py-01jb... --wait -o out/
4013
+ """
4014
+ require_auth()
4015
+ token = auth.get_access_token()
4016
+ try:
4017
+ result = _python_wait(run_id, token) if wait else call_tool(
4018
+ "python_run", "result", {"run_id": run_id}, token=token, timeout=API_TIMEOUT)
4019
+ if not result.get("finished", True) and result.get("status") in ("queued", "running"):
4020
+ print_info(f"python run {run_id} is still {result.get('status')}")
4021
+ typer.echo(f" dataspring python result {run_id} --wait", err=True)
4022
+ raise typer.Exit(1)
4023
+ _python_report(result, output_dir=output_dir, token=token, as_json=as_json)
4024
+ except Exception as e:
4025
+ fail(e, role_action="read python runs")
4026
+
4027
+
3749
4028
  # ============================================================================
3750
4029
  # The generated tree: every registry operation, by its key
3751
4030
  # ============================================================================
@@ -8,6 +8,18 @@ codes are suite standards §6's (0 ok, 1 failure, 2 invalid input, 3 not
8
8
  found, 4 forbidden, 5 invalid state, 6 conflict, 7 rate limited, 64 usage).
9
9
  ``EN`` is the package's English text, reused where a message needs it.
10
10
 
11
+ ``allowance_exhausted`` is the ninth (hub D-105, 2026-10-23: the suite's
12
+ metered products refuse with it when an org's allowance is used up, the fix
13
+ being the account's billing page). It is not in ``@forspring/ui`` yet (its
14
+ latest tag, v0.10.0, has the eight), so its status and exit code are the
15
+ suite standards' for rate limiting, 429 and exit 7, until the package names
16
+ them; DataSpring's ``python_run`` is its first user (tick r2v).
17
+
18
+ ``statement_not_allowed`` is DataSpring's own beside them (tick w3c):
19
+ ``run_sql`` on an external warehouse runs only a single read statement and
20
+ refuses anything else with it. It is not an access refusal and not in
21
+ ``@forspring/ui``; it takes the suite standards' forbidden, 403 and exit 4.
22
+
11
23
  This module ships in the CLI wheel (it imports nothing but the standard
12
24
  library) and the server imports it too (``mcp_caller``), so the MCP's
13
25
  ``ToolError`` text and the CLI's error line spell a refusal the same way:
@@ -43,6 +55,10 @@ EXIT_CODES: dict[str, int] = {
43
55
  "role_too_low": EXIT_FORBIDDEN,
44
56
  "org_mismatch": EXIT_FORBIDDEN,
45
57
  "account_unavailable": EXIT_FAILURE,
58
+ # Hub D-105; not in @forspring/ui v0.10.0 yet: suite standards §6's 7.
59
+ "allowance_exhausted": EXIT_RATE_LIMITED,
60
+ # DataSpring's own (tick w3c), not in @forspring/ui: forbidden, exit 4.
61
+ "statement_not_allowed": EXIT_FORBIDDEN,
46
62
  }
47
63
 
48
64
  #: The HTTP status for each (``REFUSALS[reason].status``).
@@ -55,6 +71,8 @@ HTTP_STATUS: dict[str, int] = {
55
71
  "role_too_low": 403,
56
72
  "org_mismatch": 403,
57
73
  "account_unavailable": 503,
74
+ "allowance_exhausted": 429,
75
+ "statement_not_allowed": 403,
58
76
  }
59
77
 
60
78
  #: ``REFUSALS[reason].en``: the package's English title, text and fix.
@@ -99,6 +117,16 @@ EN: dict[str, dict[str, str]] = {
99
117
  "text": "Forspring's account is not answering, so we cannot check your access.",
100
118
  "fix": "Nothing is lost; try again in a minute.",
101
119
  },
120
+ "allowance_exhausted": {
121
+ "title": "The organization's allowance is used up",
122
+ "text": "This would use more than the organization has left this period.",
123
+ "fix": "An owner can raise the allowance on the billing page.",
124
+ },
125
+ "statement_not_allowed": {
126
+ "title": "Only read statements run here",
127
+ "text": "On an external warehouse run_sql runs one read statement at a time, and this is not one.",
128
+ "fix": "Change data in your warehouse with your own tools.",
129
+ },
102
130
  }
103
131
 
104
132
  REASONS = tuple(EXIT_CODES)
@@ -9,7 +9,10 @@ taken for pip, which uv environments do not have (tick ghl). A uv tool
9
9
  environment holds ``uv-receipt.toml`` (and lives under ``.../uv/tools/
10
10
  dataspring-cli/``), pipx's lives under ``.../pipx/venvs/dataspring-cli/``,
11
11
  and anything else is a pip install when pip is there, else unknown: then
12
- the CLI prints the commands to run instead of failing.
12
+ the CLI prints the commands to run instead of failing. A uv tool upgrades
13
+ with ``uv tool install dataspring-cli@latest``, not ``uv tool upgrade``:
14
+ the latter keeps an exact version pin and leaves a pinned tool where it is
15
+ with exit 0 (tick 250).
13
16
  ``--check`` only compares the installed version with PyPI's latest.
14
17
  """
15
18
 
@@ -29,7 +32,7 @@ PYPI_JSON = f"https://pypi.org/pypi/{DIST}/json"
29
32
 
30
33
 
31
34
  MANUAL_COMMANDS = (
32
- f"uv tool upgrade {DIST}",
35
+ f"uv tool install {DIST}@latest",
33
36
  f"pipx upgrade {DIST}",
34
37
  f"python -m pip install --upgrade {DIST}",
35
38
  )
@@ -51,7 +54,7 @@ def detect_install_method(prefix: str | None = None, *, has_pip: bool | None = N
51
54
  def upgrade_command(method: str) -> list[str] | None:
52
55
  """The command for ``method``; None when there is none to run (``unknown``)."""
53
56
  if method == "uv":
54
- return ["uv", "tool", "upgrade", DIST]
57
+ return ["uv", "tool", "install", f"{DIST}@latest"]
55
58
  if method == "pipx":
56
59
  return ["pipx", "upgrade", DIST]
57
60
  if method == "pip":
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "dataspring-cli"
3
- version = "0.5.1"
3
+ version = "0.6.0"
4
4
  description = "DataSpring CLI - Query metrics and manage dashboards from the terminal"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.12"
@@ -93,6 +93,16 @@ server = [
93
93
  # schemas (backend/generated/datacore_catalog.json)
94
94
  "jsonschema>=4.23",
95
95
  "ruamel-yaml>=0.18",
96
+ # python_run (tick r2v, services/python_run.py): inputs fetched as Arrow
97
+ # (through the Storage Read API, effectively free, rather than paging
98
+ # tabledata.list) and handed to the executor as Parquet, over HTTP/2
99
+ # (Cloud Run caps an HTTP/1 request at 32 MiB; inputs go to 500 MB).
100
+ "pyarrow>=21",
101
+ "google-cloud-bigquery-storage>=2.30",
102
+ "h2>=4.1",
103
+ # run_sql parses every statement on an external warehouse before it runs
104
+ # (services/sql_guard.py, tick w3c); was only MetricFlow's dependency.
105
+ "sqlglot>=30.18.0",
96
106
  ]
97
107
 
98
108
  [tool.uv]
@@ -79,6 +79,26 @@ class Settings(BaseSettings):
79
79
  # warehouse (BigQuery via `result(max_results=)`, DuckDB via `fetchmany`).
80
80
  query_max_rows: int = 10_000
81
81
 
82
+ # python_run (epic kfh, tick r2v; services/python_run.py). The executor
83
+ # is the Cloud Run service python-executor (scripts/setup-python-executor.sh).
84
+ # Unset, its deterministic URL is derived on Cloud Run from the project
85
+ # number (https://python-executor-<number>.europe-west1.run.app), so a
86
+ # deploy needs no setting; set it to point a backend at another one.
87
+ python_executor_url: str | None = None
88
+ # The local per-org allowance until the account's meter exists (hub
89
+ # D-105): runs per UTC day, sandbox CPU-seconds and stored output bytes
90
+ # per calendar month. An org's own values live in
91
+ # organizations/<org>/config/python ({runs_per_day, cpu_seconds_per_month,
92
+ # cpu_seconds_hard_cap, output_bytes_per_month}).
93
+ python_runs_per_day: int = 200
94
+ python_cpu_seconds_per_month: int = 36_000
95
+ python_output_bytes_per_month: int = 5 * 1024**3
96
+ # Runs that outlive the inline wait (over 45 s) carry on as a task on the
97
+ # backend instance, which request-based billing throttles between
98
+ # requests, so they can stall or be lost with the instance. Off until
99
+ # they run as their own Cloud Run job (tick filed on epic kfh).
100
+ python_long_runs_enabled: bool = False
101
+
82
102
  # Feature flags
83
103
  schedules_enabled: bool = False # Scheduled reports (email delivery not implemented yet)
84
104
 
@@ -308,6 +308,15 @@ over raw), `<org>_core`, `<org>_marts` and `<org>_dev` (what the last
308
308
  - A script of several statements runs as one job and answers the last
309
309
  one's rows; `CREATE TEMP TABLE` in it is scratch space for an analysis
310
310
  that ends with the script.
311
+ - On an **external** warehouse there is no read identity: the statement
312
+ runs with the customer's own key, so `run_sql` runs ONE read statement
313
+ at a time there (`SELECT`, `WITH ... SELECT`, `EXPLAIN`, `DESCRIBE` /
314
+ `SHOW`, `INFORMATION_SCHEMA` selects, in that warehouse's dialect) and
315
+ refuses a script, temp tables, any DML or DDL, `MERGE`, `CALL`, `EXPORT
316
+ DATA`, `LOAD DATA`, `GRANT`, `SET`, `SELECT ... INTO` and SQL it cannot
317
+ read with `statement_not_allowed`. Do not retry it another way: tell the
318
+ person to change data in their warehouse with their own tools, and use a
319
+ CTE where a temp table would have gone.
311
320
 
312
321
  ## 8.4 Workspace files keep the customer's text
313
322
 
@@ -632,7 +641,7 @@ MCP tool `warehouse_edit`; `POST /api/dispatch/warehouse_edit`. Manage the org's
632
641
 
633
642
  | Parameter | Type | Required | Description |
634
643
  |---|---|---|---|
635
- | `label` | string | yes | Display name, e.g. "Noon's BigQuery" |
644
+ | `label` | string | yes | Display name, e.g. "Acme's BigQuery" |
636
645
  | `project` | string | yes | The customer's GCP project id |
637
646
  | `dataset` | string | yes | The dataset the semantic manifest points at |
638
647
  | `location` | string | yes | BigQuery location of that dataset, e.g. 'europe-north2' |
@@ -804,11 +804,11 @@ MCP tool `explain_query`; `POST /api/dispatch/explain_query`. Show what a query
804
804
 
805
805
  #### `run_sql`
806
806
 
807
- MCP tool `run_sql`; `POST /api/dispatch/run_sql`. Run read-only SQL (one statement, or a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.
807
+ MCP tool `run_sql`; `POST /api/dispatch/run_sql`. Run read-only SQL (one statement, or on the managed warehouse a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.
808
808
 
809
809
  | Parameter | Type | Required | Description |
810
810
  |---|---|---|---|
811
- | `sql` | string | yes | One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse. |
811
+ | `sql` | string | yes | One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse. On an external warehouse (the customer's own key) only ONE read statement runs: SELECT, WITH ... SELECT, EXPLAIN, DESCRIBE/SHOW or an INFORMATION_SCHEMA select, in that warehouse's dialect; a script, DML, DDL, MERGE, CALL, EXPORT DATA, LOAD DATA, GRANT, SET, SELECT ... INTO or unparseable SQL is refused as statement_not_allowed. |
812
812
  | `warehouse` | string | no | One of the org's warehouses (see dataspring://warehouses); omit for the active one. |
813
813
  | `max_rows` | integer | no, default `1000` | Rows to return at most (default 1000, cap 10000). Aggregate in SQL rather than paging. |
814
814
  | `format` | one of `json`, `csv`, `markdown` | no, default `json` | 'json' answers rows in `data`; 'csv' or 'markdown' answer one document in `content`. |
File without changes