dataspring-cli 0.4.0__py3-none-any.whl → 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli/_skills/dataspring-author/SKILL.md +28 -1
- cli/_skills/dataspring-consume/SKILL.md +3 -3
- cli/auth.py +260 -49
- cli/bundled_manifest.py +1 -1
- cli/contract.py +63 -7
- cli/generated.py +30 -7
- cli/main.py +384 -61
- cli/refusals.py +249 -0
- cli/runtime.py +1 -2
- cli/upgrade.py +29 -10
- {dataspring_cli-0.4.0.dist-info → dataspring_cli-0.4.2.dist-info}/METADATA +9 -5
- dataspring_cli-0.4.2.dist-info/RECORD +21 -0
- dataspring_cli-0.4.0.dist-info/RECORD +0 -20
- {dataspring_cli-0.4.0.dist-info → dataspring_cli-0.4.2.dist-info}/WHEEL +0 -0
- {dataspring_cli-0.4.0.dist-info → dataspring_cli-0.4.2.dist-info}/entry_points.txt +0 -0
|
@@ -206,6 +206,18 @@ version produced each issue. Edit the board and the next issue follows.
|
|
|
206
206
|
and is what stops later history columns moving. Rendering and checking
|
|
207
207
|
prose is the consumer's job and is covered in `dataspring-consume`.
|
|
208
208
|
|
|
209
|
+
## 8.2 Iterating on a dbt model (managed tenants)
|
|
210
|
+
|
|
211
|
+
A model change does not need a load. The loop is `datacore_edit` (kind
|
|
212
|
+
`model`), `datacore_run` `check`, `deploy`, then `build`: a run that loads
|
|
213
|
+
no resource and runs `dbt build` of the deployed project on the raw tables
|
|
214
|
+
already there, with an optional `select` (default: the named pipeline's
|
|
215
|
+
`then.dbt_select`, or every model). `deploy` with `then_build: true` (and
|
|
216
|
+
`build_select`) queues it in the same call when the deploy changed a file
|
|
217
|
+
under `models/`. Do not use `run` to see a model change: it reloads every
|
|
218
|
+
resource of the pipeline first. CLI: `dataspring datacore build --select
|
|
219
|
+
"marts.finance+" --wait`.
|
|
220
|
+
|
|
209
221
|
## 9. Parameter reference
|
|
210
222
|
|
|
211
223
|
<!-- dispatch-params: semantic_model_edit metric_edit board_edit briefing_edit import_manifest business_context_edit warehouse_edit datacore_edit datacore_run -->
|
|
@@ -461,20 +473,35 @@ MCP tool `datacore_run`; `POST /api/dispatch/datacore_run`. The datacore's check
|
|
|
461
473
|
|---|---|---|---|
|
|
462
474
|
| `snapshot` | string | no | Omit to promote the current draft (needs a green check bound to it). A previously deployed snapshot id rolls back to it |
|
|
463
475
|
| `note` | string | no | Why, recorded on the learned trail |
|
|
476
|
+
| `then_build` | boolean | no, default `False` | When the deploy changed any file under models/, also queue a models-only build (the `build` action) of the newly deployed snapshot on the raw tables already loaded; build_select narrows it. Without it, a deploy that changed models answers with the command that does it |
|
|
477
|
+
| `build_select` | string | no | With then_build: the dbt selector to rebuild (default: every model, `fqn:*`) |
|
|
464
478
|
|
|
465
479
|
**`run`** —
|
|
466
480
|
|
|
467
481
|
| Parameter | Type | Required | Description |
|
|
468
482
|
|---|---|---|---|
|
|
469
483
|
| `pipeline` | string | yes | Pipeline name (a deployed pipelines/<name>.yaml) |
|
|
470
|
-
| `window` | RunWindowParams | no | A date window to load instead of the pipeline's own; wider than backfill_chunk becomes a chunked backfill |
|
|
484
|
+
| `window` | RunWindowParams | no | A date window to load instead of the pipeline's own; wider than backfill_chunk becomes a chunked backfill. Both ends are inclusive days, and so is every chunk; a resource whose API reads its end bound as exclusive declares `window.end: exclusive` in the pipeline and is sent the day after each chunk's last day |
|
|
471
485
|
| `cursor` | RunCursorParams | no | {from: <value>}: a cursor to re-pull from |
|
|
472
486
|
| `backfill_chunk` | string | no | Chunk size for a windowed run: 7d, 2w or 1M (calendar months, the default) |
|
|
473
487
|
|
|
488
|
+
**`build`** — Rebuild the deployed dbt models on the raw tables already loaded: no resource is loaded.
|
|
489
|
+
|
|
490
|
+
| Parameter | Type | Required | Description |
|
|
491
|
+
|---|---|---|---|
|
|
492
|
+
| `pipeline` | string | no | A deployed pipeline whose `then` block names what to build (its dbt_select, dbt_exclude and tests). Omit to build every model of the deployed project |
|
|
493
|
+
| `select` | string | no | A dbt selector (`dbt build --select`), e.g. `marts.finance+` or `+fct_postings`; replaces the pipeline's dbt_select and dbt_exclude. Default: the pipeline's then.dbt_select, or every model (`fqn:*`) when no pipeline is named |
|
|
494
|
+
|
|
474
495
|
**`reset`** —
|
|
475
496
|
|
|
476
497
|
| Parameter | Type | Required | Description |
|
|
477
498
|
|---|---|---|---|
|
|
478
499
|
| `pipeline` | string | yes | The pipeline whose dlt state and raw tables to remove |
|
|
479
500
|
| `approve` | boolean | no, default `False` | Must be true: the reset is irreversible. Ask the user first |
|
|
501
|
+
|
|
502
|
+
**`cancel`** —
|
|
503
|
+
|
|
504
|
+
| Parameter | Type | Required | Description |
|
|
505
|
+
|---|---|---|---|
|
|
506
|
+
| `run_id` | string | yes | The queued or running run to stop, or a backfill's parent run (its running child is stopped and no further chunk starts); ids from dataspring://datacore/runs |
|
|
480
507
|
<!-- /dispatch-params -->
|
|
@@ -656,7 +656,7 @@ MCP tool `query_metrics`; `POST /api/dispatch/query_metrics`. Fetch metric data
|
|
|
656
656
|
| `order_by` | string | no | Column to sort by, append ' desc' for descending (e.g., 'total_revenue desc') |
|
|
657
657
|
| `where` | list of string | no | Row filters in MetricFlow's constraint syntax, one string each, e.g. "{{ Dimension('customer__segment') }} NOT IN ('Direct')". Use qualified names from dataspring://dimensions - an unresolvable name comes back as an error whose suggestion names the qualified form. `where` decides WHICH ROWS the numbers cover; `dimensions` breaks the numbers OUT by a column; send both for a breakdown of a filtered slice. A `where` on a dimension also overrides the user's standing filter on that same dimension for this call. |
|
|
658
658
|
| `warehouse` | string | no | Which of the org's warehouses to run against - an id from dataspring://warehouses ('managed', 'external', ...). Omit for the active one. During a migration ask both and compare. |
|
|
659
|
-
| `compare_to` | one of `previous_period`, `same_period_last_year` | no | Compare the window against the one before it (previous_period, stepped back by whole calendar periods when the window is whole ones) or the same window a year earlier (same_period_last_year). The deltas come back BESIDE the values - each metric column gains <metric>__previous, <metric>__delta and <metric>__delta_pct - so there is no second result set to line up and no arithmetic to do. `comparison_period` says which window they were measured against. |
|
|
659
|
+
| `compare_to` | one of `previous_period`, `same_period_last_year` | no | Compare the window against the one before it (previous_period, stepped back by whole calendar periods when the window is whole ones, and by as many periods as it touches, keeping its position, when it cuts a period short: a year to date compares with the same days a year earlier) or the same window a year earlier (same_period_last_year). The deltas come back BESIDE the values - each metric column gains <metric>__previous, <metric>__delta and <metric>__delta_pct - so there is no second result set to line up and no arithmetic to do. `comparison_period` says which window they were measured against. |
|
|
660
660
|
| `suggest_visualization` | boolean | no, default `False` | Also return a suggested visualization type for the result. |
|
|
661
661
|
| `format` | one of `json`, `csv` | no, default `json` | 'json' (default) answers rows in `data`; 'csv' answers the same result as one CSV document in `content` (header row first) for a file or a spreadsheet, with `columns` and `row_count` alongside. |
|
|
662
662
|
|
|
@@ -672,7 +672,7 @@ MCP tool `board_fetch`; `POST /api/dispatch/board_fetch`. Run every row of a boa
|
|
|
672
672
|
| `grain` | one of `day`, `week`, `month`, `quarter`, `year` | no | Bucket grain for each row's history. Defaults to the period's own grain. |
|
|
673
673
|
| `start_date` | string | no | Start date in YYYY-MM-DD format, instead of a named period |
|
|
674
674
|
| `end_date` | string | no | End date in YYYY-MM-DD format, instead of a named period |
|
|
675
|
-
| `compare_to` | one of `previous_period`, `same_period_last_year` | no | Compare the window against the one before it (previous_period, stepped back by whole calendar periods when the window is whole ones) or the same window a year earlier (same_period_last_year). The deltas come back BESIDE the values - each metric column gains <metric>__previous, <metric>__delta and <metric>__delta_pct - so there is no second result set to line up and no arithmetic to do. `comparison_period` says which window they were measured against. |
|
|
675
|
+
| `compare_to` | one of `previous_period`, `same_period_last_year` | no | Compare the window against the one before it (previous_period, stepped back by whole calendar periods when the window is whole ones, and by as many periods as it touches, keeping its position, when it cuts a period short: a year to date compares with the same days a year earlier) or the same window a year earlier (same_period_last_year). The deltas come back BESIDE the values - each metric column gains <metric>__previous, <metric>__delta and <metric>__delta_pct - so there is no second result set to line up and no arithmetic to do. `comparison_period` says which window they were measured against. |
|
|
676
676
|
| `detail` | one of `salient`, `full` | no, default `salient` | 'salient' (default) answers EVERY row compactly - value, delta, and whether it is notable - and adds the full detail (history, sparkline, receipt, freshness, which rule fired) only for the rows that are notable plus any named in `rows`. 'full' answers everything for every row, for an export or a spreadsheet. |
|
|
677
677
|
| `rows` | list of string | no | Row ids to return in full detail whether or not they are notable. |
|
|
678
678
|
| `history` | integer | no, default `13` | How many periods of history to fetch per row, for the sparkline and the volatility measure. |
|
|
@@ -703,7 +703,7 @@ MCP tool `run_sql`; `POST /api/dispatch/run_sql`. Run ONE read-only SQL statemen
|
|
|
703
703
|
|
|
704
704
|
| Parameter | Type | Required | Description |
|
|
705
705
|
|---|---|---|---|
|
|
706
|
-
| `sql` | string | yes | One SQL statement (BigQuery Standard SQL on a managed warehouse). Name tables as <org>_marts.<table>, <org>_core.<table> or <org>_staging.<table
|
|
706
|
+
| `sql` | string | yes | One SQL statement (BigQuery Standard SQL on a managed warehouse). Name tables as <org>_marts.<table>, <org>_core.<table> or <org>_staging.<table>, <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those and nothing else, so a write or a reach outside them is refused by the warehouse. |
|
|
707
707
|
| `warehouse` | string | no | One of the org's warehouses (see dataspring://warehouses); omit for the active one. |
|
|
708
708
|
| `max_rows` | integer | no, default `1000` | Rows to return at most (default 1000, cap 10000). Aggregate in SQL rather than paging. |
|
|
709
709
|
| `format` | one of `json`, `csv`, `markdown` | no, default `json` | 'json' answers rows in `data`; 'csv' or 'markdown' answer one document in `content`. |
|
cli/auth.py
CHANGED
|
@@ -4,15 +4,26 @@
|
|
|
4
4
|
backend/routes_device.py): the CLI shows a code and a URL, a person signed in
|
|
5
5
|
to DataSpring in any browser (this machine needs none) approves it, and the
|
|
6
6
|
CLI collects an ordinary ``ds_`` API key labelled ``CLI on <hostname>`` (one
|
|
7
|
-
per org with ``--all-orgs
|
|
7
|
+
per org with ``--all-orgs``).
|
|
8
8
|
|
|
9
9
|
Credentials live in ``~/.dataspring/credentials.json`` (mode 0600): a profile
|
|
10
|
-
per org
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
10
|
+
(a key) per org. Nothing machine-wide says which one a command uses (hub
|
|
11
|
+
D-99: ``auth use`` went, because two agents on one machine moved each other
|
|
12
|
+
between orgs); every command resolves its org afresh, in this order
|
|
13
|
+
(:meth:`CLIAuthManager.resolve`):
|
|
14
|
+
|
|
15
|
+
1. ``--org`` on the command;
|
|
16
|
+
2. ``DATASPRING_ORG``, then ``FORSPRING_ORG``;
|
|
17
|
+
3. ``.forspring.json`` (``{"org": "noon"}``) in this directory or a parent;
|
|
18
|
+
4. the key's own org, when ``DATASPRING_API_KEY`` is set (a key belongs to
|
|
19
|
+
one org);
|
|
20
|
+
5. the only stored profile;
|
|
21
|
+
|
|
22
|
+
else it refuses with ``org_required``, the stored orgs and the three ways to
|
|
23
|
+
name one, and never prompts. ``DATASPRING_API_KEY`` always wins over the
|
|
24
|
+
stored keys: with it, a named org that is not the key's is ``org_mismatch``.
|
|
25
|
+
``--key`` stores a key you already have, for CI. ``dataspring auth logout``
|
|
26
|
+
revokes the stored keys on the server, then forgets them.
|
|
16
27
|
|
|
17
28
|
Who the person is and which org a key is for are the server's answer to
|
|
18
29
|
``/api/me``; nothing here opens Firestore.
|
|
@@ -31,8 +42,30 @@ from pathlib import Path
|
|
|
31
42
|
|
|
32
43
|
import httpx
|
|
33
44
|
|
|
45
|
+
from cli.refusals import EN, NAME_AN_ORG, Refusal, classify, from_body
|
|
46
|
+
|
|
34
47
|
ENV_KEY = "DATASPRING_API_KEY"
|
|
35
48
|
|
|
49
|
+
#: Step 2 of the org order: the product's own variable, then the suite's.
|
|
50
|
+
ORG_ENV_VARS = ("DATASPRING_ORG", "FORSPRING_ORG")
|
|
51
|
+
|
|
52
|
+
#: Step 3: the per-project file, committed next to the code (``{"org": "noon"}``).
|
|
53
|
+
PROJECT_FILE = ".forspring.json"
|
|
54
|
+
|
|
55
|
+
#: ``org_from`` for steps 4 and 5.
|
|
56
|
+
FROM_KEY = ENV_KEY
|
|
57
|
+
FROM_ONLY_PROFILE = "only_profile"
|
|
58
|
+
|
|
59
|
+
#: Step 1, ``--org``: set by the root command (``cli.main``) for this
|
|
60
|
+
#: invocation; ``None`` when not given.
|
|
61
|
+
_org_flag: str | None = None
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def set_org_flag(org: str | None) -> None:
|
|
65
|
+
"""``--org`` for this invocation (``cli.main``'s root callback)."""
|
|
66
|
+
global _org_flag
|
|
67
|
+
_org_flag = (org or "").strip() or None
|
|
68
|
+
|
|
36
69
|
|
|
37
70
|
def _host() -> str:
|
|
38
71
|
"""The backend's root (https://dataspring.app; locally http://localhost:8000)."""
|
|
@@ -67,12 +100,100 @@ class UserContext:
|
|
|
67
100
|
|
|
68
101
|
|
|
69
102
|
class AuthenticationError(Exception):
|
|
70
|
-
"""Raised when authentication fails or user is not logged in.
|
|
71
|
-
|
|
103
|
+
"""Raised when authentication fails or user is not logged in.
|
|
104
|
+
|
|
105
|
+
``exit_code`` is what the CLI ends with (suite standards §6): 1, a plain
|
|
106
|
+
failure, unless a subclass says otherwise.
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
exit_code = 1
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class Refused(AuthenticationError):
|
|
113
|
+
"""A D-99 refusal (``cli.refusals``): not signed in, no org named, a key
|
|
114
|
+
for another org, or the server's own refusal of the key."""
|
|
115
|
+
|
|
116
|
+
def __init__(self, reason: str, note: str, fix: str | None = None):
|
|
117
|
+
self.refusal = Refusal(reason, note, fix)
|
|
118
|
+
super().__init__(self.refusal.message)
|
|
119
|
+
|
|
120
|
+
@property
|
|
121
|
+
def reason(self) -> str:
|
|
122
|
+
return self.refusal.reason
|
|
123
|
+
|
|
124
|
+
@property
|
|
125
|
+
def exit_code(self) -> int: # type: ignore[override]
|
|
126
|
+
return self.refusal.exit_code
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class ProjectFileError(AuthenticationError):
|
|
130
|
+
"""A ``.forspring.json`` that names no org: invalid input (2)."""
|
|
131
|
+
|
|
132
|
+
exit_code = 2
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@dataclass(frozen=True)
|
|
136
|
+
class ResolvedOrg:
|
|
137
|
+
"""Which key a command sends, for which org, and where the org came from.
|
|
138
|
+
|
|
139
|
+
``org_id`` is ``None`` only for ``DATASPRING_API_KEY`` with no org named
|
|
140
|
+
and no stored profile holding that key: the key knows its org, and the
|
|
141
|
+
server says which (``GET /api/me``). ``org_from`` is ``--org``,
|
|
142
|
+
``DATASPRING_ORG``, ``FORSPRING_ORG``, ``.forspring.json``,
|
|
143
|
+
``DATASPRING_API_KEY`` or ``only_profile``; ``org_file`` is the file
|
|
144
|
+
step 3 read.
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
key: str
|
|
148
|
+
org_id: str | None
|
|
149
|
+
org_from: str
|
|
150
|
+
org_file: str | None = None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _not_signed_in(detail: str = "Not logged in.") -> Refused:
|
|
154
|
+
return Refused("not_signed_in", detail, "Run 'dataspring auth login' (or set DATASPRING_API_KEY).")
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def find_project_file(start: Path | None = None) -> Path | None:
|
|
158
|
+
"""The nearest ``.forspring.json`` in ``start`` (default: the working
|
|
159
|
+
directory) or a parent."""
|
|
160
|
+
here = (start or Path.cwd()).resolve()
|
|
161
|
+
for directory in (here, *here.parents):
|
|
162
|
+
candidate = directory / PROJECT_FILE
|
|
163
|
+
if candidate.is_file():
|
|
164
|
+
return candidate
|
|
165
|
+
return None
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def read_project_org(path: Path) -> str:
|
|
169
|
+
"""The ``org`` a ``.forspring.json`` names; a file that names none is an error."""
|
|
170
|
+
shape = '{"org": "<org>"}'
|
|
171
|
+
try:
|
|
172
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
173
|
+
except (OSError, json.JSONDecodeError) as e:
|
|
174
|
+
raise ProjectFileError(f"{path} is not valid JSON ({e}); it should hold {shape}.")
|
|
175
|
+
org = data.get("org") if isinstance(data, dict) else None
|
|
176
|
+
if not isinstance(org, str) or not org.strip():
|
|
177
|
+
raise ProjectFileError(f"{path} names no org; it should hold {shape}.")
|
|
178
|
+
return org.strip()
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def named_org() -> tuple[str, str, str | None] | None:
|
|
182
|
+
"""Steps 1-3 of the org order: ``(org, org_from, file)``, or None."""
|
|
183
|
+
if _org_flag:
|
|
184
|
+
return _org_flag, "--org", None
|
|
185
|
+
for var in ORG_ENV_VARS:
|
|
186
|
+
value = (os.environ.get(var) or "").strip()
|
|
187
|
+
if value:
|
|
188
|
+
return value, var, None
|
|
189
|
+
path = find_project_file()
|
|
190
|
+
if path is not None:
|
|
191
|
+
return read_project_org(path), PROJECT_FILE, str(path)
|
|
192
|
+
return None
|
|
72
193
|
|
|
73
194
|
|
|
74
195
|
class CLIAuthManager:
|
|
75
|
-
"""API keys for the CLI: device-flow login,
|
|
196
|
+
"""API keys for the CLI: device-flow login, a profile per org, the env var."""
|
|
76
197
|
|
|
77
198
|
CONFIG_DIR = Path.home() / ".dataspring"
|
|
78
199
|
CREDENTIALS_FILE = CONFIG_DIR / "credentials.json"
|
|
@@ -89,9 +210,10 @@ class CLIAuthManager:
|
|
|
89
210
|
# -- stored profiles -------------------------------------------------
|
|
90
211
|
|
|
91
212
|
def _load(self) -> dict:
|
|
92
|
-
"""``{
|
|
93
|
-
|
|
94
|
-
|
|
213
|
+
"""``{profiles: {org: {api_key, key_id, org_name, email}}}``; empty
|
|
214
|
+
for none, or for what an older CLI stored (Google credentials, or CLI
|
|
215
|
+
tokens): those sign nobody in now. A ``current`` an older CLI wrote
|
|
216
|
+
(``auth use``) is ignored: nothing machine-wide picks the org (D-99)."""
|
|
95
217
|
if not self.CREDENTIALS_FILE.exists():
|
|
96
218
|
return {}
|
|
97
219
|
try:
|
|
@@ -101,6 +223,7 @@ class CLIAuthManager:
|
|
|
101
223
|
return data if isinstance(data.get("profiles"), dict) else {}
|
|
102
224
|
|
|
103
225
|
def _save(self, data: dict) -> None:
|
|
226
|
+
data.pop("current", None)
|
|
104
227
|
self.CREDENTIALS_FILE.write_text(json.dumps(data, indent=2))
|
|
105
228
|
self.CREDENTIALS_FILE.chmod(0o600)
|
|
106
229
|
|
|
@@ -115,19 +238,72 @@ class CLIAuthManager:
|
|
|
115
238
|
def profiles(self) -> dict[str, dict]:
|
|
116
239
|
return self._load().get("profiles", {})
|
|
117
240
|
|
|
118
|
-
|
|
119
|
-
return self._load().get("current")
|
|
241
|
+
# -- which org, which key (hub D-99) ---------------------------------
|
|
120
242
|
|
|
121
|
-
def
|
|
122
|
-
"""
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
243
|
+
def resolve(self) -> ResolvedOrg:
|
|
244
|
+
"""The org and key for this command, in the D-99 order (module doc).
|
|
245
|
+
|
|
246
|
+
Raises:
|
|
247
|
+
Refused: ``not_signed_in`` (no key at all, or none stored for the
|
|
248
|
+
org named), ``org_required`` (several stored, none named;
|
|
249
|
+
never a prompt), ``org_mismatch`` (``DATASPRING_API_KEY`` is
|
|
250
|
+
a key for another org than the one named).
|
|
251
|
+
ProjectFileError: a ``.forspring.json`` that names no org.
|
|
252
|
+
"""
|
|
253
|
+
named = named_org()
|
|
254
|
+
env_key = os.environ.get(ENV_KEY)
|
|
255
|
+
profiles = self.profiles()
|
|
256
|
+
|
|
257
|
+
if env_key:
|
|
258
|
+
stored_org = next((org for org, p in profiles.items() if p.get("api_key") == env_key), None)
|
|
259
|
+
if named is None:
|
|
260
|
+
return ResolvedOrg(env_key, stored_org, FROM_KEY)
|
|
261
|
+
org, source, path = named
|
|
262
|
+
key_org = stored_org or self._key_org(env_key)
|
|
263
|
+
if key_org != org:
|
|
264
|
+
raise Refused(
|
|
265
|
+
"org_mismatch",
|
|
266
|
+
f"DATASPRING_API_KEY is a key for '{key_org}', but {source} names '{org}'.",
|
|
267
|
+
f"Name '{key_org}', or set DATASPRING_API_KEY to a key for '{org}' "
|
|
268
|
+
"(or unset it to use a stored key).",
|
|
269
|
+
)
|
|
270
|
+
return ResolvedOrg(env_key, org, source, path)
|
|
271
|
+
|
|
272
|
+
if not profiles:
|
|
273
|
+
raise _not_signed_in()
|
|
274
|
+
if named is not None:
|
|
275
|
+
org, source, path = named
|
|
276
|
+
profile = profiles.get(org)
|
|
277
|
+
if profile is None:
|
|
278
|
+
raise Refused(
|
|
279
|
+
"not_signed_in",
|
|
280
|
+
f"No key for '{org}' on this machine ({source} names it); "
|
|
281
|
+
f"stored: {', '.join(sorted(profiles))}.",
|
|
282
|
+
f"Run 'dataspring auth login' and approve {org}, or name one of the stored orgs.",
|
|
283
|
+
)
|
|
284
|
+
return ResolvedOrg(profile["api_key"], org, source, path)
|
|
285
|
+
if len(profiles) == 1:
|
|
286
|
+
org, profile = next(iter(profiles.items()))
|
|
287
|
+
return ResolvedOrg(profile["api_key"], org, FROM_ONLY_PROFILE)
|
|
288
|
+
raise Refused(
|
|
289
|
+
"org_required",
|
|
290
|
+
"Several organizations have keys on this machine and this command names none: "
|
|
291
|
+
f"{', '.join(sorted(profiles))}.",
|
|
292
|
+
f"Name one: {NAME_AN_ORG}.",
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
def _key_org(self, key: str) -> str | None:
|
|
296
|
+
"""The org a key not stored here belongs to, as the server says.
|
|
297
|
+
Synchronous: :meth:`resolve` runs inside the async commands too."""
|
|
298
|
+
r = httpx.get(f"{_host()}/api/me", headers={"Authorization": f"Bearer {key}"}, timeout=15.0)
|
|
299
|
+
return self._me_answer(r).get("org_id")
|
|
300
|
+
|
|
301
|
+
def resolved_org(self) -> str | None:
|
|
302
|
+
"""The org a command here would use, or None when none resolves."""
|
|
303
|
+
try:
|
|
304
|
+
return self.resolve().org_id
|
|
305
|
+
except AuthenticationError:
|
|
306
|
+
return None
|
|
131
307
|
|
|
132
308
|
# -- login -----------------------------------------------------------
|
|
133
309
|
|
|
@@ -145,8 +321,11 @@ class CLIAuthManager:
|
|
|
145
321
|
if r.status_code != 200:
|
|
146
322
|
raise AuthenticationError(f"Could not start the sign-in (HTTP {r.status_code})")
|
|
147
323
|
start = r.json()
|
|
324
|
+
# flush: an agent runs this with stdout redirected, where Python
|
|
325
|
+
# buffers it; unflushed, the code appeared only after the CLI exited,
|
|
326
|
+
# too late for anyone to approve it (tick 373).
|
|
148
327
|
print(f"To sign in, open {start['verification_uri']} and enter the code {start['user_code']}")
|
|
149
|
-
print(f" (or open {start['verification_uri_complete']})")
|
|
328
|
+
print(f" (or open {start['verification_uri_complete']})", flush=True)
|
|
150
329
|
if open_browser:
|
|
151
330
|
try:
|
|
152
331
|
webbrowser.open(start["verification_uri_complete"])
|
|
@@ -164,14 +343,15 @@ class CLIAuthManager:
|
|
|
164
343
|
self._add_profile(data, body, body.get("email"))
|
|
165
344
|
for other in body.get("other_orgs") or []:
|
|
166
345
|
self._add_profile(data, other, body.get("email"))
|
|
167
|
-
data["current"] = body["org_id"]
|
|
168
346
|
self._save(data)
|
|
169
347
|
if body.get("email"):
|
|
170
348
|
self._save_config({"email": body["email"]})
|
|
171
|
-
|
|
349
|
+
# The approved org's key: the one this login is about, whatever
|
|
350
|
+
# else is stored.
|
|
351
|
+
return self._context(await self._me(body["api_key"]))
|
|
172
352
|
error = (r.json() or {}).get("error") if r.headers.get("content-type", "").startswith("application/json") else None
|
|
173
|
-
if error
|
|
174
|
-
continue
|
|
353
|
+
if error in ("authorization_pending", "temporarily_unavailable"):
|
|
354
|
+
continue # temporarily_unavailable: the Forspring account did not answer; the approval waits
|
|
175
355
|
if error == "slow_down":
|
|
176
356
|
interval += 5
|
|
177
357
|
continue
|
|
@@ -185,36 +365,52 @@ class CLIAuthManager:
|
|
|
185
365
|
me = await self._me(key)
|
|
186
366
|
data = self._load()
|
|
187
367
|
self._add_profile(data, {"api_key": key, "key_id": None, "org_id": me["org_id"], "org_name": me.get("org_name")}, me.get("email"))
|
|
188
|
-
data["current"] = me["org_id"]
|
|
189
368
|
self._save(data)
|
|
190
369
|
return self._context(me)
|
|
191
370
|
|
|
192
371
|
# -- using the key ---------------------------------------------------
|
|
193
372
|
|
|
194
373
|
def get_access_token(self) -> str:
|
|
195
|
-
"""The key
|
|
374
|
+
"""The key this command sends: ``DATASPRING_API_KEY`` if set, else the
|
|
375
|
+
stored key for the org :meth:`resolve` names.
|
|
196
376
|
|
|
197
377
|
Raises:
|
|
198
|
-
|
|
378
|
+
Refused: not signed in, no org named among several, or a mismatch
|
|
199
379
|
"""
|
|
200
|
-
|
|
201
|
-
return os.environ[ENV_KEY]
|
|
202
|
-
data = self._load()
|
|
203
|
-
profile = data.get("profiles", {}).get(data.get("current") or "")
|
|
204
|
-
if not profile:
|
|
205
|
-
raise AuthenticationError("Not logged in. Run 'dataspring auth login' first.")
|
|
206
|
-
return profile["api_key"]
|
|
380
|
+
return self.resolve().key
|
|
207
381
|
|
|
208
382
|
async def _me(self, key: str) -> dict:
|
|
209
383
|
async with httpx.AsyncClient(timeout=15.0) as client:
|
|
210
384
|
r = await client.get(f"{_host()}/api/me", headers={"Authorization": f"Bearer {key}"})
|
|
385
|
+
return self._me_answer(r)
|
|
386
|
+
|
|
387
|
+
@staticmethod
|
|
388
|
+
def _me_answer(r: httpx.Response) -> dict:
|
|
389
|
+
"""``GET /api/me``'s body, or the refusal it answered with (D-99)."""
|
|
390
|
+
if r.status_code == 200:
|
|
391
|
+
return r.json()
|
|
392
|
+
try:
|
|
393
|
+
body = r.json()
|
|
394
|
+
except ValueError:
|
|
395
|
+
body = None
|
|
396
|
+
refusal = from_body(body)
|
|
397
|
+
if refusal is not None:
|
|
398
|
+
raise Refused(refusal.reason, refusal.note, refusal.fix)
|
|
211
399
|
if r.status_code == 401:
|
|
212
|
-
raise
|
|
400
|
+
raise _not_signed_in("This key is unknown, revoked or expired.")
|
|
401
|
+
# ``note`` since tick 3sb, ``detail`` from older servers and FastAPI.
|
|
402
|
+
detail = (body.get("note") or body.get("detail")) if isinstance(body, dict) else None
|
|
213
403
|
if r.status_code == 403:
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
404
|
+
text = str(detail or "Access denied")
|
|
405
|
+
raise Refused(
|
|
406
|
+
classify(text) or "not_a_member", text,
|
|
407
|
+
"An owner or admin of the org fixes this at https://account.forspring.ai; "
|
|
408
|
+
"or run 'dataspring auth login' again.",
|
|
409
|
+
)
|
|
410
|
+
if r.status_code == 503:
|
|
411
|
+
raise Refused("account_unavailable", str(detail or "The Forspring account cannot be reached."),
|
|
412
|
+
EN["account_unavailable"]["fix"])
|
|
413
|
+
raise AuthenticationError(f"Failed to resolve user context (HTTP {r.status_code})")
|
|
218
414
|
|
|
219
415
|
@staticmethod
|
|
220
416
|
def _context(me: dict) -> "UserContext":
|
|
@@ -226,6 +422,22 @@ class CLIAuthManager:
|
|
|
226
422
|
"""The person and the key's org, as ``GET /api/me`` reports them."""
|
|
227
423
|
return self._context(await self._me(self.get_access_token()))
|
|
228
424
|
|
|
425
|
+
async def whoami(self) -> dict:
|
|
426
|
+
"""D-99's ``whoami --json``: the person, the org, where the org came
|
|
427
|
+
from (:class:`ResolvedOrg`'s ``org_from``), the role."""
|
|
428
|
+
resolved = self.resolve()
|
|
429
|
+
me = await self._me(resolved.key)
|
|
430
|
+
out = {
|
|
431
|
+
"person": me.get("email"),
|
|
432
|
+
"org": me.get("org_id"),
|
|
433
|
+
"org_name": me.get("org_name"),
|
|
434
|
+
"org_from": resolved.org_from,
|
|
435
|
+
"role": me.get("org_role"),
|
|
436
|
+
}
|
|
437
|
+
if resolved.org_file:
|
|
438
|
+
out["org_file"] = resolved.org_file
|
|
439
|
+
return out
|
|
440
|
+
|
|
229
441
|
async def logout(self):
|
|
230
442
|
"""Revoke every stored key on the server, then forget them."""
|
|
231
443
|
for profile in self.profiles().values():
|
|
@@ -242,10 +454,9 @@ class CLIAuthManager:
|
|
|
242
454
|
self.CONFIG_FILE.unlink()
|
|
243
455
|
|
|
244
456
|
def is_logged_in(self) -> bool:
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
return bool(data.get("profiles", {}).get(data.get("current") or ""))
|
|
457
|
+
"""A key exists to use: ``DATASPRING_API_KEY`` or a stored one (which
|
|
458
|
+
one a command uses is :meth:`resolve`'s)."""
|
|
459
|
+
return bool(os.environ.get(ENV_KEY)) or bool(self.profiles())
|
|
249
460
|
|
|
250
461
|
def get_stored_email(self) -> str | None:
|
|
251
462
|
"""Get stored email from config (doesn't validate the key)."""
|
cli/bundled_manifest.py
CHANGED
|
@@ -6,7 +6,7 @@ the server's to say whether the server has operations this CLI does not
|
|
|
6
6
|
know. tests/test_skills_surface.py asserts it is current.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
|
-
MANIFEST_HASH = "
|
|
9
|
+
MANIFEST_HASH = "b9fdc845fd9d3a22cea19e234b330e7e41d17f5d5ba6b5e1c713123d38109a6d"
|
|
10
10
|
TOOL_KEYS = (
|
|
11
11
|
"business_context_edit",
|
|
12
12
|
"semantic_model_edit",
|
cli/contract.py
CHANGED
|
@@ -636,6 +636,18 @@ class DeployDatacoreAction(DispatchParams):
|
|
|
636
636
|
),
|
|
637
637
|
)
|
|
638
638
|
note: str | None = Field(default=None, description="Why, recorded on the learned trail")
|
|
639
|
+
then_build: bool = Field(
|
|
640
|
+
default=False,
|
|
641
|
+
description=(
|
|
642
|
+
"When the deploy changed any file under models/, also queue a models-only build (the `build` action) "
|
|
643
|
+
"of the newly deployed snapshot on the raw tables already loaded; build_select narrows it. "
|
|
644
|
+
"Without it, a deploy that changed models answers with the command that does it"
|
|
645
|
+
),
|
|
646
|
+
)
|
|
647
|
+
build_select: str | None = Field(
|
|
648
|
+
default=None,
|
|
649
|
+
description="With then_build: the dbt selector to rebuild (default: every model, `fqn:*`)",
|
|
650
|
+
)
|
|
639
651
|
|
|
640
652
|
|
|
641
653
|
class RunWindowParams(DispatchParams):
|
|
@@ -656,7 +668,11 @@ class RunPipelineAction(DispatchParams):
|
|
|
656
668
|
pipeline: str = Field(description="Pipeline name (a deployed pipelines/<name>.yaml)")
|
|
657
669
|
window: RunWindowParams | None = Field(
|
|
658
670
|
default=None,
|
|
659
|
-
description=
|
|
671
|
+
description=(
|
|
672
|
+
"A date window to load instead of the pipeline's own; wider than backfill_chunk becomes a chunked backfill. "
|
|
673
|
+
"Both ends are inclusive days, and so is every chunk; a resource whose API reads its end bound as exclusive "
|
|
674
|
+
"declares `window.end: exclusive` in the pipeline and is sent the day after each chunk's last day"
|
|
675
|
+
),
|
|
660
676
|
)
|
|
661
677
|
cursor: RunCursorParams | None = Field(default=None, description="{from: <value>}: a cursor to re-pull from")
|
|
662
678
|
backfill_chunk: str | None = Field(
|
|
@@ -665,6 +681,27 @@ class RunPipelineAction(DispatchParams):
|
|
|
665
681
|
)
|
|
666
682
|
|
|
667
683
|
|
|
684
|
+
class BuildModelsAction(DispatchParams):
|
|
685
|
+
"""Rebuild the deployed dbt models on the raw tables already loaded: no resource is loaded."""
|
|
686
|
+
|
|
687
|
+
action: Literal["build"]
|
|
688
|
+
pipeline: str | None = Field(
|
|
689
|
+
default=None,
|
|
690
|
+
description=(
|
|
691
|
+
"A deployed pipeline whose `then` block names what to build (its dbt_select, dbt_exclude and tests). "
|
|
692
|
+
"Omit to build every model of the deployed project"
|
|
693
|
+
),
|
|
694
|
+
)
|
|
695
|
+
select: str | None = Field(
|
|
696
|
+
default=None,
|
|
697
|
+
description=(
|
|
698
|
+
"A dbt selector (`dbt build --select`), e.g. `marts.finance+` or `+fct_postings`; replaces the "
|
|
699
|
+
"pipeline's dbt_select and dbt_exclude. Default: the pipeline's then.dbt_select, or every model "
|
|
700
|
+
"(`fqn:*`) when no pipeline is named"
|
|
701
|
+
),
|
|
702
|
+
)
|
|
703
|
+
|
|
704
|
+
|
|
668
705
|
class ResetPipelineAction(DispatchParams):
|
|
669
706
|
action: Literal["reset"]
|
|
670
707
|
pipeline: str = Field(description="The pipeline whose dlt state and raw tables to remove")
|
|
@@ -674,8 +711,23 @@ class ResetPipelineAction(DispatchParams):
|
|
|
674
711
|
)
|
|
675
712
|
|
|
676
713
|
|
|
714
|
+
class CancelRunAction(DispatchParams):
|
|
715
|
+
action: Literal["cancel"]
|
|
716
|
+
run_id: str = Field(
|
|
717
|
+
description=(
|
|
718
|
+
"The queued or running run to stop, or a backfill's parent run (its running child "
|
|
719
|
+
"is stopped and no further chunk starts); ids from dataspring://datacore/runs"
|
|
720
|
+
),
|
|
721
|
+
)
|
|
722
|
+
|
|
723
|
+
|
|
677
724
|
DatacoreRunAction = Annotated[
|
|
678
|
-
CheckDatacoreAction
|
|
725
|
+
CheckDatacoreAction
|
|
726
|
+
| DeployDatacoreAction
|
|
727
|
+
| RunPipelineAction
|
|
728
|
+
| BuildModelsAction
|
|
729
|
+
| ResetPipelineAction
|
|
730
|
+
| CancelRunAction,
|
|
679
731
|
Field(discriminator="action"),
|
|
680
732
|
]
|
|
681
733
|
|
|
@@ -701,8 +753,10 @@ _PERIOD_DOC = (
|
|
|
701
753
|
|
|
702
754
|
_COMPARE_TO_DOC = (
|
|
703
755
|
"Compare the window against the one before it (previous_period, stepped "
|
|
704
|
-
"back by whole calendar periods when the window is whole ones
|
|
705
|
-
"
|
|
756
|
+
"back by whole calendar periods when the window is whole ones, and by as "
|
|
757
|
+
"many periods as it touches, keeping its position, when it cuts a period "
|
|
758
|
+
"short: a year to date compares with the same days a year earlier) or the "
|
|
759
|
+
"same window a year earlier (same_period_last_year). The deltas come back "
|
|
706
760
|
"BESIDE the values - each metric column gains <metric>__previous, "
|
|
707
761
|
"<metric>__delta and <metric>__delta_pct - so there is no second result "
|
|
708
762
|
"set to line up and no arithmetic to do. `comparison_period` says which "
|
|
@@ -1108,7 +1162,8 @@ class RunSqlParams(DispatchParams):
|
|
|
1108
1162
|
description=(
|
|
1109
1163
|
"One SQL statement (BigQuery Standard SQL on a managed warehouse). "
|
|
1110
1164
|
"Name tables as <org>_marts.<table>, <org>_core.<table> or "
|
|
1111
|
-
"<org>_staging.<table
|
|
1165
|
+
"<org>_staging.<table>, <org> being the organization id with every "
|
|
1166
|
+
"hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those and "
|
|
1112
1167
|
"nothing else, so a write or a reach outside them is refused by "
|
|
1113
1168
|
"the warehouse."
|
|
1114
1169
|
)
|
|
@@ -1150,9 +1205,10 @@ class CreateTenantAction(DispatchParams):
|
|
|
1150
1205
|
slug: str = Field(
|
|
1151
1206
|
description=(
|
|
1152
1207
|
"The tenant id, and therefore the organization id and the name in "
|
|
1153
|
-
"every Google Cloud resource it owns
|
|
1208
|
+
"every Google Cloud resource it owns, each derived from it by "
|
|
1209
|
+
"services.tenants (t-<slug>@, <slug>_raw with hyphens as underscores, "
|
|
1154
1210
|
"datacore-runner--<slug>, gs://dataspring-tenants/<slug>/). "
|
|
1155
|
-
"Lower-case [a-z0-9-], at most
|
|
1211
|
+
"Lower-case [a-z0-9-], at most 63 characters. Choose it "
|
|
1156
1212
|
"deliberately: it cannot be renamed afterwards."
|
|
1157
1213
|
)
|
|
1158
1214
|
)
|