dataspring-cli 0.4.2__tar.gz → 0.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/PKG-INFO +1 -1
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/bundled_manifest.py +1 -1
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/contract.py +62 -13
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/generated.py +66 -48
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/main.py +291 -28
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/pyproject.toml +1 -1
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/skills/dataspring-author/SKILL.md +139 -18
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/skills/dataspring-consume/SKILL.md +104 -22
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/skills/dataspring-correct/SKILL.md +4 -3
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/.gitignore +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/README.md +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/__init__.py +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/auth.py +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/output.py +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/refusals.py +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/runtime.py +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/skills_commands.py +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/skilltree.py +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/upgrade.py +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/version.py +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/hatch_build.py +0 -0
- {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/settings.py +0 -0
|
@@ -6,7 +6,7 @@ the server's to say whether the server has operations this CLI does not
|
|
|
6
6
|
know. tests/test_skills_surface.py asserts it is current.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
|
-
MANIFEST_HASH = "
|
|
9
|
+
MANIFEST_HASH = "e61e4a47f3c193beb2f6d731efde82839f5e62788bfa18560277459487bb26a7"
|
|
10
10
|
TOOL_KEYS = (
|
|
11
11
|
"business_context_edit",
|
|
12
12
|
"semantic_model_edit",
|
|
@@ -54,6 +54,8 @@ _EXPECT_VERSION_DOC = (
|
|
|
54
54
|
|
|
55
55
|
|
|
56
56
|
class CreateSemanticModelAction(DispatchParams):
|
|
57
|
+
"""Create a semantic model (admin/owner)."""
|
|
58
|
+
|
|
57
59
|
action: Literal["create"]
|
|
58
60
|
model_data: dict = Field(
|
|
59
61
|
description="Full semantic-model definition (name, measures, dimensions, entities)"
|
|
@@ -62,6 +64,8 @@ class CreateSemanticModelAction(DispatchParams):
|
|
|
62
64
|
|
|
63
65
|
|
|
64
66
|
class UpdateSemanticModelAction(DispatchParams):
|
|
67
|
+
"""Change a semantic model (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once."""
|
|
68
|
+
|
|
65
69
|
action: Literal["update"]
|
|
66
70
|
name: str
|
|
67
71
|
updates: dict
|
|
@@ -77,6 +81,8 @@ class UpdateSemanticModelAction(DispatchParams):
|
|
|
77
81
|
|
|
78
82
|
|
|
79
83
|
class DeleteSemanticModelAction(DispatchParams):
|
|
84
|
+
"""Delete a semantic model (admin/owner)."""
|
|
85
|
+
|
|
80
86
|
action: Literal["delete"]
|
|
81
87
|
name: str
|
|
82
88
|
expect_version: str | None = Field(default=None, description=_EXPECT_VERSION_DOC)
|
|
@@ -113,12 +119,16 @@ class SemanticModelEditParams(DispatchParams):
|
|
|
113
119
|
|
|
114
120
|
|
|
115
121
|
class CreateMetricAction(DispatchParams):
|
|
122
|
+
"""Create a metric, native to DataSpring (admin/owner). A ratio's numerator and denominator, a derived metric's inputs and a cumulative or conversion metric's base are metrics, not measures."""
|
|
123
|
+
|
|
116
124
|
action: Literal["create"]
|
|
117
125
|
metric_data: dict = Field(description="Metric definition (name, type, type_params)")
|
|
118
126
|
expect_version: str | None = Field(default=None, description=_EXPECT_VERSION_DOC)
|
|
119
127
|
|
|
120
128
|
|
|
121
129
|
class UpdateMetricAction(DispatchParams):
|
|
130
|
+
"""Change a metric (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once."""
|
|
131
|
+
|
|
122
132
|
action: Literal["update"]
|
|
123
133
|
name: str
|
|
124
134
|
updates: dict
|
|
@@ -134,12 +144,16 @@ class UpdateMetricAction(DispatchParams):
|
|
|
134
144
|
|
|
135
145
|
|
|
136
146
|
class DeleteMetricAction(DispatchParams):
|
|
147
|
+
"""Delete a metric (admin/owner; irreversible)."""
|
|
148
|
+
|
|
137
149
|
action: Literal["delete"]
|
|
138
150
|
name: str
|
|
139
151
|
expect_version: str | None = Field(default=None, description=_EXPECT_VERSION_DOC)
|
|
140
152
|
|
|
141
153
|
|
|
142
154
|
class PreviewMetricAction(DispatchParams):
|
|
155
|
+
"""Test a metric before saving (any role; writes nothing): sample query results or validation errors, plus the `dbt_yaml` that would be committed for it."""
|
|
156
|
+
|
|
143
157
|
action: Literal["preview"]
|
|
144
158
|
metric_data: dict = Field(description="Metric definition to preview")
|
|
145
159
|
sample_query: dict | None = Field(
|
|
@@ -178,6 +192,8 @@ class MetricEditParams(DispatchParams):
|
|
|
178
192
|
|
|
179
193
|
|
|
180
194
|
class CreateQuickMetricAction(DispatchParams):
|
|
195
|
+
"""Save a personal quick metric: arithmetic over existing metrics, queried as `qm:<name>`."""
|
|
196
|
+
|
|
181
197
|
action: Literal["create"]
|
|
182
198
|
name: str = Field(description='Metric name (e.g., "revenue_per_order")')
|
|
183
199
|
expression: str = Field(
|
|
@@ -187,6 +203,8 @@ class CreateQuickMetricAction(DispatchParams):
|
|
|
187
203
|
|
|
188
204
|
|
|
189
205
|
class UpdateQuickMetricAction(DispatchParams):
|
|
206
|
+
"""Change one of your quick metrics."""
|
|
207
|
+
|
|
190
208
|
action: Literal["update"]
|
|
191
209
|
metric_id: str
|
|
192
210
|
name: str | None = None
|
|
@@ -195,6 +213,8 @@ class UpdateQuickMetricAction(DispatchParams):
|
|
|
195
213
|
|
|
196
214
|
|
|
197
215
|
class DeleteQuickMetricAction(DispatchParams):
|
|
216
|
+
"""Delete one of your quick metrics."""
|
|
217
|
+
|
|
198
218
|
action: Literal["delete"]
|
|
199
219
|
metric_id: str
|
|
200
220
|
|
|
@@ -565,9 +585,11 @@ DatacoreKind = Literal["connection", "pipeline", "model"]
|
|
|
565
585
|
|
|
566
586
|
|
|
567
587
|
class ApplyDatacoreFileAction(DispatchParams):
|
|
568
|
-
"""Write one workspace file whole (create or replace): a
|
|
569
|
-
pipeline as
|
|
570
|
-
|
|
588
|
+
"""Write one workspace file whole (create or replace): a dbt project file
|
|
589
|
+
as ``content``; a connection or pipeline as ``content`` (its YAML text,
|
|
590
|
+
stored exactly as sent, comments and layout kept) or as a ``document``
|
|
591
|
+
(an object, stored as a fresh dump), validated against its schema and
|
|
592
|
+
the catalog either way."""
|
|
571
593
|
|
|
572
594
|
action: Literal["apply"]
|
|
573
595
|
kind: DatacoreKind = Field(description="connection (admin), pipeline (admin) or model (member)")
|
|
@@ -582,14 +604,24 @@ class ApplyDatacoreFileAction(DispatchParams):
|
|
|
582
604
|
"Secrets are named, never valued; every name must exist"
|
|
583
605
|
),
|
|
584
606
|
)
|
|
585
|
-
content: str | None = Field(
|
|
607
|
+
content: str | None = Field(
|
|
608
|
+
default=None,
|
|
609
|
+
description=(
|
|
610
|
+
"The whole file as text. kind model: required. kind connection or pipeline: the YAML "
|
|
611
|
+
"text, validated like a document and stored byte for byte (comments and layout kept); "
|
|
612
|
+
"give content or document, not both"
|
|
613
|
+
),
|
|
614
|
+
)
|
|
586
615
|
expect_version: str | None = Field(default=None, description=_DATACORE_VERSION_DOC)
|
|
587
616
|
|
|
588
617
|
|
|
589
618
|
class PatchDatacoreFileAction(DispatchParams):
|
|
590
619
|
"""Apply a JSON merge patch (RFC 7386: a key set to null is removed) to an
|
|
591
620
|
existing YAML file: a connection, a pipeline (a schedule, a window, a
|
|
592
|
-
check), or a YAML file under models/ (not SQL).
|
|
621
|
+
check), or a YAML file under models/ (not SQL). Only the lines of the
|
|
622
|
+
keys the patch changes are rewritten; every other line, comments and
|
|
623
|
+
indentation included, stays as written (an answer with ``reformatted``
|
|
624
|
+
says the file had to be rewritten whole)."""
|
|
593
625
|
|
|
594
626
|
action: Literal["patch"]
|
|
595
627
|
kind: DatacoreKind = Field(description="connection (admin), pipeline (admin) or model (member)")
|
|
@@ -620,13 +652,15 @@ class DatacoreEditParams(DispatchParams):
|
|
|
620
652
|
|
|
621
653
|
|
|
622
654
|
class CheckDatacoreAction(DispatchParams):
|
|
623
|
-
"""Check the current draft
|
|
655
|
+
"""Check the current draft (member): compile every pipeline, build the changed dbt models into <org>_dev with their tests, and validate the semantic manifest per metric. Nothing reaches production."""
|
|
624
656
|
|
|
625
657
|
action: Literal["check"]
|
|
626
658
|
note: str | None = Field(default=None, description="Optional note, recorded on the check run")
|
|
627
659
|
|
|
628
660
|
|
|
629
661
|
class DeployDatacoreAction(DispatchParams):
|
|
662
|
+
"""Promote the checked draft to deployed (admin), or roll back to an earlier deployed snapshot; then_build also rebuilds changed models."""
|
|
663
|
+
|
|
630
664
|
action: Literal["deploy"]
|
|
631
665
|
snapshot: str | None = Field(
|
|
632
666
|
default=None,
|
|
@@ -664,6 +698,8 @@ class RunCursorParams(DispatchParams):
|
|
|
664
698
|
|
|
665
699
|
|
|
666
700
|
class RunPipelineAction(DispatchParams):
|
|
701
|
+
"""Load a pipeline now (admin): its own window, a given window, or a chunked backfill whose chunks run back to back."""
|
|
702
|
+
|
|
667
703
|
action: Literal["run"]
|
|
668
704
|
pipeline: str = Field(description="Pipeline name (a deployed pipelines/<name>.yaml)")
|
|
669
705
|
window: RunWindowParams | None = Field(
|
|
@@ -677,7 +713,11 @@ class RunPipelineAction(DispatchParams):
|
|
|
677
713
|
cursor: RunCursorParams | None = Field(default=None, description="{from: <value>}: a cursor to re-pull from")
|
|
678
714
|
backfill_chunk: str | None = Field(
|
|
679
715
|
default=None,
|
|
680
|
-
description=
|
|
716
|
+
description=(
|
|
717
|
+
"Chunk size for a windowed run: 7d, 2w or 1M (calendar months, the default). Every chunk before the last "
|
|
718
|
+
"loads only the windowed resources (and the ones they read rows from) and skips its dbt step, each skip "
|
|
719
|
+
"recorded with its reason on the child run; the last chunk loads every resource and builds the models once"
|
|
720
|
+
),
|
|
681
721
|
)
|
|
682
722
|
|
|
683
723
|
|
|
@@ -703,6 +743,8 @@ class BuildModelsAction(DispatchParams):
|
|
|
703
743
|
|
|
704
744
|
|
|
705
745
|
class ResetPipelineAction(DispatchParams):
|
|
746
|
+
"""Delete a pipeline's dlt state and drop its raw tables (admin; irreversible)."""
|
|
747
|
+
|
|
706
748
|
action: Literal["reset"]
|
|
707
749
|
pipeline: str = Field(description="The pipeline whose dlt state and raw tables to remove")
|
|
708
750
|
approve: bool = Field(
|
|
@@ -712,6 +754,8 @@ class ResetPipelineAction(DispatchParams):
|
|
|
712
754
|
|
|
713
755
|
|
|
714
756
|
class CancelRunAction(DispatchParams):
|
|
757
|
+
"""Stop a queued or running run, or a whole backfill (admin); the schedule fires again at its next slot."""
|
|
758
|
+
|
|
715
759
|
action: Literal["cancel"]
|
|
716
760
|
run_id: str = Field(
|
|
717
761
|
description=(
|
|
@@ -1160,12 +1204,17 @@ class RunSqlParams(DispatchParams):
|
|
|
1160
1204
|
|
|
1161
1205
|
sql: str = Field(
|
|
1162
1206
|
description=(
|
|
1163
|
-
"One SQL statement (BigQuery Standard SQL on a managed warehouse)
|
|
1164
|
-
"
|
|
1165
|
-
"
|
|
1166
|
-
"
|
|
1167
|
-
"
|
|
1168
|
-
"the
|
|
1207
|
+
"One SQL statement (BigQuery Standard SQL on a managed warehouse), or a "
|
|
1208
|
+
"script of several separated by ';' whose last statement's rows are "
|
|
1209
|
+
"returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the "
|
|
1210
|
+
"scratch space, gone when the script ends). Name tables as "
|
|
1211
|
+
"<org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, "
|
|
1212
|
+
"<org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the "
|
|
1213
|
+
"last datacore check built; expire after 7 days), <org> being the "
|
|
1214
|
+
"organization id with every hyphen as an underscore (acme-dk is "
|
|
1215
|
+
"acme_dk_marts); the identity it runs as can read those five datasets of "
|
|
1216
|
+
"its own org and nothing else, so a write to a dataset or a reach outside "
|
|
1217
|
+
"them is refused by the warehouse."
|
|
1169
1218
|
)
|
|
1170
1219
|
)
|
|
1171
1220
|
warehouse: str | None = Field(
|
|
@@ -135,18 +135,18 @@ def business_context_edit__append(
|
|
|
135
135
|
semantic_model_edit = typer.Typer(name='semantic_model_edit', help='Edit semantic models in the org manifest. Admin/owner only for writes.', no_args_is_help=True)
|
|
136
136
|
|
|
137
137
|
|
|
138
|
-
@semantic_model_edit.command('create', short_help='
|
|
138
|
+
@semantic_model_edit.command('create', short_help='Create a semantic model (admin/owner).')
|
|
139
139
|
def semantic_model_edit__create(
|
|
140
140
|
model_data: Annotated[Optional[str], typer.Option('--model-data-json', help='Full semantic-model definition (name, measures, dimensions, entities) (JSON, or @path to a .json/.yaml file)')],
|
|
141
141
|
expect_version: Annotated[Optional[str], typer.Option('--expect-version', help="Managed tenants only: the datacore version (draft snapshot id) this edit was written against, from the previous edit's `version` or dataspring://datacore/files; refused with the current one when stale")] = None,
|
|
142
142
|
from_file: FromFile = None,
|
|
143
143
|
output: Output = None,
|
|
144
144
|
) -> None:
|
|
145
|
-
"""
|
|
145
|
+
"""Create a semantic model (admin/owner)."""
|
|
146
146
|
run_tool('semantic_model_edit', contract.SemanticModelEditParams, 'create', {'model_data': J(model_data), 'expect_version': expect_version}, from_file=from_file, output=output)
|
|
147
147
|
|
|
148
148
|
|
|
149
|
-
@semantic_model_edit.command('update', short_help='
|
|
149
|
+
@semantic_model_edit.command('update', short_help='Change a semantic model (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once.')
|
|
150
150
|
def semantic_model_edit__update(
|
|
151
151
|
name: Annotated[str, typer.Option('--name', help='name')],
|
|
152
152
|
updates: Annotated[Optional[str], typer.Option('--updates-json', help='updates (JSON, or @path to a .json/.yaml file)')],
|
|
@@ -155,18 +155,18 @@ def semantic_model_edit__update(
|
|
|
155
155
|
from_file: FromFile = None,
|
|
156
156
|
output: Output = None,
|
|
157
157
|
) -> None:
|
|
158
|
-
"""
|
|
158
|
+
"""Change a semantic model (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once."""
|
|
159
159
|
run_tool('semantic_model_edit', contract.SemanticModelEditParams, 'update', {'name': name, 'updates': J(updates), 'confirmed': confirmed, 'expect_version': expect_version}, from_file=from_file, output=output)
|
|
160
160
|
|
|
161
161
|
|
|
162
|
-
@semantic_model_edit.command('delete', short_help='
|
|
162
|
+
@semantic_model_edit.command('delete', short_help='Delete a semantic model (admin/owner).')
|
|
163
163
|
def semantic_model_edit__delete(
|
|
164
164
|
name: Annotated[str, typer.Option('--name', help='name')],
|
|
165
165
|
expect_version: Annotated[Optional[str], typer.Option('--expect-version', help="Managed tenants only: the datacore version (draft snapshot id) this edit was written against, from the previous edit's `version` or dataspring://datacore/files; refused with the current one when stale")] = None,
|
|
166
166
|
from_file: FromFile = None,
|
|
167
167
|
output: Output = None,
|
|
168
168
|
) -> None:
|
|
169
|
-
"""
|
|
169
|
+
"""Delete a semantic model (admin/owner)."""
|
|
170
170
|
run_tool('semantic_model_edit', contract.SemanticModelEditParams, 'delete', {'name': name, 'expect_version': expect_version}, from_file=from_file, output=output)
|
|
171
171
|
|
|
172
172
|
|
|
@@ -188,18 +188,18 @@ def semantic_model_edit__preview(
|
|
|
188
188
|
metric_edit = typer.Typer(name='metric_edit', help='Edit metrics in the org manifest. Admin/owner only for writes.', no_args_is_help=True)
|
|
189
189
|
|
|
190
190
|
|
|
191
|
-
@metric_edit.command('create', short_help='
|
|
191
|
+
@metric_edit.command('create', short_help="Create a metric, native to DataSpring (admin/owner). A ratio's numerator and denominator, a derived metric's inputs and a cumulative or conversion metric's base are metrics, not measures.")
|
|
192
192
|
def metric_edit__create(
|
|
193
193
|
metric_data: Annotated[Optional[str], typer.Option('--metric-data-json', help='Metric definition (name, type, type_params) (JSON, or @path to a .json/.yaml file)')],
|
|
194
194
|
expect_version: Annotated[Optional[str], typer.Option('--expect-version', help="Managed tenants only: the datacore version (draft snapshot id) this edit was written against, from the previous edit's `version` or dataspring://datacore/files; refused with the current one when stale")] = None,
|
|
195
195
|
from_file: FromFile = None,
|
|
196
196
|
output: Output = None,
|
|
197
197
|
) -> None:
|
|
198
|
-
"""
|
|
198
|
+
"""Create a metric, native to DataSpring (admin/owner). A ratio's numerator and denominator, a derived metric's inputs and a cumulative or conversion metric's base are metrics, not measures."""
|
|
199
199
|
run_tool('metric_edit', contract.MetricEditParams, 'create', {'metric_data': J(metric_data), 'expect_version': expect_version}, from_file=from_file, output=output)
|
|
200
200
|
|
|
201
201
|
|
|
202
|
-
@metric_edit.command('update', short_help='
|
|
202
|
+
@metric_edit.command('update', short_help='Change a metric (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once.')
|
|
203
203
|
def metric_edit__update(
|
|
204
204
|
name: Annotated[str, typer.Option('--name', help='name')],
|
|
205
205
|
updates: Annotated[Optional[str], typer.Option('--updates-json', help='updates (JSON, or @path to a .json/.yaml file)')],
|
|
@@ -208,29 +208,29 @@ def metric_edit__update(
|
|
|
208
208
|
from_file: FromFile = None,
|
|
209
209
|
output: Output = None,
|
|
210
210
|
) -> None:
|
|
211
|
-
"""
|
|
211
|
+
"""Change a metric (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once."""
|
|
212
212
|
run_tool('metric_edit', contract.MetricEditParams, 'update', {'name': name, 'updates': J(updates), 'confirmed': confirmed, 'expect_version': expect_version}, from_file=from_file, output=output)
|
|
213
213
|
|
|
214
214
|
|
|
215
|
-
@metric_edit.command('delete', short_help='
|
|
215
|
+
@metric_edit.command('delete', short_help='Delete a metric (admin/owner; irreversible).')
|
|
216
216
|
def metric_edit__delete(
|
|
217
217
|
name: Annotated[str, typer.Option('--name', help='name')],
|
|
218
218
|
expect_version: Annotated[Optional[str], typer.Option('--expect-version', help="Managed tenants only: the datacore version (draft snapshot id) this edit was written against, from the previous edit's `version` or dataspring://datacore/files; refused with the current one when stale")] = None,
|
|
219
219
|
from_file: FromFile = None,
|
|
220
220
|
output: Output = None,
|
|
221
221
|
) -> None:
|
|
222
|
-
"""
|
|
222
|
+
"""Delete a metric (admin/owner; irreversible)."""
|
|
223
223
|
run_tool('metric_edit', contract.MetricEditParams, 'delete', {'name': name, 'expect_version': expect_version}, from_file=from_file, output=output)
|
|
224
224
|
|
|
225
225
|
|
|
226
|
-
@metric_edit.command('preview', short_help='`
|
|
226
|
+
@metric_edit.command('preview', short_help='Test a metric before saving (any role; writes nothing): sample query results or validation errors, plus the `dbt_yaml` that would be committed for it.')
|
|
227
227
|
def metric_edit__preview(
|
|
228
228
|
metric_data: Annotated[Optional[str], typer.Option('--metric-data-json', help='Metric definition to preview (JSON, or @path to a .json/.yaml file)')],
|
|
229
229
|
sample_query: Annotated[Optional[str], typer.Option('--sample-query-json', help='Optional query params (start_date, end_date, grain, limit) (JSON, or @path to a .json/.yaml file)')] = None,
|
|
230
230
|
from_file: FromFile = None,
|
|
231
231
|
output: Output = None,
|
|
232
232
|
) -> None:
|
|
233
|
-
"""`
|
|
233
|
+
"""Test a metric before saving (any role; writes nothing): sample query results or validation errors, plus the `dbt_yaml` that would be committed for it."""
|
|
234
234
|
run_tool('metric_edit', contract.MetricEditParams, 'preview', {'metric_data': J(metric_data), 'sample_query': J(sample_query)}, from_file=from_file, output=output)
|
|
235
235
|
|
|
236
236
|
|
|
@@ -253,7 +253,7 @@ def metric_edit__impact(
|
|
|
253
253
|
quick_metric_edit = typer.Typer(name='quick_metric_edit', help='Edit user-defined quick metrics (arithmetic expressions over existing metrics).', no_args_is_help=True)
|
|
254
254
|
|
|
255
255
|
|
|
256
|
-
@quick_metric_edit.command('create', short_help='`
|
|
256
|
+
@quick_metric_edit.command('create', short_help='Save a personal quick metric: arithmetic over existing metrics, queried as `qm:<name>`.')
|
|
257
257
|
def quick_metric_edit__create(
|
|
258
258
|
name: Annotated[str, typer.Option('--name', help='Metric name (e.g., "revenue_per_order")')],
|
|
259
259
|
expression: Annotated[str, typer.Option('--expression', help='Arithmetic expression over existing metrics (e.g., "total_revenue / order_count")')],
|
|
@@ -261,11 +261,11 @@ def quick_metric_edit__create(
|
|
|
261
261
|
from_file: FromFile = None,
|
|
262
262
|
output: Output = None,
|
|
263
263
|
) -> None:
|
|
264
|
-
"""`
|
|
264
|
+
"""Save a personal quick metric: arithmetic over existing metrics, queried as `qm:<name>`."""
|
|
265
265
|
run_tool('quick_metric_edit', contract.QuickMetricEditParams, 'create', {'name': name, 'expression': expression, 'description': description}, from_file=from_file, output=output)
|
|
266
266
|
|
|
267
267
|
|
|
268
|
-
@quick_metric_edit.command('update', short_help='
|
|
268
|
+
@quick_metric_edit.command('update', short_help='Change one of your quick metrics.')
|
|
269
269
|
def quick_metric_edit__update(
|
|
270
270
|
metric_id: Annotated[str, typer.Option('--metric-id', help='metric id')],
|
|
271
271
|
name: Annotated[Optional[str], typer.Option('--name', help='name')] = None,
|
|
@@ -274,17 +274,17 @@ def quick_metric_edit__update(
|
|
|
274
274
|
from_file: FromFile = None,
|
|
275
275
|
output: Output = None,
|
|
276
276
|
) -> None:
|
|
277
|
-
"""
|
|
277
|
+
"""Change one of your quick metrics."""
|
|
278
278
|
run_tool('quick_metric_edit', contract.QuickMetricEditParams, 'update', {'metric_id': metric_id, 'name': name, 'expression': expression, 'description': description}, from_file=from_file, output=output)
|
|
279
279
|
|
|
280
280
|
|
|
281
|
-
@quick_metric_edit.command('delete', short_help='
|
|
281
|
+
@quick_metric_edit.command('delete', short_help='Delete one of your quick metrics.')
|
|
282
282
|
def quick_metric_edit__delete(
|
|
283
283
|
metric_id: Annotated[str, typer.Option('--metric-id', help='metric id')],
|
|
284
284
|
from_file: FromFile = None,
|
|
285
285
|
output: Output = None,
|
|
286
286
|
) -> None:
|
|
287
|
-
"""
|
|
287
|
+
"""Delete one of your quick metrics."""
|
|
288
288
|
run_tool('quick_metric_edit', contract.QuickMetricEditParams, 'delete', {'metric_id': metric_id}, from_file=from_file, output=output)
|
|
289
289
|
|
|
290
290
|
|
|
@@ -556,6 +556,13 @@ def query_metrics(
|
|
|
556
556
|
A ``where`` you send on the same dimension overrides that standing
|
|
557
557
|
filter and nothing is reported for it — say that too.
|
|
558
558
|
|
|
559
|
+
A grouped query is clamped to the window asked for. When that cuts a
|
|
560
|
+
bucket short (a year to date at year grain is one row labelled with the
|
|
561
|
+
whole year), the answer carries ``effective_window``: ``covered`` (the
|
|
562
|
+
days the numbers cover) and ``partial_periods`` (each short bucket and
|
|
563
|
+
the days it holds). Name those days beside the number, and compare with
|
|
564
|
+
``compare_to``, which measures the same days, never with a whole period.
|
|
565
|
+
|
|
559
566
|
Examples:
|
|
560
567
|
- Monthly revenue: metrics=['total_revenue'], grain='month', start_date='2024-01-01'
|
|
561
568
|
- Revenue by region: metrics=['total_revenue'], dimensions=['region']
|
|
@@ -770,7 +777,7 @@ def secret_edit__reveal(
|
|
|
770
777
|
|
|
771
778
|
|
|
772
779
|
def run_sql(
|
|
773
|
-
sql: Annotated[str, typer.Option('--sql', help=
|
|
780
|
+
sql: Annotated[str, typer.Option('--sql', help="One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse.")],
|
|
774
781
|
warehouse: Annotated[Optional[str], typer.Option('--warehouse', help="One of the org's warehouses (see dataspring://warehouses); omit for the active one.")] = None,
|
|
775
782
|
max_rows: Annotated[Optional[int], typer.Option('--max-rows', help='Rows to return at most (default 1000, cap 10000). Aggregate in SQL rather than paging. (default: 1000)')] = None,
|
|
776
783
|
format: Annotated[Optional[str], typer.Option('--format', help="'json' answers rows in `data`; 'csv' or 'markdown' answer one document in `content`. (one of: json, csv, markdown) (default: json)")] = None,
|
|
@@ -778,21 +785,27 @@ def run_sql(
|
|
|
778
785
|
from_file: FromFile = None,
|
|
779
786
|
output: Output = None,
|
|
780
787
|
) -> None:
|
|
781
|
-
"""Run
|
|
782
|
-
the rows back, ungoverned.
|
|
788
|
+
"""Run read-only SQL (one statement, or a script whose last statement
|
|
789
|
+
answers) against the org's warehouse and get the rows back, ungoverned.
|
|
783
790
|
|
|
784
791
|
Metrics first: ``list_metrics`` / ``query_metrics`` answer with the
|
|
785
792
|
org's own definitions (units, filters, caveats) and are what a number
|
|
786
793
|
in an answer should come from. Use ``run_sql`` for what the semantic
|
|
787
794
|
layer does not expose: a column it has no dimension for, a row-level
|
|
788
|
-
look at a mart, a join the models do not make, checking a load
|
|
789
|
-
|
|
790
|
-
|
|
795
|
+
look at a mart, a join the models do not make, checking a load, the
|
|
796
|
+
rows a staging model or a datacore check produced. Say in the answer
|
|
797
|
+
that such numbers are not metric definitions (every result carries
|
|
798
|
+
``governed: false`` and a note).
|
|
791
799
|
|
|
792
800
|
On a managed warehouse the statement runs as the tenant's read-only
|
|
793
|
-
identity (``t-<org>-reader
|
|
794
|
-
|
|
795
|
-
|
|
801
|
+
identity (``t-<org>-reader@``), which reads the org's own marts, core,
|
|
802
|
+
staging, raw (what the loads wrote) and dev (what the last
|
|
803
|
+
``datacore_run check`` built; tables there expire after 7 days) and
|
|
804
|
+
nothing else, so a write to a dataset, an export or a reach outside
|
|
805
|
+
those is refused by BigQuery itself. A script of several statements
|
|
806
|
+
runs as one job and answers the last one's rows; ``CREATE TEMP TABLE``
|
|
807
|
+
in it is scratch space that ends with the script. It is dry-run first
|
|
808
|
+
and refused over the byte cap;
|
|
796
809
|
the timeout and ``max_rows`` bound it further. On an external
|
|
797
810
|
warehouse it runs with the customer's own key. Every call is logged
|
|
798
811
|
with the statement and the caller.
|
|
@@ -808,24 +821,26 @@ def run_sql(
|
|
|
808
821
|
datacore_edit = typer.Typer(name='datacore_edit', help='Edit the datacore workspace: connections, pipelines and the dbt project of a managed tenant, as files. ``kind`` picks the family:', no_args_is_help=True)
|
|
809
822
|
|
|
810
823
|
|
|
811
|
-
@datacore_edit.command('apply', short_help='Write one workspace file whole (create or replace): a connection or pipeline as a ``document`` (validated against its schema and the catalog
|
|
824
|
+
@datacore_edit.command('apply', short_help='Write one workspace file whole (create or replace): a dbt project file as ``content``; a connection or pipeline as ``content`` (its YAML text, stored exactly as sent, comments and layout kept) or as a ``document`` (an object, stored as a fresh dump), validated against its schema and the catalog either way.')
|
|
812
825
|
def datacore_edit__apply(
|
|
813
826
|
kind: Annotated[str, typer.Option('--kind', help='connection (admin), pipeline (admin) or model (member) (one of: connection, pipeline, model)')],
|
|
814
827
|
name: Annotated[str, typer.Option('--name', help='For kind connection or pipeline: the name, i.e. the file name without .yaml (connections/<name>.yaml, pipelines/<name>.yaml). For kind model: the path under models/, the dbt project (models/models/marts/orders.sql, models/models/staging/schema.yml, models/seeds/x.csv, models/dbt_project.yml); no traversal, .sql .yml .yaml .csv .md .txt .json, never target/ or dbt_packages/')],
|
|
815
828
|
document: Annotated[Optional[str], typer.Option('--document-json', help='kind connection: the whole document (connection.schema.json: kind, base_url, auth with secret NAMES). kind pipeline: the whole document (pipeline.schema.json: a catalog `connector` with config, secrets, overrides; or a `connection` reference, an inline `source` or named `sources`, with `resources`; schedule, checks, then). Secrets are named, never valued; every name must exist (JSON, or @path to a .json/.yaml file)')] = None,
|
|
816
|
-
content: Annotated[Optional[str], typer.Option('--content', help='kind model: the
|
|
829
|
+
content: Annotated[Optional[str], typer.Option('--content', help='The whole file as text. kind model: required. kind connection or pipeline: the YAML text, validated like a document and stored byte for byte (comments and layout kept); give content or document, not both')] = None,
|
|
817
830
|
expect_version: Annotated[Optional[str], typer.Option('--expect-version', help='The datacore version this edit was written against: the `version` the previous edit returned, or dataspring://datacore/files. An edit against a stale version is refused with the current one. Omit to skip the check.')] = None,
|
|
818
831
|
from_file: FromFile = None,
|
|
819
832
|
output: Output = None,
|
|
820
833
|
) -> None:
|
|
821
|
-
"""Write one workspace file whole (create or replace): a
|
|
822
|
-
pipeline as
|
|
823
|
-
|
|
834
|
+
"""Write one workspace file whole (create or replace): a dbt project file
|
|
835
|
+
as ``content``; a connection or pipeline as ``content`` (its YAML text,
|
|
836
|
+
stored exactly as sent, comments and layout kept) or as a ``document``
|
|
837
|
+
(an object, stored as a fresh dump), validated against its schema and
|
|
838
|
+
the catalog either way.
|
|
824
839
|
"""
|
|
825
840
|
run_tool('datacore_edit', contract.DatacoreEditParams, 'apply', {'kind': kind, 'name': name, 'document': J(document), 'content': content, 'expect_version': expect_version}, from_file=from_file, output=output)
|
|
826
841
|
|
|
827
842
|
|
|
828
|
-
@datacore_edit.command('patch', short_help='Apply a JSON merge patch (RFC 7386: a key set to null is removed) to an existing YAML file: a connection, a pipeline (a schedule, a window, a check), or a YAML file under models/ (not SQL).')
|
|
843
|
+
@datacore_edit.command('patch', short_help='Apply a JSON merge patch (RFC 7386: a key set to null is removed) to an existing YAML file: a connection, a pipeline (a schedule, a window, a check), or a YAML file under models/ (not SQL). Only the lines of the keys the patch changes are rewritten; every other line, comments and indentation included, stays as written (an answer with ``reformatted`` says the file had to be rewritten whole).')
|
|
829
844
|
def datacore_edit__patch(
|
|
830
845
|
kind: Annotated[str, typer.Option('--kind', help='connection (admin), pipeline (admin) or model (member) (one of: connection, pipeline, model)')],
|
|
831
846
|
name: Annotated[str, typer.Option('--name', help='For kind connection or pipeline: the name, i.e. the file name without .yaml (connections/<name>.yaml, pipelines/<name>.yaml). For kind model: the path under models/, the dbt project (models/models/marts/orders.sql, models/models/staging/schema.yml, models/seeds/x.csv, models/dbt_project.yml); no traversal, .sql .yml .yaml .csv .md .txt .json, never target/ or dbt_packages/')],
|
|
@@ -836,7 +851,10 @@ def datacore_edit__patch(
|
|
|
836
851
|
) -> None:
|
|
837
852
|
"""Apply a JSON merge patch (RFC 7386: a key set to null is removed) to an
|
|
838
853
|
existing YAML file: a connection, a pipeline (a schedule, a window, a
|
|
839
|
-
check), or a YAML file under models/ (not SQL).
|
|
854
|
+
check), or a YAML file under models/ (not SQL). Only the lines of the
|
|
855
|
+
keys the patch changes are rewritten; every other line, comments and
|
|
856
|
+
indentation included, stays as written (an answer with ``reformatted``
|
|
857
|
+
says the file had to be rewritten whole).
|
|
840
858
|
"""
|
|
841
859
|
run_tool('datacore_edit', contract.DatacoreEditParams, 'patch', {'kind': kind, 'name': name, 'patch': J(patch), 'expect_version': expect_version}, from_file=from_file, output=output)
|
|
842
860
|
|
|
@@ -861,17 +879,17 @@ def datacore_edit__delete(
|
|
|
861
879
|
datacore_run = typer.Typer(name='datacore_run', help="The datacore's check-then-deploy loop and its operations.", no_args_is_help=True)
|
|
862
880
|
|
|
863
881
|
|
|
864
|
-
@datacore_run.command('check', short_help='Check the current draft
|
|
882
|
+
@datacore_run.command('check', short_help='Check the current draft (member): compile every pipeline, build the changed dbt models into <org>_dev with their tests, and validate the semantic manifest per metric. Nothing reaches production.')
|
|
865
883
|
def datacore_run__check(
|
|
866
884
|
note: Annotated[Optional[str], typer.Option('--note', help='Optional note, recorded on the check run')] = None,
|
|
867
885
|
from_file: FromFile = None,
|
|
868
886
|
output: Output = None,
|
|
869
887
|
) -> None:
|
|
870
|
-
"""Check the current draft
|
|
888
|
+
"""Check the current draft (member): compile every pipeline, build the changed dbt models into <org>_dev with their tests, and validate the semantic manifest per metric. Nothing reaches production."""
|
|
871
889
|
run_tool('datacore_run', contract.DatacoreRunParams, 'check', {'note': note}, from_file=from_file, output=output)
|
|
872
890
|
|
|
873
891
|
|
|
874
|
-
@datacore_run.command('deploy', short_help='
|
|
892
|
+
@datacore_run.command('deploy', short_help='Promote the checked draft to deployed (admin), or roll back to an earlier deployed snapshot; then_build also rebuilds changed models.')
|
|
875
893
|
def datacore_run__deploy(
|
|
876
894
|
snapshot: Annotated[Optional[str], typer.Option('--snapshot', help='Omit to promote the current draft (needs a green check bound to it). A previously deployed snapshot id rolls back to it')] = None,
|
|
877
895
|
note: Annotated[Optional[str], typer.Option('--note', help='Why, recorded on the learned trail')] = None,
|
|
@@ -880,20 +898,20 @@ def datacore_run__deploy(
|
|
|
880
898
|
from_file: FromFile = None,
|
|
881
899
|
output: Output = None,
|
|
882
900
|
) -> None:
|
|
883
|
-
"""
|
|
901
|
+
"""Promote the checked draft to deployed (admin), or roll back to an earlier deployed snapshot; then_build also rebuilds changed models."""
|
|
884
902
|
run_tool('datacore_run', contract.DatacoreRunParams, 'deploy', {'snapshot': snapshot, 'note': note, 'then_build': then_build, 'build_select': build_select}, from_file=from_file, output=output)
|
|
885
903
|
|
|
886
904
|
|
|
887
|
-
@datacore_run.command('run', short_help='
|
|
905
|
+
@datacore_run.command('run', short_help='Load a pipeline now (admin): its own window, a given window, or a chunked backfill whose chunks run back to back.')
|
|
888
906
|
def datacore_run__run(
|
|
889
907
|
pipeline: Annotated[str, typer.Option('--pipeline', help='Pipeline name (a deployed pipelines/<name>.yaml)')],
|
|
890
908
|
window: Annotated[Optional[str], typer.Option('--window-json', help="A date window to load instead of the pipeline's own; wider than backfill_chunk becomes a chunked backfill. Both ends are inclusive days, and so is every chunk; a resource whose API reads its end bound as exclusive declares `window.end: exclusive` in the pipeline and is sent the day after each chunk's last day (JSON object with keys: start, end, or @path to a .json/.yaml file)")] = None,
|
|
891
909
|
cursor: Annotated[Optional[str], typer.Option('--cursor-json', help='{from: <value>}: a cursor to re-pull from (JSON object with keys: from, or @path to a .json/.yaml file)')] = None,
|
|
892
|
-
backfill_chunk: Annotated[Optional[str], typer.Option('--backfill-chunk', help='Chunk size for a windowed run: 7d, 2w or 1M (calendar months, the default)')] = None,
|
|
910
|
+
backfill_chunk: Annotated[Optional[str], typer.Option('--backfill-chunk', help='Chunk size for a windowed run: 7d, 2w or 1M (calendar months, the default). Every chunk before the last loads only the windowed resources (and the ones they read rows from) and skips its dbt step, each skip recorded with its reason on the child run; the last chunk loads every resource and builds the models once')] = None,
|
|
893
911
|
from_file: FromFile = None,
|
|
894
912
|
output: Output = None,
|
|
895
913
|
) -> None:
|
|
896
|
-
"""
|
|
914
|
+
"""Load a pipeline now (admin): its own window, a given window, or a chunked backfill whose chunks run back to back."""
|
|
897
915
|
run_tool('datacore_run', contract.DatacoreRunParams, 'run', {'pipeline': pipeline, 'window': J(window), 'cursor': J(cursor), 'backfill_chunk': backfill_chunk}, from_file=from_file, output=output)
|
|
898
916
|
|
|
899
917
|
|
|
@@ -908,24 +926,24 @@ def datacore_run__build(
|
|
|
908
926
|
run_tool('datacore_run', contract.DatacoreRunParams, 'build', {'pipeline': pipeline, 'select': select}, from_file=from_file, output=output)
|
|
909
927
|
|
|
910
928
|
|
|
911
|
-
@datacore_run.command('reset', short_help='
|
|
929
|
+
@datacore_run.command('reset', short_help="Delete a pipeline's dlt state and drop its raw tables (admin; irreversible).")
|
|
912
930
|
def datacore_run__reset(
|
|
913
931
|
pipeline: Annotated[str, typer.Option('--pipeline', help='The pipeline whose dlt state and raw tables to remove')],
|
|
914
932
|
approve: Annotated[Optional[bool], typer.Option('--approve/--no-approve', help='Must be true: the reset is irreversible. Ask the user first')] = None,
|
|
915
933
|
from_file: FromFile = None,
|
|
916
934
|
output: Output = None,
|
|
917
935
|
) -> None:
|
|
918
|
-
"""
|
|
936
|
+
"""Delete a pipeline's dlt state and drop its raw tables (admin; irreversible)."""
|
|
919
937
|
run_tool('datacore_run', contract.DatacoreRunParams, 'reset', {'pipeline': pipeline, 'approve': approve}, from_file=from_file, output=output)
|
|
920
938
|
|
|
921
939
|
|
|
922
|
-
@datacore_run.command('cancel', short_help='
|
|
940
|
+
@datacore_run.command('cancel', short_help='Stop a queued or running run, or a whole backfill (admin); the schedule fires again at its next slot.')
|
|
923
941
|
def datacore_run__cancel(
|
|
924
942
|
run_id: Annotated[str, typer.Option('--run-id', help="The queued or running run to stop, or a backfill's parent run (its running child is stopped and no further chunk starts); ids from dataspring://datacore/runs")],
|
|
925
943
|
from_file: FromFile = None,
|
|
926
944
|
output: Output = None,
|
|
927
945
|
) -> None:
|
|
928
|
-
"""
|
|
946
|
+
"""Stop a queued or running run, or a whole backfill (admin); the schedule fires again at its next slot."""
|
|
929
947
|
run_tool('datacore_run', contract.DatacoreRunParams, 'cancel', {'run_id': run_id}, from_file=from_file, output=output)
|
|
930
948
|
|
|
931
949
|
|
|
@@ -1167,7 +1185,7 @@ def mount(app: typer.Typer) -> None:
|
|
|
1167
1185
|
app.command('import_manifest', short_help='Import a semantic manifest using replace semantics.')(import_manifest)
|
|
1168
1186
|
app.command('submit_error_report', short_help='Submit an error report to help improve DataSpring.')(submit_error_report)
|
|
1169
1187
|
app.add_typer(secret_edit, name='secret_edit')
|
|
1170
|
-
app.command('run_sql', short_help="Run
|
|
1188
|
+
app.command('run_sql', short_help="Run read-only SQL (one statement, or a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.")(run_sql)
|
|
1171
1189
|
app.add_typer(datacore_edit, name='datacore_edit')
|
|
1172
1190
|
app.add_typer(datacore_run, name='datacore_run')
|
|
1173
1191
|
app.add_typer(tenant_edit, name='tenant_edit')
|