dataspring-cli 0.4.2__tar.gz → 0.4.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/PKG-INFO +1 -1
  2. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/bundled_manifest.py +1 -1
  3. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/contract.py +62 -13
  4. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/generated.py +66 -48
  5. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/main.py +291 -28
  6. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/pyproject.toml +1 -1
  7. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/skills/dataspring-author/SKILL.md +139 -18
  8. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/skills/dataspring-consume/SKILL.md +104 -22
  9. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/skills/dataspring-correct/SKILL.md +4 -3
  10. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/.gitignore +0 -0
  11. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/README.md +0 -0
  12. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/__init__.py +0 -0
  13. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/auth.py +0 -0
  14. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/output.py +0 -0
  15. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/refusals.py +0 -0
  16. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/runtime.py +0 -0
  17. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/skills_commands.py +0 -0
  18. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/skilltree.py +0 -0
  19. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/upgrade.py +0 -0
  20. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/cli/version.py +0 -0
  21. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/hatch_build.py +0 -0
  22. {dataspring_cli-0.4.2 → dataspring_cli-0.4.3}/settings.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: dataspring-cli
3
- Version: 0.4.2
3
+ Version: 0.4.3
4
4
  Summary: DataSpring CLI - Query metrics and manage dashboards from the terminal
5
5
  Project-URL: Homepage, https://dataspring.app
6
6
  Project-URL: Documentation, https://dataspring.app/docs
@@ -6,7 +6,7 @@ the server's to say whether the server has operations this CLI does not
6
6
  know. tests/test_skills_surface.py asserts it is current.
7
7
  """
8
8
 
9
- MANIFEST_HASH = "b9fdc845fd9d3a22cea19e234b330e7e41d17f5d5ba6b5e1c713123d38109a6d"
9
+ MANIFEST_HASH = "e61e4a47f3c193beb2f6d731efde82839f5e62788bfa18560277459487bb26a7"
10
10
  TOOL_KEYS = (
11
11
  "business_context_edit",
12
12
  "semantic_model_edit",
@@ -54,6 +54,8 @@ _EXPECT_VERSION_DOC = (
54
54
 
55
55
 
56
56
  class CreateSemanticModelAction(DispatchParams):
57
+ """Create a semantic model (admin/owner)."""
58
+
57
59
  action: Literal["create"]
58
60
  model_data: dict = Field(
59
61
  description="Full semantic-model definition (name, measures, dimensions, entities)"
@@ -62,6 +64,8 @@ class CreateSemanticModelAction(DispatchParams):
62
64
 
63
65
 
64
66
  class UpdateSemanticModelAction(DispatchParams):
67
+ """Change a semantic model (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once."""
68
+
65
69
  action: Literal["update"]
66
70
  name: str
67
71
  updates: dict
@@ -77,6 +81,8 @@ class UpdateSemanticModelAction(DispatchParams):
77
81
 
78
82
 
79
83
  class DeleteSemanticModelAction(DispatchParams):
84
+ """Delete a semantic model (admin/owner)."""
85
+
80
86
  action: Literal["delete"]
81
87
  name: str
82
88
  expect_version: str | None = Field(default=None, description=_EXPECT_VERSION_DOC)
@@ -113,12 +119,16 @@ class SemanticModelEditParams(DispatchParams):
113
119
 
114
120
 
115
121
  class CreateMetricAction(DispatchParams):
122
+ """Create a metric, native to DataSpring (admin/owner). A ratio's numerator and denominator, a derived metric's inputs and a cumulative or conversion metric's base are metrics, not measures."""
123
+
116
124
  action: Literal["create"]
117
125
  metric_data: dict = Field(description="Metric definition (name, type, type_params)")
118
126
  expect_version: str | None = Field(default=None, description=_EXPECT_VERSION_DOC)
119
127
 
120
128
 
121
129
  class UpdateMetricAction(DispatchParams):
130
+ """Change a metric (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once."""
131
+
122
132
  action: Literal["update"]
123
133
  name: str
124
134
  updates: dict
@@ -134,12 +144,16 @@ class UpdateMetricAction(DispatchParams):
134
144
 
135
145
 
136
146
  class DeleteMetricAction(DispatchParams):
147
+ """Delete a metric (admin/owner; irreversible)."""
148
+
137
149
  action: Literal["delete"]
138
150
  name: str
139
151
  expect_version: str | None = Field(default=None, description=_EXPECT_VERSION_DOC)
140
152
 
141
153
 
142
154
  class PreviewMetricAction(DispatchParams):
155
+ """Test a metric before saving (any role; writes nothing): sample query results or validation errors, plus the `dbt_yaml` that would be committed for it."""
156
+
143
157
  action: Literal["preview"]
144
158
  metric_data: dict = Field(description="Metric definition to preview")
145
159
  sample_query: dict | None = Field(
@@ -178,6 +192,8 @@ class MetricEditParams(DispatchParams):
178
192
 
179
193
 
180
194
  class CreateQuickMetricAction(DispatchParams):
195
+ """Save a personal quick metric: arithmetic over existing metrics, queried as `qm:<name>`."""
196
+
181
197
  action: Literal["create"]
182
198
  name: str = Field(description='Metric name (e.g., "revenue_per_order")')
183
199
  expression: str = Field(
@@ -187,6 +203,8 @@ class CreateQuickMetricAction(DispatchParams):
187
203
 
188
204
 
189
205
  class UpdateQuickMetricAction(DispatchParams):
206
+ """Change one of your quick metrics."""
207
+
190
208
  action: Literal["update"]
191
209
  metric_id: str
192
210
  name: str | None = None
@@ -195,6 +213,8 @@ class UpdateQuickMetricAction(DispatchParams):
195
213
 
196
214
 
197
215
  class DeleteQuickMetricAction(DispatchParams):
216
+ """Delete one of your quick metrics."""
217
+
198
218
  action: Literal["delete"]
199
219
  metric_id: str
200
220
 
@@ -565,9 +585,11 @@ DatacoreKind = Literal["connection", "pipeline", "model"]
565
585
 
566
586
 
567
587
  class ApplyDatacoreFileAction(DispatchParams):
568
- """Write one workspace file whole (create or replace): a connection or
569
- pipeline as a ``document`` (validated against its schema and the
570
- catalog), a dbt project file as ``content``."""
588
+ """Write one workspace file whole (create or replace): a dbt project file
589
+ as ``content``; a connection or pipeline as ``content`` (its YAML text,
590
+ stored exactly as sent, comments and layout kept) or as a ``document``
591
+ (an object, stored as a fresh dump), validated against its schema and
592
+ the catalog either way."""
571
593
 
572
594
  action: Literal["apply"]
573
595
  kind: DatacoreKind = Field(description="connection (admin), pipeline (admin) or model (member)")
@@ -582,14 +604,24 @@ class ApplyDatacoreFileAction(DispatchParams):
582
604
  "Secrets are named, never valued; every name must exist"
583
605
  ),
584
606
  )
585
- content: str | None = Field(default=None, description="kind model: the whole file content")
607
+ content: str | None = Field(
608
+ default=None,
609
+ description=(
610
+ "The whole file as text. kind model: required. kind connection or pipeline: the YAML "
611
+ "text, validated like a document and stored byte for byte (comments and layout kept); "
612
+ "give content or document, not both"
613
+ ),
614
+ )
586
615
  expect_version: str | None = Field(default=None, description=_DATACORE_VERSION_DOC)
587
616
 
588
617
 
589
618
  class PatchDatacoreFileAction(DispatchParams):
590
619
  """Apply a JSON merge patch (RFC 7386: a key set to null is removed) to an
591
620
  existing YAML file: a connection, a pipeline (a schedule, a window, a
592
- check), or a YAML file under models/ (not SQL)."""
621
+ check), or a YAML file under models/ (not SQL). Only the lines of the
622
+ keys the patch changes are rewritten; every other line, comments and
623
+ indentation included, stays as written (an answer with ``reformatted``
624
+ says the file had to be rewritten whole)."""
593
625
 
594
626
  action: Literal["patch"]
595
627
  kind: DatacoreKind = Field(description="connection (admin), pipeline (admin) or model (member)")
@@ -620,13 +652,15 @@ class DatacoreEditParams(DispatchParams):
620
652
 
621
653
 
622
654
  class CheckDatacoreAction(DispatchParams):
623
- """Check the current draft; no arguments beyond an optional note."""
655
+ """Check the current draft (member): compile every pipeline, build the changed dbt models into <org>_dev with their tests, and validate the semantic manifest per metric. Nothing reaches production."""
624
656
 
625
657
  action: Literal["check"]
626
658
  note: str | None = Field(default=None, description="Optional note, recorded on the check run")
627
659
 
628
660
 
629
661
  class DeployDatacoreAction(DispatchParams):
662
+ """Promote the checked draft to deployed (admin), or roll back to an earlier deployed snapshot; then_build also rebuilds changed models."""
663
+
630
664
  action: Literal["deploy"]
631
665
  snapshot: str | None = Field(
632
666
  default=None,
@@ -664,6 +698,8 @@ class RunCursorParams(DispatchParams):
664
698
 
665
699
 
666
700
  class RunPipelineAction(DispatchParams):
701
+ """Load a pipeline now (admin): its own window, a given window, or a chunked backfill whose chunks run back to back."""
702
+
667
703
  action: Literal["run"]
668
704
  pipeline: str = Field(description="Pipeline name (a deployed pipelines/<name>.yaml)")
669
705
  window: RunWindowParams | None = Field(
@@ -677,7 +713,11 @@ class RunPipelineAction(DispatchParams):
677
713
  cursor: RunCursorParams | None = Field(default=None, description="{from: <value>}: a cursor to re-pull from")
678
714
  backfill_chunk: str | None = Field(
679
715
  default=None,
680
- description="Chunk size for a windowed run: 7d, 2w or 1M (calendar months, the default)",
716
+ description=(
717
+ "Chunk size for a windowed run: 7d, 2w or 1M (calendar months, the default). Every chunk before the last "
718
+ "loads only the windowed resources (and the ones they read rows from) and skips its dbt step, each skip "
719
+ "recorded with its reason on the child run; the last chunk loads every resource and builds the models once"
720
+ ),
681
721
  )
682
722
 
683
723
 
@@ -703,6 +743,8 @@ class BuildModelsAction(DispatchParams):
703
743
 
704
744
 
705
745
  class ResetPipelineAction(DispatchParams):
746
+ """Delete a pipeline's dlt state and drop its raw tables (admin; irreversible)."""
747
+
706
748
  action: Literal["reset"]
707
749
  pipeline: str = Field(description="The pipeline whose dlt state and raw tables to remove")
708
750
  approve: bool = Field(
@@ -712,6 +754,8 @@ class ResetPipelineAction(DispatchParams):
712
754
 
713
755
 
714
756
  class CancelRunAction(DispatchParams):
757
+ """Stop a queued or running run, or a whole backfill (admin); the schedule fires again at its next slot."""
758
+
715
759
  action: Literal["cancel"]
716
760
  run_id: str = Field(
717
761
  description=(
@@ -1160,12 +1204,17 @@ class RunSqlParams(DispatchParams):
1160
1204
 
1161
1205
  sql: str = Field(
1162
1206
  description=(
1163
- "One SQL statement (BigQuery Standard SQL on a managed warehouse). "
1164
- "Name tables as <org>_marts.<table>, <org>_core.<table> or "
1165
- "<org>_staging.<table>, <org> being the organization id with every "
1166
- "hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those and "
1167
- "nothing else, so a write or a reach outside them is refused by "
1168
- "the warehouse."
1207
+ "One SQL statement (BigQuery Standard SQL on a managed warehouse), or a "
1208
+ "script of several separated by ';' whose last statement's rows are "
1209
+ "returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the "
1210
+ "scratch space, gone when the script ends). Name tables as "
1211
+ "<org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, "
1212
+ "<org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the "
1213
+ "last datacore check built; expire after 7 days), <org> being the "
1214
+ "organization id with every hyphen as an underscore (acme-dk is "
1215
+ "acme_dk_marts); the identity it runs as can read those five datasets of "
1216
+ "its own org and nothing else, so a write to a dataset or a reach outside "
1217
+ "them is refused by the warehouse."
1169
1218
  )
1170
1219
  )
1171
1220
  warehouse: str | None = Field(
@@ -135,18 +135,18 @@ def business_context_edit__append(
135
135
  semantic_model_edit = typer.Typer(name='semantic_model_edit', help='Edit semantic models in the org manifest. Admin/owner only for writes.', no_args_is_help=True)
136
136
 
137
137
 
138
- @semantic_model_edit.command('create', short_help='`semantic_model_edit` create.')
138
+ @semantic_model_edit.command('create', short_help='Create a semantic model (admin/owner).')
139
139
  def semantic_model_edit__create(
140
140
  model_data: Annotated[Optional[str], typer.Option('--model-data-json', help='Full semantic-model definition (name, measures, dimensions, entities) (JSON, or @path to a .json/.yaml file)')],
141
141
  expect_version: Annotated[Optional[str], typer.Option('--expect-version', help="Managed tenants only: the datacore version (draft snapshot id) this edit was written against, from the previous edit's `version` or dataspring://datacore/files; refused with the current one when stale")] = None,
142
142
  from_file: FromFile = None,
143
143
  output: Output = None,
144
144
  ) -> None:
145
- """`semantic_model_edit` create."""
145
+ """Create a semantic model (admin/owner)."""
146
146
  run_tool('semantic_model_edit', contract.SemanticModelEditParams, 'create', {'model_data': J(model_data), 'expect_version': expect_version}, from_file=from_file, output=output)
147
147
 
148
148
 
149
- @semantic_model_edit.command('update', short_help='`semantic_model_edit` update.')
149
+ @semantic_model_edit.command('update', short_help='Change a semantic model (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once.')
150
150
  def semantic_model_edit__update(
151
151
  name: Annotated[str, typer.Option('--name', help='name')],
152
152
  updates: Annotated[Optional[str], typer.Option('--updates-json', help='updates (JSON, or @path to a .json/.yaml file)')],
@@ -155,18 +155,18 @@ def semantic_model_edit__update(
155
155
  from_file: FromFile = None,
156
156
  output: Output = None,
157
157
  ) -> None:
158
- """`semantic_model_edit` update."""
158
+ """Change a semantic model (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once."""
159
159
  run_tool('semantic_model_edit', contract.SemanticModelEditParams, 'update', {'name': name, 'updates': J(updates), 'confirmed': confirmed, 'expect_version': expect_version}, from_file=from_file, output=output)
160
160
 
161
161
 
162
- @semantic_model_edit.command('delete', short_help='`semantic_model_edit` delete.')
162
+ @semantic_model_edit.command('delete', short_help='Delete a semantic model (admin/owner).')
163
163
  def semantic_model_edit__delete(
164
164
  name: Annotated[str, typer.Option('--name', help='name')],
165
165
  expect_version: Annotated[Optional[str], typer.Option('--expect-version', help="Managed tenants only: the datacore version (draft snapshot id) this edit was written against, from the previous edit's `version` or dataspring://datacore/files; refused with the current one when stale")] = None,
166
166
  from_file: FromFile = None,
167
167
  output: Output = None,
168
168
  ) -> None:
169
- """`semantic_model_edit` delete."""
169
+ """Delete a semantic model (admin/owner)."""
170
170
  run_tool('semantic_model_edit', contract.SemanticModelEditParams, 'delete', {'name': name, 'expect_version': expect_version}, from_file=from_file, output=output)
171
171
 
172
172
 
@@ -188,18 +188,18 @@ def semantic_model_edit__preview(
188
188
  metric_edit = typer.Typer(name='metric_edit', help='Edit metrics in the org manifest. Admin/owner only for writes.', no_args_is_help=True)
189
189
 
190
190
 
191
- @metric_edit.command('create', short_help='`metric_edit` create.')
191
+ @metric_edit.command('create', short_help="Create a metric, native to DataSpring (admin/owner). A ratio's numerator and denominator, a derived metric's inputs and a cumulative or conversion metric's base are metrics, not measures.")
192
192
  def metric_edit__create(
193
193
  metric_data: Annotated[Optional[str], typer.Option('--metric-data-json', help='Metric definition (name, type, type_params) (JSON, or @path to a .json/.yaml file)')],
194
194
  expect_version: Annotated[Optional[str], typer.Option('--expect-version', help="Managed tenants only: the datacore version (draft snapshot id) this edit was written against, from the previous edit's `version` or dataspring://datacore/files; refused with the current one when stale")] = None,
195
195
  from_file: FromFile = None,
196
196
  output: Output = None,
197
197
  ) -> None:
198
- """`metric_edit` create."""
198
+ """Create a metric, native to DataSpring (admin/owner). A ratio's numerator and denominator, a derived metric's inputs and a cumulative or conversion metric's base are metrics, not measures."""
199
199
  run_tool('metric_edit', contract.MetricEditParams, 'create', {'metric_data': J(metric_data), 'expect_version': expect_version}, from_file=from_file, output=output)
200
200
 
201
201
 
202
- @metric_edit.command('update', short_help='`metric_edit` update.')
202
+ @metric_edit.command('update', short_help='Change a metric (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once.')
203
203
  def metric_edit__update(
204
204
  name: Annotated[str, typer.Option('--name', help='name')],
205
205
  updates: Annotated[Optional[str], typer.Option('--updates-json', help='updates (JSON, or @path to a .json/.yaml file)')],
@@ -208,29 +208,29 @@ def metric_edit__update(
208
208
  from_file: FromFile = None,
209
209
  output: Output = None,
210
210
  ) -> None:
211
- """`metric_edit` update."""
211
+ """Change a metric (admin/owner). On an imported one a change to how it computes answers a dbt YAML proposal and writes nothing; descriptions write at once."""
212
212
  run_tool('metric_edit', contract.MetricEditParams, 'update', {'name': name, 'updates': J(updates), 'confirmed': confirmed, 'expect_version': expect_version}, from_file=from_file, output=output)
213
213
 
214
214
 
215
- @metric_edit.command('delete', short_help='`metric_edit` delete.')
215
+ @metric_edit.command('delete', short_help='Delete a metric (admin/owner; irreversible).')
216
216
  def metric_edit__delete(
217
217
  name: Annotated[str, typer.Option('--name', help='name')],
218
218
  expect_version: Annotated[Optional[str], typer.Option('--expect-version', help="Managed tenants only: the datacore version (draft snapshot id) this edit was written against, from the previous edit's `version` or dataspring://datacore/files; refused with the current one when stale")] = None,
219
219
  from_file: FromFile = None,
220
220
  output: Output = None,
221
221
  ) -> None:
222
- """`metric_edit` delete."""
222
+ """Delete a metric (admin/owner; irreversible)."""
223
223
  run_tool('metric_edit', contract.MetricEditParams, 'delete', {'name': name, 'expect_version': expect_version}, from_file=from_file, output=output)
224
224
 
225
225
 
226
- @metric_edit.command('preview', short_help='`metric_edit` preview.')
226
+ @metric_edit.command('preview', short_help='Test a metric before saving (any role; writes nothing): sample query results or validation errors, plus the `dbt_yaml` that would be committed for it.')
227
227
  def metric_edit__preview(
228
228
  metric_data: Annotated[Optional[str], typer.Option('--metric-data-json', help='Metric definition to preview (JSON, or @path to a .json/.yaml file)')],
229
229
  sample_query: Annotated[Optional[str], typer.Option('--sample-query-json', help='Optional query params (start_date, end_date, grain, limit) (JSON, or @path to a .json/.yaml file)')] = None,
230
230
  from_file: FromFile = None,
231
231
  output: Output = None,
232
232
  ) -> None:
233
- """`metric_edit` preview."""
233
+ """Test a metric before saving (any role; writes nothing): sample query results or validation errors, plus the `dbt_yaml` that would be committed for it."""
234
234
  run_tool('metric_edit', contract.MetricEditParams, 'preview', {'metric_data': J(metric_data), 'sample_query': J(sample_query)}, from_file=from_file, output=output)
235
235
 
236
236
 
@@ -253,7 +253,7 @@ def metric_edit__impact(
253
253
  quick_metric_edit = typer.Typer(name='quick_metric_edit', help='Edit user-defined quick metrics (arithmetic expressions over existing metrics).', no_args_is_help=True)
254
254
 
255
255
 
256
- @quick_metric_edit.command('create', short_help='`quick_metric_edit` create.')
256
+ @quick_metric_edit.command('create', short_help='Save a personal quick metric: arithmetic over existing metrics, queried as `qm:<name>`.')
257
257
  def quick_metric_edit__create(
258
258
  name: Annotated[str, typer.Option('--name', help='Metric name (e.g., "revenue_per_order")')],
259
259
  expression: Annotated[str, typer.Option('--expression', help='Arithmetic expression over existing metrics (e.g., "total_revenue / order_count")')],
@@ -261,11 +261,11 @@ def quick_metric_edit__create(
261
261
  from_file: FromFile = None,
262
262
  output: Output = None,
263
263
  ) -> None:
264
- """`quick_metric_edit` create."""
264
+ """Save a personal quick metric: arithmetic over existing metrics, queried as `qm:<name>`."""
265
265
  run_tool('quick_metric_edit', contract.QuickMetricEditParams, 'create', {'name': name, 'expression': expression, 'description': description}, from_file=from_file, output=output)
266
266
 
267
267
 
268
- @quick_metric_edit.command('update', short_help='`quick_metric_edit` update.')
268
+ @quick_metric_edit.command('update', short_help='Change one of your quick metrics.')
269
269
  def quick_metric_edit__update(
270
270
  metric_id: Annotated[str, typer.Option('--metric-id', help='metric id')],
271
271
  name: Annotated[Optional[str], typer.Option('--name', help='name')] = None,
@@ -274,17 +274,17 @@ def quick_metric_edit__update(
274
274
  from_file: FromFile = None,
275
275
  output: Output = None,
276
276
  ) -> None:
277
- """`quick_metric_edit` update."""
277
+ """Change one of your quick metrics."""
278
278
  run_tool('quick_metric_edit', contract.QuickMetricEditParams, 'update', {'metric_id': metric_id, 'name': name, 'expression': expression, 'description': description}, from_file=from_file, output=output)
279
279
 
280
280
 
281
- @quick_metric_edit.command('delete', short_help='`quick_metric_edit` delete.')
281
+ @quick_metric_edit.command('delete', short_help='Delete one of your quick metrics.')
282
282
  def quick_metric_edit__delete(
283
283
  metric_id: Annotated[str, typer.Option('--metric-id', help='metric id')],
284
284
  from_file: FromFile = None,
285
285
  output: Output = None,
286
286
  ) -> None:
287
- """`quick_metric_edit` delete."""
287
+ """Delete one of your quick metrics."""
288
288
  run_tool('quick_metric_edit', contract.QuickMetricEditParams, 'delete', {'metric_id': metric_id}, from_file=from_file, output=output)
289
289
 
290
290
 
@@ -556,6 +556,13 @@ def query_metrics(
556
556
  A ``where`` you send on the same dimension overrides that standing
557
557
  filter and nothing is reported for it — say that too.
558
558
 
559
+ A grouped query is clamped to the window asked for. When that cuts a
560
+ bucket short (a year to date at year grain is one row labelled with the
561
+ whole year), the answer carries ``effective_window``: ``covered`` (the
562
+ days the numbers cover) and ``partial_periods`` (each short bucket and
563
+ the days it holds). Name those days beside the number, and compare with
564
+ ``compare_to``, which measures the same days, never with a whole period.
565
+
559
566
  Examples:
560
567
  - Monthly revenue: metrics=['total_revenue'], grain='month', start_date='2024-01-01'
561
568
  - Revenue by region: metrics=['total_revenue'], dimensions=['region']
@@ -770,7 +777,7 @@ def secret_edit__reveal(
770
777
 
771
778
 
772
779
  def run_sql(
773
- sql: Annotated[str, typer.Option('--sql', help='One SQL statement (BigQuery Standard SQL on a managed warehouse). Name tables as <org>_marts.<table>, <org>_core.<table> or <org>_staging.<table>, <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those and nothing else, so a write or a reach outside them is refused by the warehouse.')],
780
+ sql: Annotated[str, typer.Option('--sql', help="One SQL statement (BigQuery Standard SQL on a managed warehouse), or a script of several separated by ';' whose last statement's rows are returned (CREATE TEMP TABLE ... ; SELECT ... works: temp tables are the scratch space, gone when the script ends). Name tables as <org>_marts.<table>, <org>_core.<table>, <org>_staging.<table>, <org>_raw.<table> (what the loads wrote) or <org>_dev.<table> (what the last datacore check built; expire after 7 days), <org> being the organization id with every hyphen as an underscore (acme-dk is acme_dk_marts); the identity it runs as can read those five datasets of its own org and nothing else, so a write to a dataset or a reach outside them is refused by the warehouse.")],
774
781
  warehouse: Annotated[Optional[str], typer.Option('--warehouse', help="One of the org's warehouses (see dataspring://warehouses); omit for the active one.")] = None,
775
782
  max_rows: Annotated[Optional[int], typer.Option('--max-rows', help='Rows to return at most (default 1000, cap 10000). Aggregate in SQL rather than paging. (default: 1000)')] = None,
776
783
  format: Annotated[Optional[str], typer.Option('--format', help="'json' answers rows in `data`; 'csv' or 'markdown' answer one document in `content`. (one of: json, csv, markdown) (default: json)")] = None,
@@ -778,21 +785,27 @@ def run_sql(
778
785
  from_file: FromFile = None,
779
786
  output: Output = None,
780
787
  ) -> None:
781
- """Run ONE read-only SQL statement against the org's warehouse and get
782
- the rows back, ungoverned.
788
+ """Run read-only SQL (one statement, or a script whose last statement
789
+ answers) against the org's warehouse and get the rows back, ungoverned.
783
790
 
784
791
  Metrics first: ``list_metrics`` / ``query_metrics`` answer with the
785
792
  org's own definitions (units, filters, caveats) and are what a number
786
793
  in an answer should come from. Use ``run_sql`` for what the semantic
787
794
  layer does not expose: a column it has no dimension for, a row-level
788
- look at a mart, a join the models do not make, checking a load. Say in
789
- the answer that such numbers are not metric definitions (every result
790
- carries ``governed: false`` and a note).
795
+ look at a mart, a join the models do not make, checking a load, the
796
+ rows a staging model or a datacore check produced. Say in the answer
797
+ that such numbers are not metric definitions (every result carries
798
+ ``governed: false`` and a note).
791
799
 
792
800
  On a managed warehouse the statement runs as the tenant's read-only
793
- identity (``t-<org>-reader@``: marts, core and staging, nothing else),
794
- so any write, export or reach outside those datasets is refused by
795
- BigQuery itself. It is dry-run first and refused over the byte cap;
801
+ identity (``t-<org>-reader@``), which reads the org's own marts, core,
802
+ staging, raw (what the loads wrote) and dev (what the last
803
+ ``datacore_run check`` built; tables there expire after 7 days) and
804
+ nothing else, so a write to a dataset, an export or a reach outside
805
+ those is refused by BigQuery itself. A script of several statements
806
+ runs as one job and answers the last one's rows; ``CREATE TEMP TABLE``
807
+ in it is scratch space that ends with the script. It is dry-run first
808
+ and refused over the byte cap;
796
809
  the timeout and ``max_rows`` bound it further. On an external
797
810
  warehouse it runs with the customer's own key. Every call is logged
798
811
  with the statement and the caller.
@@ -808,24 +821,26 @@ def run_sql(
808
821
  datacore_edit = typer.Typer(name='datacore_edit', help='Edit the datacore workspace: connections, pipelines and the dbt project of a managed tenant, as files. ``kind`` picks the family:', no_args_is_help=True)
809
822
 
810
823
 
811
- @datacore_edit.command('apply', short_help='Write one workspace file whole (create or replace): a connection or pipeline as a ``document`` (validated against its schema and the catalog), a dbt project file as ``content``.')
824
+ @datacore_edit.command('apply', short_help='Write one workspace file whole (create or replace): a dbt project file as ``content``; a connection or pipeline as ``content`` (its YAML text, stored exactly as sent, comments and layout kept) or as a ``document`` (an object, stored as a fresh dump), validated against its schema and the catalog either way.')
812
825
  def datacore_edit__apply(
813
826
  kind: Annotated[str, typer.Option('--kind', help='connection (admin), pipeline (admin) or model (member) (one of: connection, pipeline, model)')],
814
827
  name: Annotated[str, typer.Option('--name', help='For kind connection or pipeline: the name, i.e. the file name without .yaml (connections/<name>.yaml, pipelines/<name>.yaml). For kind model: the path under models/, the dbt project (models/models/marts/orders.sql, models/models/staging/schema.yml, models/seeds/x.csv, models/dbt_project.yml); no traversal, .sql .yml .yaml .csv .md .txt .json, never target/ or dbt_packages/')],
815
828
  document: Annotated[Optional[str], typer.Option('--document-json', help='kind connection: the whole document (connection.schema.json: kind, base_url, auth with secret NAMES). kind pipeline: the whole document (pipeline.schema.json: a catalog `connector` with config, secrets, overrides; or a `connection` reference, an inline `source` or named `sources`, with `resources`; schedule, checks, then). Secrets are named, never valued; every name must exist (JSON, or @path to a .json/.yaml file)')] = None,
816
- content: Annotated[Optional[str], typer.Option('--content', help='kind model: the whole file content')] = None,
829
+ content: Annotated[Optional[str], typer.Option('--content', help='The whole file as text. kind model: required. kind connection or pipeline: the YAML text, validated like a document and stored byte for byte (comments and layout kept); give content or document, not both')] = None,
817
830
  expect_version: Annotated[Optional[str], typer.Option('--expect-version', help='The datacore version this edit was written against: the `version` the previous edit returned, or dataspring://datacore/files. An edit against a stale version is refused with the current one. Omit to skip the check.')] = None,
818
831
  from_file: FromFile = None,
819
832
  output: Output = None,
820
833
  ) -> None:
821
- """Write one workspace file whole (create or replace): a connection or
822
- pipeline as a ``document`` (validated against its schema and the
823
- catalog), a dbt project file as ``content``.
834
+ """Write one workspace file whole (create or replace): a dbt project file
835
+ as ``content``; a connection or pipeline as ``content`` (its YAML text,
836
+ stored exactly as sent, comments and layout kept) or as a ``document``
837
+ (an object, stored as a fresh dump), validated against its schema and
838
+ the catalog either way.
824
839
  """
825
840
  run_tool('datacore_edit', contract.DatacoreEditParams, 'apply', {'kind': kind, 'name': name, 'document': J(document), 'content': content, 'expect_version': expect_version}, from_file=from_file, output=output)
826
841
 
827
842
 
828
- @datacore_edit.command('patch', short_help='Apply a JSON merge patch (RFC 7386: a key set to null is removed) to an existing YAML file: a connection, a pipeline (a schedule, a window, a check), or a YAML file under models/ (not SQL).')
843
+ @datacore_edit.command('patch', short_help='Apply a JSON merge patch (RFC 7386: a key set to null is removed) to an existing YAML file: a connection, a pipeline (a schedule, a window, a check), or a YAML file under models/ (not SQL). Only the lines of the keys the patch changes are rewritten; every other line, comments and indentation included, stays as written (an answer with ``reformatted`` says the file had to be rewritten whole).')
829
844
  def datacore_edit__patch(
830
845
  kind: Annotated[str, typer.Option('--kind', help='connection (admin), pipeline (admin) or model (member) (one of: connection, pipeline, model)')],
831
846
  name: Annotated[str, typer.Option('--name', help='For kind connection or pipeline: the name, i.e. the file name without .yaml (connections/<name>.yaml, pipelines/<name>.yaml). For kind model: the path under models/, the dbt project (models/models/marts/orders.sql, models/models/staging/schema.yml, models/seeds/x.csv, models/dbt_project.yml); no traversal, .sql .yml .yaml .csv .md .txt .json, never target/ or dbt_packages/')],
@@ -836,7 +851,10 @@ def datacore_edit__patch(
836
851
  ) -> None:
837
852
  """Apply a JSON merge patch (RFC 7386: a key set to null is removed) to an
838
853
  existing YAML file: a connection, a pipeline (a schedule, a window, a
839
- check), or a YAML file under models/ (not SQL).
854
+ check), or a YAML file under models/ (not SQL). Only the lines of the
855
+ keys the patch changes are rewritten; every other line, comments and
856
+ indentation included, stays as written (an answer with ``reformatted``
857
+ says the file had to be rewritten whole).
840
858
  """
841
859
  run_tool('datacore_edit', contract.DatacoreEditParams, 'patch', {'kind': kind, 'name': name, 'patch': J(patch), 'expect_version': expect_version}, from_file=from_file, output=output)
842
860
 
@@ -861,17 +879,17 @@ def datacore_edit__delete(
861
879
  datacore_run = typer.Typer(name='datacore_run', help="The datacore's check-then-deploy loop and its operations.", no_args_is_help=True)
862
880
 
863
881
 
864
- @datacore_run.command('check', short_help='Check the current draft; no arguments beyond an optional note.')
882
+ @datacore_run.command('check', short_help='Check the current draft (member): compile every pipeline, build the changed dbt models into <org>_dev with their tests, and validate the semantic manifest per metric. Nothing reaches production.')
865
883
  def datacore_run__check(
866
884
  note: Annotated[Optional[str], typer.Option('--note', help='Optional note, recorded on the check run')] = None,
867
885
  from_file: FromFile = None,
868
886
  output: Output = None,
869
887
  ) -> None:
870
- """Check the current draft; no arguments beyond an optional note."""
888
+ """Check the current draft (member): compile every pipeline, build the changed dbt models into <org>_dev with their tests, and validate the semantic manifest per metric. Nothing reaches production."""
871
889
  run_tool('datacore_run', contract.DatacoreRunParams, 'check', {'note': note}, from_file=from_file, output=output)
872
890
 
873
891
 
874
- @datacore_run.command('deploy', short_help='`datacore_run` deploy.')
892
+ @datacore_run.command('deploy', short_help='Promote the checked draft to deployed (admin), or roll back to an earlier deployed snapshot; then_build also rebuilds changed models.')
875
893
  def datacore_run__deploy(
876
894
  snapshot: Annotated[Optional[str], typer.Option('--snapshot', help='Omit to promote the current draft (needs a green check bound to it). A previously deployed snapshot id rolls back to it')] = None,
877
895
  note: Annotated[Optional[str], typer.Option('--note', help='Why, recorded on the learned trail')] = None,
@@ -880,20 +898,20 @@ def datacore_run__deploy(
880
898
  from_file: FromFile = None,
881
899
  output: Output = None,
882
900
  ) -> None:
883
- """`datacore_run` deploy."""
901
+ """Promote the checked draft to deployed (admin), or roll back to an earlier deployed snapshot; then_build also rebuilds changed models."""
884
902
  run_tool('datacore_run', contract.DatacoreRunParams, 'deploy', {'snapshot': snapshot, 'note': note, 'then_build': then_build, 'build_select': build_select}, from_file=from_file, output=output)
885
903
 
886
904
 
887
- @datacore_run.command('run', short_help='`datacore_run` run.')
905
+ @datacore_run.command('run', short_help='Load a pipeline now (admin): its own window, a given window, or a chunked backfill whose chunks run back to back.')
888
906
  def datacore_run__run(
889
907
  pipeline: Annotated[str, typer.Option('--pipeline', help='Pipeline name (a deployed pipelines/<name>.yaml)')],
890
908
  window: Annotated[Optional[str], typer.Option('--window-json', help="A date window to load instead of the pipeline's own; wider than backfill_chunk becomes a chunked backfill. Both ends are inclusive days, and so is every chunk; a resource whose API reads its end bound as exclusive declares `window.end: exclusive` in the pipeline and is sent the day after each chunk's last day (JSON object with keys: start, end, or @path to a .json/.yaml file)")] = None,
891
909
  cursor: Annotated[Optional[str], typer.Option('--cursor-json', help='{from: <value>}: a cursor to re-pull from (JSON object with keys: from, or @path to a .json/.yaml file)')] = None,
892
- backfill_chunk: Annotated[Optional[str], typer.Option('--backfill-chunk', help='Chunk size for a windowed run: 7d, 2w or 1M (calendar months, the default)')] = None,
910
+ backfill_chunk: Annotated[Optional[str], typer.Option('--backfill-chunk', help='Chunk size for a windowed run: 7d, 2w or 1M (calendar months, the default). Every chunk before the last loads only the windowed resources (and the ones they read rows from) and skips its dbt step, each skip recorded with its reason on the child run; the last chunk loads every resource and builds the models once')] = None,
893
911
  from_file: FromFile = None,
894
912
  output: Output = None,
895
913
  ) -> None:
896
- """`datacore_run` run."""
914
+ """Load a pipeline now (admin): its own window, a given window, or a chunked backfill whose chunks run back to back."""
897
915
  run_tool('datacore_run', contract.DatacoreRunParams, 'run', {'pipeline': pipeline, 'window': J(window), 'cursor': J(cursor), 'backfill_chunk': backfill_chunk}, from_file=from_file, output=output)
898
916
 
899
917
 
@@ -908,24 +926,24 @@ def datacore_run__build(
908
926
  run_tool('datacore_run', contract.DatacoreRunParams, 'build', {'pipeline': pipeline, 'select': select}, from_file=from_file, output=output)
909
927
 
910
928
 
911
- @datacore_run.command('reset', short_help='`datacore_run` reset.')
929
+ @datacore_run.command('reset', short_help="Delete a pipeline's dlt state and drop its raw tables (admin; irreversible).")
912
930
  def datacore_run__reset(
913
931
  pipeline: Annotated[str, typer.Option('--pipeline', help='The pipeline whose dlt state and raw tables to remove')],
914
932
  approve: Annotated[Optional[bool], typer.Option('--approve/--no-approve', help='Must be true: the reset is irreversible. Ask the user first')] = None,
915
933
  from_file: FromFile = None,
916
934
  output: Output = None,
917
935
  ) -> None:
918
- """`datacore_run` reset."""
936
+ """Delete a pipeline's dlt state and drop its raw tables (admin; irreversible)."""
919
937
  run_tool('datacore_run', contract.DatacoreRunParams, 'reset', {'pipeline': pipeline, 'approve': approve}, from_file=from_file, output=output)
920
938
 
921
939
 
922
- @datacore_run.command('cancel', short_help='`datacore_run` cancel.')
940
+ @datacore_run.command('cancel', short_help='Stop a queued or running run, or a whole backfill (admin); the schedule fires again at its next slot.')
923
941
  def datacore_run__cancel(
924
942
  run_id: Annotated[str, typer.Option('--run-id', help="The queued or running run to stop, or a backfill's parent run (its running child is stopped and no further chunk starts); ids from dataspring://datacore/runs")],
925
943
  from_file: FromFile = None,
926
944
  output: Output = None,
927
945
  ) -> None:
928
- """`datacore_run` cancel."""
946
+ """Stop a queued or running run, or a whole backfill (admin); the schedule fires again at its next slot."""
929
947
  run_tool('datacore_run', contract.DatacoreRunParams, 'cancel', {'run_id': run_id}, from_file=from_file, output=output)
930
948
 
931
949
 
@@ -1167,7 +1185,7 @@ def mount(app: typer.Typer) -> None:
1167
1185
  app.command('import_manifest', short_help='Import a semantic manifest using replace semantics.')(import_manifest)
1168
1186
  app.command('submit_error_report', short_help='Submit an error report to help improve DataSpring.')(submit_error_report)
1169
1187
  app.add_typer(secret_edit, name='secret_edit')
1170
- app.command('run_sql', short_help="Run ONE read-only SQL statement against the org's warehouse and get the rows back, ungoverned.")(run_sql)
1188
+ app.command('run_sql', short_help="Run read-only SQL (one statement, or a script whose last statement answers) against the org's warehouse and get the rows back, ungoverned.")(run_sql)
1171
1189
  app.add_typer(datacore_edit, name='datacore_edit')
1172
1190
  app.add_typer(datacore_run, name='datacore_run')
1173
1191
  app.add_typer(tenant_edit, name='tenant_edit')