dataspring-cli 0.4.3__tar.gz → 0.4.5__tar.gz

{dataspring_cli-0.4.3 → dataspring_cli-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dataspring-cli
-Version: 0.4.3
+Version: 0.4.5
 Summary: DataSpring CLI - Query metrics and manage dashboards from the terminal
 Project-URL: Homepage, https://dataspring.app
 Project-URL: Documentation, https://dataspring.app/docs

{dataspring_cli-0.4.3 → dataspring_cli-0.4.5}/cli/bundled_manifest.py

@@ -6,7 +6,7 @@ the server's to say whether the server has operations this CLI does not
 know. tests/test_skills_surface.py asserts it is current.
 """
 
-MANIFEST_HASH = "e61e4a47f3c193beb2f6d731efde82839f5e62788bfa18560277459487bb26a7"
+MANIFEST_HASH = "54af1f0a9d07ef692b63e0648d835ce5efc020bd9588db6d75897b36af4e0232"
 TOOL_KEYS = (
     "business_context_edit",
     "semantic_model_edit",

{dataspring_cli-0.4.3 → dataspring_cli-0.4.5}/cli/contract.py

@@ -656,6 +656,14 @@ class CheckDatacoreAction(DispatchParams):
 
     action: Literal["check"]
     note: str | None = Field(default=None, description="Optional note, recorded on the check run")
+    build: bool = Field(
+        default=True,
+        description=(
+            "false = validate only: compile the pipelines, check the connections, dbt parse, the semantic "
+            "validation per metric and the manifest import's dry run, with no dbt build (an answer in well under "
+            "a minute, beside a running check or load). It is NEVER the green check deploy or auto_deploy needs"
+        ),
+    )
 
 
 class DeployDatacoreAction(DispatchParams):
@@ -707,16 +715,19 @@ class RunPipelineAction(DispatchParams):
         description=(
             "A date window to load instead of the pipeline's own; wider than backfill_chunk becomes a chunked backfill. "
             "Both ends are inclusive days, and so is every chunk; a resource whose API reads its end bound as exclusive "
-            "declares `window.end: exclusive` in the pipeline and is sent the day after each chunk's last day"
+            "declares `window.end: exclusive` in the pipeline and is sent the day after each chunk's last day. A rest resource "
+            "whose window has a `step` sends one request per step of the window or chunk, not one for the whole window"
         ),
     )
     cursor: RunCursorParams | None = Field(default=None, description="{from: <value>}: a cursor to re-pull from")
     backfill_chunk: str | None = Field(
         default=None,
         description=(
-            "Chunk size for a windowed run: 7d, 2w or 1M (calendar months, the default). Every chunk before the last "
-            "loads only the windowed resources (and the ones they read rows from) and skips its dbt step, each skip "
-            "recorded with its reason on the child run; the last chunk loads every resource and builds the models once"
+            "Chunk size for a windowed run: <n>d, <n>w or <n>M (days, weeks or calendar months; 1M is the default, "
+            "3M a quarter, 12M a year; there is no Y). Every chunk before the last "
+            "loads only the windowed resources (with their resolve children and the ones they read rows from) and "
+            "skips its dbt step, each skip recorded with its reason on the child run; the last chunk loads every "
+            "resource and builds the models once"
         ),
     )
 

{dataspring_cli-0.4.3 → dataspring_cli-0.4.5}/cli/generated.py

@@ -882,11 +882,12 @@ datacore_run = typer.Typer(name='datacore_run', help="The datacore's check-then-
 @datacore_run.command('check', short_help='Check the current draft (member): compile every pipeline, build the changed dbt models into <org>_dev with their tests, and validate the semantic manifest per metric. Nothing reaches production.')
 def datacore_run__check(
     note: Annotated[Optional[str], typer.Option('--note', help='Optional note, recorded on the check run')] = None,
+    build: Annotated[Optional[bool], typer.Option('--build/--no-build', help="false = validate only: compile the pipelines, check the connections, dbt parse, the semantic validation per metric and the manifest import's dry run, with no dbt build (an answer in well under a minute, beside a running check or load). It is NEVER the green check deploy or auto_deploy needs (default: True)")] = None,
     from_file: FromFile = None,
     output: Output = None,
 ) -> None:
     """Check the current draft (member): compile every pipeline, build the changed dbt models into <org>_dev with their tests, and validate the semantic manifest per metric. Nothing reaches production."""
-    run_tool('datacore_run', contract.DatacoreRunParams, 'check', {'note': note}, from_file=from_file, output=output)
+    run_tool('datacore_run', contract.DatacoreRunParams, 'check', {'note': note, 'build': build}, from_file=from_file, output=output)
 
 
 @datacore_run.command('deploy', short_help='Promote the checked draft to deployed (admin), or roll back to an earlier deployed snapshot; then_build also rebuilds changed models.')
@@ -905,9 +906,9 @@ def datacore_run__deploy(
 @datacore_run.command('run', short_help='Load a pipeline now (admin): its own window, a given window, or a chunked backfill whose chunks run back to back.')
 def datacore_run__run(
     pipeline: Annotated[str, typer.Option('--pipeline', help='Pipeline name (a deployed pipelines/<name>.yaml)')],
-    window: Annotated[Optional[str], typer.Option('--window-json', help="A date window to load instead of the pipeline's own; wider than backfill_chunk becomes a chunked backfill. Both ends are inclusive days, and so is every chunk; a resource whose API reads its end bound as exclusive declares `window.end: exclusive` in the pipeline and is sent the day after each chunk's last day (JSON object with keys: start, end, or @path to a .json/.yaml file)")] = None,
+    window: Annotated[Optional[str], typer.Option('--window-json', help="A date window to load instead of the pipeline's own; wider than backfill_chunk becomes a chunked backfill. Both ends are inclusive days, and so is every chunk; a resource whose API reads its end bound as exclusive declares `window.end: exclusive` in the pipeline and is sent the day after each chunk's last day. A rest resource whose window has a `step` sends one request per step of the window or chunk, not one for the whole window (JSON object with keys: start, end, or @path to a .json/.yaml file)")] = None,
     cursor: Annotated[Optional[str], typer.Option('--cursor-json', help='{from: <value>}: a cursor to re-pull from (JSON object with keys: from, or @path to a .json/.yaml file)')] = None,
-    backfill_chunk: Annotated[Optional[str], typer.Option('--backfill-chunk', help='Chunk size for a windowed run: 7d, 2w or 1M (calendar months, the default). Every chunk before the last loads only the windowed resources (and the ones they read rows from) and skips its dbt step, each skip recorded with its reason on the child run; the last chunk loads every resource and builds the models once')] = None,
+    backfill_chunk: Annotated[Optional[str], typer.Option('--backfill-chunk', help='Chunk size for a windowed run: <n>d, <n>w or <n>M (days, weeks or calendar months; 1M is the default, 3M a quarter, 12M a year; there is no Y). Every chunk before the last loads only the windowed resources (with their resolve children and the ones they read rows from) and skips its dbt step, each skip recorded with its reason on the child run; the last chunk loads every resource and builds the models once')] = None,
     from_file: FromFile = None,
     output: Output = None,
 ) -> None:

{dataspring_cli-0.4.3 → dataspring_cli-0.4.5}/cli/main.py

@@ -94,7 +94,9 @@ class DataSpringGroup(TyperGroup):
 #: exit codes an agent branches on (hub D-99, suite standards §6).
 ROOT_HELP = """DataSpring CLI: query metrics, run the datacore and manage boards from the terminal.
 
-Sign in with `dataspring auth login`. Every command acts in one org: --org, then DATASPRING_ORG or FORSPRING_ORG, then .forspring.json, then the key's own org, then the only stored key; `dataspring whoami` says which. No command changes the org for later ones.
+Sign in with `dataspring auth login`: it prints a code and a link; open the link on any device and approve (works over SSH and in containers too). Every command acts in one org: --org, then DATASPRING_ORG or FORSPRING_ORG, then .forspring.json, then the key's own org, then the only stored key; `dataspring whoami` says which. No command changes the org for later ones.
+
+Agents over MCP need no CLI: `claude mcp add --transport http --scope user dataspring https://dataspring.app/api/mcp/`, then sign in and approve the org in the browser. For CI with no person present: `export DATASPRING_API_KEY=<key>` (a key from the console's "Automation and CI"; it acts in its own org).
 
 A refusal prints `<reason>: <what happened> Fix: <what fixes it>` and exits 4 (not_signed_in, product_off, org_suspended, role_too_low, org_mismatch), 3 (not_a_member), 2 (org_required) or 1 (account_unavailable: retry). Any other failure exits 1, a usage error 2, a removed command 64."""
 
@@ -136,7 +138,7 @@ learned_app = typer.Typer(help="The learned trail: what DataSpring learned for t
 app.add_typer(learned_app, name="learned")
 warehouses_app = typer.Typer(help="The org's warehouses: external and managed, and which is active")
 app.add_typer(warehouses_app, name="warehouses")
-datacore_app = typer.Typer(help="The datacore workspace: pull, edit locally, push, check, deploy, build, run, cancel; schema, freshness and sample of its tables")
+datacore_app = typer.Typer(help="The datacore workspace: pull, edit locally, push, check, deploy, build, run, cancel; schema, freshness and sample of its tables; the reference (pipeline and connection schemas, the connector catalog)")
 app.add_typer(datacore_app, name="datacore")
 from cli.skills_commands import skills_app  # noqa: E402
 app.add_typer(skills_app, name="skills")
@@ -593,17 +595,25 @@ def auth_login(
     no_browser: Annotated[bool, typer.Option("--no-browser", help="Do not try to open a browser; just print the URL")] = False,
     all_orgs: Annotated[bool, typer.Option("--all-orgs", help="Ask for a key in every org you belong to, not only the one you approve (the browser says so)")] = False,
 ):
-    """Sign in with the device flow: approve in any browser where you are signed in with your Forspring account.
+    """Sign in: the CLI prints a code and a link; open the link on any device and approve.
 
-    Prints the URL and the code at once (also when the output is redirected,
-    so an agent can run it in the background and hand both to a person),
-    then waits for the approval. Mints an ordinary API key labelled 'CLI on
-    <hostname>' for the org you approve (one per org you are in with
-    --all-orgs), stored in ~/.dataspring/credentials.json.
+    It works over SSH and in containers too: the machine running the CLI
+    needs no browser. The URL and the code are printed at once (also when the
+    output is redirected, so an agent can run it in the background and hand
+    both to a person), then it waits for the approval. Mints an ordinary API
+    key labelled 'CLI on <hostname>' for the org you approve (one per org you
+    are in with --all-orgs), stored in ~/.dataspring/credentials.json.
 
     Logging in picks no org for later commands: with more than one key
     stored, name it per command (--org, DATASPRING_ORG, .forspring.json).
-    DATASPRING_API_KEY, when set, always wins over the stored keys.
+
+    Automation and CI, with no person present: export DATASPRING_API_KEY=<key>
+    (a key made in the console's connect page, under "Automation and CI"); it
+    always wins over the stored keys and acts in its own org. --key stores
+    such a key instead.
+
+    Agents over MCP do not need this: claude mcp add --transport http
+    --scope user dataspring https://dataspring.app/api/mcp/
     """
     try:
         if key:
@@ -2844,7 +2854,8 @@ def _check_finding(entry: dict) -> str:
 def _print_check_report(run: dict) -> None:
     report = run.get("check") or {}
     status = run.get("status")
-    line = f"check {run.get('id')}: {status}"
+    validate_only = run.get("build") is False or report.get("build") is False
+    line = f"{'validate-only check' if validate_only else 'check'} {run.get('id')}: {status}"
     if report.get("snapshot"):
         line += f" (snapshot {report['snapshot']})"
     (print_success if status == "succeeded" else print_error)(line)
@@ -2857,6 +2868,17 @@ def _print_check_report(run: dict) -> None:
     dbt = report.get("dbt") or {}
     if dbt.get("status") == "skipped" and dbt.get("reason"):
         typer.echo(f"  dbt        skipped: {dbt['reason']}")
+    elif dbt and validate_only:
+        # Tick ip2: parsed and validated, nothing built; the counts are what the parse found.
+        parsed = dbt.get("parsed") or {}
+        counts = ", ".join(f"{parsed[k]} {k.replace('_', ' ')}" for k in ("models", "semantic_models", "metrics") if k in parsed)
+        typer.echo(f"  dbt        {dbt.get('status')}: parsed and validated, nothing built" + (f" ({counts})" if counts else ""))
+        if dbt.get("error"):
+            typer.echo(f"             {dbt['error']}")
+        for failure in dbt.get("failures") or []:
+            typer.echo(f"             FAIL {_check_finding(failure)}")
+        for warning in dbt.get("warnings") or []:
+            typer.echo(f"             warn {_check_finding(warning)}")
     elif dbt:
         typer.echo(
             f"  dbt        {dbt.get('status')}: {dbt.get('models', 0)} models, {dbt.get('tests', 0)} tests, "
@@ -2882,6 +2904,8 @@ def _print_check_report(run: dict) -> None:
     if target and dbt.get("models"):
         typer.echo(f"  read it    dataspring sql \"SELECT * FROM {target}.<model> LIMIT 20\" "
                    f"(the models this check built; {target} expires tables after 7 days)")
+    if validate_only:
+        typer.echo("  note       validate-only: never the green check a deploy needs; `dataspring datacore check` builds and tests")
 
 
 def _poll_run(run_id: str, token: str, *, interval: float = 10.0, timeout: float = 1800.0) -> dict:
@@ -2905,10 +2929,25 @@ def _poll_run(run_id: str, token: str, *, interval: float = 10.0, timeout: float
 @datacore_app.command("check")
 def datacore_check_cmd(
     wait: Annotated[bool, typer.Option("--wait/--no-wait", help="Poll the check run to its end")] = True,
+    validate_only: Annotated[
+        bool,
+        typer.Option(
+            "--validate-only",
+            help="Parse and validate only (pipelines, connections, dbt parse, semantic and import validation), no dbt "
+            "build; never the green check deploy needs",
+        ),
+    ] = False,
 ):
     """Check the draft: schema validation here, then a check run of the
     tenant's job (compile, dbt build into <org>_dev, tests, dry-run bytes).
 
+    --validate-only is the edit loop's cheap answer: the same file checks,
+    pipelines compiled, connections checked, `dbt parse`, the semantic
+    validation per metric and the manifest import's dry run, and NO dbt
+    build, so it starts beside a running check or load and answers in well
+    under a minute. It never counts as the green check `datacore deploy`
+    (or auto_deploy) needs: run a full check before deploying.
+
     The check runs as the org's check identity, which can write <org>_dev and
     nothing else, so it never touches production; a dbt project that would
     build a model elsewhere is refused, naming the models. A workspace with
@@ -2923,24 +2962,27 @@ def datacore_check_cmd(
 
     Example:
         dataspring datacore check
+        dataspring datacore check --validate-only
     """
     require_auth()
     token = auth.get_access_token()
+    what = "validate-only check" if validate_only else "check"
     try:
-        result = call_tool("datacore_run", "check", {}, token=token)
+        result = call_tool("datacore_run", "check", {"build": False} if validate_only else {}, token=token)
     except Exception as e:
         fail(e, role_action="check the datacore")
     if not result.get("ok"):
-        print_error(f"check failed at {result.get('stage')} for snapshot {result.get('snapshot')}")
+        print_error(f"{what} failed at {result.get('stage')} for snapshot {result.get('snapshot')}")
         for error in result.get("errors") or []:
             typer.echo(f"  {error}")
         raise typer.Exit(1)
     run_id = result["run_id"]
-    print_info(f"check run {run_id} {result.get('status')} for snapshot {result.get('snapshot')}")
+    print_info(f"{what} run {run_id} {result.get('status')} for snapshot {result.get('snapshot')}")
     if not wait:
         return
     try:
-        run = _poll_run(run_id, token)
+        # A validate-only run is over in seconds; a build takes minutes.
+        run = _poll_run(run_id, token, interval=3.0 if validate_only else 10.0)
     except typer.Exit:
         raise
     except Exception as e:
@@ -3069,7 +3111,7 @@ def datacore_run_cmd(
     pipeline: Annotated[str, typer.Argument(help="Pipeline name")],
     start: Annotated[Optional[str], typer.Option("--start", help="Window start, YYYY-MM-DD")] = None,
     end: Annotated[Optional[str], typer.Option("--end", help="Window end, YYYY-MM-DD")] = None,
-    chunk: Annotated[Optional[str], typer.Option("--chunk", help="Backfill chunk: 7d, 2w or 1M. Chunks before the last load only windowed resources and skip dbt; the last loads everything and builds the models")] = None,
+    chunk: Annotated[Optional[str], typer.Option("--chunk", help="Backfill chunk: <n>d, <n>w or <n>M (7d, 2w, 1M the default, 3M a quarter, 12M a year). Chunks before the last load only windowed resources and skip dbt; the last loads everything and builds the models")] = None,
     cursor_from: Annotated[Optional[str], typer.Option("--from", help="Cursor value to re-pull from")] = None,
     wait: Annotated[bool, typer.Option("--wait", help="Poll the run to its end")] = False,
 ):
@@ -3077,7 +3119,10 @@ def datacore_run_cmd(
 
     --start and --end are both inclusive days, and so is every chunk. A
     resource whose API reads its end date as exclusive declares
-    `end: exclusive` in its `window` and is sent the day after.
+    `end: exclusive` in its `window` and is sent the day after. A rest
+    resource whose `window` has a `step` (7d, 2w, 1M) sends one request
+    per step of the run's window, and of every chunk, rather than one for
+    the whole window (for a source with a per-request row cap).
 
     A window wider than --chunk is a backfill: one child run per chunk, each
     started as the previous one ends, one backfill per pipeline at a time.
@@ -3302,6 +3347,125 @@ def datacore_sample_cmd(
         console.print(f"[dim]{statement}  as {result.get('identity')}; job {result.get('job_id')}[/]")
 
 
+DATACORE_REFERENCE_PATH = "/api/datacore/reference"
+ReferencePart = Literal["pipeline", "connection", "connector", "catalog"]
+
+
+def _print_reference_index(payload: dict) -> None:
+    for part, info in (payload.get("schemas") or {}).items():
+        typer.echo(f"{part:<11} {info.get('file')}")
+    typer.echo("")
+    rows = [
+        {"name": c["name"], "kind": "connector", "version": c.get("version"),
+         "secrets": ", ".join(c.get("secrets") or []), "tables": ", ".join(c.get("tables") or []),
+         "description": c.get("description", "")}
+        for c in payload.get("connectors") or []
+    ] + [
+        {"name": p["name"], "kind": "plugin", "version": p.get("version"),
+         "secrets": ", ".join(p.get("secret_fields") or []), "tables": ", ".join(p.get("tables") or []),
+         "description": ""}
+        for p in payload.get("plugins") or []
+    ]
+    format_output(rows, format="table", columns=["name", "kind", "version", "secrets", "tables", "description"],
+                  title="the connector catalog")
+    typer.echo("")
+    typer.echo(payload.get("secret_names", ""))
+    typer.echo("Whole parts: dataspring datacore reference pipeline|connection|connector|catalog [<name>]")
+
+
+def _print_reference_entry(payload: dict) -> None:
+    import yaml
+
+    typer.echo(f"{payload['name']} ({payload['kind']} version {payload.get('version')}): {payload.get('description', '')}".rstrip(": "))
+    props = (payload.get("config_schema") or {}).get("properties") or {}
+    required = set((payload.get("config_schema") or {}).get("required") or [])
+    if props:
+        format_output(
+            [{"config": key, "required": "yes" if key in required else "no",
+              "default": json.dumps(spec["default"]) if "default" in spec else "",
+              "description": spec.get("description", "")} for key, spec in props.items()],
+            format="table", columns=["config", "required", "default", "description"], title="config")
+    secrets = payload.get("secrets") or []
+    if secrets and isinstance(secrets[0], dict):
+        format_output(
+            [{"secret": s["field"], "required": "no" if s.get("optional") else "yes", "description": s.get("description", "")}
+             for s in secrets],
+            format="table", columns=["secret", "required", "description"], title="secrets")
+    resources = payload.get("resources") or {}
+    format_output(
+        [{"resource": name, "table": res.get("table", name), "when": res.get("when", "")} for name, res in resources.items()],
+        format="table", columns=["resource", "table", "when"], title="resources")
+    if payload.get("example"):
+        typer.echo("\npipelines/<name>.yaml:\n")
+        typer.echo(yaml.safe_dump(payload["example"], sort_keys=False).rstrip())
+    elif payload.get("usage"):
+        typer.echo(f"\n{payload['usage']}")
+
+
+@datacore_app.command("reference")
+def datacore_reference_cmd(
+    part: Annotated[Optional[ReferencePart], typer.Argument(help="pipeline, connection or connector (that JSON schema), or catalog")] = None,
+    name: Annotated[Optional[str], typer.Argument(help="With catalog: one connector or plugin")] = None,
+    format: Annotated[OutputFormat, typer.Option("--format", "-f", help="Output format (a schema prints as JSON unless yaml is asked for)")] = "table",
+):
+    """What a pipeline or connection file may say: the schemas and the connector catalog.
+
+    The same files the runner validates against, served by the server you
+    are signed in to, so they match what datacore_edit and datacore check
+    accept. With no argument: the schemas and every catalog connector and
+    plugin, with the secrets it needs and the tables it loads.
+
+    Examples:
+        dataspring datacore reference                       # the index and the catalog
+        dataspring datacore reference pipeline > pipeline.schema.json
+        dataspring datacore reference connection -f yaml
+        dataspring datacore reference catalog               # every connector, whole
+        dataspring datacore reference catalog kanpla        # one, with a pipeline file that uses it
+    """
+    from urllib.parse import quote
+
+    if name and part != "catalog":
+        print_error("a name goes with catalog: dataspring datacore reference catalog <name>")
+        raise typer.Exit(2)
+    path = DATACORE_REFERENCE_PATH
+    if part:
+        path += f"/{part}" + (f"/{quote(name, safe='')}" if name else "")
+    require_auth()
+    token = auth.get_access_token()
+    try:
+        payload = api_get(path, token)
+    except Exception as e:
+        fail(e, role_action="read the datacore reference")
+    if part in ("pipeline", "connection", "connector"):
+        if format == "yaml":
+            format_output(payload["schema"], format="yaml")
+        else:
+            typer.echo(json.dumps(payload["schema"], indent=2, ensure_ascii=False))
+        return
+    if format != "table":
+        format_output(payload, format=format)
+        return
+    if name:
+        _print_reference_entry(payload)
+    elif part == "catalog":
+        _print_reference_index({
+            "connectors": [
+                {"name": n, "version": c.get("version"), "description": c.get("description", ""),
+                 "secrets": [f + ("" if not s.get("optional") else " (optional)") for f, s in (c.get("secrets") or {}).items()],
+                 "tables": [r.get("table", rn) for rn, r in (c.get("resources") or {}).items()]}
+                for n, c in (payload.get("connectors") or {}).items()
+            ],
+            "plugins": [
+                {"name": n, "version": p.get("version"), "secret_fields": p.get("secrets") or [],
+                 "tables": [r.get("table", rn) for rn, r in (p.get("resources") or {}).items()]}
+                for n, p in (payload.get("plugins") or {}).items()
+            ],
+            "secret_names": payload.get("secret_names", ""),
+        })
+    else:
+        _print_reference_index(payload)
+
+
 @datacore_app.command("status")
 def datacore_status(
     format: Annotated[OutputFormat, typer.Option("--format", "-f", help="Output format")] = "table",

{dataspring_cli-0.4.3 → dataspring_cli-0.4.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataspring-cli"
-version = "0.4.3"
+version = "0.4.5"
 description = "DataSpring CLI - Query metrics and manage dashboards from the terminal"
 readme = "README.md"
 requires-python = ">=3.12"

{dataspring_cli-0.4.3 → dataspring_cli-0.4.5}/skills/dataspring-author/SKILL.md

@@ -21,8 +21,10 @@ at once (run it in the background and hand both to the person), then
 echo '{"org": "acme"}' > .forspring.json && dataspring whoami --json
 ```
 
-MCP: one connection per org, `https://dataspring.app/api/mcp/?org=<org>`, or
-`Authorization: Bearer ds_...` for a headless agent (the key's own org).
+MCP: `claude mcp add --transport http --scope user dataspring
+https://dataspring.app/api/mcp/` (or the same address in any client); the
+person signs in and the consent page names the org, bound to that connection.
+With no person present: `Authorization: Bearer ds_...` (the key's own org).
 
 A refusal leads with a reason and ends with its fix; match the reason. The
 ones you meet while building: `role_too_low` (connections, pipelines,
@@ -237,13 +239,14 @@ production.
 ```bash
 dataspring datacore pull ./ws            # deployed files, each exactly as pushed
 dataspring datacore push ./ws            # changed files only, into a new draft
+dataspring datacore check --validate-only  # seconds: parse + semantic validation, no build
 dataspring datacore check                # waits; exits 1 unless it passes
 dataspring datacore deploy --then-build  # admin; rebuilds changed models too
 ```
 
 - **A new org** pulls its starting workspace (`datacore.yaml`, empty
-  `pipelines/`, `connections/`, `models/`); `datacore
-  status` says `deployed none yet` until the first deploy.
+  `pipelines/` and `connections/`, and in `models/` a starting dbt project,
+  see 8.7).
 - **A check** compiles every pipeline, builds the changed models and their
   tests into `<org>_dev`, and dry-runs their SQL. It runs as an identity
   that can write `<org>_dev` only: a project whose `generate_schema_name`
@@ -262,6 +265,12 @@ dataspring datacore deploy --then-build  # admin; rebuilds changed models too
   type_params: {numerator: gross_margin, denominator: revenue}  # two metrics
 ```
 
+- **Validate-only while you edit** (`datacore_run` `check` with `build:
+  false`; CLI `datacore check --validate-only`): pipelines compiled,
+  connections checked, `dbt parse`, the per-metric semantic validation and
+  the import's dry run, and no `dbt build`, so a broken metric or model is
+  reported in well under a minute and it never waits behind a check or a
+  load. It is never a green check: run a full `check` before `deploy`.
 - **Deploy** needs a green check of that exact draft (any edit makes a new
   draft) and starts a run of each windowed pipeline it changed.
 
@@ -328,10 +337,41 @@ whose bytes equal the stored one's.
 
   Test an API's end bound with a one-day window before a backfill: a wrong
   guess silently drops the last day of every chunk.
+- **A window is one request** unless it has a `step`. For a source that caps
+  rows per request and does not page (e-regnskab: 20,000 item lines), a long
+  re-pull is cut into steps, one request each (`7d`, `2w`, `1M`; calendar
+  months, as backfill chunks are cut). `end: exclusive` holds per step, and
+  the scheduled window, a manual run and every backfill chunk all step:
+
+  ```yaml
+  window: { lookback: 365d, step: 1M, param_start: createdFrom, param_end: createdTo, end: exclusive }
+  ```
+
+  A rest resource only, and not a `resolve` child: step its parent.
+- **`resolve` fans out over the parent's own window** (its lookback, or the
+  run's window), not the child's. When the child's rows lack the parent's
+  key, carry it: `include_from_parent: [hnInvoiceSalesID]` adds the column
+  `_<parent resource>_<field>` (`_invoices_hnInvoiceSalesID`) to every child
+  row; use it in the child's `primary_key` or rename it in `shape`.
+  `param` fills a `{param}` placeholder in the endpoint, or else is sent as
+  a query parameter.
+- **An endpoint that answers a list of scalars** (`["budget", "forecast"]`)
+  loads one row per element in a column `value`; fan out over it with
+  `resolve: { field: value, ... }`.
 - **A backfill** is `dataspring datacore run <pipeline> --start ... --end ...
+  --chunk 1M`: one child run per chunk, each started as the previous one
+  ends. The chunk is `<n>d`, `<n>w` or `<n>M` (`7d`, `2w`, `1M`, `3M` a
+  quarter, `12M` a year; no `Y`). A chunk before the last loads only the
+  windowed resources (and those they read rows from) and skips dbt; the last
+  chunk loads everything and builds the models once. A failed child fails
+  the backfill. One backfill per pipeline at a time.
+- **One request per window.** A rest resource without a `paginator` sends
+  one request per window and nothing splits it, so a source that caps an
+  answer at N rows needs a `lookback`, and a `--chunk`, under N rows.
   --chunk 1M` (or `3M`, `7d`, `2w`): one child run per chunk, each started as
   the previous one ends. A chunk before the last loads only the windowed
-  resources (and those they read rows from) and skips dbt; the last chunk
+  resources, their `resolve` children (fetched for that chunk's parent
+  rows) and those they read rows from, and skips dbt; the last chunk
   loads everything and builds the models once. A failed child fails the
   backfill. One backfill per pipeline at a time.
 - **Stop** a queued or running run, or a whole backfill, with `dataspring
@@ -339,6 +379,63 @@ whose bytes equal the stored one's.
 - **`reset`** drops a pipeline's raw tables and load state. It is
   irreversible: ask first.
 
+## 8.6 Writing a pipeline file
+
+Do not read DataSpring's source for the grammar: it is served, cut from the
+files the runner validates against.
+
+```bash
+dataspring datacore reference                   # the schemas, every connector: secrets, tables
+dataspring datacore reference catalog kanpla    # one connector, whole, with a file that uses it
+dataspring datacore reference pipeline          # the pipeline JSON schema (also: connection)
+```
+
+MCP: `dataspring://datacore/reference[/<part>]` or `read(kind=
+"datacore_reference", id=<part>)`; REST `GET /api/datacore/reference`.
+Check the catalog first: a source with a connector is `connector: <name>`
+plus `config` and `secrets`. Otherwise write an inline `source` (or
+`sources`) and `resources`. The rules that bite:
+
+- **Secrets are names**, `t-<org>-<connection>-<field>`, lowercase letters,
+  digits and hyphens only (no underscore). Set each with `dataspring secret
+  set <name> --stdin` before pushing the file that names it.
+- **Windows** are whole inclusive days, `lookback` before today through
+  `lookahead` after, in the pipeline's timezone; declare `end: exclusive`
+  for an API whose end date is exclusive.
+- **Loads**: `replace` is a full refresh; `merge` needs `primary_key` (or
+  `merge_key`); `window_scope: replace` deletes by the `partition` date, so
+  use it only when that is the date the API filters on.
+- **Checks**: `min_rows` is refused on a `cursor` resource; use `freshness`.
+- **A `resolve` child of a windowed parent** needs its own `window`, and the
+  parent its window in `params` (`{{ window.start }}`), or backfill chunks
+  skip the child.
+
+The full guide with complete examples is "Writing a pipeline" in
+`dataspring docs --full`.
+
+## 8.7 The managed dbt project
+
+The workspace's `models/` is a dbt project (dbt-core 1.12, BigQuery). A new
+org starts with one that already keeps the contract; `dataspring datacore
+pull` shows it. The contract:
+
+- `profile: 'dataspring'` (the runner writes the profile; dataset
+  `<org>_marts`).
+- Sources at `schema: "{{ env_var('DBT_RAW_SCHEMA') }}"` and `database:
+  "{{ env_var('DBT_GCP_PROJECT') }}"`, never a dataset name.
+- `generate_schema_name` returns `DBT_SCHEMA_OVERRIDE` when set (a check:
+  `<org>_dev`), else `DBT_SCHEMA_PREFIX` + the layer (`+schema: staging` is
+  `<org>_staging`).
+- A `metricflow_time_spine` model with `time_spine:` configured; include it
+  in `then.dbt_select`.
+- Semantic models and metrics in YAML anywhere under `models/models/`;
+  `metric_edit` creates them in `models/models/semantic/metrics/`.
+- `packages.yml` is fetched by `dbt deps` on every run (`dbt_utils`).
+
+A deploy builds nothing: `deploy --then-build`, `datacore build`, or a
+pipeline's `then.dbt_select` does. The full contract is "The managed dbt
+project" in `dataspring docs --full`.
+
 ## 9. Parameter reference
 
 <!-- dispatch-params: semantic_model_edit metric_edit board_edit briefing_edit import_manifest business_context_edit warehouse_edit datacore_edit datacore_run -->
@@ -587,6 +684,7 @@ MCP tool `datacore_run`; `POST /api/dispatch/datacore_run`. The datacore's check
 | Parameter | Type | Required | Description |
 |---|---|---|---|
 | `note` | string | no | Optional note, recorded on the check run |
+| `build` | boolean | no, default `True` | false = validate only: compile the pipelines, check the connections, dbt parse, the semantic validation per metric and the manifest import's dry run, with no dbt build (an answer in well under a minute, beside a running check or load). It is NEVER the green check deploy or auto_deploy needs |
 
 **`deploy`** — Promote the checked draft to deployed (admin), or roll back to an earlier deployed snapshot; then_build also rebuilds changed models.
 
@@ -602,9 +700,9 @@ MCP tool `datacore_run`; `POST /api/dispatch/datacore_run`. The datacore's check
 | Parameter | Type | Required | Description |
 |---|---|---|---|
 | `pipeline` | string | yes | Pipeline name (a deployed pipelines/<name>.yaml) |
-| `window` | RunWindowParams | no | A date window to load instead of the pipeline's own; wider than backfill_chunk becomes a chunked backfill. Both ends are inclusive days, and so is every chunk; a resource whose API reads its end bound as exclusive declares `window.end: exclusive` in the pipeline and is sent the day after each chunk's last day |
+| `window` | RunWindowParams | no | A date window to load instead of the pipeline's own; wider than backfill_chunk becomes a chunked backfill. Both ends are inclusive days, and so is every chunk; a resource whose API reads its end bound as exclusive declares `window.end: exclusive` in the pipeline and is sent the day after each chunk's last day. A rest resource whose window has a `step` sends one request per step of the window or chunk, not one for the whole window |
 | `cursor` | RunCursorParams | no | {from: <value>}: a cursor to re-pull from |
-| `backfill_chunk` | string | no | Chunk size for a windowed run: 7d, 2w or 1M (calendar months, the default). Every chunk before the last loads only the windowed resources (and the ones they read rows from) and skips its dbt step, each skip recorded with its reason on the child run; the last chunk loads every resource and builds the models once |
+| `backfill_chunk` | string | no | Chunk size for a windowed run: <n>d, <n>w or <n>M (days, weeks or calendar months; 1M is the default, 3M a quarter, 12M a year; there is no Y). Every chunk before the last loads only the windowed resources (with their resolve children and the ones they read rows from) and skips its dbt step, each skip recorded with its reason on the child run; the last chunk loads every resource and builds the models once |
 
 **`build`** — Rebuild the deployed dbt models on the raw tables already loaded: no resource is loaded.
 

{dataspring_cli-0.4.3 → dataspring_cli-0.4.5}/skills/dataspring-consume/SKILL.md

@@ -8,8 +8,8 @@ description: How to query DataSpring correctly to answer analytical questions. U
 You are querying DataSpring. Three surfaces are available — pick the
 one the caller is using:
 
-- **MCP** — agent connected to a DataSpring MCP server (Claude
-  Desktop, Claude Code via stdio, etc.). Tool calls.
+- **MCP** — agent connected to DataSpring's MCP server (Claude.ai,
+  Claude Code, Cursor, etc.). Tool calls.
 - **CLI** — `dataspring` from the terminal. Shell commands.
 - **REST** — direct HTTP. Scripts, integrations.
 
@@ -80,11 +80,18 @@ which role, is the account's answer on every request. DataSpring has no
   parent. `DATASPRING_API_KEY=ds_...` (CI) acts in the key's own org.
   `dataspring whoami --json` says which org a command here acts in and why
   (`org_from`). There is no command that switches org for later commands.
-- **MCP.** One connection per org: `https://dataspring.app/api/mcp/?org=<org>`
-  (the slash before `?` is required), signed in with the Forspring account.
-  To work in two orgs, add two connections; no tool moves a connection. A
-  headless agent connects to the same URL with `Authorization: Bearer ds_...`;
-  the key acts in its own org, and `?org=` may only repeat it.
+- **MCP.** The address is `https://dataspring.app/api/mcp/`, with no org
+  in it. Claude Code: `claude mcp add --transport http --scope user
+  dataspring https://dataspring.app/api/mcp/`, then `/mcp` → *Authenticate*
+  (`claude mcp login dataspring` with no browser); Claude.ai: *Customize →
+  Connectors* → *Add custom connector* with that URL. The person signs in
+  with their Forspring account and DataSpring's consent page names the org
+  the connection works in (their only one, or the one they choose); the
+  choice is bound to the connection. To work in two orgs, add two
+  connections; no tool moves a connection. With no person present (CI), the
+  same address takes `Authorization: Bearer ds_...` (a key made in the
+  console's connect page, under "Automation and CI"): the key acts in its
+  own org, and `?org=` may only repeat it.
 - **REST.** `Authorization: Bearer ds_...`; the key is the org.
 
 Never pick an org for the person. When the org is ambiguous you are refused
@@ -595,12 +602,12 @@ the reason, never on the prose, and pass the fix on verbatim.
 | Reason | CLI exit | What you do |
 |---|---|---|
 | `not_signed_in` | 4 | CLI: run `dataspring auth login` and hand the person its URL and code. MCP: reconnect and sign in. Never ask for a password or a key in chat. |
-| `org_required` | 2 | Several orgs, none named. Ask the person which one (the refusal lists them); then `--org`, `DATASPRING_ORG` or `.forspring.json`. Never guess. |
+| `org_required` | 2 | Several orgs, none named. Ask the person which one (the refusal lists them); then `--org`, `DATASPRING_ORG` or `.forspring.json`. MCP: they reconnect and choose it on the consent page. Never guess. |
 | `not_a_member` | 3 | Check the org id's spelling against the list. Otherwise only an owner or admin of that org can add the person, at the Forspring account; say so and stop. |
 | `product_off` | 4 | DataSpring is not switched on for this person in that org. Tell them who can switch it on (`contacts`, `fix_url`); do not retry. |
 | `org_suspended` | 4 | Billing stopped DataSpring for the whole org. Tell them an owner can restart it on the billing page; do not retry. |
 | `role_too_low` | 4 | Their role cannot do this. Offer what it can do (a personal preference or quick metric instead of an org-wide change), or name the role needed. |
-| `org_mismatch` | 4 | The key or connection is for another org than the one named. Use that org's key or URL, or drop the conflicting `--org` / `?org=`. |
+| `org_mismatch` | 4 | The key or connection (approved for one org) is for another org than the one named. Use that org's key or connection, or drop the conflicting `--org` / `?org=`. |
 | `account_unavailable` | 1 | Forspring's account did not answer, so access could not be checked. Retry once after a few seconds; nothing changed. |
 
 Other CLI exits: 0 success, 1 any other failure, 2 a usage error, 64 a