cassis-cli 3.3.0__tar.gz → 3.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/PKG-INFO +52 -3
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/README.md +51 -2
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/api.py +91 -0
- cassis_cli-3.5.0/cassis_cli/chats.py +204 -0
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/dbt.py +9 -4
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/eval.py +212 -11
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/issues.py +6 -3
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/main.py +2 -0
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/schema.py +1 -1
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/status.py +1 -1
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/verify.py +1 -0
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/pyproject.toml +1 -1
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/LICENSE +0 -0
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/NOTICE +0 -0
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/__init__.py +0 -0
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/common.py +0 -0
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/guide.py +0 -0
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/ontology.py +0 -0
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/ontology_design_guide.md +0 -0
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/projects.py +0 -0
- {cassis_cli-3.3.0 → cassis_cli-3.5.0}/cassis_cli/schema_plan.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cassis-cli
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.5.0
|
|
4
4
|
Summary: Validate, test and evaluate your Cassis context from your terminal, then publish it
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -33,7 +33,8 @@ Validate, test and evaluate your context from your terminal, then publish it. Th
|
|
|
33
33
|
- The CLI identifies itself to the API (`User-Agent: cassis-cli/<version>`), and successful API responses advertise the newest published version. When you are behind, commands print a one-line upgrade notice on stderr (purely informational; output and exit codes are unchanged).
|
|
34
34
|
- `cassis eval run` runs the project's eval suite against your local context files (scored in-memory, nothing is pushed to Cassis) and prints per-question results, so you can test the changes on your git branch before merging.
|
|
35
35
|
- `cassis context test` runs individual questions through the text-to-SQL agent using your local context files, so you can check that a change actually works (e.g. a new column gets picked), where `eval run` only checks for regressions on existing eval cases.
|
|
36
|
-
- `cassis eval add-case` adds a gold question/SQL case to the project's eval suite: after fixing a context issue, add the question users were failing on so `eval run` guards it from regressing.
|
|
36
|
+
- `cassis eval add-case` adds a gold question/SQL case to the project's eval suite: after fixing a context issue, add the question users were failing on so `eval run` guards it from regressing. Cassis then runs the new case three times in the background, and `eval list-cases` flags it if the attempts fail or disagree.
|
|
37
|
+
- `cassis eval run --repeat N` runs every case N times (2 to 5) on the same context and model and compares the attempts per case: which cases flipped, and whether a flip came from the agent writing different SQL or the judge ruling differently on the same SQL. `cassis eval compare <run-id>` prints that comparison again. A repeat run measures noise: it opens no issue, is not the run shown before publishing, and exits 1 only when a case failed every attempt.
|
|
37
38
|
- `cassis eval list-cases`, `cassis eval history` and `cassis eval delete-case` maintain the suite: list the current cases with their ids (flagging those whose gold SQL reads tables the context lacks), see how one case has behaved across runs (pass rate, whether repeated runs under one configuration disagree, divergences and timings), and prune one that is stale or wrong (e.g. its gold SQL encodes a definition the context has since changed).
|
|
38
39
|
- `cassis schema plan <ddl>` (or `--warehouse` on a project connected to a warehouse) previews what a schema update would change before anything is applied: the source schema diff, the context changes Cassis will make (every change on a table placed in the context, with everything a drop takes with it) and warnings, terraform-style. `cassis schema apply <ddl>` (or `--plan <id>`) writes the resulting context files into the local checkout, app untouched, for review with `git diff`. `cassis schema push <ddl> [--publish]` pushes the new schema and the local context to the app (`--yes` in CI). The file speaks only for the schemas it contains: pass `--complete` when it is the project's complete source schema so schemas absent from it are treated as dropped. With `--warehouse` the server introspects the connected warehouse instead of parsing a file; the plan is always whole-source. `cassis schema plan <ddl> --dry-run` is the prepare-ahead variant: the plan is computed synchronously and nothing is kept in Cassis (no plan to apply or resume, the current plan untouched), so the schema snapshot for a change still in a PR can be planned against safely; `--write-checkout` writes the context files it would produce into the checkout, to commit alongside the schema change.
|
|
39
40
|
- `cassis projects list` lists the projects your API key can reach: id (what `--project` and `CASSIS_PROJECT_ID` take), name, published context version, and data-source dialect. A pipeline or agent can discover the project id from the terminal instead of fishing it out of a webapp URL.
|
|
@@ -141,6 +142,10 @@ cassis eval list-cases --project ...
|
|
|
141
142
|
cassis eval history 019f0000-0000-7000-8000-0000000000ca --project ...
|
|
142
143
|
cassis eval delete-case 019f0000-0000-7000-8000-0000000000ca --project ...
|
|
143
144
|
|
|
145
|
+
# Run the suite 3 times to see which cases flip, and why; print it again later:
|
|
146
|
+
cassis eval run --repeat 3
|
|
147
|
+
cassis eval compare 019f0000-0000-7000-8000-00000000aaaa --project ...
|
|
148
|
+
|
|
144
149
|
# Pull the source schema into <base-path>/.schema.json (gitignored local snapshot):
|
|
145
150
|
cassis schema pull
|
|
146
151
|
|
|
@@ -203,7 +208,7 @@ Configuration (flags take precedence over env vars):
|
|
|
203
208
|
| `--api-key` | `CASSIS_API_KEY` | none (required) |
|
|
204
209
|
| `--api-url` | `CASSIS_API_URL` | `https://app.getcassis.com` |
|
|
205
210
|
| `--base-path` | `CASSIS_BASE_PATH` | `cassis`, must match the project's git-sync "Path" setting |
|
|
206
|
-
| `--project` (check, pull, upload, schema pull, eval run, eval add-case, eval list-cases, eval history, eval delete-case, test, issues, dbt push, dbt check) | `CASSIS_PROJECT_ID` | the id in `<base-path>/project.yml` (required before the first pull; `check` alone falls back to the project-less validation when unbound) |
|
|
211
|
+
| `--project` (check, pull, upload, schema pull, eval run, eval add-case, eval list-cases, eval history, eval compare, eval delete-case, test, issues, dbt push, dbt check) | `CASSIS_PROJECT_ID` | the id in `<base-path>/project.yml` (required before the first pull; `check` alone falls back to the project-less validation when unbound) |
|
|
207
212
|
|
|
208
213
|
`cassis eval run` also accepts `--case <id>` (repeatable; run only the named
|
|
209
214
|
cases, ids from `eval list-cases` or `add-case`), `--label` (run label in the Evals page; defaults
|
|
@@ -343,3 +348,47 @@ requires an account.
|
|
|
343
348
|
|
|
344
349
|
Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).
|
|
345
350
|
|
|
351
|
+
|
|
352
|
+
## Read conversations and their analyses
|
|
353
|
+
|
|
354
|
+
These commands read saved conversations and reports. They never start an analysis,
|
|
355
|
+
retry a failed run, execute SQL or make a model call. Use the same API key and
|
|
356
|
+
project selection as other commands (`CASSIS_API_KEY`, `--project`, or the
|
|
357
|
+
checkout's `cassis/project.yml`). Requires a server with the conversation API.
|
|
358
|
+
|
|
359
|
+
```bash
|
|
360
|
+
cassis chats list --limit 20
|
|
361
|
+
cassis chats list --review needs-attention --feedback down --since 2026-11-01
|
|
362
|
+
cassis chats list --search revenue --offset 20
|
|
363
|
+
cassis chats show <chat-id>
|
|
364
|
+
cassis chats show <chat-id> --analysis-only
|
|
365
|
+
cassis chats show <chat-id> --trace
|
|
366
|
+
cassis chats show <chat-id> --trace-entry trace:<message-id>:<index>
|
|
367
|
+
cassis chats show <chat-id> --json
|
|
368
|
+
```
|
|
369
|
+
|
|
370
|
+
`list` sorts by latest message, returns up to 20 chats by default (maximum 100
|
|
371
|
+
per page), and reports the total and next offset. Filters combine with AND.
|
|
372
|
+
`--feedback` accepts `up`, `down`, `any`, or `none`; a conversation with both
|
|
373
|
+
positive and negative feedback matches both `up` and `down`. `--review
|
|
374
|
+
needs-attention` selects completed latest reports containing a potential problem,
|
|
375
|
+
not improvement suggestions alone. `--since` includes conversations created or
|
|
376
|
+
with a message posted at or after the given ISO date or timestamp; dates and
|
|
377
|
+
naive timestamps use UTC.
|
|
378
|
+
|
|
379
|
+
`show` prints messages, generated SQL, feedback, then the saved analysis with its
|
|
380
|
+
citations and next steps. Traces are compact by default; `--trace` adds previews
|
|
381
|
+
limited to 500 characters per entry, and `--trace-entry` opens one entry in full.
|
|
382
|
+
The entry identifiers match the analysis's trace references. `--json` returns the
|
|
383
|
+
full messages and traces without those display limits; `--analysis-only --json`
|
|
384
|
+
returns the analysis view. If a retry failed, the previous completed report is
|
|
385
|
+
shown with the failed status. Missing analyses are reported explicitly.
|
|
386
|
+
|
|
387
|
+
Members can read their own chats; administrators can read chats in accessible
|
|
388
|
+
projects. API keys scoped to a project cannot reach another one. `--app-url`
|
|
389
|
+
sets the base URL for browser links when the webapp and API use different hosts
|
|
390
|
+
or ports. JSON list responses include `url` on each item and `next_offset`.
|
|
391
|
+
Exit codes: 0 on a successful read (even with no report or an empty list), 1 for
|
|
392
|
+
an inaccessible/missing chat or trace entry, 2 for invalid arguments, 3 for API
|
|
393
|
+
or authentication errors.
|
|
394
|
+
|
|
@@ -11,7 +11,8 @@ Validate, test and evaluate your context from your terminal, then publish it. Th
|
|
|
11
11
|
- The CLI identifies itself to the API (`User-Agent: cassis-cli/<version>`), and successful API responses advertise the newest published version. When you are behind, commands print a one-line upgrade notice on stderr (purely informational; output and exit codes are unchanged).
|
|
12
12
|
- `cassis eval run` runs the project's eval suite against your local context files (scored in-memory, nothing is pushed to Cassis) and prints per-question results, so you can test the changes on your git branch before merging.
|
|
13
13
|
- `cassis context test` runs individual questions through the text-to-SQL agent using your local context files, so you can check that a change actually works (e.g. a new column gets picked), where `eval run` only checks for regressions on existing eval cases.
|
|
14
|
-
- `cassis eval add-case` adds a gold question/SQL case to the project's eval suite: after fixing a context issue, add the question users were failing on so `eval run` guards it from regressing.
|
|
14
|
+
- `cassis eval add-case` adds a gold question/SQL case to the project's eval suite: after fixing a context issue, add the question users were failing on so `eval run` guards it from regressing. Cassis then runs the new case three times in the background, and `eval list-cases` flags it if the attempts fail or disagree.
|
|
15
|
+
- `cassis eval run --repeat N` runs every case N times (2 to 5) on the same context and model and compares the attempts per case: which cases flipped, and whether a flip came from the agent writing different SQL or the judge ruling differently on the same SQL. `cassis eval compare <run-id>` prints that comparison again. A repeat run measures noise: it opens no issue, is not the run shown before publishing, and exits 1 only when a case failed every attempt.
|
|
15
16
|
- `cassis eval list-cases`, `cassis eval history` and `cassis eval delete-case` maintain the suite: list the current cases with their ids (flagging those whose gold SQL reads tables the context lacks), see how one case has behaved across runs (pass rate, whether repeated runs under one configuration disagree, divergences and timings), and prune one that is stale or wrong (e.g. its gold SQL encodes a definition the context has since changed).
|
|
16
17
|
- `cassis schema plan <ddl>` (or `--warehouse` on a project connected to a warehouse) previews what a schema update would change before anything is applied: the source schema diff, the context changes Cassis will make (every change on a table placed in the context, with everything a drop takes with it) and warnings, terraform-style. `cassis schema apply <ddl>` (or `--plan <id>`) writes the resulting context files into the local checkout, app untouched, for review with `git diff`. `cassis schema push <ddl> [--publish]` pushes the new schema and the local context to the app (`--yes` in CI). The file speaks only for the schemas it contains: pass `--complete` when it is the project's complete source schema so schemas absent from it are treated as dropped. With `--warehouse` the server introspects the connected warehouse instead of parsing a file; the plan is always whole-source. `cassis schema plan <ddl> --dry-run` is the prepare-ahead variant: the plan is computed synchronously and nothing is kept in Cassis (no plan to apply or resume, the current plan untouched), so the schema snapshot for a change still in a PR can be planned against safely; `--write-checkout` writes the context files it would produce into the checkout, to commit alongside the schema change.
|
|
17
18
|
- `cassis projects list` lists the projects your API key can reach: id (what `--project` and `CASSIS_PROJECT_ID` take), name, published context version, and data-source dialect. A pipeline or agent can discover the project id from the terminal instead of fishing it out of a webapp URL.
|
|
@@ -119,6 +120,10 @@ cassis eval list-cases --project ...
|
|
|
119
120
|
cassis eval history 019f0000-0000-7000-8000-0000000000ca --project ...
|
|
120
121
|
cassis eval delete-case 019f0000-0000-7000-8000-0000000000ca --project ...
|
|
121
122
|
|
|
123
|
+
# Run the suite 3 times to see which cases flip, and why; print it again later:
|
|
124
|
+
cassis eval run --repeat 3
|
|
125
|
+
cassis eval compare 019f0000-0000-7000-8000-00000000aaaa --project ...
|
|
126
|
+
|
|
122
127
|
# Pull the source schema into <base-path>/.schema.json (gitignored local snapshot):
|
|
123
128
|
cassis schema pull
|
|
124
129
|
|
|
@@ -181,7 +186,7 @@ Configuration (flags take precedence over env vars):
|
|
|
181
186
|
| `--api-key` | `CASSIS_API_KEY` | none (required) |
|
|
182
187
|
| `--api-url` | `CASSIS_API_URL` | `https://app.getcassis.com` |
|
|
183
188
|
| `--base-path` | `CASSIS_BASE_PATH` | `cassis`, must match the project's git-sync "Path" setting |
|
|
184
|
-
| `--project` (check, pull, upload, schema pull, eval run, eval add-case, eval list-cases, eval history, eval delete-case, test, issues, dbt push, dbt check) | `CASSIS_PROJECT_ID` | the id in `<base-path>/project.yml` (required before the first pull; `check` alone falls back to the project-less validation when unbound) |
|
|
189
|
+
| `--project` (check, pull, upload, schema pull, eval run, eval add-case, eval list-cases, eval history, eval compare, eval delete-case, test, issues, dbt push, dbt check) | `CASSIS_PROJECT_ID` | the id in `<base-path>/project.yml` (required before the first pull; `check` alone falls back to the project-less validation when unbound) |
|
|
185
190
|
|
|
186
191
|
`cassis eval run` also accepts `--case <id>` (repeatable; run only the named
|
|
187
192
|
cases, ids from `eval list-cases` or `add-case`), `--label` (run label in the Evals page; defaults
|
|
@@ -320,3 +325,47 @@ requires an account.
|
|
|
320
325
|
## License
|
|
321
326
|
|
|
322
327
|
Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
## Read conversations and their analyses
|
|
331
|
+
|
|
332
|
+
These commands read saved conversations and reports. They never start an analysis,
|
|
333
|
+
retry a failed run, execute SQL or make a model call. Use the same API key and
|
|
334
|
+
project selection as other commands (`CASSIS_API_KEY`, `--project`, or the
|
|
335
|
+
checkout's `cassis/project.yml`). Requires a server with the conversation API.
|
|
336
|
+
|
|
337
|
+
```bash
|
|
338
|
+
cassis chats list --limit 20
|
|
339
|
+
cassis chats list --review needs-attention --feedback down --since 2026-11-01
|
|
340
|
+
cassis chats list --search revenue --offset 20
|
|
341
|
+
cassis chats show <chat-id>
|
|
342
|
+
cassis chats show <chat-id> --analysis-only
|
|
343
|
+
cassis chats show <chat-id> --trace
|
|
344
|
+
cassis chats show <chat-id> --trace-entry trace:<message-id>:<index>
|
|
345
|
+
cassis chats show <chat-id> --json
|
|
346
|
+
```
|
|
347
|
+
|
|
348
|
+
`list` sorts by latest message, returns up to 20 chats by default (maximum 100
|
|
349
|
+
per page), and reports the total and next offset. Filters combine with AND.
|
|
350
|
+
`--feedback` accepts `up`, `down`, `any`, or `none`; a conversation with both
|
|
351
|
+
positive and negative feedback matches both `up` and `down`. `--review
|
|
352
|
+
needs-attention` selects completed latest reports containing a potential problem,
|
|
353
|
+
not improvement suggestions alone. `--since` includes conversations created or
|
|
354
|
+
with a message posted at or after the given ISO date or timestamp; dates and
|
|
355
|
+
naive timestamps use UTC.
|
|
356
|
+
|
|
357
|
+
`show` prints messages, generated SQL, feedback, then the saved analysis with its
|
|
358
|
+
citations and next steps. Traces are compact by default; `--trace` adds previews
|
|
359
|
+
limited to 500 characters per entry, and `--trace-entry` opens one entry in full.
|
|
360
|
+
The entry identifiers match the analysis's trace references. `--json` returns the
|
|
361
|
+
full messages and traces without those display limits; `--analysis-only --json`
|
|
362
|
+
returns the analysis view. If a retry failed, the previous completed report is
|
|
363
|
+
shown with the failed status. Missing analyses are reported explicitly.
|
|
364
|
+
|
|
365
|
+
Members can read their own chats; administrators can read chats in accessible
|
|
366
|
+
projects. API keys scoped to a project cannot reach another one. `--app-url`
|
|
367
|
+
sets the base URL for browser links when the webapp and API use different hosts
|
|
368
|
+
or ports. JSON list responses include `url` on each item and `next_offset`.
|
|
369
|
+
Exit codes: 0 on a successful read (even with no report or an empty list), 1 for
|
|
370
|
+
an inaccessible/missing chat or trace entry, 2 for invalid arguments, 3 for API
|
|
371
|
+
or authentication errors.
|
|
@@ -594,6 +594,7 @@ def post_eval_run_start(
|
|
|
594
594
|
branch: Optional[str] = None,
|
|
595
595
|
label: Optional[str] = None,
|
|
596
596
|
case_ids: Optional[list[str]] = None,
|
|
597
|
+
repeat_count: Optional[int] = None,
|
|
597
598
|
transport: Optional[httpx.BaseTransport] = None,
|
|
598
599
|
) -> dict[str, Any]:
|
|
599
600
|
"""POST to /api/ci/projects/{project_id}/eval/runs and return the run record."""
|
|
@@ -607,6 +608,8 @@ def post_eval_run_start(
|
|
|
607
608
|
body["label"] = label
|
|
608
609
|
if case_ids is not None:
|
|
609
610
|
body["test_case_ids"] = case_ids
|
|
611
|
+
if repeat_count is not None:
|
|
612
|
+
body["repeat_count"] = repeat_count
|
|
610
613
|
try:
|
|
611
614
|
# May carry the whole ontology tree in `files` — same budget as the
|
|
612
615
|
# other whole-tree endpoints.
|
|
@@ -833,6 +836,41 @@ def get_eval_run_results(
|
|
|
833
836
|
return result
|
|
834
837
|
|
|
835
838
|
|
|
839
|
+
class EvalCompareUnsupportedError(ApiError):
|
|
840
|
+
"""The server predates run comparisons."""
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
def get_eval_run_comparison(
|
|
844
|
+
*,
|
|
845
|
+
api_url: str,
|
|
846
|
+
api_key: str,
|
|
847
|
+
project_id: str,
|
|
848
|
+
run_id: str,
|
|
849
|
+
transport: Optional[httpx.BaseTransport] = None,
|
|
850
|
+
) -> dict[str, Any]:
|
|
851
|
+
"""GET /api/ci/projects/{project_id}/eval/runs/{run_id}/comparison: the run's attempts compared per case."""
|
|
852
|
+
url = api_url.rstrip("/") + f"/api/ci/projects/{project_id}/eval/runs/{run_id}/comparison"
|
|
853
|
+
try:
|
|
854
|
+
with _client(transport=transport) as client:
|
|
855
|
+
response = client.get(url, headers={"Authorization": f"Bearer {api_key}"})
|
|
856
|
+
except httpx.HTTPError as exc:
|
|
857
|
+
raise ApiError(f"Could not reach the Cassis API at {url}: {exc}") from exc
|
|
858
|
+
if response.status_code == 401:
|
|
859
|
+
raise AuthError("The Cassis API rejected the API key (invalid or expired).")
|
|
860
|
+
# A server without the route answers the framework's own "Not Found",
|
|
861
|
+
# distinct from the "Eval run not found" a real lookup returns.
|
|
862
|
+
if response.status_code == 404 and _detail_or_text(response) == "Not Found":
|
|
863
|
+
raise EvalCompareUnsupportedError("This Cassis server does not compare runs yet; it needs a newer server.")
|
|
864
|
+
if response.status_code in (403, 404):
|
|
865
|
+
raise _project_scope_error(response)
|
|
866
|
+
if response.status_code >= 400:
|
|
867
|
+
raise ApiError(f"Cassis API returned HTTP {response.status_code}: {response.text[:500]}")
|
|
868
|
+
result = _parse_json_response(response, url)
|
|
869
|
+
if not isinstance(result, dict) or not all(key in result for key in ("run_id", "summary", "cases")):
|
|
870
|
+
raise ApiError(f"Unexpected response shape from the Cassis API at {url}.")
|
|
871
|
+
return result
|
|
872
|
+
|
|
873
|
+
|
|
836
874
|
def post_eval_run_cancel(
|
|
837
875
|
*,
|
|
838
876
|
api_url: str,
|
|
@@ -1290,3 +1328,56 @@ def get_dbt_observation(
|
|
|
1290
1328
|
except httpx.HTTPError as exc:
|
|
1291
1329
|
raise ApiError(f"Could not reach the Cassis API at {url}: {exc}") from exc
|
|
1292
1330
|
return _dbt_observation_response(response, url)
|
|
1331
|
+
|
|
1332
|
+
|
|
1333
|
+
class ChatNotFoundError(ApiError):
|
|
1334
|
+
"""The conversation does not exist or is not readable by this key."""
|
|
1335
|
+
|
|
1336
|
+
|
|
1337
|
+
def get_chats(
|
|
1338
|
+
*,
|
|
1339
|
+
api_url: str,
|
|
1340
|
+
api_key: str,
|
|
1341
|
+
project_id: str,
|
|
1342
|
+
chat_id: Optional[str] = None,
|
|
1343
|
+
params: Optional[dict[str, Any]] = None,
|
|
1344
|
+
transport: Optional[httpx.BaseTransport] = None,
|
|
1345
|
+
) -> dict[str, Any]:
|
|
1346
|
+
"""Read a conversation page or a conversation with its saved analysis."""
|
|
1347
|
+
url = api_url.rstrip("/") + f"/api/ci/projects/{project_id}/chats"
|
|
1348
|
+
if chat_id:
|
|
1349
|
+
url += f"/{chat_id}"
|
|
1350
|
+
try:
|
|
1351
|
+
with _client(transport=transport) as client:
|
|
1352
|
+
response = client.get(url, params=params, headers={"Authorization": f"Bearer {api_key}"})
|
|
1353
|
+
except httpx.HTTPError as exc:
|
|
1354
|
+
raise ApiError(f"Could not reach the Cassis API at {url}: {exc}") from exc
|
|
1355
|
+
if response.status_code == 401:
|
|
1356
|
+
raise AuthError("The Cassis API rejected the API key (invalid or expired).")
|
|
1357
|
+
if response.status_code == 404 and chat_id and _detail_or_text(response) == "Chat not found":
|
|
1358
|
+
raise ChatNotFoundError("Conversation not found or not accessible to this API key.")
|
|
1359
|
+
if response.status_code in (403, 404):
|
|
1360
|
+
raise ApiError(
|
|
1361
|
+
f"Conversation access denied (HTTP {response.status_code}). Check --project and the API key’s read access and project scope."
|
|
1362
|
+
)
|
|
1363
|
+
if response.status_code >= 400:
|
|
1364
|
+
raise ApiError(f"Cassis API returned HTTP {response.status_code}: {response.text[:500]}")
|
|
1365
|
+
data = _parse_json_response(response, url)
|
|
1366
|
+
valid = isinstance(data, dict)
|
|
1367
|
+
if valid and chat_id:
|
|
1368
|
+
valid = (
|
|
1369
|
+
isinstance(data.get("chat"), dict)
|
|
1370
|
+
and "id" in data["chat"]
|
|
1371
|
+
and isinstance(data.get("messages"), list)
|
|
1372
|
+
and isinstance(data.get("analysis"), dict)
|
|
1373
|
+
)
|
|
1374
|
+
elif valid:
|
|
1375
|
+
valid = (
|
|
1376
|
+
isinstance(data.get("items"), list)
|
|
1377
|
+
and isinstance(data.get("total"), int)
|
|
1378
|
+
and isinstance(data.get("offset"), int)
|
|
1379
|
+
and all(isinstance(row, dict) and "id" in row for row in data["items"])
|
|
1380
|
+
)
|
|
1381
|
+
if not valid:
|
|
1382
|
+
raise ApiError(f"Unexpected response shape from the Cassis API at {url}.")
|
|
1383
|
+
return data
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Read conversations and saved analyses without running model calls."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Optional
|
|
9
|
+
from urllib.parse import urlencode
|
|
10
|
+
from uuid import UUID
|
|
11
|
+
|
|
12
|
+
import typer
|
|
13
|
+
from cassis_cli.api import DEFAULT_API_URL, ApiError, ChatNotFoundError, get_chats
|
|
14
|
+
from cassis_cli.common import api_failure, one_line, require_api_key, resolve_project_id
|
|
15
|
+
|
|
16
|
+
app = typer.Typer(no_args_is_help=True, help="Read conversations and their saved analyses.")
|
|
17
|
+
_PATH = typer.Option(Path("."), "--path", help="Checkout root containing the context directory.")
|
|
18
|
+
_PROJECT = typer.Option(None, "--project", envvar="CASSIS_PROJECT_ID", help="Project ID; defaults to project.yml.")
|
|
19
|
+
_KEY = typer.Option(None, "--api-key", envvar="CASSIS_API_KEY", help="Cassis API key.")
|
|
20
|
+
_URL = typer.Option(DEFAULT_API_URL, "--api-url", envvar="CASSIS_API_URL", help="Cassis API base URL.")
|
|
21
|
+
_BASE = typer.Option("cassis", "--base-path", envvar="CASSIS_BASE_PATH", help="Context directory within the checkout.")
|
|
22
|
+
_APP_URL = typer.Option(None, "--app-url", help="Webapp base URL for conversation links; defaults to --api-url.")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _url(base: str, project: str, chat: str) -> str:
|
|
26
|
+
return base.rstrip("/") + "/chat?" + urlencode({"projectId": project, "chatId": chat})
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _status(summary: Optional[dict[str, Any]]) -> str:
|
|
30
|
+
if not summary:
|
|
31
|
+
return "Not analyzed"
|
|
32
|
+
if summary.get("status") in ("running", "scheduled", "deferred"):
|
|
33
|
+
return "In progress"
|
|
34
|
+
if summary.get("status") == "completed" and summary.get("problems", 0):
|
|
35
|
+
return "Needs attention"
|
|
36
|
+
if summary.get("status") != "completed" or summary.get("conversation_changed"):
|
|
37
|
+
return "Not analyzed"
|
|
38
|
+
return "Analyzed"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _read(**kwargs: Any) -> dict[str, Any]:
|
|
42
|
+
try:
|
|
43
|
+
return get_chats(**kwargs)
|
|
44
|
+
except ChatNotFoundError as exc:
|
|
45
|
+
typer.echo(str(exc), err=True)
|
|
46
|
+
raise typer.Exit(1) from exc
|
|
47
|
+
except ApiError as exc:
|
|
48
|
+
raise api_failure(exc) from exc
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@app.command("list")
|
|
52
|
+
def list_chats(
|
|
53
|
+
path: Path = _PATH,
|
|
54
|
+
project_id: Optional[str] = _PROJECT,
|
|
55
|
+
api_key: Optional[str] = _KEY,
|
|
56
|
+
api_url: str = _URL,
|
|
57
|
+
base_path: str = _BASE,
|
|
58
|
+
app_url: Optional[str] = _APP_URL,
|
|
59
|
+
limit: int = typer.Option(20, min=1, max=100),
|
|
60
|
+
offset: int = typer.Option(0, min=0),
|
|
61
|
+
since: Optional[str] = typer.Option(None, help="Last-message cutoff: ISO date or timestamp; dates use UTC."),
|
|
62
|
+
feedback: Optional[str] = typer.Option(None, help="Filter: up, down, any or none."),
|
|
63
|
+
review: Optional[str] = typer.Option(None, help="Filter: needs-attention (detected problems only)."),
|
|
64
|
+
query: Optional[str] = typer.Option(None, "--search", help="Search title, author name or email."),
|
|
65
|
+
json_output: bool = typer.Option(False, "--json", help="Print the page as JSON, including links and pagination."),
|
|
66
|
+
) -> None:
|
|
67
|
+
"""List readable chats by latest message. Reads saved results; never starts an analysis."""
|
|
68
|
+
if feedback not in (None, "up", "down", "any", "none"):
|
|
69
|
+
raise typer.BadParameter("Choose up, down, any or none.", param_hint="--feedback")
|
|
70
|
+
if review not in (None, "needs-attention"):
|
|
71
|
+
raise typer.BadParameter("Choose needs-attention.", param_hint="--review")
|
|
72
|
+
if since:
|
|
73
|
+
try:
|
|
74
|
+
dt = datetime.fromisoformat(since.replace("Z", "+00:00"))
|
|
75
|
+
since = (dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)).isoformat()
|
|
76
|
+
except ValueError as exc:
|
|
77
|
+
raise typer.BadParameter("Use an ISO date or timestamp.", param_hint="--since") from exc
|
|
78
|
+
key = require_api_key(api_key)
|
|
79
|
+
project = resolve_project_id(project_id, path / base_path, quiet=json_output)
|
|
80
|
+
assert project is not None
|
|
81
|
+
params = {
|
|
82
|
+
k: v
|
|
83
|
+
for k, v in dict(limit=limit, offset=offset, since=since, feedback=feedback, review=review, q=query).items()
|
|
84
|
+
if v is not None
|
|
85
|
+
}
|
|
86
|
+
data = _read(api_url=api_url, api_key=key, project_id=project, params=params)
|
|
87
|
+
for row in data["items"]:
|
|
88
|
+
row["url"] = _url(app_url or api_url, project, row["id"])
|
|
89
|
+
data["next_offset"] = (
|
|
90
|
+
data["offset"] + len(data["items"])
|
|
91
|
+
if data["items"] and data["offset"] + len(data["items"]) < data["total"]
|
|
92
|
+
else None
|
|
93
|
+
)
|
|
94
|
+
if json_output:
|
|
95
|
+
typer.echo(json.dumps(data, indent=2, ensure_ascii=False))
|
|
96
|
+
return
|
|
97
|
+
if not data["items"]:
|
|
98
|
+
typer.echo("No conversations match these filters.")
|
|
99
|
+
for row in data["items"]:
|
|
100
|
+
feedback_text = f"up {row.get('feedback_up', 0)} / down {row.get('feedback_down', 0)}"
|
|
101
|
+
typer.echo(
|
|
102
|
+
f"{row['id']} {_status(row.get('analysis_summary'))} {feedback_text} {row.get('last_message_at') or row.get('created_at')} {one_line(row.get('title') or 'Untitled')}"
|
|
103
|
+
)
|
|
104
|
+
typer.echo(f" {row['url']}")
|
|
105
|
+
typer.echo(f"{len(data['items'])} of {data['total']} conversations (offset {data['offset']}).")
|
|
106
|
+
if data["next_offset"] is not None:
|
|
107
|
+
typer.echo(f"Next page: repeat with --offset {data['next_offset']}")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _analysis(view: dict[str, Any]) -> None:
|
|
111
|
+
typer.echo("\nAnalysis")
|
|
112
|
+
latest = view.get("analysis")
|
|
113
|
+
if latest:
|
|
114
|
+
typer.echo(f"Status: {latest['status']} | Launched: {latest.get('created_at')}")
|
|
115
|
+
if latest.get("error"):
|
|
116
|
+
typer.echo(f"Error: {latest['error']}")
|
|
117
|
+
shown = latest if latest and latest.get("status") == "completed" else view.get("previous_analysis")
|
|
118
|
+
if not shown or not shown.get("result"):
|
|
119
|
+
typer.echo(
|
|
120
|
+
"No completed analysis available." if view.get("enabled") else "Analysis is not enabled for this project."
|
|
121
|
+
)
|
|
122
|
+
return
|
|
123
|
+
if shown is not latest:
|
|
124
|
+
typer.echo(f"Previous completed analysis ({shown['id']}):")
|
|
125
|
+
if shown.get("conversation_changed"):
|
|
126
|
+
typer.echo("New messages are not covered by this analysis.")
|
|
127
|
+
if shown.get("context_changed"):
|
|
128
|
+
typer.echo("Context has changed since this analysis.")
|
|
129
|
+
report = shown["result"]
|
|
130
|
+
typer.echo(report["summary"])
|
|
131
|
+
for index, obs in enumerate(report["observations"], 1):
|
|
132
|
+
typer.echo(f"\n{index}. {obs['title']} [{obs['kind']} / {obs['confidence']}]")
|
|
133
|
+
typer.echo(obs["explanation"])
|
|
134
|
+
typer.echo(f"Next step ({obs['destination']}): {obs['next_step']}")
|
|
135
|
+
for citation in obs["evidence"]:
|
|
136
|
+
typer.echo(f" [{citation['document_id']}] {citation['quote']}")
|
|
137
|
+
for issue in view.get("issues", []):
|
|
138
|
+
typer.echo(f"Linked issue: {issue['id']} — {issue['title']} ({issue['status']})")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@app.command("show")
|
|
142
|
+
def show_chat(
|
|
143
|
+
chat_id: UUID,
|
|
144
|
+
path: Path = _PATH,
|
|
145
|
+
project_id: Optional[str] = _PROJECT,
|
|
146
|
+
api_key: Optional[str] = _KEY,
|
|
147
|
+
api_url: str = _URL,
|
|
148
|
+
base_path: str = _BASE,
|
|
149
|
+
app_url: Optional[str] = _APP_URL,
|
|
150
|
+
trace: bool = typer.Option(False, "--trace", help="Show bounded previews of execution trace entries."),
|
|
151
|
+
trace_entry: Optional[str] = typer.Option(
|
|
152
|
+
None, "--trace-entry", help="Show one full entry: trace:<message-id>:<index>."
|
|
153
|
+
),
|
|
154
|
+
analysis_only: bool = typer.Option(False, "--analysis-only", help="Show only the saved analysis."),
|
|
155
|
+
json_output: bool = typer.Option(False, "--json", help="Print the full conversation, traces and analysis as JSON."),
|
|
156
|
+
) -> None:
|
|
157
|
+
"""Read a conversation followed by its saved analysis. No model calls or automatic retries."""
|
|
158
|
+
if analysis_only and (trace or trace_entry):
|
|
159
|
+
raise typer.BadParameter("--analysis-only cannot be combined with trace options.")
|
|
160
|
+
key = require_api_key(api_key)
|
|
161
|
+
project = resolve_project_id(project_id, path / base_path, quiet=json_output)
|
|
162
|
+
assert project is not None
|
|
163
|
+
data = _read(api_url=api_url, api_key=key, project_id=project, chat_id=str(chat_id))
|
|
164
|
+
data["url"] = _url(app_url or api_url, project, str(chat_id))
|
|
165
|
+
if trace_entry:
|
|
166
|
+
for message in data["messages"]:
|
|
167
|
+
for i, entry in enumerate(message.get("process_log") or []):
|
|
168
|
+
if trace_entry == f"trace:{message['id']}:{i}":
|
|
169
|
+
typer.echo(json.dumps(entry, indent=2, ensure_ascii=False))
|
|
170
|
+
return
|
|
171
|
+
typer.echo("Trace entry not found in this conversation.", err=True)
|
|
172
|
+
raise typer.Exit(1)
|
|
173
|
+
if json_output:
|
|
174
|
+
typer.echo(json.dumps(data["analysis"] if analysis_only else data, indent=2, ensure_ascii=False))
|
|
175
|
+
return
|
|
176
|
+
chat = data["chat"]
|
|
177
|
+
typer.echo(f"{chat.get('title') or 'Untitled'} [{chat['id']}]")
|
|
178
|
+
typer.echo(f"Created: {chat.get('created_at')} | Updated: {chat.get('updated_at')}")
|
|
179
|
+
typer.echo(data["url"])
|
|
180
|
+
if not analysis_only:
|
|
181
|
+
for message in data["messages"]:
|
|
182
|
+
typer.echo(f"\n{message['role']} — {message.get('created_at')} [message:{message['id']}]")
|
|
183
|
+
if message.get("content"):
|
|
184
|
+
typer.echo(message["content"])
|
|
185
|
+
if message.get("generated_sql"):
|
|
186
|
+
typer.echo(f"SQL:\n{message['generated_sql']}")
|
|
187
|
+
if message.get("rating"):
|
|
188
|
+
# A feedback message: its content, printed above, is the justification.
|
|
189
|
+
typer.echo(f"Feedback: {message['rating']}")
|
|
190
|
+
if message.get("error"):
|
|
191
|
+
typer.echo(f"Error: {message['error']}")
|
|
192
|
+
entries = message.get("process_log") or []
|
|
193
|
+
if entries and not trace:
|
|
194
|
+
typer.echo(f"Execution: {len(entries)} trace entries (use --trace to inspect).")
|
|
195
|
+
if trace:
|
|
196
|
+
for i, entry in enumerate(entries):
|
|
197
|
+
ref = f"trace:{message['id']}:{i}"
|
|
198
|
+
typer.echo(f" [{ref}] {entry.get('name') or entry.get('type')}")
|
|
199
|
+
detail = json.dumps(entry, ensure_ascii=False)
|
|
200
|
+
typer.echo(
|
|
201
|
+
f" {detail[:500]}"
|
|
202
|
+
+ (f"… (use --trace-entry {ref} for full details)" if len(detail) > 500 else "")
|
|
203
|
+
)
|
|
204
|
+
_analysis(data["analysis"])
|
|
@@ -111,7 +111,9 @@ _BASE_PATH_OPTION = typer.Option(
|
|
|
111
111
|
DEFAULT_BASE_PATH, "--base-path", envvar="CASSIS_BASE_PATH", help="Context directory holding project.yml."
|
|
112
112
|
)
|
|
113
113
|
_POLL_INTERVAL_OPTION = typer.Option(2.0, "--poll-interval", help="Seconds between polls.")
|
|
114
|
-
_TIMEOUT_OPTION = typer.Option(
|
|
114
|
+
_TIMEOUT_OPTION = typer.Option(
|
|
115
|
+
600.0, "--timeout", help="Give up waiting after this many seconds (exit 3; the upload keeps processing)."
|
|
116
|
+
)
|
|
115
117
|
_JSON_OPTION = typer.Option(False, "--json", help="Print the upload record as raw JSON (the summary goes to stderr).")
|
|
116
118
|
|
|
117
119
|
|
|
@@ -139,7 +141,9 @@ def push(
|
|
|
139
141
|
baseline, and files each part of the context the change puts in doubt as
|
|
140
142
|
an issue (impact "at risk"). Exits 0 when accepted or superseded by a
|
|
141
143
|
newer upload, 1 when rejected (invalid files, or no relation matches the
|
|
142
|
-
data source), 2 on usage errors, 3 on transport errors
|
|
144
|
+
data source), 2 on usage errors, 3 on transport errors or a timeout (the
|
|
145
|
+
upload keeps processing server-side), or when Cassis fails on its side
|
|
146
|
+
while processing it.
|
|
143
147
|
"""
|
|
144
148
|
record = _upload_and_wait(
|
|
145
149
|
"push",
|
|
@@ -189,7 +193,8 @@ def check(
|
|
|
189
193
|
nothing is filed and the baseline doesn't move. Exits 0 with or without
|
|
190
194
|
findings, 1 when the upload is rejected (invalid files, or no baseline
|
|
191
195
|
yet: run `cassis dbt push` on the production branch first), 2 on usage
|
|
192
|
-
errors, 3 on transport errors
|
|
196
|
+
errors, 3 on transport errors or a timeout (the upload keeps processing
|
|
197
|
+
server-side), or when Cassis fails on its side while processing it.
|
|
193
198
|
"""
|
|
194
199
|
record = _upload_and_wait(
|
|
195
200
|
"check",
|
|
@@ -287,7 +292,7 @@ def _upload_and_wait(
|
|
|
287
292
|
|
|
288
293
|
def on_timeout(last: "dict[str, Any]") -> None:
|
|
289
294
|
typer.secho(
|
|
290
|
-
f"Timed out after {timeout
|
|
295
|
+
f"Timed out after {timeout:g}s: upload {observation_id} is still {last.get('status', '?')}. "
|
|
291
296
|
"It keeps processing server-side.",
|
|
292
297
|
fg=typer.colors.YELLOW,
|
|
293
298
|
err=True,
|
|
@@ -25,6 +25,7 @@ from cassis_cli.api import (
|
|
|
25
25
|
get_eval_case_history,
|
|
26
26
|
get_eval_cases,
|
|
27
27
|
get_eval_run,
|
|
28
|
+
get_eval_run_comparison,
|
|
28
29
|
get_eval_run_results,
|
|
29
30
|
post_eval_case_create,
|
|
30
31
|
post_eval_run_cancel,
|
|
@@ -272,6 +273,22 @@ def _footprint_note(case: dict[str, Any]) -> Optional[str]:
|
|
|
272
273
|
return "; ".join(reasons)
|
|
273
274
|
|
|
274
275
|
|
|
276
|
+
def _entry_check_note(case: dict[str, Any]) -> Optional[str]:
|
|
277
|
+
"""One line on a case whose creation check flags it; None when stable or never checked."""
|
|
278
|
+
check = case.get("entry_check")
|
|
279
|
+
if not isinstance(check, dict):
|
|
280
|
+
return None
|
|
281
|
+
state = check.get("state")
|
|
282
|
+
score = f"{check.get('passed')}/{check.get('total')}" if check.get("total") else ""
|
|
283
|
+
if state == "flaky":
|
|
284
|
+
return f"flaky at creation ({score} attempts passed): its verdict is noise until the question or gold SQL is tightened"
|
|
285
|
+
if state == "failing":
|
|
286
|
+
return f"failing at creation ({score} attempts passed): check the gold SQL, or the context gap it exposes"
|
|
287
|
+
if state == "running":
|
|
288
|
+
return "entry check running (3 attempts on the current context)"
|
|
289
|
+
return None
|
|
290
|
+
|
|
291
|
+
|
|
275
292
|
@app.command(name="add-case")
|
|
276
293
|
def add_case(
|
|
277
294
|
path: Path = typer.Argument(
|
|
@@ -328,6 +345,8 @@ def add_case(
|
|
|
328
345
|
case is stored, so a case that cannot execute never enters the suite.
|
|
329
346
|
The SQL comes from --gold-sql (inline) or --gold-sql-file (a file path;
|
|
330
347
|
prefer it for multi-line SQL, which shell quoting mangles inline).
|
|
348
|
+
Cassis then runs the new case three times in the background, and
|
|
349
|
+
`cassis eval list-cases` flags it if the attempts fail or disagree.
|
|
331
350
|
Exits 0 on creation, 1 on a duplicate question or gold SQL that does not
|
|
332
351
|
run, 2 on usage errors, 3 on transport/API errors.
|
|
333
352
|
"""
|
|
@@ -379,6 +398,12 @@ def add_case(
|
|
|
379
398
|
fg=typer.colors.YELLOW,
|
|
380
399
|
err=True,
|
|
381
400
|
)
|
|
401
|
+
check = case.get("entry_check")
|
|
402
|
+
if isinstance(check, dict) and check.get("state") == "running":
|
|
403
|
+
typer.echo(
|
|
404
|
+
" Checking it: 3 attempts run on the current context in the background; "
|
|
405
|
+
"`cassis eval list-cases` flags it if they fail or disagree."
|
|
406
|
+
)
|
|
382
407
|
raise typer.Exit(EXIT_OK)
|
|
383
408
|
|
|
384
409
|
|
|
@@ -427,7 +452,9 @@ def list_cases(
|
|
|
427
452
|
what `cassis eval delete-case` and `cassis eval history` take. A case
|
|
428
453
|
whose gold SQL does not fully resolve against the context (a table the
|
|
429
454
|
context lacks, an ambiguous name, SQL that does not parse) is flagged
|
|
430
|
-
under its line, since such a case cannot be trusted to score a change
|
|
455
|
+
under its line, since such a case cannot be trusted to score a change,
|
|
456
|
+
and so is a case whose three-attempt check after creation failed or
|
|
457
|
+
disagreed.
|
|
431
458
|
Exits 0 on success, 2 on usage errors (including a --branch the project
|
|
432
459
|
does not have), 3 on transport/API errors.
|
|
433
460
|
"""
|
|
@@ -459,6 +486,13 @@ def list_cases(
|
|
|
459
486
|
if note:
|
|
460
487
|
flagged += 1
|
|
461
488
|
typer.secho(f" ⚠ {note}", fg=typer.colors.YELLOW)
|
|
489
|
+
check_note = _entry_check_note(case)
|
|
490
|
+
if check_note:
|
|
491
|
+
running = (case.get("entry_check") or {}).get("state") == "running"
|
|
492
|
+
typer.secho(
|
|
493
|
+
f" {'…' if running else '⚠'} {check_note}",
|
|
494
|
+
fg=None if running else typer.colors.YELLOW,
|
|
495
|
+
)
|
|
462
496
|
if flagged:
|
|
463
497
|
typer.secho(
|
|
464
498
|
f"{flagged} case(s) have a gold SQL that does not fully resolve against the context; "
|
|
@@ -469,6 +503,55 @@ def list_cases(
|
|
|
469
503
|
raise typer.Exit(EXIT_OK)
|
|
470
504
|
|
|
471
505
|
|
|
506
|
+
# How a flip reads to the person fixing it: which side of the evaluation moved.
|
|
507
|
+
_CAUSE_HINTS = {
|
|
508
|
+
"judge": "the judge ruled differently on the same SQL: check the gold SQL and the question's wording",
|
|
509
|
+
"generation": "the agent wrote different SQL from the same context: the context leaves room for both",
|
|
510
|
+
"both": "the agent wrote different SQL and the judge also wavered",
|
|
511
|
+
"unattributed": "the evidence decides neither side (a re-judge failed, or the same SQL returned different rows)",
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def _print_comparison(comparison: dict[str, Any]) -> None:
|
|
516
|
+
"""A repeat run's attempts per case: the same comparison the webapp's run page shows."""
|
|
517
|
+
summary = comparison.get("summary") or {}
|
|
518
|
+
repeat = comparison.get("repeat_count") or 1
|
|
519
|
+
cases = comparison.get("cases") or []
|
|
520
|
+
typer.secho(f"Repeat comparison: {repeat} attempts per case, {summary.get('cases', len(cases))} cases", bold=True)
|
|
521
|
+
causes = summary.get("causes") or {}
|
|
522
|
+
flipped = f"{summary.get('flipped', 0)} flipped"
|
|
523
|
+
if causes:
|
|
524
|
+
flipped += " (" + ", ".join(f"{k} {n}" for k, n in causes.items()) + ")"
|
|
525
|
+
typer.echo(
|
|
526
|
+
f" {summary.get('all_passed', 0)} passed every attempt · "
|
|
527
|
+
f"{summary.get('all_failed', 0)} failed every attempt · {flipped}"
|
|
528
|
+
)
|
|
529
|
+
if repeat < 2:
|
|
530
|
+
typer.echo(" Not a repeat run: start one with `cassis eval run --repeat N` to measure noise.")
|
|
531
|
+
return
|
|
532
|
+
shown = [c for c in cases if c.get("agreement") != "all_passed"]
|
|
533
|
+
if not shown:
|
|
534
|
+
return
|
|
535
|
+
typer.echo("")
|
|
536
|
+
for c in shown:
|
|
537
|
+
agreement = c.get("agreement")
|
|
538
|
+
icon = "~" if agreement == "flipped" else "✗"
|
|
539
|
+
color = typer.colors.YELLOW if agreement == "flipped" else typer.colors.RED
|
|
540
|
+
distinct = int(c.get("distinct_sql") or 0)
|
|
541
|
+
sqls = f"{distinct} SQL" if distinct == 1 else f"{distinct} SQLs"
|
|
542
|
+
cause = str(c.get("cause") or "")
|
|
543
|
+
question = str(c.get("question") or "").replace("\n", " ")
|
|
544
|
+
if len(question) > 60:
|
|
545
|
+
question = question[:57] + "..."
|
|
546
|
+
typer.secho(f" {icon} {c.get('passed', 0)}/{c.get('total', 0)} {cause:<13} {sqls:<8} {question}", fg=color)
|
|
547
|
+
typer.echo(f" {c.get('test_case_id')}")
|
|
548
|
+
seen = [k for k in _CAUSE_HINTS if k in causes]
|
|
549
|
+
if seen:
|
|
550
|
+
typer.echo("")
|
|
551
|
+
for k in seen:
|
|
552
|
+
typer.echo(f" {k}: {_CAUSE_HINTS[k]}")
|
|
553
|
+
|
|
554
|
+
|
|
472
555
|
def _format_when(iso: str) -> str:
|
|
473
556
|
"""`2026-10-23 18:14` from an ISO timestamp; the raw text when it does not parse."""
|
|
474
557
|
try:
|
|
@@ -633,6 +716,76 @@ def history(
|
|
|
633
716
|
raise typer.Exit(EXIT_OK)
|
|
634
717
|
|
|
635
718
|
|
|
719
|
+
@app.command()
|
|
720
|
+
def compare(
|
|
721
|
+
run_id: str = typer.Argument(
|
|
722
|
+
...,
|
|
723
|
+
help="Id of the eval run (printed by `cassis eval run`).",
|
|
724
|
+
),
|
|
725
|
+
path: Path = typer.Option(
|
|
726
|
+
Path("."),
|
|
727
|
+
"--path",
|
|
728
|
+
help="Repository checkout root (holds <base-path>/project.yml for the --project default).",
|
|
729
|
+
),
|
|
730
|
+
project_id: Optional[str] = typer.Option(
|
|
731
|
+
None,
|
|
732
|
+
"--project",
|
|
733
|
+
envvar="CASSIS_PROJECT_ID",
|
|
734
|
+
help="Target Cassis project ID (UUID). Defaults to the id in <base-path>/project.yml.",
|
|
735
|
+
),
|
|
736
|
+
api_key: Optional[str] = typer.Option(
|
|
737
|
+
None,
|
|
738
|
+
"--api-key",
|
|
739
|
+
envvar="CASSIS_API_KEY",
|
|
740
|
+
help="Cassis API key (sk-k6-...). Create one in Organization settings -> API keys.",
|
|
741
|
+
),
|
|
742
|
+
api_url: str = typer.Option(
|
|
743
|
+
DEFAULT_API_URL,
|
|
744
|
+
"--api-url",
|
|
745
|
+
envvar="CASSIS_API_URL",
|
|
746
|
+
help="Cassis API base URL.",
|
|
747
|
+
),
|
|
748
|
+
base_path: str = typer.Option(
|
|
749
|
+
DEFAULT_BASE_PATH,
|
|
750
|
+
"--base-path",
|
|
751
|
+
envvar="CASSIS_BASE_PATH",
|
|
752
|
+
help="Repository directory the context is exported under (holds project.yml for the --project default).",
|
|
753
|
+
),
|
|
754
|
+
json_output: bool = typer.Option(False, "--json", help="Print the comparison as raw JSON."),
|
|
755
|
+
) -> None:
|
|
756
|
+
"""Compare the attempts of a repeat run, case by case.
|
|
757
|
+
|
|
758
|
+
For a run started with `cassis eval run --repeat N`: how many cases passed
|
|
759
|
+
every attempt, failed every attempt or flipped, and for each flipping case
|
|
760
|
+
how many distinct SQLs the agent wrote and what moved between attempts:
|
|
761
|
+
`generation` (the agent wrote different SQL), `judge` (the judge ruled
|
|
762
|
+
differently on the same SQL), `both`, or `unattributed`. The same
|
|
763
|
+
comparison the webapp's run page shows. Exits 0 on success, 2 on usage
|
|
764
|
+
errors, 3 on transport/API errors (including a server too old to compare).
|
|
765
|
+
"""
|
|
766
|
+
api_key = require_api_key(api_key)
|
|
767
|
+
project_id = resolve_project_id(project_id, path / Path(base_path))
|
|
768
|
+
try:
|
|
769
|
+
UUID(run_id)
|
|
770
|
+
except ValueError:
|
|
771
|
+
typer.secho(f"RUN_ID must be a UUID, got {run_id!r}.", fg=typer.colors.RED, err=True)
|
|
772
|
+
raise typer.Exit(EXIT_USAGE)
|
|
773
|
+
try:
|
|
774
|
+
comparison = get_eval_run_comparison(api_url=api_url, api_key=api_key, project_id=project_id, run_id=run_id)
|
|
775
|
+
except ApiError as exc: # covers AuthError and EvalCompareUnsupportedError
|
|
776
|
+
typer.secho(str(exc), fg=typer.colors.RED, err=True)
|
|
777
|
+
raise typer.Exit(EXIT_TRANSPORT) from exc
|
|
778
|
+
|
|
779
|
+
if json_output:
|
|
780
|
+
typer.echo(json.dumps(comparison, indent=2))
|
|
781
|
+
raise typer.Exit(EXIT_OK)
|
|
782
|
+
if comparison.get("status") == "running":
|
|
783
|
+
typer.echo("The run is still in progress; its attempts are compared once it completes.")
|
|
784
|
+
raise typer.Exit(EXIT_OK)
|
|
785
|
+
_print_comparison(comparison)
|
|
786
|
+
raise typer.Exit(EXIT_OK)
|
|
787
|
+
|
|
788
|
+
|
|
636
789
|
@app.command(name="delete-case")
|
|
637
790
|
def delete_case(
|
|
638
791
|
case_id: str = typer.Argument(
|
|
@@ -745,6 +898,13 @@ def run(
|
|
|
745
898
|
"--label",
|
|
746
899
|
help="Run label shown in the webapp's Evals page (default: the local git branch name).",
|
|
747
900
|
),
|
|
901
|
+
repeat: Optional[int] = typer.Option(
|
|
902
|
+
None,
|
|
903
|
+
"--repeat",
|
|
904
|
+
min=2,
|
|
905
|
+
max=5,
|
|
906
|
+
help="Run every case N times (2 to 5) as one diagnostic run, then compare the attempts per case.",
|
|
907
|
+
),
|
|
748
908
|
wait: bool = typer.Option(
|
|
749
909
|
True,
|
|
750
910
|
"--wait/--no-wait",
|
|
@@ -767,9 +927,14 @@ def run(
|
|
|
767
927
|
an existing Cassis branch instead (no files are sent). With --case, only the
|
|
768
928
|
named case(s) run — e.g. proving one fresh `add-case` in seconds instead of
|
|
769
929
|
rerunning the whole suite; passed cases then print their SQL and evidence
|
|
770
|
-
too, so you can check the case passes for the right reason.
|
|
771
|
-
|
|
772
|
-
|
|
930
|
+
too, so you can check the case passes for the right reason. With --repeat
|
|
931
|
+
N, every case runs N times under the same context and model, and the
|
|
932
|
+
command prints how the attempts compare per case (`cassis eval compare`);
|
|
933
|
+
a repeat run is diagnostic, so it files no issue and is not the run the
|
|
934
|
+
webapp shows before publishing. Exits 0 when
|
|
935
|
+
the run completes with every case passed (with --repeat: no case failed
|
|
936
|
+
every attempt, since a flip is the noise being measured), 1 on any failed
|
|
937
|
+
case / failed run / invalid tree, 2 on usage errors (including a --branch the project
|
|
773
938
|
does not have), 3 on transport errors or --timeout.
|
|
774
939
|
|
|
775
940
|
Failed cases (and, with --case, passed ones) print the generated SQL as
|
|
@@ -808,6 +973,7 @@ def run(
|
|
|
808
973
|
branch=branch,
|
|
809
974
|
label=label,
|
|
810
975
|
case_ids=case,
|
|
976
|
+
repeat_count=repeat,
|
|
811
977
|
)
|
|
812
978
|
except EvalStartValidationError as exc:
|
|
813
979
|
_print_validation_failure(exc.detail, base_path)
|
|
@@ -831,6 +997,15 @@ def run(
|
|
|
831
997
|
total = run_record.get("total_cases", 0)
|
|
832
998
|
ontology_label = run_record.get("ontology_label") or "unpublished context"
|
|
833
999
|
run_url = _run_page_url(app_url or api_url, project_id, run_id)
|
|
1000
|
+
repeat_count = run_record.get("repeat_count")
|
|
1001
|
+
if repeat is not None and repeat > 1 and not repeat_count:
|
|
1002
|
+
# An older server drops the field: the run is an ordinary one, so say so
|
|
1003
|
+
# rather than print a comparison it never measured.
|
|
1004
|
+
typer.secho(
|
|
1005
|
+
"This Cassis server ignored --repeat: the run executes each case once, as an ordinary run.",
|
|
1006
|
+
fg=typer.colors.YELLOW,
|
|
1007
|
+
err=True,
|
|
1008
|
+
)
|
|
834
1009
|
|
|
835
1010
|
if not wait:
|
|
836
1011
|
if json_output:
|
|
@@ -841,7 +1016,10 @@ def run(
|
|
|
841
1016
|
raise typer.Exit(EXIT_OK)
|
|
842
1017
|
|
|
843
1018
|
# To stderr under --json so `cassis eval run --json | jq` gets only the record.
|
|
844
|
-
|
|
1019
|
+
started = f"Eval run {run_id} started: {total} cases against {ontology_label!r}"
|
|
1020
|
+
if repeat_count:
|
|
1021
|
+
started += f", {repeat_count} attempts each"
|
|
1022
|
+
typer.echo(f"{started}.", err=json_output)
|
|
845
1023
|
|
|
846
1024
|
try:
|
|
847
1025
|
final_run, results = _wait_for_run(
|
|
@@ -849,7 +1027,7 @@ def run(
|
|
|
849
1027
|
api_key=api_key,
|
|
850
1028
|
project_id=project_id,
|
|
851
1029
|
run_id=run_id,
|
|
852
|
-
total=total,
|
|
1030
|
+
total=total * (repeat_count or 1),
|
|
853
1031
|
poll_interval=poll_interval,
|
|
854
1032
|
timeout=timeout,
|
|
855
1033
|
json_output=json_output,
|
|
@@ -865,21 +1043,44 @@ def run(
|
|
|
865
1043
|
typer.secho(f"Could not cancel the run: {exc}", fg=typer.colors.RED, err=True)
|
|
866
1044
|
raise typer.Exit(EXIT_INTERRUPTED)
|
|
867
1045
|
|
|
1046
|
+
comparison: Optional[dict[str, Any]] = None
|
|
1047
|
+
if repeat_count and final_run.get("status") == "completed":
|
|
1048
|
+
try:
|
|
1049
|
+
comparison = get_eval_run_comparison(api_url=api_url, api_key=api_key, project_id=project_id, run_id=run_id)
|
|
1050
|
+
except ApiError as exc:
|
|
1051
|
+
typer.secho(f"Could not compare the attempts: {exc}", fg=typer.colors.YELLOW, err=True)
|
|
1052
|
+
|
|
868
1053
|
if json_output:
|
|
869
|
-
|
|
1054
|
+
record: dict[str, Any] = {"run": final_run, "results": results, "run_url": run_url}
|
|
1055
|
+
if comparison is not None:
|
|
1056
|
+
record["comparison"] = comparison
|
|
1057
|
+
typer.echo(json.dumps(record, indent=2))
|
|
870
1058
|
else:
|
|
871
1059
|
typer.echo("")
|
|
872
|
-
|
|
1060
|
+
if comparison is not None:
|
|
1061
|
+
_print_comparison(comparison)
|
|
1062
|
+
else:
|
|
1063
|
+
_print_results_table(results, passed_evidence=bool(case))
|
|
873
1064
|
typer.echo("")
|
|
874
1065
|
_print_summary(final_run)
|
|
875
1066
|
typer.echo(f"View details: {run_url}")
|
|
876
1067
|
|
|
877
|
-
|
|
878
|
-
if status == "completed" and all(r.get("status") == _PASSED for r in results):
|
|
1068
|
+
if final_run.get("status") == "completed" and _suite_passed(results, repeated=bool(repeat_count)):
|
|
879
1069
|
raise typer.Exit(EXIT_OK)
|
|
880
1070
|
raise typer.Exit(EXIT_VALIDATION_FAILED)
|
|
881
1071
|
|
|
882
1072
|
|
|
1073
|
+
def _suite_passed(results: "list[dict[str, Any]]", *, repeated: bool) -> bool:
|
|
1074
|
+
"""Whether a completed run passes: every case passed, or on a repeat run, no case failed every attempt."""
|
|
1075
|
+
if not repeated:
|
|
1076
|
+
return all(r.get("status") == _PASSED for r in results)
|
|
1077
|
+
passed_once: dict[str, bool] = {}
|
|
1078
|
+
for r in results:
|
|
1079
|
+
key = str(r.get("test_case_id") or r.get("question"))
|
|
1080
|
+
passed_once[key] = passed_once.get(key, False) or r.get("status") == _PASSED
|
|
1081
|
+
return all(passed_once.values())
|
|
1082
|
+
|
|
1083
|
+
|
|
883
1084
|
def _wait_for_run(
|
|
884
1085
|
*,
|
|
885
1086
|
api_url: str,
|
|
@@ -928,7 +1129,7 @@ def _wait_for_run(
|
|
|
928
1129
|
|
|
929
1130
|
def on_timeout(_snapshot: "tuple[dict[str, Any], list[dict[str, Any]]]") -> None:
|
|
930
1131
|
typer.secho(
|
|
931
|
-
f"Timed out after {timeout
|
|
1132
|
+
f"Timed out after {timeout:g}s waiting for run {run_id}. "
|
|
932
1133
|
"The run keeps going server-side — see the webapp's Evals page.",
|
|
933
1134
|
fg=typer.colors.YELLOW,
|
|
934
1135
|
err=True,
|
|
@@ -133,8 +133,9 @@ def list_issues(
|
|
|
133
133
|
|
|
134
134
|
The order: editorial priority (P1, P2, P3, then unranked), then impact,
|
|
135
135
|
then reach, then how the issue was found (dbt findings first), then
|
|
136
|
-
recurrence. Prints each issue's
|
|
137
|
-
|
|
136
|
+
recurrence. Prints each issue's priority (`-` when unranked), id, impact,
|
|
137
|
+
origin (conversation, eval or dbt), occurrence count, reach (the chat
|
|
138
|
+
answers that used any of its context tables in the last 30 days; `-` when
|
|
138
139
|
not measured, because its sources name no context table), status, primary
|
|
139
140
|
context domain (`-` when Cassis could not attach one) and title; the id is what
|
|
140
141
|
`cassis issues show`, `resolve`, `dismiss` and `reopen` take. Triage one
|
|
@@ -176,6 +177,8 @@ def list_issues(
|
|
|
176
177
|
reach = issue.get("reach_count")
|
|
177
178
|
rows.append(
|
|
178
179
|
[
|
|
180
|
+
# The editorial priority the list is sorted by first; `-` when unranked.
|
|
181
|
+
str(issue.get("priority") or "-"),
|
|
179
182
|
str(issue.get("id")),
|
|
180
183
|
str(issue.get("impact")),
|
|
181
184
|
# Always printed (conversation, eval or dbt), so every line has the same columns.
|
|
@@ -621,7 +624,7 @@ def _wait_for_analysis(
|
|
|
621
624
|
|
|
622
625
|
def on_timeout(_run: "dict[str, Any]") -> None:
|
|
623
626
|
typer.secho(
|
|
624
|
-
f"Timed out after {timeout
|
|
627
|
+
f"Timed out after {timeout:g}s waiting for analysis run {run_id}. "
|
|
625
628
|
"The analysis keeps going server-side — see the webapp's Issues page.",
|
|
626
629
|
fg=typer.colors.YELLOW,
|
|
627
630
|
err=True,
|
|
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import typer
|
|
6
6
|
from cassis_cli import __version__
|
|
7
|
+
from cassis_cli.chats import app as chats_app
|
|
7
8
|
from cassis_cli.dbt import app as dbt_app
|
|
8
9
|
from cassis_cli.eval import app as eval_app
|
|
9
10
|
from cassis_cli.issues import app as issues_app
|
|
@@ -29,6 +30,7 @@ app.add_typer(schema_app, name="schema")
|
|
|
29
30
|
app.add_typer(dbt_app, name="dbt")
|
|
30
31
|
app.add_typer(projects_app, name="projects")
|
|
31
32
|
app.add_typer(issues_app, name="issues")
|
|
33
|
+
app.add_typer(chats_app, name="chats")
|
|
32
34
|
app.command()(status)
|
|
33
35
|
app.command()(verify)
|
|
34
36
|
|
|
@@ -889,7 +889,7 @@ def _wait_for_plan(
|
|
|
889
889
|
else "`cassis status` shows the outcome once it finishes"
|
|
890
890
|
)
|
|
891
891
|
typer.secho(
|
|
892
|
-
f"Timed out after {timeout
|
|
892
|
+
f"Timed out after {timeout:g}s: the {what} is still {record.get('status', '?')}. "
|
|
893
893
|
f"It keeps running server-side; {resume}",
|
|
894
894
|
fg=typer.colors.YELLOW,
|
|
895
895
|
err=True,
|
|
@@ -197,7 +197,7 @@ def status(
|
|
|
197
197
|
|
|
198
198
|
def on_timeout(_snapshot: "Tuple[dict[str, Any], str, bool]") -> None:
|
|
199
199
|
typer.secho(
|
|
200
|
-
f"Timed out after {timeout
|
|
200
|
+
f"Timed out after {timeout:g}s: the published version still does not match the local HEAD.",
|
|
201
201
|
fg=typer.colors.YELLOW,
|
|
202
202
|
err=True,
|
|
203
203
|
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|