talkviz 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- talkviz-0.1.0/LICENSE +21 -0
- talkviz-0.1.0/PKG-INFO +118 -0
- talkviz-0.1.0/README.md +100 -0
- talkviz-0.1.0/pyproject.toml +39 -0
- talkviz-0.1.0/pyproject.toml.orig +38 -0
- talkviz-0.1.0/src/talkviz/__init__.py +0 -0
- talkviz-0.1.0/src/talkviz/charts.py +123 -0
- talkviz-0.1.0/src/talkviz/cli.py +104 -0
- talkviz-0.1.0/src/talkviz/evaluate.py +84 -0
- talkviz-0.1.0/src/talkviz/llm.py +347 -0
- talkviz-0.1.0/src/talkviz/manifest.py +55 -0
- talkviz-0.1.0/src/talkviz/sources.py +68 -0
- talkviz-0.1.0/src/talkviz/speech.py +76 -0
- talkviz-0.1.0/src/talkviz/sql.py +189 -0
- talkviz-0.1.0/src/talkviz/turn.py +53 -0
- talkviz-0.1.0/src/talkviz/workspace.py +91 -0
talkviz-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jake Van Slyke
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
talkviz-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: talkviz
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Talk to your data and see it charted.
|
|
5
|
+
Author: Jake Van Slyke
|
|
6
|
+
Author-email: Jake Van Slyke <jakervanslyke@gmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: duckdb>=2.0.0
|
|
10
|
+
Requires-Dist: platformdirs>=4.14.7
|
|
11
|
+
Requires-Dist: pydantic>=2.14.3
|
|
12
|
+
Requires-Dist: pydantic-ai-slim[openrouter]>=2.71.0
|
|
13
|
+
Requires-Dist: python-dotenv>=1.2.4
|
|
14
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
15
|
+
Requires-Dist: sqlglot>=30.33.0
|
|
16
|
+
Requires-Python: >=3.12
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# talkviz
|
|
20
|
+
|
|
21
|
+
Talk to your data and see it charted. Ask a question in plain language;
|
|
22
|
+
talkviz writes a read-only SQL query, checks it, runs it, picks a chart, and
|
|
23
|
+
answers in a sentence or two.
|
|
24
|
+
|
|
25
|
+
> Early development. The query engine and command line work today. The web
|
|
26
|
+
> interface and voice are not built yet.
|
|
27
|
+
|
|
28
|
+
## How it stays grounded
|
|
29
|
+
|
|
30
|
+
Every step is checked against the one before it:
|
|
31
|
+
|
|
32
|
+
1. A **manifest** describes the tables, columns, and caveats of your data.
|
|
33
|
+
The model sees the manifest and nothing else.
|
|
34
|
+
2. The model's SQL must be a single `SELECT` over the manifest's tables.
|
|
35
|
+
3. DuckDB plans the query without running it, which checks names and types.
|
|
36
|
+
4. The query runs on a read-only connection with file and network access
|
|
37
|
+
off, a row limit, and a timeout.
|
|
38
|
+
5. The chart spec must fit the columns the query returned.
|
|
39
|
+
6. Every number in the reply must appear in the query results.
|
|
40
|
+
|
|
41
|
+
When a check fails, the reason goes back to the model and it tries again.
|
|
42
|
+
|
|
43
|
+
## Setup
|
|
44
|
+
|
|
45
|
+
```sh
|
|
46
|
+
uv sync
|
|
47
|
+
cp .env.example .env # then add your OpenRouter key
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
`OPENROUTER_API_KEY` is required. `TALKVIZ_MODEL` picks the model (any
|
|
51
|
+
OpenRouter model id); the default is `anthropic/claude-opus-5.5`.
|
|
52
|
+
|
|
53
|
+
## Use
|
|
54
|
+
|
|
55
|
+
Register a DuckDB database with the manifest that describes it, then ask:
|
|
56
|
+
|
|
57
|
+
```sh
|
|
58
|
+
uv run talkviz add path/to/data.duckdb path/to/manifest.yaml
|
|
59
|
+
uv run talkviz ask "How many orders were there each month?"
|
|
60
|
+
uv run talkviz ask # no question: start a conversation
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
In a conversation you can refer back to charts: "make that a bar chart",
|
|
64
|
+
"only since 2020", "show the first one again".
|
|
65
|
+
|
|
66
|
+
## Manifest
|
|
67
|
+
|
|
68
|
+
```yaml
|
|
69
|
+
name: shop
|
|
70
|
+
title: Shop orders
|
|
71
|
+
description: Orders placed at a small shop.
|
|
72
|
+
dialect: duckdb
|
|
73
|
+
data_version_sql: SELECT max(ordered) FROM orders
|
|
74
|
+
caveats:
|
|
75
|
+
- Refunds are not included.
|
|
76
|
+
tables:
|
|
77
|
+
- name: orders
|
|
78
|
+
description: One row per order.
|
|
79
|
+
columns:
|
|
80
|
+
- name: category
|
|
81
|
+
type: VARCHAR
|
|
82
|
+
description: Product category.
|
|
83
|
+
values: [books, games, toys] # optional: every value it can hold
|
|
84
|
+
examples:
|
|
85
|
+
- question: How many orders?
|
|
86
|
+
sql: SELECT count(*) FROM orders
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Answers are only as good as the manifest. Describe each column in plain
|
|
90
|
+
words and state the traps in `caveats`; the reply will mention them.
|
|
91
|
+
|
|
92
|
+
## Scoring a model
|
|
93
|
+
|
|
94
|
+
An eval file is a YAML list of questions with a query that answers each
|
|
95
|
+
correctly. A model passes a question when its query returns the same data.
|
|
96
|
+
|
|
97
|
+
```sh
|
|
98
|
+
uv run talkviz eval path/to/eval.yaml --model anthropic/claude-haiku-5.5
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Development
|
|
102
|
+
|
|
103
|
+
```sh
|
|
104
|
+
uv sync
|
|
105
|
+
uv run pre-commit install # once per clone
|
|
106
|
+
uv run pytest
|
|
107
|
+
uv run ruff check . && uv run ruff format --check . && uv run ty check
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
The hooks run ruff, ty, and a lockfile check on commit, and the tests on
|
|
111
|
+
push. CI runs the same checks on every pull request.
|
|
112
|
+
|
|
113
|
+
Commit messages follow [Conventional Commits](https://www.conventionalcommits.org)
|
|
114
|
+
(`feat: ...`, `fix: ...`, `docs: ...`); a hook checks the format. Releases are
|
|
115
|
+
automated from them: on each push to `main`, release-please updates a release
|
|
116
|
+
pull request with the next version and changelog. `fix` bumps the patch
|
|
117
|
+
version, `feat` the minor, and `feat!` marks a breaking change. Merging that
|
|
118
|
+
pull request tags the release and publishes it to PyPI.
|
talkviz-0.1.0/README.md
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# talkviz
|
|
2
|
+
|
|
3
|
+
Talk to your data and see it charted. Ask a question in plain language;
|
|
4
|
+
talkviz writes a read-only SQL query, checks it, runs it, picks a chart, and
|
|
5
|
+
answers in a sentence or two.
|
|
6
|
+
|
|
7
|
+
> Early development. The query engine and command line work today. The web
|
|
8
|
+
> interface and voice are not built yet.
|
|
9
|
+
|
|
10
|
+
## How it stays grounded
|
|
11
|
+
|
|
12
|
+
Every step is checked against the one before it:
|
|
13
|
+
|
|
14
|
+
1. A **manifest** describes the tables, columns, and caveats of your data.
|
|
15
|
+
The model sees the manifest and nothing else.
|
|
16
|
+
2. The model's SQL must be a single `SELECT` over the manifest's tables.
|
|
17
|
+
3. DuckDB plans the query without running it, which checks names and types.
|
|
18
|
+
4. The query runs on a read-only connection with file and network access
|
|
19
|
+
off, a row limit, and a timeout.
|
|
20
|
+
5. The chart spec must fit the columns the query returned.
|
|
21
|
+
6. Every number in the reply must appear in the query results.
|
|
22
|
+
|
|
23
|
+
When a check fails, the reason goes back to the model and it tries again.
|
|
24
|
+
|
|
25
|
+
## Setup
|
|
26
|
+
|
|
27
|
+
```sh
|
|
28
|
+
uv sync
|
|
29
|
+
cp .env.example .env # then add your OpenRouter key
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
`OPENROUTER_API_KEY` is required. `TALKVIZ_MODEL` picks the model (any
|
|
33
|
+
OpenRouter model id); the default is `anthropic/claude-opus-5.5`.
|
|
34
|
+
|
|
35
|
+
## Use
|
|
36
|
+
|
|
37
|
+
Register a DuckDB database with the manifest that describes it, then ask:
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
uv run talkviz add path/to/data.duckdb path/to/manifest.yaml
|
|
41
|
+
uv run talkviz ask "How many orders were there each month?"
|
|
42
|
+
uv run talkviz ask # no question: start a conversation
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
In a conversation you can refer back to charts: "make that a bar chart",
|
|
46
|
+
"only since 2020", "show the first one again".
|
|
47
|
+
|
|
48
|
+
## Manifest
|
|
49
|
+
|
|
50
|
+
```yaml
|
|
51
|
+
name: shop
|
|
52
|
+
title: Shop orders
|
|
53
|
+
description: Orders placed at a small shop.
|
|
54
|
+
dialect: duckdb
|
|
55
|
+
data_version_sql: SELECT max(ordered) FROM orders
|
|
56
|
+
caveats:
|
|
57
|
+
- Refunds are not included.
|
|
58
|
+
tables:
|
|
59
|
+
- name: orders
|
|
60
|
+
description: One row per order.
|
|
61
|
+
columns:
|
|
62
|
+
- name: category
|
|
63
|
+
type: VARCHAR
|
|
64
|
+
description: Product category.
|
|
65
|
+
values: [books, games, toys] # optional: every value it can hold
|
|
66
|
+
examples:
|
|
67
|
+
- question: How many orders?
|
|
68
|
+
sql: SELECT count(*) FROM orders
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Answers are only as good as the manifest. Describe each column in plain
|
|
72
|
+
words and state the traps in `caveats`; the reply will mention them.
|
|
73
|
+
|
|
74
|
+
## Scoring a model
|
|
75
|
+
|
|
76
|
+
An eval file is a YAML list of questions with a query that answers each
|
|
77
|
+
correctly. A model passes a question when its query returns the same data.
|
|
78
|
+
|
|
79
|
+
```sh
|
|
80
|
+
uv run talkviz eval path/to/eval.yaml --model anthropic/claude-haiku-5.5
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Development
|
|
84
|
+
|
|
85
|
+
```sh
|
|
86
|
+
uv sync
|
|
87
|
+
uv run pre-commit install # once per clone
|
|
88
|
+
uv run pytest
|
|
89
|
+
uv run ruff check . && uv run ruff format --check . && uv run ty check
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
The hooks run ruff, ty, and a lockfile check on commit, and the tests on
|
|
93
|
+
push. CI runs the same checks on every pull request.
|
|
94
|
+
|
|
95
|
+
Commit messages follow [Conventional Commits](https://www.conventionalcommits.org)
|
|
96
|
+
(`feat: ...`, `fix: ...`, `docs: ...`); a hook checks the format. Releases are
|
|
97
|
+
automated from them: on each push to `main`, release-please updates a release
|
|
98
|
+
pull request with the next version and changelog. `fix` bumps the patch
|
|
99
|
+
version, `feat` the minor, and `feat!` marks a breaking change. Merging that
|
|
100
|
+
pull request tags the release and publishes it to PyPI.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "talkviz"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Talk to your data and see it charted."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.12"
|
|
9
|
+
dependencies = [
|
|
10
|
+
"duckdb>=2.0.0",
|
|
11
|
+
"platformdirs>=4.14.7",
|
|
12
|
+
"pydantic>=2.14.3",
|
|
13
|
+
"pydantic-ai-slim[openrouter]>=2.71.0",
|
|
14
|
+
"python-dotenv>=1.2.4",
|
|
15
|
+
"pyyaml>=6.0.3",
|
|
16
|
+
"sqlglot>=30.33.0",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[[project.authors]]
|
|
20
|
+
name = "Jake Van Slyke"
|
|
21
|
+
email = "jakervanslyke@gmail.com"
|
|
22
|
+
|
|
23
|
+
[project.scripts]
|
|
24
|
+
talkviz = "talkviz.cli:main"
|
|
25
|
+
|
|
26
|
+
[build-system]
|
|
27
|
+
requires = ["uv_build>=0.13.6,<0.14.0"]
|
|
28
|
+
build-backend = "uv_build"
|
|
29
|
+
|
|
30
|
+
[dependency-groups]
|
|
31
|
+
dev = [
|
|
32
|
+
"pre-commit>=4.7.0",
|
|
33
|
+
"pytest>=9.2.0",
|
|
34
|
+
"ruff>=0.17.5",
|
|
35
|
+
"ty>=0.0.107",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
[tool.pytest.ini_options]
|
|
39
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "talkviz"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Talk to your data and see it charted."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "Jake Van Slyke", email = "jakervanslyke@gmail.com" }
|
|
10
|
+
]
|
|
11
|
+
requires-python = ">=3.12"
|
|
12
|
+
dependencies = [
|
|
13
|
+
"duckdb>=2.0.0",
|
|
14
|
+
"platformdirs>=4.14.7",
|
|
15
|
+
"pydantic>=2.14.3",
|
|
16
|
+
"pydantic-ai-slim[openrouter]>=2.71.0",
|
|
17
|
+
"python-dotenv>=1.2.4",
|
|
18
|
+
"pyyaml>=6.0.3",
|
|
19
|
+
"sqlglot>=30.33.0",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.scripts]
|
|
23
|
+
talkviz = "talkviz.cli:main"
|
|
24
|
+
|
|
25
|
+
[build-system]
|
|
26
|
+
requires = ["uv_build>=0.13.6,<0.14.0"]
|
|
27
|
+
build-backend = "uv_build"
|
|
28
|
+
|
|
29
|
+
[dependency-groups]
|
|
30
|
+
dev = [
|
|
31
|
+
"pre-commit>=4.7.0",
|
|
32
|
+
"pytest>=9.2.0",
|
|
33
|
+
"ruff>=0.17.5",
|
|
34
|
+
"ty>=0.0.107",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[tool.pytest.ini_options]
|
|
38
|
+
testpaths = ["tests"]
|
|
File without changes
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Chart specs: what the agent may ask the interface to draw.
|
|
2
|
+
|
|
3
|
+
A spec names columns of a query result; it never contains data. check_spec()
|
|
4
|
+
confirms a spec fits the result it will be drawn from, so the interface is
|
|
5
|
+
only ever handed charts it can render.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from typing import Annotated, Literal
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
11
|
+
|
|
12
|
+
from talkviz.sql import QueryResult
|
|
13
|
+
|
|
14
|
+
MAX_BARS = 60
|
|
15
|
+
MAX_SERIES = 12
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class _Spec(BaseModel):
|
|
19
|
+
model_config = ConfigDict(extra="forbid")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Stat(_Spec):
|
|
23
|
+
"""One headline number. The query must return exactly one row."""
|
|
24
|
+
|
|
25
|
+
kind: Literal["stat"]
|
|
26
|
+
value: str = Field(description="Numeric column holding the number.")
|
|
27
|
+
label: str = Field(description="Short caption, e.g. 'complaints in 2024'.")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class Table(_Spec):
|
|
31
|
+
"""Rows and columns as they are. Use when no chart fits."""
|
|
32
|
+
|
|
33
|
+
kind: Literal["table"]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Bar(_Spec):
|
|
37
|
+
"""Compare a number across categories; `series` splits each into groups."""
|
|
38
|
+
|
|
39
|
+
kind: Literal["bar"]
|
|
40
|
+
x: str = Field(description="Column of categories, one bar (or group) each.")
|
|
41
|
+
y: str = Field(description="Numeric column for bar height.")
|
|
42
|
+
series: str | None = Field(None, description="Optional column to group bars by.")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Line(_Spec):
|
|
46
|
+
"""A number over time or another ordered axis; `series` draws several lines."""
|
|
47
|
+
|
|
48
|
+
kind: Literal["line"]
|
|
49
|
+
x: str = Field(description="Date or numeric column, e.g. a year.")
|
|
50
|
+
y: str = Field(description="Numeric column for line height.")
|
|
51
|
+
series: str | None = Field(None, description="Optional column, one line each.")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
ChartSpec = Annotated[Stat | Table | Bar | Line, Field(discriminator="kind")]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def check_spec(spec: Stat | Table | Bar | Line, result: QueryResult) -> list[str]:
|
|
58
|
+
"""Reasons the spec cannot be drawn from the result; empty if it can."""
|
|
59
|
+
if not result.rows:
|
|
60
|
+
return ["The query returned no rows, so there is nothing to draw."]
|
|
61
|
+
if isinstance(spec, Table):
|
|
62
|
+
return []
|
|
63
|
+
available = ", ".join(f"{c.name} ({c.kind})" for c in result.columns)
|
|
64
|
+
problems = []
|
|
65
|
+
|
|
66
|
+
def need(role: str, name: str, kinds: tuple[str, ...]) -> bool:
|
|
67
|
+
column = result.column(name)
|
|
68
|
+
if column is None:
|
|
69
|
+
problems.append(
|
|
70
|
+
f"{role} column {name!r} is not in the query result."
|
|
71
|
+
f" Its columns are: {available}."
|
|
72
|
+
)
|
|
73
|
+
elif column.kind not in kinds:
|
|
74
|
+
problems.append(
|
|
75
|
+
f"{role} column {name!r} is {column.kind}; it must be"
|
|
76
|
+
f" {' or '.join(kinds)}."
|
|
77
|
+
)
|
|
78
|
+
return not problems
|
|
79
|
+
|
|
80
|
+
if isinstance(spec, Stat):
|
|
81
|
+
need("value", spec.value, ("number",))
|
|
82
|
+
if len(result.rows) != 1:
|
|
83
|
+
problems.append(
|
|
84
|
+
f"A stat needs a query that returns one row; this returns"
|
|
85
|
+
f" {len(result.rows)}."
|
|
86
|
+
)
|
|
87
|
+
return problems
|
|
88
|
+
|
|
89
|
+
x_kinds = ("number", "time") if isinstance(spec, Line) else _DISCRETE
|
|
90
|
+
ok = need("x", spec.x, x_kinds)
|
|
91
|
+
ok = need("y", spec.y, ("number",)) and ok
|
|
92
|
+
if spec.series is not None:
|
|
93
|
+
ok = need("series", spec.series, _DISCRETE) and ok
|
|
94
|
+
if not ok:
|
|
95
|
+
return problems
|
|
96
|
+
if result.truncated:
|
|
97
|
+
problems.append(
|
|
98
|
+
"The query returns too many rows to chart; aggregate or filter it."
|
|
99
|
+
)
|
|
100
|
+
xs = result.values(spec.x)
|
|
101
|
+
series = result.values(spec.series) if spec.series else [None] * len(xs)
|
|
102
|
+
if len(set(zip(xs, series, strict=True))) < len(xs):
|
|
103
|
+
split = f" and {spec.series!r}" if spec.series else ""
|
|
104
|
+
problems.append(
|
|
105
|
+
f"The result has several rows for the same {spec.x!r}{split};"
|
|
106
|
+
" group the query so there is one row each."
|
|
107
|
+
)
|
|
108
|
+
if len(set(series)) > MAX_SERIES:
|
|
109
|
+
problems.append(
|
|
110
|
+
f"{spec.series!r} has {len(set(series))} values; a chart can show at"
|
|
111
|
+
f" most {MAX_SERIES} series. Keep the top few in the query."
|
|
112
|
+
)
|
|
113
|
+
if isinstance(spec, Bar) and len(set(xs)) > MAX_BARS:
|
|
114
|
+
problems.append(
|
|
115
|
+
f"{len(set(xs))} bars is too many; keep at most {MAX_BARS} with"
|
|
116
|
+
" ORDER BY and LIMIT."
|
|
117
|
+
)
|
|
118
|
+
if isinstance(spec, Line) and len(set(xs)) < 2:
|
|
119
|
+
problems.append("A line needs at least two x values; use a stat or bar.")
|
|
120
|
+
return problems
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
_DISCRETE = ("text", "number", "boolean", "time")
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import sys
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from dotenv import find_dotenv, load_dotenv
|
|
6
|
+
|
|
7
|
+
from talkviz import llm
|
|
8
|
+
from talkviz.evaluate import evaluate
|
|
9
|
+
from talkviz.llm import Exchange
|
|
10
|
+
from talkviz.sources import add_source, list_sources, load_source
|
|
11
|
+
from talkviz.turn import Turn, take_turn
|
|
12
|
+
from talkviz.workspace import Workspace
|
|
13
|
+
|
|
14
|
+
PREVIEW_ROWS = 12
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def main() -> None:
|
|
18
|
+
parser = argparse.ArgumentParser(
|
|
19
|
+
prog="talkviz", description="Talk to your data and see it charted."
|
|
20
|
+
)
|
|
21
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
22
|
+
|
|
23
|
+
add = commands.add_parser("add", help="register a data source")
|
|
24
|
+
add.add_argument("database", type=Path, help="DuckDB database file")
|
|
25
|
+
add.add_argument("manifest", type=Path, help="YAML manifest describing it")
|
|
26
|
+
|
|
27
|
+
commands.add_parser("sources", help="list registered data sources")
|
|
28
|
+
|
|
29
|
+
ask = commands.add_parser("ask", help="ask one question, or chat if none is given")
|
|
30
|
+
ask.add_argument("question", nargs="?")
|
|
31
|
+
_common(ask)
|
|
32
|
+
|
|
33
|
+
evaluation = commands.add_parser("eval", help="score a model on an eval file")
|
|
34
|
+
evaluation.add_argument("eval_file", type=Path)
|
|
35
|
+
_common(evaluation)
|
|
36
|
+
|
|
37
|
+
args = parser.parse_args()
|
|
38
|
+
load_dotenv(find_dotenv(usecwd=True))
|
|
39
|
+
try:
|
|
40
|
+
_run(args)
|
|
41
|
+
except (LookupError, FileNotFoundError, llm.ModelFailed) as error:
|
|
42
|
+
sys.exit(f"talkviz: {error}")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _common(command: argparse.ArgumentParser) -> None:
|
|
46
|
+
command.add_argument("--source", help="source name, if more than one is registered")
|
|
47
|
+
command.add_argument(
|
|
48
|
+
"--model", help=f"OpenRouter model (default {llm.DEFAULT_MODEL})"
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _run(args: argparse.Namespace) -> None:
|
|
53
|
+
if args.command == "add":
|
|
54
|
+
source = add_source(args.database, args.manifest)
|
|
55
|
+
print(f"added {source.name} ({source.database})")
|
|
56
|
+
elif args.command == "sources":
|
|
57
|
+
print("\n".join(list_sources()) or "no sources; add one with `talkviz add`")
|
|
58
|
+
elif args.command == "ask":
|
|
59
|
+
_ask(args)
|
|
60
|
+
elif args.command == "eval":
|
|
61
|
+
source, model = load_source(args.source), llm.openrouter(args.model)
|
|
62
|
+
outcomes = evaluate(model, source, args.eval_file)
|
|
63
|
+
for outcome in outcomes:
|
|
64
|
+
print(f"{'pass' if outcome.correct else 'FAIL'} {outcome.question}")
|
|
65
|
+
if not outcome.correct:
|
|
66
|
+
print(f" {outcome.detail}")
|
|
67
|
+
if outcome.sql:
|
|
68
|
+
print(f" {' '.join(outcome.sql.split())}")
|
|
69
|
+
correct = sum(outcome.correct for outcome in outcomes)
|
|
70
|
+
print(f"\n{correct}/{len(outcomes)} correct ({model.model_name})")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _ask(args: argparse.Namespace) -> None:
|
|
74
|
+
source, model = load_source(args.source), llm.openrouter(args.model)
|
|
75
|
+
workspace, history = Workspace(), list[Exchange]()
|
|
76
|
+
with source.connect() as con:
|
|
77
|
+
print(f"{source.manifest.title}, data as of {source.data_version(con)}")
|
|
78
|
+
questions = [args.question] if args.question else iter(_prompt, None)
|
|
79
|
+
for question in questions:
|
|
80
|
+
turn = take_turn(model, question, source.manifest, con, workspace, history)
|
|
81
|
+
_show(turn)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _prompt() -> str | None:
|
|
85
|
+
try:
|
|
86
|
+
return input("\n> ").strip() or None
|
|
87
|
+
except EOFError:
|
|
88
|
+
return None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _show(turn: Turn) -> None:
|
|
92
|
+
for run in turn.runs:
|
|
93
|
+
chart, result = run.chart, run.result
|
|
94
|
+
print(f"\n[{chart.id} r{chart.revision}] {chart.title} ({run.op})")
|
|
95
|
+
print(f" spec: {chart.spec.model_dump_json(exclude_none=True)}")
|
|
96
|
+
print(f" sql: {' '.join(chart.sql.split())}")
|
|
97
|
+
print(" " + " | ".join(column.name for column in result.columns))
|
|
98
|
+
for row in result.rows[:PREVIEW_ROWS]:
|
|
99
|
+
print(" " + " | ".join(str(value) for value in row))
|
|
100
|
+
if len(result.rows) > PREVIEW_ROWS:
|
|
101
|
+
print(f" … {len(result.rows)} rows")
|
|
102
|
+
flag = "" if turn.reply_verified else " [unverified fallback]"
|
|
103
|
+
print(f"\n{turn.reply}{flag}")
|
|
104
|
+
print(f" (plan {turn.seconds['plan']:.1f}s, speak {turn.seconds['speak']:.1f}s)")
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Score a model on questions with known answers.
|
|
2
|
+
|
|
3
|
+
An eval file is YAML: a list of {question, sql}, where sql is a query that
|
|
4
|
+
answers the question correctly. The model's first chart for the question is
|
|
5
|
+
scored by whether its query returns the same data.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from datetime import date, datetime
|
|
10
|
+
from decimal import Decimal
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import yaml
|
|
15
|
+
from pydantic_ai.models import Model
|
|
16
|
+
|
|
17
|
+
from talkviz import llm
|
|
18
|
+
from talkviz.sources import Source
|
|
19
|
+
from talkviz.sql import QueryResult, run
|
|
20
|
+
from talkviz.workspace import Workspace
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class Outcome:
|
|
25
|
+
question: str
|
|
26
|
+
correct: bool
|
|
27
|
+
sql: str | None
|
|
28
|
+
"""The model's query, or None if it made no chart."""
|
|
29
|
+
detail: str
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def evaluate(model: Model, source: Source, eval_file: Path) -> list[Outcome]:
|
|
33
|
+
cases = yaml.safe_load(eval_file.read_text())
|
|
34
|
+
outcomes = []
|
|
35
|
+
with source.connect() as con:
|
|
36
|
+
for case in cases:
|
|
37
|
+
expected = run(con, case["sql"])
|
|
38
|
+
try:
|
|
39
|
+
planned = llm.plan(
|
|
40
|
+
model, case["question"], source.manifest, con, Workspace(), []
|
|
41
|
+
)
|
|
42
|
+
except llm.ModelFailed as error:
|
|
43
|
+
outcomes.append(Outcome(case["question"], False, None, str(error)))
|
|
44
|
+
continue
|
|
45
|
+
if not planned.runs:
|
|
46
|
+
detail = f"no chart: {planned.plan.note}"
|
|
47
|
+
outcomes.append(Outcome(case["question"], False, None, detail))
|
|
48
|
+
continue
|
|
49
|
+
got = planned.runs[0]
|
|
50
|
+
correct = same_data(expected, got.result)
|
|
51
|
+
detail = "" if correct else "result differs from the expected data"
|
|
52
|
+
outcomes.append(Outcome(case["question"], correct, got.chart.sql, detail))
|
|
53
|
+
return outcomes
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def same_data(expected: QueryResult, got: QueryResult) -> bool:
|
|
57
|
+
"""True if every expected column appears in `got` with the same values.
|
|
58
|
+
|
|
59
|
+
Column names, column order, row order, and extra columns in `got` are
|
|
60
|
+
ignored; floats are compared to 4 significant figures.
|
|
61
|
+
"""
|
|
62
|
+
if len(expected.rows) != len(got.rows):
|
|
63
|
+
return False
|
|
64
|
+
available = [_column(got, c.name) for c in got.columns]
|
|
65
|
+
for column in expected.columns:
|
|
66
|
+
wanted = _column(expected, column.name)
|
|
67
|
+
if wanted not in available:
|
|
68
|
+
return False
|
|
69
|
+
available.remove(wanted)
|
|
70
|
+
return True
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _column(result: QueryResult, name: str) -> list[str]:
|
|
74
|
+
return sorted(_normal(value) for value in result.values(name))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _normal(value: Any) -> str:
|
|
78
|
+
if isinstance(value, bool) or value is None:
|
|
79
|
+
return str(value)
|
|
80
|
+
if isinstance(value, int | float | Decimal):
|
|
81
|
+
return f"{float(value):.4g}"
|
|
82
|
+
if isinstance(value, datetime | date):
|
|
83
|
+
return value.isoformat()
|
|
84
|
+
return str(value)
|