evals-promptdb 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evals_promptdb-1.2.0.dist-info/METADATA +301 -0
- evals_promptdb-1.2.0.dist-info/RECORD +17 -0
- evals_promptdb-1.2.0.dist-info/WHEEL +5 -0
- evals_promptdb-1.2.0.dist-info/entry_points.txt +2 -0
- evals_promptdb-1.2.0.dist-info/licenses/LICENSE +21 -0
- evals_promptdb-1.2.0.dist-info/top_level.txt +1 -0
- promptdb/__init__.py +27 -0
- promptdb/__main__.py +3 -0
- promptdb/_snapshots.py +163 -0
- promptdb/_values.py +102 -0
- promptdb/_version.py +1 -0
- promptdb/cli.py +352 -0
- promptdb/database.py +664 -0
- promptdb/py.typed +1 -0
- promptdb/records.py +128 -0
- promptdb/schema.py +73 -0
- promptdb/scorers.py +44 -0
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: evals-promptdb
|
|
3
|
+
Version: 1.2.0
|
|
4
|
+
Summary: A local, embedded SQLite database for versioned LLM prompts
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Repository, https://github.com/Arcynox/EvalsPromptDB
|
|
7
|
+
Project-URL: Documentation, https://github.com/Arcynox/EvalsPromptDB#readme
|
|
8
|
+
Project-URL: Issues, https://github.com/Arcynox/EvalsPromptDB/issues
|
|
9
|
+
Keywords: llm,prompts,versioning,evaluation,sqlite
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
21
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
22
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
23
|
+
Classifier: Topic :: Database
|
|
24
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
<div align="center">
|
|
32
|
+
<img src="assets/evals-promptdb.png" width="128" height="128" alt="Evals PromptDB icon">
|
|
33
|
+
<h1>Evals PromptDB</h1>
|
|
34
|
+
<p><strong>Local SQLite storage for versioned LLM prompts and evaluations.</strong></p>
|
|
35
|
+
<p>
|
|
36
|
+
<a href="https://github.com/Arcynox/EvalsPromptDB/actions/workflows/ci.yml"><img src="https://github.com/Arcynox/EvalsPromptDB/actions/workflows/ci.yml/badge.svg" alt="GitHub Actions CI status"></a>
|
|
37
|
+
<a href="#install"><img src="https://img.shields.io/badge/Python-3.10%2B-6d4aff" alt="Python 3.10 and later"></a>
|
|
38
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-27896d" alt="MIT license"></a>
|
|
39
|
+
</p>
|
|
40
|
+
<p>
|
|
41
|
+
<a href="README.es.md">Español</a> · <a href="#install">Install</a> · <a href="#quickstart-in-python">Python quickstart</a> · <a href="#quickstart-in-the-terminal">CLI quickstart</a> · <a href="#provider-examples">Providers</a>
|
|
42
|
+
</p>
|
|
43
|
+
</div>
|
|
44
|
+
|
|
45
|
+
Keep templates, test cases, model outputs, and scores in one `.db` file. Use the Python API in your application or the CLI in scripts and CI. The core uses the Python standard library.
|
|
46
|
+
|
|
47
|
+
## Prompt workflow
|
|
48
|
+
|
|
49
|
+
| Step | What you can do |
|
|
50
|
+
| --- | --- |
|
|
51
|
+
| **Version** | Save immutable templates and select the active version by tag. |
|
|
52
|
+
| **Evaluate** | Run the same cases across versions with a generator and scorer you supply. |
|
|
53
|
+
| **Compare and promote** | Inspect diffs, scores, and coverage. Gate promotion in CI and roll back by moving the tag. |
|
|
54
|
+
| **Share** | Track deterministic JSON in Git or transfer a consistent SQLite backup. |
|
|
55
|
+
|
|
56
|
+
## Install
|
|
57
|
+
|
|
58
|
+
Use Python 3.10 or later. Install from this checkout.
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
python -m pip install .
|
|
62
|
+
promptdb --version
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The distribution is named `evals-promptdb`. Import `PromptDB` from `promptdb`, and run `promptdb` or `python -m promptdb`. You can run `python -m promptdb` from the checkout without installing it.
|
|
66
|
+
|
|
67
|
+
If you installed PromptDB 0.1.0 or 0.2.0 from this project, remove the old distribution (`promptdb` or `promptdb-local`, respectively) from that environment before installing `evals-promptdb`. Opening an older SQLite file with a writable connection migrates it in a single transaction. Use the upgraded package to open migrated files.
|
|
68
|
+
|
|
69
|
+
## Quickstart in Python
|
|
70
|
+
|
|
71
|
+
Run this in a directory without a `prompts.db` file. The demo functions stand in for model calls so the example runs offline.
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from promptdb import PromptDB
|
|
75
|
+
|
|
76
|
+
with PromptDB("prompts.db") as db:
|
|
77
|
+
v1 = db.save(
|
|
78
|
+
"country",
|
|
79
|
+
"Give the country containing {{city}}.",
|
|
80
|
+
tags=["production"],
|
|
81
|
+
metadata={"team": "examples"},
|
|
82
|
+
)
|
|
83
|
+
v2 = db.save(
|
|
84
|
+
"country",
|
|
85
|
+
"Return only the name of the country containing {{city}}.",
|
|
86
|
+
tags=["experiment"],
|
|
87
|
+
)
|
|
88
|
+
case = db.add_test("country", input={"city": "Madrid"}, expected="Spain")
|
|
89
|
+
|
|
90
|
+
current = db.get("country", tag="production")
|
|
91
|
+
print(current.render(case.input))
|
|
92
|
+
|
|
93
|
+
db.run_tests("country", version=v1.version, model="demo",
|
|
94
|
+
generate=lambda text: "Madrid is in Spain.", run_id="quickstart")
|
|
95
|
+
db.run_tests("country", version=v2.version, model="demo",
|
|
96
|
+
generate=lambda text: "Spain", run_id="quickstart")
|
|
97
|
+
|
|
98
|
+
report = db.compare("country", [v1.version, v2.version], model="demo", run_id="quickstart")
|
|
99
|
+
print(report.versions[1].diff)
|
|
100
|
+
print(report.versions[1].coverage) # 1.0
|
|
101
|
+
print(report.best.prompt.version) # 2
|
|
102
|
+
|
|
103
|
+
db.promote("country", v2.version, tag="production")
|
|
104
|
+
db.promote("country", v1.version, tag="production") # Roll back
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The same case applies to both versions. By default, `run_tests()` scores text by exact match after trimming surrounding whitespace. A custom `scorer(output, expected)` can return any numeric score from 0 to 1 and is required for non-text expectations.
|
|
108
|
+
|
|
109
|
+
## Chat prompts
|
|
110
|
+
|
|
111
|
+
Store ordered text messages when the model needs distinct instruction and conversation roles. Variables work in every message. The supported roles are `system`, `developer`, `user`, and `assistant`.
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from promptdb import PromptDB
|
|
115
|
+
|
|
116
|
+
with PromptDB(":memory:") as db:
|
|
117
|
+
prompt = db.save("translate", messages=[
|
|
118
|
+
{"role": "system", "content": "Translate the user's text to {{language}}."},
|
|
119
|
+
{"role": "user", "content": "{{text}}"},
|
|
120
|
+
])
|
|
121
|
+
inputs = {"language": "Spanish", "text": "Good morning"}
|
|
122
|
+
messages = prompt.render_messages(inputs)
|
|
123
|
+
db.add_test("translate", input=inputs, expected="Buenos días")
|
|
124
|
+
db.run_tests("translate", model="demo", generate_messages=lambda conversation: "Buenos días")
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
`Prompt.kind` distinguishes `text` and `chat`. `render_messages()` also accepts text prompts and wraps their rendered text in a single user message. `render()` returns JSON text for chat prompts. Use `generate_messages` to pass the rendered messages directly to your model client.
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
promptdb --db chat.db save translate --messages - <<'JSON'
|
|
131
|
+
[{"role":"system","content":"Translate to {{language}}."},{"role":"user","content":"{{text}}"}]
|
|
132
|
+
JSON
|
|
133
|
+
promptdb --db chat.db get translate --input '{"language":"Spanish","text":"Good morning"}'
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
## Quickstart in the terminal
|
|
137
|
+
|
|
138
|
+
These commands use a fresh database named `cli.db`.
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
promptdb --db cli.db save country --template 'Give the country containing {{city}}.' --tag production
|
|
142
|
+
promptdb --db cli.db save country --template 'Return only the country name for {{city}}.' --tag experiment
|
|
143
|
+
promptdb --db cli.db get country --tag production --input '{"city":"Madrid"}'
|
|
144
|
+
promptdb --db cli.db add-test country --input '{"city":"Madrid"}' --expected '"Spain"'
|
|
145
|
+
promptdb --db cli.db eval country --version 1 --model demo --run-id demo-1 --test-id 1 --score 0.83
|
|
146
|
+
promptdb --db cli.db eval country --version 2 --model demo --run-id demo-1 --test-id 1 --score 0.87
|
|
147
|
+
promptdb --db cli.db compare country 1 2 --model demo --run-id demo-1 --min-score 0.85 --min-coverage 1 --max-regression 0.02
|
|
148
|
+
promptdb --db cli.db promote country 2 --tag production
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
<details>
|
|
152
|
+
<summary>Example comparison output</summary>
|
|
153
|
+
|
|
154
|
+
```text
|
|
155
|
+
Version Variables Added Removed Evals Avg score Cases Case avg Paired delta
|
|
156
|
+
1 city - - 1 0.830 1/1 0.830 -
|
|
157
|
+
2 city - - 1 0.870 1/1 0.870 +0.040 (1)
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
</details>
|
|
161
|
+
|
|
162
|
+
Use `save --from-file template.txt` for a multiline UTF-8 template and `--from-file -` for standard input. Use `get --json` for a full record, `get --input` for a rendered template, and `compare --json` for a machine-readable comparison. For the full command list, run `promptdb --help`.
|
|
163
|
+
|
|
164
|
+
### Quality gates in CI
|
|
165
|
+
|
|
166
|
+
The last version in the comparison is the candidate. The first is the baseline. Pass the same `--model` and `--run-id` used when recording the scores so the gate reads one model and one run.
|
|
167
|
+
|
|
168
|
+
| Flag | Condition for passing |
|
|
169
|
+
| --- | --- |
|
|
170
|
+
| `--min-score 0.85` | The candidate meets the score threshold. With a coverage or regression gate, each scored case has equal weight and only scores linked to a case count. On its own, `--min-score` uses the average of all matching evaluation rows. |
|
|
171
|
+
| `--min-coverage 1` | The candidate has scores for 100% of eligible cases. Without `--test-id`, all saved cases for the prompt are eligible. Repeated scores on one case count once toward coverage. |
|
|
172
|
+
| `--max-regression 0.02` | The candidate's case average drops by at most 0.02 against the baseline on their shared cases. Shared coverage must reach `--min-coverage`, or 100% if that flag is omitted. |
|
|
173
|
+
|
|
174
|
+
A failed gate returns status `3`, including when required scores or cases are missing. A passing result returns `0`. Operational errors return `1`, and command-line syntax errors return `2`. The JSON report includes `case_scores`, `coverage`, `case_average_score`, and `paired` for each version, alongside the original row average.
|
|
175
|
+
|
|
176
|
+
## Python API
|
|
177
|
+
|
|
178
|
+
| Operation | Behavior |
|
|
179
|
+
| --- | --- |
|
|
180
|
+
| `save(name, template, ...)` | Creates a version of a text prompt or accepts `messages=` for chat. Also accepts `version`, `variables`, `model`, `temperature`, `metadata`, and `tags`. |
|
|
181
|
+
| `get(name, version="latest", tag=None)` | Returns the highest matching version. Passing a number and a tag requires both to match. Raises `KeyError` when there is no match. |
|
|
182
|
+
| `Prompt.render(values)` | Replaces detected placeholders in one pass. Extra inputs are allowed. A missing input raises `KeyError`. |
|
|
183
|
+
| `list(tags=())` and `history(name)` | List versions. Filtering by tags requires every requested tag. History orders versions from oldest to newest. |
|
|
184
|
+
| `set_tags(name, version, tags)` | Replaces tags on one version. The CLI equivalent is `tag`. |
|
|
185
|
+
| `promote(name, version, tag="produccion")` | Moves a tag to exactly one version of that prompt in a single transaction. |
|
|
186
|
+
| `add_test(name, input=..., expected=...)` and `list_tests(name)` | Store cases shared by all versions of a prompt and list the active cases. |
|
|
187
|
+
| `get_test(name, test_id)` | Read a case by ID, including an archived case. |
|
|
188
|
+
| `set_test_archived(name, test_id, archived=True)` and `replace_test(...)` | Manage the active suite while keeping the previous case and its evaluations available. |
|
|
189
|
+
| `add_eval(name, version=..., score=..., model=..., ...)` | Records a score and optionally `test_id`, `run_id`, `output`, `metadata`, and an existing `created_at`. |
|
|
190
|
+
| `run_tests(name, model=..., generate=..., ...)` | Renders active cases, calls the generator, and stores the scored results in one transaction. Select a version or tag and optionally pass `test_ids` to run a subset. |
|
|
191
|
+
| `await arun_tests(name, model=..., generate=..., ...)` | Awaits a text or message generator and supports a synchronous or asynchronous scorer. |
|
|
192
|
+
| `compare(name, versions=None, ...)` and `list_evals(name, ...)` | Read stored evaluations with filters for run ID, model, case, and date. |
|
|
193
|
+
| `delete(name, version)` | Removes one version, its tags, and its evaluations. Associated test cases remain attached to the prompt name. |
|
|
194
|
+
|
|
195
|
+
Placeholder names such as `{{city}}` and `{{ city }}` are detected in first-use order. If you pass `variables` explicitly, use the same set as in the template. Version numbers increase per name, including after deletion. Templates and model settings are immutable once saved. Use a new version for edits.
|
|
196
|
+
|
|
197
|
+
Pass an aware `datetime` for `created_at`, `since`, and `until`. The CLI accepts ISO 8601 timestamps with offsets, such as `2026-11-12T10:00:00-03:00`. Time windows include `since` and exclude `until`. The comparison uses the first requested version as the base of the diff. `report.best` uses the highest `average_score` across matching rows and breaks ties in comparison order. Versions without matching runs have an average of `None`. Compare the same model and a consistent set of cases when using a score as a gate.
|
|
198
|
+
|
|
199
|
+
A batch starts from the prompt and cases visible at the beginning of the run. Model calls and scoring happen outside the SQLite transaction. If rendering, generation, scoring, or persistence fails, the batch leaves no partial results. `run_tests()` assigns an ID to the batch by default. Pass the same `run_id` to the baseline and candidate when evaluating them together.
|
|
200
|
+
|
|
201
|
+
Use `arun_tests()` with an async client in an event loop. It keeps SQLite transactions out of provider `await` calls. Both runners accept `tag` to select a deployed version and `test_ids` to run selected active cases in the requested order. A comparison measures coverage against the cases eligible for that comparison.
|
|
202
|
+
|
|
203
|
+
### Maintaining test cases
|
|
204
|
+
|
|
205
|
+
Use `archive-test` to retire a case or `restore-test` to make it active again. Use `replace-test NAME ID --input ... --expected ...` to archive the old case and create a new case with its own ID in one transaction. `get-test` and `evals` still show the original case and scores. By default, `run_tests`, `tests`, and `compare` use active cases. Pass `include_archived=True` to `list_tests()` or `compare()`, or `--include-archived` to `tests` or `compare`, to include archived cases in an audit.
|
|
206
|
+
|
|
207
|
+
### Scoring structured outputs
|
|
208
|
+
|
|
209
|
+
Use `from promptdb.scorers import json_match` and pass `scorer=json_match` to `run_tests()` for a JSON expectation. Field order and insignificant whitespace are ignored. Array order and the difference between numbers and booleans matter. Malformed JSON receives a zero score and its output stays in the evaluation record.
|
|
210
|
+
|
|
211
|
+
To score output produced by another tool, pass `eval --test-id ID --scorer json --output-file response.json` with the prompt name, version, and model. Use `--output-file -` for stdin. The provider example also accepts `--scorer json`.
|
|
212
|
+
|
|
213
|
+
## JSON snapshots and SQLite backups
|
|
214
|
+
|
|
215
|
+
Use JSON when you want to inspect changes in Git, transfer the logical data, or load it into a new database.
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
promptdb --db prompts.db export > promptdb.json
|
|
219
|
+
promptdb --db restored.db import promptdb.json
|
|
220
|
+
promptdb --db restored.db history country
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
The snapshot includes templates, metadata, tags, inputs, expected values, outputs, dates, IDs, and version counters. The same database state produces the same UTF-8 JSON. Import requires a new or unused database and validates the complete snapshot in one transaction. Use `db.export_json()` to get a string and `db.import_json(payload)` to restore it from Python.
|
|
224
|
+
|
|
225
|
+
The JSON format is identified by `format: "evals-promptdb"` and `version: 4`. It stores chat messages as JSON arrays and preserves which cases are archived. Imports also accept formats 1–3. You can check a snapshot into Git and import it into a new SQLite file in CI.
|
|
226
|
+
|
|
227
|
+
Use SQLite backup when you want a standalone database file. The source can be open in other connections. The destination must be a new path in an existing directory on a filesystem that supports hard links.
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
promptdb --db prompts.db backup prompts-copy.db
|
|
231
|
+
promptdb --db prompts-copy.db history country
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
The equivalent API is `db.backup("prompts-copy.db")`. Open the copied file directly to use it. Use a separate `PromptDB` instance per thread. Set `PromptDB("prompts.db", timeout=30)` to configure the lock wait in seconds. Set `create=False` to require an existing database, or use `PromptDB(":memory:")` for a temporary database.
|
|
235
|
+
|
|
236
|
+
Use `PromptDB("prompts.db", read_only=True)` or `promptdb --read-only` for an existing, current-schema database that should stay untouched. Reads, comparisons, exports, and backups can use that connection. `db.check()` returns SQLite integrity and foreign-key findings.
|
|
237
|
+
|
|
238
|
+
```bash
|
|
239
|
+
promptdb --db empty.db init
|
|
240
|
+
promptdb --db prompts.db --read-only check
|
|
241
|
+
promptdb --db prompts.db --read-only export --output snapshot.json
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
`export --output` creates a new UTF-8 file without replacing an existing path. The CLI also uses UTF-8 for prompt and JSON pipes, and `import` accepts snapshots with a UTF-8 BOM.
|
|
245
|
+
|
|
246
|
+
## Provider examples
|
|
247
|
+
|
|
248
|
+
Run the Python quickstart first. From the checkout, `python -m examples.evaluate` runs saved cases over any number of versions with one shared run ID. It supports Ollama, OpenAI, Anthropic, OpenAI-compatible Chat Completions servers, and a deterministic uppercase function for offline tests.
|
|
249
|
+
|
|
250
|
+
```bash
|
|
251
|
+
python -m examples.evaluate ollama country 1 2 --model qwen2.5-coder:7b --run-id ollama-smoke
|
|
252
|
+
promptdb compare country 1 2 --model ollama/qwen2.5-coder:7b --run-id ollama-smoke --min-coverage 1
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
Ollama uses its local HTTP API and an installed model. Set `--ollama-host` to choose a server. OpenAI uses the optional `openai` SDK and `OPENAI_API_KEY`. Anthropic uses the optional `anthropic` SDK and `ANTHROPIC_API_KEY`. Pass `--model` with the ID for that provider. The script prints the model key and run ID to use with `promptdb compare`. [Ollama Chat API](https://docs.ollama.com/api/chat) · [Anthropic Messages API](https://platform.claude.com/docs/en/api/messages)
|
|
256
|
+
|
|
257
|
+
Use `--timeout` to allow for a cold start, `--max-tokens` to limit generated tokens, or `--ollama-context` to set the local context size.
|
|
258
|
+
|
|
259
|
+
Use `openai-compatible --api-base` with a base URL such as `http://127.0.0.1:8000/v1` for a server that exposes Chat Completions. Set `PROMPTDB_ENDPOINT_API_KEY` if that server requires a bearer token. The example sends developer messages as system instructions to Ollama and compatible servers. Anthropic takes system and developer instructions before the first conversation turn. [vLLM compatible server documentation](https://docs.vllm.ai/en/latest/serving/openai_compatible_server/)
|
|
260
|
+
|
|
261
|
+
### OpenAI from Python
|
|
262
|
+
|
|
263
|
+
Run the Python quickstart first. Install the optional SDK with `python -m pip install openai` and configure `OPENAI_API_KEY`. The generator below sends the rendered template to the Responses API. Evals PromptDB stores the returned text and score locally. [OpenAI Responses API reference](https://developers.openai.com/api/reference/python/resources/responses/methods/create).
|
|
264
|
+
|
|
265
|
+
```python
|
|
266
|
+
from openai import OpenAI
|
|
267
|
+
from promptdb import PromptDB
|
|
268
|
+
|
|
269
|
+
client = OpenAI()
|
|
270
|
+
model = "gpt-4o-mini"
|
|
271
|
+
|
|
272
|
+
def generate(text):
|
|
273
|
+
return client.responses.create(
|
|
274
|
+
model=model,
|
|
275
|
+
input=text,
|
|
276
|
+
store=False,
|
|
277
|
+
).output_text
|
|
278
|
+
|
|
279
|
+
with PromptDB("prompts.db", create=False) as db:
|
|
280
|
+
runs = db.run_tests("country", version=2, model=model, generate=generate)
|
|
281
|
+
for run in runs:
|
|
282
|
+
print(run.test_id, run.score, run.output)
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
## Development
|
|
286
|
+
|
|
287
|
+
Run the test suite with the standard library.
|
|
288
|
+
|
|
289
|
+
```bash
|
|
290
|
+
python -m unittest discover -s tests -v
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
GitHub Actions tests CPython 3.10–3.15 and PyPy 3.11 on Linux, CPython 3.10 and 3.15 on Windows, and CPython 3.15 on macOS. The wheel includes type annotations for consumers and has no required third-party runtime dependencies. Use the packaging tools below to build and check the distributions locally.
|
|
294
|
+
|
|
295
|
+
```bash
|
|
296
|
+
python -m pip install build twine
|
|
297
|
+
python -m build
|
|
298
|
+
python -m twine check --strict dist/*
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
[Changelog](CHANGELOG.md) · [MIT license](LICENSE)
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
evals_promptdb-1.2.0.dist-info/licenses/LICENSE,sha256=9mmbCs1opIuCp1-mNmqf-YQouhhdz-sSX6zosnQQYhg,1078
|
|
2
|
+
promptdb/__init__.py,sha256=6BJQe9HuMCDd5HESAiTqs82q1wcRiazHspBuyIkD_o4,492
|
|
3
|
+
promptdb/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
|
|
4
|
+
promptdb/_snapshots.py,sha256=1F-n0VFZQnrTevYmIg1xH2-uq0F4fAtbDCDnjuP0rTE,7179
|
|
5
|
+
promptdb/_values.py,sha256=_4Ng3NPsQpfNoDBzMLh5ub0rR2n8wLspdiBf48diVVM,4317
|
|
6
|
+
promptdb/_version.py,sha256=MpAT5hgNoHnTtG1XRD_GV_A7QrHVU6vJjGSw_8qMGA4,22
|
|
7
|
+
promptdb/cli.py,sha256=AxmyiNIaRTjgEiBp06JWS2Pfy1KZ8EJuID1hF6pCa_s,17314
|
|
8
|
+
promptdb/database.py,sha256=becsbjjNUotcWccyWKUZB0SBPTCPZHF7HzSpvUKDEGk,31971
|
|
9
|
+
promptdb/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
10
|
+
promptdb/records.py,sha256=euezOJpEUfSN8XUq3KLDZJZve6LrlZRMaXByPGDOUs0,3459
|
|
11
|
+
promptdb/schema.py,sha256=YsLUCW--dPH3cwiiAJaHwJ7tl2W-85H49o_I4PtDvBU,2322
|
|
12
|
+
promptdb/scorers.py,sha256=DEAqR7rLFSeUAgFmf1MWW3d9-uVFhPDV7x2EHV6Ao2w,1615
|
|
13
|
+
evals_promptdb-1.2.0.dist-info/METADATA,sha256=aGZ6Y-U_eRxf2yV2XUaYerfiSTAezi9_mboBd0guwRk,19149
|
|
14
|
+
evals_promptdb-1.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
15
|
+
evals_promptdb-1.2.0.dist-info/entry_points.txt,sha256=xU9HGet0aNKRtEdGnLPXMgPNPZ3eRM47DV5jhTnSlUE,47
|
|
16
|
+
evals_promptdb-1.2.0.dist-info/top_level.txt,sha256=yrn6rJlMLP8oQdT-BaDpC6hOq7Ct8NJmRuig5YzS-cI,9
|
|
17
|
+
evals_promptdb-1.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 PromptDB contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
promptdb
|
promptdb/__init__.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Local storage for versioned LLM prompts, test cases and evaluations."""
|
|
2
|
+
|
|
3
|
+
from ._version import __version__
|
|
4
|
+
from .database import PromptDB
|
|
5
|
+
from .records import (
|
|
6
|
+
CaseScore,
|
|
7
|
+
ChatMessage,
|
|
8
|
+
Comparison,
|
|
9
|
+
Evaluation,
|
|
10
|
+
PairedScore,
|
|
11
|
+
Prompt,
|
|
12
|
+
TestCase,
|
|
13
|
+
VersionComparison,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"CaseScore",
|
|
18
|
+
"ChatMessage",
|
|
19
|
+
"Comparison",
|
|
20
|
+
"Evaluation",
|
|
21
|
+
"PairedScore",
|
|
22
|
+
"Prompt",
|
|
23
|
+
"PromptDB",
|
|
24
|
+
"TestCase",
|
|
25
|
+
"VersionComparison",
|
|
26
|
+
"__version__",
|
|
27
|
+
]
|
promptdb/__main__.py
ADDED
promptdb/_snapshots.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import sqlite3
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from ._values import (
|
|
7
|
+
chat_messages,
|
|
8
|
+
chat_template,
|
|
9
|
+
evaluation_output,
|
|
10
|
+
json_object,
|
|
11
|
+
json_value,
|
|
12
|
+
label,
|
|
13
|
+
message_variables,
|
|
14
|
+
number,
|
|
15
|
+
positive_integer,
|
|
16
|
+
template_variables,
|
|
17
|
+
timestamp,
|
|
18
|
+
unique_json_object,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
FORMAT = "evals-promptdb"
|
|
22
|
+
FORMAT_VERSION = 4
|
|
23
|
+
|
|
24
|
+
# Foreign-key dependencies determine import order. Each order-by key is unique.
|
|
25
|
+
TABLES = (
|
|
26
|
+
("prompt_names", "name last_version", "name"),
|
|
27
|
+
("prompts", "name version template variables model temperature metadata created_at kind", "name, version"),
|
|
28
|
+
("tags", "name version tag", "name, version, tag"),
|
|
29
|
+
("test_cases", "id name input expected created_at archived_at", "id"),
|
|
30
|
+
("evaluations", "id name version score model test_id output metadata created_at run_id", "id"),
|
|
31
|
+
)
|
|
32
|
+
JSON_COLUMNS = {
|
|
33
|
+
"prompts": ("variables", "metadata"),
|
|
34
|
+
"test_cases": ("input", "expected"),
|
|
35
|
+
"evaluations": ("metadata",),
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def encode(connection: sqlite3.Connection) -> str:
|
|
40
|
+
sequence = connection.execute("SELECT seq FROM sqlite_sequence WHERE name = 'evaluations'").fetchone()
|
|
41
|
+
snapshot: dict[str, Any] = {
|
|
42
|
+
"format": FORMAT,
|
|
43
|
+
"version": FORMAT_VERSION,
|
|
44
|
+
"last_evaluation_id": 0 if sequence is None else sequence[0],
|
|
45
|
+
}
|
|
46
|
+
for table, columns, order in TABLES:
|
|
47
|
+
selection = ", ".join(columns.split())
|
|
48
|
+
records = []
|
|
49
|
+
for row in connection.execute(f"SELECT {selection} FROM {table} ORDER BY {order}"):
|
|
50
|
+
record = dict(row)
|
|
51
|
+
for column in JSON_COLUMNS.get(table, ()):
|
|
52
|
+
record[column] = json.loads(record[column])
|
|
53
|
+
if table == "prompts" and record["kind"] == "chat":
|
|
54
|
+
record["template"] = json.loads(record["template"])
|
|
55
|
+
records.append(record)
|
|
56
|
+
snapshot[table] = records
|
|
57
|
+
return json.dumps(snapshot, ensure_ascii=False, allow_nan=False, sort_keys=True, indent=2) + "\n"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def decode(payload: str) -> dict[str, Any]:
|
|
61
|
+
snapshot = json.loads(payload, object_pairs_hook=unique_json_object)
|
|
62
|
+
keys = {"format", "version", "last_evaluation_id", *(table for table, _, _ in TABLES)}
|
|
63
|
+
if not isinstance(snapshot, dict) or snapshot.keys() != keys:
|
|
64
|
+
raise ValueError("Invalid snapshot structure")
|
|
65
|
+
version = snapshot["version"]
|
|
66
|
+
if snapshot["format"] != FORMAT or type(version) is not int or not 1 <= version <= FORMAT_VERSION:
|
|
67
|
+
raise ValueError("Unsupported snapshot format or version")
|
|
68
|
+
last_eval = snapshot["last_evaluation_id"]
|
|
69
|
+
if type(last_eval) is not int or not 0 <= last_eval <= 2**63 - 1:
|
|
70
|
+
raise ValueError("last_evaluation_id must be a non-negative 64-bit integer")
|
|
71
|
+
return snapshot
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _row_values(table: str, columns: list[str], record: Any) -> tuple[Any, ...]:
|
|
75
|
+
if not isinstance(record, dict) or record.keys() != set(columns):
|
|
76
|
+
raise ValueError("Unexpected record fields")
|
|
77
|
+
row = dict(record)
|
|
78
|
+
label(row["name"], "name")
|
|
79
|
+
if "version" in row:
|
|
80
|
+
positive_integer(row["version"], "version")
|
|
81
|
+
if "id" in row:
|
|
82
|
+
positive_integer(row["id"], "id")
|
|
83
|
+
if "created_at" in row:
|
|
84
|
+
row["created_at"] = timestamp(datetime.fromisoformat(row["created_at"]))
|
|
85
|
+
if "metadata" in row:
|
|
86
|
+
row["metadata"] = json_object(row["metadata"], "metadata")
|
|
87
|
+
|
|
88
|
+
match table:
|
|
89
|
+
case "prompt_names":
|
|
90
|
+
positive_integer(row["last_version"], "last_version")
|
|
91
|
+
case "prompts":
|
|
92
|
+
kind = row.get("kind", "text")
|
|
93
|
+
if kind == "chat":
|
|
94
|
+
if not isinstance(row["template"], list):
|
|
95
|
+
raise ValueError("A chat template must be a JSON array")
|
|
96
|
+
messages = chat_messages(row["template"])
|
|
97
|
+
detected = message_variables(messages)
|
|
98
|
+
row["template"] = chat_template(messages)
|
|
99
|
+
elif kind == "text":
|
|
100
|
+
detected = template_variables(row["template"])
|
|
101
|
+
else:
|
|
102
|
+
raise ValueError("Prompt kind must be text or chat")
|
|
103
|
+
if row["variables"] != list(detected):
|
|
104
|
+
raise ValueError("variables must match the detected variables in order")
|
|
105
|
+
row["variables"] = json_value(detected)
|
|
106
|
+
if row["model"] is not None:
|
|
107
|
+
label(row["model"], "model")
|
|
108
|
+
if row["temperature"] is not None:
|
|
109
|
+
row["temperature"] = number(row["temperature"], 2, "temperature")
|
|
110
|
+
case "tags":
|
|
111
|
+
label(row["tag"], "tag")
|
|
112
|
+
case "test_cases":
|
|
113
|
+
row["input"] = json_object(row["input"], "input")
|
|
114
|
+
row["expected"] = json_value(row["expected"])
|
|
115
|
+
if row.get("archived_at") is not None:
|
|
116
|
+
row["archived_at"] = timestamp(datetime.fromisoformat(row["archived_at"]))
|
|
117
|
+
case "evaluations":
|
|
118
|
+
row["score"] = number(row["score"], 1, "score")
|
|
119
|
+
label(row["model"], "model")
|
|
120
|
+
if row["test_id"] is not None:
|
|
121
|
+
positive_integer(row["test_id"], "test_id")
|
|
122
|
+
if row.get("run_id") is not None:
|
|
123
|
+
label(row["run_id"], "run_id")
|
|
124
|
+
evaluation_output(row["output"])
|
|
125
|
+
|
|
126
|
+
return tuple(row[column] for column in columns)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def restore(connection: sqlite3.Connection, snapshot: dict[str, Any]) -> None:
|
|
130
|
+
if connection.execute("SELECT 1 FROM prompt_names LIMIT 1").fetchone() is not None:
|
|
131
|
+
raise ValueError("Import requires a new or unused PromptDB database")
|
|
132
|
+
for table, fields, _ in TABLES:
|
|
133
|
+
records = snapshot[table]
|
|
134
|
+
if not isinstance(records, list):
|
|
135
|
+
raise ValueError(f"{table} must be an array")
|
|
136
|
+
columns = fields.split()
|
|
137
|
+
if table == "prompts" and snapshot["version"] < 3:
|
|
138
|
+
columns.remove("kind")
|
|
139
|
+
if table == "test_cases" and snapshot["version"] < 4:
|
|
140
|
+
columns.remove("archived_at")
|
|
141
|
+
if table == "evaluations" and snapshot["version"] == 1:
|
|
142
|
+
columns.remove("run_id")
|
|
143
|
+
names = ", ".join(columns)
|
|
144
|
+
parameters = ", ".join("?" for _ in columns)
|
|
145
|
+
insert = f"INSERT INTO {table} ({names}) VALUES ({parameters})"
|
|
146
|
+
for index, record in enumerate(records, start=1):
|
|
147
|
+
try:
|
|
148
|
+
connection.execute(insert, _row_values(table, columns, record))
|
|
149
|
+
except (TypeError, ValueError, OverflowError, sqlite3.IntegrityError) as exc:
|
|
150
|
+
raise ValueError(f"Invalid {table} record {index}: {exc}") from exc
|
|
151
|
+
|
|
152
|
+
if connection.execute("""
|
|
153
|
+
SELECT 1 FROM prompts p JOIN prompt_names n USING (name) WHERE p.version > n.last_version LIMIT 1
|
|
154
|
+
""").fetchone() is not None:
|
|
155
|
+
raise ValueError("last_version cannot be less than a saved version")
|
|
156
|
+
|
|
157
|
+
maximum_id = connection.execute("SELECT COALESCE(MAX(id), 0) FROM evaluations").fetchone()[0]
|
|
158
|
+
last_eval = snapshot["last_evaluation_id"]
|
|
159
|
+
if last_eval < maximum_id:
|
|
160
|
+
raise ValueError("last_evaluation_id cannot be less than a saved evaluation ID")
|
|
161
|
+
if last_eval:
|
|
162
|
+
connection.execute("DELETE FROM sqlite_sequence WHERE name = 'evaluations'")
|
|
163
|
+
connection.execute("INSERT INTO sqlite_sequence(name, seq) VALUES ('evaluations', ?)", (last_eval,))
|
promptdb/_values.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from collections.abc import Iterable, Mapping, Set
|
|
3
|
+
from datetime import datetime, timezone
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from .records import PLACEHOLDER, ChatMessage
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def label(value: str, field: str) -> str:
|
|
10
|
+
if not isinstance(value, str) or not value.strip() or "\0" in value:
|
|
11
|
+
raise ValueError(f"{field} must be a non-empty string without null characters")
|
|
12
|
+
return value
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def labels(values: Iterable[str], field: str) -> tuple[str, ...]:
|
|
16
|
+
if isinstance(values, str):
|
|
17
|
+
raise ValueError(f"{field} must be a collection of strings")
|
|
18
|
+
return tuple(dict.fromkeys(label(value, field) for value in values))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def positive_integer(value: int, field: str) -> int:
|
|
22
|
+
if type(value) is not int or not 1 <= value <= 2**63 - 1:
|
|
23
|
+
raise ValueError(f"{field} must be a positive 64-bit integer")
|
|
24
|
+
return value
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def number(value: float, maximum: float, field: str) -> float:
|
|
28
|
+
if type(value) not in (int, float) or not 0 <= value <= maximum:
|
|
29
|
+
raise ValueError(f"{field} must be a number between 0 and {maximum:g}")
|
|
30
|
+
return float(value)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def json_value(value: Any) -> str:
|
|
34
|
+
return json.dumps(value, ensure_ascii=False, allow_nan=False)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def json_object(value: Mapping[str, Any], field: str) -> str:
|
|
38
|
+
if not isinstance(value, Mapping) or any(not isinstance(key, str) for key in value):
|
|
39
|
+
raise ValueError(f"{field} must be a JSON object with string keys")
|
|
40
|
+
return json_value(dict(value))
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def unique_json_object(fields: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
44
|
+
result = dict(fields)
|
|
45
|
+
if len(result) != len(fields):
|
|
46
|
+
raise ValueError("Duplicate field in JSON")
|
|
47
|
+
return result
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def timestamp(value: datetime | None = None) -> str:
|
|
51
|
+
if value is None:
|
|
52
|
+
value = datetime.now(timezone.utc)
|
|
53
|
+
if not isinstance(value, datetime) or value.utcoffset() is None:
|
|
54
|
+
raise ValueError("Timestamps must be timezone-aware datetimes")
|
|
55
|
+
return value.astimezone(timezone.utc).isoformat(timespec="microseconds")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _detect_variables(texts: Iterable[str], variables: Iterable[str] | None) -> tuple[str, ...]:
|
|
59
|
+
detected = tuple(dict.fromkeys(name for text in texts for name in PLACEHOLDER.findall(text)))
|
|
60
|
+
if variables is not None and set(labels(variables, "variables")) != set(detected):
|
|
61
|
+
raise ValueError("variables must match the placeholders in the template")
|
|
62
|
+
return detected
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def template_variables(template: str, variables: Iterable[str] | None = None) -> tuple[str, ...]:
|
|
66
|
+
if not isinstance(template, str) or not template.strip():
|
|
67
|
+
raise ValueError("template must be a non-empty string")
|
|
68
|
+
return _detect_variables((template,), variables)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def chat_messages(messages: Iterable[ChatMessage | Mapping[str, str]]) -> tuple[ChatMessage, ...]:
|
|
72
|
+
if isinstance(messages, (str, bytes, Mapping, Set)) or not isinstance(messages, Iterable):
|
|
73
|
+
raise ValueError("messages must be an ordered collection of role/content messages")
|
|
74
|
+
result = []
|
|
75
|
+
for message in messages:
|
|
76
|
+
if isinstance(message, ChatMessage):
|
|
77
|
+
role, content = message.role, message.content
|
|
78
|
+
elif isinstance(message, Mapping) and message.keys() == {"role", "content"}:
|
|
79
|
+
role, content = message["role"], message["content"]
|
|
80
|
+
else:
|
|
81
|
+
raise ValueError("Each chat message must have role and content fields")
|
|
82
|
+
if role not in ("system", "developer", "user", "assistant") or not isinstance(content, str):
|
|
83
|
+
raise ValueError("Chat messages require a system, developer, user or assistant role and string content")
|
|
84
|
+
result.append(ChatMessage(role, content))
|
|
85
|
+
if not result:
|
|
86
|
+
raise ValueError("A chat prompt needs at least one message")
|
|
87
|
+
return tuple(result)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def message_variables(messages: Iterable[ChatMessage], variables: Iterable[str] | None = None) -> tuple[str, ...]:
|
|
91
|
+
return _detect_variables((message.content for message in messages), variables)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def chat_template(messages: Iterable[ChatMessage]) -> str:
|
|
95
|
+
return json.dumps([{"role": message.role, "content": message.content} for message in messages],
|
|
96
|
+
ensure_ascii=False, indent=2)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def evaluation_output(value: str | None) -> str | None:
|
|
100
|
+
if value is not None and not isinstance(value, str):
|
|
101
|
+
raise ValueError("output must be a string")
|
|
102
|
+
return value
|
promptdb/_version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.2.0"
|