theoremql 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- theoremql-0.3.1/PKG-INFO +203 -0
- theoremql-0.3.1/README.md +173 -0
- theoremql-0.3.1/pyproject.toml +80 -0
- theoremql-0.3.1/pyproject.toml.orig +64 -0
- theoremql-0.3.1/src/theorem/__init__.py +54 -0
- theoremql-0.3.1/src/theorem/__main__.py +3 -0
- theoremql-0.3.1/src/theorem/ast_nodes.py +175 -0
- theoremql-0.3.1/src/theorem/canonical.py +200 -0
- theoremql-0.3.1/src/theorem/cli.py +367 -0
- theoremql-0.3.1/src/theorem/engine/__init__.py +0 -0
- theoremql-0.3.1/src/theorem/engine/dedup.py +107 -0
- theoremql-0.3.1/src/theorem/engine/executor.py +1154 -0
- theoremql-0.3.1/src/theorem/engine/health.py +24 -0
- theoremql-0.3.1/src/theorem/engine/storage.py +693 -0
- theoremql-0.3.1/src/theorem/engine/text.py +53 -0
- theoremql-0.3.1/src/theorem/engine/writes.py +500 -0
- theoremql-0.3.1/src/theorem/ingest/__init__.py +4 -0
- theoremql-0.3.1/src/theorem/ingest/bulk.py +213 -0
- theoremql-0.3.1/src/theorem/ingest/chunk.py +88 -0
- theoremql-0.3.1/src/theorem/ingest/envelope.py +31 -0
- theoremql-0.3.1/src/theorem/ingest/extract.py +125 -0
- theoremql-0.3.1/src/theorem/ingest/normalize.py +393 -0
- theoremql-0.3.1/src/theorem/ingest/playbook.py +199 -0
- theoremql-0.3.1/src/theorem/ingest/runners.py +141 -0
- theoremql-0.3.1/src/theorem/ingest/sniff.py +118 -0
- theoremql-0.3.1/src/theorem/ingest/stage.py +196 -0
- theoremql-0.3.1/src/theorem/parser.py +772 -0
- theoremql-0.3.1/src/theorem/prompt.py +224 -0
- theoremql-0.3.1/src/theorem/py.typed +0 -0
- theoremql-0.3.1/src/theorem/schema.py +151 -0
- theoremql-0.3.1/src/theorem/session.py +278 -0
- theoremql-0.3.1/src/theorem/verifier.py +518 -0
theoremql-0.3.1/PKG-INFO
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: theoremql
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: A graph query and construction language for AI agents. Every query verified whole before it runs.
|
|
5
|
+
Keywords: graph,query-language,ai-agents,llm,knowledge-graph,cypher-alternative
|
|
6
|
+
Author: VishiATChoudhary
|
|
7
|
+
Author-email: VishiATChoudhary <vishisht.choudhary@tum.de>
|
|
8
|
+
License-Expression: Apache-2.0
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Topic :: Database :: Front-Ends
|
|
16
|
+
Classifier: Topic :: Software Development :: Interpreters
|
|
17
|
+
Requires-Dist: python-docx>=1.1 ; extra == 'office'
|
|
18
|
+
Requires-Dist: openpyxl>=3.1 ; extra == 'office'
|
|
19
|
+
Requires-Dist: python-pptx>=1.0 ; extra == 'office'
|
|
20
|
+
Requires-Dist: pdfplumber>=0.11 ; extra == 'pdf'
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Project-URL: Homepage, https://github.com/VishiATChoudhary/theorem
|
|
23
|
+
Project-URL: Documentation, https://vishiatchoudhary.github.io/theorem/
|
|
24
|
+
Project-URL: Repository, https://github.com/VishiATChoudhary/theorem
|
|
25
|
+
Project-URL: Issues, https://github.com/VishiATChoudhary/theorem/issues
|
|
26
|
+
Project-URL: Changelog, https://github.com/VishiATChoudhary/theorem/blob/main/CHANGELOG.md
|
|
27
|
+
Provides-Extra: office
|
|
28
|
+
Provides-Extra: pdf
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
<p align="center">
|
|
32
|
+
<img src="https://raw.githubusercontent.com/VishiATChoudhary/theorem/main/docs/assets/wordmark.svg" alt="theorem" width="340">
|
|
33
|
+
</p>
|
|
34
|
+
|
|
35
|
+
<p align="center"><b>A graph language agents can't get wrong.</b><br>
|
|
36
|
+
Every query is verified whole against the live schema before anything runs.</p>
|
|
37
|
+
|
|
38
|
+
<p align="center">
|
|
39
|
+
<a href="https://vishiatchoudhary.github.io/theorem/">Documentation</a> ·
|
|
40
|
+
<a href="https://vishiatchoudhary.github.io/theorem/tutorial/">Tutorial</a> ·
|
|
41
|
+
<a href="https://vishiatchoudhary.github.io/theorem/using-theorem/">Use it in a project</a> ·
|
|
42
|
+
<a href="https://vishiatchoudhary.github.io/theorem/benchmarks/">Benchmarks</a>
|
|
43
|
+
</p>
|
|
44
|
+
|
|
45
|
+
<p align="center">
|
|
46
|
+
<a href="https://github.com/VishiATChoudhary/theorem/actions/workflows/test.yml"><img src="https://github.com/VishiATChoudhary/theorem/actions/workflows/test.yml/badge.svg" alt="CI"></a>
|
|
47
|
+
<img src="https://img.shields.io/badge/python-3.11%20|%203.12%20|%203.13-blue" alt="Python versions">
|
|
48
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-green" alt="License"></a>
|
|
49
|
+
</p>
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
LLMs get roughly **40% of Cypher queries wrong** on realistic schemas. Two model generations of scaling have not fixed it: frontier models still sit at 51-61% execution accuracy. The failure modes are structural: reversed arrows, hallucinated labels, implicit grouping, long-range brackets.
|
|
54
|
+
|
|
55
|
+
**theorem** removes each failure mode by construction. On the full public CypherBench test set, all 2,348 questions, the small Haiku model writing theorem reaches **78.0%** execution accuracy where the same model writing Cypher reaches **70.4%**, ahead on all seven graphs.
|
|
56
|
+
|
|
57
|
+
<p align="center">
|
|
58
|
+
<img src="https://raw.githubusercontent.com/VishiATChoudhary/theorem/main/docs/assets/demo.gif" alt="theorem REPL: a typo is caught before execution with a suggestion; a five-step pipeline aggregates suppliers" width="720">
|
|
59
|
+
</p>
|
|
60
|
+
|
|
61
|
+
## Install
|
|
62
|
+
|
|
63
|
+
Not on PyPI yet. Install from the repository, pinning a commit if you want
|
|
64
|
+
the language and the storage format to hold still:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
pip install "git+https://github.com/VishiATChoudhary/theorem.git@main"
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
The core package has no dependencies. On PyPI the distribution is named
|
|
71
|
+
**`theoremql`**, because PyPI prohibits `theorem`; the module you import
|
|
72
|
+
and the command you run are `theorem` either way.
|
|
73
|
+
|
|
74
|
+
Run a program or an interactive session:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
theorem program.thm --db ./db
|
|
78
|
+
theorem --repl
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Or embed it, and let a model write the queries:
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
from theorem import Schema, Session, answer
|
|
85
|
+
|
|
86
|
+
with Session("./db", Schema()) as db:
|
|
87
|
+
db.execute("derive class supplier from entity with {country: str}")
|
|
88
|
+
got = answer(db, "Which suppliers are in Germany?", your_model)
|
|
89
|
+
print(got.rows, got.turns, got.errors)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
`answer` is the loop the benchmarks measure: write, run, and on an error hand the error back verbatim and try again. Nothing is ever partly applied, so a repair is a fresh attempt rather than a cleanup.
|
|
93
|
+
|
|
94
|
+
Use `execute` (raises) and `rows` (raises, reads only) from your own code, and
|
|
95
|
+
`run` (renders the error as text) when a model is the caller. Getting that choice
|
|
96
|
+
right is the whole of [Using theorem in a project](docs/using-theorem.md), which
|
|
97
|
+
also covers schemas, locking, bulk loading and query limits.
|
|
98
|
+
|
|
99
|
+
## Sixty seconds of theorem
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
find product where launch_year > 2024 as recent
|
|
103
|
+
follow recent uses component as parts
|
|
104
|
+
follow parts supplied_by source as sups
|
|
105
|
+
group by sups as g
|
|
106
|
+
count distinct g.parts as n_parts
|
|
107
|
+
return sups.name, n_parts order by n_parts desc budget 2000 tokens
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
One line, one step, one name. Reading it top to bottom is the whole mental model.
|
|
111
|
+
|
|
112
|
+
## Ingest anything
|
|
113
|
+
|
|
114
|
+
Any file, not just CSVs, lands as queryable graph nodes with page-level provenance: markdown, text, CSV, JSON/JSONL, PDFs, docx/xlsx/pptx, and images (stored with metadata) all normalize into document, chunk, table, and media nodes joined by `part_of` and traceable back to their source page. CSVs stay first-class and refinable as before. For domain-specific schemas, a playbook is a markdown file describing a use case in prose; any agent CLI compiles it into verified `derive class`/`derive edge` statements with lineage back to the playbook.
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
pip install "theoremql[pdf,office] @ git+https://github.com/VishiATChoudhary/theorem.git@main"
|
|
118
|
+
theorem ingest report.pdf --db ./db
|
|
119
|
+
theorem playbook compile playbook.md --db ./db --agent claude
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
For data whose columns you already know, skip the model entirely:
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
theorem load parts.csv --db ./db --class part
|
|
126
|
+
theorem load links.csv --db ./db --edge supplied_by --role item=part --role source=supplier
|
|
127
|
+
theorem stats --db ./db
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
`--role` maps a role to the **column** naming the node that fills it, so
|
|
131
|
+
`links.csv` there has columns `part` and `supplier`. A program that fails
|
|
132
|
+
exits non-zero, so `theorem build.thm --db ./db && deploy` does what it looks
|
|
133
|
+
like it does.
|
|
134
|
+
|
|
135
|
+
## Why agents stop failing
|
|
136
|
+
|
|
137
|
+
- **No direction glyphs.** Edges traverse by role name: `follow parts supplied_by source as sups`. A wrong role is a type error caught before execution, not a silently empty result, except where an edge's two roles hold the same class and no schema check can distinguish them. That exception is [6.1% of our own broken queries](docs/benchmarks/silent-failure.md), and it is the honest ceiling on the headline of this README.
|
|
138
|
+
- **One line, one step, one name.** No chaining, no nesting, nothing to balance.
|
|
139
|
+
- **Explicit staged aggregation.** `group by sups as g`, then `count distinct g.parts as n`. Adding a return column can never change the grouping.
|
|
140
|
+
- **Schema-closed vocabulary.** Every query is verified whole against the live schema before anything runs; errors name the line, suggest the fix, and confirm nothing executed.
|
|
141
|
+
- **Token budgets.** `budget 2000 tokens` caps serialized results with explicit truncation and `continue @c...` handles.
|
|
142
|
+
|
|
143
|
+
And a write surface no existing query language has: `assert` with provenance, receipts carrying dedup candidates, `merge`/`distinct` resolution, `refine`/`compact` granularity verbs with full lineage, `retire`, `flag`, `derive class`, and queryable per-node health (`find nodes where health.loss > 0.8`).
|
|
144
|
+
|
|
145
|
+
## Benchmarks
|
|
146
|
+
|
|
147
|
+
The complete public [CypherBench](https://github.com/megagonlabs/cypherbench) test set (ACL 2025): all 2,348 questions across all 7 test graphs, every match category, the full unsampled graphs, zero-shot with one generation and no repair retry, scored with the benchmark's own execution-accuracy comparator against its published answers.
|
|
148
|
+
|
|
149
|
+
| Condition | EX | Multi-hop EX | Executable | Mean result tokens |
|
|
150
|
+
|-----------|---:|-------------:|-----------:|-------------------:|
|
|
151
|
+
| theorem + Haiku 4.5 | **78.0%** | **78.7%** | 96.6% | **167** |
|
|
152
|
+
| text2cypher + Haiku 4.5 | 70.4% | 69.6% | 95.3% | 242 |
|
|
153
|
+
| text2cypher + Claude 3.5 Sonnet (published) | 61.6% | — | 96.3% | — |
|
|
154
|
+
| text2cypher + GPT-4o (published) | 60.2% | — | 94.9% | — |
|
|
155
|
+
|
|
156
|
+
The text2cypher row is a control, not a citation: same model, same questions, same comparator, the official zero-shot prompt, executed on the official Neo4j image. Excluding `nba`, the one graph theorem's prompt was written against, theorem scores 76.9%. Median execution latency is 0.2 ms against 67 ms over Bolt.
|
|
157
|
+
|
|
158
|
+
theorem's prompt carries a tutorial the model has never seen, so it costs more per question on these graphs, which have 9 to 13 classes each. It costs 39 tokens per class against text2cypher's 85, and the lines cross at 31: on the seven schemas unioned, 40 classes, theorem's prompt is the smaller one ([prompt cost](docs/benchmarks/prompt-cost.md)).
|
|
159
|
+
|
|
160
|
+
Full method, per-category results and the caveats that matter, including the prompt asymmetry between the two arms, are in [docs/benchmarks/cypherbench.md](docs/benchmarks/cypherbench.md). That benchmark measures one-shot translation; for convergence under retry and tokens across a whole agent loop, on graphs nothing was tuned on, see [docs/benchmarks/agent-loop.md](docs/benchmarks/agent-loop.md). Per-question queries and errors for both arms are in `eval/out/public/`.
|
|
161
|
+
|
|
162
|
+
Reproduce (needs the `claude` CLI, and docker for the text2cypher control):
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
uv run python -m eval.run_public all --model claude-haiku-4-5-20251001
|
|
166
|
+
uv run python -m eval.run_cypher_public all --model claude-haiku-4-5-20251001
|
|
167
|
+
uv run python -m eval.make_report
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
## Development
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
git clone https://github.com/VishiATChoudhary/theorem
|
|
174
|
+
cd theorem
|
|
175
|
+
uv sync
|
|
176
|
+
uv run pytest -q # unit, property-based, and end-to-end deployment tests
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
| Path | What |
|
|
180
|
+
|------|------|
|
|
181
|
+
| `src/theorem/parser.py` | Tokenizer + line-oriented parser |
|
|
182
|
+
| `src/theorem/verifier.py` | Whole-program verify-before-execute |
|
|
183
|
+
| `src/theorem/engine/storage.py` | WAL, automatic compaction, one-writer lock |
|
|
184
|
+
| `src/theorem/engine/executor.py` | Binding-table reads, budgets, serialization |
|
|
185
|
+
| `src/theorem/engine/writes.py` | Structural writes with receipts |
|
|
186
|
+
| `src/theorem/engine/dedup.py` | Blocking + similarity dedup pipeline |
|
|
187
|
+
| `src/theorem/engine/health.py` | Four health subscores |
|
|
188
|
+
| `src/theorem/session.py` | Session facade (parse, verify, execute, rows) |
|
|
189
|
+
| `src/theorem/prompt.py` | The prompt and agent loop the benchmarks measure |
|
|
190
|
+
| `src/theorem/ingest/bulk.py` | CSV/JSONL bulk load |
|
|
191
|
+
| `eval/` | CypherBench, agent-loop, frontier, broken-query and prompt-cost harnesses |
|
|
192
|
+
| `skills/theorem/` | The agent skill: how to use the language, for a model |
|
|
193
|
+
| `docs/language-spec.md` | Normative grammar and semantics |
|
|
194
|
+
|
|
195
|
+
## Community
|
|
196
|
+
|
|
197
|
+
theorem is a community project under Apache-2.0. The language grows spec-first: proposals are discussed as issues before syntax lands ([how it works](CONTRIBUTING.md)).
|
|
198
|
+
|
|
199
|
+
- [CHANGELOG.md](CHANGELOG.md): what changed in each release, and why
|
|
200
|
+
- [RELEASING.md](RELEASING.md): how a version is cut, and the PyPI setup that is still pending
|
|
201
|
+
- [CONTRIBUTING.md](CONTRIBUTING.md): setup, test loop, DCO sign-off
|
|
202
|
+
- [ROADMAP.md](ROADMAP.md): every open objective states the number that closes it
|
|
203
|
+
- [Good first issues](https://github.com/VishiATChoudhary/theorem/labels/good%20first%20issue)
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/VishiATChoudhary/theorem/main/docs/assets/wordmark.svg" alt="theorem" width="340">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center"><b>A graph language agents can't get wrong.</b><br>
|
|
6
|
+
Every query is verified whole against the live schema before anything runs.</p>
|
|
7
|
+
|
|
8
|
+
<p align="center">
|
|
9
|
+
<a href="https://vishiatchoudhary.github.io/theorem/">Documentation</a> ·
|
|
10
|
+
<a href="https://vishiatchoudhary.github.io/theorem/tutorial/">Tutorial</a> ·
|
|
11
|
+
<a href="https://vishiatchoudhary.github.io/theorem/using-theorem/">Use it in a project</a> ·
|
|
12
|
+
<a href="https://vishiatchoudhary.github.io/theorem/benchmarks/">Benchmarks</a>
|
|
13
|
+
</p>
|
|
14
|
+
|
|
15
|
+
<p align="center">
|
|
16
|
+
<a href="https://github.com/VishiATChoudhary/theorem/actions/workflows/test.yml"><img src="https://github.com/VishiATChoudhary/theorem/actions/workflows/test.yml/badge.svg" alt="CI"></a>
|
|
17
|
+
<img src="https://img.shields.io/badge/python-3.11%20|%203.12%20|%203.13-blue" alt="Python versions">
|
|
18
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-Apache--2.0-green" alt="License"></a>
|
|
19
|
+
</p>
|
|
20
|
+
|
|
21
|
+
---
|
|
22
|
+
|
|
23
|
+
LLMs get roughly **40% of Cypher queries wrong** on realistic schemas. Two model generations of scaling have not fixed it: frontier models still sit at 51-61% execution accuracy. The failure modes are structural: reversed arrows, hallucinated labels, implicit grouping, long-range brackets.
|
|
24
|
+
|
|
25
|
+
**theorem** removes each failure mode by construction. On the full public CypherBench test set, all 2,348 questions, the small Haiku model writing theorem reaches **78.0%** execution accuracy where the same model writing Cypher reaches **70.4%**, ahead on all seven graphs.
|
|
26
|
+
|
|
27
|
+
<p align="center">
|
|
28
|
+
<img src="https://raw.githubusercontent.com/VishiATChoudhary/theorem/main/docs/assets/demo.gif" alt="theorem REPL: a typo is caught before execution with a suggestion; a five-step pipeline aggregates suppliers" width="720">
|
|
29
|
+
</p>
|
|
30
|
+
|
|
31
|
+
## Install
|
|
32
|
+
|
|
33
|
+
Not on PyPI yet. Install from the repository, pinning a commit if you want
|
|
34
|
+
the language and the storage format to hold still:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install "git+https://github.com/VishiATChoudhary/theorem.git@main"
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The core package has no dependencies. On PyPI the distribution is named
|
|
41
|
+
**`theoremql`**, because PyPI prohibits `theorem`; the module you import
|
|
42
|
+
and the command you run are `theorem` either way.
|
|
43
|
+
|
|
44
|
+
Run a program or an interactive session:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
theorem program.thm --db ./db
|
|
48
|
+
theorem --repl
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Or embed it, and let a model write the queries:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from theorem import Schema, Session, answer
|
|
55
|
+
|
|
56
|
+
with Session("./db", Schema()) as db:
|
|
57
|
+
db.execute("derive class supplier from entity with {country: str}")
|
|
58
|
+
got = answer(db, "Which suppliers are in Germany?", your_model)
|
|
59
|
+
print(got.rows, got.turns, got.errors)
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`answer` is the loop the benchmarks measure: write, run, and on an error hand the error back verbatim and try again. Nothing is ever partly applied, so a repair is a fresh attempt rather than a cleanup.
|
|
63
|
+
|
|
64
|
+
Use `execute` (raises) and `rows` (raises, reads only) from your own code, and
|
|
65
|
+
`run` (renders the error as text) when a model is the caller. Getting that choice
|
|
66
|
+
right is the whole of [Using theorem in a project](docs/using-theorem.md), which
|
|
67
|
+
also covers schemas, locking, bulk loading and query limits.
|
|
68
|
+
|
|
69
|
+
## Sixty seconds of theorem
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
find product where launch_year > 2024 as recent
|
|
73
|
+
follow recent uses component as parts
|
|
74
|
+
follow parts supplied_by source as sups
|
|
75
|
+
group by sups as g
|
|
76
|
+
count distinct g.parts as n_parts
|
|
77
|
+
return sups.name, n_parts order by n_parts desc budget 2000 tokens
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
One line, one step, one name. Reading it top to bottom is the whole mental model.
|
|
81
|
+
|
|
82
|
+
## Ingest anything
|
|
83
|
+
|
|
84
|
+
Any file, not just CSVs, lands as queryable graph nodes with page-level provenance: markdown, text, CSV, JSON/JSONL, PDFs, docx/xlsx/pptx, and images (stored with metadata) all normalize into document, chunk, table, and media nodes joined by `part_of` and traceable back to their source page. CSVs stay first-class and refinable as before. For domain-specific schemas, a playbook is a markdown file describing a use case in prose; any agent CLI compiles it into verified `derive class`/`derive edge` statements with lineage back to the playbook.
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
pip install "theoremql[pdf,office] @ git+https://github.com/VishiATChoudhary/theorem.git@main"
|
|
88
|
+
theorem ingest report.pdf --db ./db
|
|
89
|
+
theorem playbook compile playbook.md --db ./db --agent claude
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
For data whose columns you already know, skip the model entirely:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
theorem load parts.csv --db ./db --class part
|
|
96
|
+
theorem load links.csv --db ./db --edge supplied_by --role item=part --role source=supplier
|
|
97
|
+
theorem stats --db ./db
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
`--role` maps a role to the **column** naming the node that fills it, so
|
|
101
|
+
`links.csv` there has columns `part` and `supplier`. A program that fails
|
|
102
|
+
exits non-zero, so `theorem build.thm --db ./db && deploy` does what it looks
|
|
103
|
+
like it does.
|
|
104
|
+
|
|
105
|
+
## Why agents stop failing
|
|
106
|
+
|
|
107
|
+
- **No direction glyphs.** Edges traverse by role name: `follow parts supplied_by source as sups`. A wrong role is a type error caught before execution, not a silently empty result, except where an edge's two roles hold the same class and no schema check can distinguish them. That exception is [6.1% of our own broken queries](docs/benchmarks/silent-failure.md), and it is the honest ceiling on the headline of this README.
|
|
108
|
+
- **One line, one step, one name.** No chaining, no nesting, nothing to balance.
|
|
109
|
+
- **Explicit staged aggregation.** `group by sups as g`, then `count distinct g.parts as n`. Adding a return column can never change the grouping.
|
|
110
|
+
- **Schema-closed vocabulary.** Every query is verified whole against the live schema before anything runs; errors name the line, suggest the fix, and confirm nothing executed.
|
|
111
|
+
- **Token budgets.** `budget 2000 tokens` caps serialized results with explicit truncation and `continue @c...` handles.
|
|
112
|
+
|
|
113
|
+
And a write surface no existing query language has: `assert` with provenance, receipts carrying dedup candidates, `merge`/`distinct` resolution, `refine`/`compact` granularity verbs with full lineage, `retire`, `flag`, `derive class`, and queryable per-node health (`find nodes where health.loss > 0.8`).
|
|
114
|
+
|
|
115
|
+
## Benchmarks
|
|
116
|
+
|
|
117
|
+
The complete public [CypherBench](https://github.com/megagonlabs/cypherbench) test set (ACL 2025): all 2,348 questions across all 7 test graphs, every match category, the full unsampled graphs, zero-shot with one generation and no repair retry, scored with the benchmark's own execution-accuracy comparator against its published answers.
|
|
118
|
+
|
|
119
|
+
| Condition | EX | Multi-hop EX | Executable | Mean result tokens |
|
|
120
|
+
|-----------|---:|-------------:|-----------:|-------------------:|
|
|
121
|
+
| theorem + Haiku 4.5 | **78.0%** | **78.7%** | 96.6% | **167** |
|
|
122
|
+
| text2cypher + Haiku 4.5 | 70.4% | 69.6% | 95.3% | 242 |
|
|
123
|
+
| text2cypher + Claude 3.5 Sonnet (published) | 61.6% | — | 96.3% | — |
|
|
124
|
+
| text2cypher + GPT-4o (published) | 60.2% | — | 94.9% | — |
|
|
125
|
+
|
|
126
|
+
The text2cypher row is a control, not a citation: same model, same questions, same comparator, the official zero-shot prompt, executed on the official Neo4j image. Excluding `nba`, the one graph theorem's prompt was written against, theorem scores 76.9%. Median execution latency is 0.2 ms against 67 ms over Bolt.
|
|
127
|
+
|
|
128
|
+
theorem's prompt carries a tutorial the model has never seen, so it costs more per question on these graphs, which have 9 to 13 classes each. It costs 39 tokens per class against text2cypher's 85, and the lines cross at 31: on the seven schemas unioned, 40 classes, theorem's prompt is the smaller one ([prompt cost](docs/benchmarks/prompt-cost.md)).
|
|
129
|
+
|
|
130
|
+
Full method, per-category results and the caveats that matter, including the prompt asymmetry between the two arms, are in [docs/benchmarks/cypherbench.md](docs/benchmarks/cypherbench.md). That benchmark measures one-shot translation; for convergence under retry and tokens across a whole agent loop, on graphs nothing was tuned on, see [docs/benchmarks/agent-loop.md](docs/benchmarks/agent-loop.md). Per-question queries and errors for both arms are in `eval/out/public/`.
|
|
131
|
+
|
|
132
|
+
Reproduce (needs the `claude` CLI, and docker for the text2cypher control):
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
uv run python -m eval.run_public all --model claude-haiku-4-5-20251001
|
|
136
|
+
uv run python -m eval.run_cypher_public all --model claude-haiku-4-5-20251001
|
|
137
|
+
uv run python -m eval.make_report
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## Development
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
git clone https://github.com/VishiATChoudhary/theorem
|
|
144
|
+
cd theorem
|
|
145
|
+
uv sync
|
|
146
|
+
uv run pytest -q # unit, property-based, and end-to-end deployment tests
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
| Path | What |
|
|
150
|
+
|------|------|
|
|
151
|
+
| `src/theorem/parser.py` | Tokenizer + line-oriented parser |
|
|
152
|
+
| `src/theorem/verifier.py` | Whole-program verify-before-execute |
|
|
153
|
+
| `src/theorem/engine/storage.py` | WAL, automatic compaction, one-writer lock |
|
|
154
|
+
| `src/theorem/engine/executor.py` | Binding-table reads, budgets, serialization |
|
|
155
|
+
| `src/theorem/engine/writes.py` | Structural writes with receipts |
|
|
156
|
+
| `src/theorem/engine/dedup.py` | Blocking + similarity dedup pipeline |
|
|
157
|
+
| `src/theorem/engine/health.py` | Four health subscores |
|
|
158
|
+
| `src/theorem/session.py` | Session facade (parse, verify, execute, rows) |
|
|
159
|
+
| `src/theorem/prompt.py` | The prompt and agent loop the benchmarks measure |
|
|
160
|
+
| `src/theorem/ingest/bulk.py` | CSV/JSONL bulk load |
|
|
161
|
+
| `eval/` | CypherBench, agent-loop, frontier, broken-query and prompt-cost harnesses |
|
|
162
|
+
| `skills/theorem/` | The agent skill: how to use the language, for a model |
|
|
163
|
+
| `docs/language-spec.md` | Normative grammar and semantics |
|
|
164
|
+
|
|
165
|
+
## Community
|
|
166
|
+
|
|
167
|
+
theorem is a community project under Apache-2.0. The language grows spec-first: proposals are discussed as issues before syntax lands ([how it works](CONTRIBUTING.md)).
|
|
168
|
+
|
|
169
|
+
- [CHANGELOG.md](CHANGELOG.md): what changed in each release, and why
|
|
170
|
+
- [RELEASING.md](RELEASING.md): how a version is cut, and the PyPI setup that is still pending
|
|
171
|
+
- [CONTRIBUTING.md](CONTRIBUTING.md): setup, test loop, DCO sign-off
|
|
172
|
+
- [ROADMAP.md](ROADMAP.md): every open objective states the number that closes it
|
|
173
|
+
- [Good first issues](https://github.com/VishiATChoudhary/theorem/labels/good%20first%20issue)
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "theoremql"
|
|
3
|
+
version = "0.3.1"
|
|
4
|
+
description = "A graph query and construction language for AI agents. Every query verified whole before it runs."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "Apache-2.0"
|
|
7
|
+
requires-python = ">=3.11"
|
|
8
|
+
dependencies = []
|
|
9
|
+
keywords = [
|
|
10
|
+
"graph",
|
|
11
|
+
"query-language",
|
|
12
|
+
"ai-agents",
|
|
13
|
+
"llm",
|
|
14
|
+
"knowledge-graph",
|
|
15
|
+
"cypher-alternative",
|
|
16
|
+
]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 4 - Beta",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Topic :: Database :: Front-Ends",
|
|
25
|
+
"Topic :: Software Development :: Interpreters",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[[project.authors]]
|
|
29
|
+
name = "VishiATChoudhary"
|
|
30
|
+
email = "vishisht.choudhary@tum.de"
|
|
31
|
+
|
|
32
|
+
[project.urls]
|
|
33
|
+
Homepage = "https://github.com/VishiATChoudhary/theorem"
|
|
34
|
+
Documentation = "https://vishiatchoudhary.github.io/theorem/"
|
|
35
|
+
Repository = "https://github.com/VishiATChoudhary/theorem"
|
|
36
|
+
Issues = "https://github.com/VishiATChoudhary/theorem/issues"
|
|
37
|
+
Changelog = "https://github.com/VishiATChoudhary/theorem/blob/main/CHANGELOG.md"
|
|
38
|
+
|
|
39
|
+
[project.optional-dependencies]
|
|
40
|
+
pdf = ["pdfplumber>=0.11"]
|
|
41
|
+
office = [
|
|
42
|
+
"python-docx>=1.1",
|
|
43
|
+
"openpyxl>=3.1",
|
|
44
|
+
"python-pptx>=1.0",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
[project.scripts]
|
|
48
|
+
theorem = "theorem.cli:main"
|
|
49
|
+
|
|
50
|
+
[tool.uv.build-backend]
|
|
51
|
+
module-name = "theorem"
|
|
52
|
+
|
|
53
|
+
[tool.pytest.ini_options]
|
|
54
|
+
pythonpath = ["."]
|
|
55
|
+
|
|
56
|
+
[tool.ruff]
|
|
57
|
+
target-version = "py311"
|
|
58
|
+
|
|
59
|
+
[tool.ruff.lint]
|
|
60
|
+
select = [
|
|
61
|
+
"E4",
|
|
62
|
+
"E7",
|
|
63
|
+
"E9",
|
|
64
|
+
"F",
|
|
65
|
+
"B",
|
|
66
|
+
"RUF",
|
|
67
|
+
"FURB",
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
[build-system]
|
|
71
|
+
requires = ["uv_build>=0.9.15,<0.10.0"]
|
|
72
|
+
build-backend = "uv_build"
|
|
73
|
+
|
|
74
|
+
[dependency-groups]
|
|
75
|
+
dev = [
|
|
76
|
+
"hypothesis>=6.165.10",
|
|
77
|
+
"matplotlib>=3.11.1",
|
|
78
|
+
"neo4j>=6.3.0",
|
|
79
|
+
"pytest>=9.1.1",
|
|
80
|
+
]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "theoremql"
|
|
3
|
+
version = "0.3.1"
|
|
4
|
+
description = "A graph query and construction language for AI agents. Every query verified whole before it runs."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "Apache-2.0"
|
|
7
|
+
authors = [
|
|
8
|
+
{ name = "VishiATChoudhary", email = "vishisht.choudhary@tum.de" }
|
|
9
|
+
]
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
dependencies = []
|
|
12
|
+
keywords = ["graph", "query-language", "ai-agents", "llm", "knowledge-graph", "cypher-alternative"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 4 - Beta",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Programming Language :: Python :: 3.13",
|
|
20
|
+
"Topic :: Database :: Front-Ends",
|
|
21
|
+
"Topic :: Software Development :: Interpreters",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/VishiATChoudhary/theorem"
|
|
26
|
+
Documentation = "https://vishiatchoudhary.github.io/theorem/"
|
|
27
|
+
Repository = "https://github.com/VishiATChoudhary/theorem"
|
|
28
|
+
Issues = "https://github.com/VishiATChoudhary/theorem/issues"
|
|
29
|
+
Changelog = "https://github.com/VishiATChoudhary/theorem/blob/main/CHANGELOG.md"
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
pdf = ["pdfplumber>=0.11"]
|
|
33
|
+
office = ["python-docx>=1.1", "openpyxl>=3.1", "python-pptx>=1.0"]
|
|
34
|
+
|
|
35
|
+
[project.scripts]
|
|
36
|
+
theorem = "theorem.cli:main"
|
|
37
|
+
|
|
38
|
+
# The distribution is `theoremql` because PyPI prohibits the name
|
|
39
|
+
# `theorem`, but the module stays `theorem`: that is what the docs, the
|
|
40
|
+
# benchmarks and every existing program import. uv infers the module name
|
|
41
|
+
# from the project name, so it has to be told.
|
|
42
|
+
[tool.uv.build-backend]
|
|
43
|
+
module-name = "theorem"
|
|
44
|
+
|
|
45
|
+
[build-system]
|
|
46
|
+
requires = ["uv_build>=0.9.15,<0.10.0"]
|
|
47
|
+
build-backend = "uv_build"
|
|
48
|
+
|
|
49
|
+
[dependency-groups]
|
|
50
|
+
dev = [
|
|
51
|
+
"hypothesis>=6.165.10",
|
|
52
|
+
"matplotlib>=3.11.1",
|
|
53
|
+
"neo4j>=6.3.0",
|
|
54
|
+
"pytest>=9.1.1",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
[tool.pytest.ini_options]
|
|
58
|
+
pythonpath = ["."]
|
|
59
|
+
|
|
60
|
+
[tool.ruff]
|
|
61
|
+
target-version = "py311"
|
|
62
|
+
|
|
63
|
+
[tool.ruff.lint]
|
|
64
|
+
select = ["E4", "E7", "E9", "F", "B", "RUF", "FURB"]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""theorem: a graph query and construction language agents can't get wrong.
|
|
2
|
+
|
|
3
|
+
The whole surface an embedding application needs:
|
|
4
|
+
|
|
5
|
+
from theorem import Schema, Session
|
|
6
|
+
|
|
7
|
+
with Session("mydb", Schema()) as db:
|
|
8
|
+
print(db.run('derive class supplier from entity with {country: str}'))
|
|
9
|
+
print(db.run('assert supplier {name: "VoltaChem", country: "DE"} as v'))
|
|
10
|
+
print(db.run('find supplier where country = "DE" as s\\nreturn s.name'))
|
|
11
|
+
|
|
12
|
+
`Schema()` is the base schema: `entity` to derive domain classes from,
|
|
13
|
+
plus the document classes the ingest pipeline uses. `Schema.supply_chain()`
|
|
14
|
+
adds the demo classes the tutorial is written against.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from .canonical import CanonicalError, canonical
|
|
18
|
+
from .engine.executor import ExecError, Limits, limits
|
|
19
|
+
from .engine.storage import Store, StoreError, StoreLocked
|
|
20
|
+
from .ingest.bulk import LoadError, load_edges, load_nodes
|
|
21
|
+
from .parser import ParseError, parse
|
|
22
|
+
from .prompt import Answer, agent_prompt, answer, repair_prompt
|
|
23
|
+
from .schema import ClassDef, EdgeDef, Schema
|
|
24
|
+
from .session import Session
|
|
25
|
+
from .verifier import VerifyError, verify
|
|
26
|
+
|
|
27
|
+
__version__ = "0.3.1"
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
"Answer",
|
|
31
|
+
"CanonicalError",
|
|
32
|
+
"ClassDef",
|
|
33
|
+
"EdgeDef",
|
|
34
|
+
"ExecError",
|
|
35
|
+
"Limits",
|
|
36
|
+
"LoadError",
|
|
37
|
+
"ParseError",
|
|
38
|
+
"Schema",
|
|
39
|
+
"Session",
|
|
40
|
+
"Store",
|
|
41
|
+
"StoreError",
|
|
42
|
+
"StoreLocked",
|
|
43
|
+
"VerifyError",
|
|
44
|
+
"__version__",
|
|
45
|
+
"agent_prompt",
|
|
46
|
+
"answer",
|
|
47
|
+
"canonical",
|
|
48
|
+
"limits",
|
|
49
|
+
"load_edges",
|
|
50
|
+
"load_nodes",
|
|
51
|
+
"parse",
|
|
52
|
+
"repair_prompt",
|
|
53
|
+
"verify",
|
|
54
|
+
]
|