text2sql-engine 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. text2sql_engine-0.1.0/LICENSE +21 -0
  2. text2sql_engine-0.1.0/PKG-INFO +304 -0
  3. text2sql_engine-0.1.0/README.md +270 -0
  4. text2sql_engine-0.1.0/pyproject.toml +71 -0
  5. text2sql_engine-0.1.0/setup.cfg +4 -0
  6. text2sql_engine-0.1.0/tests/test_config.py +91 -0
  7. text2sql_engine-0.1.0/tests/test_pipeline.py +142 -0
  8. text2sql_engine-0.1.0/tests/test_prompts.py +51 -0
  9. text2sql_engine-0.1.0/tests/test_schema.py +69 -0
  10. text2sql_engine-0.1.0/text2sql/__init__.py +67 -0
  11. text2sql_engine-0.1.0/text2sql/config.py +251 -0
  12. text2sql_engine-0.1.0/text2sql/errors.py +43 -0
  13. text2sql_engine-0.1.0/text2sql/logging_utils.py +54 -0
  14. text2sql_engine-0.1.0/text2sql/pipeline/__init__.py +9 -0
  15. text2sql_engine-0.1.0/text2sql/pipeline/generation.py +110 -0
  16. text2sql_engine-0.1.0/text2sql/pipeline/json_utils.py +108 -0
  17. text2sql_engine-0.1.0/text2sql/pipeline/linking.py +206 -0
  18. text2sql_engine-0.1.0/text2sql/pipeline/pipeline.py +156 -0
  19. text2sql_engine-0.1.0/text2sql/prompts/__init__.py +96 -0
  20. text2sql_engine-0.1.0/text2sql/prompts/generation.txt +28 -0
  21. text2sql_engine-0.1.0/text2sql/prompts/linking.txt +28 -0
  22. text2sql_engine-0.1.0/text2sql/providers/__init__.py +19 -0
  23. text2sql_engine-0.1.0/text2sql/providers/anthropic_provider.py +123 -0
  24. text2sql_engine-0.1.0/text2sql/providers/base.py +131 -0
  25. text2sql_engine-0.1.0/text2sql/providers/openai_provider.py +125 -0
  26. text2sql_engine-0.1.0/text2sql/providers/registry.py +38 -0
  27. text2sql_engine-0.1.0/text2sql/py.typed +0 -0
  28. text2sql_engine-0.1.0/text2sql/schema/__init__.py +18 -0
  29. text2sql_engine-0.1.0/text2sql/schema/file_provider.py +140 -0
  30. text2sql_engine-0.1.0/text2sql/schema/models.py +84 -0
  31. text2sql_engine-0.1.0/text2sql/schema/provider.py +30 -0
  32. text2sql_engine-0.1.0/text2sql/schema/serialize.py +55 -0
  33. text2sql_engine-0.1.0/text2sql/types.py +138 -0
  34. text2sql_engine-0.1.0/text2sql_engine.egg-info/PKG-INFO +304 -0
  35. text2sql_engine-0.1.0/text2sql_engine.egg-info/SOURCES.txt +36 -0
  36. text2sql_engine-0.1.0/text2sql_engine.egg-info/dependency_links.txt +1 -0
  37. text2sql_engine-0.1.0/text2sql_engine.egg-info/requires.txt +15 -0
  38. text2sql_engine-0.1.0/text2sql_engine.egg-info/top_level.txt +1 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 text2sql contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,304 @@
1
+ Metadata-Version: 2.4
2
+ Name: text2sql-engine
3
+ Version: 0.1.0
4
+ Summary: Schema-agnostic, config-driven Text-to-SQL library (2-stage LLM pipeline). Produces a SQL string; never connects to or executes against a database.
5
+ Author: stajyer14
6
+ License: MIT
7
+ Keywords: text-to-sql,nl2sql,llm,sql-generation,schema-linking
8
+ Classifier: Development Status :: 4 - Beta
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Topic :: Database
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Classifier: Topic :: Software Development :: Libraries
18
+ Classifier: Typing :: Typed
19
+ Requires-Python: >=3.11
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Requires-Dist: python-dotenv>=1.0
23
+ Requires-Dist: PyYAML>=6.0
24
+ Provides-Extra: openai
25
+ Requires-Dist: openai>=1.0; extra == "openai"
26
+ Provides-Extra: anthropic
27
+ Requires-Dist: anthropic>=0.34; extra == "anthropic"
28
+ Provides-Extra: all
29
+ Requires-Dist: openai>=1.0; extra == "all"
30
+ Requires-Dist: anthropic>=0.34; extra == "all"
31
+ Provides-Extra: dev
32
+ Requires-Dist: pytest>=7.4; extra == "dev"
33
+ Dynamic: license-file
34
+
35
+ # text2sql
36
+
37
+ A **schema-agnostic, config-driven Text-to-SQL library**. Give it a
38
+ natural-language request and metadata for any relational schema, and it returns
39
+ a SQL string.
40
+
41
+ It is a **library**, not an app. No UI, no web server, no CLI. It **never
42
+ connects to or runs against a database**. Generating the SQL is the last step.
43
+ Running it is the caller's job.
44
+
45
+ ## How it works — a 2-stage LLM pipeline
46
+
47
+ ```
48
+ user request
49
+
50
+
51
+ ┌─────────────────────────────┐ full schema metadata
52
+ │ Stage 1: Schema Linking LLM │◄──────────────────────
53
+ │ reduce schema → subset │
54
+ └─────────────────────────────┘
55
+ │ linked schema (tables, columns, joins, entities, filters, rationale)
56
+
57
+ ┌─────────────────────────────┐
58
+ │ Stage 2: SQL Generation LLM │
59
+ │ produce SQL in dialect │
60
+ └─────────────────────────────┘
61
+
62
+
63
+ Text2SQLResult(sql, linked_schema, metadata, explanation)
64
+ ```
65
+
66
+ - **Stage 1 (linking)** takes the request and the *full* schema metadata. It
67
+ reduces the schema to the relevant part and returns structured JSON. Output is
68
+ forced via `response_format` (OpenAI) or tool-calling (Anthropic), with
69
+ strict-JSON parsing and retry as a fallback.
70
+ - **Stage 2 (generation)** takes the request and that reduced subset. It writes
71
+ SQL in the target dialect, plus a short explanation.
72
+ - The two stages can use **different providers and models**. For example a cheap
73
+ general model for linking, a stronger SQL model for generation.
74
+
75
+ ## Installation
76
+
77
+ ```bash
78
+ pip install text2sql-engine # from PyPI (import name: text2sql)
79
+ pip install "text2sql-engine[all]" # + openai and anthropic SDKs
80
+ ```
81
+
82
+ The PyPI name is `text2sql-engine`. The import name is `text2sql`
83
+ (`import text2sql`). Different names on purpose, since `text2sql` was taken.
84
+
85
+ From a checkout of this repo:
86
+
87
+ ```bash
88
+ pip install -e . # core library
89
+ pip install -e ".[all]" # + openai and anthropic SDKs
90
+ pip install -e ".[dev]" # + pytest
91
+ ```
92
+
93
+ The provider SDKs (`openai`, `anthropic`) are imported lazily. So the library
94
+ imports, and the tests run, without them installed.
95
+
96
+ ## Quick start
97
+
98
+ ```python
99
+ from text2sql import Text2SQL
100
+
101
+ engine = Text2SQL.from_config() # reads .env + config.toml
102
+ result = engine.run("get me the top 5 best-selling products last month")
103
+
104
+ print(result.sql) # the generated SQL string
105
+ print(result.linked_schema) # Stage 1 output (reduced schema)
106
+ print(result.metadata) # models, tokens, latency, attempts
107
+ ```
108
+
109
+ 1. Copy `.env.example` to `.env`. Fill in provider selection and API keys.
110
+ 2. Adjust `config.toml` for behavior (dialect, sampling, retries).
111
+ 3. Point `SCHEMA_PATH` at your metadata file. See `examples/schema.yaml`.
112
+
113
+ ## Configuration — two separate layers
114
+
115
+ Config is split by concern. **Secrets, paths, and provider/model *selection* go
116
+ in `.env`. Everything tunable about *behavior* goes in `config.toml`.** One typed
117
+ layer (`text2sql.config.Settings`) loads both. It fails fast with a clear error
118
+ if a required `.env` variable is missing.
119
+
120
+ ### `.env` — secrets, paths, provider & model selection
121
+
122
+ | Variable | Required | Description |
123
+ |----------|----------|-------------|
124
+ | `SCHEMA_PATH` | yes | Path to the schema file (`.json`, `.yaml`, `.yml`). |
125
+ | `PROMPT_DIR` | no | Directory with prompt templates (`linking.txt`, `generation.txt`). If unset, the templates **packaged inside the library** are used. So it works after pip install, from any working directory. |
126
+ | `CONFIG_PATH` | no | Override the `config.toml` location. Defaults to `./config.toml`. |
127
+ | `LINKING_PROVIDER` | yes | Provider for Stage 1. One of the registered names: `openai`, `anthropic`. |
128
+ | `LINKING_MODEL` | yes | Model id for Stage 1 (schema linking). |
129
+ | `LINKING_BASE_URL` | no | Endpoint override for Stage 1 (for OpenAI-compatible gateways). |
130
+ | `SQL_PROVIDER` | yes | Provider for Stage 2. |
131
+ | `SQL_MODEL` | yes | Model id for Stage 2 (SQL generation). |
132
+ | `SQL_BASE_URL` | no | Endpoint override for Stage 2. |
133
+ | `OPENAI_API_KEY` | if used | API key. Needed only if a stage uses provider `openai`. |
134
+ | `ANTHROPIC_API_KEY` | if used | API key. Needed only if a stage uses provider `anthropic`. |
135
+
136
+ Provider, model, and `base_url` are picked **per stage**, independently.
137
+
138
+ ### `config.toml` — behavior
139
+
140
+ No magic numbers in code. Every tunable is here.
141
+
142
+ #### `[general]`
143
+
144
+ | Key | Type | Default | Description |
145
+ |-----|------|---------|-------------|
146
+ | `sql_dialect` | string | `postgres` | Target dialect (e.g. `postgres`, `mysql`, `sqlite`). Passed to the prompts. |
147
+ | `log_level` | string | `INFO` | `DEBUG` / `INFO` / `WARNING` / `ERROR`. |
148
+ | `strict_json` | bool | `true` | `true`: parse LLM JSON strictly. `false`: tolerate fences / extra prose. |
149
+
150
+ #### `[linking]` — Stage 1
151
+
152
+ | Key | Type | Default | Description |
153
+ |-----|------|---------|-------------|
154
+ | `temperature` | float | `0.0` | Sampling temperature. |
155
+ | `top_p` | float | `1.0` | Nucleus-sampling cutoff. |
156
+ | `max_tokens` | int | `1024` | Max tokens for the linking response. |
157
+ | `prompt_template` | string | `linking.txt` | Template file, looked up in `PROMPT_DIR`. |
158
+ | `max_tables` | int | `8` | Max tables linking may return. |
159
+ | `timeout_seconds` | float | `60` | Per-request timeout. |
160
+
161
+ #### `[generation]` — Stage 2
162
+
163
+ | Key | Type | Default | Description |
164
+ |-----|------|---------|-------------|
165
+ | `temperature` | float | `0.0` | Sampling temperature. |
166
+ | `top_p` | float | `1.0` | Nucleus-sampling cutoff. |
167
+ | `max_tokens` | int | `1024` | Max tokens for the SQL response. |
168
+ | `prompt_template` | string | `generation.txt` | Template file, looked up in `PROMPT_DIR`. |
169
+ | `timeout_seconds` | float | `60` | Per-request timeout. |
170
+
171
+ #### `[retry]` — shared by both stages
172
+
173
+ | Key | Type | Default | Description |
174
+ |-----|------|---------|-------------|
175
+ | `max_attempts` | int | `3` | Attempts per LLM call before failing. Retries on JSON-parse and transient provider errors. |
176
+ | `backoff_seconds` | float | `1.0` | First delay between retries. |
177
+ | `backoff_factor` | float | `2.0` | Delay is multiplied by this each retry. |
178
+
179
+ ## Schema metadata format
180
+
181
+ The schema is **never hardcoded**. It is loaded from an external file through a
182
+ `SchemaProvider`. `examples/schema.yaml` (and the same `examples/schema.json`)
183
+ show the full format: a small e-commerce database with `customers`,
184
+ `categories`, `products`, `orders`, and `order_items`. Each table has a
185
+ description. Each column has a name, type, description, primary-key flag, and
186
+ sample values. Foreign keys are listed. Both JSON and YAML work, picked by
187
+ extension.
188
+
189
+ The example runs out of the box. The tests exercise the pipeline against it with
190
+ mocked LLM calls.
191
+
192
+ ## Extending
193
+
194
+ Everything is swappable via **config + dependency injection**:
195
+
196
+ ```python
197
+ from text2sql import Text2SQL, SchemaProvider, PromptTemplates
198
+ from text2sql.schema import Schema
199
+
200
+ # 1. Custom schema source, e.g. live DB introspection instead of a file.
201
+ class MyIntrospectionProvider(SchemaProvider):
202
+ def load(self) -> Schema:
203
+ ... # build and return a Schema
204
+
205
+ # 2. Inject any part. Anything left None is built from config.
206
+ engine = Text2SQL.from_config(
207
+ schema_provider=MyIntrospectionProvider(),
208
+ prompts=PromptTemplates("/path/to/my/templates"),
209
+ # linking_provider=..., generation_provider=...,
210
+ )
211
+ ```
212
+
213
+ ### Adding a new LLM provider
214
+
215
+ Subclass `LLMProvider`, implement `complete` (and optionally `complete_json` for
216
+ native structured output), and register it. One class, one decorator:
217
+
218
+ ```python
219
+ from text2sql import LLMProvider, register_provider
220
+ from text2sql.types import LLMResponse
221
+
222
+ @register_provider("myprovider")
223
+ class MyProvider(LLMProvider):
224
+ def complete(self, *, system, user, temperature, top_p, max_tokens) -> LLMResponse:
225
+ ...
226
+ ```
227
+
228
+ Then set `LINKING_PROVIDER=myprovider` (or `SQL_PROVIDER`) in `.env`.
229
+
230
+ ### Prompts
231
+
232
+ Prompt templates are plain, editable text files in `PROMPT_DIR`. Each has a
233
+ `SYSTEM:` and a `USER:` section with `{placeholder}` substitution. Edit them
234
+ without touching code.
235
+
236
+ ## Result object
237
+
238
+ `engine.run(...)` returns a `Text2SQLResult`:
239
+
240
+ | Field | Description |
241
+ |-------|-------------|
242
+ | `sql` | The generated SQL string. The deliverable. |
243
+ | `explanation` | Short explanation from Stage 2. |
244
+ | `linked_schema` | Stage 1 output: `tables`, `columns`, `joins`, `entities`, `filters`, `rationale`. |
245
+ | `metadata` | `RunMetadata`: per-stage provider/model, token usage, latency, attempts. Plus `total_usage` and `total_latency_seconds`. |
246
+ | `request` | The original request. |
247
+
248
+ ## Error handling
249
+
250
+ Every error subclasses `Text2SQLError`:
251
+
252
+ - `ConfigError` — missing/invalid `.env` var or `config.toml`.
253
+ - `SchemaError` — schema file missing or invalid.
254
+ - `ProviderError` — provider build or API call failed.
255
+ - `StructuredOutputError` — output not parseable as JSON after all retries.
256
+ - `EmptyLinkingError` — Stage 1 matched no table/column.
257
+
258
+ Uses stdlib `logging` throughout, never `print`. Secrets are never logged.
259
+
260
+ ## Try it end to end
261
+
262
+ With a real provider and key set in `.env`:
263
+
264
+ ```bash
265
+ python examples/run_example.py "top 5 best-selling products last month"
266
+ ```
267
+
268
+ Makes real LLM calls for both stages. Prints the linked schema, the SQL, and the
269
+ metadata. Never connects to a database.
270
+
271
+ ## Development
272
+
273
+ ```bash
274
+ pip install -e ".[dev]"
275
+ pytest
276
+ ```
277
+
278
+ Tests mock all LLM calls (no network) and run on `examples/schema.yaml`.
279
+
280
+ ## Project layout
281
+
282
+ ```
283
+ text2sql/
284
+ config.py .env + config.toml loading, typed settings
285
+ types.py result dataclasses
286
+ errors.py exceptions
287
+ logging_utils.py logging setup
288
+ schema/ SchemaProvider, file loader, models, serializer
289
+ providers/ base + registry + openai + anthropic
290
+ prompts/ editable linking.txt / generation.txt templates
291
+ pipeline/ linking (Stage 1), generation (Stage 2), orchestrator
292
+ examples/
293
+ schema.yaml / schema.json sample metadata "spec"
294
+ run_example.py runnable end-to-end example (real LLM call)
295
+ config.toml behavioral parameters
296
+ .env.example secrets / paths / model selection template
297
+ tests/ pipeline, schema, config, and prompt tests
298
+ .github/workflows/ci.yml GitHub Actions: pytest on 3.11 and 3.12
299
+ LICENSE MIT
300
+ ```
301
+
302
+ ## License
303
+
304
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,270 @@
1
+ # text2sql
2
+
3
+ A **schema-agnostic, config-driven Text-to-SQL library**. Give it a
4
+ natural-language request and metadata for any relational schema, and it returns
5
+ a SQL string.
6
+
7
+ It is a **library**, not an app. No UI, no web server, no CLI. It **never
8
+ connects to or runs against a database**. Generating the SQL is the last step.
9
+ Running it is the caller's job.
10
+
11
+ ## How it works — a 2-stage LLM pipeline
12
+
13
+ ```
14
+ user request
15
+
16
+
17
+ ┌─────────────────────────────┐ full schema metadata
18
+ │ Stage 1: Schema Linking LLM │◄──────────────────────
19
+ │ reduce schema → subset │
20
+ └─────────────────────────────┘
21
+ │ linked schema (tables, columns, joins, entities, filters, rationale)
22
+
23
+ ┌─────────────────────────────┐
24
+ │ Stage 2: SQL Generation LLM │
25
+ │ produce SQL in dialect │
26
+ └─────────────────────────────┘
27
+
28
+
29
+ Text2SQLResult(sql, linked_schema, metadata, explanation)
30
+ ```
31
+
32
+ - **Stage 1 (linking)** takes the request and the *full* schema metadata. It
33
+ reduces the schema to the relevant part and returns structured JSON. Output is
34
+ forced via `response_format` (OpenAI) or tool-calling (Anthropic), with
35
+ strict-JSON parsing and retry as a fallback.
36
+ - **Stage 2 (generation)** takes the request and that reduced subset. It writes
37
+ SQL in the target dialect, plus a short explanation.
38
+ - The two stages can use **different providers and models**. For example a cheap
39
+ general model for linking, a stronger SQL model for generation.
40
+
41
+ ## Installation
42
+
43
+ ```bash
44
+ pip install text2sql-engine # from PyPI (import name: text2sql)
45
+ pip install "text2sql-engine[all]" # + openai and anthropic SDKs
46
+ ```
47
+
48
+ The PyPI name is `text2sql-engine`. The import name is `text2sql`
49
+ (`import text2sql`). Different names on purpose, since `text2sql` was taken.
50
+
51
+ From a checkout of this repo:
52
+
53
+ ```bash
54
+ pip install -e . # core library
55
+ pip install -e ".[all]" # + openai and anthropic SDKs
56
+ pip install -e ".[dev]" # + pytest
57
+ ```
58
+
59
+ The provider SDKs (`openai`, `anthropic`) are imported lazily. So the library
60
+ imports, and the tests run, without them installed.
61
+
62
+ ## Quick start
63
+
64
+ ```python
65
+ from text2sql import Text2SQL
66
+
67
+ engine = Text2SQL.from_config() # reads .env + config.toml
68
+ result = engine.run("get me the top 5 best-selling products last month")
69
+
70
+ print(result.sql) # the generated SQL string
71
+ print(result.linked_schema) # Stage 1 output (reduced schema)
72
+ print(result.metadata) # models, tokens, latency, attempts
73
+ ```
74
+
75
+ 1. Copy `.env.example` to `.env`. Fill in provider selection and API keys.
76
+ 2. Adjust `config.toml` for behavior (dialect, sampling, retries).
77
+ 3. Point `SCHEMA_PATH` at your metadata file. See `examples/schema.yaml`.
78
+
79
+ ## Configuration — two separate layers
80
+
81
+ Config is split by concern. **Secrets, paths, and provider/model *selection* go
82
+ in `.env`. Everything tunable about *behavior* goes in `config.toml`.** One typed
83
+ layer (`text2sql.config.Settings`) loads both. It fails fast with a clear error
84
+ if a required `.env` variable is missing.
85
+
86
+ ### `.env` — secrets, paths, provider & model selection
87
+
88
+ | Variable | Required | Description |
89
+ |----------|----------|-------------|
90
+ | `SCHEMA_PATH` | yes | Path to the schema file (`.json`, `.yaml`, `.yml`). |
91
+ | `PROMPT_DIR` | no | Directory with prompt templates (`linking.txt`, `generation.txt`). If unset, the templates **packaged inside the library** are used. So it works after pip install, from any working directory. |
92
+ | `CONFIG_PATH` | no | Override the `config.toml` location. Defaults to `./config.toml`. |
93
+ | `LINKING_PROVIDER` | yes | Provider for Stage 1. One of the registered names: `openai`, `anthropic`. |
94
+ | `LINKING_MODEL` | yes | Model id for Stage 1 (schema linking). |
95
+ | `LINKING_BASE_URL` | no | Endpoint override for Stage 1 (for OpenAI-compatible gateways). |
96
+ | `SQL_PROVIDER` | yes | Provider for Stage 2. |
97
+ | `SQL_MODEL` | yes | Model id for Stage 2 (SQL generation). |
98
+ | `SQL_BASE_URL` | no | Endpoint override for Stage 2. |
99
+ | `OPENAI_API_KEY` | if used | API key. Needed only if a stage uses provider `openai`. |
100
+ | `ANTHROPIC_API_KEY` | if used | API key. Needed only if a stage uses provider `anthropic`. |
101
+
102
+ Provider, model, and `base_url` are picked **per stage**, independently.
103
+
104
+ ### `config.toml` — behavior
105
+
106
+ No magic numbers in code. Every tunable is here.
107
+
108
+ #### `[general]`
109
+
110
+ | Key | Type | Default | Description |
111
+ |-----|------|---------|-------------|
112
+ | `sql_dialect` | string | `postgres` | Target dialect (e.g. `postgres`, `mysql`, `sqlite`). Passed to the prompts. |
113
+ | `log_level` | string | `INFO` | `DEBUG` / `INFO` / `WARNING` / `ERROR`. |
114
+ | `strict_json` | bool | `true` | `true`: parse LLM JSON strictly. `false`: tolerate fences / extra prose. |
115
+
116
+ #### `[linking]` — Stage 1
117
+
118
+ | Key | Type | Default | Description |
119
+ |-----|------|---------|-------------|
120
+ | `temperature` | float | `0.0` | Sampling temperature. |
121
+ | `top_p` | float | `1.0` | Nucleus-sampling cutoff. |
122
+ | `max_tokens` | int | `1024` | Max tokens for the linking response. |
123
+ | `prompt_template` | string | `linking.txt` | Template file, looked up in `PROMPT_DIR`. |
124
+ | `max_tables` | int | `8` | Max tables linking may return. |
125
+ | `timeout_seconds` | float | `60` | Per-request timeout. |
126
+
127
+ #### `[generation]` — Stage 2
128
+
129
+ | Key | Type | Default | Description |
130
+ |-----|------|---------|-------------|
131
+ | `temperature` | float | `0.0` | Sampling temperature. |
132
+ | `top_p` | float | `1.0` | Nucleus-sampling cutoff. |
133
+ | `max_tokens` | int | `1024` | Max tokens for the SQL response. |
134
+ | `prompt_template` | string | `generation.txt` | Template file, looked up in `PROMPT_DIR`. |
135
+ | `timeout_seconds` | float | `60` | Per-request timeout. |
136
+
137
+ #### `[retry]` — shared by both stages
138
+
139
+ | Key | Type | Default | Description |
140
+ |-----|------|---------|-------------|
141
+ | `max_attempts` | int | `3` | Attempts per LLM call before failing. Retries on JSON-parse and transient provider errors. |
142
+ | `backoff_seconds` | float | `1.0` | First delay between retries. |
143
+ | `backoff_factor` | float | `2.0` | Delay is multiplied by this each retry. |
144
+
145
+ ## Schema metadata format
146
+
147
+ The schema is **never hardcoded**. It is loaded from an external file through a
148
+ `SchemaProvider`. `examples/schema.yaml` (and the same `examples/schema.json`)
149
+ show the full format: a small e-commerce database with `customers`,
150
+ `categories`, `products`, `orders`, and `order_items`. Each table has a
151
+ description. Each column has a name, type, description, primary-key flag, and
152
+ sample values. Foreign keys are listed. Both JSON and YAML work, picked by
153
+ extension.
154
+
155
+ The example runs out of the box. The tests exercise the pipeline against it with
156
+ mocked LLM calls.
157
+
158
+ ## Extending
159
+
160
+ Everything is swappable via **config + dependency injection**:
161
+
162
+ ```python
163
+ from text2sql import Text2SQL, SchemaProvider, PromptTemplates
164
+ from text2sql.schema import Schema
165
+
166
+ # 1. Custom schema source, e.g. live DB introspection instead of a file.
167
+ class MyIntrospectionProvider(SchemaProvider):
168
+ def load(self) -> Schema:
169
+ ... # build and return a Schema
170
+
171
+ # 2. Inject any part. Anything left None is built from config.
172
+ engine = Text2SQL.from_config(
173
+ schema_provider=MyIntrospectionProvider(),
174
+ prompts=PromptTemplates("/path/to/my/templates"),
175
+ # linking_provider=..., generation_provider=...,
176
+ )
177
+ ```
178
+
179
+ ### Adding a new LLM provider
180
+
181
+ Subclass `LLMProvider`, implement `complete` (and optionally `complete_json` for
182
+ native structured output), and register it. One class, one decorator:
183
+
184
+ ```python
185
+ from text2sql import LLMProvider, register_provider
186
+ from text2sql.types import LLMResponse
187
+
188
+ @register_provider("myprovider")
189
+ class MyProvider(LLMProvider):
190
+ def complete(self, *, system, user, temperature, top_p, max_tokens) -> LLMResponse:
191
+ ...
192
+ ```
193
+
194
+ Then set `LINKING_PROVIDER=myprovider` (or `SQL_PROVIDER`) in `.env`.
195
+
196
+ ### Prompts
197
+
198
+ Prompt templates are plain, editable text files in `PROMPT_DIR`. Each has a
199
+ `SYSTEM:` and a `USER:` section with `{placeholder}` substitution. Edit them
200
+ without touching code.
201
+
202
+ ## Result object
203
+
204
+ `engine.run(...)` returns a `Text2SQLResult`:
205
+
206
+ | Field | Description |
207
+ |-------|-------------|
208
+ | `sql` | The generated SQL string. The deliverable. |
209
+ | `explanation` | Short explanation from Stage 2. |
210
+ | `linked_schema` | Stage 1 output: `tables`, `columns`, `joins`, `entities`, `filters`, `rationale`. |
211
+ | `metadata` | `RunMetadata`: per-stage provider/model, token usage, latency, attempts. Plus `total_usage` and `total_latency_seconds`. |
212
+ | `request` | The original request. |
213
+
214
+ ## Error handling
215
+
216
+ Every error subclasses `Text2SQLError`:
217
+
218
+ - `ConfigError` — missing/invalid `.env` var or `config.toml`.
219
+ - `SchemaError` — schema file missing or invalid.
220
+ - `ProviderError` — provider build or API call failed.
221
+ - `StructuredOutputError` — output not parseable as JSON after all retries.
222
+ - `EmptyLinkingError` — Stage 1 matched no table/column.
223
+
224
+ Uses stdlib `logging` throughout, never `print`. Secrets are never logged.
225
+
226
+ ## Try it end to end
227
+
228
+ With a real provider and key set in `.env`:
229
+
230
+ ```bash
231
+ python examples/run_example.py "top 5 best-selling products last month"
232
+ ```
233
+
234
+ Makes real LLM calls for both stages. Prints the linked schema, the SQL, and the
235
+ metadata. Never connects to a database.
236
+
237
+ ## Development
238
+
239
+ ```bash
240
+ pip install -e ".[dev]"
241
+ pytest
242
+ ```
243
+
244
+ Tests mock all LLM calls (no network) and run on `examples/schema.yaml`.
245
+
246
+ ## Project layout
247
+
248
+ ```
249
+ text2sql/
250
+ config.py .env + config.toml loading, typed settings
251
+ types.py result dataclasses
252
+ errors.py exceptions
253
+ logging_utils.py logging setup
254
+ schema/ SchemaProvider, file loader, models, serializer
255
+ providers/ base + registry + openai + anthropic
256
+ prompts/ editable linking.txt / generation.txt templates
257
+ pipeline/ linking (Stage 1), generation (Stage 2), orchestrator
258
+ examples/
259
+ schema.yaml / schema.json sample metadata "spec"
260
+ run_example.py runnable end-to-end example (real LLM call)
261
+ config.toml behavioral parameters
262
+ .env.example secrets / paths / model selection template
263
+ tests/ pipeline, schema, config, and prompt tests
264
+ .github/workflows/ci.yml GitHub Actions: pytest on 3.11 and 3.12
265
+ LICENSE MIT
266
+ ```
267
+
268
+ ## License
269
+
270
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,71 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ # Distribution name on PyPI. The IMPORT name stays `text2sql`:
7
+ # pip install text2sql-engine -> import text2sql
8
+ name = "text2sql-engine"
9
+ version = "0.1.0"
10
+ description = "Schema-agnostic, config-driven Text-to-SQL library (2-stage LLM pipeline). Produces a SQL string; never connects to or executes against a database."
11
+ readme = "README.md"
12
+ requires-python = ">=3.11"
13
+ license = { text = "MIT" }
14
+ authors = [{ name = "stajyer14" }]
15
+ keywords = ["text-to-sql", "nl2sql", "llm", "sql-generation", "schema-linking"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Intended Audience :: Developers",
19
+ "License :: OSI Approved :: MIT License",
20
+ "Operating System :: OS Independent",
21
+ "Programming Language :: Python :: 3",
22
+ "Programming Language :: Python :: 3.11",
23
+ "Programming Language :: Python :: 3.12",
24
+ "Topic :: Database",
25
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
26
+ "Topic :: Software Development :: Libraries",
27
+ "Typing :: Typed",
28
+ ]
29
+
30
+ # Core runtime dependencies. The provider SDKs (openai, anthropic) are imported
31
+ # lazily inside their provider modules, so the library imports and tests run
32
+ # without them installed.
33
+ dependencies = [
34
+ "python-dotenv>=1.0",
35
+ "PyYAML>=6.0",
36
+ ]
37
+
38
+ [project.optional-dependencies]
39
+ openai = ["openai>=1.0"]
40
+ anthropic = ["anthropic>=0.34"]
41
+ # Everything needed to talk to both providers.
42
+ all = ["openai>=1.0", "anthropic>=0.34"]
43
+ # Test toolchain.
44
+ dev = ["pytest>=7.4"]
45
+
46
+ # Replace OWNER with your GitHub username/org once the repo is pushed, then
47
+ # uncomment. Dead links look worse than no links, so this stays commented out
48
+ # until the repo exists.
49
+ # [project.urls]
50
+ # Homepage = "https://github.com/OWNER/text2sql-engine"
51
+ # Repository = "https://github.com/OWNER/text2sql-engine"
52
+ # Issues = "https://github.com/OWNER/text2sql-engine/issues"
53
+
54
+ [tool.setuptools.packages.find]
55
+ include = ["text2sql*"]
56
+
57
+ [tool.setuptools.package-data]
58
+ "text2sql" = ["prompts/*.txt", "py.typed"]
59
+
60
+ [tool.setuptools]
61
+ license-files = ["LICENSE"]
62
+
63
+ [tool.pytest.ini_options]
64
+ testpaths = ["tests"]
65
+
66
+ [tool.ruff]
67
+ line-length = 100
68
+ target-version = "py311"
69
+
70
+ [tool.ruff.lint]
71
+ select = ["E", "F", "I", "UP", "B"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+