pyoq-sql 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (264) hide show
  1. pyoq/__init__.py +7 -0
  2. pyoq/__main__.py +5 -0
  3. pyoq/_native.pyi +5 -0
  4. pyoq/cli/__init__.py +5 -0
  5. pyoq/cli/commands.py +270 -0
  6. pyoq/cli/defaults.py +98 -0
  7. pyoq/cli/services.py +97 -0
  8. pyoq/config/__init__.py +19 -0
  9. pyoq/config/loader.py +204 -0
  10. pyoq/config/models.py +243 -0
  11. pyoq/descriptors.py +165 -0
  12. pyoq/diagnostics/__init__.py +68 -0
  13. pyoq/diagnostics/budget.py +136 -0
  14. pyoq/diagnostics/events.py +137 -0
  15. pyoq/diagnostics/fingerprint.py +267 -0
  16. pyoq/diagnostics/instrumented.py +237 -0
  17. pyoq/diagnostics/metrics.py +61 -0
  18. pyoq/diagnostics/observation.py +227 -0
  19. pyoq/diagnostics/scoped.py +103 -0
  20. pyoq/django/__init__.py +15 -0
  21. pyoq/django/apps.py +17 -0
  22. pyoq/django/execution.py +317 -0
  23. pyoq/django/generation.py +59 -0
  24. pyoq/django/management/__init__.py +0 -0
  25. pyoq/django/management/commands/__init__.py +0 -0
  26. pyoq/django/management/commands/makemigrations.py +53 -0
  27. pyoq/django/management/commands/pyoq_codegen.py +75 -0
  28. pyoq/django/parameters.py +101 -0
  29. pyoq/django/schema.py +379 -0
  30. pyoq/django/settings.py +87 -0
  31. pyoq/django/timeouts.py +105 -0
  32. pyoq/dsl/__init__.py +64 -0
  33. pyoq/dsl/aio/__init__.py +31 -0
  34. pyoq/dsl/aio/context.py +295 -0
  35. pyoq/dsl/aio/queries.py +335 -0
  36. pyoq/dsl/aio/writes.py +368 -0
  37. pyoq/dsl/context.py +326 -0
  38. pyoq/dsl/entry.py +37 -0
  39. pyoq/dsl/labels.py +36 -0
  40. pyoq/dsl/queries.py +339 -0
  41. pyoq/dsl/result.py +164 -0
  42. pyoq/dsl/writes.py +360 -0
  43. pyoq/errors.py +317 -0
  44. pyoq/fastapi/__init__.py +32 -0
  45. pyoq/fastapi/dependencies.py +167 -0
  46. pyoq/fastapi/lifespan.py +119 -0
  47. pyoq/fetching/__init__.py +55 -0
  48. pyoq/fetching/collections.py +136 -0
  49. pyoq/fetching/execution.py +587 -0
  50. pyoq/fetching/joined.py +79 -0
  51. pyoq/fetching/nesting.py +183 -0
  52. pyoq/fetching/plans.py +541 -0
  53. pyoq/fetching/select_in.py +149 -0
  54. pyoq/fetching/tables.py +110 -0
  55. pyoq/generation/__init__.py +54 -0
  56. pyoq/generation/cleanup.py +44 -0
  57. pyoq/generation/contracts.py +248 -0
  58. pyoq/generation/drift.py +169 -0
  59. pyoq/generation/lock.py +33 -0
  60. pyoq/generation/manifest.py +114 -0
  61. pyoq/generation/model.py +1001 -0
  62. pyoq/generation/pipeline.py +119 -0
  63. pyoq/generation/rendering/__init__.py +5 -0
  64. pyoq/generation/rendering/domains.py +51 -0
  65. pyoq/generation/rendering/enums.py +29 -0
  66. pyoq/generation/rendering/exports.py +70 -0
  67. pyoq/generation/rendering/imports.py +63 -0
  68. pyoq/generation/rendering/package.py +56 -0
  69. pyoq/generation/rendering/relations.py +133 -0
  70. pyoq/generation/rendering/routines.py +396 -0
  71. pyoq/generation/rendering/rows.py +79 -0
  72. pyoq/generation/rendering/source.py +121 -0
  73. pyoq/generation/rendering/tables.py +300 -0
  74. pyoq/generation/rendering/writes.py +514 -0
  75. pyoq/generation/validation.py +27 -0
  76. pyoq/generation/writer.py +184 -0
  77. pyoq/hydration/__init__.py +24 -0
  78. pyoq/hydration/engine.py +155 -0
  79. pyoq/hydration/identity.py +194 -0
  80. pyoq/hydration/plan.py +116 -0
  81. pyoq/migrations/__init__.py +9 -0
  82. pyoq/migrations/alembic.py +106 -0
  83. pyoq/migrations/hooks.py +75 -0
  84. pyoq/naming.py +261 -0
  85. pyoq/policies/__init__.py +47 -0
  86. pyoq/policies/bypass.py +122 -0
  87. pyoq/policies/governed.py +430 -0
  88. pyoq/policies/model.py +242 -0
  89. pyoq/policies/rewriting.py +263 -0
  90. pyoq/py.typed +1 -0
  91. pyoq/query/__init__.py +312 -0
  92. pyoq/query/aggregates.py +172 -0
  93. pyoq/query/arrays.py +65 -0
  94. pyoq/query/binding.py +52 -0
  95. pyoq/query/capabilities.py +317 -0
  96. pyoq/query/casts.py +73 -0
  97. pyoq/query/choices.py +185 -0
  98. pyoq/query/decoding.py +360 -0
  99. pyoq/query/documents.py +56 -0
  100. pyoq/query/execution/__init__.py +63 -0
  101. pyoq/query/execution/aio/__init__.py +31 -0
  102. pyoq/query/execution/aio/operations.py +228 -0
  103. pyoq/query/execution/aio/pooling.py +233 -0
  104. pyoq/query/execution/aio/streaming.py +161 -0
  105. pyoq/query/execution/aio/transactions.py +105 -0
  106. pyoq/query/execution/batch.py +96 -0
  107. pyoq/query/execution/binding_style.py +30 -0
  108. pyoq/query/execution/compilation.py +48 -0
  109. pyoq/query/execution/context.py +61 -0
  110. pyoq/query/execution/control.py +50 -0
  111. pyoq/query/execution/operations.py +224 -0
  112. pyoq/query/execution/planning.py +107 -0
  113. pyoq/query/execution/pooling.py +279 -0
  114. pyoq/query/execution/results.py +36 -0
  115. pyoq/query/execution/streaming.py +178 -0
  116. pyoq/query/execution/transactions.py +95 -0
  117. pyoq/query/expressions.py +1200 -0
  118. pyoq/query/fields.py +60 -0
  119. pyoq/query/mysql/__init__.py +59 -0
  120. pyoq/query/mysql/aio/__init__.py +38 -0
  121. pyoq/query/mysql/aio/commands.py +389 -0
  122. pyoq/query/mysql/aio/driver.py +196 -0
  123. pyoq/query/mysql/aio/executor.py +123 -0
  124. pyoq/query/mysql/aio/factory.py +26 -0
  125. pyoq/query/mysql/aio/operations.py +38 -0
  126. pyoq/query/mysql/aio/pool.py +53 -0
  127. pyoq/query/mysql/aio/transactions.py +313 -0
  128. pyoq/query/mysql/commands.py +354 -0
  129. pyoq/query/mysql/compiler.py +134 -0
  130. pyoq/query/mysql/context.py +20 -0
  131. pyoq/query/mysql/executor.py +126 -0
  132. pyoq/query/mysql/expressions.py +244 -0
  133. pyoq/query/mysql/factory.py +46 -0
  134. pyoq/query/mysql/health.py +66 -0
  135. pyoq/query/mysql/identifiers.py +9 -0
  136. pyoq/query/mysql/model.py +79 -0
  137. pyoq/query/mysql/operations.py +43 -0
  138. pyoq/query/mysql/parameters.py +69 -0
  139. pyoq/query/mysql/planning.py +20 -0
  140. pyoq/query/mysql/pool.py +67 -0
  141. pyoq/query/mysql/transactions.py +331 -0
  142. pyoq/query/mysql/writes.py +73 -0
  143. pyoq/query/nodes.py +750 -0
  144. pyoq/query/postgres/__init__.py +48 -0
  145. pyoq/query/postgres/aio/__init__.py +25 -0
  146. pyoq/query/postgres/aio/bulk.py +56 -0
  147. pyoq/query/postgres/aio/commands.py +264 -0
  148. pyoq/query/postgres/aio/executor.py +152 -0
  149. pyoq/query/postgres/aio/factory.py +26 -0
  150. pyoq/query/postgres/aio/operations.py +26 -0
  151. pyoq/query/postgres/aio/pool.py +40 -0
  152. pyoq/query/postgres/aio/transactions.py +295 -0
  153. pyoq/query/postgres/bulk.py +62 -0
  154. pyoq/query/postgres/commands.py +238 -0
  155. pyoq/query/postgres/compiler.py +114 -0
  156. pyoq/query/postgres/context.py +20 -0
  157. pyoq/query/postgres/executor.py +147 -0
  158. pyoq/query/postgres/expressions.py +311 -0
  159. pyoq/query/postgres/factory.py +24 -0
  160. pyoq/query/postgres/health.py +24 -0
  161. pyoq/query/postgres/identifiers.py +9 -0
  162. pyoq/query/postgres/model.py +81 -0
  163. pyoq/query/postgres/operations.py +25 -0
  164. pyoq/query/postgres/parameters.py +71 -0
  165. pyoq/query/postgres/planning.py +20 -0
  166. pyoq/query/postgres/pool.py +52 -0
  167. pyoq/query/postgres/transactions.py +295 -0
  168. pyoq/query/postgres/writes.py +37 -0
  169. pyoq/query/projections.py +105 -0
  170. pyoq/query/raw.py +90 -0
  171. pyoq/query/recursion.py +265 -0
  172. pyoq/query/rendering/__init__.py +1 -0
  173. pyoq/query/rendering/expressions.py +913 -0
  174. pyoq/query/rendering/identifiers.py +40 -0
  175. pyoq/query/rendering/projections.py +63 -0
  176. pyoq/query/rendering/queries.py +334 -0
  177. pyoq/query/rendering/sources.py +66 -0
  178. pyoq/query/rendering/writes.py +176 -0
  179. pyoq/query/results.py +459 -0
  180. pyoq/query/routines.py +196 -0
  181. pyoq/query/rows.py +156 -0
  182. pyoq/query/select.py +793 -0
  183. pyoq/query/select_nodes.py +277 -0
  184. pyoq/query/sources.py +236 -0
  185. pyoq/query/sqlite/__init__.py +43 -0
  186. pyoq/query/sqlite/commands.py +201 -0
  187. pyoq/query/sqlite/compiler.py +139 -0
  188. pyoq/query/sqlite/context.py +20 -0
  189. pyoq/query/sqlite/executor.py +119 -0
  190. pyoq/query/sqlite/expressions.py +224 -0
  191. pyoq/query/sqlite/factory.py +32 -0
  192. pyoq/query/sqlite/health.py +28 -0
  193. pyoq/query/sqlite/identifiers.py +9 -0
  194. pyoq/query/sqlite/model.py +73 -0
  195. pyoq/query/sqlite/operations.py +36 -0
  196. pyoq/query/sqlite/parameters.py +50 -0
  197. pyoq/query/sqlite/planning.py +20 -0
  198. pyoq/query/sqlite/pool.py +50 -0
  199. pyoq/query/sqlite/streaming.py +13 -0
  200. pyoq/query/sqlite/transactions.py +274 -0
  201. pyoq/query/sqlite/writes.py +35 -0
  202. pyoq/query/statements.py +27 -0
  203. pyoq/query/values.py +23 -0
  204. pyoq/query/vendor.py +162 -0
  205. pyoq/query/windows.py +424 -0
  206. pyoq/query/write_nodes.py +174 -0
  207. pyoq/query/writes.py +628 -0
  208. pyoq/relations/__init__.py +66 -0
  209. pyoq/relations/batching.py +219 -0
  210. pyoq/relations/derivation.py +111 -0
  211. pyoq/relations/fetching.py +355 -0
  212. pyoq/relations/graph.py +245 -0
  213. pyoq/relations/loading.py +74 -0
  214. pyoq/relations/model.py +75 -0
  215. pyoq/relations/planning.py +206 -0
  216. pyoq/runtime/__init__.py +9 -0
  217. pyoq/runtime/kernels.py +25 -0
  218. pyoq/runtime/python.py +43 -0
  219. pyoq/runtime/selection.py +73 -0
  220. pyoq/sanic/__init__.py +32 -0
  221. pyoq/sanic/scope.py +197 -0
  222. pyoq/sanic/workers.py +129 -0
  223. pyoq/schema/__init__.py +108 -0
  224. pyoq/schema/codec.py +711 -0
  225. pyoq/schema/models.py +604 -0
  226. pyoq/schema/mysql/__init__.py +16 -0
  227. pyoq/schema/mysql/connection.py +72 -0
  228. pyoq/schema/mysql/dsn.py +72 -0
  229. pyoq/schema/mysql/records.py +354 -0
  230. pyoq/schema/mysql/reflection.py +309 -0
  231. pyoq/schema/mysql/source.py +30 -0
  232. pyoq/schema/mysql/sql.py +128 -0
  233. pyoq/schema/mysql/types.py +105 -0
  234. pyoq/schema/postgres/__init__.py +13 -0
  235. pyoq/schema/postgres/connection.py +62 -0
  236. pyoq/schema/postgres/records.py +384 -0
  237. pyoq/schema/postgres/reflection.py +466 -0
  238. pyoq/schema/postgres/source.py +30 -0
  239. pyoq/schema/postgres/sql.py +246 -0
  240. pyoq/schema/postgres/types.py +98 -0
  241. pyoq/schema/registry.py +45 -0
  242. pyoq/schema/source.py +15 -0
  243. pyoq/schema/sqlite/__init__.py +6 -0
  244. pyoq/schema/sqlite/connection.py +54 -0
  245. pyoq/schema/sqlite/records.py +167 -0
  246. pyoq/schema/sqlite/reflection.py +393 -0
  247. pyoq/schema/sqlite/source.py +30 -0
  248. pyoq/schema/sqlite/sql.py +254 -0
  249. pyoq/schema/sqlite/types.py +74 -0
  250. pyoq/serving/__init__.py +19 -0
  251. pyoq/serving/databases.py +107 -0
  252. pyoq/snapshots/__init__.py +20 -0
  253. pyoq/snapshots/drift.py +312 -0
  254. pyoq/snapshots/files.py +96 -0
  255. pyoq/snapshots/routing.py +40 -0
  256. pyoq/snapshots/source.py +33 -0
  257. pyoq/tracing/__init__.py +5 -0
  258. pyoq/tracing/spans.py +89 -0
  259. pyoq/unset.py +14 -0
  260. pyoq_sql-1.0.0.dist-info/METADATA +3034 -0
  261. pyoq_sql-1.0.0.dist-info/RECORD +264 -0
  262. pyoq_sql-1.0.0.dist-info/WHEEL +4 -0
  263. pyoq_sql-1.0.0.dist-info/entry_points.txt +3 -0
  264. pyoq_sql-1.0.0.dist-info/licenses/LICENSE +373 -0
@@ -0,0 +1,3034 @@
1
+ Metadata-Version: 2.5
2
+ Name: pyoq-sql
3
+ Version: 1.0.0
4
+ Summary: A fully typed, database-first SQL toolkit for Python.
5
+ License-Expression: MPL-2.0
6
+ License-File: LICENSE
7
+ Keywords: database,query-builder,sql,typing
8
+ Classifier: Development Status :: 5 - Production/Stable
9
+ Classifier: Framework :: AsyncIO
10
+ Classifier: Framework :: Django
11
+ Classifier: Framework :: FastAPI
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Operating System :: OS Independent
14
+ Classifier: Programming Language :: Python
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3 :: Only
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
21
+ Classifier: Topic :: Database
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.11
24
+ Provides-Extra: django
25
+ Requires-Dist: django<7,>=4.2; extra == 'django'
26
+ Provides-Extra: fastapi
27
+ Requires-Dist: fastapi<1,>=0.115; extra == 'fastapi'
28
+ Provides-Extra: mysql
29
+ Requires-Dist: pymysql<2,>=1.1; extra == 'mysql'
30
+ Provides-Extra: mysql-async
31
+ Requires-Dist: asyncmy<1,>=0.2.14; extra == 'mysql-async'
32
+ Requires-Dist: pymysql<2,>=1.1; extra == 'mysql-async'
33
+ Provides-Extra: postgres
34
+ Requires-Dist: psycopg<4,>=3.2; extra == 'postgres'
35
+ Provides-Extra: sanic
36
+ Requires-Dist: sanic<26,>=24.6; extra == 'sanic'
37
+ Provides-Extra: tracing
38
+ Requires-Dist: opentelemetry-api<2,>=1.25; extra == 'tracing'
39
+ Description-Content-Type: text/markdown
40
+
41
+ # PyOQ
42
+
43
+ PyOQ is a fully typed, database-first SQL toolkit for Python. It is designed to
44
+ provide explicit query construction, generated schema types, predictable
45
+ execution, and efficient result hydration without hiding SQL behavior.
46
+
47
+ SQLite, PostgreSQL and MySQL are supported, synchronously and asynchronously.
48
+ Schema reflection generates row, key, insert, update, table, column, relation,
49
+ enum, domain and routine types that hold under strict type checking. Relations
50
+ are fetched by a plan that keeps the query count independent of the result,
51
+ and Django, FastAPI and Sanic each run on the same core rather than on a copy
52
+ of it.
53
+
54
+ Every public behaviour is covered by tests at complete statement and branch
55
+ coverage, run against real PostgreSQL and MySQL servers rather than recorded
56
+ stand-ins.
57
+
58
+ ## Installation
59
+
60
+ ```console
61
+ python -m pip install pyoq-sql
62
+ ```
63
+
64
+ The distribution name is `pyoq-sql`; the Python import package is `pyoq`.
65
+
66
+ Releases provide native platform wheels and a portable fallback wheel, so
67
+ installation does not require Rust. Building from the source archive does
68
+ require a supported Rust toolchain. When the portable wheel is used, select
69
+ its runtime explicitly:
70
+
71
+ ```console
72
+ PYOQ_RUNTIME=python python -m your_application
73
+ ```
74
+
75
+ The fallback preserves behavior and typing but is outside the native
76
+ performance contract. `PYOQ_RUNTIME=native` makes an unavailable native engine
77
+ a startup error. The default selects the native engine when present and emits
78
+ a runtime warning before using the fallback when it is absent.
79
+
80
+ ## Package check
81
+
82
+ ```python
83
+ from pyoq import __version__
84
+
85
+ print(__version__)
86
+ ```
87
+
88
+ ## Package architecture
89
+
90
+ Implementation code is grouped by capability. Each feature package owns its
91
+ public facade and private implementation details, while package-wide errors,
92
+ naming policy, typing metadata, and the native extension remain at the root.
93
+
94
+ ```mermaid
95
+ flowchart TD
96
+ Package[pyoq] --> CLI[cli]
97
+ Package --> Config[config]
98
+ Package --> Generation[generation]
99
+ Package --> Query[query]
100
+ Package --> Runtime[runtime]
101
+ Package --> Schema[schema]
102
+ Package --> Errors[errors]
103
+ Package --> Naming[naming]
104
+ Runtime --> Native[private native extension]
105
+ ```
106
+
107
+ Applications should import feature APIs from their stable facades, such as
108
+ `pyoq.config`, `pyoq.schema`, `pyoq.generation`, and `pyoq.runtime`. Nested
109
+ modules separate component responsibilities and are not compatibility
110
+ surfaces.
111
+
112
+ ## Configuration
113
+
114
+ Configuration is read from `pyproject.toml`. Generated code always lives in a
115
+ dedicated directory below the project root. Database credentials are referenced
116
+ by environment variable name and are never stored in the configuration file.
117
+
118
+ ```toml
119
+ [tool.pyoq]
120
+ codegen-directory = "generated/database"
121
+ codegen-package = "application.database"
122
+ selected-profile = "default"
123
+
124
+ [tool.pyoq.profiles.default]
125
+ dialect = "postgres"
126
+ dsn-environment = "DATABASE_URL"
127
+
128
+ [tool.pyoq.profiles.testing]
129
+ dialect = "sqlite"
130
+ database-path = "database/testing.sqlite"
131
+ ```
132
+
133
+ Load the selected profile through the fully typed configuration API:
134
+
135
+ ```python
136
+ from pathlib import Path
137
+
138
+ from pyoq.config import load_configuration
139
+
140
+ configuration = load_configuration(Path.cwd())
141
+ profile = configuration.active_profile
142
+
143
+ if profile.dsn_environment is not None:
144
+ database_url = profile.dsn_environment.resolve()
145
+ connect_value = database_url.reveal()
146
+ ```
147
+
148
+ SQLite file paths use `database-path`. They are resolved relative to the project
149
+ root, must remain inside it, and are opened read-only during inspection. An
150
+ environment reference may be used instead when the database location should be
151
+ supplied by the process environment. A profile cannot define both settings.
152
+
153
+ `SecretValue` renders as `[REDACTED]`; access to its underlying value is always
154
+ explicit. An explicit `Configuration` object takes precedence over the file,
155
+ and the `selected_profile` argument can override profile selection. Unknown
156
+ settings, missing profiles, invalid dialects, unavailable environment values,
157
+ and paths outside the project root fail with exceptions from `pyoq.errors`.
158
+
159
+ ## What each dialect can do
160
+
161
+ Every row here is one operation, not a family, because a family name hides
162
+ which parts of it exist. Every entry was taken from a running server rather
163
+ than from a specification.
164
+
165
+ **refused** means PyOQ raises before the statement is sent, naming what the
166
+ dialect has not got. It never emits SQL a database cannot run, and never
167
+ quietly does something narrower than what was asked. **written by PyOQ** means
168
+ the dialect has no such operation and PyOQ writes it out of what the dialect
169
+ does have, so one word means one thing on all three.
170
+
171
+ This table is generated from the capability matrix the source keeps. A
172
+ contract compiles every row against every dialect, checks that each answers
173
+ with the kind of value the row claims, and names the dialect-neutral contract
174
+ that runs it against a real server, so a claim here that is not true stops the
175
+ test suite.
176
+
177
+ <!-- capability matrix -->
178
+
179
+ | Operation | sqlite | postgres | mysql | reads back as | run by |
180
+ | --- | --- | --- | --- | --- | --- |
181
+ | window: row number | yes | yes | yes | numeric | `window_contract` |
182
+ | window: rank | yes | yes | yes | numeric | `window_contract` |
183
+ | window: dense rank | yes | yes | yes | numeric | `window_contract` |
184
+ | window: percent rank | yes | yes | yes | numeric | `window_contract` |
185
+ | window: cumulative distribution | yes | yes | yes | numeric | `window_contract` |
186
+ | window: ntile | yes | yes | yes | numeric | `window_contract` |
187
+ | window: lag | yes | yes | yes | numeric | `window_contract` |
188
+ | window: lead | yes | yes | yes | numeric | `window_contract` |
189
+ | window: first value | yes | yes | yes | numeric | `window_contract` |
190
+ | window: last value | yes | yes | yes | numeric | `window_contract` |
191
+ | window: nth value | yes | yes | yes | numeric | `window_contract` |
192
+ | window: an aggregate over a window | yes | yes | yes | numeric | `window_contract` |
193
+ | window: partition by | yes | yes | yes | numeric | `window_contract` |
194
+ | window: order by | yes | yes | yes | numeric | `window_contract` |
195
+ | window frame: rows | yes | yes | yes | numeric | `window_contract` |
196
+ | window frame: range | yes | yes | yes | numeric | `window_contract` |
197
+ | window frame: groups | yes | yes | refused | numeric | `window_contract` |
198
+ | window frame: exclude | yes | yes | refused | numeric | `window_contract` |
199
+ | window: declared by name | yes | yes | yes | numeric | `window_contract` |
200
+ | json: read a value | yes | yes | yes | json | `json_contract` |
201
+ | json: read text | yes | yes | yes | string | `json_contract` |
202
+ | json: how long an array is | yes | yes | yes | numeric | `json_contract` |
203
+ | json: whether a path is there | yes | yes | yes | boolean | `json_contract` |
204
+ | json: whether it holds a value | refused | yes | yes | boolean | `json_contract` |
205
+ | json: set | yes | yes | yes | json | `json_contract` |
206
+ | json: insert | yes | written by PyOQ | yes | json | `json_contract` |
207
+ | json: replace | yes | yes | yes | json | `json_contract` |
208
+ | json: merge all the way down | yes | refused | yes | json | `json_contract` |
209
+ | json: write over members, one level | refused | yes | refused | json | `json_contract` |
210
+ | json: remove | yes | yes | yes | json | `json_contract` |
211
+ | json: build an object | yes | yes | yes | json | `json_contract` |
212
+ | json: build an array | yes | yes | yes | json | `json_contract` |
213
+ | json: name the members | written by PyOQ | written by PyOQ | yes | json | `json_contract` |
214
+ | json: gather rows into an array | yes | yes | yes | json | `json_contract` |
215
+ | json: gather rows into an object | yes | yes | yes | json | `json_contract` |
216
+ | json: read an array as rows | yes | yes | yes | string | `json_contract` |
217
+ | array: holds a value | refused | yes | refused | boolean | `array_contract` |
218
+ | array: holds a value nowhere | refused | yes | refused | boolean | `array_contract` |
219
+ | array: holds every value | refused | yes | refused | boolean | `array_contract` |
220
+ | array: is among | refused | yes | refused | boolean | `array_contract` |
221
+ | array: shares a value | refused | yes | refused | boolean | `array_contract` |
222
+ | array: append | refused | yes | refused | other | `array_contract` |
223
+ | array: prepend | refused | yes | refused | other | `array_contract` |
224
+ | array: concatenate | refused | yes | refused | other | `array_contract` |
225
+ | array: without a value | refused | yes | refused | other | `array_contract` |
226
+ | array: replace a value | refused | yes | refused | other | `array_contract` |
227
+ | array: one element | refused | yes | refused | other | `array_contract` |
228
+ | array: how many elements | refused | yes | refused | numeric | `array_contract` |
229
+ | array: how many along a dimension | refused | yes | refused | numeric | `array_contract` |
230
+ | array: how many dimensions | refused | yes | refused | numeric | `array_contract` |
231
+ | array: built from values | refused | yes | refused | other | `array_contract` |
232
+ | lock: for update | refused | yes | yes | numeric | `locking_contract` |
233
+ | lock: for share | refused | yes | yes | numeric | `locking_contract` |
234
+ | lock: fail rather than wait | refused | yes | yes | numeric | `locking_contract` |
235
+ | lock: skip what is locked | refused | yes | yes | numeric | `locking_contract` |
236
+ | lock: named tables | refused | yes | yes | numeric | `locking_contract` |
237
+ | lock: named fields | refused | yes | yes | numeric | `locking_contract` |
238
+ | query: lateral source | refused | yes | yes | numeric | `lateral_contract` |
239
+ | query: recursive common table | yes | yes | yes | numeric | `recursion_contract` |
240
+ | query: row value | yes | yes | yes | boolean | `row_value_contract` |
241
+ | query: cast | yes | yes | yes | string | `cast_contract` |
242
+ | query: nulls first or last | yes | yes | written by PyOQ | numeric | `null_order_contract` |
243
+ | query: right join | refused | yes | yes | numeric | `joined_contract` |
244
+ | query: full join | refused | yes | refused | numeric | `joined_contract` |
245
+ | routine: run a stored procedure | refused | yes | yes | rows | `routine_contract` |
246
+ | routine: ask a function by name | yes | yes | yes | numeric | `routine_contract` |
247
+ | routine: more than one result set | refused | refused | refused | rows | `routine_contract` |
248
+
249
+ <!-- capability matrix -->
250
+
251
+ A JSON path is steps rather than text, because PostgreSQL reads another
252
+ dialect's written path as a member of that name and answers null without
253
+ complaining. PostgreSQL's own insert raises where a member already exists,
254
+ while the other two leave what is there, so PyOQ asks first on PostgreSQL.
255
+
256
+ Merging and concatenating are two operations, not one. `json_merge` is
257
+ merge-patch: a member given as null is removed, and an object inside an object
258
+ is merged rather than replaced. MySQL and SQLite do that themselves, and
259
+ PostgreSQL has no merge-patch at all. `json_concat` is one level deep: a member
260
+ is replaced whole and a null is stored as a null. PostgreSQL does that itself,
261
+ and the other two have nothing that means it. Each is refused where the
262
+ dialect does not have it, rather than quietly doing the other one.
263
+
264
+ Only PostgreSQL has an array type. SQLite will accept `TEXT[]` as a type name
265
+ with no array behind it, which is the kind of tolerance an emulation would be
266
+ built on, so the other two refuse instead.
267
+
268
+ SQLite locks a database rather than rows and keeps no stored procedures. A
269
+ timed lock wait is in no supported dialect: `FOR UPDATE WAIT` is a syntax error
270
+ on both PostgreSQL and MySQL. Null treatment on a window function is likewise
271
+ in none of them: `IGNORE NULLS` is refused by all three servers.
272
+
273
+ A procedure that answers with more than one result set is refused rather than
274
+ read down to its first, because a dropped result set is a silent one.
275
+
276
+ ### Schema objects
277
+
278
+ | Object | SQLite | PostgreSQL | MySQL |
279
+ | --- | --- | --- | --- |
280
+ | tables, views, keys, relations, indexes, checks | yes | yes | yes |
281
+ | enums | none | reflected and generated | reflected per column |
282
+ | domains | none | reflected and generated | none |
283
+ | routines | none | reflected and generated | reflected with typed calls or typed rejection |
284
+
285
+ Only PostgreSQL keeps overloaded routines; MySQL refuses a second routine of
286
+ the same name, and SQLite keeps no routine catalog at all.
287
+
288
+ A procedure answers through the parameters it writes. A generated PostgreSQL
289
+ call asks the caller only for IN and INOUT values, supplies required OUT
290
+ placeholders internally, and reads the answer back with the catalog's types and
291
+ nullability. MySQL needs a session-variable adapter for OUT and INOUT values.
292
+ Those routines retain a generated, typed signature that raises
293
+ `UnsupportedQueryError` before any invalid SQL is sent.
294
+
295
+ A generated call names the schema the catalog said keeps the routine. Without
296
+ it a routine outside the search path resolves to nothing, and one whose name
297
+ another schema shares resolves to the wrong one.
298
+
299
+ ## Supported dialects
300
+
301
+ Every configured dialect resolves to a schema source, so inspection and
302
+ generation work for all of them. SQLite is implemented end to end. PostgreSQL
303
+ has a compiler, type mapping, schema reflection, and both synchronous and
304
+ asynchronous drivers with pooling, transactions, savepoints, and streaming.
305
+ MySQL has all of that too, so all three dialects are now complete vertical
306
+ slices.
307
+
308
+ Database drivers are optional extras, so a SQLite installation stays dependency
309
+ free:
310
+
311
+ ```console
312
+ python -m pip install "pyoq-sql[postgres]"
313
+ python -m pip install "pyoq-sql[mysql]"
314
+ python -m pip install "pyoq-sql[mysql-async]"
315
+ ```
316
+
317
+ Without them, PostgreSQL and MySQL query construction and compilation still
318
+ work because they are pure Python. Only connecting requires a driver, and its
319
+ absence raises an explicit error naming the extra to install. PostgreSQL and
320
+ MySQL profiles supply their connection string through `dsn-environment`, and
321
+ inspection opens the connection read-only.
322
+
323
+ A MySQL connection string names one database, which is reflected as the
324
+ snapshot's schema:
325
+
326
+ ```console
327
+ mysql://user:password@host:3306/database
328
+ ```
329
+
330
+ Query and fragment options are refused rather than silently ignored, so an
331
+ unsupported setting can never be mistaken for an applied one.
332
+
333
+ `schema_source_for()` resolves a dialect to its schema source, and
334
+ `supported_dialects()` reports every dialect that has one. Command services
335
+ route through the configured dialect at load time, so adding a dialect is a
336
+ registry entry rather than a change to the command layer.
337
+
338
+ The layers a dialect plugs into are deliberately neutral. `CompiledQuery`,
339
+ `StatementCompiler`, `SchemaSource`, `QueryOperations`, `BulkPlan`, and
340
+ `DatabaseCursor` carry no dialect assumptions, so a new database supplies a
341
+ compiler, a schema source, and a cursor-backed executor without reimplementing
342
+ result cardinality, bulk planning, or ordered multi-operation execution.
343
+
344
+ Execution is shared too. `ConnectionPool` owns the lease lifecycle, size and
345
+ timeout policy, idle reuse, invalidation, and close semantics; `RowStream` owns
346
+ bounded batch buffering, iteration ownership, and deterministic cleanup; and
347
+ `TransactionState` owns scope nesting, savepoint naming, owner-thread checks,
348
+ and stream interaction. A dialect supplies how it resets and validates a
349
+ connection, how it opens a cursor, and how it begins a transaction.
350
+
351
+ SQL rendering is shared as well. `ExpressionRenderer`, `QueryRenderer`, and
352
+ `WriteRenderer` own the standard SQL that every relational dialect writes the
353
+ same way: projections, sources, joins, common tables, ordering, set operations,
354
+ inserts, assignments, row scoping, conflict resolution, and returning. A dialect
355
+ declares only where it genuinely differs, such as how it spells distinctness,
356
+ how it extracts a date part, how it renders string predicates, which joins and
357
+ capabilities it allows, and how it paginates.
358
+
359
+ Parameter placeholders are part of that declaration. `ParameterStyle` covers
360
+ `?`, `$1`, and `%s`, and the compilation context escapes literal percent signs
361
+ in structural SQL when the driver style requires it.
362
+
363
+ `PostgresCompiler` shows the shape of a dialect built this way. It renders the
364
+ same query model as SQLite while spelling distinctness as `IS DISTINCT FROM`,
365
+ extracting date parts with `EXTRACT`, rendering string predicates with
366
+ `POSITION`, `LEFT`, and `RIGHT`, allowing RIGHT and FULL joins and a native
367
+ `NULLS` clause, emitting a bare `OFFSET` without a placeholder limit, and
368
+ budgeting at the 65535 parameter ceiling of the extended query protocol. It
369
+ rejects catalog-qualified identifiers and adding two temporal values, because
370
+ PostgreSQL has no operator for either.
371
+
372
+ MySQL has a compiler, type mapping, schema reflection, and synchronous and
373
+ asynchronous drivers. It shows how far a dialect can diverge while still using
374
+ the shared renderers: identifiers are quoted with backticks, string
375
+ concatenation renders as `CONCAT` because `||` means logical OR in MySQL,
376
+ distinctness uses the null-safe `<=>` operator, a bare `OFFSET` is given the
377
+ maximum row limit MySQL requires, explicit null placement is written by
378
+ ordering on whether a value is null because MySQL has no `NULLS` clause, and
379
+ upsert renders as a row alias with `ON DUPLICATE KEY UPDATE`. The row alias form requires
380
+ MySQL 8.0.19 or later; earlier servers and MariaDB use a different spelling
381
+ that PyOQ does not emit.
382
+
383
+ Where MySQL cannot express something, it says so instead of emulating it.
384
+ `RETURNING` is refused with a message pointing at a follow-up query. A conflict
385
+ target is refused because MySQL infers the key. A conflict condition is refused
386
+ because `ON DUPLICATE KEY UPDATE` has no `WHERE`. Do-nothing conflict
387
+ resolution is refused rather than emulated with `INSERT IGNORE`, which would
388
+ also swallow unrelated errors, or with a self-assignment, which would report a
389
+ different affected row count.
390
+
391
+ `postgres_type()` normalizes declared PostgreSQL type names into the shared
392
+ schema model, including serial types, spelled-out variants such as
393
+ `timestamp with time zone`, length and precision arguments, and array types
394
+ with their element type. A declaration it cannot parse stays opaque rather than
395
+ being guessed at.
396
+
397
+ MySQL executes through `MySQLExecutor` over a `MySQLPool`, with the same typed
398
+ operations SQLite and PostgreSQL use. Four dialect behaviors differ and are
399
+ documented rather than hidden.
400
+
401
+ An affected row count means matched rows. MySQL counts changed rows by default,
402
+ so an update writing a column's existing value would report nothing, and PyOQ
403
+ connects with the client flag that restores the meaning every other dialect
404
+ has.
405
+
406
+ An insert reports the first identifier it generated, not the last, because that
407
+ is what MySQL reports for a multi-row insert.
408
+
409
+ A statement timeout is enforced from the client. MySQL applies its own
410
+ `max_execution_time` only to read-only SELECT statements, so PyOQ sets that
411
+ and also runs a watchdog that asks the server to stop the statement once the
412
+ deadline passes. Cancellation works the same way, because MySQL has no
413
+ client-side cancel. Both open a short-lived second connection, and a statement
414
+ carrying neither a timeout nor a cancellation token starts no watchdog and pays
415
+ nothing.
416
+
417
+ A deadlock rolls the whole transaction back on the server while leaving the
418
+ connection healthy, and a later commit then succeeds having committed nothing.
419
+ PyOQ refuses to report that commit and raises instead, so work that was
420
+ discarded is never reported as saved. A failed statement that is not a deadlock
421
+ leaves a MySQL transaction usable, which is the opposite of PostgreSQL and
422
+ needs no savepoint to recover from.
423
+
424
+ Isolation is set before a transaction starts rather than as part of starting
425
+ it, and the default is MySQL's own repeatable read rather than a level PyOQ
426
+ imposes. Streaming reads through an unbuffered cursor, and a transaction allows
427
+ only one open stream at a time so that a second statement cannot silently
428
+ discard the rows the first has not read.
429
+
430
+ `MySQLAsyncExecutor` over a `MySQLAsyncPool` provides the same operations
431
+ without blocking, and behaves identically on every point above. Three things are
432
+ worth knowing about it.
433
+
434
+ It has its own extra, because its driver is a compiled one:
435
+
436
+ ```console
437
+ python -m pip install "pyoq-sql[mysql-async]"
438
+ ```
439
+
440
+ The synchronous extra stays pure Python. The asynchronous driver is compiled
441
+ because the read path is worth it: PyOQ adds only a few percent above its
442
+ driver on a large read, so the driver is the cost, and a pure-Python driver
443
+ measured roughly an order of magnitude slower end to end on the same query. A
444
+ throughput contract holds the read path to that standard so it cannot quietly
445
+ regress.
446
+
447
+ Its connections are encrypted. The synchronous driver negotiates TLS on its
448
+ own, and the asynchronous one is given the same settings so that choosing
449
+ `async` never quietly downgrades the transport. Encryption is what lets a
450
+ server's password check happen without a separate cryptography library, so a
451
+ server with TLS switched off needs `PyMySQL[rsa]` installed for either driver.
452
+
453
+ Its driver publishes only generated type stubs that leave most of its surface
454
+ unknown, and raises its own exception hierarchy rather than the synchronous
455
+ driver's. PyOQ therefore declares the shape it depends on and confines every
456
+ reference to the driver to one module. Failure classification is shared, since
457
+ every MySQL driver reports the same numeric codes while raising different
458
+ exception types.
459
+
460
+ `mysql_type()` normalizes declared MySQL type names, including width and
461
+ precision arguments, `unsigned`, `signed`, and `zerofill` modifiers, and the
462
+ inline `ENUM` and `SET` declarations that carry their values in the column type
463
+ rather than in a named type.
464
+
465
+ MySQL reflection reads the connected database through `information_schema`.
466
+ Auto-increment columns are reported as identity columns, stored and virtual
467
+ generated columns keep their generation expression, and column defaults are
468
+ normalized into valid SQL so that a literal default such as an empty string is
469
+ quoted while an expression default such as `json_object()` is not. Descending
470
+ and expression index terms, composite foreign keys with their update and delete
471
+ rules, and CHECK constraints are all carried across. The placeholder comment
472
+ MySQL stores on every view is discarded rather than reported as a comment.
473
+
474
+ PostgreSQL executes through `PostgresExecutor` over a `PostgresPool`, with the
475
+ same typed operations SQLite uses. Two dialect behaviors differ and are not
476
+ hidden. A statement timeout is enforced by the server through
477
+ `statement_timeout` rather than a client-side interrupt, and a cancellation
478
+ token is honored mid-statement by a watchdog that issues a server cancel.
479
+ Streaming uses a server-side cursor inside its own transaction, which is what
480
+ keeps memory bounded for large results.
481
+
482
+ PostgreSQL also executes asynchronously through `PostgresAsyncExecutor` over a
483
+ `PostgresAsyncPool`. The asynchronous surface mirrors the synchronous one
484
+ method for method, including pooling, transactions, savepoints, server-side
485
+ cursor streaming, server-enforced timeouts, and cancellation:
486
+
487
+ ```python
488
+ async with PostgresAsyncPool(PostgresAsyncConnectionFactory(dsn)) as pool:
489
+ database = PostgresAsyncExecutor(pool)
490
+ async with database.transaction() as transaction:
491
+ await transaction.execute(statement)
492
+ async with database.stream(query) as rows:
493
+ async for row in rows:
494
+ ...
495
+ ```
496
+
497
+ Bulk writes use PostgreSQL pipeline mode on both paths. The planner still
498
+ splits a large insert into chunks that fit the parameter budget, but the chunks
499
+ are sent without waiting for each result, which removes a round trip per chunk.
500
+ This changes one behavior for the better and it is worth knowing: a pipelined
501
+ batch is atomic, so a failure discards the whole batch rather than leaving
502
+ earlier chunks applied. Sequential chunking keeps the earlier chunks, as it
503
+ always did.
504
+
505
+ Pipelining is governed by the `pipeline` compiler capability and by whether the
506
+ installed libpq supports it. When either says no, chunk execution falls back to
507
+ the sequential path with identical results.
508
+
509
+ Ownership is per task rather than per thread, so a stream or transaction used
510
+ from a different task fails explicitly instead of corrupting its state. Both
511
+ paths are held to the same dialect-neutral execution contract, so behavior does
512
+ not drift between them.
513
+
514
+ The most important difference is transactional: in PostgreSQL a failed
515
+ statement aborts the whole transaction, and every later statement in that
516
+ transaction fails until it ends. Recovery is a savepoint taken before the
517
+ statement that might fail:
518
+
519
+ ```python
520
+ with database.transaction() as transaction:
521
+ with transaction.savepoint() as attempt:
522
+ attempt.execute(statement_that_may_conflict)
523
+ transaction.execute(next_statement)
524
+ ```
525
+
526
+ PyOQ does not wrap every statement in an implicit savepoint to paper over this,
527
+ because that would add a hidden round trip to every write.
528
+
529
+ PostgreSQL reflection reads the system catalogs directly rather than
530
+ `information_schema`, so it recovers identity and generated columns, array
531
+ element types, per-column comments, foreign key referential actions, expression
532
+ and partial indexes with their sort direction, check constraints, materialized
533
+ views, and multiple schemas. Catalog rows are validated as they are read, so a
534
+ row of unexpected shape fails as a schema error instead of producing a silently
535
+ wrong model.
536
+
537
+ ## Schema model
538
+
539
+ Database metadata is normalized into an immutable, dialect-neutral snapshot.
540
+ Source identifiers retain their exact case, spacing, and Unicode content. The
541
+ model does not apply Python naming rules or depend on database drivers and web
542
+ frameworks.
543
+
544
+ ```mermaid
545
+ flowchart TD
546
+ Snapshot[SchemaSnapshot] --> Catalog[Catalog]
547
+ Catalog --> Schema[Schema]
548
+ Schema --> Table[Table]
549
+ Schema --> View[View]
550
+ Schema --> Enum[EnumType]
551
+ Table --> Column[Column]
552
+ Table --> Key[Key]
553
+ Table --> Relation[Relation]
554
+ Table --> Index[Index]
555
+ Table --> Check[CheckConstraint]
556
+ ```
557
+
558
+ Construct snapshots directly when implementing metadata sources:
559
+
560
+ ```python
561
+ from pyoq.config import DatabaseDialect
562
+ from pyoq.schema import (
563
+ Catalog,
564
+ Column,
565
+ Identifier,
566
+ Schema,
567
+ SchemaSnapshot,
568
+ SqlType,
569
+ Table,
570
+ TypeKind,
571
+ )
572
+
573
+ user_id = Column(
574
+ name=Identifier("id"),
575
+ data_type=SqlType(TypeKind.INTEGER, "INTEGER"),
576
+ nullable=False,
577
+ )
578
+ users = Table(name=Identifier("users"), columns=(user_id,))
579
+ snapshot = SchemaSnapshot(
580
+ dialect=DatabaseDialect.SQLITE,
581
+ catalogs=(Catalog(None, (Schema(None, tables=(users,)),)),),
582
+ )
583
+ ```
584
+
585
+ `SchemaSnapshot.to_json()` produces compact, versioned, deterministic JSON with
586
+ Unicode preserved. `SchemaSnapshot.from_json()` performs strict field and type
587
+ validation before constructing model objects. The round trip preserves ordered
588
+ columns, nullability, raw defaults, identity and computed values, SQL type
589
+ details, qualified references, comments, indexes, checks, relations, enums,
590
+ views, and dialect capabilities.
591
+
592
+ Every model uses frozen slots. Duplicate identifiers, invalid type dimensions,
593
+ misaligned foreign-key columns, unknown local key or index columns, conflicting
594
+ generated-value metadata, and unsupported snapshot versions fail with typed
595
+ schema exceptions. Qualified relation targets may remain outside a snapshot so
596
+ metadata sources can represent intentionally partial introspection safely.
597
+
598
+ ## Relations
599
+
600
+ A foreign key is one fact that can be read from either side, so PyOQ derives
601
+ both. `derive_relations()` turns a schema snapshot into typed relations that
602
+ carry direction, cardinality, and whether a to-one relation may resolve to
603
+ nothing:
604
+
605
+ ```python
606
+ from pyoq.relations import derive_relations
607
+
608
+ for relation in derive_relations(snapshot):
609
+ print(relation.direction, relation.cardinality, relation.optional)
610
+ ```
611
+
612
+ Cardinality and optionality are read from the schema rather than declared:
613
+
614
+ - A foreign key names at most one row of the table it points at, so following
615
+ it forward is always to-one. It is optional when any of its columns is
616
+ nullable, because a foreign key with a null part references nothing at all.
617
+ - Following it backwards is to-many, unless the referencing columns are
618
+ themselves unique, in which case it is a to-one that may be absent. A unique
619
+ key over part of the referencing columns is enough, since a subset that is
620
+ already unique makes the wider set unique too. This is what recognizes a
621
+ table whose primary key is also its foreign key as a one-to-one extension
622
+ rather than a collection.
623
+ - A to-many relation is never optional. The absence of children is an empty
624
+ collection, not a missing value.
625
+
626
+ A self-referencing foreign key produces both directions on the same table.
627
+
628
+ `build_relation_graph()` indexes those relations by the table they start from,
629
+ so a schema can be navigated in either direction:
630
+
631
+ ```python
632
+ from pyoq.relations import build_relation_graph
633
+
634
+ graph = build_relation_graph(snapshot)
635
+ for relation in graph.to_many_from(graph.resolve(team_reference)):
636
+ print(relation.target.table.name.value)
637
+ ```
638
+
639
+ Endpoints and lookups are resolved to the way the snapshot names a table, so a
640
+ foreign key written against a bare name reaches the qualified table it means,
641
+ and a caller need not know how the snapshot qualified anything. A bare name
642
+ that two schemas both carry is left alone, because it names neither.
643
+
644
+ A snapshot describes one database and a foreign key may point beyond it, so a
645
+ target the snapshot does not describe is reported through
646
+ `unresolved_targets` rather than refused. Such a table stays navigable, because
647
+ the table that declared the key still knows its side of it; only asking for the
648
+ missing table's definition fails, and it fails by name. A key naming a column
649
+ that a table it did resolve does not have is refused outright, because that is
650
+ corruption rather than partial coverage.
651
+
652
+ Generation emits both directions of every key as typed constants, each
653
+ carrying its cardinality and whether it may resolve to nothing:
654
+
655
+ ```python
656
+ from generated.relations import (
657
+ EMPLOYEE_TEAM_TEAM_ID,
658
+ TEAM_EMPLOYEE_TEAM_ID_REVERSE,
659
+ )
660
+
661
+ EMPLOYEE_TEAM_TEAM_ID.to_one # True: an employee has one team
662
+ TEAM_EMPLOYEE_TEAM_ID_REVERSE.to_one # False: a team has many employees
663
+ ```
664
+
665
+ A relation is named after the table across the key and the columns that carry
666
+ it, with the reverse direction saying so. Nothing in the name depends on where
667
+ a relation falls in a list, so adding a key to one table cannot rename the
668
+ relations already generated for another. A key that points at its own table
669
+ produces both directions on that table, which is why the direction is always
670
+ part of the name rather than only when it would otherwise collide. Where a
671
+ constraint has a name of its own, that name is used instead.
672
+
673
+ Two keys that no name can tell apart are reported as a naming collision rather
674
+ than resolved by guessing, and naming the constraint in the database is the fix.
675
+
676
+ ### Fetch plans
677
+
678
+ A fetch plan says which relations a query brings back and how, and it is read
679
+ back before anything runs:
680
+
681
+ ```python
682
+ from pyoq.relations import FetchPlan, FetchRequest, FetchStrategy
683
+
684
+ plan = FetchPlan(
685
+ team_reference,
686
+ (FetchRequest(employees, FetchStrategy.NESTED, (FetchRequest(badge),)),),
687
+ )
688
+ print("\n".join(plan.describe()))
689
+ ```
690
+
691
+ ```console
692
+ employee [] via nested
693
+ badge via auto
694
+ ```
695
+
696
+ Four strategies are available. `joined` brings a relation back in the same
697
+ query, `nested` brings a collection back as one correlated result, `select_in`
698
+ issues one further query per level, and `auto` leaves the choice to be made
699
+ against the dialect's capabilities when the query is planned, so one plan can
700
+ take the best route each database offers.
701
+
702
+ `resolve_fetch_plan()` turns that intent into a decision against what a dialect
703
+ can actually do, and records why:
704
+
705
+ ```console
706
+ posts ordered by id via select-in: this dialect cannot order inside an
707
+ aggregate, and this collection's order matters
708
+ users via joined: a to-one relation is one row of the join
709
+ ```
710
+
711
+ A fetch request says what a collection is ordered by with `by_column()` and
712
+ `by_expression()`, each of which takes a direction:
713
+
714
+ ```python
715
+ from pyoq.relations import by_column, by_expression
716
+
717
+ FetchRequest(
718
+ employees,
719
+ order_by=(by_expression("lower(name)"), by_column("id", descending=True)),
720
+ )
721
+ ```
722
+
723
+ A column is checked against the related table, because the plan can see the
724
+ schema. An expression is named rather than carried, because a plan is built
725
+ below the layer that writes queries and does not reach up into one. The name
726
+ is what the plan reports when it explains itself, and naming it is enough to
727
+ plan with: whether a collection is ordered at all is what decides how it can
728
+ be fetched, not what it is ordered by. The expression itself is given to
729
+ whatever carries the plan out.
730
+
731
+ A to-one relation rides the join, because it is one row the join already
732
+ carries. A collection is nested where the dialect can aggregate one, and falls
733
+ back to select-in where it cannot, or where the collection's order matters and
734
+ the dialect cannot order inside an aggregate. That last case is measured rather
735
+ than assumed: SQLite from 3.44 and PostgreSQL both accept an order inside their
736
+ aggregate, while MySQL rejects it outright, and MySQL discards an order given in
737
+ a derived table instead of honouring it. An aggregate in the wrong order is
738
+ worse than a second query, so PyOQ takes the second query.
739
+
740
+ A plan that names a strategy the dialect cannot carry out is refused rather than
741
+ quietly changed, because a caller who asked for an ordered nested collection and
742
+ received an unordered one has no way to notice.
743
+
744
+ `validate_fetch_plan()` checks a plan against the relation graph once, before
745
+ any query runs, so a mistake is reported against the schema rather than part-way
746
+ through a fetch. A relation the table does not have is refused, a nested
747
+ relation is checked against its own table rather than the root, and fetching one
748
+ relation twice at the same level is refused. The same relation may appear at
749
+ different levels, because a key pointing back is a different fetch rather than a
750
+ repeat, and that is also why plan depth is bounded.
751
+
752
+ ### Relation loading state
753
+
754
+ A relation is loaded, known to be absent, or was never fetched. Rows are
755
+ detached values, so reading one never reaches a database, which is what makes
756
+ the third state necessary: without it a relation nobody asked for would be
757
+ indistinguishable from one that resolved to nothing.
758
+
759
+ ```python
760
+ team.employees.value # the fetched rows
761
+ person.manager.value # None, when the key resolved to nothing
762
+ person.badge.value # raises: this relation was not fetched
763
+ ```
764
+
765
+ Reading a relation nobody fetched raises rather than answering, because any
766
+ answer would be a guess and no answer can be produced without I/O.
767
+
768
+ Run the model against a reflected database with:
769
+
770
+ ```console
771
+ python examples/relation_model.py
772
+ ```
773
+
774
+ ## Row identity
775
+
776
+ A join repeats a parent row once per child, and an outer join invents a child
777
+ made entirely of nulls. Both are answered by asking what identifies a row, so
778
+ identity is decided before any row is built:
779
+
780
+ ```python
781
+ from pyoq.hydration import IdentityMap, row_key_for
782
+
783
+ key = row_key_for(table, reference, projection.position_of)
784
+ identity = key.identify(result_row) # None when the result holds no row here
785
+ ```
786
+
787
+ A primary key identifies a table. Failing that, a unique key no part of which
788
+ is nullable. A table offering neither is identified by everything projected
789
+ from it, which is the only sound answer left: rows it cannot tell apart are
790
+ rows nobody can tell apart. A null among the identifying values means the
791
+ result holds no row there rather than a row whose identity happens to be null,
792
+ which is how an absent child is recognized.
793
+
794
+ Identity is by value however a driver spells it, so buffers and arrays compare
795
+ by their contents. A value no dictionary can hold, or a key column the query
796
+ does not select, is reported against the table it belongs to rather than
797
+ surfacing as a bare type or lookup error from inside a fetch.
798
+
799
+ `IdentityMap` builds each distinct row once. It lives for one fetch and is
800
+ discarded with it, so a row it returns can never be a stale row from an earlier
801
+ fetch.
802
+
803
+ ## Hydration
804
+
805
+ A join hands back one row per combination, so the same parent arrives once per
806
+ child and the same child can arrive under several parents. `hydrate()` reads a
807
+ result back into rows that carry what the plan asked for:
808
+
809
+ ```python
810
+ from pyoq.hydration import hydrate
811
+
812
+ teams = hydrate(node, result_rows)
813
+ ```
814
+
815
+ Rows come back in the order the result first mentioned them, a row the result
816
+ mentions twice is built once, and children keep the order they first appeared
817
+ in. A null among a child's identifying values means the result holds no child
818
+ there, so an outer join yields an empty collection or an absent value rather
819
+ than a row made of nulls.
820
+
821
+ The result is read twice: once to group it and once to construct it. A frozen
822
+ row cannot be handed its children after it exists, so its children have to be
823
+ known before it is built.
824
+
825
+ One table read at two places in a plan yields separate rows, because each
826
+ carries different children and one row cannot hold both. Within one place, a
827
+ child several parents share is one object.
828
+
829
+ A to-one relation that returns several distinct rows for one parent is
830
+ reported. The schema said those columns were unique and the result disagreed,
831
+ and discarding rows would hide the disagreement.
832
+
833
+ Throughput and memory are held to a contract:
834
+
835
+ ```console
836
+ python -m benchmarks.hydration
837
+ ```
838
+
839
+ ## Query diagnostics
840
+
841
+ A query issued once per row of a previous result is invisible from inside the
842
+ loop that causes it. Recording what a scope executed makes it visible from
843
+ outside:
844
+
845
+ ```python
846
+ from pyoq.diagnostics import QueryObserver
847
+
848
+ observer = QueryObserver()
849
+ ...
850
+ for repeated in observer.repeated():
851
+ print(repeated.describe())
852
+ ```
853
+
854
+ ```console
855
+ 4 executions of SELECT id FROM person WHERE team_id = ? from app/teams.py:31 in members
856
+ ```
857
+
858
+ One query is recognized across the many times it is executed. Its bound values
859
+ are already placeholders, a list of any length collapses to one, a write chunked
860
+ to fit a parameter limit collapses however many chunks it took, and the three
861
+ placeholder styles reach the same shape, so one query compiled for three
862
+ dialects is one shape. Placeholders that are not a list stay apart, so a query
863
+ with two conditions never merges into one with a single condition.
864
+
865
+ A report carries the first place outside PyOQ that issued the query, which is
866
+ the caller's own code rather than the library's. An interpreter that offers no
867
+ frames reports no place rather than refusing to run.
868
+
869
+ Diagnostics are safe to log. A shape carries no bound value, so nothing a query
870
+ was asked about appears in a report, and that holds whether or not a parameter
871
+ was marked sensitive.
872
+
873
+ Diagnostics must not become the thing that exhausts a process, so the number of
874
+ distinct shapes and the number of places per shape are both capped. Executions
875
+ keep being counted after the caps are reached, and shapes seen beyond the cap
876
+ are counted through `unrecorded_shapes`.
877
+
878
+ An observer is safe to share. A pool hands connections to whichever thread asks,
879
+ so reading a report while several threads record must not fail.
880
+
881
+ ### Query budgets
882
+
883
+ A repeated query that is only reported is a repeated query that still ships. A
884
+ budget turns the same observation into a refusal:
885
+
886
+ ```python
887
+ from pyoq.diagnostics import QueryBudget, QueryScope
888
+
889
+ scope = QueryScope(QueryBudget(maximum_repeats=3))
890
+ ```
891
+
892
+ ```console
893
+ one statement ran 4 times in this scope, beyond the 3 it was allowed
894
+ from app/teams.py:31 in members: SELECT id FROM person WHERE team_id = ?
895
+ ```
896
+
897
+ `maximum_repeats` is the one that catches an N+1 access, because the shape
898
+ executed once per row of an earlier result is the shape that repeats.
899
+ `maximum_queries` caps the scope as a whole. A refusal names the shape, the
900
+ count, and the place, and carries no bound value. The statement that caused it
901
+ is recorded before it is judged, so a report taken afterwards explains the
902
+ refusal.
903
+
904
+ Holding a budget costs almost nothing per statement, because only the shape
905
+ just recorded is judged rather than every shape a scope has seen.
906
+
907
+ ### Select-in fetching
908
+
909
+ Select-in is the strategy that turns an N+1 access into a bounded number of
910
+ statements. `select_in_conditions()` produces one predicate per batch:
911
+
912
+ ```python
913
+ from pyoq.fetching import select_in_conditions
914
+
915
+ for condition in select_in_conditions(
916
+ relation, columns, parent_keys, maximum_parameters=999
917
+ ):
918
+ rows = database.many(select(...).from_(source).where(condition))
919
+ ```
920
+
921
+ A key of one column becomes a membership test. A key of several becomes a
922
+ choice between equalities rather than a row value, because every dialect
923
+ renders the first alike and they do not all render the second alike.
924
+
925
+ The predicate is built from expression nodes rather than through the typed
926
+ query API. Keys come back from a database as values whose types the caller has
927
+ already checked, so routing them through a typed column would fight the column's
928
+ own type without protecting the query the caller wrote.
929
+
930
+ A contract proves the reduction on SQLite, PostgreSQL, and MySQL alike: the same
931
+ rows come back from far fewer statements than asking once per parent.
932
+
933
+ ### Joined fetching
934
+
935
+ A to-one relation is one row the join already carries, so it costs no extra
936
+ statement. What it costs is knowing where each table's values land:
937
+
938
+ ```python
939
+ from pyoq.fetching import join_condition, projection_layout
940
+
941
+ child_at, parent_at = projection_layout((2, 2))
942
+ query = (
943
+ select(*child_columns, *parent_columns)
944
+ .from_(table_source(children))
945
+ .left_join(table_source(parents))
946
+ .on(join_condition(child_key, parent_key))
947
+ )
948
+ ```
949
+
950
+ Hydration reads a result by position, so the layout a projection is laid out
951
+ with and the layout it is read back by have to be the same one. Deriving both
952
+ from `projection_layout()` keeps them from drifting apart as a plan grows.
953
+
954
+ A left join leaves a to-one relation absent where nothing matched, which the
955
+ relation states already distinguish from a relation nobody fetched. This is
956
+ proved on SQLite, PostgreSQL, and MySQL alike: one query, hydrated back into
957
+ parents carrying their relation, with the unmatched half absent.
958
+
959
+ ### Nested collections
960
+
961
+ A nested collection gathers a relation's rows into one value beside their
962
+ parent, so one statement returns the graph:
963
+
964
+ ```console
965
+ SELECT "parent"."id",
966
+ (SELECT COALESCE(json_group_array(json_object('id', "child"."id")
967
+ ORDER BY "child"."id"), json_array())
968
+ FROM "child" WHERE ("child"."parent_id" = "parent"."id"))
969
+ FROM "parent"
970
+ ```
971
+
972
+ Each dialect gathers rows with its own functions: `json_group_array` and
973
+ `json_object` on SQLite, `jsonb_agg` and `jsonb_build_object` on PostgreSQL,
974
+ `JSON_ARRAYAGG` and `JSON_OBJECT` on MySQL. An empty relation is coalesced to an
975
+ empty collection rather than left null, so a caller need not tell a relation
976
+ with no rows from a fetch that did not happen.
977
+
978
+ MySQL refuses an ordered collection rather than returning one in the wrong
979
+ order, and says to fetch it with select-in instead. Ordering inside a SQLite
980
+ aggregate needs SQLite 3.44 or later.
981
+
982
+ A collection reads one table, because a collection is the rows of one relation
983
+ and anything wider is a query rather than a relation.
984
+
985
+ `decode_collection()` reads the value back into rows. What a driver hands over
986
+ differs: SQLite and MySQL return JSON text while PostgreSQL returns a list it
987
+ has already decoded, and both arrive at the same rows. Values come back in the
988
+ order their names were given, so a caller reads them like any other row.
989
+
990
+ Nothing is trusted to hold what it promised. A collection that is not a list of
991
+ objects is refused, and a row missing a value it was asked for is reported by
992
+ name and position. A value that is null is a value, because a column that held
993
+ null is not a column that was missing. Values a query did not ask for are left
994
+ alone, so a table gaining a column does not break a fetch.
995
+
996
+ `hydrate_collection()` turns that into a loaded relation. An empty collection is
997
+ loaded and holds nothing, which is not the same as a relation nobody fetched.
998
+
999
+ ### Relation batching
1000
+
1001
+ An N+1 access asks for one parent's children at a time. Gathering the keys first
1002
+ turns that into one question, and a dialect's parameter limit turns it into a
1003
+ bounded number of questions rather than one per parent:
1004
+
1005
+ ```python
1006
+ from pyoq.relations import RelationBatchLoader
1007
+
1008
+ loader = RelationBatchLoader()
1009
+ for team in teams:
1010
+ loader.enqueue(members, team.key)
1011
+ batches = loader.batches(maximum_parameters=999)
1012
+ ```
1013
+
1014
+ A key spanning several columns costs one bound value per column, so a limit
1015
+ counts values rather than keys, and a key wider than the limit is refused rather
1016
+ than split. Keys keep their order and never repeat. A key that is null reaches
1017
+ nothing, so it is counted through `skipped_keys` rather than asked about, which
1018
+ would cost a parameter to learn nothing.
1019
+
1020
+ A loader belongs to one scope, and each relation it holds is bounded on its own.
1021
+
1022
+ ## One chain, from building to reading
1023
+
1024
+ A builder describes a statement and never reaches a database. That is what makes
1025
+ one safe to share, cache, and test, and it does not change. A context adds the
1026
+ other half: it holds the database to run against and hands back builders that
1027
+ carry it, so a chain can end in a fetch instead of being handed to an executor.
1028
+
1029
+ ```python
1030
+ from pyoq.dsl import using
1031
+
1032
+ dsl = using(database)
1033
+
1034
+ dsl.select(TITLE_ID, TITLE_NAME, TITLE_PRICE) \
1035
+ .from_(table_source(TITLES)) \
1036
+ .where(TITLE_PRICE.gt(Decimal("10.00"))) \
1037
+ .order_by(TITLE_ID) \
1038
+ .fetch_all() \
1039
+ .to_list()
1040
+ ```
1041
+
1042
+ A generated table names its own columns, so a query over all of them does not
1043
+ have to list them:
1044
+
1045
+ ```python
1046
+ dsl.select_from(TITLES) \
1047
+ .where(TITLE_PRICE.gt(Decimal("10.00"))) \
1048
+ .order_by(TITLE_ID) \
1049
+ .fetch_all() \
1050
+ .into(TitleRow)
1051
+ ```
1052
+
1053
+ The SQL is `SELECT *`. PyOQ still carries the column list, so the row keeps
1054
+ its names and each value is read back as the type the column holds, and that
1055
+ order is the order the generated row type takes its arguments in. A package
1056
+ that no longer matches its schema is refused rather than read by the wrong
1057
+ columns.
1058
+
1059
+ The shape a caller wants is asked for at the end rather than assembled around
1060
+ the call:
1061
+
1062
+ ```python
1063
+ rows = dsl.select(...).from_(...).fetch_all()
1064
+
1065
+ rows.to_list() # [(1, "A Guide", Decimal("12.50")), ...]
1066
+ rows.into(Title) # [Title(1, "A Guide", Decimal("12.50")), ...]
1067
+ rows.map(lambda r: r[1])
1068
+ rows.to_dicts() # [{"id": 1, "name": "A Guide", ...}, ...]
1069
+ rows.to_dict() # one row keyed, or an error saying which way it failed
1070
+ rows.one() # exactly one row, or an error saying which way it failed
1071
+ rows.first() # the row at the front, or None
1072
+ ```
1073
+
1074
+ A chain that ends in one row asks the same questions of it. The row is a
1075
+ tuple, so it still unpacks, indexes, and compares the way it always did:
1076
+
1077
+ ```python
1078
+ row = dsl.select(TITLE_ID, TITLE_NAME).from_(...).fetch_one()
1079
+
1080
+ row.to_dict() # {"id": 1, "name": "A Guide"}
1081
+ row.to_list() # [1, "A Guide"]
1082
+ row.to_tuple() # (1, "A Guide"), typed as it was selected
1083
+ row.into(Title) # Title(1, "A Guide")
1084
+ identifier, name = row # still a tuple
1085
+ ```
1086
+
1087
+ A row that may be missing is the one shape that had to be tested for before
1088
+ it could be built, so the chain takes the type instead:
1089
+
1090
+ ```python
1091
+ dsl.select(...).where(...).fetch_optional_into(Title) # Title | None
1092
+ ```
1093
+
1094
+ `into` spreads each row across a constructor in the order it was selected, so a
1095
+ dataclass or a named tuple needs nothing else. `to_dicts` keys by the name each
1096
+ column came back under, and refuses when a column has none rather than inventing
1097
+ one.
1098
+
1099
+ A query can end in a single row or a single value instead:
1100
+
1101
+ ```python
1102
+ dsl.select(TITLE_NAME).from_(source).where(TITLE_ID.eq(1)).fetch_one()
1103
+ dsl.select(TITLE_NAME).from_(source).where(TITLE_ID.eq(1)).fetch_optional()
1104
+ dsl.select(TITLE_NAME).from_(source).where(TITLE_ID.eq(1)).fetch_value()
1105
+ ```
1106
+
1107
+ `fetch_value` is only offered to a query that selected one column. That is a
1108
+ statement about the type rather than a check when it runs.
1109
+
1110
+ Writes chain the same way and end in the number of rows the database wrote:
1111
+
1112
+ ```python
1113
+ dsl.insert_into(TITLES, TITLE_ID, TITLE_NAME).values(1, "A Guide").execute()
1114
+ dsl.update(TITLES).set(TITLE_PRICE, Decimal("20.00")).where(TITLE_ID.eq(1)).execute()
1115
+ dsl.delete_from(TITLES).where(TITLE_ID.eq(1)).execute()
1116
+
1117
+ dsl.insert_into(TITLES, TITLE_ID, TITLE_NAME) \
1118
+ .values(1, "A Guide") \
1119
+ .on_conflict_do_update(TITLE_ID) \
1120
+ .set(TITLE_NAME, "A Guide") \
1121
+ .returning(TITLE_ID, TITLE_NAME) \
1122
+ .fetch_all() \
1123
+ .into(Title)
1124
+ ```
1125
+
1126
+ A set operation orders by a column it selected, because its operands may read
1127
+ different tables and a table-qualified name means nothing once they are
1128
+ combined. Naming anything else there is refused rather than sent to a server
1129
+ that would reject it.
1130
+
1131
+ Types are carried the whole way. `select(TITLE_ID, TITLE_NAME, TITLE_PRICE)`
1132
+ gives a chain over `tuple[int, str, Decimal]`, `fetch_all().to_list()` is a
1133
+ `list[tuple[int, str, Decimal]]`, and `fetch_one()` is the tuple itself.
1134
+
1135
+ Unwrapping a bound query with `query`, or a bound write with `statement`, gives
1136
+ back the ordinary builder, which cannot reach a database. Anything built that
1137
+ way still runs through `dsl.fetch(...)` or `dsl.execute(...)`.
1138
+
1139
+ ### An asynchronous database
1140
+
1141
+ `using` answers the same question either way, so an asynchronous database gives
1142
+ an asynchronous context. Only the ending changes:
1143
+
1144
+ ```python
1145
+ rows = await (
1146
+ using(database)
1147
+ .select(TITLE_ID, TITLE_NAME)
1148
+ .from_(table_source(TITLES))
1149
+ .order_by(TITLE_ID)
1150
+ .fetch_all()
1151
+ )
1152
+ rows.into(Title)
1153
+
1154
+ await using(database).insert_into(TITLES, TITLE_ID).values(1).execute()
1155
+ ```
1156
+
1157
+ The chain is built the same way and describes the same statement. `fetch_all`,
1158
+ `fetch_one`, `fetch_optional`, `fetch_value`, and `execute` are awaited because
1159
+ the database is, and the types carry through exactly as they do synchronously.
1160
+
1161
+ ## Django
1162
+
1163
+ Django 4.2 through 6.x, on any Python each of them supports.
1164
+
1165
+ [`docs/django.md`](docs/django.md) is a step by step guide from an empty project
1166
+ to typed reads, writes, transactions, policies, and events. What follows is the
1167
+ reference.
1168
+
1169
+ PyOQ runs on the connection Django already has open:
1170
+
1171
+ ```python
1172
+ from pyoq.django import DjangoOperations
1173
+
1174
+ database = DjangoOperations() # or DjangoOperations("reporting")
1175
+ rows = database.many(select(...).from_(source))
1176
+ ```
1177
+
1178
+ ```console
1179
+ python -m pip install "pyoq-sql[django]"
1180
+ ```
1181
+
1182
+ No pool is opened beside Django's. Two pools against one database is two views
1183
+ of what has been committed, and the point of running inside Django is that there
1184
+ is one. The connection is resolved for each statement rather than held, because
1185
+ Django gives each thread a different one and closes them between requests.
1186
+
1187
+ The dialect comes from the connection rather than from the caller, since it is
1188
+ the connection that is on the socket. Values are adapted the same way they are
1189
+ on PyOQ's own connections, because a backend does not care whose connection it
1190
+ is: a decimal or a date still has to reach it in the shape its driver accepts.
1191
+ A backend PyOQ has no dialect for is refused by name.
1192
+
1193
+ A timeout is applied where the backend has a setting for one, and the session is
1194
+ left exactly as it was found. Inside a transaction the setting is local, so the
1195
+ database reverts it when the transaction ends; outside one, the previous value is
1196
+ read first and put back after. A restore never replaces the failure that made it
1197
+ necessary, because a statement that failed can leave a connection unable to
1198
+ accept the next one. SQLite has no such setting, and MySQL's covers reads only,
1199
+ so a caller is given what the backend can honour rather than a promise it
1200
+ cannot keep. A caller's cancellation is honoured before and after a statement
1201
+ regardless.
1202
+
1203
+ Routing is a question about a model, because that is the only thing a Django
1204
+ router is given to decide on:
1205
+
1206
+ ```python
1207
+ database = DjangoOperations.for_model(Report) # wherever this is routed
1208
+ database = DjangoOperations.for_model(Report, write=True)
1209
+ ```
1210
+
1211
+ A caller without a model names the alias instead.
1212
+
1213
+ ### Transactions
1214
+
1215
+ PyOQ takes part in the transaction Django opened and never opens one of its own.
1216
+ There is no commit or rollback on `DjangoOperations`, because the block that
1217
+ starts a transaction is the one that ends it:
1218
+
1219
+ ```python
1220
+ with database.atomic():
1221
+ database.execute(insert_into(REPORTS, name).values("quarterly"))
1222
+ database.on_commit(lambda: notify("saved"))
1223
+ ```
1224
+
1225
+ Ask the database for its own block rather than reaching for Django's. Django's
1226
+ `transaction.atomic()` covers the default alias unless told which to use, so a
1227
+ caller working against another database inside a bare block has no transaction
1228
+ there at all and nothing reports it. `database.atomic()` always means this
1229
+ database. The same holds for `database.on_commit()`, which waits for this
1230
+ database to commit.
1231
+
1232
+ Nesting a block is a savepoint, and `atomic(savepoint=False)`,
1233
+ `atomic(durable=True)`, and `on_commit(robust=True)` behave as Django defines
1234
+ them. `database.in_transaction` reports whether a transaction is open.
1235
+
1236
+ Test isolation needs nothing extra. A Django test case wraps its test in a block
1237
+ and rolls it back, and PyOQ writes through the connection that block belongs to.
1238
+
1239
+ An async view reaches a statement through Django's own bridging:
1240
+
1241
+ ```python
1242
+ await sync_to_async(read_reports, thread_sensitive=True)(database)
1243
+ ```
1244
+
1245
+ Django refuses a synchronous statement called directly from a coroutine, and
1246
+ PyOQ inherits that guard. Nothing here shares transaction state across the
1247
+ boundary, because a transaction belongs to the connection that a thread holds.
1248
+
1249
+ ### Generating from migrations
1250
+
1251
+ Add PyOQ to the project and tell it where generated code belongs:
1252
+
1253
+ ```python
1254
+ INSTALLED_APPS = ["pyoq.django", ...]
1255
+
1256
+ PYOQ = {
1257
+ "codegen_directory": "app/_generated",
1258
+ "codegen_package": "app._generated",
1259
+ }
1260
+ ```
1261
+
1262
+ ```bash
1263
+ python manage.py pyoq_codegen
1264
+ python manage.py pyoq_codegen catalog --database reporting
1265
+ python manage.py pyoq_codegen --dry-run
1266
+ python manage.py pyoq_codegen --check
1267
+ ```
1268
+
1269
+ The schema comes from the migration files, not from a database. Migrations are
1270
+ what the schema is going to be, and a developer generating types has usually
1271
+ not applied them yet. Nothing connects: Django answers what column a field
1272
+ takes on a backend without consulting a server, so the alias selects the
1273
+ dialect rather than opening a socket.
1274
+
1275
+ What that yields is the schema the database will really have, which is not
1276
+ always the one the models appear to describe:
1277
+
1278
+ - A Django default is applied in Python, so the column has no default. PyOQ
1279
+ says so rather than inviting an insert the database would refuse. The model's
1280
+ default is kept as column metadata.
1281
+ - `on_delete` is carried out by Django, and the constraint it writes names no
1282
+ action at all. The relation records what the database will do, with the
1283
+ model's choice beside it.
1284
+ - A unique constraint carrying a condition is not a key. It is unique among the
1285
+ rows it matches and not among the others, and treating it as a key would
1286
+ describe a relation reaching many rows as reaching one.
1287
+ - A many-to-many field owns a table that migration state never lists, so PyOQ
1288
+ describes it from the field. A through model the project wrote is already a
1289
+ model and is left alone.
1290
+
1291
+ Generation itself is the same pipeline every other entry point uses, so
1292
+ staging, validation, the manifest, drift detection, and the atomic replacement
1293
+ behave identically here.
1294
+
1295
+ To generate after every migration, put PyOQ ahead of the app that supplies the
1296
+ original command and ask for it:
1297
+
1298
+ ```python
1299
+ PYOQ = {..., "codegen_after_makemigrations": True}
1300
+ ```
1301
+
1302
+ Django's own command runs first and unchanged. A run that writes no migration,
1303
+ such as `--dry-run` or `--check`, generates nothing: the files on disk still
1304
+ describe the schema before the change, and generating from them would look like
1305
+ it had worked.
1306
+
1307
+ ### A project to read
1308
+
1309
+ [`examples/django_project`](examples/django_project) is a project laid out the
1310
+ way a real one is: settings, a URLconf, WSGI and ASGI entry points, the admin,
1311
+ two applications with models and migrations, a template, and views. It has two
1312
+ databases and a router, and its models are chosen to show what reaches the
1313
+ database and what stays in Python.
1314
+
1315
+ [`examples/django_integration.py`](examples/django_integration.py) applies its
1316
+ migrations, generates from them, writes through the ORM and PyOQ on the same
1317
+ connection, and then exercises the project through its own views, including an
1318
+ async one, along with transactions, savepoints, and commit hooks:
1319
+
1320
+ ```bash
1321
+ python examples/django_integration.py
1322
+ ```
1323
+
1324
+ ## FastAPI
1325
+
1326
+ The pool belongs to the application, so the lifespan opens it once and gives it
1327
+ back once:
1328
+
1329
+ ```python
1330
+ from pyoq.fastapi import SyncDatabase, database_lifespan
1331
+
1332
+ def open_notes() -> SyncDatabase:
1333
+ pool = SQLitePool(SQLiteConnectionFactory(path))
1334
+ return SyncDatabase(SQLiteExecutor(pool), pool.close)
1335
+
1336
+ app = FastAPI(lifespan=database_lifespan(synchronous={"default": open_notes}))
1337
+ ```
1338
+
1339
+ A database that fails to open takes the application down, and the ones already
1340
+ open are given back first. Starting up halfway and serving requests against a
1341
+ partly opened application is worse than not starting.
1342
+
1343
+ Routes ask for what they need:
1344
+
1345
+ ```python
1346
+ import pyoq.fastapi as pyoq_fastapi
1347
+
1348
+ Database = Annotated[QueryOperations, Depends(pyoq_fastapi.database())]
1349
+ Transaction = Annotated[QueryOperations, Depends(pyoq_fastapi.transaction())]
1350
+
1351
+ @app.get("/notes")
1352
+ def list_notes(database: Database) -> dict[str, object]:
1353
+ return {"notes": using(database).select(...).fetch_all().to_dicts()}
1354
+
1355
+ @app.post("/notes")
1356
+ def add_note(database: Transaction) -> dict[str, int]:
1357
+ return {"written": using(database).insert_into(...).values(...).execute()}
1358
+ ```
1359
+
1360
+ A request that returns commits. A request that raises rolls back, and so does
1361
+ one the client gave up on: FastAPI tears the dependency down either way, and the
1362
+ transaction is told what tore it down. An `HTTPException` counts as raising,
1363
+ because a refused request is still a refused request.
1364
+
1365
+ Asking twice for the same dependency gives back the same object, so an override
1366
+ matches:
1367
+
1368
+ ```python
1369
+ app.dependency_overrides[pyoq_fastapi.database()] = lambda: substitute
1370
+ ```
1371
+
1372
+ ### Which database a route may use
1373
+
1374
+ `database()` and `transaction()` are synchronous. `async_database()` and
1375
+ `async_transaction()` are their counterparts.
1376
+
1377
+ A synchronous transaction belongs to a synchronous route. FastAPI runs a
1378
+ synchronous dependency in a worker thread and an async route body on the event
1379
+ loop, and a transaction belongs to the thread that opened it. PyOQ refuses the
1380
+ mismatch rather than running a statement somewhere it does not belong, so an
1381
+ async route takes `async_transaction()`.
1382
+
1383
+ ### Request budgets
1384
+
1385
+ A budget is per request, so one request cannot spend another's allowance:
1386
+
1387
+ ```python
1388
+ Budgeted = Annotated[
1389
+ QueryOperations,
1390
+ Depends(pyoq_fastapi.scoped(QueryBudget(maximum_queries=20))),
1391
+ ]
1392
+ ```
1393
+
1394
+ Every read and write is recorded before the driver sees it, so the statement
1395
+ that goes beyond the allowance never reaches the database. `maximum_repeats` is
1396
+ the one that catches a query running once per row of an earlier result.
1397
+
1398
+ `ScopedOperations` and `AsyncScopedOperations` do the counting and are not
1399
+ specific to FastAPI. Either wraps any database:
1400
+
1401
+ ```python
1402
+ from pyoq.diagnostics import QueryScope, ScopedOperations
1403
+
1404
+ scope = QueryScope(QueryBudget(maximum_queries=20))
1405
+ counted = ScopedOperations(database, scope)
1406
+ ```
1407
+
1408
+ See [`examples/fastapi_application.py`](examples/fastapi_application.py) for a
1409
+ runnable application covering all of it.
1410
+
1411
+ ## Sanic
1412
+
1413
+ Sanic starts its workers as separate processes. A connection made before they
1414
+ exist would be shared by all of them, and two processes talking over one socket
1415
+ corrupt each other's results. So every pool is opened from the listener that
1416
+ runs inside each worker:
1417
+
1418
+ ```python
1419
+ from pyoq.sanic import AsyncDatabase, attach_databases
1420
+
1421
+ async def open_notes() -> AsyncDatabase:
1422
+ pool = await PostgresAsyncPool(PostgresAsyncConnectionFactory(dsn)).open()
1423
+ return AsyncDatabase(PostgresAsyncExecutor(pool), pool.close)
1424
+
1425
+ attach_databases(app, asynchronous={"default": open_notes})
1426
+ ```
1427
+
1428
+ A database that fails to open takes the worker down, and the ones already open
1429
+ are given back first. Every one a worker opened is closed when it stops.
1430
+
1431
+ ### One transaction per request
1432
+
1433
+ A transaction belongs to the block the handler enters:
1434
+
1435
+ ```python
1436
+ import pyoq.sanic as pyoq_sanic
1437
+
1438
+ @app.post("/notes")
1439
+ async def add_note(request):
1440
+ async with pyoq_sanic.transaction(request) as database:
1441
+ ...
1442
+ ```
1443
+
1444
+ Leaving the block normally commits. Leaving it any other way rolls back: a
1445
+ handler that raised, and a request the server abandoned.
1446
+
1447
+ It is a block rather than a pair of middlewares for a reason. A server that is
1448
+ told the client has gone cancels the task it called the application on, and
1449
+ nothing after the handler runs, response middleware included. A transaction
1450
+ ended there would keep its connection for as long as the worker lives, inside an
1451
+ open transaction, holding whatever it locked. Python ends a block whatever
1452
+ happens to the task running it, so the connection goes back to the pool.
1453
+
1454
+ A commit that fails raises on the way out of the block, which Sanic answers the
1455
+ way it answers any other failure in a handler.
1456
+
1457
+ `synchronous_transaction` is the same for a database that is not awaited.
1458
+
1459
+ ### Request budgets
1460
+
1461
+ ```python
1462
+ pyoq_sanic.attach_request_budget(app, QueryBudget(maximum_queries=20))
1463
+
1464
+ @app.get("/notes")
1465
+ async def list_notes(request):
1466
+ database = pyoq_sanic.budgeted_of(request)
1467
+ ```
1468
+
1469
+ A budget belongs to the request, so one request cannot spend another's.
1470
+ `attach_synchronous_request_budget` and `synchronous_budgeted_of` are the
1471
+ counterparts.
1472
+
1473
+ See [`examples/sanic_application.py`](examples/sanic_application.py) for a
1474
+ runnable application covering all of it.
1475
+
1476
+ ## Policies
1477
+
1478
+ A rule written against SQL text can be walked around with an alias, a subquery,
1479
+ or a common table. A policy is written against the statement's own nodes, before
1480
+ any SQL exists, so every one of those routes carries it:
1481
+
1482
+ ```python
1483
+ from pyoq.policies import SoftDelete, TenantScope, governed
1484
+
1485
+ scoped = governed(database, [TenantScope("tenant_id", 7), SoftDelete("deleted_at")])
1486
+ ```
1487
+
1488
+ ```sql
1489
+ SELECT COUNT(*) FROM "notes" AS "n"
1490
+ WHERE (("n"."tenant_id" = ?) AND ("n"."deleted_at" IS NULL))
1491
+ ```
1492
+
1493
+ The condition follows the name the statement gave the table, joins are scoped
1494
+ beside it, a subquery and a common table are scoped inside themselves, and both
1495
+ halves of a set operation are scoped. A write carries the scope as a value, so a
1496
+ row cannot be created outside the scope that would then be unable to see it, and
1497
+ a caller that names the column itself has its value replaced rather than
1498
+ honoured. An update or a delete that said it meant every row now means every row
1499
+ the policy admits.
1500
+
1501
+ ### What is available
1502
+
1503
+ - `TenantScope(column, value)` puts every read and every write inside one scope
1504
+ - `SoftDelete(column)` keeps a row marked deleted out of every read
1505
+ - `AllowedTables(names)` refuses a statement that touches anything else
1506
+ - `RowConstraint(table, build)` narrows one table by a condition of your own
1507
+
1508
+ ### What cannot be governed
1509
+
1510
+ SQL that was compiled elsewhere has no statement left to read, so it is refused
1511
+ rather than quietly allowed. So is raw SQL written into an expression. A project
1512
+ that means to run them says so once:
1513
+
1514
+ ```python
1515
+ governed(database, [...], raw_sql=True)
1516
+ ```
1517
+
1518
+ ### Transactions and streams
1519
+
1520
+ A governed database opens its own transactions and streams its own reads, both
1521
+ held to the same rules:
1522
+
1523
+ ```python
1524
+ with scoped.transaction() as active:
1525
+ ...
1526
+ with scoped.stream(query) as rows:
1527
+ ...
1528
+ ```
1529
+
1530
+ Without those a caller would reach past the policy to the database underneath to
1531
+ open one, and everything inside it would be ungoverned. A dialect's own
1532
+ streaming settings belong to the database underneath, which is where it was
1533
+ given them.
1534
+
1535
+ ### Going around a policy
1536
+
1537
+ ```python
1538
+ with administrative_bypass(scoped, "restoring a withdrawn note", audit) as free:
1539
+ ...
1540
+ ```
1541
+
1542
+ The ungoverned database is yielded rather than returned, so it belongs to the
1543
+ block and cannot be kept past it. A reason is required, and entering and leaving
1544
+ are both written to the audit.
1545
+
1546
+ See [`examples/policies.py`](examples/policies.py) for a runnable walkthrough.
1547
+
1548
+ ## Reading a column as the type it was declared to hold
1549
+
1550
+ A driver answers with what its own protocol carries, which is not always what a
1551
+ column was declared to be. SQLite hands back a float for a decimal and a string
1552
+ for a date, and psycopg hands back a view of its buffer for bytes. A descriptor
1553
+ that says `Decimal` and yields a float is a promise the runtime does not keep,
1554
+ and money is the usual casualty.
1555
+
1556
+ A descriptor carries the type it was declared with, so a row is given back as
1557
+ the row was described:
1558
+
1559
+ ```python
1560
+ PRICE = ColumnDescriptor[Decimal](..., value_type=Decimal)
1561
+
1562
+ row = using(database).select(PRICE).from_(source).fetch_one()
1563
+ # Decimal('12.50'), not 12.5
1564
+ ```
1565
+
1566
+ Generated descriptors declare it for you. `Decimal`, `date`, `datetime`,
1567
+ `time`, `timedelta`, `UUID`, `bytes`, `bool`, `int`, `float`, `str`, and JSON
1568
+ are read from whatever a driver answered with. A generated enumeration reads
1569
+ back as a member of itself rather than as the string stored for it, and an
1570
+ array column reads back as the tuple its descriptor declares rather than as
1571
+ the list a driver hands over.
1572
+
1573
+ This happens on every path that answers with rows: `fetch_one`, `fetch_all`,
1574
+ scalars, bulk statements that return rows, streams, and the asynchronous
1575
+ counterpart of each. A driver decides what it carries and decoding decides
1576
+ what a caller is given, so two paths that decode differently would answer the
1577
+ same query with different types.
1578
+
1579
+ A column that declared nothing is handed back exactly as it came, so a caller
1580
+ that never said what it wanted pays nothing for the question.
1581
+
1582
+ A value that cannot be read as what it was declared to be raises, because
1583
+ handing back the wrong type silently is the defect this exists to stop. What a
1584
+ driver already lost is lost: PyOQ converts what it is given and cannot recover
1585
+ precision a database did not keep.
1586
+
1587
+ A nested collection arrives as JSON, which spells a date as a string and a
1588
+ decimal as a number, so it is told what its columns hold in the same way:
1589
+
1590
+ ```python
1591
+ decode_collection(payload, ("id", "released"), (int, date))
1592
+ hydrate_collection(payload, ("id", "released"), construct, (int, date))
1593
+ ```
1594
+
1595
+ ## Observability
1596
+
1597
+ Instrumentation is a database that wraps another one, so a project that wants
1598
+ none holds the plain database and pays nothing at all:
1599
+
1600
+ ```python
1601
+ from pyoq.diagnostics import CollectingSink, InstrumentedOperations
1602
+
1603
+ watched = InstrumentedOperations(database, sink)
1604
+ ```
1605
+
1606
+ Every statement is reported before it reaches a driver and again when it is
1607
+ answered, at the one place they all pass. An event carries the shape of the
1608
+ statement, how many parameters it had, how long it took, and how many rows came
1609
+ back.
1610
+
1611
+ ### What an event may carry
1612
+
1613
+ Nothing a query was asked about, unless a policy says so:
1614
+
1615
+ ```python
1616
+ InstrumentedOperations(database, sink, EventPolicy(
1617
+ include_values=True, # never the ones marked sensitive
1618
+ include_failure_detail=True, # a driver names the offending value in it
1619
+ slow_after=0.5,
1620
+ ))
1621
+ ```
1622
+
1623
+ A failure is reported by the name of its type. PostgreSQL states the value that
1624
+ caused it inside the message it raises, so the message is withheld until a
1625
+ project decides otherwise. A value the compiler marked sensitive is left out
1626
+ even when a policy asks for values, because marking it was the decision that it
1627
+ must not be shown.
1628
+
1629
+ A shape has its literals taken out as well as its placeholders, so a statement
1630
+ written by hand can be logged as safely as one PyOQ built. The statement is read
1631
+ once, left to right, because what a character means depends on what it is
1632
+ inside: a quote inside a comment starts nothing, and two dashes inside a string
1633
+ are not a comment. Quoted strings, dollar quoted strings, and every kind of
1634
+ comment go, nested ones whole; quoted names stay, because a name is not a value.
1635
+ A number is taken out however it was written, and an opening that was never
1636
+ closed takes the rest of the statement with it, because nothing after it can be
1637
+ shown to be safe. What a shape carries is
1638
+ bounded, while its digest is taken from the whole statement, so two that differ
1639
+ only past the bound are still told apart.
1640
+
1641
+ ### Tracing
1642
+
1643
+ ```python
1644
+ from pyoq.tracing import TracingSink
1645
+
1646
+ watched = InstrumentedOperations(database, TracingSink())
1647
+ ```
1648
+
1649
+ Each answered statement becomes a span carrying the same facts the event does.
1650
+ A span is opened and closed when the statement is answered, because that is the
1651
+ event that knows how long it took.
1652
+
1653
+ ### Readings
1654
+
1655
+ ```python
1656
+ from pyoq.diagnostics import metrics_of, shape_cache_metrics
1657
+
1658
+ metrics_of(pool) # open, idle, checked out, and how much is in use
1659
+ shape_cache_metrics() # how often a shape was already known
1660
+ ```
1661
+
1662
+ Events say what happened; a reading says what is happening now. The shape cache
1663
+ is bounded, because the statements a process runs are not.
1664
+
1665
+ See [`examples/observability.py`](examples/observability.py) for a runnable
1666
+ walkthrough, and `benchmarks/observability.py` for the cost it is held to.
1667
+
1668
+ ## Generated names
1669
+
1670
+ `NamingPolicy` converts source identifiers into valid Python names as one
1671
+ immutable batch. It supports snake case, Pascal case, and upper snake case.
1672
+ Normalization uses Unicode NFKC and Python's locale-independent case rules.
1673
+ Python keywords, soft keywords, and names reserved by a generated component
1674
+ receive trailing underscores until the name is available.
1675
+
1676
+ Each request has a stable key chosen by the generation planner. Scopes isolate
1677
+ namespaces such as table types, row fields, and enum members:
1678
+
1679
+ ```python
1680
+ from pyoq.naming import (
1681
+ NameRequest,
1682
+ NamingPolicy,
1683
+ NamingScope,
1684
+ PythonNameStyle,
1685
+ )
1686
+ from pyoq.schema import Identifier
1687
+
1688
+ fields = NamingScope(
1689
+ name="user-fields",
1690
+ style=PythonNameStyle.SNAKE_CASE,
1691
+ requests=(
1692
+ NameRequest("first-name", Identifier("First Name")),
1693
+ NameRequest("class", Identifier("class")),
1694
+ ),
1695
+ reserved_names=("builder", "build"),
1696
+ )
1697
+ names = NamingPolicy().resolve((fields,))
1698
+
1699
+ assert names.get("user-fields", "first-name") == "first_name"
1700
+ assert names.get("user-fields", "class") == "class_"
1701
+ ```
1702
+
1703
+ Ordering requests or scopes differently does not change the result. Names that
1704
+ become equal after case conversion, Unicode normalization, punctuation removal,
1705
+ or reserved-name escaping are not renamed by sequence. Instead,
1706
+ `NamingCollisionError` reports every conflict in the batch before a renderer
1707
+ can receive a partial result. This keeps generated APIs stable when database
1708
+ reflection order changes.
1709
+
1710
+ ## Schema snapshots and drift
1711
+
1712
+ A snapshot is the one canonical record of what a database holds. Written to a
1713
+ file it becomes reviewable in a diff and usable without a database, which is
1714
+ what lets generation and staleness checks run where no credentials exist.
1715
+
1716
+ ```toml
1717
+ [tool.pyoq]
1718
+ schema-snapshot = "schema.json"
1719
+ ```
1720
+
1721
+ With that set, generation reads the file instead of connecting. Nothing else
1722
+ changes: the same pipeline serves both and does not know which it got.
1723
+
1724
+ ```bash
1725
+ pyoq snapshot # record what the database holds
1726
+ pyoq drift # compare the record against the database
1727
+ pyoq generate # generate from the record, with no connection
1728
+ ```
1729
+
1730
+ Recording the same database twice writes the same bytes, so the file changes
1731
+ only when the schema does. It is written beside its destination and moved into
1732
+ place, so a reader never sees half a snapshot and a failed write leaves the
1733
+ previous record intact.
1734
+
1735
+ `pyoq drift` names what moved in terms of the schema rather than of the file,
1736
+ so the answer is the column that changed rather than the line that differs.
1737
+
1738
+ ```text
1739
+ 1 schema difference(s): added column warehouse.public.parts.weight
1740
+ ```
1741
+
1742
+ Named things are matched by name and then by value, so a renamed table reads
1743
+ as one removed and one added rather than as a change to something that is no
1744
+ longer there.
1745
+
1746
+ ### After a migration
1747
+
1748
+ A migration changes the schema, which makes the generated package stale and
1749
+ the recorded snapshot wrong. One call puts both back in step, in that order,
1750
+ because a package regenerated before the snapshot is written would be built
1751
+ from the schema that has just been replaced.
1752
+
1753
+ ```python
1754
+ from pyoq.migrations import after_migration_at
1755
+
1756
+ after_migration_at("/path/to/project")
1757
+ ```
1758
+
1759
+ Nothing there imports a migration tool, and no migration tool is a dependency
1760
+ of PyOQ.
1761
+
1762
+ It runs after the migration has been applied, which for Alembic means after
1763
+ `alembic upgrade`. A post-write hook is not that moment: Alembic runs those
1764
+ inside `alembic revision`, on the revision file it has just written, while
1765
+ the database is still whatever it was. Wiring PyOQ there would record the
1766
+ schema the migration is about to replace, so the installed command refuses a
1767
+ revision file and says where the step belongs.
1768
+
1769
+ Call it from `env.py`, once the migrations have run:
1770
+
1771
+ ```python
1772
+ with connectable.connect() as connection:
1773
+ context.configure(connection=connection, target_metadata=target_metadata)
1774
+ with context.begin_transaction():
1775
+ context.run_migrations()
1776
+ after_migration_at(PROJECT_ROOT)
1777
+ ```
1778
+
1779
+ Or run it as a step of its own:
1780
+
1781
+ ```bash
1782
+ alembic upgrade head && pyoq-after-migration
1783
+ ```
1784
+
1785
+ The command takes the project directory, or nothing and uses the working
1786
+ directory, walking upwards to the first directory holding a `pyproject.toml`.
1787
+ Django, a shell script, or a CI job call `after_migration_at` directly and get
1788
+ the same two steps.
1789
+
1790
+ ## Generation pipeline
1791
+
1792
+ Generation is a staged application service assembled from small typed
1793
+ components. Schema loading, concern rendering, syntax validation, drift
1794
+ inspection, locking, cleanup, manifest storage, and package replacement have
1795
+ independent interfaces and one responsibility each.
1796
+
1797
+ ```mermaid
1798
+ flowchart LR
1799
+ Source[Schema source] --> Snapshot[Immutable snapshot]
1800
+ Snapshot --> Planner[Generation planner]
1801
+ Planner --> Renderers[Concern renderers]
1802
+ Renderers --> Plan[Canonical plan]
1803
+ Plan --> Validator[Python syntax validator]
1804
+ Validator --> Lock[Project lock]
1805
+ Lock --> Drift[Drift checker]
1806
+ Drift --> DryRun[Dry run report]
1807
+ Drift --> Check[Drift check]
1808
+ Drift --> Writer[Atomic writer]
1809
+ Writer --> Stage[Staged package]
1810
+ Stage --> Manifest[Ownership manifest]
1811
+ Manifest --> Swap[Package replacement]
1812
+ ```
1813
+
1814
+ Every rendered path is relative and portable. Plans sort paths before
1815
+ validation, normalize line endings, reject duplicate outputs from any concern,
1816
+ and hash UTF-8 bytes with SHA-256. Python and stub files are parsed before the
1817
+ project lock or generated directory can be changed.
1818
+
1819
+ The ownership manifest is compact, versioned, and deterministic. A generated
1820
+ file is replaceable or removable only when its path is recorded and its current
1821
+ checksum still matches the manifest. Existing paths without ownership records,
1822
+ modified generated files, symlinks, and unsafe parent paths fail closed.
1823
+ Unowned files elsewhere in the generated directory are copied forward without
1824
+ content changes. Cleanup considers only stale manifest entries.
1825
+
1826
+ Generation uses a fail-closed project lock at `.pyoq-generation-lock`. If a
1827
+ process is interrupted without releasing it, verify that no generation command
1828
+ is active before removing that directory.
1829
+
1830
+ Write mode copies the current package into a sibling staging directory, applies
1831
+ the validated plan there, and replaces the destination package only after the
1832
+ new manifest is complete. A failed replacement restores the previous package.
1833
+ Check mode fails on creates, updates, removals, ownership conflicts, or manifest
1834
+ drift. Dry-run mode returns the same categorized change counts without writing
1835
+ the generated package.
1836
+
1837
+ The complete in-memory example assembles every production component and runs
1838
+ dry-run, write, and check modes without a database or network connection:
1839
+
1840
+ ```console
1841
+ python examples/generation_pipeline.py
1842
+ ```
1843
+
1844
+ See [`examples/generation_pipeline.py`](examples/generation_pipeline.py) for
1845
+ the typed schema source and pipeline assembly. Database reflection is
1846
+ introduced in a later package phase. Until a built-in schema source is
1847
+ available, the default command-line services continue to report generation as
1848
+ unavailable.
1849
+
1850
+ ## Generated database types
1851
+
1852
+ `GeneratedTypesRenderer` turns one immutable schema snapshot into a complete
1853
+ Python package. The renderer resolves all names and type mappings once, then
1854
+ passes that canonical model to focused file renderers.
1855
+
1856
+ ```mermaid
1857
+ flowchart LR
1858
+ Snapshot[Schema snapshot] --> Model[Canonical generation model]
1859
+ Model --> Enums[enums.py]
1860
+ Model --> Rows[rows.py]
1861
+ Model --> Writes[writes.py]
1862
+ Model --> Tables[tables.py]
1863
+ Model --> Relations[relations.py]
1864
+ Model --> Facade[package facade]
1865
+ ```
1866
+
1867
+ Add the renderer as one concern in the generation planner:
1868
+
1869
+ ```python
1870
+ from pyoq.generation import GeneratedTypesRenderer, GenerationPlanner
1871
+
1872
+ planner = GenerationPlanner((GeneratedTypesRenderer(),))
1873
+ ```
1874
+
1875
+ For a `user` table, generated names follow these roles:
1876
+
1877
+ | Generated name | Responsibility |
1878
+ |---|---|
1879
+ | `User` and `USER` | Typed table descriptor and its shared instance |
1880
+ | `UserRow` | Frozen result value containing every readable column |
1881
+ | `UserInsert` | Frozen insert value with required and omitted-field semantics |
1882
+ | `UserUpdate` | Frozen update value where every writable field may be omitted |
1883
+ | `UserInsertValues` | Required and optional dictionary shape for typed boundaries |
1884
+ | `UserUpdateValues` | Optional dictionary shape for update boundaries |
1885
+ | `UserPrimaryKey` | Frozen primary-key value |
1886
+ | `UserBuilder` | Immutable insert builder without `build()` until complete |
1887
+ | `UserUpdateBuilder` | Immutable update builder with concrete field setters |
1888
+
1889
+ Table and column descriptors support SQL-shaped discovery. Constants use the
1890
+ source database name while carrying its exact value as metadata:
1891
+
1892
+ ```python
1893
+ from application.database import USER, User
1894
+
1895
+ identifier_column = USER.ID
1896
+ new_values = User.builder().tenant_id(7).name("Ada").build()
1897
+ ```
1898
+
1899
+ Every writable column produces a concrete setter with its exact mapped Python
1900
+ type. Nullable setters accept `None`. Generated, identity, computed, and other
1901
+ read-only columns expose no insert or update setter. Calling a setter returns a
1902
+ new frozen builder and never performs database I/O.
1903
+
1904
+ Required fields use `Missing` and `Present` type states. Each required setter
1905
+ adds two overloads, so generated typing grows linearly with required-column
1906
+ count. The final required setter returns a distinct complete-builder subtype.
1907
+ Only that subtype defines `build()`. Mypy and Pyright therefore reject both an
1908
+ empty build and a partially complete build, while editors can omit `build()`
1909
+ from their completion lists. Runtime incomplete builder objects also lack that
1910
+ attribute.
1911
+
1912
+ `UserInsert` is a reusable value object. The column-oriented insert statement
1913
+ API accepts it through `values_many()` alongside positional multi-row values.
1914
+ Constructing it has no connection, transaction, or persistence side effect.
1915
+
1916
+ Each generated column descriptor records both its database name and the Python
1917
+ field name used by the generated row and write values. That pairing is what
1918
+ lets a write statement map a generated value onto the columns it selected
1919
+ without dynamic name guessing.
1920
+
1921
+ Primary and unique keys become frozen exact-type values. Relationships become
1922
+ typed descriptors containing aligned source and target columns, referential
1923
+ actions, and source names. Relationship descriptors describe metadata only;
1924
+ they do not trigger lazy loading or hidden queries.
1925
+
1926
+ SQL scalars map to narrow Python types, including `Decimal`, `UUID`, date and
1927
+ time values, immutable tuples for arrays, generated `StrEnum` classes, and a
1928
+ recursive `JsonValue` alias. Unknown source types map to `object`, never
1929
+ `Any`. Generated source is deterministic and passes Ruff formatting, Mypy
1930
+ strict mode, and Pyright strict mode without suppressions.
1931
+
1932
+ A domain is a named type with rules attached, and it maps to whatever it is
1933
+ written over: a domain over `text` generates `str`, and one over
1934
+ `numeric(12, 2)` generates `Decimal`. The schema keeps the name as well as the
1935
+ base, because a schema that forgot it would no longer describe the database it
1936
+ was read from. A domain that forbids null makes every column of it not
1937
+ nullable, whatever the column itself said, because the server refuses a null
1938
+ there either way. PostgreSQL has domains; SQLite and MySQL have none.
1939
+
1940
+ Run the in-memory rendering example without a database or network connection:
1941
+
1942
+ ```console
1943
+ python examples/generated_types.py
1944
+ ```
1945
+
1946
+ ## Typed expressions
1947
+
1948
+ Fields, bound values, computed expressions, and conditions form an immutable
1949
+ typed expression tree. Named methods keep SQL semantics explicit and preserve
1950
+ the result type in Mypy and Pyright strict modes.
1951
+
1952
+ ```python
1953
+ from decimal import Decimal
1954
+
1955
+ from pyoq.query import bind, field
1956
+
1957
+ USER_ID = field(int, "id", table_name="users")
1958
+ USER_NAME = field(str, "name", table_name="users")
1959
+ USER_BALANCE = field(Decimal, "balance", table_name="users")
1960
+ USER_TENANT = field(int, "tenant_id", table_name="users")
1961
+
1962
+ predicate = USER_ID.gt(0) & USER_NAME.starts_with("A")
1963
+ adjusted = USER_BALANCE.add(Decimal("5.00"))
1964
+ selected = USER_ID.in_(bind(1), bind(2))
1965
+ ```
1966
+
1967
+ Generated column descriptors implement the same `Expression[T]` contract, so
1968
+ the generated database package is the primary field source. Comparisons,
1969
+ numeric arithmetic, string operations, temporal extraction and duration
1970
+ arithmetic, null predicates, ranges, membership, and boolean composition are
1971
+ available without converting descriptors or losing their value types.
1972
+
1973
+ ```mermaid
1974
+ flowchart LR
1975
+ Field[Typed field] --> Node[Immutable expression node]
1976
+ Value[Python value] --> Bind[Bound value node]
1977
+ Bind --> Node
1978
+ Node --> Computed[Typed computed expression]
1979
+ Node --> Condition[Boolean condition]
1980
+ Raw[Explicit typed raw template] --> Node
1981
+ ```
1982
+
1983
+ Python values supplied to expression methods always become bound-value nodes.
1984
+ They cannot become field names, operators, clauses, or raw SQL text. Null
1985
+ equality is normalized to `is_null()` or `is_not_null()` nodes. Explicit raw
1986
+ expressions use named placeholders that accept expression objects only:
1987
+
1988
+ ```python
1989
+ from pyoq.query import raw
1990
+
1991
+ distance = raw(
1992
+ float,
1993
+ "distance({origin}, {target})",
1994
+ origin=USER_ID,
1995
+ target=bind(10),
1996
+ )
1997
+ ```
1998
+
1999
+ Raw placeholders reject plain values, attribute access, indexing, conversion,
2000
+ and format specifications. Use `raw_condition()` when the template produces a
2001
+ boolean condition. SQL rendering and ordered parameter extraction are owned by
2002
+ the compiler introduced with statement construction.
2003
+
2004
+ Run the expression example with:
2005
+
2006
+ ```console
2007
+ python examples/expressions.py
2008
+ ```
2009
+
2010
+ ### Choosing one value out of several
2011
+
2012
+ `case()` reads its branches in order and stops at the first that holds. The
2013
+ type of the whole expression is the type of its first result, so a later branch
2014
+ that disagrees is refused where it is written.
2015
+
2016
+ ```python
2017
+ from pyoq.query import case, coalesce, greatest, least, nullif
2018
+
2019
+ tier = (
2020
+ case()
2021
+ .when(USER_BALANCE.gt(Decimal("500.00")), "premium")
2022
+ .when(USER_BALANCE.gt(Decimal("100.00")), "standard")
2023
+ .otherwise("basic")
2024
+ )
2025
+ ```
2026
+
2027
+ `otherwise()` gives the value for rows no branch claimed, and the result is not
2028
+ nullable. `end()` closes the case without one, and the result is `T | None`,
2029
+ because SQL answers null for a row nothing matched:
2030
+
2031
+ ```python
2032
+ flagged = case().when(USER_BALANCE.lt(Decimal("0.00")), "overdrawn").end()
2033
+ ```
2034
+
2035
+ The other three answer the same question with fixed rules. `coalesce()` takes
2036
+ the first argument that is not null, and it drops the `None` from the type when
2037
+ the fallback cannot be null. `USER.NICKNAME` below is a generated descriptor for
2038
+ a nullable column, typed `Expression[str | None]`:
2039
+
2040
+ ```python
2041
+ from generated.database.tables import USER
2042
+
2043
+ label = coalesce(USER.NICKNAME, "unknown")
2044
+ ```
2045
+
2046
+ `label` is `ComputedExpression[str]`, so a row read through it needs no null
2047
+ check and no cast. `nullif()` answers null when its two arguments agree, `greatest()` takes the
2048
+ widest of its arguments and `least()` the narrowest:
2049
+
2050
+ ```python
2051
+ blank_as_null = nullif(USER_NAME, "")
2052
+ floor_price = greatest(USER_BALANCE, Decimal("0.00"))
2053
+ capped = least(USER_BALANCE, Decimal("1000.00"))
2054
+ ```
2055
+
2056
+ Every one of these is an expression like any other, so it can be selected,
2057
+ ordered by, grouped by, compared, or nested inside another. Values reach them
2058
+ as bound parameters, never as SQL text. SQLite has no `GREATEST` or `LEAST` and
2059
+ spells them `MAX` and `MIN` over several arguments, which the SQLite compiler
2060
+ emits without changing what the expression means.
2061
+
2062
+ ### Asking for a value as another type
2063
+
2064
+ `cast()` gives the conversion to the database and declares what it answers
2065
+ with, so the value is read back as that type rather than as whatever the driver
2066
+ carried:
2067
+
2068
+ ```python
2069
+ from pyoq.query import cast
2070
+
2071
+ as_number = cast(USER_NAME, Decimal)
2072
+ as_text = cast(USER_ID, str)
2073
+ ```
2074
+
2075
+ `as_number` is `ComputedExpression[Decimal]`, and a row read through it holds a
2076
+ `Decimal` on every dialect. A cast is the only computed expression that names
2077
+ its own type, so it is the only one the row decoder can hold to a promise.
2078
+
2079
+ Each dialect names its own types, and the names were taken from running
2080
+ servers rather than from a specification:
2081
+
2082
+ | Target | SQLite | PostgreSQL | MySQL |
2083
+ | --- | --- | --- | --- |
2084
+ | `bool` | `INTEGER` | `BOOLEAN` | refused |
2085
+ | `int` | `INTEGER` | `INTEGER` | `SIGNED` |
2086
+ | `float` | `REAL` | `DOUBLE PRECISION` | `DOUBLE` |
2087
+ | `Decimal` | `NUMERIC` | `NUMERIC` | `DECIMAL(65, 30)` |
2088
+ | `str` | `TEXT` | `TEXT` | `CHAR` |
2089
+ | `bytes` | `BLOB` | `BYTEA` | `BINARY` |
2090
+ | `date` | refused | `DATE` | `DATE` |
2091
+ | `time` | refused | `TIME` | `TIME` |
2092
+ | `datetime` | refused | `TIMESTAMP` | `DATETIME` |
2093
+ | `timedelta` | refused | `INTERVAL` | refused |
2094
+ | `UUID` | refused | `UUID` | refused |
2095
+ | `dict`, `list` | refused | `JSONB` | `JSON` |
2096
+
2097
+ A dialect that has no such type refuses the cast. SQLite accepts
2098
+ `CAST(x AS DATE)` and answers with an integer, because an unfamiliar type name
2099
+ falls back to storage affinity there instead of being rejected, so PyOQ
2100
+ refuses rather than emitting a name that would return the wrong kind of value
2101
+ under a promise of the right one. MySQL is asked for its largest decimal
2102
+ precision, because a bare `DECIMAL` means `DECIMAL(10, 0)` and would answer
2103
+ `123` for `123.45`.
2104
+
2105
+ A target no database type answers to is refused where it is written, not when
2106
+ the query runs.
2107
+
2108
+ ### Measuring a row against the rows around it
2109
+
2110
+ An aggregate collapses a group into one row. A window leaves the rows alone
2111
+ and gives each one an answer computed from its neighbours, so a rank, a
2112
+ running total, or the previous row's value can be selected beside the row
2113
+ itself.
2114
+
2115
+ The clauses read in the order SQL writes them: the function, then the window
2116
+ it looks through, then how that window is divided and ordered.
2117
+
2118
+ ```python
2119
+ from pyoq.query import dense_rank, lag, ntile, rank, row_number, sum_
2120
+
2121
+ position = row_number().over().partition_by(USER_TENANT).order_by(
2122
+ USER_BALANCE.desc()
2123
+ )
2124
+ standing = rank().over().order_by(USER_BALANCE.desc())
2125
+ running = sum_(USER_BALANCE).over().partition_by(USER_TENANT).order_by(USER_ID)
2126
+ previous = lag(USER_BALANCE).over().order_by(USER_ID)
2127
+ ```
2128
+
2129
+ `row_number()` counts from one and breaks ties arbitrarily. `rank()` gives
2130
+ tied rows the same position and leaves a gap after them; `dense_rank()` gives
2131
+ them the same position and leaves no gap. `ntile(n)` says which of `n` equal
2132
+ buckets a row falls in. `lag()` and `lead()` read the value that many rows
2133
+ back or ahead, and answer null past the edge of the window, so they are typed
2134
+ `T | None`.
2135
+
2136
+ Any aggregate can be taken over a window instead of over a group, through the
2137
+ same `over()`. `count().over()` counts every row in the window without naming
2138
+ a column.
2139
+
2140
+ A window with no `partition_by()` covers every row. A window orders rows the
2141
+ way a query does, using the same terms and the same rules, so asking for nulls
2142
+ last inside a window works on every dialect just as it does outside one.
2143
+
2144
+ Window functions require SQLite 3.25 or later. They are governed by the
2145
+ `window_functions` capability, so a build without them refuses the query
2146
+ rather than emitting SQL it cannot run.
2147
+
2148
+ ### Comparing several values as one
2149
+
2150
+ A composite key is one key, and asking whether a row is among a set of them
2151
+ should read that way. `row()` puts columns side by side and compares them
2152
+ against tuples in one predicate, instead of an OR of ANDs a reader has to
2153
+ reassemble:
2154
+
2155
+ ```python
2156
+ from pyoq.query import row
2157
+
2158
+ wanted = row(USER_TENANT, USER_NAME).in_((1, "Ada"), (2, "Grace"))
2159
+ exact = row(USER_TENANT, USER_NAME).eq((1, "Ada"))
2160
+ after = row(USER_TENANT, USER_ID).gt((1, 100))
2161
+ ```
2162
+
2163
+ The tuples are checked against the columns by position, so a value of the
2164
+ wrong type, in the wrong order, or a tuple of the wrong width is refused where
2165
+ it is written rather than when the query runs. `row(USER_TENANT,
2166
+ USER_NAME).in_((1, 2))` does not type-check, because the second column is a
2167
+ string.
2168
+
2169
+ `in_()`, `not_in()`, `eq()`, `ne()`, `gt()`, `ge()`, `lt()`, and `le()` are
2170
+ available. The ordering comparisons compare left to right, the way SQL orders
2171
+ a row value, so `(tenant, id) > (1, 100)` means every row of a later tenant
2172
+ and the rows of tenant one after id 100. Matching an empty set of rows is the
2173
+ same nothing that an empty `in_()` already means.
2174
+
2175
+ A row value is two columns or more. SQL reads a single bracketed value as that
2176
+ value, so a row of one is refused. Every value travels bound, exactly as it
2177
+ does in any other predicate.
2178
+
2179
+ Membership in the rows of another query is a `semi_join()` rather than a row
2180
+ value, because that is where the join conditions and their typing already
2181
+ live.
2182
+
2183
+ ### Holding the rows a query read
2184
+
2185
+ A row read inside a transaction can be changed by somebody else before the
2186
+ transaction acts on it. A lock holds it until the transaction ends, and the
2187
+ clauses read in the order SQL writes them:
2188
+
2189
+ ```python
2190
+ job = (
2191
+ select(JOB_ID, JOB_PAYLOAD)
2192
+ .from_(JOBS)
2193
+ .where(JOB_STATE.eq("pending"))
2194
+ .limit(1)
2195
+ .for_update()
2196
+ .skip_locked()
2197
+ )
2198
+ ```
2199
+
2200
+ That is a work queue: each worker takes a row nobody else holds, and passes
2201
+ over the ones already taken instead of waiting behind them.
2202
+
2203
+ `for_update()` holds the whole row. `for_share()` holds it against change
2204
+ while letting others read it. `for_no_key_update()` and `for_key_share()` are
2205
+ the weaker PostgreSQL locks that leave a key referenceable.
2206
+
2207
+ `nowait()` fails instead of waiting for a row somebody else holds, and
2208
+ `skip_locked()` passes over it. Waiting is what a lock does when told neither,
2209
+ so nothing is written for it. `of()` narrows the lock to some of what the
2210
+ query read, which is what keeps a join from holding rows it only looked at.
2211
+
2212
+ Each dialect refuses what it has not got, measured against running servers.
2213
+ SQLite locks the whole database rather than rows, so it refuses locking
2214
+ outright and points at a transaction. MySQL has `FOR UPDATE` and `FOR SHARE`
2215
+ with `NOWAIT`, `SKIP LOCKED`, and `OF`, but no weaker lock, so it refuses
2216
+ `for_no_key_update()` and `for_key_share()`. PostgreSQL has all four.
2217
+
2218
+ A lock belongs to the query that read the rows, so a set operation carries
2219
+ none.
2220
+
2221
+ ### Reading inside a JSON value
2222
+
2223
+ A path is steps rather than text, because the three dialects do not agree on
2224
+ how a path is written, and PostgreSQL reads another's spelling as a member
2225
+ that is simply absent and answers null without complaining. A string step is a
2226
+ member and an integer step is an element:
2227
+
2228
+ ```python
2229
+ country = EVENT_BODY.json_text("actor", "country")
2230
+ first_tag = EVENT_BODY.json_text("tags", 0)
2231
+ tag_count = EVENT_BODY.json_length("tags")
2232
+ ```
2233
+
2234
+ `json_get()` answers with JSON and `json_text()` with what that JSON says, so
2235
+ `json_text()` is typed `str | None` and gives you text on every dialect even
2236
+ where the driver would have handed back the number it looked like.
2237
+
2238
+ `json_has()` asks whether anything is at a path at all, and `json_contains()`
2239
+ asks whether one JSON value holds another:
2240
+
2241
+ ```python
2242
+ verified = EVENT_BODY.json_has("actor", "verified")
2243
+ from_london = EVENT_BODY.json_contains({"actor": {"city": "London"}})
2244
+ ```
2245
+
2246
+ The path is a value, so it is bound like any other and never enters the SQL.
2247
+ Each dialect writes what it has:
2248
+
2249
+ | Asked | SQLite | PostgreSQL | MySQL |
2250
+ | --- | --- | --- | --- |
2251
+ | `json_get` | `-> '$.a.b'` | `#> '{a,b}'` | `JSON_EXTRACT` |
2252
+ | `json_text` | `JSON_EXTRACT` | `#>> '{a,b}'` | `JSON_UNQUOTE(JSON_EXTRACT(...))` |
2253
+ | `json_length` | `JSON_ARRAY_LENGTH` | `JSONB_ARRAY_LENGTH` | `JSON_LENGTH` |
2254
+ | `json_has` | `JSON_TYPE(...) IS NOT NULL` | `#> ... IS NOT NULL` | `JSON_CONTAINS_PATH` |
2255
+ | `json_contains` | refused | `@>` | `JSON_CONTAINS` |
2256
+
2257
+ SQLite has no containment operator, so it refuses rather than emulating one.
2258
+ Asking any of this of a column that is not JSON is refused where it is
2259
+ written.
2260
+
2261
+ ### Asking about an array
2262
+
2263
+ An array column is generated as `tuple[T, ...]`, and these questions are typed
2264
+ by that element type, so a value of the wrong kind is refused where it is
2265
+ written:
2266
+
2267
+ ```python
2268
+ tagged = POST_TAGS.has("python")
2269
+ untagged = POST_TAGS.lacks("draft")
2270
+ both = POST_TAGS.contains_all(("python", "sql"))
2271
+ any_of = POST_TAGS.overlaps(("python", "rust"))
2272
+ inside = POST_TAGS.contained_by(("python", "sql", "rust"))
2273
+ first = POST_TAGS.element(0)
2274
+ count = POST_TAGS.length()
2275
+ ```
2276
+
2277
+ **Elements are counted from zero.** The column reads back as a tuple and a
2278
+ tuple counts from zero, so `POST_TAGS.element(0)` is the same element as
2279
+ `row.tags[0]`. SQL counts an array from one, and PyOQ writes that difference
2280
+ out rather than leaving a caller to remember it. Past the end there is no
2281
+ element, so the answer is null and the type is `T | None`.
2282
+
2283
+ `length()` is the same method that measures text, because how long a thing is
2284
+ is one question whichever kind of thing it is.
2285
+
2286
+ Only PostgreSQL has an array type. SQLite and MySQL refuse these and say to
2287
+ hold a collection as JSON or in a table of its own, rather than emulating an
2288
+ array they have not got.
2289
+
2290
+ ### Calling a function this database has
2291
+
2292
+ Every database grows functions the others have not got, and a toolkit that
2293
+ offered only what all three share would be smaller than any of them. A vendor
2294
+ function is declared once with the type it answers with, and called like any
2295
+ other expression:
2296
+
2297
+ ```python
2298
+ from pyoq.config import DatabaseDialect
2299
+ from pyoq.query import vendor_function, vendor_predicate
2300
+
2301
+ similarity = vendor_function(
2302
+ float, "similarity", dialects=(DatabaseDialect.POSTGRES,)
2303
+ )
2304
+ starts_with = vendor_predicate(
2305
+ "starts_with", dialects=(DatabaseDialect.POSTGRES,)
2306
+ )
2307
+
2308
+ close = similarity(USER_NAME, bind("Ada")).gt(0.3)
2309
+ prefixed = starts_with(USER_NAME, bind("Ad"))
2310
+ ```
2311
+
2312
+ `dialects` says which databases the function exists on, and a compiler for any
2313
+ other refuses the query rather than sending SQL that cannot run. A declaration
2314
+ that names no dialect is written wherever it is asked for, because saying
2315
+ nothing means the caller did not say.
2316
+
2317
+ `vendor_function()` answers with the type given; `vendor_predicate()` answers
2318
+ with a `Condition`, so it composes with `and_()`, `or_()`, and `where()` like
2319
+ any other predicate.
2320
+
2321
+ Arguments are expressions and travel bound, exactly as they do everywhere
2322
+ else. The **name** is not a value, because no database takes a function name
2323
+ as a parameter, so it is written into the SQL and held to being a name: it
2324
+ must be an identifier, optionally qualified by the schema that holds it.
2325
+ Anything else is refused where it is declared.
2326
+
2327
+ Use `raw()` instead when what you need is not a call at all but a fragment of
2328
+ SQL with a shape of its own.
2329
+
2330
+ ### Running a stored procedure
2331
+
2332
+ A procedure is invoked rather than selected from, so it is a statement rather
2333
+ than an expression:
2334
+
2335
+ ```python
2336
+ from pyoq.query import call
2337
+
2338
+ database.execute(call("record_one", bind(7)))
2339
+ ```
2340
+
2341
+ Arguments travel bound like any other value. The name is written into the SQL,
2342
+ because no database takes a routine name as a parameter, so it is held to
2343
+ being an identifier optionally qualified by the schema that holds it.
2344
+
2345
+ A procedure can answer with rows, and a schema does not describe what they
2346
+ hold, so nothing can be inferred. Naming the columns is what lets the rows be
2347
+ read back as the types they were said to be:
2348
+
2349
+ ```python
2350
+ rows = database.many(call("read_one", bind(0)).returning(RECORDED_VALUE))
2351
+ ```
2352
+
2353
+ PostgreSQL has no result set from a procedure and passes a value back through
2354
+ an `INOUT` parameter, which `CALL` answers with as one row. MySQL answers with
2355
+ whatever the procedure selected. SQLite keeps no stored procedures at all and
2356
+ refuses, because there is nothing for it to run and nothing to fall back on.
2357
+
2358
+ A stored **function** is a function, not a statement, so it is declared with
2359
+ `vendor_function()` and called wherever an expression goes.
2360
+
2361
+ ## SELECT queries
2362
+
2363
+ Every join type is its own method and the clause qualifying it comes after,
2364
+ so a chain reads in the order SQL is written:
2365
+
2366
+ ```python
2367
+ select(TITLE_NAME, PUBLISHER_NAME).from_(TITLES).left_join(PUBLISHERS).on(
2368
+ TITLE_PUBLISHER_ID.eq(PUBLISHER_ID)
2369
+ )
2370
+ ```
2371
+
2372
+ | SQL | PyOQ |
2373
+ | --- | --- |
2374
+ | `INNER JOIN t ON c` | `.join(t).on(c)`, or `.inner_join(t).on(c)` |
2375
+ | `LEFT JOIN t ON c` | `.left_join(t).on(c)` |
2376
+ | `RIGHT JOIN t ON c` | `.right_join(t).on(c)` |
2377
+ | `FULL JOIN t ON c` | `.full_join(t).on(c)` |
2378
+ | `CROSS JOIN t` | `.cross_join(t)` |
2379
+ | `NATURAL JOIN t` | `.natural_join(t)` |
2380
+ | `NATURAL LEFT JOIN t` | `.natural_left_join(t)` |
2381
+ | `NATURAL RIGHT JOIN t` | `.natural_right_join(t)` |
2382
+ | `NATURAL FULL JOIN t` | `.natural_full_join(t)` |
2383
+ | `JOIN t USING (a, b)` | `.join(t).using("a", "b")` |
2384
+ | `WHERE EXISTS (...)` | `.semi_join(t).on(c)`, or `where(exists(q))` |
2385
+ | `WHERE NOT EXISTS (...)` | `.anti_join(t).on(c)`, or `where(not_exists(q))` |
2386
+ | `JOIN LATERAL (...)` | `.cross_join(q.as_lateral("name"))` |
2387
+
2388
+ A join is qualified once, by `ON`, `USING`, or `NATURAL`. `on_key()` takes a
2389
+ generated relationship descriptor and builds the `ON` from the foreign key the
2390
+ schema already declares, including one equality per column of a composite key.
2391
+
2392
+ A cross join and a natural join finish the chain on their own. Every other
2393
+ kind is not a query until it is qualified, which both type checkers enforce.
2394
+
2395
+ A semi join keeps rows that have a match without bringing the match back, and
2396
+ an anti join keeps rows that have none. No database writes either as a join,
2397
+ so neither does the SQL: they become `EXISTS` and `NOT EXISTS`, which every
2398
+ supported dialect understands. They neither collide with `where` nor depend
2399
+ on being written after it.
2400
+
2401
+ A lateral source may read the rows to its left, one row at a time. SQLite has
2402
+ none, so it refuses the query rather than evaluating the subquery once and
2403
+ quietly meaning something else.
2404
+
2405
+ A recursive table declares typed columns before either term is built. The
2406
+ column object is reused when reading from the recursive source, so autocomplete
2407
+ and static checking cannot replace its type with a caller assertion:
2408
+
2409
+ ```python
2410
+ from pyoq.query import bind, column, recursive_table, select
2411
+
2412
+ number = column("n", int)
2413
+ walk = recursive_table("walk", number)
2414
+ walk_definition = walk.define(
2415
+ select(bind(1)),
2416
+ select(walk.field(number).add(1))
2417
+ .from_(walk)
2418
+ .where(walk.field(number).lt(5)),
2419
+ )
2420
+ numbers = select(walk.field(number)).with_(walk_definition).from_(walk_definition)
2421
+ ```
2422
+
2423
+ A write returns every column its table declares with `returning_all()`, which
2424
+ reads the list generation put on the table.
2425
+
2426
+ `select()` retains the exact ordered projection tuple through eight fields.
2427
+ Every clause returns a new query and leaves its input reusable. Fields and
2428
+ generated column descriptors are the only normal structural inputs, while
2429
+ Python values remain bound expression values.
2430
+
2431
+ ```python
2432
+ from pyoq.descriptors import TableDescriptor
2433
+ from pyoq.query import count, field, select
2434
+
2435
+ USERS = TableDescriptor[object, object, object]("users")
2436
+ USER_ID = field(int, "id", table_name="users")
2437
+ USER_NAME = field(str, "name", table_name="users")
2438
+
2439
+ active_users = (
2440
+ select(USER_ID.as_("user_id"), USER_NAME, count())
2441
+ .from_(USERS)
2442
+ .where(USER_ID.gt(0))
2443
+ .group_by(USER_ID, USER_NAME)
2444
+ .having(count().gt(0))
2445
+ .order_by(USER_NAME.asc())
2446
+ .limit(20)
2447
+ )
2448
+ ```
2449
+
2450
+ A query source is a table descriptor instance, never a generated table class.
2451
+ The generated class is the typed column and builder namespace, and the
2452
+ generated constant beside it is the value that `from_()`, `join()`,
2453
+ `cross_join()`, and `table_source()` accept. Passing the class is rejected by
2454
+ both strict type checkers and, for untyped callers, by a `QueryValidationError`
2455
+ that states the instance requirement.
2456
+
2457
+ The immutable model covers projection aliases, distinct selection, table and
2458
+ aliased-table sources, inner and outer joins, cross joins, predicates,
2459
+ grouping, aggregate filters, ordering with explicit null placement,
2460
+ pagination, derived-table subqueries, common table expressions, and set
2461
+ operations. Aggregates preserve numeric and scalar result types while marking
2462
+ empty-set results nullable where required.
2463
+
2464
+ ```mermaid
2465
+ flowchart LR
2466
+ Projection[Typed projections] --> Select[Immutable SELECT node]
2467
+ Source[Table or derived source] --> Select
2468
+ Join[Typed joins] --> Select
2469
+ Predicate[Boolean conditions] --> Select
2470
+ Select --> Derived[Subquery or common table]
2471
+ Select --> Set[Typed set operation]
2472
+ ```
2473
+
2474
+ Set operations require the same projection type and validate projection counts
2475
+ again at runtime. Common table column lists must match the selected arity.
2476
+ Structural identifiers reject empty strings and null characters. The model
2477
+ does not render or execute SQL; compilation and execution own those separate
2478
+ responsibilities.
2479
+
2480
+ Run the SELECT construction example with:
2481
+
2482
+ ```console
2483
+ python examples/select_queries.py
2484
+ ```
2485
+
2486
+ ## Typed INSERT statements
2487
+
2488
+ `insert_into(table, *columns)` mirrors SQL and types the values by the selected
2489
+ columns. Each `values()` call adds one row, and repeated calls compile into a
2490
+ single multi-row statement rather than separate statements. Statement
2491
+ construction is immutable and performs no I/O.
2492
+
2493
+ ```python
2494
+ from pyoq.query import insert_into
2495
+
2496
+ created = (
2497
+ insert_into(USERS, USER_ID, USER_NAME).values(100, "Hermann").values(101, "Alfred")
2498
+ )
2499
+ ```
2500
+
2501
+ The selected columns determine the exact positional types and arity that
2502
+ `values()` accepts, so a wrong order, type, or count is a static error in both
2503
+ strict type checkers. Values become bound parameters; typed expressions may be
2504
+ passed where a computed value is required.
2505
+
2506
+ Generated immutable insert values feed the same statement through
2507
+ `values_many()`:
2508
+
2509
+ ```python
2510
+ statement = insert_into(USERS, USER_ID, USER_NAME).values_many(
2511
+ (User.builder().id(100).name("Hermann").build(),)
2512
+ )
2513
+ ```
2514
+
2515
+ `values_many()` maps each generated value onto the selected columns by the
2516
+ field name recorded on the generated column descriptor. A value that leaves a
2517
+ selected column unset is rejected, because a selected column always requires a
2518
+ value. Handwritten `field()` columns carry no generated field name and are
2519
+ therefore positional only.
2520
+
2521
+ Generated columns cannot be written. Selecting one, or selecting a column the
2522
+ schema marks read-only, fails with a `QueryValidationError` before any SQL is
2523
+ produced.
2524
+
2525
+ `returning()` changes the terminal result type and produces a statement that
2526
+ result operations can read:
2527
+
2528
+ ```python
2529
+ row = database.one(
2530
+ insert_into(USERS, USER_NAME).values("Hermann").returning(USER_ID, USER_NAME)
2531
+ )
2532
+ ```
2533
+
2534
+ RETURNING requires SQLite 3.35 or later. It is governed by the `returning`
2535
+ compiler capability, and disabling that capability makes the clause fail closed
2536
+ with `UnsupportedQueryError`.
2537
+
2538
+ Statements without a returning clause execute through `execute()`, which
2539
+ reports the affected row count and the last inserted row identifier:
2540
+
2541
+ ```python
2542
+ result = database.execute(insert_into(USERS, USER_ID, USER_NAME).values(100, "Hermann"))
2543
+ ```
2544
+
2545
+ Run the typed INSERT example with:
2546
+
2547
+ ```console
2548
+ python examples/typed_inserts.py
2549
+ ```
2550
+
2551
+ ## Typed UPDATE and DELETE
2552
+
2553
+ `update(table)` and `delete_from(table)` build the same kind of immutable
2554
+ statement. Assignments are typed by their column, and values become bound
2555
+ parameters.
2556
+
2557
+ ```python
2558
+ from pyoq.query import delete_from, update
2559
+
2560
+ renamed = update(USERS).set(USER_NAME, "Ada Lovelace").where(USER_ID.eq(1))
2561
+ removed = delete_from(USERS).where(USER_ID.eq(1))
2562
+ ```
2563
+
2564
+ A statement that would touch every row must say so. An UPDATE or DELETE with no
2565
+ WHERE condition fails with a `CompilationError` before any SQL reaches the
2566
+ database, so a forgotten condition cannot rewrite or empty a table:
2567
+
2568
+ ```python
2569
+ update(USERS).set(USER_ACTIVE, False).all_rows()
2570
+ delete_from(USERS).all_rows()
2571
+ ```
2572
+
2573
+ `all_rows()` and `where()` exclude each other. Defining both, or defining either
2574
+ twice, raises `QueryStateError`.
2575
+
2576
+ Generated immutable update values assign only the fields they actually set:
2577
+
2578
+ ```python
2579
+ statement = update(USERS).set_values(
2580
+ User.update_builder().name("Ada").build(),
2581
+ USER_NAME,
2582
+ USER_ACTIVE,
2583
+ )
2584
+ ```
2585
+
2586
+ Unset fields are skipped, which is what makes an update value a partial update.
2587
+ A value that leaves every named column unset is rejected, because that statement
2588
+ would assign nothing. As with inserts, generated columns and read-only columns
2589
+ cannot be assigned.
2590
+
2591
+ `returning()` is available on every write form and produces the same typed
2592
+ terminal statement:
2593
+
2594
+ ```python
2595
+ row = database.one(
2596
+ delete_from(USERS).where(USER_ID.eq(1)).returning(USER_ID, USER_NAME)
2597
+ )
2598
+ ```
2599
+
2600
+ Run the typed UPDATE and DELETE example with:
2601
+
2602
+ ```console
2603
+ python examples/typed_updates.py
2604
+ ```
2605
+
2606
+ ## Conflict resolution
2607
+
2608
+ An INSERT can resolve a uniqueness conflict instead of failing. The conflict
2609
+ target names the columns whose constraint is being resolved:
2610
+
2611
+ ```python
2612
+ from pyoq.query import excluded, insert_into
2613
+
2614
+ ignored = (
2615
+ insert_into(USERS, USER_ID, USER_NAME)
2616
+ .values(1, "Ada")
2617
+ .on_conflict_do_nothing(USER_ID)
2618
+ )
2619
+
2620
+ merged = (
2621
+ insert_into(USERS, USER_ID, USER_NAME)
2622
+ .values(1, "Ada Lovelace")
2623
+ .on_conflict_do_update(USER_ID)
2624
+ .set(USER_NAME, excluded(USER_NAME))
2625
+ )
2626
+ ```
2627
+
2628
+ `excluded()` refers to the row the statement tried to insert, keeping the value
2629
+ type of the column it names. It is how a conflicting update reads the new value
2630
+ rather than the stored one.
2631
+
2632
+ A conflict target must be one of the columns the statement inserts, so a typo or
2633
+ a column from another table fails with a `QueryValidationError` rather than
2634
+ producing a statement that never matches. `on_conflict_do_nothing()` may omit the
2635
+ target, which resolves a conflict on any constraint. Resolution that updates
2636
+ always requires a target, because SQLite cannot infer one.
2637
+
2638
+ A conflicting update can also filter which rows it touches:
2639
+
2640
+ ```python
2641
+ merged = (
2642
+ insert_into(USERS, USER_ID, USER_NAME)
2643
+ .values(1, "Ada Lovelace")
2644
+ .on_conflict_do_update(USER_ID)
2645
+ .set(USER_NAME, excluded(USER_NAME))
2646
+ .where(USER_NAME.ne("locked"))
2647
+ )
2648
+ ```
2649
+
2650
+ Conflict resolution can be defined once per statement, and an update resolution
2651
+ that assigns nothing fails at compilation. The `upsert` compiler capability
2652
+ disables the whole feature and makes it fail closed with
2653
+ `UnsupportedQueryError`.
2654
+
2655
+ Run the conflict resolution example with:
2656
+
2657
+ ```console
2658
+ python examples/typed_upserts.py
2659
+ ```
2660
+
2661
+ ## Bulk and multi-operation writes
2662
+
2663
+ SQLite binds a bounded number of parameters per statement, so a large multi-row
2664
+ insert cannot be one statement. The bulk planner splits it into the fewest
2665
+ statements that each stay inside the budget, and it does so without executing
2666
+ anything:
2667
+
2668
+ ```python
2669
+ plan = database.plan_bulk(statement)
2670
+
2671
+ plan.statement_count
2672
+ plan.input_rows
2673
+ plan.row_counts
2674
+ ```
2675
+
2676
+ Chunks are contiguous and cover every input row exactly once, so each planned
2677
+ statement maps back to a known range of input rows through `start` and
2678
+ `length`. That is the correlation the plan guarantees. SQLite does not define
2679
+ the row order a `RETURNING` clause produces, so returned rows correlate to a
2680
+ chunk rather than to an individual input row.
2681
+
2682
+ Planning accounts for the parameters a statement spends outside its rows.
2683
+ Conflict assignments, conflict conditions, and returning projections reserve
2684
+ their share of the budget before rows are packed, and rows whose values are not
2685
+ plain bound parameters are measured individually rather than assumed.
2686
+
2687
+ ```python
2688
+ result = database.execute_bulk(statement)
2689
+
2690
+ result.rows_affected
2691
+ result.statement_count
2692
+ result.row_counts
2693
+ ```
2694
+
2695
+ A bulk write is only atomic inside a transaction. Executed directly, each
2696
+ planned statement commits on its own, so a failure in a later chunk leaves the
2697
+ earlier chunks applied. Wrap the call in a transaction when the whole batch must
2698
+ succeed or fail together:
2699
+
2700
+ ```python
2701
+ with database.transaction() as transaction:
2702
+ transaction.execute_bulk(statement)
2703
+ ```
2704
+
2705
+ `many_bulk()` runs the same plan and returns the returned rows from every chunk.
2706
+
2707
+ Ordered multi-operation execution runs a sequence of write statements in order
2708
+ under an explicit budget, so an unbounded batch cannot be submitted by accident:
2709
+
2710
+ ```python
2711
+ from pyoq.query.execution import OperationBudget
2712
+
2713
+ results = database.execute_all(
2714
+ (first_statement, second_statement),
2715
+ budget=OperationBudget(maximum_operations=8),
2716
+ )
2717
+ ```
2718
+
2719
+ An empty sequence executes nothing and returns an empty result tuple. Execution
2720
+ stops at the first failure, and the same atomicity rule applies: use a
2721
+ transaction when the earlier operations must not survive a later failure.
2722
+
2723
+ Run the bulk write example with:
2724
+
2725
+ ```console
2726
+ python examples/bulk_writes.py
2727
+ ```
2728
+
2729
+ ## SQLite compilation
2730
+
2731
+ `SQLiteCompiler` renders a query into immutable SQL text and a separate ordered
2732
+ parameter tuple. Python values never enter the SQL string. Pagination values
2733
+ use the same parameter path, and sensitive bound values are recorded by their
2734
+ zero-based parameter positions.
2735
+
2736
+ ```python
2737
+ from pyoq.query import bind, select
2738
+ from pyoq.query.sqlite import SQLiteCompiler
2739
+
2740
+ compiled = SQLiteCompiler().compile(
2741
+ select(USER_ID, USER_NAME)
2742
+ .from_(USERS)
2743
+ .where(USER_ID.gt(bind(100, sensitive=True)))
2744
+ .limit(20)
2745
+ )
2746
+
2747
+ assert compiled.parameters == (100, 20)
2748
+ assert compiled.sensitive_parameter_indexes == frozenset({0})
2749
+ ```
2750
+
2751
+ Identifiers are quoted component by component. Operator expressions are
2752
+ parenthesized deterministically, null equality is rendered with `IS NULL`, and
2753
+ empty membership predicates become constant boolean expressions. Derived
2754
+ tables, common tables, joins, aggregates, and compound queries preserve the
2755
+ textual order of their parameters.
2756
+
2757
+ ```mermaid
2758
+ flowchart LR
2759
+ Query[Immutable query tree] --> Capability[SQLite capability checks]
2760
+ Capability --> Compiler[SQLite compiler]
2761
+ Compiler --> SQL[Quoted SQL structure]
2762
+ Compiler --> Parameters[Ordered parameter tuple]
2763
+ Compiler --> Sensitive[Sensitive parameter positions]
2764
+ ```
2765
+
2766
+ Compiler capabilities are explicit and immutable. RIGHT JOIN and FULL JOIN are
2767
+ disabled by default because support depends on the SQLite runtime. Recursive
2768
+ common tables, explicit null ordering, raw expressions, write returning,
2769
+ conflict resolution, and the maximum parameter budget can also be restricted. A query requiring a
2770
+ disabled or unrepresentable feature fails with an exception from `pyoq.errors`.
2771
+
2772
+ The compiler also renders write statements. An INSERT compiles into one
2773
+ statement whose rows share the same quoted column list, an UPDATE renders its
2774
+ assignments in the order they were added, and every value in either form is a
2775
+ bound parameter. An UPDATE or DELETE that has neither a WHERE condition nor an
2776
+ explicit full-table opt-in fails here rather than reaching the database.
2777
+
2778
+ Temporal arithmetic is not translated into SQLite numeric operators. Use an
2779
+ explicit typed raw expression when the intended SQLite date function is known.
2780
+ Raw expression text cannot introduce bind markers, statement separators, or
2781
+ SQL comment syntax. Catalog-qualified identifiers also fail because SQLite has
2782
+ no matching catalog level.
2783
+
2784
+ Run the compiler example with:
2785
+
2786
+ ```console
2787
+ python examples/sqlite_compilation.py
2788
+ ```
2789
+
2790
+ ## Synchronous SQLite execution
2791
+
2792
+ `SQLitePool` provides bounded, exclusive connection leases. One checked-out
2793
+ connection belongs to one caller until the lease exits. Application exceptions
2794
+ return healthy connections, while failures that indicate a broken connection
2795
+ discard only that connection. Checkout waits are bounded by policy.
2796
+
2797
+ ```python
2798
+ from pyoq.query import select
2799
+ from pyoq.query.sqlite import (
2800
+ SQLiteConnectionFactory,
2801
+ SQLiteExecutor,
2802
+ SQLitePool,
2803
+ SQLitePoolPolicy,
2804
+ )
2805
+
2806
+ pool = SQLitePool(
2807
+ SQLiteConnectionFactory("application.sqlite"),
2808
+ SQLitePoolPolicy(maximum_size=10, checkout_timeout=5.0),
2809
+ )
2810
+ db = SQLiteExecutor(pool)
2811
+
2812
+ rows: list[tuple[int, str]] = db.many(
2813
+ select(USER_ID, USER_NAME).from_(USERS).order_by(USER_ID)
2814
+ )
2815
+ selected: tuple[int, str] = db.one(
2816
+ select(USER_ID, USER_NAME).from_(USERS).where(USER_ID.eq(1))
2817
+ )
2818
+ name: str = db.scalar(select(USER_NAME).from_(USERS).where(USER_ID.eq(1)))
2819
+
2820
+ pool.close()
2821
+ ```
2822
+
2823
+ `one` requires exactly one row, `one_or_none` accepts zero or one, `many`
2824
+ returns all rows, and `scalar` requires a single projected value. Their return
2825
+ types follow the query projection. `execute` accepts an immutable compiled
2826
+ statement and returns affected-row metadata. Insert identifiers are populated
2827
+ only for statements carrying explicit insert classification, which prevents
2828
+ stale driver metadata from escaping.
2829
+
2830
+ SQLite-compatible scalar parameters are adapted before a connection is
2831
+ checked out. Text, numbers, bytes, temporal values, decimal values, UUIDs,
2832
+ enums, memory views, and JSON containers have deterministic representations.
2833
+ Unsupported values fail with `ParameterBindingError` before pool capacity is
2834
+ used.
2835
+
2836
+ ```mermaid
2837
+ flowchart LR
2838
+ Query[Typed query] --> Compiler[SQLite compiler]
2839
+ Compiler --> Adapter[Parameter adapter]
2840
+ Adapter --> Pool[Exclusive pool lease]
2841
+ Pool --> Driver[SQLite driver]
2842
+ Driver --> Cardinality[Typed result policy]
2843
+ Cardinality --> Result[Row, scalar, or execute result]
2844
+ ```
2845
+
2846
+ Run the execution example with:
2847
+
2848
+ ```console
2849
+ python examples/sqlite_execution.py
2850
+ ```
2851
+
2852
+ ## Transactions and bounded streaming
2853
+
2854
+ `SQLiteTransaction` owns one pool lease for its complete context. Normal exit
2855
+ commits, while every exception derived from `BaseException` rolls back. A
2856
+ transaction can only be used by its owner thread, and only its innermost active
2857
+ scope may issue work. Nested scopes use generated savepoint identifiers.
2858
+
2859
+ ```python
2860
+ from pyoq.query.sqlite import TransactionMode
2861
+
2862
+ with db.transaction(TransactionMode.IMMEDIATE) as transaction:
2863
+ transaction.execute(FIRST_INSERT)
2864
+
2865
+ try:
2866
+ with transaction.savepoint() as nested:
2867
+ nested.execute(OPTIONAL_INSERT)
2868
+ raise ValueError("discard optional work")
2869
+ except ValueError:
2870
+ pass
2871
+
2872
+ transaction.execute(FINAL_INSERT)
2873
+ ```
2874
+
2875
+ Statement failures that leave the SQLite connection healthy can be handled
2876
+ inside the transaction. Connection failures make the complete transaction
2877
+ unusable. A commit or savepoint failure discards the connection because its
2878
+ outcome cannot be assumed safely.
2879
+
2880
+ `stream` returns a typed iterator that fetches at most the configured batch
2881
+ size. It checks out no connection until it is entered or iterated, returns the
2882
+ lease automatically on exhaustion, and closes on iteration failure. Use its
2883
+ context manager whenever iteration may stop early.
2884
+
2885
+ ```python
2886
+ from pyoq.query.execution import CancellationToken, ExecutionControl
2887
+ from pyoq.query.sqlite import SQLiteStreamPolicy
2888
+
2889
+ token = CancellationToken()
2890
+ policy = SQLiteStreamPolicy(
2891
+ batch_size=128,
2892
+ control=ExecutionControl(
2893
+ timeout=2.5,
2894
+ cancellation_token=token,
2895
+ progress_steps=500,
2896
+ ),
2897
+ )
2898
+
2899
+ with db.stream(
2900
+ select(USER_ID, USER_NAME).from_(USERS).order_by(USER_ID),
2901
+ policy=policy,
2902
+ ) as rows:
2903
+ for user_id, user_name in rows:
2904
+ consume(user_id, user_name)
2905
+ ```
2906
+
2907
+ Timeout and cancellation checks run before execution, through SQLite progress
2908
+ callbacks, and between streamed rows. `QueryTimeoutError` and
2909
+ `QueryCancelledError` are distinct typed failures. Cancellation is cooperative:
2910
+ another thread may call `token.cancel()`, but it must not use the owned
2911
+ connection or consume the row stream.
2912
+
2913
+ ```mermaid
2914
+ flowchart TD
2915
+ Executor[SQLiteExecutor] --> Lease[Exclusive pool lease]
2916
+ Lease --> Transaction[Owned transaction]
2917
+ Transaction --> Savepoint[Nested savepoint]
2918
+ Transaction --> Stream[Bounded row stream]
2919
+ Stream --> Batch[At most batch_size buffered rows]
2920
+ Control[Timeout or cancellation] --> Progress[SQLite progress callback]
2921
+ Progress --> Cleanup[Cursor, handler, transaction, and lease cleanup]
2922
+ Batch --> Cleanup
2923
+ Savepoint --> Cleanup
2924
+ ```
2925
+
2926
+ Run the transaction and streaming example with:
2927
+
2928
+ ```console
2929
+ python examples/sqlite_transactions.py
2930
+ ```
2931
+
2932
+ ## Command line
2933
+
2934
+ The package installs the `pyoq` command and also supports `python -m pyoq`.
2935
+
2936
+ ```console
2937
+ pyoq --version
2938
+ pyoq generate --project-root .
2939
+ pyoq generate --project-root . --check
2940
+ pyoq generate --project-root . --dry-run
2941
+ pyoq inspect --project-root .
2942
+ ```
2943
+
2944
+ Both database commands accept `--config` for a configuration file below the
2945
+ project root and `--profile` for explicit profile selection. `--check` and
2946
+ `--dry-run` are mutually exclusive.
2947
+
2948
+ Command exit codes are stable:
2949
+
2950
+ - `0`: success
2951
+ - `1`: command service failure
2952
+ - `2`: invalid command usage
2953
+ - `3`: configuration failure
2954
+ - `4`: requested operation is unavailable
2955
+
2956
+ SQLite generation and inspection are available through the default command
2957
+ services. Inspection reads tables, views, columns, defaults, generated values,
2958
+ primary and unique keys, indexes, CHECK constraints, and foreign-key
2959
+ relationships. Metadata is normalized before rendering, and the connection is
2960
+ closed before generated files are checked or written.
2961
+
2962
+ Generation writes six deterministic, fully typed modules and an ownership
2963
+ manifest into `codegen-directory`. Repeating a write against an unchanged
2964
+ database leaves the generated directory untouched. `--check` fails on drift,
2965
+ while `--dry-run` reports planned creation, update, removal, and conflict counts
2966
+ without changing files. Generation runs against SQLite, PostgreSQL, and
2967
+ MySQL.
2968
+
2969
+ Run the complete SQLite reflection and generation example with:
2970
+
2971
+ ```console
2972
+ python examples/sqlite_generation.py
2973
+ ```
2974
+
2975
+ ## Performance contract
2976
+
2977
+ Run the native and portable kernel comparison from an editable native build:
2978
+
2979
+ ```console
2980
+ python -m benchmarks.performance_contract
2981
+ ```
2982
+
2983
+ The command checks relative throughput, traced peak memory, retained
2984
+ allocations, and native artifact size. It exits unsuccessfully when a measured
2985
+ kernel exceeds its calibrated budget.
2986
+
2987
+ ## Distribution checks
2988
+
2989
+ Build the native wheel with the release backend:
2990
+
2991
+ ```console
2992
+ maturin build --release --locked --out wheelhouse
2993
+ ```
2994
+
2995
+ Build the universal portable wheel without compiling the native extension:
2996
+
2997
+ ```console
2998
+ hatchling build -t wheel -d wheelhouse
2999
+ ```
3000
+
3001
+ When both artifact kinds are present, validate their tags, native contents,
3002
+ license, type information, and shared package metadata:
3003
+
3004
+ ```console
3005
+ python -m scripts.verify_wheels wheelhouse
3006
+ ```
3007
+
3008
+ Continuous integration builds native wheels for glibc and musl Linux on x86-64
3009
+ and ARM64, macOS on Apple Silicon and x86-64, and Windows on x86-64 and ARM64.
3010
+ The stable native ABI is installed and imported on every supported Python
3011
+ version. The universal wheel is also installed through an incompatible-platform
3012
+ resolver test to prove that it does not require Rust.
3013
+
3014
+ ## Design goals
3015
+
3016
+ - Fully typed public APIs compatible with mypy and Pyright strict modes.
3017
+ - Database-first schema generation with deterministic output.
3018
+ - Explicit, immutable SQL query models and bound parameters.
3019
+ - PostgreSQL, MySQL, and SQLite support.
3020
+ - Django, FastAPI, and Sanic integrations.
3021
+ - Connection pooling, transaction safety, and observable query execution.
3022
+ - Fetch planning that prevents hidden I/O and N+1 query behavior.
3023
+ - Native hot paths with measured throughput, allocation, and peak-memory gates.
3024
+
3025
+ ## Supported Python
3026
+
3027
+ PyOQ requires Python 3.11 or newer.
3028
+
3029
+ ## License
3030
+
3031
+ PyOQ is licensed under the Mozilla Public License 2.0. Applications may use it
3032
+ commercially, including as part of larger proprietary products. Changes to
3033
+ covered PyOQ source files remain subject to the MPL terms. See `LICENSE` for the
3034
+ complete terms.