sendproof-sqlalchemy 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sendproof_sqlalchemy-0.1.0a1/.gitignore +34 -0
- sendproof_sqlalchemy-0.1.0a1/LICENSE +21 -0
- sendproof_sqlalchemy-0.1.0a1/PKG-INFO +127 -0
- sendproof_sqlalchemy-0.1.0a1/README.md +96 -0
- sendproof_sqlalchemy-0.1.0a1/pyproject.toml +44 -0
- sendproof_sqlalchemy-0.1.0a1/src/sendproof_sqlalchemy/__init__.py +7 -0
- sendproof_sqlalchemy-0.1.0a1/src/sendproof_sqlalchemy/changes.py +115 -0
- sendproof_sqlalchemy-0.1.0a1/src/sendproof_sqlalchemy/connection.py +152 -0
- sendproof_sqlalchemy-0.1.0a1/src/sendproof_sqlalchemy/provider.py +371 -0
- sendproof_sqlalchemy-0.1.0a1/src/sendproof_sqlalchemy/py.typed +0 -0
- sendproof_sqlalchemy-0.1.0a1/src/sendproof_sqlalchemy/redaction.py +5 -0
- sendproof_sqlalchemy-0.1.0a1/src/sendproof_sqlalchemy/turns.py +51 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
.DS_Store
|
|
2
|
+
.vs/
|
|
3
|
+
.idea/
|
|
4
|
+
**/bin/
|
|
5
|
+
# The Ruby binding's scripts (prepare, test, fixture) are source, not build output.
|
|
6
|
+
!adapters/ruby/bin/
|
|
7
|
+
**/obj/
|
|
8
|
+
**/node_modules/
|
|
9
|
+
artifacts/
|
|
10
|
+
TestResults/
|
|
11
|
+
*.user
|
|
12
|
+
*.db
|
|
13
|
+
*.db-shm
|
|
14
|
+
*.db-wal
|
|
15
|
+
*.nupkg
|
|
16
|
+
*.snupkg
|
|
17
|
+
__pycache__/
|
|
18
|
+
|
|
19
|
+
# Binding build output, local toolchains and dependency caches
|
|
20
|
+
**/target/
|
|
21
|
+
**/vendor/
|
|
22
|
+
**/.venv/
|
|
23
|
+
**/.gradle/
|
|
24
|
+
**/.gradle-home/
|
|
25
|
+
adapters/java/**/build/
|
|
26
|
+
adapters/java/build/
|
|
27
|
+
adapters/python/dist/
|
|
28
|
+
adapters/php/dist/
|
|
29
|
+
*.egg-info/
|
|
30
|
+
.pytest_cache/
|
|
31
|
+
.phpunit.cache/
|
|
32
|
+
|
|
33
|
+
# Demo cookie keys, kept beside the demo database
|
|
34
|
+
store-demo.keys/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sendproof contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sendproof-sqlalchemy
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: SQLAlchemy rollback previews and change capture for Sendproof.
|
|
5
|
+
Project-URL: Homepage, https://github.com/mprync/sendproof/tree/main/adapters/python/sqlalchemy
|
|
6
|
+
Project-URL: Repository, https://github.com/mprync/sendproof
|
|
7
|
+
Project-URL: Issues, https://github.com/mprync/sendproof/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/mprync/sendproof/blob/main/CHANGELOG.md
|
|
9
|
+
Author: Sendproof contributors
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: api-testing,openapi,rollback,sqlalchemy,swagger,testing
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Database
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Requires-Dist: sendproof-core==0.1.0a1
|
|
27
|
+
Requires-Dist: sqlalchemy>=2.0
|
|
28
|
+
Provides-Extra: asyncio
|
|
29
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0; extra == 'asyncio'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# sendproof-sqlalchemy
|
|
33
|
+
|
|
34
|
+
Rollback previews, SQL capture and ORM change capture for [Sendproof](../../spec/README.md)
|
|
35
|
+
applications that use SQLAlchemy 2.x.
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
from sendproof_sqlalchemy import SQLAlchemyPreview
|
|
39
|
+
|
|
40
|
+
provider = SQLAlchemyPreview(engine, database="inventory_dev") # Engine or AsyncEngine
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Pass the provider to a framework adapter, for example `Sendproof(app, provider=provider, ...)`.
|
|
44
|
+
Install `sendproof-sqlalchemy[asyncio]` for `AsyncEngine` support (needs `greenlet`).
|
|
45
|
+
`database` is a public label; never put a connection string in it.
|
|
46
|
+
|
|
47
|
+
## What a preview guarantees
|
|
48
|
+
|
|
49
|
+
While a preview runs, every `Session` and `Connection` of the request that uses a registered
|
|
50
|
+
engine shares **one** database connection. A hook on the engine's `raw_connection()` provides it
|
|
51
|
+
only inside a preview; other requests are untouched. On that connection:
|
|
52
|
+
|
|
53
|
+
- `commit` releases an internal `SAVEPOINT`, so the writes stay visible to the rest of the request
|
|
54
|
+
(commit-then-`refresh`, a second `Session` in a dependency, and so on) but are never durable.
|
|
55
|
+
- `rollback` rolls back to that savepoint, so a handler that commits and then fails behaves as it
|
|
56
|
+
would for real.
|
|
57
|
+
- At the end the adapter performs a real rollback and confirms it. If the database refuses, the
|
|
58
|
+
adapter withholds the success envelope and the connection is invalidated rather than reused.
|
|
59
|
+
|
|
60
|
+
An independent read after the preview, from a separate request or connection, finds the data
|
|
61
|
+
unchanged.
|
|
62
|
+
|
|
63
|
+
## Change capture
|
|
64
|
+
|
|
65
|
+
Inserts, updates and deletes flushed through the ORM `Session` become changes with the table, the
|
|
66
|
+
primary key and before/after column snapshots (keys use column names). Observed Send records the
|
|
67
|
+
same changes while committing normally.
|
|
68
|
+
|
|
69
|
+
## Coverage notes
|
|
70
|
+
|
|
71
|
+
Read these before advertising an operation for Preview.
|
|
72
|
+
|
|
73
|
+
- **Not captured:** Core DML (`session.execute(update(...))`), ORM bulk operations, raw SQL,
|
|
74
|
+
database triggers and cascades performed by the database. Their SQL appears in the logs and
|
|
75
|
+
they are still rolled back, but no rows are shown. Rows are never synthesized.
|
|
76
|
+
- **One connection per engine:** statements that run concurrently inside a single preview share a
|
|
77
|
+
connection and its locks. With SQLite, two previews at once block each other until the first
|
|
78
|
+
rolls back; PostgreSQL and MySQL do not.
|
|
79
|
+
- **Verified dialects:** SQLite, PostgreSQL and MySQL. The ANSI savepoint path also permits
|
|
80
|
+
MariaDB and Oracle, which have yet to be exercised by the conformance matrix. Other dialects
|
|
81
|
+
make Preview unavailable.
|
|
82
|
+
- **Refused:** engines or connections in `AUTOCOMMIT` mode, because the driver commits each
|
|
83
|
+
statement itself.
|
|
84
|
+
- **Transaction-control SQL is refused before it is sent:** `COMMIT`, `ROLLBACK` (but not
|
|
85
|
+
`ROLLBACK TO SAVEPOINT`), `BEGIN`, `SET autocommit` in any form, `XA`, and the internal
|
|
86
|
+
`sendproof_preview` savepoint. On MySQL and MariaDB also every implicitly committing statement
|
|
87
|
+
(DDL other than `TEMPORARY` tables, `LOCK`/`UNLOCK TABLES`, `GRANT`, `REVOKE`, `ANALYZE`,
|
|
88
|
+
`CHECK`, `OPTIMIZE`, `REPAIR` and `CACHE INDEX` on tables, `SET PASSWORD`, replication control),
|
|
89
|
+
SQL-level `PREPARE` and `EXECUTE` (and MariaDB's `EXECUTE IMMEDIATE`), executable comments (`/*! ... */`) that contain any
|
|
90
|
+
of these, and `CALL`. The Preview then answers 500 and rolls back.
|
|
91
|
+
- **Stored procedures (MySQL and MariaDB):** a procedure can `COMMIT`, so `CALL` is refused unless
|
|
92
|
+
you declare the procedure safe by exact name:
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
SQLAlchemyPreview(engine, safe_procedures=["refresh_totals", "reports.rebuild_cache"])
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Names are exact (a schema-qualified call needs the qualified name) and, like MySQL,
|
|
99
|
+
case-insensitive; wildcards are rejected. This is your statement that the procedure never
|
|
100
|
+
commits: the binding trusts it.
|
|
101
|
+
- **Backstop (every database):** the Preview opens its transaction with
|
|
102
|
+
`SAVEPOINT sendproof_preview` and, before rolling back, rolls back to it. Any commit erases it,
|
|
103
|
+
so if something committed anyway (a declared procedure that does commit, a driver call around
|
|
104
|
+
the engine, after which the driver opens a new transaction by itself), the Preview answers 500
|
|
105
|
+
without confirmation, whatever the handler returned. A handler's `RELEASE` or `ROLLBACK TO` of
|
|
106
|
+
that savepoint is refused. The writes made before that commit are durable; the check only stops the
|
|
107
|
+
Preview from claiming otherwise.
|
|
108
|
+
- **Deadlocks (MySQL and MariaDB):** a deadlock (1213), or a lock wait timeout (1205) with
|
|
109
|
+
`innodb_rollback_on_timeout` on, rolls back the whole Preview transaction. The handler still gets
|
|
110
|
+
the original error, but every later statement in that Preview is refused before it is sent (a
|
|
111
|
+
retry would run outside the transaction, and with autocommit on commit for real), and the
|
|
112
|
+
Preview answers 500 without confirmation. The lock wait timeout case shares the deadlock code
|
|
113
|
+
path and is covered by unit tests only, because `innodb_rollback_on_timeout` can be set only at
|
|
114
|
+
server startup.
|
|
115
|
+
- **Outside the database:** caches, files, other services, other engines you did not register and
|
|
116
|
+
work queued after the response are not rolled back. Sequences may advance.
|
|
117
|
+
- **Connections the application holds across requests** are not enrolled.
|
|
118
|
+
- Other engines you registered are enrolled separately but are not coordinated, as with any
|
|
119
|
+
non-distributed transaction.
|
|
120
|
+
|
|
121
|
+
## Redaction
|
|
122
|
+
|
|
123
|
+
SQL parameter values are shown only in the Development environment (override with the adapter's
|
|
124
|
+
`include_sql_parameter_values`). A parameter whose name contains a secret word (`password`, `secret`,
|
|
125
|
+
`token`, `hash`, `salt`, `credential`, `apikey`, or `key` after `api`, `private`, `secret`,
|
|
126
|
+
`access`, `signing`, `encryption`, `client`) shows `[REDACTED]`. If the statement itself mentions
|
|
127
|
+
such a word, every value in it is hidden.
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# sendproof-sqlalchemy
|
|
2
|
+
|
|
3
|
+
Rollback previews, SQL capture and ORM change capture for [Sendproof](../../spec/README.md)
|
|
4
|
+
applications that use SQLAlchemy 2.x.
|
|
5
|
+
|
|
6
|
+
```python
|
|
7
|
+
from sendproof_sqlalchemy import SQLAlchemyPreview
|
|
8
|
+
|
|
9
|
+
provider = SQLAlchemyPreview(engine, database="inventory_dev") # Engine or AsyncEngine
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Pass the provider to a framework adapter, for example `Sendproof(app, provider=provider, ...)`.
|
|
13
|
+
Install `sendproof-sqlalchemy[asyncio]` for `AsyncEngine` support (needs `greenlet`).
|
|
14
|
+
`database` is a public label; never put a connection string in it.
|
|
15
|
+
|
|
16
|
+
## What a preview guarantees
|
|
17
|
+
|
|
18
|
+
While a preview runs, every `Session` and `Connection` of the request that uses a registered
|
|
19
|
+
engine shares **one** database connection. A hook on the engine's `raw_connection()` provides it
|
|
20
|
+
only inside a preview; other requests are untouched. On that connection:
|
|
21
|
+
|
|
22
|
+
- `commit` releases an internal `SAVEPOINT`, so the writes stay visible to the rest of the request
|
|
23
|
+
(commit-then-`refresh`, a second `Session` in a dependency, and so on) but are never durable.
|
|
24
|
+
- `rollback` rolls back to that savepoint, so a handler that commits and then fails behaves as it
|
|
25
|
+
would for real.
|
|
26
|
+
- At the end the adapter performs a real rollback and confirms it. If the database refuses, the
|
|
27
|
+
adapter withholds the success envelope and the connection is invalidated rather than reused.
|
|
28
|
+
|
|
29
|
+
An independent read after the preview, from a separate request or connection, finds the data
|
|
30
|
+
unchanged.
|
|
31
|
+
|
|
32
|
+
## Change capture
|
|
33
|
+
|
|
34
|
+
Inserts, updates and deletes flushed through the ORM `Session` become changes with the table, the
|
|
35
|
+
primary key and before/after column snapshots (keys use column names). Observed Send records the
|
|
36
|
+
same changes while committing normally.
|
|
37
|
+
|
|
38
|
+
## Coverage notes
|
|
39
|
+
|
|
40
|
+
Read these before advertising an operation for Preview.
|
|
41
|
+
|
|
42
|
+
- **Not captured:** Core DML (`session.execute(update(...))`), ORM bulk operations, raw SQL,
|
|
43
|
+
database triggers and cascades performed by the database. Their SQL appears in the logs and
|
|
44
|
+
they are still rolled back, but no rows are shown. Rows are never synthesized.
|
|
45
|
+
- **One connection per engine:** statements that run concurrently inside a single preview share a
|
|
46
|
+
connection and its locks. With SQLite, two previews at once block each other until the first
|
|
47
|
+
rolls back; PostgreSQL and MySQL do not.
|
|
48
|
+
- **Verified dialects:** SQLite, PostgreSQL and MySQL. The ANSI savepoint path also permits
|
|
49
|
+
MariaDB and Oracle, which have yet to be exercised by the conformance matrix. Other dialects
|
|
50
|
+
make Preview unavailable.
|
|
51
|
+
- **Refused:** engines or connections in `AUTOCOMMIT` mode, because the driver commits each
|
|
52
|
+
statement itself.
|
|
53
|
+
- **Transaction-control SQL is refused before it is sent:** `COMMIT`, `ROLLBACK` (but not
|
|
54
|
+
`ROLLBACK TO SAVEPOINT`), `BEGIN`, `SET autocommit` in any form, `XA`, and the internal
|
|
55
|
+
`sendproof_preview` savepoint. On MySQL and MariaDB also every implicitly committing statement
|
|
56
|
+
(DDL other than `TEMPORARY` tables, `LOCK`/`UNLOCK TABLES`, `GRANT`, `REVOKE`, `ANALYZE`,
|
|
57
|
+
`CHECK`, `OPTIMIZE`, `REPAIR` and `CACHE INDEX` on tables, `SET PASSWORD`, replication control),
|
|
58
|
+
SQL-level `PREPARE` and `EXECUTE` (and MariaDB's `EXECUTE IMMEDIATE`), executable comments (`/*! ... */`) that contain any
|
|
59
|
+
of these, and `CALL`. The Preview then answers 500 and rolls back.
|
|
60
|
+
- **Stored procedures (MySQL and MariaDB):** a procedure can `COMMIT`, so `CALL` is refused unless
|
|
61
|
+
you declare the procedure safe by exact name:
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
SQLAlchemyPreview(engine, safe_procedures=["refresh_totals", "reports.rebuild_cache"])
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Names are exact (a schema-qualified call needs the qualified name) and, like MySQL,
|
|
68
|
+
case-insensitive; wildcards are rejected. This is your statement that the procedure never
|
|
69
|
+
commits: the binding trusts it.
|
|
70
|
+
- **Backstop (every database):** the Preview opens its transaction with
|
|
71
|
+
`SAVEPOINT sendproof_preview` and, before rolling back, rolls back to it. Any commit erases it,
|
|
72
|
+
so if something committed anyway (a declared procedure that does commit, a driver call around
|
|
73
|
+
the engine, after which the driver opens a new transaction by itself), the Preview answers 500
|
|
74
|
+
without confirmation, whatever the handler returned. A handler's `RELEASE` or `ROLLBACK TO` of
|
|
75
|
+
that savepoint is refused. The writes made before that commit are durable; the check only stops the
|
|
76
|
+
Preview from claiming otherwise.
|
|
77
|
+
- **Deadlocks (MySQL and MariaDB):** a deadlock (1213), or a lock wait timeout (1205) with
|
|
78
|
+
`innodb_rollback_on_timeout` on, rolls back the whole Preview transaction. The handler still gets
|
|
79
|
+
the original error, but every later statement in that Preview is refused before it is sent (a
|
|
80
|
+
retry would run outside the transaction, and with autocommit on commit for real), and the
|
|
81
|
+
Preview answers 500 without confirmation. The lock wait timeout case shares the deadlock code
|
|
82
|
+
path and is covered by unit tests only, because `innodb_rollback_on_timeout` can be set only at
|
|
83
|
+
server startup.
|
|
84
|
+
- **Outside the database:** caches, files, other services, other engines you did not register and
|
|
85
|
+
work queued after the response are not rolled back. Sequences may advance.
|
|
86
|
+
- **Connections the application holds across requests** are not enrolled.
|
|
87
|
+
- Other engines you registered are enrolled separately but are not coordinated, as with any
|
|
88
|
+
non-distributed transaction.
|
|
89
|
+
|
|
90
|
+
## Redaction
|
|
91
|
+
|
|
92
|
+
SQL parameter values are shown only in the Development environment (override with the adapter's
|
|
93
|
+
`include_sql_parameter_values`). A parameter whose name contains a secret word (`password`, `secret`,
|
|
94
|
+
`token`, `hash`, `salt`, `credential`, `apikey`, or `key` after `api`, `private`, `secret`,
|
|
95
|
+
`access`, `signing`, `encryption`, `client`) shows `[REDACTED]`. If the statement itself mentions
|
|
96
|
+
such a word, every value in it is hidden.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "sendproof-sqlalchemy"
|
|
7
|
+
version = "0.1.0a1"
|
|
8
|
+
description = "SQLAlchemy rollback previews and change capture for Sendproof."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Sendproof contributors" }]
|
|
14
|
+
keywords = ["swagger", "api-testing", "sqlalchemy", "testing", "rollback", "openapi"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
26
|
+
"Topic :: Database",
|
|
27
|
+
"Typing :: Typed",
|
|
28
|
+
]
|
|
29
|
+
dependencies = ["sendproof-core==0.1.0a1", "sqlalchemy>=2.0"]
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
asyncio = ["sqlalchemy[asyncio]>=2.0"]
|
|
33
|
+
|
|
34
|
+
[tool.hatch.build.targets.wheel]
|
|
35
|
+
packages = ["src/sendproof_sqlalchemy"]
|
|
36
|
+
|
|
37
|
+
[tool.hatch.build.targets.sdist]
|
|
38
|
+
include = ["src/sendproof_sqlalchemy", "README.md", "LICENSE"]
|
|
39
|
+
|
|
40
|
+
[project.urls]
|
|
41
|
+
Homepage = "https://github.com/mprync/sendproof/tree/main/adapters/python/sqlalchemy"
|
|
42
|
+
Repository = "https://github.com/mprync/sendproof"
|
|
43
|
+
Issues = "https://github.com/mprync/sendproof/issues"
|
|
44
|
+
Changelog = "https://github.com/mprync/sendproof/blob/main/CHANGELOG.md"
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""ORM change capture through ``Session`` flush events."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from sqlalchemy import event, inspect
|
|
8
|
+
from sqlalchemy.orm import Session
|
|
9
|
+
from sqlalchemy.orm.attributes import NO_VALUE
|
|
10
|
+
|
|
11
|
+
from sendproof_core import PreviewChange
|
|
12
|
+
from sendproof_core.scope import current_scope
|
|
13
|
+
|
|
14
|
+
STATE_KEY = "sqlalchemy"
|
|
15
|
+
_PENDING = "sendproof.pending"
|
|
16
|
+
_installed = False
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def install() -> None:
|
|
20
|
+
"""Register the flush listeners once per process."""
|
|
21
|
+
global _installed
|
|
22
|
+
if not _installed:
|
|
23
|
+
event.listen(Session, "before_flush", _before_flush)
|
|
24
|
+
event.listen(Session, "after_flush", _after_flush)
|
|
25
|
+
event.listen(Session, "after_rollback", _discard_pending)
|
|
26
|
+
_installed = True
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def uninstall() -> None:
|
|
30
|
+
"""Remove the flush listeners (used by tests)."""
|
|
31
|
+
global _installed
|
|
32
|
+
if _installed:
|
|
33
|
+
event.remove(Session, "before_flush", _before_flush)
|
|
34
|
+
event.remove(Session, "after_flush", _after_flush)
|
|
35
|
+
event.remove(Session, "after_rollback", _discard_pending)
|
|
36
|
+
_installed = False
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _state_for(session: Session, mapper: Any):
|
|
40
|
+
scope = current_scope()
|
|
41
|
+
state = scope.extensions.get(STATE_KEY) if scope else None
|
|
42
|
+
if state is None or state.finished:
|
|
43
|
+
return None
|
|
44
|
+
try:
|
|
45
|
+
bind = session.get_bind(mapper=mapper)
|
|
46
|
+
except Exception:
|
|
47
|
+
return None
|
|
48
|
+
return state if getattr(bind, "engine", bind) in state.engines else None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _columns(mapper: Any):
|
|
52
|
+
for prop in mapper.column_attrs:
|
|
53
|
+
yield prop.key, prop.columns[0].name
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _current_row(instance_state: Any, mapper: Any) -> dict[str, Any]:
|
|
57
|
+
return {name: instance_state.dict[key] for key, name in _columns(mapper) if key in instance_state.dict}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _original_row(instance_state: Any, mapper: Any) -> dict[str, Any]:
|
|
61
|
+
row: dict[str, Any] = {}
|
|
62
|
+
for key, name in _columns(mapper):
|
|
63
|
+
if key in instance_state.committed_state:
|
|
64
|
+
old = instance_state.committed_state[key]
|
|
65
|
+
if old is not NO_VALUE:
|
|
66
|
+
row[name] = old
|
|
67
|
+
elif key in instance_state.dict:
|
|
68
|
+
row[name] = instance_state.dict[key]
|
|
69
|
+
return row
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _key(mapper: Any, row: dict[str, Any], instance_state: Any) -> dict[str, Any]:
|
|
73
|
+
key = {}
|
|
74
|
+
for column in mapper.primary_key:
|
|
75
|
+
prop = mapper.get_property_by_column(column)
|
|
76
|
+
key[column.name] = row.get(column.name, instance_state.dict.get(prop.key))
|
|
77
|
+
return key
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _before_flush(session: Session, flush_context: Any, instances: Any) -> None:
|
|
81
|
+
pending: list[tuple[str, Any, Any, dict | None]] = []
|
|
82
|
+
for instance in list(session.new):
|
|
83
|
+
instance_state = inspect(instance)
|
|
84
|
+
if _state_for(session, instance_state.mapper) is not None:
|
|
85
|
+
pending.append(("insert", instance_state, instance_state.mapper, None))
|
|
86
|
+
for instance in list(session.dirty):
|
|
87
|
+
instance_state = inspect(instance)
|
|
88
|
+
if instance_state.modified and session.is_modified(instance) and \
|
|
89
|
+
_state_for(session, instance_state.mapper) is not None:
|
|
90
|
+
pending.append(("update", instance_state, instance_state.mapper, _original_row(instance_state, instance_state.mapper)))
|
|
91
|
+
for instance in list(session.deleted):
|
|
92
|
+
instance_state = inspect(instance)
|
|
93
|
+
if _state_for(session, instance_state.mapper) is not None:
|
|
94
|
+
pending.append(("delete", instance_state, instance_state.mapper, _original_row(instance_state, instance_state.mapper)))
|
|
95
|
+
session.info[_PENDING] = pending
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _after_flush(session: Session, flush_context: Any) -> None:
|
|
99
|
+
pending = session.info.pop(_PENDING, None)
|
|
100
|
+
if not pending:
|
|
101
|
+
return
|
|
102
|
+
for operation, instance_state, mapper, before in pending:
|
|
103
|
+
state = _state_for(session, mapper)
|
|
104
|
+
if state is None:
|
|
105
|
+
continue
|
|
106
|
+
after = None if operation == "delete" else _current_row(instance_state, mapper)
|
|
107
|
+
key = _key(mapper, after or before or {}, instance_state)
|
|
108
|
+
state.add_change(PreviewChange(operation, mapper.local_table.fullname, key, before, after))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _discard_pending(session: Session) -> None:
|
|
112
|
+
session.info.pop(_PENDING, None)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
__all__ = ["STATE_KEY", "install", "uninstall"]
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""Rollback-only connection sharing for one preview.
|
|
2
|
+
|
|
3
|
+
While a preview is active, every ``Engine.raw_connection()`` call for an enrolled engine is
|
|
4
|
+
answered from a private one-connection pool. That pool hands out the same DBAPI connection to
|
|
5
|
+
every ``Session`` and ``Connection`` of the request, so later statements see earlier writes
|
|
6
|
+
the way they would after a real commit.
|
|
7
|
+
|
|
8
|
+
Transaction boundaries are emulated with one SAVEPOINT inside a transaction that is never
|
|
9
|
+
committed:
|
|
10
|
+
|
|
11
|
+
* ``commit()`` -> ``RELEASE SAVEPOINT`` then ``SAVEPOINT`` (writes stay, nothing is durable)
|
|
12
|
+
* ``rollback()`` -> ``ROLLBACK TO SAVEPOINT`` (undoes work since the last emulated commit)
|
|
13
|
+
|
|
14
|
+
So a handler that commits and later rolls back behaves as it would for real. The adapter ends
|
|
15
|
+
the preview with a genuine rollback and returns the connection to the real pool.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from collections.abc import Callable
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from sqlalchemy.pool import StaticPool
|
|
24
|
+
|
|
25
|
+
from sendproof_core import PreviewRollbackError, PreviewUnavailable
|
|
26
|
+
|
|
27
|
+
#: Dialects with ANSI ``SAVEPOINT`` / ``RELEASE SAVEPOINT`` / ``ROLLBACK TO SAVEPOINT``.
|
|
28
|
+
SUPPORTED_DIALECTS = frozenset({"sqlite", "postgresql", "mysql", "mariadb", "oracle"})
|
|
29
|
+
SAVEPOINT = "sendproof_preview"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def ensure_supported(engine: Any) -> None:
|
|
33
|
+
"""Raise :class:`PreviewUnavailable` when rollback cannot be guaranteed for ``engine``."""
|
|
34
|
+
name = engine.dialect.name
|
|
35
|
+
if name not in SUPPORTED_DIALECTS:
|
|
36
|
+
raise PreviewUnavailable(f"The '{name}' dialect is not supported for rollback previews.")
|
|
37
|
+
level = getattr(engine.dialect, "_on_connect_isolation_level", None) or engine.get_execution_options().get(
|
|
38
|
+
"isolation_level"
|
|
39
|
+
)
|
|
40
|
+
if level == "AUTOCOMMIT":
|
|
41
|
+
raise PreviewUnavailable(
|
|
42
|
+
"An engine uses AUTOCOMMIT, so its writes cannot be rolled back. Previews are unavailable."
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _switches_autocommit_on(name: str, args: tuple[Any, ...], kwargs: dict[str, Any]) -> bool:
|
|
47
|
+
"""Whether a DBAPI method call would put the connection in autocommit mode.
|
|
48
|
+
|
|
49
|
+
Covers ``autocommit(True)`` (PyMySQL, mysqlclient), ``set_autocommit(True)`` (psycopg),
|
|
50
|
+
``set_session(autocommit=True)`` and ``set_isolation_level(0)`` (psycopg2).
|
|
51
|
+
"""
|
|
52
|
+
if name in ("autocommit", "set_autocommit"):
|
|
53
|
+
return bool(args[0] if args else kwargs.get("value", kwargs.get("on", False)))
|
|
54
|
+
if name == "set_session":
|
|
55
|
+
return bool(kwargs.get("autocommit", args[3] if len(args) > 3 else False))
|
|
56
|
+
if name == "set_isolation_level":
|
|
57
|
+
return (args[0] if args else kwargs.get("level")) == 0
|
|
58
|
+
return False
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
_AUTOCOMMIT_METHODS = frozenset({"autocommit", "set_autocommit", "set_session", "set_isolation_level"})
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class RollbackOnlyConnection:
|
|
65
|
+
"""A DBAPI connection proxy that never commits.
|
|
66
|
+
|
|
67
|
+
Everything not overridden is forwarded to the real connection. Switching the connection to
|
|
68
|
+
autocommit is refused before it reaches the driver, because several drivers commit the open
|
|
69
|
+
transaction as part of that switch (``sqlite3`` when ``isolation_level`` becomes ``None``,
|
|
70
|
+
MySQL on ``SET autocommit = 1``).
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
__slots__ = ("_tb_real", "_tb_hold")
|
|
74
|
+
|
|
75
|
+
def __init__(self, real: Any, hold: "Hold") -> None:
|
|
76
|
+
object.__setattr__(self, "_tb_real", real)
|
|
77
|
+
object.__setattr__(self, "_tb_hold", hold)
|
|
78
|
+
|
|
79
|
+
def commit(self) -> None:
|
|
80
|
+
"""Keep the writes inside the preview transaction instead of committing them."""
|
|
81
|
+
if self._tb_hold.ended:
|
|
82
|
+
raise PreviewRollbackError(self._tb_hold.ended)
|
|
83
|
+
self._tb_hold.run("RELEASE SAVEPOINT " + SAVEPOINT, "SAVEPOINT " + SAVEPOINT)
|
|
84
|
+
|
|
85
|
+
def rollback(self) -> None:
|
|
86
|
+
"""Undo the work since the last emulated commit; nothing to undo once the server ended the transaction."""
|
|
87
|
+
if self._tb_hold.ended:
|
|
88
|
+
return
|
|
89
|
+
self._tb_hold.run("ROLLBACK TO SAVEPOINT " + SAVEPOINT)
|
|
90
|
+
|
|
91
|
+
def close(self) -> None:
|
|
92
|
+
"""Keep the connection open until the preview rolls it back."""
|
|
93
|
+
|
|
94
|
+
def __getattr__(self, name: str) -> Any:
|
|
95
|
+
attribute = getattr(self._tb_real, name)
|
|
96
|
+
if name in _AUTOCOMMIT_METHODS and callable(attribute):
|
|
97
|
+
hold = self._tb_hold
|
|
98
|
+
|
|
99
|
+
def guarded(*args: Any, **kwargs: Any) -> Any:
|
|
100
|
+
if _switches_autocommit_on(name, args, kwargs):
|
|
101
|
+
hold.refuse("The connection tried to switch to autocommit.")
|
|
102
|
+
return attribute(*args, **kwargs)
|
|
103
|
+
|
|
104
|
+
return guarded
|
|
105
|
+
return attribute
|
|
106
|
+
|
|
107
|
+
def __setattr__(self, name: str, value: Any) -> None:
|
|
108
|
+
if (name == "autocommit" and value) or (name == "isolation_level" and value is None):
|
|
109
|
+
self._tb_hold.refuse("The connection tried to switch to autocommit.")
|
|
110
|
+
setattr(self._tb_real, name, value)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class Hold:
|
|
114
|
+
"""The connection one engine contributes to a preview."""
|
|
115
|
+
|
|
116
|
+
def __init__(self, engine: Any, fairy: Any) -> None:
|
|
117
|
+
self.engine = engine
|
|
118
|
+
self.fairy = fairy # real pooled connection, checked out for the whole preview
|
|
119
|
+
self.unsafe: str | None = None
|
|
120
|
+
#: Set when the server itself ended the Preview transaction (a MySQL deadlock, or a lock wait
|
|
121
|
+
#: timeout with innodb_rollback_on_timeout): every later statement is refused before sending.
|
|
122
|
+
self.ended: str | None = None
|
|
123
|
+
# Hands back the engine's MySQL preview turn once the connection is released (see ``turns``).
|
|
124
|
+
self.release_turn: Callable[[], None] | None = None
|
|
125
|
+
self.proxy = RollbackOnlyConnection(fairy.dbapi_connection, self)
|
|
126
|
+
# SQLite starts a transaction on SAVEPOINT but would commit it on the final RELEASE. IMMEDIATE takes the write
|
|
127
|
+
# lock up front, so concurrent previews queue behind the driver's busy timeout. A deferred BEGIN lets two
|
|
128
|
+
# previews take the read lock and then fail at once with "database is locked" when both try to write.
|
|
129
|
+
opening = ["BEGIN IMMEDIATE"] if engine.dialect.name == "sqlite" else []
|
|
130
|
+
self.run(*opening, "SAVEPOINT " + SAVEPOINT)
|
|
131
|
+
self.pool = StaticPool(
|
|
132
|
+
lambda: self.proxy, reset_on_return=None, dialect=engine.dialect, logging_name="sendproof-preview"
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
def refuse(self, reason: str) -> None:
|
|
136
|
+
"""Mark the preview unsafe and stop the operation before the driver commits anything."""
|
|
137
|
+
self.unsafe = reason
|
|
138
|
+
raise PreviewRollbackError(reason)
|
|
139
|
+
|
|
140
|
+
def run(self, *statements: str) -> None:
|
|
141
|
+
"""Execute transaction-control statements on the real DBAPI connection."""
|
|
142
|
+
real = self.proxy._tb_real
|
|
143
|
+
cursor = real.cursor()
|
|
144
|
+
try:
|
|
145
|
+
for statement in statements:
|
|
146
|
+
cursor.execute(statement)
|
|
147
|
+
finally:
|
|
148
|
+
cursor.close()
|
|
149
|
+
|
|
150
|
+
def connect(self) -> Any:
|
|
151
|
+
"""A pooled-connection handle that shares the held DBAPI connection."""
|
|
152
|
+
return self.pool.connect()
|
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
"""SQLAlchemy implementation of the Sendproof preview provider."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import Iterable
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from sqlalchemy import event
|
|
11
|
+
from sqlalchemy.engine import Engine
|
|
12
|
+
from sqlalchemy.types import NullType
|
|
13
|
+
|
|
14
|
+
from sendproof_core import (
|
|
15
|
+
PreviewChange,
|
|
16
|
+
PreviewObservation,
|
|
17
|
+
PreviewProvider,
|
|
18
|
+
PreviewRollbackError,
|
|
19
|
+
PreviewSession,
|
|
20
|
+
PreviewUnavailable,
|
|
21
|
+
RequestScope,
|
|
22
|
+
limits,
|
|
23
|
+
)
|
|
24
|
+
from sendproof_core.provider import run_in_thread
|
|
25
|
+
from sendproof_core.scope import current_scope
|
|
26
|
+
|
|
27
|
+
from . import changes as change_capture
|
|
28
|
+
from .connection import SAVEPOINT, Hold, ensure_supported
|
|
29
|
+
from .turns import take_turn
|
|
30
|
+
from sendproof_core.redaction import describe_parameters, python_type_name
|
|
31
|
+
from sendproof_core.txguard import escapes_transaction, normalize_procedures, server_ended_transaction
|
|
32
|
+
|
|
33
|
+
try: # AsyncEngine support needs greenlet; synchronous use does not.
|
|
34
|
+
from sqlalchemy.ext.asyncio import AsyncEngine
|
|
35
|
+
except ImportError: # pragma: no cover - depends on the installed extras
|
|
36
|
+
class AsyncEngine: # type: ignore[no-redef]
|
|
37
|
+
"""Placeholder so that isinstance checks work without greenlet."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
async def greenlet_spawn(function: Any, *args: Any) -> Any:
|
|
41
|
+
"""Run ``function`` where SQLAlchemy's async adapters may await."""
|
|
42
|
+
from sqlalchemy.util import greenlet_spawn as spawn
|
|
43
|
+
|
|
44
|
+
return await spawn(function, *args)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
STATE_KEY = change_capture.STATE_KEY
|
|
48
|
+
_STARTED = "_sendproof_started"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class _State:
|
|
52
|
+
"""Per-request state shared by listeners, the pool hook and the session."""
|
|
53
|
+
|
|
54
|
+
def __init__(self, engines: frozenset[Engine], rollback: bool,
|
|
55
|
+
safe_procedures: frozenset[str] = frozenset()) -> None:
|
|
56
|
+
self.engines = engines
|
|
57
|
+
self.rollback = rollback
|
|
58
|
+
#: Procedures the host declared never commit; any other MySQL ``CALL`` is refused.
|
|
59
|
+
self.safe_procedures = safe_procedures
|
|
60
|
+
self.holds: dict[Engine, Hold] = {}
|
|
61
|
+
self.changes: list[PreviewChange] = []
|
|
62
|
+
self.changes_truncated = False
|
|
63
|
+
self.finished = False
|
|
64
|
+
self.unsafe: str | None = None
|
|
65
|
+
self.lock = threading.RLock()
|
|
66
|
+
|
|
67
|
+
def add_change(self, change: PreviewChange) -> None:
|
|
68
|
+
with self.lock:
|
|
69
|
+
if len(self.changes) >= limits.CHANGES:
|
|
70
|
+
self.changes_truncated = True
|
|
71
|
+
else:
|
|
72
|
+
self.changes.append(change)
|
|
73
|
+
|
|
74
|
+
def connect(self, engine: Engine, original: Any) -> Any:
|
|
75
|
+
"""Answer ``engine.raw_connection()`` from the shared, rollback-only connection."""
|
|
76
|
+
with self.lock:
|
|
77
|
+
hold = self.holds.get(engine)
|
|
78
|
+
if hold is None:
|
|
79
|
+
# Wait for a MySQL preview's turn before checking out a connection (see ``turns``).
|
|
80
|
+
release_turn = take_turn(engine)
|
|
81
|
+
try:
|
|
82
|
+
fairy = original()
|
|
83
|
+
except BaseException:
|
|
84
|
+
if release_turn is not None:
|
|
85
|
+
release_turn()
|
|
86
|
+
raise
|
|
87
|
+
try:
|
|
88
|
+
hold = self.holds[engine] = Hold(engine, fairy)
|
|
89
|
+
except BaseException:
|
|
90
|
+
try:
|
|
91
|
+
fairy.invalidate()
|
|
92
|
+
fairy.close()
|
|
93
|
+
finally:
|
|
94
|
+
if release_turn is not None:
|
|
95
|
+
release_turn()
|
|
96
|
+
raise
|
|
97
|
+
hold.release_turn = release_turn
|
|
98
|
+
return hold.connect()
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class _Instrumentation:
|
|
102
|
+
"""Listeners and the ``raw_connection`` hook installed on one engine."""
|
|
103
|
+
|
|
104
|
+
def __init__(self, engine: Engine) -> None:
|
|
105
|
+
self.engine = engine
|
|
106
|
+
self._original = engine.raw_connection
|
|
107
|
+
engine.raw_connection = self._raw_connection # type: ignore[method-assign]
|
|
108
|
+
event.listen(engine, "before_cursor_execute", self._before)
|
|
109
|
+
event.listen(engine, "after_cursor_execute", self._after)
|
|
110
|
+
event.listen(engine, "handle_error", self._error)
|
|
111
|
+
|
|
112
|
+
def remove(self) -> None:
|
|
113
|
+
event.remove(self.engine, "before_cursor_execute", self._before)
|
|
114
|
+
event.remove(self.engine, "after_cursor_execute", self._after)
|
|
115
|
+
event.remove(self.engine, "handle_error", self._error)
|
|
116
|
+
self.engine.__dict__.pop("raw_connection", None)
|
|
117
|
+
|
|
118
|
+
def _error(self, context: Any) -> None:
|
|
119
|
+
"""Notice when the server ended the Preview transaction itself; the handler still gets the error."""
|
|
120
|
+
scope = current_scope()
|
|
121
|
+
state = scope.extensions.get(STATE_KEY) if scope else None
|
|
122
|
+
hold = state.holds.get(self.engine) if state is not None and state.rollback and not state.finished else None
|
|
123
|
+
if hold is None or hold.ended:
|
|
124
|
+
return
|
|
125
|
+
|
|
126
|
+
def rollback_on_timeout() -> bool:
|
|
127
|
+
try:
|
|
128
|
+
cursor = hold.proxy._tb_real.cursor()
|
|
129
|
+
try:
|
|
130
|
+
cursor.execute("SELECT @@innodb_rollback_on_timeout")
|
|
131
|
+
return bool(int(cursor.fetchone()[0]))
|
|
132
|
+
finally:
|
|
133
|
+
cursor.close()
|
|
134
|
+
except Exception: # noqa: BLE001 - unknown: assume the transaction is gone
|
|
135
|
+
return True
|
|
136
|
+
|
|
137
|
+
reason = server_ended_transaction(context.original_exception, self.engine.dialect.name, rollback_on_timeout)
|
|
138
|
+
if reason is not None:
|
|
139
|
+
hold.ended = hold.unsafe = reason
|
|
140
|
+
|
|
141
|
+
def _raw_connection(self) -> Any:
|
|
142
|
+
scope = current_scope()
|
|
143
|
+
state = scope.extensions.get(STATE_KEY) if scope else None
|
|
144
|
+
if state is None or not state.rollback or state.finished:
|
|
145
|
+
return self._original()
|
|
146
|
+
return state.connect(self.engine, self._original)
|
|
147
|
+
|
|
148
|
+
def _before(self, conn: Any, cursor: Any, statement: str, parameters: Any, context: Any, executemany: bool) -> None:
|
|
149
|
+
scope = current_scope()
|
|
150
|
+
if scope is None:
|
|
151
|
+
return
|
|
152
|
+
state = scope.extensions.get(STATE_KEY)
|
|
153
|
+
if state is not None and state.rollback and not state.finished:
|
|
154
|
+
hold = state.holds.get(self.engine)
|
|
155
|
+
if hold is not None and hold.ended:
|
|
156
|
+
# The server already ended the Preview transaction: this statement would run outside it.
|
|
157
|
+
raise PreviewRollbackError(hold.ended)
|
|
158
|
+
if conn.get_execution_options().get("isolation_level") == "AUTOCOMMIT":
|
|
159
|
+
state.unsafe = "A statement ran in AUTOCOMMIT mode, which cannot be rolled back."
|
|
160
|
+
# Refuse COMMIT, END, implicit-commit DDL and the like before they reach the driver:
|
|
161
|
+
# flagging them afterwards would be too late, the earlier writes would be durable.
|
|
162
|
+
reason = escapes_transaction(statement, conn.dialect.name, SAVEPOINT, state.safe_procedures)
|
|
163
|
+
if reason is not None:
|
|
164
|
+
state.unsafe = reason
|
|
165
|
+
raise PreviewRollbackError(reason)
|
|
166
|
+
try:
|
|
167
|
+
setattr(context, _STARTED, time.perf_counter())
|
|
168
|
+
except AttributeError:
|
|
169
|
+
pass
|
|
170
|
+
|
|
171
|
+
def _after(self, conn: Any, cursor: Any, statement: str, parameters: Any, context: Any, executemany: bool) -> None:
|
|
172
|
+
scope = current_scope()
|
|
173
|
+
if scope is None or not scope.capture.is_active:
|
|
174
|
+
return
|
|
175
|
+
started = getattr(context, _STARTED, None)
|
|
176
|
+
duration = 0.0 if started is None else (time.perf_counter() - started) * 1000
|
|
177
|
+
listed, truncated = _parameters(statement, parameters, context, scope.include_sql_parameter_values)
|
|
178
|
+
rows = cursor.rowcount if isinstance(cursor.rowcount, int) and cursor.rowcount >= 0 else None
|
|
179
|
+
scope.capture.record_sql(
|
|
180
|
+
statement, listed, rows_affected=rows, duration_ms=duration,
|
|
181
|
+
category="sqlalchemy.engine", truncated=truncated or executemany,
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _parameters(statement: str, parameters: Any, context: Any, show_values: bool) -> tuple[list[tuple[str, str | None, str]], bool]:
|
|
186
|
+
"""Name, stringify and redact the bound parameters of a statement."""
|
|
187
|
+
compiled = getattr(context, "compiled", None)
|
|
188
|
+
named: list[tuple[str, Any, str]] = []
|
|
189
|
+
if compiled is not None and getattr(context, "compiled_parameters", None):
|
|
190
|
+
for name, value in context.compiled_parameters[0].items():
|
|
191
|
+
bind = compiled.binds.get(name)
|
|
192
|
+
kind = type(bind.type).__name__ if bind is not None and not isinstance(bind.type, NullType) else None
|
|
193
|
+
named.append((name, value, kind or python_type_name(value)))
|
|
194
|
+
elif isinstance(parameters, dict):
|
|
195
|
+
named = [(str(k), v, python_type_name(v)) for k, v in parameters.items()]
|
|
196
|
+
elif isinstance(parameters, (list, tuple)):
|
|
197
|
+
rows = parameters[0] if parameters and isinstance(parameters[0], (list, tuple, dict)) else parameters
|
|
198
|
+
named = [(f"p{i}", v, python_type_name(v)) for i, v in enumerate(rows.values() if isinstance(rows, dict) else rows)]
|
|
199
|
+
return describe_parameters(statement, named, show_values)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
class SQLAlchemyPreview(PreviewProvider):
|
|
203
|
+
"""Rolls back and observes the writes of an application that uses SQLAlchemy.
|
|
204
|
+
|
|
205
|
+
``engines`` may be :class:`~sqlalchemy.engine.Engine` or
|
|
206
|
+
:class:`~sqlalchemy.ext.asyncio.AsyncEngine` instances. ``database`` is a public label for the
|
|
207
|
+
workbench; never put a connection string in it.
|
|
208
|
+
|
|
209
|
+
On MySQL and MariaDB a Preview refuses ``CALL``, because a stored procedure can commit the
|
|
210
|
+
Preview transaction. ``safe_procedures`` lists procedures the host declares never commit, by
|
|
211
|
+
exact name (optionally ``schema.name``; case-insensitive, like MySQL): a ``CALL`` of one of
|
|
212
|
+
them is allowed. The binding trusts the declaration, so a listed procedure that does commit
|
|
213
|
+
makes the Preview's writes durable; the savepoint check before the rollback then answers 500.
|
|
214
|
+
"""
|
|
215
|
+
|
|
216
|
+
def __init__(self, *engines: Engine | AsyncEngine, database: str | None = None,
|
|
217
|
+
safe_procedures: Iterable[str] = ()) -> None:
|
|
218
|
+
if not engines:
|
|
219
|
+
raise ValueError("Pass at least one engine.")
|
|
220
|
+
self.database = database
|
|
221
|
+
self._safe_procedures = normalize_procedures(safe_procedures)
|
|
222
|
+
self._async = {e.sync_engine for e in engines if isinstance(e, AsyncEngine)}
|
|
223
|
+
self._engines = frozenset(e.sync_engine if isinstance(e, AsyncEngine) else e for e in engines)
|
|
224
|
+
first = (engines[0].sync_engine if isinstance(engines[0], AsyncEngine) else engines[0]).dialect
|
|
225
|
+
#: The SQL provider id the workbench shows a store mark for.
|
|
226
|
+
self.system = "mariadb" if first.name == "mysql" and getattr(first, "is_mariadb", False) else first.name
|
|
227
|
+
self._instrumentation = [
|
|
228
|
+
_Instrumentation(engine) for engine in self._engines if "raw_connection" not in engine.__dict__
|
|
229
|
+
]
|
|
230
|
+
change_capture.install()
|
|
231
|
+
|
|
232
|
+
# ------------------------------------------------------------------ provider
|
|
233
|
+
def begin(self, scope: RequestScope) -> PreviewSession:
|
|
234
|
+
for engine in self._engines:
|
|
235
|
+
ensure_supported(engine)
|
|
236
|
+
state = _State(self._engines, rollback=True, safe_procedures=self._safe_procedures)
|
|
237
|
+
scope.extensions[STATE_KEY] = state
|
|
238
|
+
return _Session(state, self._async)
|
|
239
|
+
|
|
240
|
+
def begin_observation(self, scope: RequestScope) -> PreviewObservation | None:
|
|
241
|
+
state = _State(self._engines, rollback=False)
|
|
242
|
+
scope.extensions[STATE_KEY] = state
|
|
243
|
+
return _Observation(state)
|
|
244
|
+
|
|
245
|
+
def uninstall(self) -> None:
|
|
246
|
+
"""Remove the hooks from the engines. Meant for tests and shutdown."""
|
|
247
|
+
for hook in self._instrumentation:
|
|
248
|
+
hook.remove()
|
|
249
|
+
self._instrumentation.clear()
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
class _Observation(PreviewObservation):
|
|
253
|
+
def __init__(self, state: _State) -> None:
|
|
254
|
+
self._state = state
|
|
255
|
+
|
|
256
|
+
@property
|
|
257
|
+
def changes(self) -> list[PreviewChange]:
|
|
258
|
+
return self._state.changes
|
|
259
|
+
|
|
260
|
+
@property
|
|
261
|
+
def changes_truncated(self) -> bool:
|
|
262
|
+
return self._state.changes_truncated
|
|
263
|
+
|
|
264
|
+
def close(self) -> None:
|
|
265
|
+
self._state.finished = True
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
class _Session(PreviewSession):
|
|
269
|
+
"""Rolls back every connection the request used and confirms it."""
|
|
270
|
+
|
|
271
|
+
def __init__(self, state: _State, async_engines: set[Engine]) -> None:
|
|
272
|
+
self._state = state
|
|
273
|
+
self._async = async_engines
|
|
274
|
+
self._rolled_back = False
|
|
275
|
+
|
|
276
|
+
@property
|
|
277
|
+
def changes(self) -> list[PreviewChange]:
|
|
278
|
+
return self._state.changes
|
|
279
|
+
|
|
280
|
+
@property
|
|
281
|
+
def changes_truncated(self) -> bool:
|
|
282
|
+
return self._state.changes_truncated
|
|
283
|
+
|
|
284
|
+
# ----------------------------------------------------------------- rollback
|
|
285
|
+
def _check_before(self) -> None:
|
|
286
|
+
unsafe = self._state.unsafe or next((h.unsafe for h in self._state.holds.values() if h.unsafe), None)
|
|
287
|
+
if unsafe:
|
|
288
|
+
raise PreviewRollbackError(unsafe)
|
|
289
|
+
|
|
290
|
+
def _rollback_hold(self, hold: Hold) -> BaseException | None:
|
|
291
|
+
if not hold.unsafe:
|
|
292
|
+
# The backstop, on every dialect: the savepoint the Preview opened its transaction with must
|
|
293
|
+
# still exist. Any commit, explicit or implicit, erases every savepoint, and the driver (or
|
|
294
|
+
# MySQL with autocommit off) then opens a new transaction by itself, so only a savepoint made
|
|
295
|
+
# at the start, never one made now, proves the Preview transaction is the one still open.
|
|
296
|
+
# ROLLBACK TO rather than RELEASE: it also works on a PostgreSQL transaction aborted by an
|
|
297
|
+
# error, which commits nothing and must not fail the Preview.
|
|
298
|
+
try:
|
|
299
|
+
hold.run("ROLLBACK TO SAVEPOINT " + SAVEPOINT)
|
|
300
|
+
except BaseException as error: # noqa: BLE001 - "no such savepoint" (MySQL 1305), or a lost connection
|
|
301
|
+
hold.unsafe = (
|
|
302
|
+
"The Preview transaction was committed before its rollback (its savepoint is gone), "
|
|
303
|
+
"so the Preview can't be confirmed."
|
|
304
|
+
)
|
|
305
|
+
if not isinstance(error, Exception):
|
|
306
|
+
raise
|
|
307
|
+
try:
|
|
308
|
+
hold.proxy._tb_real.rollback()
|
|
309
|
+
except BaseException as error: # noqa: BLE001 - reported through PreviewRollbackError
|
|
310
|
+
return error
|
|
311
|
+
return None
|
|
312
|
+
|
|
313
|
+
def rollback(self) -> None:
|
|
314
|
+
if self._async & set(self._state.holds):
|
|
315
|
+
raise PreviewUnavailable("Use the async API for AsyncEngine previews.")
|
|
316
|
+
self._finish([self._rollback_hold(h) for h in list(self._state.holds.values())])
|
|
317
|
+
|
|
318
|
+
async def arollback(self) -> None:
|
|
319
|
+
results: list[BaseException | None] = []
|
|
320
|
+
for engine, hold in list(self._state.holds.items()):
|
|
321
|
+
if engine in self._async:
|
|
322
|
+
results.append(await greenlet_spawn(self._rollback_hold, hold))
|
|
323
|
+
else:
|
|
324
|
+
results.append(await run_in_thread(self._rollback_hold, hold))
|
|
325
|
+
self._finish(results)
|
|
326
|
+
|
|
327
|
+
def _finish(self, errors: list[BaseException | None]) -> None:
|
|
328
|
+
failures = [e for e in errors if e is not None]
|
|
329
|
+
self._check_before()
|
|
330
|
+
if failures:
|
|
331
|
+
raise PreviewRollbackError("The database rollback could not be confirmed.") from failures[0]
|
|
332
|
+
self._rolled_back = True
|
|
333
|
+
|
|
334
|
+
# -------------------------------------------------------------------- close
|
|
335
|
+
def _release(self, hold: Hold) -> BaseException | None:
|
|
336
|
+
try:
|
|
337
|
+
if not self._rolled_back:
|
|
338
|
+
# Never hand a connection with unconfirmed writes back to the pool.
|
|
339
|
+
hold.fairy.invalidate()
|
|
340
|
+
hold.fairy.close()
|
|
341
|
+
except BaseException as error: # noqa: BLE001
|
|
342
|
+
return error
|
|
343
|
+
finally:
|
|
344
|
+
# The transaction has ended with the connection, so the next MySQL preview may run.
|
|
345
|
+
release_turn, hold.release_turn = hold.release_turn, None
|
|
346
|
+
if release_turn is not None:
|
|
347
|
+
release_turn()
|
|
348
|
+
return None
|
|
349
|
+
|
|
350
|
+
def close(self) -> None:
|
|
351
|
+
self._state.finished = True
|
|
352
|
+
holds, self._state.holds = list(self._state.holds.items()), {}
|
|
353
|
+
errors = [self._release(h) for _, h in holds]
|
|
354
|
+
_raise_first(errors)
|
|
355
|
+
|
|
356
|
+
async def aclose(self) -> None:
|
|
357
|
+
self._state.finished = True
|
|
358
|
+
holds, self._state.holds = list(self._state.holds.items()), {}
|
|
359
|
+
errors = []
|
|
360
|
+
for engine, hold in holds:
|
|
361
|
+
if engine in self._async:
|
|
362
|
+
errors.append(await greenlet_spawn(self._release, hold))
|
|
363
|
+
else:
|
|
364
|
+
errors.append(await run_in_thread(self._release, hold))
|
|
365
|
+
_raise_first(errors)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _raise_first(errors: list[BaseException | None]) -> None:
|
|
369
|
+
for error in errors:
|
|
370
|
+
if error is not None:
|
|
371
|
+
raise error
|
|
File without changes
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Turns that keep MySQL previews from running at the same time.
|
|
2
|
+
|
|
3
|
+
InnoDB checks a new key for duplicates under a shared lock. When three or more transactions insert the
|
|
4
|
+
same key and the first one rolls back, as every preview does, the waiters each hold that shared lock and
|
|
5
|
+
each need the exclusive one, so InnoDB aborts all but one with a deadlock (error 1213). Sent for real,
|
|
6
|
+
the first insert commits and the others get a duplicate-key error instead. No isolation level avoids it.
|
|
7
|
+
|
|
8
|
+
A preview therefore takes its engine's turn before it checks out a MySQL or MariaDB connection, and
|
|
9
|
+
hands it back once that connection's transaction has ended, as SQLite previews already queue behind
|
|
10
|
+
``BEGIN IMMEDIATE``. Engines of other dialects take no turn.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import threading
|
|
17
|
+
import weakref
|
|
18
|
+
from collections.abc import Callable
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
#: Dialects whose previews take turns.
|
|
22
|
+
SERIAL_DIALECTS = frozenset({"mysql", "mariadb"})
|
|
23
|
+
|
|
24
|
+
#: How long an async preview sleeps between attempts to take its turn, in seconds.
|
|
25
|
+
_ASYNC_POLL_SECONDS = 0.005
|
|
26
|
+
|
|
27
|
+
_turns: weakref.WeakKeyDictionary[Any, threading.Lock] = weakref.WeakKeyDictionary()
|
|
28
|
+
_turns_guard = threading.Lock()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def take_turn(engine: Any) -> Callable[[], None] | None:
|
|
32
|
+
"""Wait for ``engine``'s preview turn and answer the callable that hands it back.
|
|
33
|
+
|
|
34
|
+
Answers ``None`` for dialects whose previews run at the same time. An async engine runs this inside
|
|
35
|
+
SQLAlchemy's greenlet on the event loop, where blocking would stall the preview that holds the turn,
|
|
36
|
+
so it awaits between attempts instead of blocking.
|
|
37
|
+
"""
|
|
38
|
+
if engine.dialect.name not in SERIAL_DIALECTS:
|
|
39
|
+
return None
|
|
40
|
+
with _turns_guard:
|
|
41
|
+
lock = _turns.get(engine)
|
|
42
|
+
if lock is None:
|
|
43
|
+
lock = _turns[engine] = threading.Lock()
|
|
44
|
+
if getattr(engine.dialect, "is_async", False):
|
|
45
|
+
from sqlalchemy.util import await_only
|
|
46
|
+
|
|
47
|
+
while not lock.acquire(blocking=False):
|
|
48
|
+
await_only(asyncio.sleep(_ASYNC_POLL_SECONDS))
|
|
49
|
+
else:
|
|
50
|
+
lock.acquire()
|
|
51
|
+
return lock.release
|