s3rds 0.1.2__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {s3rds-0.1.2/s3rds.egg-info → s3rds-0.2.0}/PKG-INFO +45 -12
- s3rds-0.1.2/PKG-INFO → s3rds-0.2.0/README.md +32 -27
- s3rds-0.2.0/pyproject.toml +55 -0
- {s3rds-0.1.2 → s3rds-0.2.0}/s3rds/__init__.py +4 -1
- {s3rds-0.1.2 → s3rds-0.2.0}/s3rds/pymysql.py +13 -2
- {s3rds-0.1.2 → s3rds-0.2.0}/s3rds/sqlite.py +236 -28
- s3rds-0.2.0/s3rds.egg-info/PKG-INFO +113 -0
- {s3rds-0.1.2 → s3rds-0.2.0}/s3rds.egg-info/SOURCES.txt +1 -0
- s3rds-0.2.0/s3rds.egg-info/entry_points.txt +2 -0
- s3rds-0.2.0/s3rds.egg-info/requires.txt +6 -0
- s3rds-0.1.2/README.md +0 -61
- s3rds-0.1.2/pyproject.toml +0 -31
- s3rds-0.1.2/s3rds.egg-info/requires.txt +0 -3
- {s3rds-0.1.2 → s3rds-0.2.0}/s3rds/psycopg2.py +0 -0
- {s3rds-0.1.2 → s3rds-0.2.0}/s3rds.egg-info/dependency_links.txt +0 -0
- {s3rds-0.1.2 → s3rds-0.2.0}/s3rds.egg-info/top_level.txt +0 -0
- {s3rds-0.1.2 → s3rds-0.2.0}/setup.cfg +0 -0
|
@@ -1,21 +1,30 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: s3rds
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary: A truly serverless RDS on your own AWS account: Lambda URL endpoint, SQLite on S3, ≈ $0 when idle. Drivers named after the standard clients: s3rds.sqlite, s3rds.pymysql
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: A truly serverless RDS on your own AWS account: Lambda URL endpoint, SQLite on S3, ≈ $0 when idle. Drivers named after the standard clients: s3rds.sqlite, s3rds.pymysql; SQLAlchemy via sqlite+s3rds://.
|
|
5
5
|
Author: Frank Zhang
|
|
6
6
|
License: MIT
|
|
7
7
|
Project-URL: Homepage, https://github.com/cloudformer/s3rds
|
|
8
|
-
|
|
8
|
+
Project-URL: Repository, https://github.com/cloudformer/s3rds
|
|
9
|
+
Project-URL: Deploy, https://github.com/cloudformer/s3rds#quick-start
|
|
10
|
+
Project-URL: Compatibility, https://github.com/cloudformer/s3rds/blob/main/tests/compat/REPORT.md
|
|
11
|
+
Project-URL: Changelog, https://github.com/cloudformer/s3rds/blob/main/CHANGELOG.md
|
|
12
|
+
Project-URL: Issues, https://github.com/cloudformer/s3rds/issues
|
|
13
|
+
Keywords: serverless,rds,sqlite,s3,lambda,database,pymysql,dbapi,sqlalchemy
|
|
9
14
|
Classifier: Development Status :: 3 - Alpha
|
|
10
15
|
Classifier: Intended Audience :: Developers
|
|
11
16
|
Classifier: License :: OSI Approved :: MIT License
|
|
12
17
|
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
20
|
Classifier: Topic :: Database
|
|
14
21
|
Classifier: Topic :: Database :: Front-Ends
|
|
15
22
|
Requires-Python: >=3.9
|
|
16
23
|
Description-Content-Type: text/markdown
|
|
24
|
+
Requires-Dist: sqlglot<31,>=30.24
|
|
17
25
|
Provides-Extra: mysql
|
|
18
|
-
|
|
26
|
+
Provides-Extra: sqlalchemy
|
|
27
|
+
Requires-Dist: SQLAlchemy<2.1,>=2.0; extra == "sqlalchemy"
|
|
19
28
|
|
|
20
29
|
# s3rds
|
|
21
30
|
|
|
@@ -24,12 +33,11 @@ a Lambda URL as your database endpoint, SQLite on S3, ≈ $0 when idle.**
|
|
|
24
33
|
|
|
25
34
|

|
|
26
35
|
|
|
27
|
-
|
|
36
|
+
One install, one dependency (sqlglot, used only by `s3rds.pymysql`). Each driver is a submodule **named after the standard
|
|
28
37
|
client it mirrors**, so moving to a real database is a one-line import change and nothing else:
|
|
29
38
|
|
|
30
39
|
```bash
|
|
31
|
-
pip install s3rds
|
|
32
|
-
pip install "s3rds[mysql]" # + s3rds.pymysql: write MySQL, translated on the client (sqlglot)
|
|
40
|
+
pip install s3rds # s3rds.sqlite, s3rds.pymysql, sqlite+s3rds:// for SQLAlchemy
|
|
33
41
|
```
|
|
34
42
|
|
|
35
43
|
```python
|
|
@@ -57,16 +65,28 @@ conn.transaction([ # all-or-nothing, single S3 w
|
|
|
57
65
|
| Submodule | SQL dialect | Mirrors | Status |
|
|
58
66
|
|---|---|---|---|
|
|
59
67
|
| `s3rds.sqlite` | SQLite (no translation) | pymysql-shaped API | available |
|
|
60
|
-
| `s3rds.pymysql` | MySQL, translated on the client | pymysql | available
|
|
68
|
+
| `s3rds.pymysql` | MySQL, translated on the client | pymysql | available |
|
|
61
69
|
| `s3rds.psycopg2` | PostgreSQL | psycopg2 | placeholder; `connect()` raises until implemented |
|
|
70
|
+
| `sqlite+s3rds://` | SQLite, written by SQLAlchemy | SQLAlchemy Core / ORM | available — the URL is the only change |
|
|
71
|
+
|
|
72
|
+
**SQLAlchemy** (Core and ORM): only the URL changes — SQLAlchemy's own SQLite dialect writes the SQL,
|
|
73
|
+
`s3rds.sqlite` replaces the standard library's `sqlite3` underneath.
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
engine = create_engine("sqlite+s3rds://admin:...@xxxx.lambda-url.us-east-2.on.aws/blog")
|
|
77
|
+
```
|
|
62
78
|
|
|
63
79
|
## Semantics
|
|
64
80
|
|
|
65
|
-
- **Transactions**: each `execute()` is its own transaction and is durable in S3 when it
|
|
66
|
-
(
|
|
81
|
+
- **Transactions**: by default each `execute()` is its own transaction and is durable in S3 when it
|
|
82
|
+
returns (`autocommit=True`). `commit()` is a no-op. `rollback()` **cannot undo statements that already
|
|
67
83
|
ran** — it warns if anything was written since the last `commit()` (and stays silent otherwise, so
|
|
68
|
-
pools
|
|
69
|
-
|
|
84
|
+
pools may call it freely). For all-or-nothing writes use `transaction([...])`, which runs several
|
|
85
|
+
statements as one commit and one S3 write.
|
|
86
|
+
- `s3rds.sqlite.connect(..., autocommit=False)` (what SQLAlchemy uses) has sqlite3's transactions: reads see
|
|
87
|
+
the transaction's own uncommitted writes, `commit()` writes S3 once, `rollback()` discards. If another
|
|
88
|
+
writer commits first, the transaction raises `OperationalError` instead of overwriting: run it again.
|
|
89
|
+
Keep transactions short — each statement replays the writes before it on the server.
|
|
70
90
|
- Large results are paged by the server (4 MB per page) and reassembled inside `execute()`; `rows`
|
|
71
91
|
is always complete. If the database changes while paging, the query is re-run rather than mixing versions.
|
|
72
92
|
- Writes **never** duplicate: the driver re-sends only when the request provably never reached the
|
|
@@ -74,6 +94,19 @@ conn.transaction([ # all-or-nothing, single S3 w
|
|
|
74
94
|
- `s3rds.pymysql` raises pymysql-style exceptions with MySQL error codes (1062, 1146, 1054, 1064, …);
|
|
75
95
|
unsupported MySQL features raise `NotSupportedError` instead of misbehaving silently.
|
|
76
96
|
|
|
97
|
+
## Versions
|
|
98
|
+
|
|
99
|
+
What the report was verified with; the full report: [REPORT.md](https://github.com/cloudformer/s3rds/blob/main/tests/compat/REPORT.md).
|
|
100
|
+
|
|
101
|
+
| | Version | Fixed by |
|
|
102
|
+
|---|---|---|
|
|
103
|
+
| Python (client) | 3.9 – 3.13: regression suite on 3.9.6, full comparison on 3.13.7 | `requires-python = ">=3.9"` |
|
|
104
|
+
| Server runtime | AWS Lambda `python3.12` | `terraform/main.tf` |
|
|
105
|
+
| SQLite (server engine) | 3.50.4, shipped in the Lambda zip (the runtime's own is 3.40.0) | `src/db/requirements.txt`: `sqlean.py==3.50.4.5` |
|
|
106
|
+
| sqlglot (MySQL translation) | 30.24 | dependency `sqlglot>=30.24,<31` |
|
|
107
|
+
| SQLAlchemy | 2.0, tested on 2.0.55 | extra `pip install "s3rds[sqlalchemy]"` = `SQLAlchemy>=2.0,<2.1` |
|
|
108
|
+
| MySQL (reference) | 8.0.46 in Docker | `tests/compat/docker-compose.yml`: `mysql:8.0.46` |
|
|
109
|
+
|
|
77
110
|
## Server
|
|
78
111
|
|
|
79
112
|
You need an s3rds server in your own AWS account — deploy it once with Terraform or CloudFormation
|
|
@@ -1,22 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: s3rds
|
|
3
|
-
Version: 0.1.2
|
|
4
|
-
Summary: A truly serverless RDS on your own AWS account: Lambda URL endpoint, SQLite on S3, ≈ $0 when idle. Drivers named after the standard clients: s3rds.sqlite, s3rds.pymysql.
|
|
5
|
-
Author: Frank Zhang
|
|
6
|
-
License: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/cloudformer/s3rds
|
|
8
|
-
Keywords: serverless,rds,sqlite,s3,lambda,database,pymysql,dbapi
|
|
9
|
-
Classifier: Development Status :: 3 - Alpha
|
|
10
|
-
Classifier: Intended Audience :: Developers
|
|
11
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
-
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Topic :: Database
|
|
14
|
-
Classifier: Topic :: Database :: Front-Ends
|
|
15
|
-
Requires-Python: >=3.9
|
|
16
|
-
Description-Content-Type: text/markdown
|
|
17
|
-
Provides-Extra: mysql
|
|
18
|
-
Requires-Dist: sqlglot>=25.0; extra == "mysql"
|
|
19
|
-
|
|
20
1
|
# s3rds
|
|
21
2
|
|
|
22
3
|
**Python drivers for [s3rds](https://github.com/cloudformer/s3rds) — a truly serverless RDS:
|
|
@@ -24,12 +5,11 @@ a Lambda URL as your database endpoint, SQLite on S3, ≈ $0 when idle.**
|
|
|
24
5
|
|
|
25
6
|

|
|
26
7
|
|
|
27
|
-
|
|
8
|
+
One install, one dependency (sqlglot, used only by `s3rds.pymysql`). Each driver is a submodule **named after the standard
|
|
28
9
|
client it mirrors**, so moving to a real database is a one-line import change and nothing else:
|
|
29
10
|
|
|
30
11
|
```bash
|
|
31
|
-
pip install s3rds
|
|
32
|
-
pip install "s3rds[mysql]" # + s3rds.pymysql: write MySQL, translated on the client (sqlglot)
|
|
12
|
+
pip install s3rds # s3rds.sqlite, s3rds.pymysql, sqlite+s3rds:// for SQLAlchemy
|
|
33
13
|
```
|
|
34
14
|
|
|
35
15
|
```python
|
|
@@ -57,16 +37,28 @@ conn.transaction([ # all-or-nothing, single S3 w
|
|
|
57
37
|
| Submodule | SQL dialect | Mirrors | Status |
|
|
58
38
|
|---|---|---|---|
|
|
59
39
|
| `s3rds.sqlite` | SQLite (no translation) | pymysql-shaped API | available |
|
|
60
|
-
| `s3rds.pymysql` | MySQL, translated on the client | pymysql | available
|
|
40
|
+
| `s3rds.pymysql` | MySQL, translated on the client | pymysql | available |
|
|
61
41
|
| `s3rds.psycopg2` | PostgreSQL | psycopg2 | placeholder; `connect()` raises until implemented |
|
|
42
|
+
| `sqlite+s3rds://` | SQLite, written by SQLAlchemy | SQLAlchemy Core / ORM | available — the URL is the only change |
|
|
43
|
+
|
|
44
|
+
**SQLAlchemy** (Core and ORM): only the URL changes — SQLAlchemy's own SQLite dialect writes the SQL,
|
|
45
|
+
`s3rds.sqlite` replaces the standard library's `sqlite3` underneath.
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
engine = create_engine("sqlite+s3rds://admin:...@xxxx.lambda-url.us-east-2.on.aws/blog")
|
|
49
|
+
```
|
|
62
50
|
|
|
63
51
|
## Semantics
|
|
64
52
|
|
|
65
|
-
- **Transactions**: each `execute()` is its own transaction and is durable in S3 when it
|
|
66
|
-
(
|
|
53
|
+
- **Transactions**: by default each `execute()` is its own transaction and is durable in S3 when it
|
|
54
|
+
returns (`autocommit=True`). `commit()` is a no-op. `rollback()` **cannot undo statements that already
|
|
67
55
|
ran** — it warns if anything was written since the last `commit()` (and stays silent otherwise, so
|
|
68
|
-
pools
|
|
69
|
-
|
|
56
|
+
pools may call it freely). For all-or-nothing writes use `transaction([...])`, which runs several
|
|
57
|
+
statements as one commit and one S3 write.
|
|
58
|
+
- `s3rds.sqlite.connect(..., autocommit=False)` (what SQLAlchemy uses) has sqlite3's transactions: reads see
|
|
59
|
+
the transaction's own uncommitted writes, `commit()` writes S3 once, `rollback()` discards. If another
|
|
60
|
+
writer commits first, the transaction raises `OperationalError` instead of overwriting: run it again.
|
|
61
|
+
Keep transactions short — each statement replays the writes before it on the server.
|
|
70
62
|
- Large results are paged by the server (4 MB per page) and reassembled inside `execute()`; `rows`
|
|
71
63
|
is always complete. If the database changes while paging, the query is re-run rather than mixing versions.
|
|
72
64
|
- Writes **never** duplicate: the driver re-sends only when the request provably never reached the
|
|
@@ -74,6 +66,19 @@ conn.transaction([ # all-or-nothing, single S3 w
|
|
|
74
66
|
- `s3rds.pymysql` raises pymysql-style exceptions with MySQL error codes (1062, 1146, 1054, 1064, …);
|
|
75
67
|
unsupported MySQL features raise `NotSupportedError` instead of misbehaving silently.
|
|
76
68
|
|
|
69
|
+
## Versions
|
|
70
|
+
|
|
71
|
+
What the report was verified with; the full report: [REPORT.md](https://github.com/cloudformer/s3rds/blob/main/tests/compat/REPORT.md).
|
|
72
|
+
|
|
73
|
+
| | Version | Fixed by |
|
|
74
|
+
|---|---|---|
|
|
75
|
+
| Python (client) | 3.9 – 3.13: regression suite on 3.9.6, full comparison on 3.13.7 | `requires-python = ">=3.9"` |
|
|
76
|
+
| Server runtime | AWS Lambda `python3.12` | `terraform/main.tf` |
|
|
77
|
+
| SQLite (server engine) | 3.50.4, shipped in the Lambda zip (the runtime's own is 3.40.0) | `src/db/requirements.txt`: `sqlean.py==3.50.4.5` |
|
|
78
|
+
| sqlglot (MySQL translation) | 30.24 | dependency `sqlglot>=30.24,<31` |
|
|
79
|
+
| SQLAlchemy | 2.0, tested on 2.0.55 | extra `pip install "s3rds[sqlalchemy]"` = `SQLAlchemy>=2.0,<2.1` |
|
|
80
|
+
| MySQL (reference) | 8.0.46 in Docker | `tests/compat/docker-compose.yml`: `mysql:8.0.46` |
|
|
81
|
+
|
|
77
82
|
## Server
|
|
78
83
|
|
|
79
84
|
You need an s3rds server in your own AWS account — deploy it once with Terraform or CloudFormation
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "s3rds"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "A truly serverless RDS on your own AWS account: Lambda URL endpoint, SQLite on S3, ≈ $0 when idle. Drivers named after the standard clients: s3rds.sqlite, s3rds.pymysql; SQLAlchemy via sqlite+s3rds://."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Frank Zhang" }]
|
|
13
|
+
keywords = ["serverless", "rds", "sqlite", "s3", "lambda", "database", "pymysql", "dbapi", "sqlalchemy"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.9",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Topic :: Database",
|
|
22
|
+
"Topic :: Database :: Front-Ends",
|
|
23
|
+
]
|
|
24
|
+
# one install for every driver. sqlglot: s3rds.pymysql's MySQL -> SQLite translation, imported only when a MySQL
|
|
25
|
+
# statement is translated; pinned to the release the compatibility report was verified with (translations
|
|
26
|
+
# change between releases). SQLAlchemy is not a dependency: the applications that use sqlite+s3rds:// have it.
|
|
27
|
+
dependencies = ["sqlglot>=30.24,<31"]
|
|
28
|
+
|
|
29
|
+
[project.optional-dependencies]
|
|
30
|
+
mysql = [] # kept so that pip install "s3rds[mysql]" from 0.1.x keeps working
|
|
31
|
+
sqlalchemy = ["SQLAlchemy>=2.0,<2.1"] # the series the compatibility report verified; pip install "s3rds[sqlalchemy]"
|
|
32
|
+
|
|
33
|
+
[project.entry-points."sqlalchemy.dialects"]
|
|
34
|
+
"sqlite.s3rds" = "s3rds.sqlite:SQLAlchemyDialect"
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://github.com/cloudformer/s3rds"
|
|
38
|
+
Repository = "https://github.com/cloudformer/s3rds"
|
|
39
|
+
Deploy = "https://github.com/cloudformer/s3rds#quick-start"
|
|
40
|
+
Compatibility = "https://github.com/cloudformer/s3rds/blob/main/tests/compat/REPORT.md"
|
|
41
|
+
Changelog = "https://github.com/cloudformer/s3rds/blob/main/CHANGELOG.md"
|
|
42
|
+
Issues = "https://github.com/cloudformer/s3rds/issues"
|
|
43
|
+
|
|
44
|
+
[tool.setuptools]
|
|
45
|
+
packages = ["s3rds"]
|
|
46
|
+
|
|
47
|
+
# The versions tests/compat/REPORT.md was verified with; README.md, README.zh.md and client/README.md state the same.
|
|
48
|
+
[tool.s3rds.verified]
|
|
49
|
+
python = ["3.9.6", "3.13.7"] # the client: regression suite on 3.9.6, the full comparison on 3.13.7
|
|
50
|
+
lambda_runtime = "python3.12" # the server (terraform/main.tf)
|
|
51
|
+
sqlite = "3.50.4" # the server's engine, shipped in the Lambda zip (the runtime's own is 3.40.0)
|
|
52
|
+
sqlean = "3.50.4.5" # sqlean.py, which carries it (src/db/requirements.txt)
|
|
53
|
+
sqlglot = "30.24.0"
|
|
54
|
+
sqlalchemy = "2.0.55"
|
|
55
|
+
mysql_reference = "8.0.46" # tests/compat/docker-compose.yml
|
|
@@ -6,8 +6,11 @@ real database you change one import line and nothing else:
|
|
|
6
6
|
from s3rds import pymysql # today: MySQL dialect on the s3rds endpoint
|
|
7
7
|
import pymysql # later: a real RDS MySQL -- pymysql.connect(...), pymysql.cursors, errors all unchanged
|
|
8
8
|
|
|
9
|
-
from s3rds import sqlite # SQLite dialect, no translation,
|
|
9
|
+
from s3rds import sqlite # SQLite dialect, no translation, standard library only
|
|
10
10
|
from s3rds import psycopg2 # PostgreSQL dialect: placeholder, connect() raises until implemented
|
|
11
|
+
|
|
12
|
+
SQLAlchemy (Core / ORM) needs no import: create_engine("sqlite+s3rds://user:password@<lambda-url>/<db>")
|
|
13
|
+
uses SQLAlchemy's SQLite dialect with s3rds.sqlite as its driver; on RDS the URL becomes mysql+pymysql://.
|
|
11
14
|
"""
|
|
12
15
|
from . import sqlite # noqa: F401
|
|
13
16
|
from . import pymysql # noqa: F401 (sqlglot is only needed when a MySQL statement is actually translated)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""s3rds.pymysql — the MySQL-dialect driver for s3rds. Write MySQL; it runs on the s3rds endpoint (SQLite engine).
|
|
2
2
|
|
|
3
|
-
pip install
|
|
3
|
+
pip install s3rds # sqlglot (the dialect translation) comes with it
|
|
4
4
|
|
|
5
5
|
from s3rds import pymysql # later: `import pymysql` on a real RDS, nothing else changes
|
|
6
6
|
conn = pymysql.connect(host="xxx.lambda-url.us-east-2.on.aws",
|
|
@@ -248,6 +248,16 @@ def _substring_index(node, args):
|
|
|
248
248
|
return template.format(*args)
|
|
249
249
|
|
|
250
250
|
|
|
251
|
+
def _concat_ws(node, args):
|
|
252
|
+
"""CONCAT_WS without SQLite's concat_ws() (3.44+; the Lambda runtime's SQLite is older): NULL arguments
|
|
253
|
+
are skipped, a NULL separator gives NULL -- as MySQL."""
|
|
254
|
+
sep, *items = [a.sql(dialect="sqlite") for a in node.expressions]
|
|
255
|
+
if not items:
|
|
256
|
+
raise NotSupportedError(1582, "wrong number of arguments for CONCAT_WS()")
|
|
257
|
+
parts = " || ".join(f"COALESCE({sep} || ({x}), '')" for x in items)
|
|
258
|
+
return f"SUBSTR({parts}, LENGTH({sep}) + 1)"
|
|
259
|
+
|
|
260
|
+
|
|
251
261
|
_FUNCS = {
|
|
252
262
|
"NOW": "CURRENT_TIMESTAMP",
|
|
253
263
|
"CURDATE": "CURRENT_DATE",
|
|
@@ -265,6 +275,7 @@ _FUNCS = {
|
|
|
265
275
|
"PAD": lambda node, args: _PAD.format(*args, left="" if node.args.get("is_left") else f"{args[0]} || ",
|
|
266
276
|
right=f" || {args[0]}" if node.args.get("is_left") else ""),
|
|
267
277
|
"SUBSTRING_INDEX": _substring_index,
|
|
278
|
+
"CONCAT_WS": _concat_ws,
|
|
268
279
|
"YEAR": "CAST(STRFTIME('%Y', {0}) AS INTEGER)",
|
|
269
280
|
"MONTH": "CAST(STRFTIME('%m', {0}) AS INTEGER)",
|
|
270
281
|
"DAY": "CAST(STRFTIME('%d', {0}) AS INTEGER)",
|
|
@@ -546,7 +557,7 @@ def _translate_stmts(sql):
|
|
|
546
557
|
"""MySQL statement -> tuple of SQLite statements (usually one; CREATE TABLE with inline indexes and
|
|
547
558
|
RENAME TABLE give several, a type-only ALTER gives none)."""
|
|
548
559
|
if sqlglot is None:
|
|
549
|
-
raise NotSupportedError(0, "the MySQL dialect needs sqlglot: pip install
|
|
560
|
+
raise NotSupportedError(0, "the MySQL dialect needs sqlglot (an s3rds dependency): pip install s3rds")
|
|
550
561
|
m = _RENAME_TABLE.match(sql)
|
|
551
562
|
if m:
|
|
552
563
|
pairs = _RENAME_PAIR.findall(m.group(1))
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""s3rds.sqlite — the native (SQLite-dialect) driver for s3rds: Lambda URL endpoint, SQLite on S3.
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Standard library only. The API is shaped like pymysql
|
|
4
4
|
(connect(host=, user=, password=, database=), cursors, %s placeholders); the SQL dialect is SQLite.
|
|
5
5
|
|
|
6
6
|
from s3rds import sqlite
|
|
@@ -19,9 +19,14 @@ Zero dependencies (standard library only). The API is shaped like pymysql
|
|
|
19
19
|
conn.transaction([("UPDATE ...", [..]), ("INSERT ...", [..])]) # one transaction, one S3 write
|
|
20
20
|
conn.last # {"ms": server time, "s3": "cache-hit"...}
|
|
21
21
|
|
|
22
|
-
Transactions: every execute() is its own transaction and is durable in S3 when it returns
|
|
23
|
-
(= pymysql autocommit=True)
|
|
24
|
-
|
|
22
|
+
Transactions: by default every execute() is its own transaction and is durable in S3 when it returns
|
|
23
|
+
(= pymysql autocommit=True); transaction() runs several atomically, commit()/rollback() are no-ops.
|
|
24
|
+
connect(..., autocommit=False) gives sqlite3's transactions: INSERT / UPDATE / DELETE open one, reads see
|
|
25
|
+
its own writes, commit() writes S3 once, rollback() discards; another writer committing in between makes
|
|
26
|
+
the next statement or commit() raise OperationalError (roll back and run it again).
|
|
27
|
+
|
|
28
|
+
SQLAlchemy: create_engine("sqlite+s3rds://user:password@xxx.lambda-url.us-east-2.on.aws/blog")
|
|
29
|
+
SQLAlchemy's SQLite dialect with this driver, autocommit=False; SQLAlchemy itself is the application's.
|
|
25
30
|
|
|
26
31
|
Retry policy -- fail rather than duplicate a write:
|
|
27
32
|
the request is re-sent exactly once, and only when it provably never reached the server
|
|
@@ -34,18 +39,28 @@ import datetime
|
|
|
34
39
|
import decimal
|
|
35
40
|
import http.client
|
|
36
41
|
import json
|
|
42
|
+
import re
|
|
37
43
|
import socket
|
|
38
44
|
import urllib.parse
|
|
39
45
|
import warnings
|
|
40
46
|
|
|
41
|
-
__version__ = "0.
|
|
47
|
+
__version__ = "0.2.0"
|
|
42
48
|
|
|
43
49
|
apilevel = "2.0"
|
|
44
50
|
threadsafety = 1
|
|
45
51
|
paramstyle = "format" # %s like pymysql; ? is accepted too
|
|
52
|
+
# the engine behind the endpoint: the Lambda runtime's SQLite (tests/compat/TASKS.md, task 11);
|
|
53
|
+
# SQLAlchemy reads it to decide what SQL it may use (RETURNING needs 3.35)
|
|
54
|
+
sqlite_version = "3.40.0"
|
|
55
|
+
sqlite_version_info = (3, 40, 0)
|
|
56
|
+
Binary = memoryview # as sqlite3
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# ---------------- exceptions: the DB-API classes sqlite3 raises ----------------
|
|
46
60
|
|
|
61
|
+
class Warning(Exception): # noqa: A001 (DB-API name, as sqlite3.Warning)
|
|
62
|
+
pass
|
|
47
63
|
|
|
48
|
-
# ---------------- exceptions ----------------
|
|
49
64
|
|
|
50
65
|
class Error(Exception):
|
|
51
66
|
def __init__(self, status, message):
|
|
@@ -54,6 +69,29 @@ class Error(Exception):
|
|
|
54
69
|
self.message = message
|
|
55
70
|
|
|
56
71
|
|
|
72
|
+
class InterfaceError(Error): pass
|
|
73
|
+
class DatabaseError(Error): pass
|
|
74
|
+
class DataError(DatabaseError): pass
|
|
75
|
+
class OperationalError(DatabaseError): pass
|
|
76
|
+
class IntegrityError(DatabaseError): pass
|
|
77
|
+
class InternalError(DatabaseError): pass
|
|
78
|
+
class ProgrammingError(DatabaseError): pass
|
|
79
|
+
class NotSupportedError(DatabaseError): pass
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
_BY_NAME = {c.__name__: c for c in (InterfaceError, DatabaseError, DataError, OperationalError,
|
|
83
|
+
IntegrityError, InternalError, ProgrammingError, NotSupportedError)}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _error(status, message):
|
|
87
|
+
"""A failed reply -> the class sqlite3 would have raised. SQLite errors arrive as "IntegrityError: ...";
|
|
88
|
+
the rest by status: 400 a bad request, 401/403/404/409 and the network (0) operational, 5xx internal."""
|
|
89
|
+
cls = _BY_NAME.get((message or "").split(":", 1)[0])
|
|
90
|
+
if cls is None:
|
|
91
|
+
cls = (ProgrammingError if status == 400 else InternalError if status >= 500 else OperationalError)
|
|
92
|
+
return cls(status, message)
|
|
93
|
+
|
|
94
|
+
|
|
57
95
|
class _NotDelivered(Exception):
|
|
58
96
|
"""The request provably never reached the server: sending failed, or no status line came back.
|
|
59
97
|
|
|
@@ -85,7 +123,7 @@ def _encode_value(value, index):
|
|
|
85
123
|
return str(value) # keeps the precision; never a float
|
|
86
124
|
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
87
125
|
return {"$b": base64.b64encode(bytes(value)).decode()}
|
|
88
|
-
raise
|
|
126
|
+
raise ProgrammingError(0, f"parameter #{index}: type {type(value).__name__} cannot be encoded")
|
|
89
127
|
|
|
90
128
|
|
|
91
129
|
def _encode_params(params):
|
|
@@ -173,11 +211,12 @@ class Cursor:
|
|
|
173
211
|
return self.rowcount
|
|
174
212
|
|
|
175
213
|
def executemany(self, sql, seq_of_params):
|
|
176
|
-
"""Same statement, several parameter sets, one transaction (all or nothing)
|
|
214
|
+
"""Same statement, several parameter sets, one transaction (all or nothing) -- or, with
|
|
215
|
+
autocommit=False, part of the open transaction.
|
|
177
216
|
|
|
178
217
|
rowcount = rows actually affected, summed; lastrowid = the last statement's.
|
|
179
218
|
"""
|
|
180
|
-
results = self.connection.
|
|
219
|
+
results = self.connection._executemany(sql, seq_of_params)
|
|
181
220
|
self.rowcount = sum(max(r.get("rowcount", 0), 0) for r in results)
|
|
182
221
|
self.lastrowid = results[-1].get("lastrowid") if results else None
|
|
183
222
|
self.description = None
|
|
@@ -234,7 +273,7 @@ class cursors: # so that sqlite.cursors.DictCursor reads like pymysql.cursors.D
|
|
|
234
273
|
class Connection:
|
|
235
274
|
_typed = False # ask the server for column-type markers (the MySQL driver does; sqlite3 returns text)
|
|
236
275
|
|
|
237
|
-
def __init__(self, url, database, user="", password="", timeout=30, cursorclass=Cursor):
|
|
276
|
+
def __init__(self, url, database, user="", password="", timeout=30, cursorclass=Cursor, autocommit=True):
|
|
238
277
|
p = urllib.parse.urlparse(url.rstrip("/"))
|
|
239
278
|
self._host = p.netloc
|
|
240
279
|
self._base_path = p.path
|
|
@@ -246,6 +285,9 @@ class Connection:
|
|
|
246
285
|
# server-side metrics of the last call: {"ms": ..., "s3": "cache-hit" | "download" | "...+upload"}
|
|
247
286
|
self.last = {}
|
|
248
287
|
self._written = False # a write landed in S3 since the last commit() (drives the rollback() warning)
|
|
288
|
+
self._autocommit = autocommit
|
|
289
|
+
# the open transaction (autocommit=False): {"log": statements that may have written, "base": ETag}
|
|
290
|
+
self._tx = None
|
|
249
291
|
|
|
250
292
|
# ---- transport ----
|
|
251
293
|
|
|
@@ -281,7 +323,7 @@ class Connection:
|
|
|
281
323
|
except ValueError:
|
|
282
324
|
data = {"error": raw[:200].decode(errors="replace")}
|
|
283
325
|
if resp.status >= 400:
|
|
284
|
-
raise
|
|
326
|
+
raise _error(resp.status, data.get("error", ""))
|
|
285
327
|
return data
|
|
286
328
|
|
|
287
329
|
def _post(self, path, payload):
|
|
@@ -291,18 +333,18 @@ class Connection:
|
|
|
291
333
|
except _NotDelivered as e:
|
|
292
334
|
self._drop()
|
|
293
335
|
if not e.reused: # a fresh connection failed: not a stale keep-alive, do not guess
|
|
294
|
-
raise
|
|
336
|
+
raise OperationalError(0, f"connection failed: {e.cause}") from None
|
|
295
337
|
try:
|
|
296
338
|
return self._send(path, body) # the only safe retry: provably not delivered
|
|
297
339
|
except _NotDelivered as e2:
|
|
298
340
|
self._drop()
|
|
299
|
-
raise
|
|
341
|
+
raise OperationalError(0, f"connection failed: {e2.cause}") from None
|
|
300
342
|
except (socket.timeout, TimeoutError):
|
|
301
343
|
self._drop()
|
|
302
|
-
raise
|
|
344
|
+
raise OperationalError(0, "timeout: result unknown (the statement may have run); do not blindly retry writes") from None
|
|
303
345
|
except (OSError, http.client.HTTPException) as e:
|
|
304
346
|
self._drop()
|
|
305
|
-
raise
|
|
347
|
+
raise OperationalError(0, f"network error (result unknown): {e}") from None
|
|
306
348
|
|
|
307
349
|
def _get(self, path):
|
|
308
350
|
"""A GET on a short-lived connection (the keep-alive path is POST-only)."""
|
|
@@ -312,7 +354,7 @@ class Connection:
|
|
|
312
354
|
resp = conn.getresponse()
|
|
313
355
|
data = json.loads(resp.read() or b"{}")
|
|
314
356
|
if resp.status >= 400:
|
|
315
|
-
raise
|
|
357
|
+
raise _error(resp.status, data.get("error", ""))
|
|
316
358
|
return data
|
|
317
359
|
finally:
|
|
318
360
|
conn.close()
|
|
@@ -323,20 +365,49 @@ class Connection:
|
|
|
323
365
|
return (cursorclass or self.cursorclass)(self)
|
|
324
366
|
|
|
325
367
|
def commit(self):
|
|
326
|
-
"""
|
|
327
|
-
|
|
368
|
+
"""autocommit=False: the open transaction's writes commit together, S3 is written once; a conflict
|
|
369
|
+
with another writer raises OperationalError and nothing lands.
|
|
370
|
+
autocommit=True: every statement is already committed and durable when execute() returns; this only
|
|
371
|
+
clears the 'written since last commit' flag (kept for pymysql compatibility)."""
|
|
372
|
+
tx, self._tx = self._tx, None
|
|
373
|
+
if tx and tx["log"]:
|
|
374
|
+
payload = {"statements": tx["log"], "replay": len(tx["log"]), "base": tx["base"], "commit": True}
|
|
375
|
+
self._note(self._replay(payload))
|
|
328
376
|
self._written = False
|
|
329
377
|
|
|
330
378
|
def rollback(self):
|
|
331
|
-
"""
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
379
|
+
"""autocommit=False: the open transaction never reached S3; forgetting it is the rollback.
|
|
380
|
+
autocommit=True: cannot undo anything, statements are durable as soon as they run. If something was
|
|
381
|
+
written since the last commit() this warns (never silent); with nothing written it stays quiet,
|
|
382
|
+
because pools and ORMs call rollback() when returning a connection."""
|
|
383
|
+
self._tx = None
|
|
384
|
+
if self._written and self._autocommit:
|
|
335
385
|
warnings.warn("s3rds: rollback() cannot undo statements that already ran; "
|
|
336
386
|
"use transaction() for all-or-nothing writes", RuntimeWarning, stacklevel=2)
|
|
337
387
|
self._written = False
|
|
338
388
|
|
|
389
|
+
@property
|
|
390
|
+
def isolation_level(self):
|
|
391
|
+
"""As sqlite3: None = autocommit, "" = transactions (implicit BEGIN before INSERT / UPDATE / DELETE /
|
|
392
|
+
REPLACE). Setting None commits the open transaction, as sqlite3 does."""
|
|
393
|
+
return None if self._autocommit else ""
|
|
394
|
+
|
|
395
|
+
@isolation_level.setter
|
|
396
|
+
def isolation_level(self, value):
|
|
397
|
+
if value is None and self._tx is not None:
|
|
398
|
+
self.commit()
|
|
399
|
+
self._autocommit = value is None
|
|
400
|
+
|
|
401
|
+
@property
|
|
402
|
+
def in_transaction(self):
|
|
403
|
+
return self._tx is not None
|
|
404
|
+
|
|
405
|
+
def create_function(self, *a, **kw):
|
|
406
|
+
raise NotSupportedError(0, "user-defined functions would run on the server; "
|
|
407
|
+
"not supported (that is the moment to move to RDS)")
|
|
408
|
+
|
|
339
409
|
def close(self):
|
|
410
|
+
self._tx = None # as sqlite3: closing without commit() discards the open transaction
|
|
340
411
|
self._drop()
|
|
341
412
|
|
|
342
413
|
def ping(self, reconnect=True):
|
|
@@ -367,7 +438,15 @@ class Connection:
|
|
|
367
438
|
Large results come back from the server in pages; all pages are fetched here, so `rows` is
|
|
368
439
|
always complete. If the database changes while paging the server refuses to mix versions
|
|
369
440
|
(409) and the whole query is re-run, up to 3 times.
|
|
441
|
+
|
|
442
|
+
autocommit=False follows sqlite3's rules: INSERT / UPDATE / DELETE / REPLACE (and SAVEPOINT) open a
|
|
443
|
+
transaction; BEGIN / COMMIT / ROLLBACK statements act as begin / commit() / rollback(); anything
|
|
444
|
+
else outside a transaction runs on its own.
|
|
370
445
|
"""
|
|
446
|
+
if not self._autocommit:
|
|
447
|
+
r = self._tx_statement(sql, params)
|
|
448
|
+
if r is not None:
|
|
449
|
+
return r
|
|
371
450
|
path = f"/api/db/{self.database}/query"
|
|
372
451
|
payload = {"sql": _to_qmark(sql), "params": _encode_params(params)}
|
|
373
452
|
if self._typed:
|
|
@@ -383,7 +462,7 @@ class Connection:
|
|
|
383
462
|
def _fetch_pages(self, path, payload):
|
|
384
463
|
first = self._post(path, payload)
|
|
385
464
|
if first.get("truncated") and not first.get("next_token"):
|
|
386
|
-
raise
|
|
465
|
+
raise InterfaceError(0, "the server truncated the result (server older than 0.1.1); upgrade the server")
|
|
387
466
|
rows, page = first["rows"], first
|
|
388
467
|
while page.get("next_token"):
|
|
389
468
|
page = self._post(path, {"next_token": page["next_token"]})
|
|
@@ -409,8 +488,7 @@ class Connection:
|
|
|
409
488
|
|
|
410
489
|
Each item is "sql" or (sql, params). Returns one result per statement.
|
|
411
490
|
"""
|
|
412
|
-
stmts = [
|
|
413
|
-
else {"sql": _to_qmark(s[0]), "params": _encode_params(s[1] if len(s) > 1 else [])}
|
|
491
|
+
stmts = [_statement(s, []) if isinstance(s, str) else _statement(s[0], s[1] if len(s) > 1 else [])
|
|
414
492
|
for s in statements]
|
|
415
493
|
payload = {"statements": stmts}
|
|
416
494
|
if self._typed:
|
|
@@ -419,14 +497,144 @@ class Connection:
|
|
|
419
497
|
self._note(r)
|
|
420
498
|
return [_decode_result(x) for x in r["results"]]
|
|
421
499
|
|
|
500
|
+
# ---- open transactions (autocommit=False) ----
|
|
501
|
+
# The server keeps no session: every statement inside a transaction is sent with the log of the
|
|
502
|
+
# statements that may have written so far. The server replays the log on the version the transaction
|
|
503
|
+
# began on (base), runs the new statements, returns their results and rolls back; commit() sends the log
|
|
504
|
+
# once more for real. Reads see the transaction's own writes; if anyone else committed in between, the
|
|
505
|
+
# base is gone and the next statement or commit() raises OperationalError: roll back, run it again.
|
|
506
|
+
|
|
507
|
+
def _tx_statement(self, sql, params):
|
|
508
|
+
"""sqlite3's rules for one statement; returns its result, or None when it runs on its own."""
|
|
509
|
+
if _BEGIN.match(sql):
|
|
510
|
+
if self._tx is not None:
|
|
511
|
+
raise OperationalError(0, "cannot start a transaction within a transaction")
|
|
512
|
+
self._tx = {"log": [], "base": None}
|
|
513
|
+
return _empty()
|
|
514
|
+
if _COMMIT.match(sql):
|
|
515
|
+
self.commit()
|
|
516
|
+
return _empty()
|
|
517
|
+
if _ROLLBACK.match(sql):
|
|
518
|
+
self.rollback()
|
|
519
|
+
return _empty()
|
|
520
|
+
if self._tx is None:
|
|
521
|
+
if not _IMPLICIT_BEGIN.match(sql):
|
|
522
|
+
return None
|
|
523
|
+
self._tx = {"log": [], "base": None}
|
|
524
|
+
return self._tx_run([(sql, params)])[0]
|
|
525
|
+
|
|
526
|
+
def _executemany(self, sql, seq_of_params):
|
|
527
|
+
statements = [(sql, p) for p in seq_of_params]
|
|
528
|
+
if not statements:
|
|
529
|
+
return []
|
|
530
|
+
if self._autocommit or (self._tx is None and not _IMPLICIT_BEGIN.match(sql)):
|
|
531
|
+
return self.transaction(statements)
|
|
532
|
+
if self._tx is None:
|
|
533
|
+
self._tx = {"log": [], "base": None}
|
|
534
|
+
return self._tx_run(statements)
|
|
535
|
+
|
|
536
|
+
def _tx_run(self, statements):
|
|
537
|
+
tx = self._tx
|
|
538
|
+
new = [_statement(sql, params) for sql, params in statements]
|
|
539
|
+
payload = {"statements": tx["log"] + new, "replay": len(tx["log"]), "commit": False}
|
|
540
|
+
if tx["base"]:
|
|
541
|
+
payload["base"] = tx["base"]
|
|
542
|
+
r = self._replay(payload)
|
|
543
|
+
tx["base"] = tx["base"] or r.get("etag")
|
|
544
|
+
tx["log"] += [s for s in new if not _READS.match(s["sql"])]
|
|
545
|
+
self._note(r)
|
|
546
|
+
return [_decode_result(x) for x in r["results"]]
|
|
547
|
+
|
|
548
|
+
def _replay(self, payload):
|
|
549
|
+
if self._typed:
|
|
550
|
+
payload["types"] = True
|
|
551
|
+
try:
|
|
552
|
+
return self._post(f"/api/db/{self.database}/replay", payload)
|
|
553
|
+
except Error as e:
|
|
554
|
+
if e.status == 404 and "no route" in e.message:
|
|
555
|
+
raise OperationalError(404, "the server has no open transactions (older than this driver); "
|
|
556
|
+
"redeploy the server") from None
|
|
557
|
+
raise
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
_BEGIN = re.compile(r"\s*BEGIN\b", re.I)
|
|
561
|
+
_COMMIT = re.compile(r"\s*(COMMIT|END)\b", re.I)
|
|
562
|
+
_ROLLBACK = re.compile(r"\s*ROLLBACK\b(?!.*\bTO\b)", re.I | re.S) # ROLLBACK TO <savepoint> is a statement
|
|
563
|
+
_IMPLICIT_BEGIN = re.compile(r"\s*(INSERT|UPDATE|DELETE|REPLACE|SAVEPOINT)\b", re.I)
|
|
564
|
+
_READS = re.compile(r"\s*(SELECT|VALUES|EXPLAIN)\b", re.I) # cannot write: never replayed
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def _statement(sql, params):
|
|
568
|
+
return {"sql": _to_qmark(sql), "params": _encode_params(params)}
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _empty():
|
|
572
|
+
return {"columns": [], "rows": [], "rowcount": -1, "lastrowid": None}
|
|
573
|
+
|
|
422
574
|
|
|
423
575
|
# ---------------- module entry point, same signature as pymysql.connect ----------------
|
|
424
576
|
|
|
425
577
|
def connect(host, user="", password="", database="", timeout=30,
|
|
426
|
-
cursorclass=Cursor, **_ignored):
|
|
427
|
-
"""Extra pymysql arguments (port, charset,
|
|
578
|
+
cursorclass=Cursor, autocommit=True, **_ignored):
|
|
579
|
+
"""Extra pymysql arguments (port, charset, ...) are accepted and ignored.
|
|
428
580
|
|
|
429
581
|
host may be given with or without https:// (Lambda Function URLs are https only).
|
|
582
|
+
autocommit=True (default): every statement commits on its own. autocommit=False: transactions as
|
|
583
|
+
sqlite3 has them -- commit() / rollback(), reads see the transaction's own writes.
|
|
430
584
|
"""
|
|
431
585
|
url = host if host.startswith("http") else "https://" + host
|
|
432
|
-
return Connection(url, database, user, password, timeout, cursorclass)
|
|
586
|
+
return Connection(url, database, user, password, timeout, cursorclass, autocommit)
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
# ---------------- SQLAlchemy: sqlite+s3rds://user:password@<lambda-url>/<database> ----------------
|
|
590
|
+
|
|
591
|
+
def __getattr__(name):
|
|
592
|
+
"""PEP 562: the dialect is built when SQLAlchemy asks for it, so SQLAlchemy stays optional."""
|
|
593
|
+
if name == "SQLAlchemyDialect":
|
|
594
|
+
global SQLAlchemyDialect
|
|
595
|
+
SQLAlchemyDialect = _sqlalchemy_dialect()
|
|
596
|
+
return SQLAlchemyDialect
|
|
597
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _sqlalchemy_dialect():
|
|
601
|
+
"""SQLAlchemy's own SQLite dialect (it writes SQLite SQL; nothing is translated) with this module as the
|
|
602
|
+
DB-API driver in place of the standard library's sqlite3."""
|
|
603
|
+
import sys
|
|
604
|
+
|
|
605
|
+
from sqlalchemy.dialects.sqlite.pysqlite import SQLiteDialect_pysqlite
|
|
606
|
+
from sqlalchemy.pool import QueuePool
|
|
607
|
+
|
|
608
|
+
module = sys.modules[__name__]
|
|
609
|
+
|
|
610
|
+
class SQLAlchemyDialect(SQLiteDialect_pysqlite):
|
|
611
|
+
driver = "s3rds"
|
|
612
|
+
supports_statement_cache = True
|
|
613
|
+
|
|
614
|
+
def __init__(self, *a, **kw):
|
|
615
|
+
kw.setdefault("paramstyle", "qmark") # the module says format (pymysql style); SQLite's own is qmark
|
|
616
|
+
super().__init__(*a, **kw)
|
|
617
|
+
|
|
618
|
+
@classmethod
|
|
619
|
+
def import_dbapi(cls):
|
|
620
|
+
return module
|
|
621
|
+
|
|
622
|
+
@classmethod
|
|
623
|
+
def get_pool_class(cls, url):
|
|
624
|
+
return QueuePool
|
|
625
|
+
|
|
626
|
+
def create_connect_args(self, url):
|
|
627
|
+
host = url.host + (f":{url.port}" if url.port else "")
|
|
628
|
+
return [], {"host": host, "user": url.username or "", "password": url.password or "",
|
|
629
|
+
"database": url.database or "", "autocommit": False}
|
|
630
|
+
|
|
631
|
+
def on_connect(self):
|
|
632
|
+
return None # pysqlite registers Python functions (regexp, floor) here; they cannot run on the server
|
|
633
|
+
|
|
634
|
+
def set_isolation_level(self, dbapi_connection, level):
|
|
635
|
+
dbapi_connection.isolation_level = None if level == "AUTOCOMMIT" else ""
|
|
636
|
+
|
|
637
|
+
def get_isolation_level(self, dbapi_connection):
|
|
638
|
+
return "SERIALIZABLE" # a transaction fails on conflict rather than read another writer's data
|
|
639
|
+
|
|
640
|
+
return SQLAlchemyDialect
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: s3rds
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: A truly serverless RDS on your own AWS account: Lambda URL endpoint, SQLite on S3, ≈ $0 when idle. Drivers named after the standard clients: s3rds.sqlite, s3rds.pymysql; SQLAlchemy via sqlite+s3rds://.
|
|
5
|
+
Author: Frank Zhang
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/cloudformer/s3rds
|
|
8
|
+
Project-URL: Repository, https://github.com/cloudformer/s3rds
|
|
9
|
+
Project-URL: Deploy, https://github.com/cloudformer/s3rds#quick-start
|
|
10
|
+
Project-URL: Compatibility, https://github.com/cloudformer/s3rds/blob/main/tests/compat/REPORT.md
|
|
11
|
+
Project-URL: Changelog, https://github.com/cloudformer/s3rds/blob/main/CHANGELOG.md
|
|
12
|
+
Project-URL: Issues, https://github.com/cloudformer/s3rds/issues
|
|
13
|
+
Keywords: serverless,rds,sqlite,s3,lambda,database,pymysql,dbapi,sqlalchemy
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Database
|
|
21
|
+
Classifier: Topic :: Database :: Front-Ends
|
|
22
|
+
Requires-Python: >=3.9
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
Requires-Dist: sqlglot<31,>=30.24
|
|
25
|
+
Provides-Extra: mysql
|
|
26
|
+
Provides-Extra: sqlalchemy
|
|
27
|
+
Requires-Dist: SQLAlchemy<2.1,>=2.0; extra == "sqlalchemy"
|
|
28
|
+
|
|
29
|
+
# s3rds
|
|
30
|
+
|
|
31
|
+
**Python drivers for [s3rds](https://github.com/cloudformer/s3rds) — a truly serverless RDS:
|
|
32
|
+
a Lambda URL as your database endpoint, SQLite on S3, ≈ $0 when idle.**
|
|
33
|
+
|
|
34
|
+

|
|
35
|
+
|
|
36
|
+
One install, one dependency (sqlglot, used only by `s3rds.pymysql`). Each driver is a submodule **named after the standard
|
|
37
|
+
client it mirrors**, so moving to a real database is a one-line import change and nothing else:
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install s3rds # s3rds.sqlite, s3rds.pymysql, sqlite+s3rds:// for SQLAlchemy
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
from s3rds import pymysql # today -- later: `import pymysql` on a real RDS, rest unchanged
|
|
45
|
+
|
|
46
|
+
conn = pymysql.connect(
|
|
47
|
+
host="xxxx.lambda-url.us-east-2.on.aws",
|
|
48
|
+
user="admin", password="...", database="blog")
|
|
49
|
+
|
|
50
|
+
with conn.cursor() as cur:
|
|
51
|
+
cur.execute("CREATE TABLE posts (id INT AUTO_INCREMENT PRIMARY KEY, title VARCHAR(200))")
|
|
52
|
+
cur.execute("INSERT INTO posts (title) VALUES (%s)", ["hello"])
|
|
53
|
+
print(cur.lastrowid)
|
|
54
|
+
|
|
55
|
+
with conn.cursor(pymysql.cursors.DictCursor) as cur:
|
|
56
|
+
cur.execute("SELECT * FROM posts WHERE id = %s", [1])
|
|
57
|
+
print(cur.fetchall()) # [{'id': 1, 'title': 'hello'}]
|
|
58
|
+
|
|
59
|
+
conn.transaction([ # all-or-nothing, single S3 write
|
|
60
|
+
("UPDATE stock SET qty = qty - 1 WHERE id = %s", [7]),
|
|
61
|
+
("INSERT INTO orders (item) VALUES (%s)", [7]),
|
|
62
|
+
])
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
| Submodule | SQL dialect | Mirrors | Status |
|
|
66
|
+
|---|---|---|---|
|
|
67
|
+
| `s3rds.sqlite` | SQLite (no translation) | pymysql-shaped API | available |
|
|
68
|
+
| `s3rds.pymysql` | MySQL, translated on the client | pymysql | available |
|
|
69
|
+
| `s3rds.psycopg2` | PostgreSQL | psycopg2 | placeholder; `connect()` raises until implemented |
|
|
70
|
+
| `sqlite+s3rds://` | SQLite, written by SQLAlchemy | SQLAlchemy Core / ORM | available — the URL is the only change |
|
|
71
|
+
|
|
72
|
+
**SQLAlchemy** (Core and ORM): only the URL changes — SQLAlchemy's own SQLite dialect writes the SQL,
|
|
73
|
+
`s3rds.sqlite` replaces the standard library's `sqlite3` underneath.
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
engine = create_engine("sqlite+s3rds://admin:...@xxxx.lambda-url.us-east-2.on.aws/blog")
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## Semantics
|
|
80
|
+
|
|
81
|
+
- **Transactions**: by default each `execute()` is its own transaction and is durable in S3 when it
|
|
82
|
+
returns (`autocommit=True`). `commit()` is a no-op. `rollback()` **cannot undo statements that already
|
|
83
|
+
ran** — it warns if anything was written since the last `commit()` (and stays silent otherwise, so
|
|
84
|
+
pools may call it freely). For all-or-nothing writes use `transaction([...])`, which runs several
|
|
85
|
+
statements as one commit and one S3 write.
|
|
86
|
+
- `s3rds.sqlite.connect(..., autocommit=False)` (what SQLAlchemy uses) has sqlite3's transactions: reads see
|
|
87
|
+
the transaction's own uncommitted writes, `commit()` writes S3 once, `rollback()` discards. If another
|
|
88
|
+
writer commits first, the transaction raises `OperationalError` instead of overwriting: run it again.
|
|
89
|
+
Keep transactions short — each statement replays the writes before it on the server.
|
|
90
|
+
- Large results are paged by the server (4 MB per page) and reassembled inside `execute()`; `rows`
|
|
91
|
+
is always complete. If the database changes while paging, the query is re-run rather than mixing versions.
|
|
92
|
+
- Writes **never** duplicate: the driver re-sends only when the request provably never reached the
|
|
93
|
+
server (stale keep-alive connection). Timeouts raise an error with an explicit "result unknown".
|
|
94
|
+
- `s3rds.pymysql` raises pymysql-style exceptions with MySQL error codes (1062, 1146, 1054, 1064, …);
|
|
95
|
+
unsupported MySQL features raise `NotSupportedError` instead of misbehaving silently.
|
|
96
|
+
|
|
97
|
+
## Versions
|
|
98
|
+
|
|
99
|
+
What the report was verified with; the full report: [REPORT.md](https://github.com/cloudformer/s3rds/blob/main/tests/compat/REPORT.md).
|
|
100
|
+
|
|
101
|
+
| | Version | Fixed by |
|
|
102
|
+
|---|---|---|
|
|
103
|
+
| Python (client) | 3.9 – 3.13: regression suite on 3.9.6, full comparison on 3.13.7 | `requires-python = ">=3.9"` |
|
|
104
|
+
| Server runtime | AWS Lambda `python3.12` | `terraform/main.tf` |
|
|
105
|
+
| SQLite (server engine) | 3.50.4, shipped in the Lambda zip (the runtime's own is 3.40.0) | `src/db/requirements.txt`: `sqlean.py==3.50.4.5` |
|
|
106
|
+
| sqlglot (MySQL translation) | 30.24 | dependency `sqlglot>=30.24,<31` |
|
|
107
|
+
| SQLAlchemy | 2.0, tested on 2.0.55 | extra `pip install "s3rds[sqlalchemy]"` = `SQLAlchemy>=2.0,<2.1` |
|
|
108
|
+
| MySQL (reference) | 8.0.46 in Docker | `tests/compat/docker-compose.yml`: `mysql:8.0.46` |
|
|
109
|
+
|
|
110
|
+
## Server
|
|
111
|
+
|
|
112
|
+
You need an s3rds server in your own AWS account — deploy it once with Terraform or CloudFormation
|
|
113
|
+
from the [main repository](https://github.com/cloudformer/s3rds). Same as any driver needs its database.
|
s3rds-0.1.2/README.md
DELETED
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
# s3rds
|
|
2
|
-
|
|
3
|
-
**Python drivers for [s3rds](https://github.com/cloudformer/s3rds) — a truly serverless RDS:
|
|
4
|
-
a Lambda URL as your database endpoint, SQLite on S3, ≈ $0 when idle.**
|
|
5
|
-
|
|
6
|
-

|
|
7
|
-
|
|
8
|
-
Zero dependencies (standard library only). Each driver is a submodule **named after the standard
|
|
9
|
-
client it mirrors**, so moving to a real database is a one-line import change and nothing else:
|
|
10
|
-
|
|
11
|
-
```bash
|
|
12
|
-
pip install s3rds # s3rds.sqlite: SQLite dialect
|
|
13
|
-
pip install "s3rds[mysql]" # + s3rds.pymysql: write MySQL, translated on the client (sqlglot)
|
|
14
|
-
```
|
|
15
|
-
|
|
16
|
-
```python
|
|
17
|
-
from s3rds import pymysql # today -- later: `import pymysql` on a real RDS, rest unchanged
|
|
18
|
-
|
|
19
|
-
conn = pymysql.connect(
|
|
20
|
-
host="xxxx.lambda-url.us-east-2.on.aws",
|
|
21
|
-
user="admin", password="...", database="blog")
|
|
22
|
-
|
|
23
|
-
with conn.cursor() as cur:
|
|
24
|
-
cur.execute("CREATE TABLE posts (id INT AUTO_INCREMENT PRIMARY KEY, title VARCHAR(200))")
|
|
25
|
-
cur.execute("INSERT INTO posts (title) VALUES (%s)", ["hello"])
|
|
26
|
-
print(cur.lastrowid)
|
|
27
|
-
|
|
28
|
-
with conn.cursor(pymysql.cursors.DictCursor) as cur:
|
|
29
|
-
cur.execute("SELECT * FROM posts WHERE id = %s", [1])
|
|
30
|
-
print(cur.fetchall()) # [{'id': 1, 'title': 'hello'}]
|
|
31
|
-
|
|
32
|
-
conn.transaction([ # all-or-nothing, single S3 write
|
|
33
|
-
("UPDATE stock SET qty = qty - 1 WHERE id = %s", [7]),
|
|
34
|
-
("INSERT INTO orders (item) VALUES (%s)", [7]),
|
|
35
|
-
])
|
|
36
|
-
```
|
|
37
|
-
|
|
38
|
-
| Submodule | SQL dialect | Mirrors | Status |
|
|
39
|
-
|---|---|---|---|
|
|
40
|
-
| `s3rds.sqlite` | SQLite (no translation) | pymysql-shaped API | available |
|
|
41
|
-
| `s3rds.pymysql` | MySQL, translated on the client | pymysql | available — `pip install "s3rds[mysql]"` |
|
|
42
|
-
| `s3rds.psycopg2` | PostgreSQL | psycopg2 | placeholder; `connect()` raises until implemented |
|
|
43
|
-
|
|
44
|
-
## Semantics
|
|
45
|
-
|
|
46
|
-
- **Transactions**: each `execute()` is its own transaction and is durable in S3 when it returns
|
|
47
|
-
(like `autocommit=True`). `commit()` is a no-op. `rollback()` **cannot undo statements that already
|
|
48
|
-
ran** — it warns if anything was written since the last `commit()` (and stays silent otherwise, so
|
|
49
|
-
pools and ORMs may call it freely). For all-or-nothing writes use `transaction([...])`, which runs
|
|
50
|
-
several statements as one commit and one S3 write.
|
|
51
|
-
- Large results are paged by the server (4 MB per page) and reassembled inside `execute()`; `rows`
|
|
52
|
-
is always complete. If the database changes while paging, the query is re-run rather than mixing versions.
|
|
53
|
-
- Writes **never** duplicate: the driver re-sends only when the request provably never reached the
|
|
54
|
-
server (stale keep-alive connection). Timeouts raise an error with an explicit "result unknown".
|
|
55
|
-
- `s3rds.pymysql` raises pymysql-style exceptions with MySQL error codes (1062, 1146, 1054, 1064, …);
|
|
56
|
-
unsupported MySQL features raise `NotSupportedError` instead of misbehaving silently.
|
|
57
|
-
|
|
58
|
-
## Server
|
|
59
|
-
|
|
60
|
-
You need an s3rds server in your own AWS account — deploy it once with Terraform or CloudFormation
|
|
61
|
-
from the [main repository](https://github.com/cloudformer/s3rds). Same as any driver needs its database.
|
s3rds-0.1.2/pyproject.toml
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = ["setuptools>=68"]
|
|
3
|
-
build-backend = "setuptools.build_meta"
|
|
4
|
-
|
|
5
|
-
[project]
|
|
6
|
-
name = "s3rds"
|
|
7
|
-
version = "0.1.2"
|
|
8
|
-
description = "A truly serverless RDS on your own AWS account: Lambda URL endpoint, SQLite on S3, ≈ $0 when idle. Drivers named after the standard clients: s3rds.sqlite, s3rds.pymysql."
|
|
9
|
-
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.9"
|
|
11
|
-
license = { text = "MIT" }
|
|
12
|
-
authors = [{ name = "Frank Zhang" }]
|
|
13
|
-
keywords = ["serverless", "rds", "sqlite", "s3", "lambda", "database", "pymysql", "dbapi"]
|
|
14
|
-
classifiers = [
|
|
15
|
-
"Development Status :: 3 - Alpha",
|
|
16
|
-
"Intended Audience :: Developers",
|
|
17
|
-
"License :: OSI Approved :: MIT License",
|
|
18
|
-
"Programming Language :: Python :: 3",
|
|
19
|
-
"Topic :: Database",
|
|
20
|
-
"Topic :: Database :: Front-Ends",
|
|
21
|
-
]
|
|
22
|
-
dependencies = [] # standard library only
|
|
23
|
-
|
|
24
|
-
[project.optional-dependencies]
|
|
25
|
-
mysql = ["sqlglot>=25.0"] # s3rds.pymysql: MySQL -> SQLite dialect translation
|
|
26
|
-
|
|
27
|
-
[project.urls]
|
|
28
|
-
Homepage = "https://github.com/cloudformer/s3rds"
|
|
29
|
-
|
|
30
|
-
[tool.setuptools]
|
|
31
|
-
packages = ["s3rds"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|