sqlodin 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlodin/__init__.py +15 -0
- sqlodin/client.py +333 -0
- sqlodin/dbapi.py +324 -0
- sqlodin/errors.py +42 -0
- sqlodin/parameters.py +86 -0
- sqlodin/py.typed +0 -0
- sqlodin/results.py +67 -0
- sqlodin/search.py +110 -0
- sqlodin/sqlalchemy.py +138 -0
- sqlodin/transaction.py +49 -0
- sqlodin/transport.py +97 -0
- sqlodin/vector.py +47 -0
- sqlodin-0.6.0.dist-info/METADATA +253 -0
- sqlodin-0.6.0.dist-info/RECORD +17 -0
- sqlodin-0.6.0.dist-info/WHEEL +4 -0
- sqlodin-0.6.0.dist-info/entry_points.txt +2 -0
- sqlodin-0.6.0.dist-info/licenses/LICENSE +21 -0
sqlodin/errors.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Errors distinguish rejected requests from writes whose outcome is unknown."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Error(Exception):
|
|
5
|
+
"""Base class for SQLodin client errors."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ConnectionError(Error):
|
|
9
|
+
"""No usable authenticated connection, or an invalid server response."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class QueryError(Error):
|
|
13
|
+
"""The server rejected a SQL statement or its bounded result."""
|
|
14
|
+
|
|
15
|
+
def __init__(self, code: str):
|
|
16
|
+
self.code = code
|
|
17
|
+
super().__init__(code)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ConstraintError(QueryError):
|
|
21
|
+
"""A constraint rejected and rolled back the entire transaction."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SerializationError(QueryError):
|
|
25
|
+
"""The read version changed; retry the complete transaction after rollback."""
|
|
26
|
+
sqlstate = "40001"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class SessionError(QueryError):
|
|
30
|
+
"""A session identity is stale, conflicted, or cannot be admitted."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class PendingWriteError(Error):
|
|
34
|
+
"""Resolve the outstanding write before submitting another one."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class UnknownOutcome(Error):
|
|
38
|
+
"""The write may have committed. Retry its identity, never a fresh write."""
|
|
39
|
+
|
|
40
|
+
def __init__(self, pending):
|
|
41
|
+
self.pending = pending
|
|
42
|
+
super().__init__("Write outcome unknown; call resolve_pending() to retry the same request")
|
sqlodin/parameters.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Bound values and qmark translation; values are never interpolated into SQL."""
|
|
2
|
+
import math
|
|
3
|
+
from collections.abc import Sequence
|
|
4
|
+
from .vector import Vector
|
|
5
|
+
|
|
6
|
+
Value = str | int | float | Vector | None
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def encode(value: Value) -> dict:
|
|
10
|
+
if isinstance(value, Vector):
|
|
11
|
+
return {"kind": "vector", "vector": list(value)}
|
|
12
|
+
if value is None:
|
|
13
|
+
return {"kind": "null"}
|
|
14
|
+
if isinstance(value, int):
|
|
15
|
+
if not -(2**63) <= value < 2**63:
|
|
16
|
+
raise ValueError("Integers must fit SQLite's signed 64-bit range")
|
|
17
|
+
return {"kind": "integer", "integer": int(value)}
|
|
18
|
+
if isinstance(value, float):
|
|
19
|
+
if not math.isfinite(value):
|
|
20
|
+
raise ValueError("Floating-point parameters must be finite")
|
|
21
|
+
return {"kind": "real", "real": value}
|
|
22
|
+
if isinstance(value, str):
|
|
23
|
+
if len(value.encode('utf-8')) > 256:
|
|
24
|
+
raise ValueError("Text parameters are limited to 256 UTF-8 bytes")
|
|
25
|
+
return {"kind": "text", "text": value}
|
|
26
|
+
raise TypeError("Parameters support str, int, float, Vector, and None")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def prepare(sql: str, parameters: Sequence[Value], offset: int = 0) -> tuple[str, tuple[Value, ...]]:
|
|
30
|
+
if not isinstance(sql, str) or not sql.strip() or '\x00' in sql:
|
|
31
|
+
raise ValueError("SQL must be a nonempty string without NUL")
|
|
32
|
+
if isinstance(parameters, (str, bytes, dict)):
|
|
33
|
+
raise TypeError("Parameters must be a sequence of values")
|
|
34
|
+
values = tuple(parameters)
|
|
35
|
+
if offset + len(values) > 16:
|
|
36
|
+
raise ValueError("A request supports at most 16 parameters")
|
|
37
|
+
if sum(len(v) for v in values if isinstance(v, Vector)) > 384:
|
|
38
|
+
raise ValueError("A request supports at most 384 vector components in total")
|
|
39
|
+
for value in values:
|
|
40
|
+
encode(value)
|
|
41
|
+
# SQLite lexical quoting and comments; reject numbered/named bindings so a
|
|
42
|
+
# batch can assign one unambiguous global parameter tuple to all statements.
|
|
43
|
+
out, count, i = [], 0, 0
|
|
44
|
+
while i < len(sql):
|
|
45
|
+
char = sql[i]
|
|
46
|
+
if sql.startswith('--', i):
|
|
47
|
+
end = sql.find('\n', i)
|
|
48
|
+
end = len(sql) if end == -1 else end
|
|
49
|
+
elif sql.startswith('/*', i):
|
|
50
|
+
end = sql.find('*/', i + 2)
|
|
51
|
+
if end == -1:
|
|
52
|
+
raise ValueError("Unterminated SQL comment")
|
|
53
|
+
end += 2
|
|
54
|
+
elif char in "'\"`[":
|
|
55
|
+
closing = ']' if char == '[' else char
|
|
56
|
+
end = i + 1
|
|
57
|
+
while end < len(sql):
|
|
58
|
+
if sql[end] == closing:
|
|
59
|
+
end += 1
|
|
60
|
+
if closing != ']' and end < len(sql) and sql[end] == closing:
|
|
61
|
+
end += 1
|
|
62
|
+
continue
|
|
63
|
+
break
|
|
64
|
+
end += 1
|
|
65
|
+
else:
|
|
66
|
+
raise ValueError("Unterminated SQL quote")
|
|
67
|
+
else:
|
|
68
|
+
if char == '?':
|
|
69
|
+
if i + 1 < len(sql) and sql[i + 1].isdigit():
|
|
70
|
+
raise ValueError("Use plain ? placeholders, not numbered parameters")
|
|
71
|
+
count += 1
|
|
72
|
+
out.append(f'?{offset + count}')
|
|
73
|
+
elif char in ':@$':
|
|
74
|
+
raise ValueError("Use plain ? placeholders, not named parameters")
|
|
75
|
+
else:
|
|
76
|
+
out.append(char)
|
|
77
|
+
i += 1
|
|
78
|
+
continue
|
|
79
|
+
out.append(sql[i:end])
|
|
80
|
+
i = end
|
|
81
|
+
if count != len(values):
|
|
82
|
+
raise ValueError(f"SQL has {count} placeholders but received {len(values)} parameters")
|
|
83
|
+
text = ''.join(out)
|
|
84
|
+
if len(text.encode('utf-8')) > 4096:
|
|
85
|
+
raise ValueError("A SQL request is limited to 4096 UTF-8 bytes")
|
|
86
|
+
return text, values
|
sqlodin/py.typed
ADDED
|
File without changes
|
sqlodin/results.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Owned, immutable results: iteration, named columns, and explicit cardinality."""
|
|
2
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any, overload
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class Row(Mapping[str, Any]):
|
|
9
|
+
_columns: tuple[str, ...]
|
|
10
|
+
_values: tuple[Any, ...]
|
|
11
|
+
|
|
12
|
+
def __getitem__(self, key: str | int) -> Any:
|
|
13
|
+
if isinstance(key, int):
|
|
14
|
+
return self._values[key]
|
|
15
|
+
try:
|
|
16
|
+
return self._values[self._columns.index(key)]
|
|
17
|
+
except ValueError:
|
|
18
|
+
raise KeyError(key) from None
|
|
19
|
+
|
|
20
|
+
def __iter__(self) -> Iterator[str]:
|
|
21
|
+
return iter(dict.fromkeys(self._columns))
|
|
22
|
+
|
|
23
|
+
def __len__(self) -> int:
|
|
24
|
+
return len(set(self._columns))
|
|
25
|
+
|
|
26
|
+
def as_tuple(self) -> tuple[Any, ...]:
|
|
27
|
+
return self._values
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class Rows(Sequence[Row]):
|
|
32
|
+
columns: tuple[str, ...]
|
|
33
|
+
_rows: tuple[Row, ...]
|
|
34
|
+
applied: int
|
|
35
|
+
node: int
|
|
36
|
+
|
|
37
|
+
@overload
|
|
38
|
+
def __getitem__(self, index: int) -> Row: ...
|
|
39
|
+
@overload
|
|
40
|
+
def __getitem__(self, index: slice) -> tuple[Row, ...]: ...
|
|
41
|
+
def __getitem__(self, index):
|
|
42
|
+
return self._rows[index]
|
|
43
|
+
|
|
44
|
+
def __len__(self) -> int:
|
|
45
|
+
return len(self._rows)
|
|
46
|
+
|
|
47
|
+
def first(self) -> Row | None:
|
|
48
|
+
return self._rows[0] if self._rows else None
|
|
49
|
+
|
|
50
|
+
def one(self) -> Row:
|
|
51
|
+
if len(self) != 1:
|
|
52
|
+
raise ValueError(f"Expected exactly one row, received {len(self)}")
|
|
53
|
+
return self._rows[0]
|
|
54
|
+
|
|
55
|
+
def scalar(self) -> Any:
|
|
56
|
+
row = self.one()
|
|
57
|
+
if len(self.columns) != 1:
|
|
58
|
+
raise ValueError(f"Expected one column, received {len(self.columns)}")
|
|
59
|
+
return row[0]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class WriteResult:
|
|
64
|
+
changes: int
|
|
65
|
+
applied: int
|
|
66
|
+
node: int
|
|
67
|
+
sequence: int
|
sqlodin/search.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""FTS5 and exact vector retrieval over one atomically maintained search index."""
|
|
2
|
+
import re
|
|
3
|
+
from .vector import Vector
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def identifier(name: str) -> str:
|
|
7
|
+
if not isinstance(name, str) or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]{0,63}', name):
|
|
8
|
+
raise ValueError('Index names need 1..64 ASCII letters, digits or underscores, starting with a letter')
|
|
9
|
+
if name.lower().startswith('sqlite_'):
|
|
10
|
+
raise ValueError('Reserved index name')
|
|
11
|
+
return '"' + name + '"'
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def bound(value, name, maximum=100):
|
|
15
|
+
if type(value) is not int or not 1 <= value <= maximum:
|
|
16
|
+
raise ValueError(f'{name} must be an integer in 1..{maximum}')
|
|
17
|
+
return value
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class SearchIndex:
|
|
21
|
+
def __init__(self, connection, name: str, *, dimensions: int):
|
|
22
|
+
self.connection, self.name = connection, name
|
|
23
|
+
if not isinstance(name, str) or len(name) > 60:
|
|
24
|
+
raise ValueError("Search index names must be at most 60 characters")
|
|
25
|
+
self.table, self.fts = identifier(name), identifier(name + '_fts')
|
|
26
|
+
self.dimensions = bound(dimensions, 'dimensions', 384)
|
|
27
|
+
|
|
28
|
+
def create(self):
|
|
29
|
+
"""Fail if either table exists; never silently accept a different schema."""
|
|
30
|
+
return self.connection.execute(
|
|
31
|
+
f'CREATE TABLE {self.table}(id INTEGER PRIMARY KEY,title TEXT NOT NULL,body TEXT NOT NULL,'
|
|
32
|
+
f'embedding BLOB NOT NULL CHECK(length(embedding)={self.dimensions * 4}));'
|
|
33
|
+
f'CREATE VIRTUAL TABLE {self.fts} USING fts5(title,body);')
|
|
34
|
+
|
|
35
|
+
def _vector(self, value):
|
|
36
|
+
value = value if isinstance(value, Vector) else Vector(value)
|
|
37
|
+
if len(value) != self.dimensions:
|
|
38
|
+
raise ValueError(f'Expected {self.dimensions} vector dimensions, received {len(value)}')
|
|
39
|
+
return value
|
|
40
|
+
|
|
41
|
+
def put(self, id: int, *, title: str, body: str, vector):
|
|
42
|
+
"""Atomically insert/update the content, embedding, and FTS document."""
|
|
43
|
+
if type(id) is not int or not 1 <= id < 2**63 - 1:
|
|
44
|
+
raise ValueError('Document ID must be a positive integer below SQLite maximum rowid')
|
|
45
|
+
value = self._vector(vector)
|
|
46
|
+
with self.connection.transaction() as tx:
|
|
47
|
+
tx.execute(f'INSERT INTO {self.table}(id,title,body,embedding) VALUES(?,?,?,?) '
|
|
48
|
+
'ON CONFLICT(id) DO UPDATE SET title=excluded.title,body=excluded.body,'
|
|
49
|
+
'embedding=excluded.embedding', (id, title, body, value))
|
|
50
|
+
tx.execute(f'DELETE FROM {self.fts} WHERE rowid=?', (id,))
|
|
51
|
+
tx.execute(f'INSERT INTO {self.fts}(rowid,title,body) VALUES(?,?,?)', (id, title, body))
|
|
52
|
+
return tx.result
|
|
53
|
+
|
|
54
|
+
def delete(self, id: int):
|
|
55
|
+
if type(id) is not int or not 1 <= id < 2**63 - 1:
|
|
56
|
+
raise ValueError('Invalid document ID')
|
|
57
|
+
with self.connection.transaction() as tx:
|
|
58
|
+
tx.execute(f'DELETE FROM {self.fts} WHERE rowid=?', (id,))
|
|
59
|
+
tx.execute(f'DELETE FROM {self.table} WHERE id=?', (id,))
|
|
60
|
+
return tx.result
|
|
61
|
+
|
|
62
|
+
def full_text(self, text: str, *, limit: int = 20):
|
|
63
|
+
"""FTS5 query syntax; lower BM25 score is better. SQL values stay bound."""
|
|
64
|
+
bound(limit, 'limit')
|
|
65
|
+
return self.connection.query(
|
|
66
|
+
f'SELECT d.id,d.title,d.body,bm25({self.fts}) AS score '
|
|
67
|
+
f'FROM {self.fts} JOIN {self.table} AS d ON d.id={self.fts}.rowid '
|
|
68
|
+
f'WHERE {self.fts} MATCH ? ORDER BY score,d.id LIMIT ?', (text, limit))
|
|
69
|
+
|
|
70
|
+
def nearest(self, vector, *, limit: int = 20, metric: str = 'l2'):
|
|
71
|
+
"""Exact distance scan, not an ANN index. Lower distance is better."""
|
|
72
|
+
function = self._metric(metric)
|
|
73
|
+
value = self._vector(vector)
|
|
74
|
+
if metric == 'cosine' and not any(value):
|
|
75
|
+
raise ValueError('Cosine distance requires a nonzero query vector')
|
|
76
|
+
bound(limit, 'limit')
|
|
77
|
+
return self.connection.query(
|
|
78
|
+
f'SELECT id,title,body,{function}(embedding,?) AS distance FROM {self.table} '
|
|
79
|
+
'ORDER BY distance,id LIMIT ?', (value, limit))
|
|
80
|
+
|
|
81
|
+
@staticmethod
|
|
82
|
+
def _metric(metric):
|
|
83
|
+
if metric not in ('l2', 'cosine'):
|
|
84
|
+
raise ValueError("Metric must be 'l2' or 'cosine'")
|
|
85
|
+
return 'vec_distance_' + metric
|
|
86
|
+
|
|
87
|
+
def hybrid(self, text: str, vector, *, limit: int = 20, candidates: int = 50,
|
|
88
|
+
rank_constant: int = 60, metric: str = 'l2'):
|
|
89
|
+
"""Reciprocal-rank fusion of FTS and vector candidates in one fenced snapshot."""
|
|
90
|
+
bound(limit, 'limit')
|
|
91
|
+
bound(candidates, 'candidates')
|
|
92
|
+
bound(rank_constant, 'rank_constant', 1000)
|
|
93
|
+
if candidates < limit:
|
|
94
|
+
raise ValueError('Candidates must be at least limit')
|
|
95
|
+
function = self._metric(metric)
|
|
96
|
+
value = self._vector(vector)
|
|
97
|
+
if metric == 'cosine' and not any(value):
|
|
98
|
+
raise ValueError('Cosine distance requires a nonzero query vector')
|
|
99
|
+
sql = f'''WITH
|
|
100
|
+
ft AS MATERIALIZED (SELECT rowid AS id,bm25({self.fts}) AS score FROM {self.fts}
|
|
101
|
+
WHERE {self.fts} MATCH ? ORDER BY score,id LIMIT ?),
|
|
102
|
+
vt AS MATERIALIZED (SELECT id,{function}(embedding,?) AS distance FROM {self.table}
|
|
103
|
+
ORDER BY distance,id LIMIT ?),
|
|
104
|
+
ranks AS (SELECT id,ROW_NUMBER() OVER(ORDER BY score,id) AS rank FROM ft
|
|
105
|
+
UNION ALL SELECT id,ROW_NUMBER() OVER(ORDER BY distance,id) AS rank FROM vt),
|
|
106
|
+
fused AS (SELECT id,SUM(1.0 / (? + rank)) AS score FROM ranks GROUP BY id)
|
|
107
|
+
SELECT d.id,d.title,d.body,f.score FROM fused AS f JOIN {self.table} AS d ON d.id=f.id
|
|
108
|
+
ORDER BY f.score DESC,d.id LIMIT ?'''
|
|
109
|
+
return self.connection.query(sql, (text, candidates, value, candidates,
|
|
110
|
+
rank_constant, limit))
|
sqlodin/sqlalchemy.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""SQLAlchemy 2.0 dialect with optimistic serializable ORM transactions."""
|
|
2
|
+
from sqlalchemy import create_engine as sa_create_engine
|
|
3
|
+
from sqlalchemy.dialects import registry
|
|
4
|
+
from sqlalchemy.dialects.sqlite.base import SQLiteDialect
|
|
5
|
+
from sqlalchemy.types import UserDefinedType
|
|
6
|
+
|
|
7
|
+
from . import dbapi
|
|
8
|
+
from .vector import Vector
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class SQLodinDialect(SQLiteDialect):
|
|
12
|
+
name = 'sqlodin'
|
|
13
|
+
driver = 'native'
|
|
14
|
+
supports_statement_cache = True
|
|
15
|
+
insert_returning = update_returning = delete_returning = False
|
|
16
|
+
use_insertmanyvalues = False
|
|
17
|
+
postfetch_lastrowid = True
|
|
18
|
+
# Preview returns sqlite3_changes for the outer statement, excluding triggers.
|
|
19
|
+
supports_sane_rowcount = True
|
|
20
|
+
supports_sane_multi_rowcount = True
|
|
21
|
+
supports_default_values = True
|
|
22
|
+
|
|
23
|
+
def __init__(self, **kwargs):
|
|
24
|
+
# SQLite's constructor assumes a local sqlite3 module. This remote driver
|
|
25
|
+
# obtains the actual server version after connecting instead.
|
|
26
|
+
module = kwargs.pop('dbapi', None)
|
|
27
|
+
super().__init__(dbapi=None, **kwargs)
|
|
28
|
+
self.dbapi = module
|
|
29
|
+
|
|
30
|
+
@classmethod
|
|
31
|
+
def import_dbapi(cls): return dbapi
|
|
32
|
+
|
|
33
|
+
def create_connect_args(self, url):
|
|
34
|
+
if url.host or url.database or url.username or url.password or url.query:
|
|
35
|
+
raise ValueError('Use sqlodin:// with explicit connect_args for endpoints, cluster and TLS')
|
|
36
|
+
return [], {}
|
|
37
|
+
|
|
38
|
+
def _get_server_version_info(self, connection):
|
|
39
|
+
value = connection.exec_driver_sql('SELECT sqlite_version()').scalar()
|
|
40
|
+
return tuple(map(int, value.split('.')))
|
|
41
|
+
|
|
42
|
+
def _get_default_schema_name(self, connection): return 'main'
|
|
43
|
+
def get_default_isolation_level(self, connection): return 'SERIALIZABLE'
|
|
44
|
+
def get_isolation_level(self, connection): return 'AUTOCOMMIT' if connection.autocommit else 'SERIALIZABLE'
|
|
45
|
+
def get_isolation_level_values(self, connection): return ['SERIALIZABLE', 'AUTOCOMMIT']
|
|
46
|
+
def detect_autocommit_setting(self, connection): return connection.autocommit
|
|
47
|
+
|
|
48
|
+
def set_isolation_level(self, connection, level):
|
|
49
|
+
if level not in ('SERIALIZABLE', 'AUTOCOMMIT'):
|
|
50
|
+
raise dbapi.NotSupportedError('Supported isolation levels: SERIALIZABLE and AUTOCOMMIT')
|
|
51
|
+
connection.autocommit = level == 'AUTOCOMMIT'
|
|
52
|
+
|
|
53
|
+
def do_savepoint(self, connection, name):
|
|
54
|
+
connection.connection.dbapi_connection.savepoint(name)
|
|
55
|
+
|
|
56
|
+
def do_rollback_to_savepoint(self, connection, name):
|
|
57
|
+
connection.connection.dbapi_connection.rollback_savepoint(name)
|
|
58
|
+
|
|
59
|
+
def do_release_savepoint(self, connection, name):
|
|
60
|
+
connection.connection.dbapi_connection.release_savepoint(name)
|
|
61
|
+
|
|
62
|
+
def do_begin_twophase(self, connection, xid):
|
|
63
|
+
raise dbapi.NotSupportedError('Two-phase transactions are unsupported')
|
|
64
|
+
|
|
65
|
+
def do_execute(self, cursor, statement, parameters, context=None):
|
|
66
|
+
cursor.execute(statement, parameters)
|
|
67
|
+
|
|
68
|
+
def do_executemany(self, cursor, statement, parameters, context=None):
|
|
69
|
+
cursor.executemany(statement, parameters)
|
|
70
|
+
|
|
71
|
+
def has_table(self, connection, table_name, schema=None, **kw):
|
|
72
|
+
if schema not in (None, 'main'): return False
|
|
73
|
+
return bool(connection.exec_driver_sql(
|
|
74
|
+
"SELECT 1 FROM sqlite_schema WHERE name=? AND type IN ('table','view')", (table_name,)).first())
|
|
75
|
+
|
|
76
|
+
def get_table_names(self, connection, schema=None, **kw):
|
|
77
|
+
if schema not in (None, 'main'): return []
|
|
78
|
+
return [r[0] for r in connection.exec_driver_sql(
|
|
79
|
+
"SELECT name FROM sqlite_schema WHERE type='table' "
|
|
80
|
+
"AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '_sqlodin_%' ORDER BY name")]
|
|
81
|
+
|
|
82
|
+
def get_columns(self, *args, **kw):
|
|
83
|
+
raise NotImplementedError('Schema reflection is not supported; declare SQLAlchemy tables explicitly')
|
|
84
|
+
|
|
85
|
+
def get_foreign_keys(self, *args, **kw):
|
|
86
|
+
raise NotImplementedError('Foreign-key reflection is not supported')
|
|
87
|
+
|
|
88
|
+
def get_indexes(self, *args, **kw):
|
|
89
|
+
raise NotImplementedError('Index reflection is not supported')
|
|
90
|
+
|
|
91
|
+
def get_pk_constraint(self, *args, **kw):
|
|
92
|
+
raise NotImplementedError('Primary-key reflection is not supported')
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class VectorType(UserDefinedType):
|
|
96
|
+
"""SQLite BLOB storage with typed network float32 bindings and Vector results."""
|
|
97
|
+
cache_ok = True
|
|
98
|
+
|
|
99
|
+
def __init__(self, dimensions: int):
|
|
100
|
+
if type(dimensions) is not int or not 1 <= dimensions <= 384:
|
|
101
|
+
raise ValueError('Vector dimensions must be in 1..384')
|
|
102
|
+
self.dimensions = dimensions
|
|
103
|
+
|
|
104
|
+
def get_col_spec(self, **kw): return 'BLOB'
|
|
105
|
+
|
|
106
|
+
def bind_processor(self, dialect):
|
|
107
|
+
def process(value):
|
|
108
|
+
if value is None: return None
|
|
109
|
+
value = value if isinstance(value, Vector) else Vector(value)
|
|
110
|
+
if len(value) != self.dimensions: raise ValueError('Vector dimension mismatch')
|
|
111
|
+
return value
|
|
112
|
+
return process
|
|
113
|
+
|
|
114
|
+
def result_processor(self, dialect, coltype):
|
|
115
|
+
def process(value):
|
|
116
|
+
if value is None: return None
|
|
117
|
+
vector = Vector.from_bytes(value)
|
|
118
|
+
if len(vector) != self.dimensions: raise ValueError('Stored vector dimension mismatch')
|
|
119
|
+
return vector
|
|
120
|
+
return process
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
registry.register('sqlodin', 'sqlodin.sqlalchemy', 'SQLodinDialect')
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def create_engine(endpoints, *, cluster, tls, autocommit=False, timeout=10, **options):
|
|
127
|
+
"""Create an ORM/Core engine with SERIALIZABLE transactions by default.
|
|
128
|
+
|
|
129
|
+
Writes are provisional until commit. Concurrent application writes cause a
|
|
130
|
+
serialization error; roll back and retry the entire transaction. Explicit
|
|
131
|
+
autocommit=True opts into independently committed statements.
|
|
132
|
+
"""
|
|
133
|
+
if type(autocommit) is not bool: raise ValueError('autocommit must be bool')
|
|
134
|
+
if 'connect_args' in options or 'isolation_level' in options:
|
|
135
|
+
raise ValueError('Connection arguments and isolation are set by this helper')
|
|
136
|
+
return sa_create_engine('sqlodin://', isolation_level='AUTOCOMMIT' if autocommit else 'SERIALIZABLE',
|
|
137
|
+
connect_args=dict(endpoints=endpoints, cluster=cluster, tls=tls,
|
|
138
|
+
timeout=timeout, autocommit=autocommit), **options)
|
sqlodin/transaction.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Atomic buffered transaction bodies, without misleading live SQL transactions."""
|
|
2
|
+
from .errors import PendingWriteError
|
|
3
|
+
from .parameters import prepare
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Transaction:
|
|
7
|
+
def __init__(self, connection):
|
|
8
|
+
self.connection = connection
|
|
9
|
+
self.result = None
|
|
10
|
+
self._statements = []
|
|
11
|
+
self._parameters = []
|
|
12
|
+
self._active = False
|
|
13
|
+
self._used = False
|
|
14
|
+
|
|
15
|
+
def __enter__(self):
|
|
16
|
+
db = self.connection
|
|
17
|
+
db._lock.acquire()
|
|
18
|
+
try:
|
|
19
|
+
db._ready()
|
|
20
|
+
if self._used or db.pending is not None:
|
|
21
|
+
raise PendingWriteError("Transaction already used or a write needs resolution")
|
|
22
|
+
self._active = self._used = db._transaction = True
|
|
23
|
+
return self
|
|
24
|
+
except BaseException:
|
|
25
|
+
db._lock.release()
|
|
26
|
+
raise
|
|
27
|
+
|
|
28
|
+
def execute(self, sql, parameters=()):
|
|
29
|
+
if not self._active:
|
|
30
|
+
raise PendingWriteError("Use transaction.execute() inside its with block")
|
|
31
|
+
if len(self._statements) >= 8:
|
|
32
|
+
raise ValueError("A transaction supports at most eight statements")
|
|
33
|
+
text, values = prepare(sql, parameters, offset=len(self._parameters))
|
|
34
|
+
# A newline terminates any trailing -- comment before our separator.
|
|
35
|
+
body = '\n;\n'.join([*self._statements, text + '\n'])
|
|
36
|
+
if len(body.encode('utf-8')) > 4096:
|
|
37
|
+
raise ValueError("A transaction is limited to 4096 UTF-8 bytes")
|
|
38
|
+
self._statements.append(text + '\n')
|
|
39
|
+
self._parameters.extend(values)
|
|
40
|
+
return self
|
|
41
|
+
|
|
42
|
+
def __exit__(self, exc_type, exc, tb):
|
|
43
|
+
db = self.connection
|
|
44
|
+
try:
|
|
45
|
+
self._active = db._transaction = False
|
|
46
|
+
if exc_type is None and self._statements:
|
|
47
|
+
self.result = db._execute_prepared('\n;\n'.join(self._statements), tuple(self._parameters))
|
|
48
|
+
finally:
|
|
49
|
+
db._lock.release()
|
sqlodin/transport.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""TLS 1.3 framed transport with a single absolute deadline per operation."""
|
|
2
|
+
import json
|
|
3
|
+
import socket
|
|
4
|
+
import ssl
|
|
5
|
+
import struct
|
|
6
|
+
import time
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from .errors import ConnectionError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class Endpoint:
|
|
15
|
+
address: str
|
|
16
|
+
server_name: str
|
|
17
|
+
|
|
18
|
+
def socket_address(self) -> tuple[str, int]:
|
|
19
|
+
host, port = self.address.rsplit(':', 1)
|
|
20
|
+
return host, int(port)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class TLS:
|
|
25
|
+
ca: str | Path
|
|
26
|
+
cert: str | Path
|
|
27
|
+
key: str | Path
|
|
28
|
+
|
|
29
|
+
def context(self) -> ssl.SSLContext:
|
|
30
|
+
ctx = ssl.create_default_context(cafile=str(self.ca))
|
|
31
|
+
ctx.minimum_version = ssl.TLSVersion.TLSv1_3
|
|
32
|
+
ctx.maximum_version = ssl.TLSVersion.TLSv1_3
|
|
33
|
+
ctx.hostname_checks_common_name = False
|
|
34
|
+
ctx.load_cert_chain(str(self.cert), str(self.key))
|
|
35
|
+
return ctx
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def remaining(deadline: float) -> float:
|
|
39
|
+
seconds = deadline - time.monotonic()
|
|
40
|
+
if seconds <= 0:
|
|
41
|
+
raise TimeoutError("SQLodin operation deadline expired")
|
|
42
|
+
return seconds
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Transport:
|
|
46
|
+
def __init__(self, tls: TLS):
|
|
47
|
+
self.context = tls.context()
|
|
48
|
+
self.socket = None
|
|
49
|
+
self.endpoint = None
|
|
50
|
+
|
|
51
|
+
def close(self):
|
|
52
|
+
if self.socket is not None:
|
|
53
|
+
self.socket.close()
|
|
54
|
+
self.socket = self.endpoint = None
|
|
55
|
+
|
|
56
|
+
def exchange(self, endpoint: Endpoint, request: dict, deadline: float) -> dict:
|
|
57
|
+
body = json.dumps(request, separators=(',', ':'), allow_nan=False).encode('utf-8')
|
|
58
|
+
if len(body) > 65536:
|
|
59
|
+
raise ValueError("Encoded request exceeds 64 KiB")
|
|
60
|
+
try:
|
|
61
|
+
if self.endpoint != endpoint or self.socket is None:
|
|
62
|
+
self.close()
|
|
63
|
+
raw = socket.create_connection(endpoint.socket_address(), timeout=remaining(deadline))
|
|
64
|
+
try:
|
|
65
|
+
raw.settimeout(remaining(deadline))
|
|
66
|
+
raw.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
|
67
|
+
self.socket = self.context.wrap_socket(raw, server_hostname=endpoint.server_name)
|
|
68
|
+
sans = self.socket.getpeercert().get('subjectAltName', ())
|
|
69
|
+
if ('DNS', endpoint.server_name) not in sans:
|
|
70
|
+
raise ssl.CertificateError("Server certificate requires an exact DNS SAN")
|
|
71
|
+
self.endpoint = endpoint
|
|
72
|
+
except BaseException:
|
|
73
|
+
raw.close()
|
|
74
|
+
self.close()
|
|
75
|
+
raise
|
|
76
|
+
self.socket.settimeout(remaining(deadline))
|
|
77
|
+
self.socket.sendall(struct.pack('<I', len(body)) + body)
|
|
78
|
+
size = struct.unpack('<I', self._receive(4, deadline))[0]
|
|
79
|
+
if not 0 < size <= 1024 * 1024:
|
|
80
|
+
raise ConnectionError("Invalid response frame length")
|
|
81
|
+
result = json.loads(self._receive(size, deadline))
|
|
82
|
+
if not isinstance(result, dict):
|
|
83
|
+
raise ConnectionError("Invalid response object")
|
|
84
|
+
return result
|
|
85
|
+
except (OSError, ValueError, ConnectionError) as exc:
|
|
86
|
+
self.close()
|
|
87
|
+
raise ConnectionError(str(exc)) from exc
|
|
88
|
+
|
|
89
|
+
def _receive(self, size, deadline):
|
|
90
|
+
data = bytearray()
|
|
91
|
+
while len(data) < size:
|
|
92
|
+
self.socket.settimeout(remaining(deadline))
|
|
93
|
+
chunk = self.socket.recv(size - len(data))
|
|
94
|
+
if not chunk:
|
|
95
|
+
raise ConnectionError("Server closed the connection")
|
|
96
|
+
data.extend(chunk)
|
|
97
|
+
return data
|
sqlodin/vector.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Immutable, finite float32 vectors with explicit dimensional bounds."""
|
|
2
|
+
import math
|
|
3
|
+
import struct
|
|
4
|
+
from collections.abc import Iterable, Iterator, Sequence
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True, init=False)
|
|
9
|
+
class Vector(Sequence[float]):
|
|
10
|
+
values: tuple[float, ...]
|
|
11
|
+
|
|
12
|
+
def __init__(self, values: Iterable[float]):
|
|
13
|
+
if isinstance(values, (str, bytes, bytearray)):
|
|
14
|
+
raise TypeError('Vector expects numeric components; use Vector.from_bytes for a BLOB')
|
|
15
|
+
normalized = []
|
|
16
|
+
for value in values:
|
|
17
|
+
if len(normalized) == 384:
|
|
18
|
+
raise ValueError('A vector supports at most 384 dimensions')
|
|
19
|
+
try:
|
|
20
|
+
number = float(value)
|
|
21
|
+
number = struct.unpack('<f', struct.pack('<f', number))[0]
|
|
22
|
+
except (TypeError, ValueError, OverflowError, struct.error) as exc:
|
|
23
|
+
raise ValueError('Vector components must fit finite float32') from exc
|
|
24
|
+
if not math.isfinite(number):
|
|
25
|
+
raise ValueError('Vector components must be finite')
|
|
26
|
+
normalized.append(number)
|
|
27
|
+
if not normalized:
|
|
28
|
+
raise ValueError('A vector must have at least one dimension')
|
|
29
|
+
object.__setattr__(self, 'values', tuple(normalized))
|
|
30
|
+
|
|
31
|
+
def __len__(self) -> int:
|
|
32
|
+
return len(self.values)
|
|
33
|
+
|
|
34
|
+
def __getitem__(self, index):
|
|
35
|
+
return self.values[index]
|
|
36
|
+
|
|
37
|
+
def __iter__(self) -> Iterator[float]:
|
|
38
|
+
return iter(self.values)
|
|
39
|
+
|
|
40
|
+
def to_bytes(self) -> bytes:
|
|
41
|
+
return struct.pack(f'<{len(self)}f', *self.values)
|
|
42
|
+
|
|
43
|
+
@classmethod
|
|
44
|
+
def from_bytes(cls, value: bytes) -> 'Vector':
|
|
45
|
+
if not value or len(value) % 4 or len(value) > 384 * 4:
|
|
46
|
+
raise ValueError('A float32 vector BLOB needs 1..384 four-byte components')
|
|
47
|
+
return cls(struct.unpack(f'<{len(value) // 4}f', value))
|