boti-data 1.2.0__tar.gz → 1.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {boti_data-1.2.0 → boti_data-1.3.1}/PKG-INFO +39 -1
- {boti_data-1.2.0 → boti_data-1.3.1}/README.md +38 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/pyproject.toml +1 -1
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/__init__.py +4 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/__init__.py +8 -1
- boti_data-1.3.1/src/boti_data/db/sql_repository.py +361 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data.egg-info/PKG-INFO +39 -1
- boti_data-1.2.0/src/boti_data/db/sql_repository.py +0 -186
- {boti_data-1.2.0 → boti_data-1.3.1}/setup.cfg +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/arrow_schema.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/connection_catalog.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/datacube/__init__.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/datacube/artifact.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/datacube/base.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/datacube/contract.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/datacube/resource.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/dataset/__init__.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/dataset/hybrid.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/dataset/hybrid_support.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/arrow_schema_mapper.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/engine_registry.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/partitioned_execution.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/partitioned_loader.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/partitioned_planner.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/partitioned_planner_async.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/partitioned_planning_math.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/partitioned_row_materialization.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/partitioned_statements.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/partitioned_types.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/sql_config.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/sql_engine.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/sql_manager.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/sql_model_builder.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/sql_model_module_injection.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/sql_model_registry.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/sql_model_registry_types.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/sql_readonly.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/sql_resource.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/sqlalchemy_async.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/db/vector_search.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/enrichment/__init__.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/enrichment/async_enricher.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/enrichment/composite.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/enrichment/protocol.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/enrichment/specs.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/enrichment/transformers.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/field_map.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/filters/__init__.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/filters/arrow_kernels.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/filters/arrow_kernels_comparison.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/filters/arrow_kernels_compile.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/filters/arrow_kernels_string.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/filters/backend_ops.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/filters/expressions.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/filters/handler.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/filters/parquet_compiler.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/filters/utils.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/filters/value_parsing.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/__init__.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/_backend_strategies.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/_factory.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/_gateway_init.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/_payloads.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/_series_filters.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/arrow_adapters.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/build_context.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/chunking.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/configured_load.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/core.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/core_load.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/datacube_strategy.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/frame_convert.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/frame_strategies.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/incremental.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/load_execution.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/load_request.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/loaders.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/normalization.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/parquet_strategy.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/planning.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/policies.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/post_process.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/requests.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/return_type.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/select_cache.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/semi_join.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/sql_guard.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/sql_partitioned_exec.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/sql_size_estimation.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/sql_strategy.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/gateway/sql_strategy_configured_async.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/helper.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/joins.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/parquet/__init__.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/parquet/config.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/parquet/discovery.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/parquet/file_info.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/parquet/filesystem.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/parquet/reader.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/parquet/reader_config.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/parquet/resource.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/parquet/schema_filters.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/pipelines/__init__.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/pipelines/base.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/pipelines/parquet_pipeline.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/pipelines/registry.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/pipelines/sinks.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/pipelines/sinks_common.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/pipelines/sinks_parquet.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/pipelines/sinks_text.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/py.typed +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/schema.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/watermark/__init__.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/watermark/incremental.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data/watermark/store.py +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data.egg-info/SOURCES.txt +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data.egg-info/dependency_links.txt +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data.egg-info/requires.txt +0 -0
- {boti_data-1.2.0 → boti_data-1.3.1}/src/boti_data.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: boti-data
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.1
|
|
4
4
|
Summary: Data infrastructure for the Boti ecosystem
|
|
5
5
|
Author: Luis Valverde
|
|
6
6
|
Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>
|
|
@@ -920,6 +920,44 @@ async with AsyncSqlRepository(config, "users", query_only=False) as users:
|
|
|
920
920
|
row = await users.get(1)
|
|
921
921
|
```
|
|
922
922
|
|
|
923
|
+
#### Multi-table atomic transactions
|
|
924
|
+
|
|
925
|
+
By default each `SqlRepository` owns its own connection and commits after every write — fine for independent single-row operations, but not when several writes (potentially across different tables) need to succeed or fail together. Pass an already-open `session` instead of `config` to bind a repository to a transaction you control; writes `flush()` instead of committing, so nothing is durable until you commit the session yourself:
|
|
926
|
+
|
|
927
|
+
```python
|
|
928
|
+
with SqlDatabaseResource(config) as db, db.session() as session:
|
|
929
|
+
with SqlRepository(session=session, table_name="audit_log") as audit_log:
|
|
930
|
+
with SqlRepository(session=session, table_name="embeddings") as embeddings:
|
|
931
|
+
audit_log.insert({"event": "enroll"})
|
|
932
|
+
embeddings.insert({"subject_id": 1, "vector": [...]})
|
|
933
|
+
session.commit() # both rows commit together, or neither does on an exception
|
|
934
|
+
```
|
|
935
|
+
|
|
936
|
+
`SqlUnitOfWork` wraps this pattern for the common case — one config, several tables, one commit/rollback:
|
|
937
|
+
|
|
938
|
+
```python
|
|
939
|
+
from boti_data import SqlDatabaseConfig, SqlUnitOfWork
|
|
940
|
+
|
|
941
|
+
config = SqlDatabaseConfig(connection_url="postgresql+psycopg://user:pass@host/mydb")
|
|
942
|
+
|
|
943
|
+
with SqlUnitOfWork(config, query_only=False) as uow:
|
|
944
|
+
uow.repository("audit_log").insert({"event": "enroll"})
|
|
945
|
+
uow.repository("embeddings").insert({"subject_id": 1, "vector": [...]})
|
|
946
|
+
# commits both on clean exit; rolls back both on any exception
|
|
947
|
+
```
|
|
948
|
+
|
|
949
|
+
`query_only` on a `session`-bound `SqlRepository` is inert — the session's own class (e.g. a `ReadOnlySession`) already determines writability, since this path never builds a new guarded session itself. `AsyncSqlUnitOfWork` mirrors `SqlUnitOfWork`, with an async `repository()` method (returns an already-set-up `AsyncSqlRepository`, so callers don't need to `async with` each one individually):
|
|
950
|
+
|
|
951
|
+
```python
|
|
952
|
+
from boti_data import AsyncSqlUnitOfWork
|
|
953
|
+
|
|
954
|
+
async with AsyncSqlUnitOfWork(config, query_only=False) as uow:
|
|
955
|
+
audit_log = await uow.repository("audit_log")
|
|
956
|
+
embeddings = await uow.repository("embeddings")
|
|
957
|
+
await audit_log.insert({"event": "enroll"})
|
|
958
|
+
await embeddings.insert({"subject_id": 1, "vector": [...]})
|
|
959
|
+
```
|
|
960
|
+
|
|
923
961
|
### Nearest-neighbour search (pgvector)
|
|
924
962
|
|
|
925
963
|
`boti_data.db.nearest_neighbors`/`vector_distance` build an `ORDER BY <distance> LIMIT k` statement against a pgvector column, with the query vector bound as a driver parameter — never string-interpolated into the SQL, regardless of how large the vector is. Requires `boti-data[pgvector]` (see [Installation](#installation)) and a table reflected via `SqlAlchemyModelBuilder`/`SqlRepository`/`DataGateway`.
|
|
@@ -885,6 +885,44 @@ async with AsyncSqlRepository(config, "users", query_only=False) as users:
|
|
|
885
885
|
row = await users.get(1)
|
|
886
886
|
```
|
|
887
887
|
|
|
888
|
+
#### Multi-table atomic transactions
|
|
889
|
+
|
|
890
|
+
By default each `SqlRepository` owns its own connection and commits after every write — fine for independent single-row operations, but not when several writes (potentially across different tables) need to succeed or fail together. Pass an already-open `session` instead of `config` to bind a repository to a transaction you control; writes `flush()` instead of committing, so nothing is durable until you commit the session yourself:
|
|
891
|
+
|
|
892
|
+
```python
|
|
893
|
+
with SqlDatabaseResource(config) as db, db.session() as session:
|
|
894
|
+
with SqlRepository(session=session, table_name="audit_log") as audit_log:
|
|
895
|
+
with SqlRepository(session=session, table_name="embeddings") as embeddings:
|
|
896
|
+
audit_log.insert({"event": "enroll"})
|
|
897
|
+
embeddings.insert({"subject_id": 1, "vector": [...]})
|
|
898
|
+
session.commit() # both rows commit together, or neither does on an exception
|
|
899
|
+
```
|
|
900
|
+
|
|
901
|
+
`SqlUnitOfWork` wraps this pattern for the common case — one config, several tables, one commit/rollback:
|
|
902
|
+
|
|
903
|
+
```python
|
|
904
|
+
from boti_data import SqlDatabaseConfig, SqlUnitOfWork
|
|
905
|
+
|
|
906
|
+
config = SqlDatabaseConfig(connection_url="postgresql+psycopg://user:pass@host/mydb")
|
|
907
|
+
|
|
908
|
+
with SqlUnitOfWork(config, query_only=False) as uow:
|
|
909
|
+
uow.repository("audit_log").insert({"event": "enroll"})
|
|
910
|
+
uow.repository("embeddings").insert({"subject_id": 1, "vector": [...]})
|
|
911
|
+
# commits both on clean exit; rolls back both on any exception
|
|
912
|
+
```
|
|
913
|
+
|
|
914
|
+
`query_only` on a `session`-bound `SqlRepository` is inert — the session's own class (e.g. a `ReadOnlySession`) already determines writability, since this path never builds a new guarded session itself. `AsyncSqlUnitOfWork` mirrors `SqlUnitOfWork`, with an async `repository()` method (returns an already-set-up `AsyncSqlRepository`, so callers don't need to `async with` each one individually):
|
|
915
|
+
|
|
916
|
+
```python
|
|
917
|
+
from boti_data import AsyncSqlUnitOfWork
|
|
918
|
+
|
|
919
|
+
async with AsyncSqlUnitOfWork(config, query_only=False) as uow:
|
|
920
|
+
audit_log = await uow.repository("audit_log")
|
|
921
|
+
embeddings = await uow.repository("embeddings")
|
|
922
|
+
await audit_log.insert({"event": "enroll"})
|
|
923
|
+
await embeddings.insert({"subject_id": 1, "vector": [...]})
|
|
924
|
+
```
|
|
925
|
+
|
|
888
926
|
### Nearest-neighbour search (pgvector)
|
|
889
927
|
|
|
890
928
|
`boti_data.db.nearest_neighbors`/`vector_distance` build an `ORDER BY <distance> LIMIT k` statement against a pgvector column, with the query vector bound as a driver parameter — never string-interpolated into the SQL, regardless of how large the vector is. Requires `boti-data[pgvector]` (see [Installation](#installation)) and a table reflected via `SqlAlchemyModelBuilder`/`SqlRepository`/`DataGateway`.
|
|
@@ -12,6 +12,7 @@ __all__ = [
|
|
|
12
12
|
"AsyncFrameEnricher",
|
|
13
13
|
"AsyncSqlDatabaseResource",
|
|
14
14
|
"AsyncSqlRepository",
|
|
15
|
+
"AsyncSqlUnitOfWork",
|
|
15
16
|
"AttachmentSpec",
|
|
16
17
|
"BaseDataCube",
|
|
17
18
|
"BaseArtifact",
|
|
@@ -61,6 +62,7 @@ __all__ = [
|
|
|
61
62
|
"SqlPartitionedLoader",
|
|
62
63
|
"SqlModelRegistry",
|
|
63
64
|
"SqlRepository",
|
|
65
|
+
"SqlUnitOfWork",
|
|
64
66
|
"TrueExpr",
|
|
65
67
|
"VectorMetric",
|
|
66
68
|
"WatermarkStore",
|
|
@@ -105,6 +107,8 @@ _LAZY = {
|
|
|
105
107
|
"SqlPartitionSpec": ("boti_data.db.partitioned_types", "SqlPartitionSpec"),
|
|
106
108
|
"SqlRepository": ("boti_data.db.sql_repository", "SqlRepository"),
|
|
107
109
|
"AsyncSqlRepository": ("boti_data.db.sql_repository", "AsyncSqlRepository"),
|
|
110
|
+
"SqlUnitOfWork": ("boti_data.db.sql_repository", "SqlUnitOfWork"),
|
|
111
|
+
"AsyncSqlUnitOfWork": ("boti_data.db.sql_repository", "AsyncSqlUnitOfWork"),
|
|
108
112
|
"nearest_neighbors": ("boti_data.db.vector_search", "nearest_neighbors"),
|
|
109
113
|
"vector_distance": ("boti_data.db.vector_search", "vector_distance"),
|
|
110
114
|
"VectorMetric": ("boti_data.db.vector_search", "VectorMetric"),
|
|
@@ -21,13 +21,19 @@ from boti_data.db.sql_model_registry import (
|
|
|
21
21
|
SqlModelRegistry,
|
|
22
22
|
get_global_registry,
|
|
23
23
|
)
|
|
24
|
-
from boti_data.db.sql_repository import
|
|
24
|
+
from boti_data.db.sql_repository import (
|
|
25
|
+
AsyncSqlRepository,
|
|
26
|
+
AsyncSqlUnitOfWork,
|
|
27
|
+
SqlRepository,
|
|
28
|
+
SqlUnitOfWork,
|
|
29
|
+
)
|
|
25
30
|
from boti_data.db.sqlalchemy_async import ensure_greenlet_available
|
|
26
31
|
from boti_data.db.vector_search import nearest_neighbors, vector_distance
|
|
27
32
|
|
|
28
33
|
__all__ = [
|
|
29
34
|
"AsyncSqlDatabaseResource",
|
|
30
35
|
"AsyncSqlRepository",
|
|
36
|
+
"AsyncSqlUnitOfWork",
|
|
31
37
|
"BuilderConfig",
|
|
32
38
|
"DefaultBase",
|
|
33
39
|
"EngineRegistry",
|
|
@@ -41,6 +47,7 @@ __all__ = [
|
|
|
41
47
|
"SqlPartitionedLoader",
|
|
42
48
|
"SqlModelRegistry",
|
|
43
49
|
"SqlRepository",
|
|
50
|
+
"SqlUnitOfWork",
|
|
44
51
|
"ensure_greenlet_available",
|
|
45
52
|
"get_global_registry",
|
|
46
53
|
"nearest_neighbors",
|
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Lightweight single-row transactional access (get/insert/update/delete by
|
|
3
|
+
primary key), returning plain dicts instead of a DataFrame — plus
|
|
4
|
+
SqlUnitOfWork/AsyncSqlUnitOfWork for composing writes across several tables
|
|
5
|
+
into one atomic transaction.
|
|
6
|
+
|
|
7
|
+
boti_data's flagship API (DataGateway/DataHelper) is dataframe-shaped and
|
|
8
|
+
bulk-oriented, pulling in the dask/pandas/polars dependency chain even for a
|
|
9
|
+
consumer that only ever touches one row at a time (e.g. an OLTP request
|
|
10
|
+
handler at p95 <300ms). SqlRepository/AsyncSqlRepository sit directly on
|
|
11
|
+
SqlDatabaseResource/AsyncSqlDatabaseResource + SqlModelRegistry — both already
|
|
12
|
+
free of that dependency chain — so this module can be imported and used
|
|
13
|
+
without dask/pandas/polars installed at all.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from contextlib import asynccontextmanager, contextmanager
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from boti.core.lifecycle import LifecycleCore
|
|
22
|
+
from boti.core.logger import Logger
|
|
23
|
+
from sqlalchemy import inspect as sa_inspect
|
|
24
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
25
|
+
from sqlalchemy.orm import Session
|
|
26
|
+
|
|
27
|
+
from boti_data.db.sql_config import SqlDatabaseConfig
|
|
28
|
+
from boti_data.db.sql_model_registry import get_global_registry
|
|
29
|
+
from boti_data.db.sql_readonly import ReadOnlyAsyncSession, ReadOnlySession
|
|
30
|
+
from boti_data.db.sql_resource import AsyncSqlDatabaseResource, SqlDatabaseResource
|
|
31
|
+
|
|
32
|
+
__all__ = ["AsyncSqlRepository", "AsyncSqlUnitOfWork", "SqlRepository", "SqlUnitOfWork"]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _row_to_dict(instance: Any) -> dict[str, Any]:
|
|
36
|
+
mapper = sa_inspect(instance).mapper
|
|
37
|
+
return {attr.key: getattr(instance, attr.key) for attr in mapper.column_attrs}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _require_table_name(table_name: str | None) -> str:
|
|
41
|
+
if table_name is None:
|
|
42
|
+
raise ValueError("table_name is required.")
|
|
43
|
+
return table_name
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _require_exactly_one_source(config: SqlDatabaseConfig | None, session: Any | None) -> None:
|
|
47
|
+
if (config is None) == (session is None):
|
|
48
|
+
raise ValueError("Provide exactly one of 'config' or 'session'.")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class SqlRepository(LifecycleCore):
|
|
52
|
+
"""Single-row get/insert/update/delete against one table, by primary key.
|
|
53
|
+
|
|
54
|
+
A thin repository layer over SqlDatabaseResource + SqlModelRegistry for
|
|
55
|
+
consumers that want cheap transactional CRUD without pulling in
|
|
56
|
+
DataGateway/DataHelper's dataframe-oriented machinery. Rows are plain
|
|
57
|
+
``dict[str, Any]``, never a DataFrame.
|
|
58
|
+
|
|
59
|
+
Exactly one of ``config``/``session`` must be given:
|
|
60
|
+
|
|
61
|
+
- ``config``: the repository owns a private ``SqlDatabaseResource`` and a
|
|
62
|
+
fresh, ``query_only``-guarded session per call, committing after every
|
|
63
|
+
write. ``query_only`` defaults to ``True`` here and always wins over
|
|
64
|
+
whatever ``config.query_only`` says — insert/update/delete raise via the
|
|
65
|
+
same ``ReadOnlySession`` guard SqlDatabaseResource uses until a caller
|
|
66
|
+
passes ``query_only=False`` explicitly. This is deliberate: a ``config``
|
|
67
|
+
object may be shared with or reused from a context where writes are
|
|
68
|
+
already enabled, and a repository shouldn't silently inherit write
|
|
69
|
+
access nobody asked it for.
|
|
70
|
+
- ``session``: an already-open ``Session`` supplied by the caller (e.g.
|
|
71
|
+
from ``SqlUnitOfWork``, or hand-rolled session code composing several
|
|
72
|
+
repositories into one transaction). Writes ``flush()`` instead of
|
|
73
|
+
``commit()`` — the caller controls the transaction boundary — and
|
|
74
|
+
closing this repository never closes the shared session. ``query_only``
|
|
75
|
+
is accepted for signature uniformity but is inert here: the session's
|
|
76
|
+
own class (``ReadOnlySession`` or not) already determines writability,
|
|
77
|
+
since this path never constructs a new guarded session.
|
|
78
|
+
|
|
79
|
+
``pk`` accepts a scalar or a tuple for composite keys, mirroring
|
|
80
|
+
``Session.get()``'s own signature.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
def __init__(
|
|
84
|
+
self,
|
|
85
|
+
config: SqlDatabaseConfig | None = None,
|
|
86
|
+
table_name: str | None = None,
|
|
87
|
+
*,
|
|
88
|
+
schema: str | None = None,
|
|
89
|
+
query_only: bool = True,
|
|
90
|
+
session: Session | None = None,
|
|
91
|
+
) -> None:
|
|
92
|
+
table_name = _require_table_name(table_name)
|
|
93
|
+
_require_exactly_one_source(config, session)
|
|
94
|
+
self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
|
|
95
|
+
super().__init__()
|
|
96
|
+
self._owns_resource = session is None
|
|
97
|
+
self._session_override = session
|
|
98
|
+
if session is not None:
|
|
99
|
+
self._resource = None
|
|
100
|
+
engine = session.bind
|
|
101
|
+
else:
|
|
102
|
+
effective_config = config.model_copy(update={"query_only": query_only})
|
|
103
|
+
self._resource = SqlDatabaseResource(effective_config)
|
|
104
|
+
engine = self._resource.engine
|
|
105
|
+
self._model = get_global_registry().get_model(engine, table_name, schema=schema)
|
|
106
|
+
|
|
107
|
+
def _cleanup(self) -> None:
|
|
108
|
+
if self._resource is not None:
|
|
109
|
+
self._resource.close()
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def engine(self) -> Any:
|
|
113
|
+
return self._resource.engine if self._resource is not None else self._session_override.bind
|
|
114
|
+
|
|
115
|
+
@contextmanager
|
|
116
|
+
def _session_scope(self):
|
|
117
|
+
if self._session_override is not None:
|
|
118
|
+
yield self._session_override
|
|
119
|
+
else:
|
|
120
|
+
with self._resource.session() as session:
|
|
121
|
+
yield session
|
|
122
|
+
|
|
123
|
+
def _finalize_write(self, session: Session) -> None:
|
|
124
|
+
if self._owns_resource:
|
|
125
|
+
session.commit()
|
|
126
|
+
else:
|
|
127
|
+
session.flush()
|
|
128
|
+
|
|
129
|
+
def get(self, pk: Any) -> dict[str, Any] | None:
|
|
130
|
+
with self._session_scope() as session:
|
|
131
|
+
instance = session.get(self._model, pk)
|
|
132
|
+
return _row_to_dict(instance) if instance is not None else None
|
|
133
|
+
|
|
134
|
+
def insert(self, values: dict[str, Any]) -> dict[str, Any]:
|
|
135
|
+
with self._session_scope() as session:
|
|
136
|
+
instance = self._model(**values)
|
|
137
|
+
session.add(instance)
|
|
138
|
+
self._finalize_write(session)
|
|
139
|
+
session.refresh(instance)
|
|
140
|
+
return _row_to_dict(instance)
|
|
141
|
+
|
|
142
|
+
def update(self, pk: Any, values: dict[str, Any]) -> dict[str, Any] | None:
|
|
143
|
+
with self._session_scope() as session:
|
|
144
|
+
instance = session.get(self._model, pk)
|
|
145
|
+
if instance is None:
|
|
146
|
+
return None
|
|
147
|
+
for key, value in values.items():
|
|
148
|
+
setattr(instance, key, value)
|
|
149
|
+
self._finalize_write(session)
|
|
150
|
+
session.refresh(instance)
|
|
151
|
+
return _row_to_dict(instance)
|
|
152
|
+
|
|
153
|
+
def delete(self, pk: Any) -> bool:
|
|
154
|
+
with self._session_scope() as session:
|
|
155
|
+
instance = session.get(self._model, pk)
|
|
156
|
+
if instance is None:
|
|
157
|
+
return False
|
|
158
|
+
session.delete(instance)
|
|
159
|
+
self._finalize_write(session)
|
|
160
|
+
return True
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class AsyncSqlRepository(LifecycleCore):
|
|
164
|
+
"""Async counterpart to :class:`SqlRepository` — see its docstring,
|
|
165
|
+
including the ``config``/``session`` and ``query_only`` semantics.
|
|
166
|
+
|
|
167
|
+
Async-only by design, mirroring ``AsyncSqlDatabaseResource``: real
|
|
168
|
+
resource acquisition (the engine, the reflected model) happens in
|
|
169
|
+
``__aenter__``, not ``__init__``, since resolving the engine requires
|
|
170
|
+
awaiting ``EngineRegistry.get_or_create_async()`` (``config`` path) or
|
|
171
|
+
``SqlModelRegistry.get_model_async()`` (``session`` path either way).
|
|
172
|
+
"""
|
|
173
|
+
|
|
174
|
+
def __init__(
|
|
175
|
+
self,
|
|
176
|
+
config: SqlDatabaseConfig | None = None,
|
|
177
|
+
table_name: str | None = None,
|
|
178
|
+
*,
|
|
179
|
+
schema: str | None = None,
|
|
180
|
+
query_only: bool = True,
|
|
181
|
+
session: AsyncSession | None = None,
|
|
182
|
+
) -> None:
|
|
183
|
+
table_name = _require_table_name(table_name)
|
|
184
|
+
_require_exactly_one_source(config, session)
|
|
185
|
+
self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
|
|
186
|
+
self._config = (
|
|
187
|
+
config.model_copy(update={"query_only": query_only}) if config is not None else None
|
|
188
|
+
)
|
|
189
|
+
self._table_name = table_name
|
|
190
|
+
self._schema = schema
|
|
191
|
+
self._session_override = session
|
|
192
|
+
self._owns_resource = session is None
|
|
193
|
+
self._resource: AsyncSqlDatabaseResource | None = None
|
|
194
|
+
self._model: type[Any] | None = None
|
|
195
|
+
super().__init__()
|
|
196
|
+
|
|
197
|
+
async def __aenter__(self) -> AsyncSqlRepository:
|
|
198
|
+
await super().__aenter__()
|
|
199
|
+
if self._session_override is not None:
|
|
200
|
+
engine = self._session_override.bind
|
|
201
|
+
else:
|
|
202
|
+
self._resource = await AsyncSqlDatabaseResource(self._config).__aenter__()
|
|
203
|
+
engine = self._resource.engine
|
|
204
|
+
self._model = await get_global_registry().get_model_async(
|
|
205
|
+
engine, self._table_name, schema=self._schema
|
|
206
|
+
)
|
|
207
|
+
return self
|
|
208
|
+
|
|
209
|
+
async def _acleanup(self) -> None:
|
|
210
|
+
if self._resource is not None:
|
|
211
|
+
await self._resource.aclose()
|
|
212
|
+
|
|
213
|
+
@property
|
|
214
|
+
def engine(self) -> Any:
|
|
215
|
+
if self._resource is not None:
|
|
216
|
+
return self._resource.engine
|
|
217
|
+
if self._session_override is not None:
|
|
218
|
+
return self._session_override.bind
|
|
219
|
+
raise RuntimeError("AsyncSqlRepository is not bound — use 'async with'.")
|
|
220
|
+
|
|
221
|
+
@asynccontextmanager
|
|
222
|
+
async def _session_scope(self):
|
|
223
|
+
if self._session_override is not None:
|
|
224
|
+
yield self._session_override
|
|
225
|
+
else:
|
|
226
|
+
async with self._resource.session() as session:
|
|
227
|
+
yield session
|
|
228
|
+
|
|
229
|
+
async def _finalize_write(self, session: AsyncSession) -> None:
|
|
230
|
+
if self._owns_resource:
|
|
231
|
+
await session.commit()
|
|
232
|
+
else:
|
|
233
|
+
await session.flush()
|
|
234
|
+
|
|
235
|
+
async def get(self, pk: Any) -> dict[str, Any] | None:
|
|
236
|
+
async with self._session_scope() as session:
|
|
237
|
+
instance = await session.get(self._model, pk)
|
|
238
|
+
return _row_to_dict(instance) if instance is not None else None
|
|
239
|
+
|
|
240
|
+
async def insert(self, values: dict[str, Any]) -> dict[str, Any]:
|
|
241
|
+
async with self._session_scope() as session:
|
|
242
|
+
instance = self._model(**values)
|
|
243
|
+
session.add(instance)
|
|
244
|
+
await self._finalize_write(session)
|
|
245
|
+
await session.refresh(instance)
|
|
246
|
+
return _row_to_dict(instance)
|
|
247
|
+
|
|
248
|
+
async def update(self, pk: Any, values: dict[str, Any]) -> dict[str, Any] | None:
|
|
249
|
+
async with self._session_scope() as session:
|
|
250
|
+
instance = await session.get(self._model, pk)
|
|
251
|
+
if instance is None:
|
|
252
|
+
return None
|
|
253
|
+
for key, value in values.items():
|
|
254
|
+
setattr(instance, key, value)
|
|
255
|
+
await self._finalize_write(session)
|
|
256
|
+
await session.refresh(instance)
|
|
257
|
+
return _row_to_dict(instance)
|
|
258
|
+
|
|
259
|
+
async def delete(self, pk: Any) -> bool:
|
|
260
|
+
async with self._session_scope() as session:
|
|
261
|
+
instance = await session.get(self._model, pk)
|
|
262
|
+
if instance is None:
|
|
263
|
+
return False
|
|
264
|
+
await session.delete(instance)
|
|
265
|
+
await self._finalize_write(session)
|
|
266
|
+
return True
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
class SqlUnitOfWork:
|
|
270
|
+
"""Shares one transaction across several ``SqlRepository`` instances,
|
|
271
|
+
potentially against different tables, so their writes commit or roll back
|
|
272
|
+
together.
|
|
273
|
+
|
|
274
|
+
Not a ``LifecycleCore`` subclass: ``LifecycleCore.__exit__`` is marked
|
|
275
|
+
``@final`` specifically so cleanup-suppression logic can't be overridden,
|
|
276
|
+
but that also means there's no hook that receives exception info to
|
|
277
|
+
decide "commit on success, roll back on exception" — exactly what a unit
|
|
278
|
+
of work needs. A small hand-rolled context manager is simpler than
|
|
279
|
+
fighting that sealed contract.
|
|
280
|
+
|
|
281
|
+
``query_only`` defaults to ``True``, consistent with ``SqlRepository``'s
|
|
282
|
+
own default — pass ``query_only=False`` explicitly to actually write.
|
|
283
|
+
"""
|
|
284
|
+
|
|
285
|
+
def __init__(self, config: SqlDatabaseConfig, *, query_only: bool = True) -> None:
|
|
286
|
+
effective_config = config.model_copy(update={"query_only": query_only})
|
|
287
|
+
self._resource = SqlDatabaseResource(effective_config)
|
|
288
|
+
self._session: Session = self._resource.session()
|
|
289
|
+
self._repositories: list[SqlRepository] = []
|
|
290
|
+
|
|
291
|
+
def repository(self, table_name: str, *, schema: str | None = None) -> SqlRepository:
|
|
292
|
+
repo = SqlRepository(session=self._session, table_name=table_name, schema=schema)
|
|
293
|
+
self._repositories.append(repo)
|
|
294
|
+
return repo
|
|
295
|
+
|
|
296
|
+
def __enter__(self) -> SqlUnitOfWork:
|
|
297
|
+
return self
|
|
298
|
+
|
|
299
|
+
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
|
300
|
+
try:
|
|
301
|
+
if exc_type is not None:
|
|
302
|
+
self._session.rollback()
|
|
303
|
+
elif not isinstance(self._session, ReadOnlySession):
|
|
304
|
+
self._session.commit()
|
|
305
|
+
# else: read-only session, exited cleanly — a read never needs a
|
|
306
|
+
# commit, and ReadOnlySession.commit() always raises regardless
|
|
307
|
+
# of whether anything was actually written.
|
|
308
|
+
finally:
|
|
309
|
+
# Each handed-out repository has its own GC leak-warning
|
|
310
|
+
# finalizer (see boti.core.lifecycle) even though its own
|
|
311
|
+
# _cleanup() is a no-op in session-injected mode — close them
|
|
312
|
+
# here so that finalizer doesn't fire spuriously.
|
|
313
|
+
for repo in self._repositories:
|
|
314
|
+
repo.close()
|
|
315
|
+
self._session.close()
|
|
316
|
+
self._resource.close()
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
class AsyncSqlUnitOfWork:
|
|
320
|
+
"""Async counterpart to :class:`SqlUnitOfWork` — see its docstring.
|
|
321
|
+
|
|
322
|
+
``repository()`` is async here (unlike the sync version) because binding
|
|
323
|
+
an ``AsyncSqlRepository`` to a shared session requires awaiting
|
|
324
|
+
``SqlModelRegistry.get_model_async()`` — the returned repository is
|
|
325
|
+
already fully set up, so callers don't need to ``async with`` it too.
|
|
326
|
+
"""
|
|
327
|
+
|
|
328
|
+
def __init__(self, config: SqlDatabaseConfig, *, query_only: bool = True) -> None:
|
|
329
|
+
self._config = config.model_copy(update={"query_only": query_only})
|
|
330
|
+
self._resource: AsyncSqlDatabaseResource | None = None
|
|
331
|
+
self._session: AsyncSession | None = None
|
|
332
|
+
self._repositories: list[AsyncSqlRepository] = []
|
|
333
|
+
|
|
334
|
+
async def repository(self, table_name: str, *, schema: str | None = None) -> AsyncSqlRepository:
|
|
335
|
+
repo = AsyncSqlRepository(session=self._session, table_name=table_name, schema=schema)
|
|
336
|
+
await repo.__aenter__()
|
|
337
|
+
self._repositories.append(repo)
|
|
338
|
+
return repo
|
|
339
|
+
|
|
340
|
+
async def __aenter__(self) -> AsyncSqlUnitOfWork:
|
|
341
|
+
self._resource = await AsyncSqlDatabaseResource(self._config).__aenter__()
|
|
342
|
+
self._session = self._resource.session()
|
|
343
|
+
return self
|
|
344
|
+
|
|
345
|
+
async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
|
346
|
+
try:
|
|
347
|
+
if exc_type is not None:
|
|
348
|
+
await self._session.rollback()
|
|
349
|
+
elif not isinstance(self._session, ReadOnlyAsyncSession):
|
|
350
|
+
await self._session.commit()
|
|
351
|
+
# else: read-only session, exited cleanly — a read never needs a
|
|
352
|
+
# commit, and ReadOnlyAsyncSession.commit() always raises
|
|
353
|
+
# regardless of whether anything was actually written.
|
|
354
|
+
finally:
|
|
355
|
+
# See SqlUnitOfWork.__exit__'s comment: each handed-out repository
|
|
356
|
+
# has its own GC leak-warning finalizer even though its _acleanup()
|
|
357
|
+
# is a no-op in session-injected mode.
|
|
358
|
+
for repo in self._repositories:
|
|
359
|
+
await repo.aclose()
|
|
360
|
+
await self._session.close()
|
|
361
|
+
await self._resource.aclose()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: boti-data
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.1
|
|
4
4
|
Summary: Data infrastructure for the Boti ecosystem
|
|
5
5
|
Author: Luis Valverde
|
|
6
6
|
Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>
|
|
@@ -920,6 +920,44 @@ async with AsyncSqlRepository(config, "users", query_only=False) as users:
|
|
|
920
920
|
row = await users.get(1)
|
|
921
921
|
```
|
|
922
922
|
|
|
923
|
+
#### Multi-table atomic transactions
|
|
924
|
+
|
|
925
|
+
By default each `SqlRepository` owns its own connection and commits after every write — fine for independent single-row operations, but not when several writes (potentially across different tables) need to succeed or fail together. Pass an already-open `session` instead of `config` to bind a repository to a transaction you control; writes `flush()` instead of committing, so nothing is durable until you commit the session yourself:
|
|
926
|
+
|
|
927
|
+
```python
|
|
928
|
+
with SqlDatabaseResource(config) as db, db.session() as session:
|
|
929
|
+
with SqlRepository(session=session, table_name="audit_log") as audit_log:
|
|
930
|
+
with SqlRepository(session=session, table_name="embeddings") as embeddings:
|
|
931
|
+
audit_log.insert({"event": "enroll"})
|
|
932
|
+
embeddings.insert({"subject_id": 1, "vector": [...]})
|
|
933
|
+
session.commit() # both rows commit together, or neither does on an exception
|
|
934
|
+
```
|
|
935
|
+
|
|
936
|
+
`SqlUnitOfWork` wraps this pattern for the common case — one config, several tables, one commit/rollback:
|
|
937
|
+
|
|
938
|
+
```python
|
|
939
|
+
from boti_data import SqlDatabaseConfig, SqlUnitOfWork
|
|
940
|
+
|
|
941
|
+
config = SqlDatabaseConfig(connection_url="postgresql+psycopg://user:pass@host/mydb")
|
|
942
|
+
|
|
943
|
+
with SqlUnitOfWork(config, query_only=False) as uow:
|
|
944
|
+
uow.repository("audit_log").insert({"event": "enroll"})
|
|
945
|
+
uow.repository("embeddings").insert({"subject_id": 1, "vector": [...]})
|
|
946
|
+
# commits both on clean exit; rolls back both on any exception
|
|
947
|
+
```
|
|
948
|
+
|
|
949
|
+
`query_only` on a `session`-bound `SqlRepository` is inert — the session's own class (e.g. a `ReadOnlySession`) already determines writability, since this path never builds a new guarded session itself. `AsyncSqlUnitOfWork` mirrors `SqlUnitOfWork`, with an async `repository()` method (returns an already-set-up `AsyncSqlRepository`, so callers don't need to `async with` each one individually):
|
|
950
|
+
|
|
951
|
+
```python
|
|
952
|
+
from boti_data import AsyncSqlUnitOfWork
|
|
953
|
+
|
|
954
|
+
async with AsyncSqlUnitOfWork(config, query_only=False) as uow:
|
|
955
|
+
audit_log = await uow.repository("audit_log")
|
|
956
|
+
embeddings = await uow.repository("embeddings")
|
|
957
|
+
await audit_log.insert({"event": "enroll"})
|
|
958
|
+
await embeddings.insert({"subject_id": 1, "vector": [...]})
|
|
959
|
+
```
|
|
960
|
+
|
|
923
961
|
### Nearest-neighbour search (pgvector)
|
|
924
962
|
|
|
925
963
|
`boti_data.db.nearest_neighbors`/`vector_distance` build an `ORDER BY <distance> LIMIT k` statement against a pgvector column, with the query vector bound as a driver parameter — never string-interpolated into the SQL, regardless of how large the vector is. Requires `boti-data[pgvector]` (see [Installation](#installation)) and a table reflected via `SqlAlchemyModelBuilder`/`SqlRepository`/`DataGateway`.
|
|
@@ -1,186 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Lightweight single-row transactional access (get/insert/update/delete by
|
|
3
|
-
primary key), returning plain dicts instead of a DataFrame.
|
|
4
|
-
|
|
5
|
-
boti_data's flagship API (DataGateway/DataHelper) is dataframe-shaped and
|
|
6
|
-
bulk-oriented, pulling in the dask/pandas/polars dependency chain even for a
|
|
7
|
-
consumer that only ever touches one row at a time (e.g. an OLTP request
|
|
8
|
-
handler at p95 <300ms). SqlRepository/AsyncSqlRepository sit directly on
|
|
9
|
-
SqlDatabaseResource/AsyncSqlDatabaseResource + SqlModelRegistry — both already
|
|
10
|
-
free of that dependency chain — so this module can be imported and used
|
|
11
|
-
without dask/pandas/polars installed at all.
|
|
12
|
-
"""
|
|
13
|
-
|
|
14
|
-
from __future__ import annotations
|
|
15
|
-
|
|
16
|
-
from typing import Any
|
|
17
|
-
|
|
18
|
-
from boti.core.lifecycle import LifecycleCore
|
|
19
|
-
from boti.core.logger import Logger
|
|
20
|
-
from sqlalchemy import inspect as sa_inspect
|
|
21
|
-
|
|
22
|
-
from boti_data.db.sql_config import SqlDatabaseConfig
|
|
23
|
-
from boti_data.db.sql_model_registry import get_global_registry
|
|
24
|
-
from boti_data.db.sql_resource import AsyncSqlDatabaseResource, SqlDatabaseResource
|
|
25
|
-
|
|
26
|
-
__all__ = ["AsyncSqlRepository", "SqlRepository"]
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def _row_to_dict(instance: Any) -> dict[str, Any]:
|
|
30
|
-
mapper = sa_inspect(instance).mapper
|
|
31
|
-
return {attr.key: getattr(instance, attr.key) for attr in mapper.column_attrs}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
class SqlRepository(LifecycleCore):
|
|
35
|
-
"""Single-row get/insert/update/delete against one table, by primary key.
|
|
36
|
-
|
|
37
|
-
A thin repository layer over SqlDatabaseResource + SqlModelRegistry for
|
|
38
|
-
consumers that want cheap transactional CRUD without pulling in
|
|
39
|
-
DataGateway/DataHelper's dataframe-oriented machinery. Rows are plain
|
|
40
|
-
``dict[str, Any]``, never a DataFrame.
|
|
41
|
-
|
|
42
|
-
``query_only`` defaults to ``True`` here and always wins over whatever
|
|
43
|
-
``config.query_only`` says — insert/update/delete raise via the same
|
|
44
|
-
``ReadOnlySession`` guard SqlDatabaseResource uses until a caller passes
|
|
45
|
-
``query_only=False`` explicitly. This is deliberate: a ``config`` object
|
|
46
|
-
may be shared with or reused from a context where writes are already
|
|
47
|
-
enabled, and a repository shouldn't silently inherit write access nobody
|
|
48
|
-
asked it for.
|
|
49
|
-
|
|
50
|
-
``pk`` accepts a scalar or a tuple for composite keys, mirroring
|
|
51
|
-
``Session.get()``'s own signature.
|
|
52
|
-
"""
|
|
53
|
-
|
|
54
|
-
def __init__(
|
|
55
|
-
self,
|
|
56
|
-
config: SqlDatabaseConfig,
|
|
57
|
-
table_name: str,
|
|
58
|
-
*,
|
|
59
|
-
schema: str | None = None,
|
|
60
|
-
query_only: bool = True,
|
|
61
|
-
) -> None:
|
|
62
|
-
self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
|
|
63
|
-
super().__init__()
|
|
64
|
-
effective_config = config.model_copy(update={"query_only": query_only})
|
|
65
|
-
self._resource = SqlDatabaseResource(effective_config)
|
|
66
|
-
self._model = get_global_registry().get_model(
|
|
67
|
-
self._resource.engine, table_name, schema=schema
|
|
68
|
-
)
|
|
69
|
-
|
|
70
|
-
def _cleanup(self) -> None:
|
|
71
|
-
self._resource.close()
|
|
72
|
-
|
|
73
|
-
@property
|
|
74
|
-
def engine(self) -> Any:
|
|
75
|
-
return self._resource.engine
|
|
76
|
-
|
|
77
|
-
def get(self, pk: Any) -> dict[str, Any] | None:
|
|
78
|
-
with self._resource.session() as session:
|
|
79
|
-
instance = session.get(self._model, pk)
|
|
80
|
-
return _row_to_dict(instance) if instance is not None else None
|
|
81
|
-
|
|
82
|
-
def insert(self, values: dict[str, Any]) -> dict[str, Any]:
|
|
83
|
-
with self._resource.session() as session:
|
|
84
|
-
instance = self._model(**values)
|
|
85
|
-
session.add(instance)
|
|
86
|
-
session.commit()
|
|
87
|
-
session.refresh(instance)
|
|
88
|
-
return _row_to_dict(instance)
|
|
89
|
-
|
|
90
|
-
def update(self, pk: Any, values: dict[str, Any]) -> dict[str, Any] | None:
|
|
91
|
-
with self._resource.session() as session:
|
|
92
|
-
instance = session.get(self._model, pk)
|
|
93
|
-
if instance is None:
|
|
94
|
-
return None
|
|
95
|
-
for key, value in values.items():
|
|
96
|
-
setattr(instance, key, value)
|
|
97
|
-
session.commit()
|
|
98
|
-
session.refresh(instance)
|
|
99
|
-
return _row_to_dict(instance)
|
|
100
|
-
|
|
101
|
-
def delete(self, pk: Any) -> bool:
|
|
102
|
-
with self._resource.session() as session:
|
|
103
|
-
instance = session.get(self._model, pk)
|
|
104
|
-
if instance is None:
|
|
105
|
-
return False
|
|
106
|
-
session.delete(instance)
|
|
107
|
-
session.commit()
|
|
108
|
-
return True
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
class AsyncSqlRepository(LifecycleCore):
|
|
112
|
-
"""Async counterpart to :class:`SqlRepository` — see its docstring,
|
|
113
|
-
including the ``query_only`` default/override behaviour.
|
|
114
|
-
|
|
115
|
-
Async-only by design, mirroring ``AsyncSqlDatabaseResource``: real
|
|
116
|
-
resource acquisition (the engine, the reflected model) happens in
|
|
117
|
-
``__aenter__``, not ``__init__``, since resolving the engine requires
|
|
118
|
-
awaiting ``EngineRegistry.get_or_create_async()``.
|
|
119
|
-
"""
|
|
120
|
-
|
|
121
|
-
def __init__(
|
|
122
|
-
self,
|
|
123
|
-
config: SqlDatabaseConfig,
|
|
124
|
-
table_name: str,
|
|
125
|
-
*,
|
|
126
|
-
schema: str | None = None,
|
|
127
|
-
query_only: bool = True,
|
|
128
|
-
) -> None:
|
|
129
|
-
self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
|
|
130
|
-
self._config = config.model_copy(update={"query_only": query_only})
|
|
131
|
-
self._table_name = table_name
|
|
132
|
-
self._schema = schema
|
|
133
|
-
self._resource: AsyncSqlDatabaseResource | None = None
|
|
134
|
-
self._model: type[Any] | None = None
|
|
135
|
-
super().__init__()
|
|
136
|
-
|
|
137
|
-
async def __aenter__(self) -> AsyncSqlRepository:
|
|
138
|
-
await super().__aenter__()
|
|
139
|
-
self._resource = await AsyncSqlDatabaseResource(self._config).__aenter__()
|
|
140
|
-
self._model = await get_global_registry().get_model_async(
|
|
141
|
-
self._resource.engine, self._table_name, schema=self._schema
|
|
142
|
-
)
|
|
143
|
-
return self
|
|
144
|
-
|
|
145
|
-
async def _acleanup(self) -> None:
|
|
146
|
-
if self._resource is not None:
|
|
147
|
-
await self._resource.aclose()
|
|
148
|
-
|
|
149
|
-
@property
|
|
150
|
-
def engine(self) -> Any:
|
|
151
|
-
if self._resource is None:
|
|
152
|
-
raise RuntimeError("AsyncSqlRepository is not bound — use 'async with'.")
|
|
153
|
-
return self._resource.engine
|
|
154
|
-
|
|
155
|
-
async def get(self, pk: Any) -> dict[str, Any] | None:
|
|
156
|
-
async with self._resource.session() as session:
|
|
157
|
-
instance = await session.get(self._model, pk)
|
|
158
|
-
return _row_to_dict(instance) if instance is not None else None
|
|
159
|
-
|
|
160
|
-
async def insert(self, values: dict[str, Any]) -> dict[str, Any]:
|
|
161
|
-
async with self._resource.session() as session:
|
|
162
|
-
instance = self._model(**values)
|
|
163
|
-
session.add(instance)
|
|
164
|
-
await session.commit()
|
|
165
|
-
await session.refresh(instance)
|
|
166
|
-
return _row_to_dict(instance)
|
|
167
|
-
|
|
168
|
-
async def update(self, pk: Any, values: dict[str, Any]) -> dict[str, Any] | None:
|
|
169
|
-
async with self._resource.session() as session:
|
|
170
|
-
instance = await session.get(self._model, pk)
|
|
171
|
-
if instance is None:
|
|
172
|
-
return None
|
|
173
|
-
for key, value in values.items():
|
|
174
|
-
setattr(instance, key, value)
|
|
175
|
-
await session.commit()
|
|
176
|
-
await session.refresh(instance)
|
|
177
|
-
return _row_to_dict(instance)
|
|
178
|
-
|
|
179
|
-
async def delete(self, pk: Any) -> bool:
|
|
180
|
-
async with self._resource.session() as session:
|
|
181
|
-
instance = await session.get(self._model, pk)
|
|
182
|
-
if instance is None:
|
|
183
|
-
return False
|
|
184
|
-
await session.delete(instance)
|
|
185
|
-
await session.commit()
|
|
186
|
-
return True
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|