boti-data 1.0.6__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {boti_data-1.0.6/src/boti_data.egg-info → boti_data-1.3.0}/PKG-INFO +130 -7
- boti_data-1.0.6/PKG-INFO → boti_data-1.3.0/README.md +116 -28
- boti_data-1.3.0/pyproject.toml +141 -0
- boti_data-1.3.0/src/boti_data/__init__.py +181 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/arrow_schema.py +12 -6
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/connection_catalog.py +5 -4
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/datacube/__init__.py +0 -1
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/datacube/artifact.py +7 -15
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/datacube/base.py +16 -22
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/datacube/contract.py +19 -1
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/datacube/resource.py +1 -4
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/dataset/__init__.py +0 -1
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/dataset/hybrid.py +64 -152
- boti_data-1.3.0/src/boti_data/dataset/hybrid_support.py +133 -0
- boti_data-1.3.0/src/boti_data/db/__init__.py +72 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/arrow_schema_mapper.py +42 -35
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/engine_registry.py +42 -17
- boti_data-1.3.0/src/boti_data/db/partitioned_execution.py +310 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/partitioned_loader.py +61 -36
- boti_data-1.3.0/src/boti_data/db/partitioned_planner.py +383 -0
- boti_data-1.3.0/src/boti_data/db/partitioned_planner_async.py +110 -0
- boti_data-1.3.0/src/boti_data/db/partitioned_planning_math.py +100 -0
- boti_data-1.3.0/src/boti_data/db/partitioned_row_materialization.py +168 -0
- boti_data-1.3.0/src/boti_data/db/partitioned_statements.py +82 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/partitioned_types.py +50 -3
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/sql_config.py +1 -3
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/sql_engine.py +3 -13
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/sql_model_builder.py +21 -12
- boti_data-1.3.0/src/boti_data/db/sql_model_module_injection.py +51 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/sql_model_registry.py +139 -155
- boti_data-1.3.0/src/boti_data/db/sql_model_registry_types.py +51 -0
- boti_data-1.3.0/src/boti_data/db/sql_repository.py +354 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/sql_resource.py +3 -1
- boti_data-1.3.0/src/boti_data/db/vector_search.py +71 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/enrichment/async_enricher.py +37 -21
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/enrichment/protocol.py +1 -3
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/enrichment/specs.py +3 -1
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/enrichment/transformers.py +3 -1
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/field_map.py +2 -2
- boti_data-1.3.0/src/boti_data/filters/arrow_kernels.py +90 -0
- boti_data-1.3.0/src/boti_data/filters/arrow_kernels_comparison.py +82 -0
- boti_data-1.3.0/src/boti_data/filters/arrow_kernels_compile.py +217 -0
- boti_data-1.3.0/src/boti_data/filters/arrow_kernels_string.py +92 -0
- boti_data-1.3.0/src/boti_data/filters/backend_ops.py +228 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/filters/expressions.py +29 -23
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/filters/handler.py +34 -32
- boti_data-1.3.0/src/boti_data/filters/utils.py +67 -0
- boti_data-1.3.0/src/boti_data/filters/value_parsing.py +314 -0
- boti_data-1.3.0/src/boti_data/gateway/_backend_strategies.py +309 -0
- boti_data-1.3.0/src/boti_data/gateway/_gateway_init.py +221 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/_series_filters.py +14 -10
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/arrow_adapters.py +88 -77
- boti_data-1.3.0/src/boti_data/gateway/build_context.py +70 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/chunking.py +44 -31
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/configured_load.py +20 -33
- boti_data-1.3.0/src/boti_data/gateway/core.py +400 -0
- boti_data-1.3.0/src/boti_data/gateway/core_load.py +199 -0
- boti_data-1.3.0/src/boti_data/gateway/datacube_strategy.py +112 -0
- boti_data-1.3.0/src/boti_data/gateway/frame_convert.py +110 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/frame_strategies.py +44 -98
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/incremental.py +5 -0
- boti_data-1.3.0/src/boti_data/gateway/load_execution.py +202 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/loaders.py +31 -13
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/normalization.py +8 -5
- boti_data-1.3.0/src/boti_data/gateway/parquet_strategy.py +170 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/planning.py +101 -33
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/post_process.py +44 -54
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/return_type.py +75 -69
- boti_data-1.3.0/src/boti_data/gateway/select_cache.py +76 -0
- boti_data-1.3.0/src/boti_data/gateway/semi_join.py +97 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/sql_guard.py +2 -6
- boti_data-1.3.0/src/boti_data/gateway/sql_partitioned_exec.py +364 -0
- boti_data-1.3.0/src/boti_data/gateway/sql_size_estimation.py +170 -0
- boti_data-1.3.0/src/boti_data/gateway/sql_strategy.py +378 -0
- boti_data-1.3.0/src/boti_data/gateway/sql_strategy_configured_async.py +97 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/helper.py +62 -48
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/joins.py +124 -48
- boti_data-1.3.0/src/boti_data/parquet/__init__.py +28 -0
- boti_data-1.3.0/src/boti_data/parquet/config.py +77 -0
- boti_data-1.3.0/src/boti_data/parquet/discovery.py +163 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/parquet/file_info.py +5 -14
- boti_data-1.3.0/src/boti_data/parquet/filesystem.py +120 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/parquet/reader.py +9 -5
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/parquet/reader_config.py +5 -4
- boti_data-1.3.0/src/boti_data/parquet/resource.py +341 -0
- boti_data-1.3.0/src/boti_data/parquet/schema_filters.py +151 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/pipelines/__init__.py +2 -1
- boti_data-1.3.0/src/boti_data/pipelines/base.py +208 -0
- boti_data-1.3.0/src/boti_data/pipelines/parquet_pipeline.py +312 -0
- boti_data-1.3.0/src/boti_data/pipelines/sinks.py +34 -0
- boti_data-1.3.0/src/boti_data/pipelines/sinks_common.py +209 -0
- boti_data-1.3.0/src/boti_data/pipelines/sinks_parquet.py +127 -0
- boti_data-1.3.0/src/boti_data/pipelines/sinks_text.py +336 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/schema.py +38 -33
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/watermark/incremental.py +22 -19
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/watermark/store.py +7 -2
- boti_data-1.0.6/README.md → boti_data-1.3.0/src/boti_data.egg-info/PKG-INFO +151 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data.egg-info/SOURCES.txt +35 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data.egg-info/requires.txt +13 -2
- boti_data-1.0.6/pyproject.toml +0 -90
- boti_data-1.0.6/src/boti_data/__init__.py +0 -142
- boti_data-1.0.6/src/boti_data/db/__init__.py +0 -38
- boti_data-1.0.6/src/boti_data/db/partitioned_execution.py +0 -415
- boti_data-1.0.6/src/boti_data/db/partitioned_planner.py +0 -633
- boti_data-1.0.6/src/boti_data/filters/arrow_kernels.py +0 -351
- boti_data-1.0.6/src/boti_data/filters/utils.py +0 -482
- boti_data-1.0.6/src/boti_data/gateway/_backend_strategies.py +0 -1341
- boti_data-1.0.6/src/boti_data/gateway/core.py +0 -883
- boti_data-1.0.6/src/boti_data/parquet/__init__.py +0 -8
- boti_data-1.0.6/src/boti_data/parquet/resource.py +0 -736
- boti_data-1.0.6/src/boti_data/pipelines/base.py +0 -519
- boti_data-1.0.6/src/boti_data/pipelines/sinks.py +0 -642
- {boti_data-1.0.6 → boti_data-1.3.0}/setup.cfg +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/sql_manager.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/sql_readonly.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/db/sqlalchemy_async.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/enrichment/__init__.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/enrichment/composite.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/filters/__init__.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/filters/parquet_compiler.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/__init__.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/_factory.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/_payloads.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/load_request.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/policies.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/gateway/requests.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/pipelines/registry.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/py.typed +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data/watermark/__init__.py +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data.egg-info/dependency_links.txt +0 -0
- {boti_data-1.0.6 → boti_data-1.3.0}/src/boti_data.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: boti-data
|
|
3
|
-
Version: 1.0
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Data infrastructure for the Boti ecosystem
|
|
5
5
|
Author: Luis Valverde
|
|
6
6
|
Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>
|
|
@@ -17,14 +17,21 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
17
17
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
18
|
Requires-Python: >=3.13
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
|
-
Requires-Dist: asyncmy<0.3.0,>=0.2.11
|
|
21
20
|
Requires-Dist: boti<2.0.0,>=1.0.2
|
|
22
|
-
Requires-Dist: boti-dask<2.0.0,>=1.0.0
|
|
23
|
-
Requires-Dist: dask[dataframe,distributed]<2027.0.0,>=2026.3.0
|
|
24
|
-
Requires-Dist: pandas<4.0.0,>=3.0.2
|
|
25
|
-
Requires-Dist: polars<2.0.0,>=1.29.0
|
|
26
|
-
Requires-Dist: pymysql<2.0.0,>=1.1.2
|
|
27
21
|
Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.49
|
|
22
|
+
Provides-Extra: mysql
|
|
23
|
+
Requires-Dist: asyncmy<0.3.0,>=0.2.11; extra == "mysql"
|
|
24
|
+
Requires-Dist: pymysql<2.0.0,>=1.1.2; extra == "mysql"
|
|
25
|
+
Provides-Extra: postgres
|
|
26
|
+
Requires-Dist: asyncpg<0.32.0,>=0.30.0; extra == "postgres"
|
|
27
|
+
Requires-Dist: psycopg[binary]<4.0.0,>=3.2.0; extra == "postgres"
|
|
28
|
+
Provides-Extra: pgvector
|
|
29
|
+
Requires-Dist: pgvector<0.6.0,>=0.3.0; extra == "pgvector"
|
|
30
|
+
Provides-Extra: dataframes
|
|
31
|
+
Requires-Dist: boti-dask<2.0.0,>=1.0.0; extra == "dataframes"
|
|
32
|
+
Requires-Dist: dask[dataframe,distributed]<2027.0.0,>=2026.3.0; extra == "dataframes"
|
|
33
|
+
Requires-Dist: pandas<4.0.0,>=3.0.2; extra == "dataframes"
|
|
34
|
+
Requires-Dist: polars<2.0.0,>=1.29.0; extra == "dataframes"
|
|
28
35
|
|
|
29
36
|
# boti-data
|
|
30
37
|
|
|
@@ -119,6 +126,27 @@ Or install through the core package extra:
|
|
|
119
126
|
pip install "boti[data]"
|
|
120
127
|
```
|
|
121
128
|
|
|
129
|
+
`boti-data` doesn't bundle a database driver by default — SQLAlchemy resolves the driver package from your connection URL's scheme (`mysql+pymysql://`, `postgresql+asyncpg://`, ...), so install the one matching your database:
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
pip install "boti-data[mysql]" # asyncmy + pymysql
|
|
133
|
+
pip install "boti-data[postgres]" # asyncpg + psycopg[binary]
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
If a Postgres table has a [pgvector](https://github.com/pgvector/pgvector) `vector` column, install `boti-data[pgvector]` too — this registers pgvector's `Vector` type with SQLAlchemy's reflection so `SqlAlchemyModelBuilder`/`SqlRepository`/`DataGateway` map that column to a usable `list[float]` (with its dimension preserved) instead of an unreflectable `NullType`:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
pip install "boti-data[pgvector]"
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
The base install is deliberately light: single-row transactional access (`SqlRepository`/`AsyncSqlRepository`, see below) works out of the box, with no dask/pandas/polars pulled in. The dataframe-shaped, bulk-oriented side of the package (`DataGateway`/`DataHelper`, `ParquetPipeline`, `HybridDataset`, the sink pipelines, ...) needs the `dataframes` extra:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
pip install "boti-data[dataframes]" # dask + pandas + polars + boti-dask
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Every name importable from `boti_data` is loaded lazily on first access, so `import boti_data` alone never touches dask/pandas/polars — only accessing a dataframe-shaped name does.
|
|
149
|
+
|
|
122
150
|
## Imports
|
|
123
151
|
|
|
124
152
|
`boti-data` uses the top-level Python package `boti_data`. Dask runtime/session/resilience helpers live in `boti_dask` and are not re-exported here.
|
|
@@ -146,9 +174,11 @@ from boti_data import (
|
|
|
146
174
|
available_sinks,
|
|
147
175
|
create_sink,
|
|
148
176
|
register_sink,
|
|
177
|
+
AsyncSqlRepository,
|
|
149
178
|
SqlAlchemyModelBuilder,
|
|
150
179
|
SqlDatabaseConfig,
|
|
151
180
|
SqlDatabaseResource,
|
|
181
|
+
SqlRepository,
|
|
152
182
|
)
|
|
153
183
|
```
|
|
154
184
|
|
|
@@ -855,6 +885,99 @@ Use the following as a guide:
|
|
|
855
885
|
|
|
856
886
|
## Examples
|
|
857
887
|
|
|
888
|
+
### Repository (lightest-weight, no dataframe dependencies)
|
|
889
|
+
|
|
890
|
+
For single-row transactional access — get/insert/update/delete by primary key — without pulling in dask/pandas/polars at all. Rows come back as plain `dict[str, Any]`, never a DataFrame.
|
|
891
|
+
|
|
892
|
+
`query_only` defaults to `True` on `SqlRepository`/`AsyncSqlRepository` themselves and always wins over whatever the passed-in `config.query_only` says — insert/update/delete raise until you opt in explicitly with `query_only=False`. This is deliberate: a `config` object might be shared with or reused from a context where writes are already enabled, and a repository shouldn't silently inherit write access nobody asked it for.
|
|
893
|
+
|
|
894
|
+
```python
|
|
895
|
+
from boti_data import SqlDatabaseConfig, SqlRepository
|
|
896
|
+
|
|
897
|
+
config = SqlDatabaseConfig(connection_url="sqlite:///example.db")
|
|
898
|
+
|
|
899
|
+
with SqlRepository(config, "users", query_only=False) as users:
|
|
900
|
+
row = users.insert({"id": 1, "name": "Ada"})
|
|
901
|
+
row = users.get(1) # -> dict | None
|
|
902
|
+
row = users.update(1, {"name": "Grace"})
|
|
903
|
+
deleted = users.delete(1) # -> bool
|
|
904
|
+
```
|
|
905
|
+
|
|
906
|
+
Omit `query_only=False` (or read-only access is all you need) and writes raise instead of silently succeeding:
|
|
907
|
+
|
|
908
|
+
```python
|
|
909
|
+
with SqlRepository(config, "users") as users:
|
|
910
|
+
users.get(1) # fine
|
|
911
|
+
users.insert({"id": 2, "name": "Bob"}) # raises SQLAlchemyError
|
|
912
|
+
```
|
|
913
|
+
|
|
914
|
+
An async counterpart, `AsyncSqlRepository`, mirrors the same four methods as coroutines and is used as an async context manager:
|
|
915
|
+
|
|
916
|
+
```python
|
|
917
|
+
from boti_data import AsyncSqlRepository
|
|
918
|
+
|
|
919
|
+
async with AsyncSqlRepository(config, "users", query_only=False) as users:
|
|
920
|
+
row = await users.get(1)
|
|
921
|
+
```
|
|
922
|
+
|
|
923
|
+
#### Multi-table atomic transactions
|
|
924
|
+
|
|
925
|
+
By default each `SqlRepository` owns its own connection and commits after every write — fine for independent single-row operations, but not when several writes (potentially across different tables) need to succeed or fail together. Pass an already-open `session` instead of `config` to bind a repository to a transaction you control; writes `flush()` instead of committing, so nothing is durable until you commit the session yourself:
|
|
926
|
+
|
|
927
|
+
```python
|
|
928
|
+
with SqlDatabaseResource(config) as db, db.session() as session:
|
|
929
|
+
with SqlRepository(session=session, table_name="audit_log") as audit_log:
|
|
930
|
+
with SqlRepository(session=session, table_name="embeddings") as embeddings:
|
|
931
|
+
audit_log.insert({"event": "enroll"})
|
|
932
|
+
embeddings.insert({"subject_id": 1, "vector": [...]})
|
|
933
|
+
session.commit() # both rows commit together, or neither does on an exception
|
|
934
|
+
```
|
|
935
|
+
|
|
936
|
+
`SqlUnitOfWork` wraps this pattern for the common case — one config, several tables, one commit/rollback:
|
|
937
|
+
|
|
938
|
+
```python
|
|
939
|
+
from boti_data import SqlDatabaseConfig, SqlUnitOfWork
|
|
940
|
+
|
|
941
|
+
config = SqlDatabaseConfig(connection_url="postgresql+psycopg://user:pass@host/mydb")
|
|
942
|
+
|
|
943
|
+
with SqlUnitOfWork(config, query_only=False) as uow:
|
|
944
|
+
uow.repository("audit_log").insert({"event": "enroll"})
|
|
945
|
+
uow.repository("embeddings").insert({"subject_id": 1, "vector": [...]})
|
|
946
|
+
# commits both on clean exit; rolls back both on any exception
|
|
947
|
+
```
|
|
948
|
+
|
|
949
|
+
`query_only` on a `session`-bound `SqlRepository` is inert — the session's own class (e.g. a `ReadOnlySession`) already determines writability, since this path never builds a new guarded session itself. `AsyncSqlUnitOfWork` mirrors `SqlUnitOfWork`, with an async `repository()` method (returns an already-set-up `AsyncSqlRepository`, so callers don't need to `async with` each one individually):
|
|
950
|
+
|
|
951
|
+
```python
|
|
952
|
+
from boti_data import AsyncSqlUnitOfWork
|
|
953
|
+
|
|
954
|
+
async with AsyncSqlUnitOfWork(config, query_only=False) as uow:
|
|
955
|
+
audit_log = await uow.repository("audit_log")
|
|
956
|
+
embeddings = await uow.repository("embeddings")
|
|
957
|
+
await audit_log.insert({"event": "enroll"})
|
|
958
|
+
await embeddings.insert({"subject_id": 1, "vector": [...]})
|
|
959
|
+
```
|
|
960
|
+
|
|
961
|
+
### Nearest-neighbour search (pgvector)
|
|
962
|
+
|
|
963
|
+
`boti_data.db.nearest_neighbors`/`vector_distance` build an `ORDER BY <distance> LIMIT k` statement against a pgvector column, with the query vector bound as a driver parameter — never string-interpolated into the SQL, regardless of how large the vector is. Requires `boti-data[pgvector]` (see [Installation](#installation)) and a table reflected via `SqlAlchemyModelBuilder`/`SqlRepository`/`DataGateway`.
|
|
964
|
+
|
|
965
|
+
```python
|
|
966
|
+
from boti_data import DataGateway, SqlDatabaseConfig
|
|
967
|
+
from boti_data.db import SqlAlchemyModelBuilder, nearest_neighbors
|
|
968
|
+
|
|
969
|
+
config = SqlDatabaseConfig(connection_url="postgresql+psycopg://user:pass@host/mydb", query_only=True)
|
|
970
|
+
gateway = DataGateway(config)
|
|
971
|
+
|
|
972
|
+
model = SqlAlchemyModelBuilder(gateway.resource.engine, "embeddings").build_model()
|
|
973
|
+
query_vector = [...] # e.g. a 512-float face embedding
|
|
974
|
+
|
|
975
|
+
stmt = nearest_neighbors(model, model.vector, query_vector, k=10, metric="cosine")
|
|
976
|
+
matches = gateway.load(statement=stmt, model=model, as_pandas=True)
|
|
977
|
+
```
|
|
978
|
+
|
|
979
|
+
Supported `metric` values map directly to pgvector's operators: `"cosine"` (`<=>`), `"l2"` (`<->`), `"inner_product"` (`<#>`), `"l1"` (`<+>`). Add ordinary filtering with `.where(...)` on the statement before passing it to `gateway.load(...)`.
|
|
980
|
+
|
|
858
981
|
### SQL resource (low-level)
|
|
859
982
|
|
|
860
983
|
```python
|
|
@@ -1,31 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: boti-data
|
|
3
|
-
Version: 1.0.6
|
|
4
|
-
Summary: Data infrastructure for the Boti ecosystem
|
|
5
|
-
Author: Luis Valverde
|
|
6
|
-
Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>
|
|
7
|
-
License-Expression: MIT
|
|
8
|
-
Project-URL: Homepage, https://github.com/lvalverdeb/boti-data
|
|
9
|
-
Project-URL: Repository, https://github.com/lvalverdeb/boti-data
|
|
10
|
-
Project-URL: Documentation, https://github.com/lvalverdeb/boti-data#readme
|
|
11
|
-
Project-URL: Issues, https://github.com/lvalverdeb/boti-data/issues
|
|
12
|
-
Classifier: Development Status :: 3 - Alpha
|
|
13
|
-
Classifier: Intended Audience :: Developers
|
|
14
|
-
Classifier: Programming Language :: Python :: 3
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
-
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
-
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
-
Requires-Python: >=3.13
|
|
19
|
-
Description-Content-Type: text/markdown
|
|
20
|
-
Requires-Dist: asyncmy<0.3.0,>=0.2.11
|
|
21
|
-
Requires-Dist: boti<2.0.0,>=1.0.2
|
|
22
|
-
Requires-Dist: boti-dask<2.0.0,>=1.0.0
|
|
23
|
-
Requires-Dist: dask[dataframe,distributed]<2027.0.0,>=2026.3.0
|
|
24
|
-
Requires-Dist: pandas<4.0.0,>=3.0.2
|
|
25
|
-
Requires-Dist: polars<2.0.0,>=1.29.0
|
|
26
|
-
Requires-Dist: pymysql<2.0.0,>=1.1.2
|
|
27
|
-
Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.49
|
|
28
|
-
|
|
29
1
|
# boti-data
|
|
30
2
|
|
|
31
3
|
`boti-data` is the **data access and data transformation layer** of the Boti ecosystem.
|
|
@@ -119,6 +91,27 @@ Or install through the core package extra:
|
|
|
119
91
|
pip install "boti[data]"
|
|
120
92
|
```
|
|
121
93
|
|
|
94
|
+
`boti-data` doesn't bundle a database driver by default — SQLAlchemy resolves the driver package from your connection URL's scheme (`mysql+pymysql://`, `postgresql+asyncpg://`, ...), so install the one matching your database:
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
pip install "boti-data[mysql]" # asyncmy + pymysql
|
|
98
|
+
pip install "boti-data[postgres]" # asyncpg + psycopg[binary]
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
If a Postgres table has a [pgvector](https://github.com/pgvector/pgvector) `vector` column, install `boti-data[pgvector]` too — this registers pgvector's `Vector` type with SQLAlchemy's reflection so `SqlAlchemyModelBuilder`/`SqlRepository`/`DataGateway` map that column to a usable `list[float]` (with its dimension preserved) instead of an unreflectable `NullType`:
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
pip install "boti-data[pgvector]"
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The base install is deliberately light: single-row transactional access (`SqlRepository`/`AsyncSqlRepository`, see below) works out of the box, with no dask/pandas/polars pulled in. The dataframe-shaped, bulk-oriented side of the package (`DataGateway`/`DataHelper`, `ParquetPipeline`, `HybridDataset`, the sink pipelines, ...) needs the `dataframes` extra:
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
pip install "boti-data[dataframes]" # dask + pandas + polars + boti-dask
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Every name importable from `boti_data` is loaded lazily on first access, so `import boti_data` alone never touches dask/pandas/polars — only accessing a dataframe-shaped name does.
|
|
114
|
+
|
|
122
115
|
## Imports
|
|
123
116
|
|
|
124
117
|
`boti-data` uses the top-level Python package `boti_data`. Dask runtime/session/resilience helpers live in `boti_dask` and are not re-exported here.
|
|
@@ -146,9 +139,11 @@ from boti_data import (
|
|
|
146
139
|
available_sinks,
|
|
147
140
|
create_sink,
|
|
148
141
|
register_sink,
|
|
142
|
+
AsyncSqlRepository,
|
|
149
143
|
SqlAlchemyModelBuilder,
|
|
150
144
|
SqlDatabaseConfig,
|
|
151
145
|
SqlDatabaseResource,
|
|
146
|
+
SqlRepository,
|
|
152
147
|
)
|
|
153
148
|
```
|
|
154
149
|
|
|
@@ -855,6 +850,99 @@ Use the following as a guide:
|
|
|
855
850
|
|
|
856
851
|
## Examples
|
|
857
852
|
|
|
853
|
+
### Repository (lightest-weight, no dataframe dependencies)
|
|
854
|
+
|
|
855
|
+
For single-row transactional access — get/insert/update/delete by primary key — without pulling in dask/pandas/polars at all. Rows come back as plain `dict[str, Any]`, never a DataFrame.
|
|
856
|
+
|
|
857
|
+
`query_only` defaults to `True` on `SqlRepository`/`AsyncSqlRepository` themselves and always wins over whatever the passed-in `config.query_only` says — insert/update/delete raise until you opt in explicitly with `query_only=False`. This is deliberate: a `config` object might be shared with or reused from a context where writes are already enabled, and a repository shouldn't silently inherit write access nobody asked it for.
|
|
858
|
+
|
|
859
|
+
```python
|
|
860
|
+
from boti_data import SqlDatabaseConfig, SqlRepository
|
|
861
|
+
|
|
862
|
+
config = SqlDatabaseConfig(connection_url="sqlite:///example.db")
|
|
863
|
+
|
|
864
|
+
with SqlRepository(config, "users", query_only=False) as users:
|
|
865
|
+
row = users.insert({"id": 1, "name": "Ada"})
|
|
866
|
+
row = users.get(1) # -> dict | None
|
|
867
|
+
row = users.update(1, {"name": "Grace"})
|
|
868
|
+
deleted = users.delete(1) # -> bool
|
|
869
|
+
```
|
|
870
|
+
|
|
871
|
+
Omit `query_only=False` (or read-only access is all you need) and writes raise instead of silently succeeding:
|
|
872
|
+
|
|
873
|
+
```python
|
|
874
|
+
with SqlRepository(config, "users") as users:
|
|
875
|
+
users.get(1) # fine
|
|
876
|
+
users.insert({"id": 2, "name": "Bob"}) # raises SQLAlchemyError
|
|
877
|
+
```
|
|
878
|
+
|
|
879
|
+
An async counterpart, `AsyncSqlRepository`, mirrors the same four methods as coroutines and is used as an async context manager:
|
|
880
|
+
|
|
881
|
+
```python
|
|
882
|
+
from boti_data import AsyncSqlRepository
|
|
883
|
+
|
|
884
|
+
async with AsyncSqlRepository(config, "users", query_only=False) as users:
|
|
885
|
+
row = await users.get(1)
|
|
886
|
+
```
|
|
887
|
+
|
|
888
|
+
#### Multi-table atomic transactions
|
|
889
|
+
|
|
890
|
+
By default each `SqlRepository` owns its own connection and commits after every write — fine for independent single-row operations, but not when several writes (potentially across different tables) need to succeed or fail together. Pass an already-open `session` instead of `config` to bind a repository to a transaction you control; writes `flush()` instead of committing, so nothing is durable until you commit the session yourself:
|
|
891
|
+
|
|
892
|
+
```python
|
|
893
|
+
with SqlDatabaseResource(config) as db, db.session() as session:
|
|
894
|
+
with SqlRepository(session=session, table_name="audit_log") as audit_log:
|
|
895
|
+
with SqlRepository(session=session, table_name="embeddings") as embeddings:
|
|
896
|
+
audit_log.insert({"event": "enroll"})
|
|
897
|
+
embeddings.insert({"subject_id": 1, "vector": [...]})
|
|
898
|
+
session.commit() # both rows commit together, or neither does on an exception
|
|
899
|
+
```
|
|
900
|
+
|
|
901
|
+
`SqlUnitOfWork` wraps this pattern for the common case — one config, several tables, one commit/rollback:
|
|
902
|
+
|
|
903
|
+
```python
|
|
904
|
+
from boti_data import SqlDatabaseConfig, SqlUnitOfWork
|
|
905
|
+
|
|
906
|
+
config = SqlDatabaseConfig(connection_url="postgresql+psycopg://user:pass@host/mydb")
|
|
907
|
+
|
|
908
|
+
with SqlUnitOfWork(config, query_only=False) as uow:
|
|
909
|
+
uow.repository("audit_log").insert({"event": "enroll"})
|
|
910
|
+
uow.repository("embeddings").insert({"subject_id": 1, "vector": [...]})
|
|
911
|
+
# commits both on clean exit; rolls back both on any exception
|
|
912
|
+
```
|
|
913
|
+
|
|
914
|
+
`query_only` on a `session`-bound `SqlRepository` is inert — the session's own class (e.g. a `ReadOnlySession`) already determines writability, since this path never builds a new guarded session itself. `AsyncSqlUnitOfWork` mirrors `SqlUnitOfWork`, with an async `repository()` method (returns an already-set-up `AsyncSqlRepository`, so callers don't need to `async with` each one individually):
|
|
915
|
+
|
|
916
|
+
```python
|
|
917
|
+
from boti_data import AsyncSqlUnitOfWork
|
|
918
|
+
|
|
919
|
+
async with AsyncSqlUnitOfWork(config, query_only=False) as uow:
|
|
920
|
+
audit_log = await uow.repository("audit_log")
|
|
921
|
+
embeddings = await uow.repository("embeddings")
|
|
922
|
+
await audit_log.insert({"event": "enroll"})
|
|
923
|
+
await embeddings.insert({"subject_id": 1, "vector": [...]})
|
|
924
|
+
```
|
|
925
|
+
|
|
926
|
+
### Nearest-neighbour search (pgvector)
|
|
927
|
+
|
|
928
|
+
`boti_data.db.nearest_neighbors`/`vector_distance` build an `ORDER BY <distance> LIMIT k` statement against a pgvector column, with the query vector bound as a driver parameter — never string-interpolated into the SQL, regardless of how large the vector is. Requires `boti-data[pgvector]` (see [Installation](#installation)) and a table reflected via `SqlAlchemyModelBuilder`/`SqlRepository`/`DataGateway`.
|
|
929
|
+
|
|
930
|
+
```python
|
|
931
|
+
from boti_data import DataGateway, SqlDatabaseConfig
|
|
932
|
+
from boti_data.db import SqlAlchemyModelBuilder, nearest_neighbors
|
|
933
|
+
|
|
934
|
+
config = SqlDatabaseConfig(connection_url="postgresql+psycopg://user:pass@host/mydb", query_only=True)
|
|
935
|
+
gateway = DataGateway(config)
|
|
936
|
+
|
|
937
|
+
model = SqlAlchemyModelBuilder(gateway.resource.engine, "embeddings").build_model()
|
|
938
|
+
query_vector = [...] # e.g. a 512-float face embedding
|
|
939
|
+
|
|
940
|
+
stmt = nearest_neighbors(model, model.vector, query_vector, k=10, metric="cosine")
|
|
941
|
+
matches = gateway.load(statement=stmt, model=model, as_pandas=True)
|
|
942
|
+
```
|
|
943
|
+
|
|
944
|
+
Supported `metric` values map directly to pgvector's operators: `"cosine"` (`<=>`), `"l2"` (`<->`), `"inner_product"` (`<#>`), `"l1"` (`<+>`). Add ordinary filtering with `.where(...)` on the statement before passing it to `gateway.load(...)`.
|
|
945
|
+
|
|
858
946
|
### SQL resource (low-level)
|
|
859
947
|
|
|
860
948
|
```python
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "boti-data"
|
|
3
|
+
version = "1.3.0"
|
|
4
|
+
description = "Data infrastructure for the Boti ecosystem"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.13"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
authors = [
|
|
9
|
+
{name = "Luis Valverde"}
|
|
10
|
+
]
|
|
11
|
+
maintainers = [
|
|
12
|
+
{name = "Luis Valverde", email = "lvalverdeb@gmail.com"}
|
|
13
|
+
]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.13",
|
|
19
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
20
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
21
|
+
]
|
|
22
|
+
dependencies = [
|
|
23
|
+
"boti>=1.0.2,<2.0.0",
|
|
24
|
+
"sqlalchemy[asyncio]>=2.0.49,<3.0.0",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
# Neither driver is imported directly by boti_data's own code — SQLAlchemy
|
|
28
|
+
# lazily imports the DBAPI module named by the connection URL's scheme
|
|
29
|
+
# (e.g. "mysql+pymysql://", "postgresql+asyncpg://") when the engine/dialect
|
|
30
|
+
# is actually constructed. So these are purely "make sure pip installed the
|
|
31
|
+
# right package for the DSN you're using" extras, not code-level optionality.
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
mysql = [
|
|
34
|
+
"asyncmy>=0.2.11,<0.3.0",
|
|
35
|
+
"pymysql>=1.1.2,<2.0.0",
|
|
36
|
+
]
|
|
37
|
+
postgres = [
|
|
38
|
+
"asyncpg>=0.30.0,<0.32.0",
|
|
39
|
+
"psycopg[binary]>=3.2.0,<4.0.0",
|
|
40
|
+
]
|
|
41
|
+
# Merely importable, not called directly: importing pgvector.sqlalchemy has the
|
|
42
|
+
# side effect of registering the 'vector' type name into SQLAlchemy's Postgres
|
|
43
|
+
# dialect (sqlalchemy.dialects.postgresql.base.ischema_names), which is what
|
|
44
|
+
# boti_data.db.sql_model_registry's reflection needs to recognize pgvector
|
|
45
|
+
# Vector columns instead of falling back to an unusable NullType.
|
|
46
|
+
pgvector = [
|
|
47
|
+
"pgvector>=0.3.0,<0.6.0",
|
|
48
|
+
]
|
|
49
|
+
# The dataframe-shaped, bulk-oriented side of the package (DataGateway/
|
|
50
|
+
# DataHelper, ParquetPipeline, HybridDataset, the sink pipelines, etc.) — not
|
|
51
|
+
# needed by the lightweight SqlRepository/AsyncSqlRepository single-row API,
|
|
52
|
+
# which only needs the base "boti-data" install.
|
|
53
|
+
dataframes = [
|
|
54
|
+
"boti-dask>=1.0.0,<2.0.0",
|
|
55
|
+
"dask[dataframe,distributed]>=2026.3.0,<2027.0.0",
|
|
56
|
+
"pandas>=3.0.2,<4.0.0",
|
|
57
|
+
"polars>=1.29.0,<2.0.0",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
[project.urls]
|
|
61
|
+
Homepage = "https://github.com/lvalverdeb/boti-data"
|
|
62
|
+
Repository = "https://github.com/lvalverdeb/boti-data"
|
|
63
|
+
Documentation = "https://github.com/lvalverdeb/boti-data#readme"
|
|
64
|
+
Issues = "https://github.com/lvalverdeb/boti-data/issues"
|
|
65
|
+
|
|
66
|
+
[build-system]
|
|
67
|
+
requires = ["setuptools>=80", "wheel"]
|
|
68
|
+
build-backend = "setuptools.build_meta"
|
|
69
|
+
|
|
70
|
+
[dependency-groups]
|
|
71
|
+
dev = [
|
|
72
|
+
# boti-data itself no longer requires these (see [project.optional-dependencies]
|
|
73
|
+
# mysql), but tests/security/test_regressions_engine.py and
|
|
74
|
+
# tests/data/test_sql_manager_registry.py construct real mysql+pymysql/
|
|
75
|
+
# mysql+asyncmy engines and expect them to succeed, not raise a
|
|
76
|
+
# missing-driver error — so the dev/test environment still needs both.
|
|
77
|
+
"asyncmy>=0.2.11,<0.3.0",
|
|
78
|
+
"pymysql>=1.1.2,<2.0.0",
|
|
79
|
+
# Real async-SQLite driver used only by tests/data/test_sql_repository.py
|
|
80
|
+
# for genuine end-to-end AsyncSqlRepository round-trips — not a runtime
|
|
81
|
+
# dependency of boti_data itself, and not part of [mysql]/[postgres]/
|
|
82
|
+
# [dataframes] since SQLite isn't one of boti-data's supported backends.
|
|
83
|
+
"aiosqlite>=0.21.0,<0.23.0",
|
|
84
|
+
# boti-data itself no longer requires these (see [project.optional-dependencies]
|
|
85
|
+
# dataframes), but the bulk of the test suite exercises DataGateway/
|
|
86
|
+
# DataHelper/ParquetPipeline/HybridDataset and friends, which do — so the
|
|
87
|
+
# dev/test environment still needs the full dataframe stack installed.
|
|
88
|
+
"boti-dask>=1.0.0,<2.0.0",
|
|
89
|
+
"dask[dataframe,distributed]>=2026.3.0,<2027.0.0",
|
|
90
|
+
"pandas>=3.0.2,<4.0.0",
|
|
91
|
+
"polars>=1.29.0,<2.0.0",
|
|
92
|
+
# Not part of [project.optional-dependencies] pgvector's runtime need — it's
|
|
93
|
+
# here so tests/data/test_sql_model_registry_pgvector.py can verify the
|
|
94
|
+
# registration side effect actually happens when the package is installed.
|
|
95
|
+
"pgvector>=0.3.0,<0.6.0",
|
|
96
|
+
"mypy>=1.14.0",
|
|
97
|
+
"pytest>=9.0.3",
|
|
98
|
+
"pytest-asyncio>=1.3.0",
|
|
99
|
+
"pytest-benchmark>=5.1.0",
|
|
100
|
+
"pytest-cov>=6.0.0",
|
|
101
|
+
"ruff>=0.11.0",
|
|
102
|
+
]
|
|
103
|
+
|
|
104
|
+
[tool.ruff]
|
|
105
|
+
line-length = 100
|
|
106
|
+
target-version = "py313"
|
|
107
|
+
|
|
108
|
+
[tool.ruff.lint]
|
|
109
|
+
select = ["E", "F", "W", "I", "UP"]
|
|
110
|
+
ignore = ["E501"]
|
|
111
|
+
|
|
112
|
+
[tool.ruff.lint.per-file-ignores]
|
|
113
|
+
"tests/**" = ["E402", "F841"]
|
|
114
|
+
"src/**" = ["UP007", "E701"]
|
|
115
|
+
# __init__.py has a deliberate non-alphabetical import order to break a circular import cycle
|
|
116
|
+
"src/boti_data/__init__.py" = ["I001"]
|
|
117
|
+
|
|
118
|
+
[tool.pytest.ini_options]
|
|
119
|
+
asyncio_mode = "auto"
|
|
120
|
+
testpaths = ["tests"]
|
|
121
|
+
markers = [
|
|
122
|
+
"perf: performance benchmark (not run in default test suite)",
|
|
123
|
+
"security_regression: focused regression coverage for security fixes and audit findings",
|
|
124
|
+
]
|
|
125
|
+
filterwarnings = [
|
|
126
|
+
'ignore:Distributed SQL task payloads will carry the raw DSN credential.*worker_connection_env_var.*:UserWarning',
|
|
127
|
+
'ignore:Closing managed Dask session with 1 live persisted collection\(s\).*Missing dependency\.:RuntimeWarning',
|
|
128
|
+
]
|
|
129
|
+
|
|
130
|
+
[tool.setuptools]
|
|
131
|
+
include-package-data = true
|
|
132
|
+
|
|
133
|
+
[tool.setuptools.package-data]
|
|
134
|
+
boti_data = ["py.typed"]
|
|
135
|
+
|
|
136
|
+
[tool.setuptools.package-dir]
|
|
137
|
+
"" = "src"
|
|
138
|
+
|
|
139
|
+
[tool.setuptools.packages.find]
|
|
140
|
+
where = ["src"]
|
|
141
|
+
include = ["boti_data*"]
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Data modules and interfaces for the Boti pipeline context.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import importlib
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"And",
|
|
12
|
+
"AsyncFrameEnricher",
|
|
13
|
+
"AsyncSqlDatabaseResource",
|
|
14
|
+
"AsyncSqlRepository",
|
|
15
|
+
"AsyncSqlUnitOfWork",
|
|
16
|
+
"AttachmentSpec",
|
|
17
|
+
"BaseDataCube",
|
|
18
|
+
"BaseArtifact",
|
|
19
|
+
"BuilderConfig",
|
|
20
|
+
"ConnectionCatalog",
|
|
21
|
+
"CsvSink",
|
|
22
|
+
"CsvSinkConfig",
|
|
23
|
+
"DataFrameOptions",
|
|
24
|
+
"DataFrameParams",
|
|
25
|
+
"DataGateway",
|
|
26
|
+
"DataHelper",
|
|
27
|
+
"DatacubeConfig",
|
|
28
|
+
"DatacubeContract",
|
|
29
|
+
"DefaultBase",
|
|
30
|
+
"EngineRegistry",
|
|
31
|
+
"Expr",
|
|
32
|
+
"FieldMap",
|
|
33
|
+
"FileWatermarkStore",
|
|
34
|
+
"FsspecWatermarkStore",
|
|
35
|
+
"FilterHandler",
|
|
36
|
+
"FrameEnricher",
|
|
37
|
+
"HybridDataset",
|
|
38
|
+
"IncrementalResult",
|
|
39
|
+
"JsonlSink",
|
|
40
|
+
"JsonlSinkConfig",
|
|
41
|
+
"Not",
|
|
42
|
+
"Or",
|
|
43
|
+
"ParquetDataConfig",
|
|
44
|
+
"ParquetLoadRequest",
|
|
45
|
+
"ParquetDataResource",
|
|
46
|
+
"ParquetMaterializationResult",
|
|
47
|
+
"ParquetPipeline",
|
|
48
|
+
"ParquetReader",
|
|
49
|
+
"ParquetSink",
|
|
50
|
+
"RegistryConfig",
|
|
51
|
+
"SchemaValidationError",
|
|
52
|
+
"SinkPipeline",
|
|
53
|
+
"SinkRegistry",
|
|
54
|
+
"SinkWriteResult",
|
|
55
|
+
"SqlLoadRequest",
|
|
56
|
+
"SqlAlchemyModelBuilder",
|
|
57
|
+
"SqlDatabaseConfig",
|
|
58
|
+
"SqlDatabaseResource",
|
|
59
|
+
"SqlPartitionPlan",
|
|
60
|
+
"SqlPartitionSpec",
|
|
61
|
+
"SqlPartitionedLoadRequest",
|
|
62
|
+
"SqlPartitionedLoader",
|
|
63
|
+
"SqlModelRegistry",
|
|
64
|
+
"SqlRepository",
|
|
65
|
+
"SqlUnitOfWork",
|
|
66
|
+
"TrueExpr",
|
|
67
|
+
"VectorMetric",
|
|
68
|
+
"WatermarkStore",
|
|
69
|
+
"advance_watermark",
|
|
70
|
+
"align_frames_for_join",
|
|
71
|
+
"apply_schema_map",
|
|
72
|
+
"available_sinks",
|
|
73
|
+
"build_incremental_filters",
|
|
74
|
+
"create_sink",
|
|
75
|
+
"ensure_greenlet_available",
|
|
76
|
+
"get_global_registry",
|
|
77
|
+
"indexed_left_join",
|
|
78
|
+
"infer_schema_map",
|
|
79
|
+
"left_join_frames",
|
|
80
|
+
"nearest_neighbors",
|
|
81
|
+
"normalize_dtype_alias",
|
|
82
|
+
"normalize_schema_map",
|
|
83
|
+
"register_sink",
|
|
84
|
+
"validate_schema",
|
|
85
|
+
"vector_distance",
|
|
86
|
+
]
|
|
87
|
+
|
|
88
|
+
# Every public name is loaded lazily (PEP 562) so that importing boti_data (or
|
|
89
|
+
# any of its dependency-clean pieces, e.g. SqlRepository) never pulls in the
|
|
90
|
+
# dask/pandas/polars/pyarrow chain used by the dataframe-shaped, bulk-oriented
|
|
91
|
+
# side of the package (DataGateway/DataHelper, ParquetPipeline, etc.) unless a
|
|
92
|
+
# consumer actually touches one of those names.
|
|
93
|
+
_LAZY = {
|
|
94
|
+
# db.* — dependency-clean.
|
|
95
|
+
"AsyncSqlDatabaseResource": ("boti_data.db.sql_manager", "AsyncSqlDatabaseResource"),
|
|
96
|
+
"EngineRegistry": ("boti_data.db.sql_manager", "EngineRegistry"),
|
|
97
|
+
"SqlDatabaseConfig": ("boti_data.db.sql_manager", "SqlDatabaseConfig"),
|
|
98
|
+
"SqlDatabaseResource": ("boti_data.db.sql_manager", "SqlDatabaseResource"),
|
|
99
|
+
"BuilderConfig": ("boti_data.db.sql_model_builder", "BuilderConfig"),
|
|
100
|
+
"SqlAlchemyModelBuilder": ("boti_data.db.sql_model_builder", "SqlAlchemyModelBuilder"),
|
|
101
|
+
"DefaultBase": ("boti_data.db.sql_model_registry", "DefaultBase"),
|
|
102
|
+
"RegistryConfig": ("boti_data.db.sql_model_registry", "RegistryConfig"),
|
|
103
|
+
"SqlModelRegistry": ("boti_data.db.sql_model_registry", "SqlModelRegistry"),
|
|
104
|
+
"get_global_registry": ("boti_data.db.sql_model_registry", "get_global_registry"),
|
|
105
|
+
"ensure_greenlet_available": ("boti_data.db.sqlalchemy_async", "ensure_greenlet_available"),
|
|
106
|
+
"SqlPartitionPlan": ("boti_data.db.partitioned_types", "SqlPartitionPlan"),
|
|
107
|
+
"SqlPartitionSpec": ("boti_data.db.partitioned_types", "SqlPartitionSpec"),
|
|
108
|
+
"SqlRepository": ("boti_data.db.sql_repository", "SqlRepository"),
|
|
109
|
+
"AsyncSqlRepository": ("boti_data.db.sql_repository", "AsyncSqlRepository"),
|
|
110
|
+
"SqlUnitOfWork": ("boti_data.db.sql_repository", "SqlUnitOfWork"),
|
|
111
|
+
"AsyncSqlUnitOfWork": ("boti_data.db.sql_repository", "AsyncSqlUnitOfWork"),
|
|
112
|
+
"nearest_neighbors": ("boti_data.db.vector_search", "nearest_neighbors"),
|
|
113
|
+
"vector_distance": ("boti_data.db.vector_search", "vector_distance"),
|
|
114
|
+
"VectorMetric": ("boti_data.db.vector_search", "VectorMetric"),
|
|
115
|
+
"FieldMap": ("boti_data.field_map", "FieldMap"),
|
|
116
|
+
# dask+pandas — the one heavy import inside the otherwise-clean db package.
|
|
117
|
+
"SqlPartitionedLoader": ("boti_data.db.partitioned_loader", "SqlPartitionedLoader"),
|
|
118
|
+
"SqlPartitionedLoadRequest": ("boti_data.db.partitioned_loader", "SqlPartitionedLoadRequest"),
|
|
119
|
+
# dask/pandas/polars/pyarrow-touching — the dataframe-shaped, bulk-oriented side.
|
|
120
|
+
"ConnectionCatalog": ("boti_data.connection_catalog", "ConnectionCatalog"),
|
|
121
|
+
"ParquetDataConfig": ("boti_data.parquet", "ParquetDataConfig"),
|
|
122
|
+
"ParquetDataResource": ("boti_data.parquet", "ParquetDataResource"),
|
|
123
|
+
"ParquetReader": ("boti_data.parquet", "ParquetReader"),
|
|
124
|
+
"FilterHandler": ("boti_data.filters", "FilterHandler"),
|
|
125
|
+
"Expr": ("boti_data.filters", "Expr"),
|
|
126
|
+
"TrueExpr": ("boti_data.filters", "TrueExpr"),
|
|
127
|
+
"And": ("boti_data.filters", "And"),
|
|
128
|
+
"Or": ("boti_data.filters", "Or"),
|
|
129
|
+
"Not": ("boti_data.filters", "Not"),
|
|
130
|
+
"DataGateway": ("boti_data.gateway", "DataGateway"),
|
|
131
|
+
"ParquetLoadRequest": ("boti_data.gateway", "ParquetLoadRequest"),
|
|
132
|
+
"SqlLoadRequest": ("boti_data.gateway", "SqlLoadRequest"),
|
|
133
|
+
"DataFrameOptions": ("boti_data.gateway", "DataFrameOptions"),
|
|
134
|
+
"DataFrameParams": ("boti_data.gateway", "DataFrameParams"),
|
|
135
|
+
"DataHelper": ("boti_data.helper", "DataHelper"),
|
|
136
|
+
"indexed_left_join": ("boti_data.joins", "indexed_left_join"),
|
|
137
|
+
"left_join_frames": ("boti_data.joins", "left_join_frames"),
|
|
138
|
+
"SchemaValidationError": ("boti_data.schema", "SchemaValidationError"),
|
|
139
|
+
"align_frames_for_join": ("boti_data.schema", "align_frames_for_join"),
|
|
140
|
+
"apply_schema_map": ("boti_data.schema", "apply_schema_map"),
|
|
141
|
+
"infer_schema_map": ("boti_data.schema", "infer_schema_map"),
|
|
142
|
+
"normalize_dtype_alias": ("boti_data.schema", "normalize_dtype_alias"),
|
|
143
|
+
"normalize_schema_map": ("boti_data.schema", "normalize_schema_map"),
|
|
144
|
+
"validate_schema": ("boti_data.schema", "validate_schema"),
|
|
145
|
+
"BaseDataCube": ("boti_data.datacube", "BaseDataCube"),
|
|
146
|
+
"BaseArtifact": ("boti_data.datacube", "BaseArtifact"),
|
|
147
|
+
"DatacubeConfig": ("boti_data.datacube", "DatacubeConfig"),
|
|
148
|
+
"DatacubeContract": ("boti_data.datacube", "DatacubeContract"),
|
|
149
|
+
"HybridDataset": ("boti_data.dataset", "HybridDataset"),
|
|
150
|
+
"AsyncFrameEnricher": ("boti_data.enrichment", "AsyncFrameEnricher"),
|
|
151
|
+
"AttachmentSpec": ("boti_data.enrichment", "AttachmentSpec"),
|
|
152
|
+
"FrameEnricher": ("boti_data.enrichment", "FrameEnricher"),
|
|
153
|
+
"CsvSink": ("boti_data.pipelines", "CsvSink"),
|
|
154
|
+
"CsvSinkConfig": ("boti_data.pipelines", "CsvSinkConfig"),
|
|
155
|
+
"JsonlSink": ("boti_data.pipelines", "JsonlSink"),
|
|
156
|
+
"JsonlSinkConfig": ("boti_data.pipelines", "JsonlSinkConfig"),
|
|
157
|
+
"ParquetMaterializationResult": ("boti_data.pipelines", "ParquetMaterializationResult"),
|
|
158
|
+
"ParquetPipeline": ("boti_data.pipelines", "ParquetPipeline"),
|
|
159
|
+
"ParquetSink": ("boti_data.pipelines", "ParquetSink"),
|
|
160
|
+
"SinkPipeline": ("boti_data.pipelines", "SinkPipeline"),
|
|
161
|
+
"SinkRegistry": ("boti_data.pipelines", "SinkRegistry"),
|
|
162
|
+
"SinkWriteResult": ("boti_data.pipelines", "SinkWriteResult"),
|
|
163
|
+
"available_sinks": ("boti_data.pipelines", "available_sinks"),
|
|
164
|
+
"create_sink": ("boti_data.pipelines", "create_sink"),
|
|
165
|
+
"register_sink": ("boti_data.pipelines", "register_sink"),
|
|
166
|
+
"FileWatermarkStore": ("boti_data.watermark", "FileWatermarkStore"),
|
|
167
|
+
"FsspecWatermarkStore": ("boti_data.watermark", "FsspecWatermarkStore"),
|
|
168
|
+
"IncrementalResult": ("boti_data.watermark", "IncrementalResult"),
|
|
169
|
+
"WatermarkStore": ("boti_data.watermark", "WatermarkStore"),
|
|
170
|
+
"advance_watermark": ("boti_data.watermark", "advance_watermark"),
|
|
171
|
+
"build_incremental_filters": ("boti_data.watermark", "build_incremental_filters"),
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def __getattr__(name: str) -> Any:
|
|
176
|
+
if name in _LAZY:
|
|
177
|
+
module_name, attr = _LAZY[name]
|
|
178
|
+
value = getattr(importlib.import_module(module_name), attr)
|
|
179
|
+
globals()[name] = value
|
|
180
|
+
return value
|
|
181
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|