boti-data 1.0.2__tar.gz → 1.0.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {boti_data-1.0.2 → boti_data-1.0.6}/PKG-INFO +5 -6
- {boti_data-1.0.2 → boti_data-1.0.6}/pyproject.toml +7 -10
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/datacube/artifact.py +12 -6
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/datacube/base.py +5 -5
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_engine.py +13 -1
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/_backend_strategies.py +11 -12
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/core.py +16 -13
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/loaders.py +10 -9
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/parquet/resource.py +3 -3
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/pipelines/base.py +3 -3
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/pipelines/sinks.py +36 -3
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data.egg-info/PKG-INFO +5 -6
- {boti_data-1.0.2 → boti_data-1.0.6}/README.md +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/setup.cfg +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/__init__.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/arrow_schema.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/connection_catalog.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/datacube/__init__.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/datacube/contract.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/datacube/resource.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/dataset/__init__.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/dataset/hybrid.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/__init__.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/arrow_schema_mapper.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/engine_registry.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/partitioned_execution.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/partitioned_loader.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/partitioned_planner.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/partitioned_types.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_config.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_manager.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_model_builder.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_model_registry.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_readonly.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_resource.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sqlalchemy_async.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/__init__.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/async_enricher.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/composite.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/protocol.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/specs.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/transformers.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/field_map.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/__init__.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/arrow_kernels.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/expressions.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/handler.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/parquet_compiler.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/utils.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/__init__.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/_factory.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/_payloads.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/_series_filters.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/arrow_adapters.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/chunking.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/configured_load.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/frame_strategies.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/incremental.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/load_request.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/normalization.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/planning.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/policies.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/post_process.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/requests.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/return_type.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/sql_guard.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/helper.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/joins.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/parquet/__init__.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/parquet/file_info.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/parquet/reader.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/parquet/reader_config.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/pipelines/__init__.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/pipelines/registry.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/py.typed +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/schema.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/watermark/__init__.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/watermark/incremental.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/watermark/store.py +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data.egg-info/SOURCES.txt +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data.egg-info/dependency_links.txt +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data.egg-info/requires.txt +0 -0
- {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data.egg-info/top_level.txt +0 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: boti-data
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.6
|
|
4
4
|
Summary: Data infrastructure for the Boti ecosystem
|
|
5
|
-
Author
|
|
5
|
+
Author: Luis Valverde
|
|
6
|
+
Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>
|
|
6
7
|
License-Expression: MIT
|
|
7
8
|
Project-URL: Homepage, https://github.com/lvalverdeb/boti-data
|
|
8
9
|
Project-URL: Repository, https://github.com/lvalverdeb/boti-data
|
|
@@ -11,12 +12,10 @@ Project-URL: Issues, https://github.com/lvalverdeb/boti-data/issues
|
|
|
11
12
|
Classifier: Development Status :: 3 - Alpha
|
|
12
13
|
Classifier: Intended Audience :: Developers
|
|
13
14
|
Classifier: Programming Language :: Python :: 3
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
17
15
|
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
17
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
-
Requires-Python: >=3.
|
|
18
|
+
Requires-Python: >=3.13
|
|
20
19
|
Description-Content-Type: text/markdown
|
|
21
20
|
Requires-Dist: asyncmy<0.3.0,>=0.2.11
|
|
22
21
|
Requires-Dist: boti<2.0.0,>=1.0.2
|
|
@@ -1,21 +1,22 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "boti-data"
|
|
3
|
-
version = "1.0.
|
|
3
|
+
version = "1.0.6"
|
|
4
4
|
description = "Data infrastructure for the Boti ecosystem"
|
|
5
5
|
readme = "README.md"
|
|
6
|
-
requires-python = ">=3.
|
|
6
|
+
requires-python = ">=3.13"
|
|
7
7
|
license = "MIT"
|
|
8
8
|
authors = [
|
|
9
|
+
{name = "Luis Valverde"}
|
|
10
|
+
]
|
|
11
|
+
maintainers = [
|
|
9
12
|
{name = "Luis Valverde", email = "lvalverdeb@gmail.com"}
|
|
10
13
|
]
|
|
11
14
|
classifiers = [
|
|
12
15
|
"Development Status :: 3 - Alpha",
|
|
13
16
|
"Intended Audience :: Developers",
|
|
14
17
|
"Programming Language :: Python :: 3",
|
|
15
|
-
"Programming Language :: Python :: 3.10",
|
|
16
|
-
"Programming Language :: Python :: 3.11",
|
|
17
|
-
"Programming Language :: Python :: 3.12",
|
|
18
18
|
"Programming Language :: Python :: 3.13",
|
|
19
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
19
20
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
20
21
|
]
|
|
21
22
|
dependencies = [
|
|
@@ -51,7 +52,7 @@ dev = [
|
|
|
51
52
|
|
|
52
53
|
[tool.ruff]
|
|
53
54
|
line-length = 100
|
|
54
|
-
target-version = "
|
|
55
|
+
target-version = "py313"
|
|
55
56
|
|
|
56
57
|
[tool.ruff.lint]
|
|
57
58
|
select = ["E", "F", "W", "I", "UP"]
|
|
@@ -75,10 +76,6 @@ filterwarnings = [
|
|
|
75
76
|
'ignore:Closing managed Dask session with 1 live persisted collection\(s\).*Missing dependency\.:RuntimeWarning',
|
|
76
77
|
]
|
|
77
78
|
|
|
78
|
-
[tool.uv.sources]
|
|
79
|
-
boti = { workspace = true }
|
|
80
|
-
boti-dask = { workspace = true }
|
|
81
|
-
|
|
82
79
|
[tool.setuptools]
|
|
83
80
|
include-package-data = true
|
|
84
81
|
|
|
@@ -84,13 +84,17 @@ class BaseArtifact(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
84
84
|
|
|
85
85
|
merged = {**self.config, **kwargs}
|
|
86
86
|
self._helper: DataHelper = DataHelper(merged)
|
|
87
|
-
self.logger = Logger.default_logger(
|
|
87
|
+
self.logger = kwargs.get("logger") or Logger.default_logger(
|
|
88
|
+
logger_name=self.__class__.__name__
|
|
89
|
+
)
|
|
88
90
|
|
|
89
91
|
self.parquet_start_date: str | None = _validate_and_format_date(
|
|
90
|
-
"parquet_start_date",
|
|
92
|
+
"parquet_start_date",
|
|
93
|
+
merged.get("parquet_start_date"),
|
|
91
94
|
)
|
|
92
95
|
self.parquet_end_date: str | None = _validate_and_format_date(
|
|
93
|
-
"parquet_end_date",
|
|
96
|
+
"parquet_end_date",
|
|
97
|
+
merged.get("parquet_end_date"),
|
|
94
98
|
)
|
|
95
99
|
self.data_wrapper_class: type[Any] | None = merged.get("data_wrapper_class")
|
|
96
100
|
self.class_params: dict[str, Any] = merged.get("class_params") or {
|
|
@@ -106,14 +110,16 @@ class BaseArtifact(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
106
110
|
instance.df = None
|
|
107
111
|
instance._helper = helper
|
|
108
112
|
instance.config = {**cls.config, **overrides}
|
|
109
|
-
instance.logger = Logger.default_logger(logger_name=cls.__name__)
|
|
113
|
+
instance.logger = overrides.get("logger") or Logger.default_logger(logger_name=cls.__name__)
|
|
110
114
|
|
|
111
115
|
merged = {**cls.config, **overrides}
|
|
112
116
|
instance.parquet_start_date = _validate_and_format_date(
|
|
113
|
-
"parquet_start_date",
|
|
117
|
+
"parquet_start_date",
|
|
118
|
+
merged.get("parquet_start_date"),
|
|
114
119
|
)
|
|
115
120
|
instance.parquet_end_date = _validate_and_format_date(
|
|
116
|
-
"parquet_end_date",
|
|
121
|
+
"parquet_end_date",
|
|
122
|
+
merged.get("parquet_end_date"),
|
|
117
123
|
)
|
|
118
124
|
instance.data_wrapper_class = merged.get("data_wrapper_class")
|
|
119
125
|
instance.class_params = merged.get("class_params") or {
|
|
@@ -60,7 +60,9 @@ class BaseDataCube(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
60
60
|
|
|
61
61
|
merged = {**self.config, **kwargs}
|
|
62
62
|
self._helper: DataHelper = DataHelper(merged)
|
|
63
|
-
self.logger = Logger.default_logger(
|
|
63
|
+
self.logger = kwargs.get("logger") or Logger.default_logger(
|
|
64
|
+
logger_name=self.__class__.__name__
|
|
65
|
+
)
|
|
64
66
|
super().__init__()
|
|
65
67
|
|
|
66
68
|
@classmethod
|
|
@@ -70,7 +72,7 @@ class BaseDataCube(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
70
72
|
instance.df = None
|
|
71
73
|
instance._helper = helper
|
|
72
74
|
instance.config = {**cls.config, **overrides}
|
|
73
|
-
instance.logger = Logger.default_logger(logger_name=cls.__name__)
|
|
75
|
+
instance.logger = overrides.get("logger") or Logger.default_logger(logger_name=cls.__name__)
|
|
74
76
|
# __new__() bypasses __init__(), so LifecycleCore's state (_state_lock,
|
|
75
77
|
# _is_closed, the GC finalizer, etc.) must be wired up explicitly here.
|
|
76
78
|
LifecycleCore.__init__(instance)
|
|
@@ -111,9 +113,7 @@ class BaseDataCube(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
111
113
|
return self.__class__.fix_data is not BaseDataCube.fix_data
|
|
112
114
|
|
|
113
115
|
def _log_no_data(self) -> None:
|
|
114
|
-
self.logger.debug(
|
|
115
|
-
"No data was found by %s loader", self.__class__.__name__
|
|
116
|
-
)
|
|
116
|
+
self.logger.debug("No data was found by %s loader", self.__class__.__name__)
|
|
117
117
|
|
|
118
118
|
# ── public API ──────────────────────────────────────────────────────
|
|
119
119
|
|
|
@@ -34,12 +34,24 @@ def _parse_connection_url(
|
|
|
34
34
|
normalized_url = _normalize_connection_url(connection_url)
|
|
35
35
|
parsed = sqlalchemy_url.make_url(normalized_url)
|
|
36
36
|
try:
|
|
37
|
-
parsed.get_dialect()
|
|
37
|
+
dialect_cls = parsed.get_dialect()
|
|
38
38
|
except NoSuchModuleError as exc:
|
|
39
39
|
raise SQLAlchemyError(
|
|
40
40
|
f"{consumer_name} received unsupported SQLAlchemy dialect '{parsed.drivername}'. "
|
|
41
41
|
"Check the driver name in the connection URL."
|
|
42
42
|
) from exc
|
|
43
|
+
# get_dialect() only resolves the SQLAlchemy dialect class; the DBAPI driver
|
|
44
|
+
# package itself (e.g. psycopg2, pymysql) is imported lazily and would
|
|
45
|
+
# otherwise only fail deep inside create_engine()/first connect().
|
|
46
|
+
try:
|
|
47
|
+
dialect_cls.import_dbapi()
|
|
48
|
+
except (ModuleNotFoundError, ImportError) as exc:
|
|
49
|
+
raise SQLAlchemyError(
|
|
50
|
+
f"{consumer_name} connection_url uses dialect '{parsed.drivername}' but its DBAPI "
|
|
51
|
+
f"driver package is not installed ({exc}). Install the driver package for this "
|
|
52
|
+
"dialect (e.g. 'psycopg2-binary' for 'postgresql+psycopg2', 'pymysql' for "
|
|
53
|
+
"'mysql+pymysql') before constructing this resource."
|
|
54
|
+
) from exc
|
|
43
55
|
return parsed
|
|
44
56
|
|
|
45
57
|
|
|
@@ -116,8 +116,7 @@ _FALLBACK_DTYPE_BYTES: int = 16
|
|
|
116
116
|
def _estimate_bytes_per_row(meta_dtypes: dict[str, str]) -> int:
|
|
117
117
|
"""Estimate the in-memory footprint of one result row from its column dtypes."""
|
|
118
118
|
total = sum(
|
|
119
|
-
_DTYPE_BYTE_WEIGHTS.get(dtype, _FALLBACK_DTYPE_BYTES)
|
|
120
|
-
for dtype in meta_dtypes.values()
|
|
119
|
+
_DTYPE_BYTE_WEIGHTS.get(dtype, _FALLBACK_DTYPE_BYTES) for dtype in meta_dtypes.values()
|
|
121
120
|
)
|
|
122
121
|
return max(1, total)
|
|
123
122
|
|
|
@@ -388,7 +387,12 @@ class SqlAlchemyStrategy(BackendStrategy):
|
|
|
388
387
|
cfg = dict(cfg)
|
|
389
388
|
raw_url = cfg.pop("connection_url", None)
|
|
390
389
|
if raw_url is None:
|
|
391
|
-
raise ValueError(
|
|
390
|
+
raise ValueError(
|
|
391
|
+
"from_config requires 'connection_url' for the 'sqlalchemy' backend "
|
|
392
|
+
"(the default used when no 'backend' key is present in config). "
|
|
393
|
+
"If you intended a different backend, pass an explicit 'backend' key "
|
|
394
|
+
"(e.g. 'parquet', 'datacube')."
|
|
395
|
+
)
|
|
392
396
|
connection_url = SecretStr(raw_url) if isinstance(raw_url, str) else raw_url
|
|
393
397
|
return SqlDatabaseConfig(connection_url=connection_url, **cfg)
|
|
394
398
|
|
|
@@ -620,7 +624,8 @@ class SqlAlchemyStrategy(BackendStrategy):
|
|
|
620
624
|
t_coerce = perf_counter()
|
|
621
625
|
if len(df) <= single_fetch_ceiling:
|
|
622
626
|
df = SqlPartitionExecutor.align_and_coerce_partition(
|
|
623
|
-
df,
|
|
627
|
+
df,
|
|
628
|
+
meta_dtypes,
|
|
624
629
|
)
|
|
625
630
|
t_dask = perf_counter()
|
|
626
631
|
result = dd.from_pandas(df, npartitions=1)
|
|
@@ -808,10 +813,7 @@ class SqlAlchemyStrategy(BackendStrategy):
|
|
|
808
813
|
)
|
|
809
814
|
t1 = perf_counter()
|
|
810
815
|
if diagnostics:
|
|
811
|
-
resource.logger.info(
|
|
812
|
-
"Configured async select "
|
|
813
|
-
f"reflect_select={t1 - t0:.3f}s"
|
|
814
|
-
)
|
|
816
|
+
resource.logger.info(f"Configured async select reflect_select={t1 - t0:.3f}s")
|
|
815
817
|
if ctx.execution_mode == "lazy":
|
|
816
818
|
t2 = perf_counter()
|
|
817
819
|
partitioned_options = build_partitioned_load_options(
|
|
@@ -826,10 +828,7 @@ class SqlAlchemyStrategy(BackendStrategy):
|
|
|
826
828
|
request = build_sql_partitioned_request(options_dict)
|
|
827
829
|
t3 = perf_counter()
|
|
828
830
|
if diagnostics:
|
|
829
|
-
resource.logger.info(
|
|
830
|
-
"Build partitioned request "
|
|
831
|
-
f"elapsed={t3 - t2:.3f}s"
|
|
832
|
-
)
|
|
831
|
+
resource.logger.info(f"Build partitioned request elapsed={t3 - t2:.3f}s")
|
|
833
832
|
return await self._run_async_partitioned_request(ctx, resource, request)
|
|
834
833
|
|
|
835
834
|
df = await read_sql_async(
|
|
@@ -139,7 +139,9 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
139
139
|
self._return_type: ReturnType = self._df_params.return_type
|
|
140
140
|
self._execution_mode: ExecutionMode = self._df_params.execution_mode
|
|
141
141
|
self._configured_select_cache: OrderedDict[tuple[str, ...], tuple[Any, Any]] = OrderedDict()
|
|
142
|
-
self._configured_async_select_cache: OrderedDict[tuple[str, ...], tuple[Any, Any]] =
|
|
142
|
+
self._configured_async_select_cache: OrderedDict[tuple[str, ...], tuple[Any, Any]] = (
|
|
143
|
+
OrderedDict()
|
|
144
|
+
)
|
|
143
145
|
self._raw_sql_policy = raw_sql_policy
|
|
144
146
|
self._policies = policies or GatewayPolicies()
|
|
145
147
|
self._strict_filter_validation = strict_filter_validation
|
|
@@ -225,7 +227,6 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
225
227
|
"""
|
|
226
228
|
return self._logger
|
|
227
229
|
|
|
228
|
-
|
|
229
230
|
def _load_planner(self) -> LoadPlanner:
|
|
230
231
|
return LoadPlanner(
|
|
231
232
|
configured=self._configured,
|
|
@@ -354,7 +355,12 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
354
355
|
provided.
|
|
355
356
|
"""
|
|
356
357
|
cfg = dict(config)
|
|
357
|
-
|
|
358
|
+
# Wrapper classes (e.g. ParquetReader -> DataHelper) always forward
|
|
359
|
+
# fs=None/fs_factory=None when the caller didn't explicitly pass them,
|
|
360
|
+
# rather than omitting the keys. A blind cfg.update(overrides) would let
|
|
361
|
+
# that incidental None clobber a real fs embedded directly in the config
|
|
362
|
+
# mapping, so only non-None overrides are allowed to win.
|
|
363
|
+
cfg.update({key: value for key, value in overrides.items() if value is not None})
|
|
358
364
|
|
|
359
365
|
backend, common = _factory.extract_config_common_options(cfg)
|
|
360
366
|
strategy = get_strategy(backend)
|
|
@@ -389,7 +395,9 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
389
395
|
# Load API — structured mode
|
|
390
396
|
# ------------------------------------------------------------------
|
|
391
397
|
|
|
392
|
-
def preview(
|
|
398
|
+
def preview(
|
|
399
|
+
self, statement: Any, model: Any, n: int = 5, npartitions: int = 1, **options: Any
|
|
400
|
+
) -> pd.DataFrame:
|
|
393
401
|
"""Load a preview (first *n* rows) as a pandas DataFrame.
|
|
394
402
|
|
|
395
403
|
Leverages lazy loading + ``safe_head`` so a distributed client is used
|
|
@@ -404,7 +412,9 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
404
412
|
)
|
|
405
413
|
return safe_head(frame, n=n, npartitions=npartitions)
|
|
406
414
|
|
|
407
|
-
async def apreview(
|
|
415
|
+
async def apreview(
|
|
416
|
+
self, statement: Any, model: Any, n: int = 5, npartitions: int = 1, **options: Any
|
|
417
|
+
) -> pd.DataFrame:
|
|
408
418
|
"""Async version of :meth:`preview`."""
|
|
409
419
|
frame = await self.aload(
|
|
410
420
|
statement=statement,
|
|
@@ -436,7 +446,6 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
436
446
|
f"Allowed fields: {sorted(self._allowed_filter_fields)}"
|
|
437
447
|
)
|
|
438
448
|
|
|
439
|
-
|
|
440
449
|
@staticmethod
|
|
441
450
|
def _perform_load_kwargs(*, request: GatewayLoadRequest | None, plan: Any) -> dict[str, Any]:
|
|
442
451
|
"""Shared kwargs for _perform_load_sync/_perform_load_async, derived
|
|
@@ -648,9 +657,7 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
648
657
|
|
|
649
658
|
# Resolve any Series values before chunked dispatch so the chunker
|
|
650
659
|
# sees plain lists (which it already knows how to split).
|
|
651
|
-
loader_options = await _series_filters.resolve_series_filters_async(
|
|
652
|
-
loader_options
|
|
653
|
-
)
|
|
660
|
+
loader_options = await _series_filters.resolve_series_filters_async(loader_options)
|
|
654
661
|
in_chunk_size, in_chunk_concurrency = self._resolve_in_chunk_controls_for_plan(
|
|
655
662
|
loader_options, plan
|
|
656
663
|
)
|
|
@@ -874,7 +881,3 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
874
881
|
)
|
|
875
882
|
kwargs[f"{on}__in"] = join_series
|
|
876
883
|
return await self.aload(**kwargs)
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
@@ -27,7 +27,9 @@ from .requests import (
|
|
|
27
27
|
)
|
|
28
28
|
|
|
29
29
|
|
|
30
|
-
def _prepare_sql_statement(
|
|
30
|
+
def _prepare_sql_statement(
|
|
31
|
+
request: SqlLoadRequest, *, logger: Any, debug: bool
|
|
32
|
+
) -> tuple[Any, dict[str, Any] | None]:
|
|
31
33
|
if request.statement is not None:
|
|
32
34
|
statement = request.statement
|
|
33
35
|
if request.columns:
|
|
@@ -61,9 +63,7 @@ def _arrow_table_from_sql_result(
|
|
|
61
63
|
if statement is None:
|
|
62
64
|
if not rows:
|
|
63
65
|
return pa.table({name: [] for name in columns})
|
|
64
|
-
return pa.Table.from_pylist(
|
|
65
|
-
[dict(zip(columns, row)) for row in rows]
|
|
66
|
-
)
|
|
66
|
+
return pa.Table.from_pylist([dict(zip(columns, row)) for row in rows])
|
|
67
67
|
|
|
68
68
|
sql_types = [getattr(selected, "type", None) for selected in statement.selected_columns]
|
|
69
69
|
schema = build_arrow_schema_from_sqlalchemy_types(columns, sql_types)
|
|
@@ -106,9 +106,7 @@ def build_sql_partitioned_request(options: dict[str, Any]) -> SqlPartitionedLoad
|
|
|
106
106
|
# Discard any extra fields that SqlPartitionedLoadRequest does not accept
|
|
107
107
|
# (e.g. dry_run, resilient — control keys passed through from the gateway).
|
|
108
108
|
partitioned_options = {
|
|
109
|
-
k: v
|
|
110
|
-
for k, v in partitioned_options.items()
|
|
111
|
-
if k in SqlPartitionedLoadRequest.model_fields
|
|
109
|
+
k: v for k, v in partitioned_options.items() if k in SqlPartitionedLoadRequest.model_fields
|
|
112
110
|
}
|
|
113
111
|
partitioned_options.setdefault("as_pandas", False)
|
|
114
112
|
partitioned_options.setdefault("partitioned", True)
|
|
@@ -164,8 +162,11 @@ def load_sql_partitioned(
|
|
|
164
162
|
resource: SqlDatabaseResource,
|
|
165
163
|
request: SqlPartitionedLoadRequest,
|
|
166
164
|
):
|
|
167
|
-
|
|
168
|
-
|
|
165
|
+
# resource is caller-owned (SqlPartitionedLoader._owns_resource is False here),
|
|
166
|
+
# so closing the loader only releases its own planner/executor state and does
|
|
167
|
+
# not touch resource — safe to close deterministically instead of relying on GC.
|
|
168
|
+
with SqlPartitionedLoader(config, resource=resource, use_arrow=request.use_arrow) as loader:
|
|
169
|
+
return loader.load_request(request)
|
|
169
170
|
|
|
170
171
|
|
|
171
172
|
async def read_sql_async(
|
|
@@ -495,9 +495,9 @@ class ParquetDataResource(SecureResource):
|
|
|
495
495
|
if modified_at is None:
|
|
496
496
|
return False
|
|
497
497
|
|
|
498
|
-
now = dt.datetime.now(dt.
|
|
498
|
+
now = dt.datetime.now(dt.UTC)
|
|
499
499
|
if modified_at.tzinfo is None:
|
|
500
|
-
modified_at = modified_at.replace(tzinfo=dt.
|
|
500
|
+
modified_at = modified_at.replace(tzinfo=dt.UTC)
|
|
501
501
|
return (now - modified_at) <= dt.timedelta(minutes=self.config.parquet_max_age_minutes)
|
|
502
502
|
|
|
503
503
|
def scan_summary(self, *, max_files: int) -> tuple[int | None, int | None]:
|
|
@@ -562,7 +562,7 @@ class ParquetDataResource(SecureResource):
|
|
|
562
562
|
if isinstance(modified, dt.datetime):
|
|
563
563
|
return modified
|
|
564
564
|
if isinstance(modified, (int, float)):
|
|
565
|
-
return dt.datetime.fromtimestamp(modified, tz=dt.
|
|
565
|
+
return dt.datetime.fromtimestamp(modified, tz=dt.UTC)
|
|
566
566
|
if isinstance(modified, str):
|
|
567
567
|
try:
|
|
568
568
|
return dt.datetime.fromisoformat(modified)
|
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import inspect
|
|
4
4
|
from collections.abc import Mapping, Sequence
|
|
5
5
|
from dataclasses import dataclass
|
|
6
|
-
from typing import Any, Protocol,
|
|
6
|
+
from typing import Any, Protocol, Union, cast
|
|
7
7
|
|
|
8
8
|
from boti.core.lifecycle import LifecycleCore
|
|
9
9
|
from boti.core.lifecycle_pickle import PicklableLifecycleCoreMixin
|
|
@@ -24,8 +24,8 @@ from boti_data.watermark import (
|
|
|
24
24
|
advance_watermark,
|
|
25
25
|
)
|
|
26
26
|
|
|
27
|
-
PipelineSource
|
|
28
|
-
PipelineDestination
|
|
27
|
+
type PipelineSource = Union[DataHelper, HybridDataset]
|
|
28
|
+
type PipelineDestination = ParquetDestination
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
class FrameEnricher(Protocol):
|
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import asyncio
|
|
4
4
|
from collections.abc import Callable, Mapping, Sequence
|
|
5
5
|
from dataclasses import dataclass
|
|
6
|
-
from typing import Any, Protocol,
|
|
6
|
+
from typing import Any, Protocol, Union, cast
|
|
7
7
|
from urllib.parse import urlparse
|
|
8
8
|
|
|
9
9
|
import dask.dataframe as dd
|
|
@@ -19,8 +19,8 @@ from pydantic import Field, field_validator
|
|
|
19
19
|
|
|
20
20
|
from boti_data.parquet import ParquetDataConfig, ParquetReader
|
|
21
21
|
|
|
22
|
-
FrameResult
|
|
23
|
-
ParquetDestination
|
|
22
|
+
type FrameResult = Union[pd.DataFrame, dd.DataFrame, pa.Table, pl.DataFrame]
|
|
23
|
+
type ParquetDestination = Union[ParquetReader, ParquetDataConfig, Mapping[str, Any]]
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
def to_dask_frame(frame: FrameResult) -> dd.DataFrame:
|
|
@@ -57,9 +57,21 @@ def prepare_partitioned_frame(
|
|
|
57
57
|
f"{sink_name} expected date_field={date_field!r} in loaded frame columns "
|
|
58
58
|
f"but only found {list(frame.columns)!r}."
|
|
59
59
|
)
|
|
60
|
+
pre_assign_dtypes = frame.dtypes
|
|
60
61
|
frame = frame.assign(
|
|
61
62
|
partition_date=dd.to_datetime(frame[date_field], errors="coerce").dt.date.astype(str)
|
|
62
63
|
)
|
|
64
|
+
# dask-expr's query optimizer can re-derive _meta for the whole frame when
|
|
65
|
+
# .assign() adds a column, silently dropping explicit per-column dtype hints
|
|
66
|
+
# (e.g. bool cast via map_partitions upstream) in favor of generic `object`.
|
|
67
|
+
# Restore any dtype that changed as an unintended side effect of the assign.
|
|
68
|
+
changed_dtypes = {
|
|
69
|
+
column: dtype
|
|
70
|
+
for column, dtype in pre_assign_dtypes.items()
|
|
71
|
+
if column in frame.columns and frame.dtypes[column] != dtype
|
|
72
|
+
}
|
|
73
|
+
if changed_dtypes:
|
|
74
|
+
frame = frame.astype(changed_dtypes)
|
|
63
75
|
missing = [name for name in partition_columns if name not in frame.columns]
|
|
64
76
|
|
|
65
77
|
if missing:
|
|
@@ -217,6 +229,13 @@ class CsvSink(SecureResource):
|
|
|
217
229
|
overwrite: bool = True,
|
|
218
230
|
persist: bool = False,
|
|
219
231
|
) -> SinkWriteResult:
|
|
232
|
+
"""Write *frame* to the configured CSV destination.
|
|
233
|
+
|
|
234
|
+
``overwrite=True`` replaces the *entire* target directory (all
|
|
235
|
+
partitions/dates), not just the partition(s) present in *frame* — see
|
|
236
|
+
:func:`_write_with_staging`. For incremental per-date writes, do not
|
|
237
|
+
rely on ``overwrite=True`` to mean "overwrite this date only".
|
|
238
|
+
"""
|
|
220
239
|
ddf = to_dask_frame(frame)
|
|
221
240
|
ddf = prepare_partitioned_frame(
|
|
222
241
|
ddf,
|
|
@@ -381,6 +400,13 @@ class JsonlSink(SecureResource):
|
|
|
381
400
|
overwrite: bool = True,
|
|
382
401
|
persist: bool = False,
|
|
383
402
|
) -> SinkWriteResult:
|
|
403
|
+
"""Write *frame* to the configured JSONL destination.
|
|
404
|
+
|
|
405
|
+
``overwrite=True`` replaces the *entire* target directory (all
|
|
406
|
+
partitions/dates), not just the partition(s) present in *frame* — see
|
|
407
|
+
:func:`_write_with_staging`. For incremental per-date writes, do not
|
|
408
|
+
rely on ``overwrite=True`` to mean "overwrite this date only".
|
|
409
|
+
"""
|
|
384
410
|
ddf = to_dask_frame(frame)
|
|
385
411
|
ddf = prepare_partitioned_frame(
|
|
386
412
|
ddf,
|
|
@@ -558,6 +584,13 @@ class ParquetSink(PicklableLifecycleCoreMixin, LifecycleCore):
|
|
|
558
584
|
overwrite: bool = True,
|
|
559
585
|
persist: bool = False,
|
|
560
586
|
) -> SinkWriteResult:
|
|
587
|
+
"""Write *frame* to the configured parquet destination.
|
|
588
|
+
|
|
589
|
+
``overwrite=True`` replaces the *entire* target directory (all
|
|
590
|
+
partitions/dates), not just the partition(s) present in *frame* — see
|
|
591
|
+
:func:`_write_with_staging`. For incremental per-date writes, do not
|
|
592
|
+
rely on ``overwrite=True`` to mean "overwrite this date only".
|
|
593
|
+
"""
|
|
561
594
|
ddf = to_dask_frame(frame)
|
|
562
595
|
ddf = prepare_partitioned_frame(
|
|
563
596
|
ddf,
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: boti-data
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.6
|
|
4
4
|
Summary: Data infrastructure for the Boti ecosystem
|
|
5
|
-
Author
|
|
5
|
+
Author: Luis Valverde
|
|
6
|
+
Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>
|
|
6
7
|
License-Expression: MIT
|
|
7
8
|
Project-URL: Homepage, https://github.com/lvalverdeb/boti-data
|
|
8
9
|
Project-URL: Repository, https://github.com/lvalverdeb/boti-data
|
|
@@ -11,12 +12,10 @@ Project-URL: Issues, https://github.com/lvalverdeb/boti-data/issues
|
|
|
11
12
|
Classifier: Development Status :: 3 - Alpha
|
|
12
13
|
Classifier: Intended Audience :: Developers
|
|
13
14
|
Classifier: Programming Language :: Python :: 3
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
17
15
|
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
17
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
-
Requires-Python: >=3.
|
|
18
|
+
Requires-Python: >=3.13
|
|
20
19
|
Description-Content-Type: text/markdown
|
|
21
20
|
Requires-Dist: asyncmy<0.3.0,>=0.2.11
|
|
22
21
|
Requires-Dist: boti<2.0.0,>=1.0.2
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|