boti-data 1.0.2__tar.gz → 1.0.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {boti_data-1.0.2 → boti_data-1.0.6}/PKG-INFO +5 -6
  2. {boti_data-1.0.2 → boti_data-1.0.6}/pyproject.toml +7 -10
  3. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/datacube/artifact.py +12 -6
  4. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/datacube/base.py +5 -5
  5. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_engine.py +13 -1
  6. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/_backend_strategies.py +11 -12
  7. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/core.py +16 -13
  8. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/loaders.py +10 -9
  9. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/parquet/resource.py +3 -3
  10. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/pipelines/base.py +3 -3
  11. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/pipelines/sinks.py +36 -3
  12. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data.egg-info/PKG-INFO +5 -6
  13. {boti_data-1.0.2 → boti_data-1.0.6}/README.md +0 -0
  14. {boti_data-1.0.2 → boti_data-1.0.6}/setup.cfg +0 -0
  15. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/__init__.py +0 -0
  16. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/arrow_schema.py +0 -0
  17. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/connection_catalog.py +0 -0
  18. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/datacube/__init__.py +0 -0
  19. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/datacube/contract.py +0 -0
  20. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/datacube/resource.py +0 -0
  21. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/dataset/__init__.py +0 -0
  22. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/dataset/hybrid.py +0 -0
  23. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/__init__.py +0 -0
  24. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/arrow_schema_mapper.py +0 -0
  25. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/engine_registry.py +0 -0
  26. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/partitioned_execution.py +0 -0
  27. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/partitioned_loader.py +0 -0
  28. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/partitioned_planner.py +0 -0
  29. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/partitioned_types.py +0 -0
  30. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_config.py +0 -0
  31. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_manager.py +0 -0
  32. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_model_builder.py +0 -0
  33. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_model_registry.py +0 -0
  34. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_readonly.py +0 -0
  35. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sql_resource.py +0 -0
  36. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/db/sqlalchemy_async.py +0 -0
  37. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/__init__.py +0 -0
  38. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/async_enricher.py +0 -0
  39. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/composite.py +0 -0
  40. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/protocol.py +0 -0
  41. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/specs.py +0 -0
  42. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/enrichment/transformers.py +0 -0
  43. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/field_map.py +0 -0
  44. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/__init__.py +0 -0
  45. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/arrow_kernels.py +0 -0
  46. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/expressions.py +0 -0
  47. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/handler.py +0 -0
  48. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/parquet_compiler.py +0 -0
  49. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/filters/utils.py +0 -0
  50. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/__init__.py +0 -0
  51. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/_factory.py +0 -0
  52. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/_payloads.py +0 -0
  53. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/_series_filters.py +0 -0
  54. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/arrow_adapters.py +0 -0
  55. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/chunking.py +0 -0
  56. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/configured_load.py +0 -0
  57. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/frame_strategies.py +0 -0
  58. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/incremental.py +0 -0
  59. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/load_request.py +0 -0
  60. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/normalization.py +0 -0
  61. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/planning.py +0 -0
  62. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/policies.py +0 -0
  63. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/post_process.py +0 -0
  64. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/requests.py +0 -0
  65. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/return_type.py +0 -0
  66. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/gateway/sql_guard.py +0 -0
  67. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/helper.py +0 -0
  68. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/joins.py +0 -0
  69. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/parquet/__init__.py +0 -0
  70. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/parquet/file_info.py +0 -0
  71. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/parquet/reader.py +0 -0
  72. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/parquet/reader_config.py +0 -0
  73. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/pipelines/__init__.py +0 -0
  74. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/pipelines/registry.py +0 -0
  75. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/py.typed +0 -0
  76. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/schema.py +0 -0
  77. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/watermark/__init__.py +0 -0
  78. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/watermark/incremental.py +0 -0
  79. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data/watermark/store.py +0 -0
  80. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data.egg-info/SOURCES.txt +0 -0
  81. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data.egg-info/dependency_links.txt +0 -0
  82. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data.egg-info/requires.txt +0 -0
  83. {boti_data-1.0.2 → boti_data-1.0.6}/src/boti_data.egg-info/top_level.txt +0 -0
@@ -1,8 +1,9 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: boti-data
3
- Version: 1.0.2
3
+ Version: 1.0.6
4
4
  Summary: Data infrastructure for the Boti ecosystem
5
- Author-email: Luis Valverde <lvalverdeb@gmail.com>
5
+ Author: Luis Valverde
6
+ Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>
6
7
  License-Expression: MIT
7
8
  Project-URL: Homepage, https://github.com/lvalverdeb/boti-data
8
9
  Project-URL: Repository, https://github.com/lvalverdeb/boti-data
@@ -11,12 +12,10 @@ Project-URL: Issues, https://github.com/lvalverdeb/boti-data/issues
11
12
  Classifier: Development Status :: 3 - Alpha
12
13
  Classifier: Intended Audience :: Developers
13
14
  Classifier: Programming Language :: Python :: 3
14
- Classifier: Programming Language :: Python :: 3.10
15
- Classifier: Programming Language :: Python :: 3.11
16
- Classifier: Programming Language :: Python :: 3.12
17
15
  Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Python :: 3 :: Only
18
17
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
19
- Requires-Python: >=3.10
18
+ Requires-Python: >=3.13
20
19
  Description-Content-Type: text/markdown
21
20
  Requires-Dist: asyncmy<0.3.0,>=0.2.11
22
21
  Requires-Dist: boti<2.0.0,>=1.0.2
@@ -1,21 +1,22 @@
1
1
  [project]
2
2
  name = "boti-data"
3
- version = "1.0.2"
3
+ version = "1.0.6"
4
4
  description = "Data infrastructure for the Boti ecosystem"
5
5
  readme = "README.md"
6
- requires-python = ">=3.10"
6
+ requires-python = ">=3.13"
7
7
  license = "MIT"
8
8
  authors = [
9
+ {name = "Luis Valverde"}
10
+ ]
11
+ maintainers = [
9
12
  {name = "Luis Valverde", email = "lvalverdeb@gmail.com"}
10
13
  ]
11
14
  classifiers = [
12
15
  "Development Status :: 3 - Alpha",
13
16
  "Intended Audience :: Developers",
14
17
  "Programming Language :: Python :: 3",
15
- "Programming Language :: Python :: 3.10",
16
- "Programming Language :: Python :: 3.11",
17
- "Programming Language :: Python :: 3.12",
18
18
  "Programming Language :: Python :: 3.13",
19
+ "Programming Language :: Python :: 3 :: Only",
19
20
  "Topic :: Software Development :: Libraries :: Python Modules",
20
21
  ]
21
22
  dependencies = [
@@ -51,7 +52,7 @@ dev = [
51
52
 
52
53
  [tool.ruff]
53
54
  line-length = 100
54
- target-version = "py310"
55
+ target-version = "py313"
55
56
 
56
57
  [tool.ruff.lint]
57
58
  select = ["E", "F", "W", "I", "UP"]
@@ -75,10 +76,6 @@ filterwarnings = [
75
76
  'ignore:Closing managed Dask session with 1 live persisted collection\(s\).*Missing dependency\.:RuntimeWarning',
76
77
  ]
77
78
 
78
- [tool.uv.sources]
79
- boti = { workspace = true }
80
- boti-dask = { workspace = true }
81
-
82
79
  [tool.setuptools]
83
80
  include-package-data = true
84
81
 
@@ -84,13 +84,17 @@ class BaseArtifact(PicklableLifecycleCoreMixin, LifecycleCore):
84
84
 
85
85
  merged = {**self.config, **kwargs}
86
86
  self._helper: DataHelper = DataHelper(merged)
87
- self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
87
+ self.logger = kwargs.get("logger") or Logger.default_logger(
88
+ logger_name=self.__class__.__name__
89
+ )
88
90
 
89
91
  self.parquet_start_date: str | None = _validate_and_format_date(
90
- "parquet_start_date", merged.get("parquet_start_date"),
92
+ "parquet_start_date",
93
+ merged.get("parquet_start_date"),
91
94
  )
92
95
  self.parquet_end_date: str | None = _validate_and_format_date(
93
- "parquet_end_date", merged.get("parquet_end_date"),
96
+ "parquet_end_date",
97
+ merged.get("parquet_end_date"),
94
98
  )
95
99
  self.data_wrapper_class: type[Any] | None = merged.get("data_wrapper_class")
96
100
  self.class_params: dict[str, Any] = merged.get("class_params") or {
@@ -106,14 +110,16 @@ class BaseArtifact(PicklableLifecycleCoreMixin, LifecycleCore):
106
110
  instance.df = None
107
111
  instance._helper = helper
108
112
  instance.config = {**cls.config, **overrides}
109
- instance.logger = Logger.default_logger(logger_name=cls.__name__)
113
+ instance.logger = overrides.get("logger") or Logger.default_logger(logger_name=cls.__name__)
110
114
 
111
115
  merged = {**cls.config, **overrides}
112
116
  instance.parquet_start_date = _validate_and_format_date(
113
- "parquet_start_date", merged.get("parquet_start_date"),
117
+ "parquet_start_date",
118
+ merged.get("parquet_start_date"),
114
119
  )
115
120
  instance.parquet_end_date = _validate_and_format_date(
116
- "parquet_end_date", merged.get("parquet_end_date"),
121
+ "parquet_end_date",
122
+ merged.get("parquet_end_date"),
117
123
  )
118
124
  instance.data_wrapper_class = merged.get("data_wrapper_class")
119
125
  instance.class_params = merged.get("class_params") or {
@@ -60,7 +60,9 @@ class BaseDataCube(PicklableLifecycleCoreMixin, LifecycleCore):
60
60
 
61
61
  merged = {**self.config, **kwargs}
62
62
  self._helper: DataHelper = DataHelper(merged)
63
- self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
63
+ self.logger = kwargs.get("logger") or Logger.default_logger(
64
+ logger_name=self.__class__.__name__
65
+ )
64
66
  super().__init__()
65
67
 
66
68
  @classmethod
@@ -70,7 +72,7 @@ class BaseDataCube(PicklableLifecycleCoreMixin, LifecycleCore):
70
72
  instance.df = None
71
73
  instance._helper = helper
72
74
  instance.config = {**cls.config, **overrides}
73
- instance.logger = Logger.default_logger(logger_name=cls.__name__)
75
+ instance.logger = overrides.get("logger") or Logger.default_logger(logger_name=cls.__name__)
74
76
  # __new__() bypasses __init__(), so LifecycleCore's state (_state_lock,
75
77
  # _is_closed, the GC finalizer, etc.) must be wired up explicitly here.
76
78
  LifecycleCore.__init__(instance)
@@ -111,9 +113,7 @@ class BaseDataCube(PicklableLifecycleCoreMixin, LifecycleCore):
111
113
  return self.__class__.fix_data is not BaseDataCube.fix_data
112
114
 
113
115
  def _log_no_data(self) -> None:
114
- self.logger.debug(
115
- "No data was found by %s loader", self.__class__.__name__
116
- )
116
+ self.logger.debug("No data was found by %s loader", self.__class__.__name__)
117
117
 
118
118
  # ── public API ──────────────────────────────────────────────────────
119
119
 
@@ -34,12 +34,24 @@ def _parse_connection_url(
34
34
  normalized_url = _normalize_connection_url(connection_url)
35
35
  parsed = sqlalchemy_url.make_url(normalized_url)
36
36
  try:
37
- parsed.get_dialect()
37
+ dialect_cls = parsed.get_dialect()
38
38
  except NoSuchModuleError as exc:
39
39
  raise SQLAlchemyError(
40
40
  f"{consumer_name} received unsupported SQLAlchemy dialect '{parsed.drivername}'. "
41
41
  "Check the driver name in the connection URL."
42
42
  ) from exc
43
+ # get_dialect() only resolves the SQLAlchemy dialect class; the DBAPI driver
44
+ # package itself (e.g. psycopg2, pymysql) is imported lazily and would
45
+ # otherwise only fail deep inside create_engine()/first connect().
46
+ try:
47
+ dialect_cls.import_dbapi()
48
+ except (ModuleNotFoundError, ImportError) as exc:
49
+ raise SQLAlchemyError(
50
+ f"{consumer_name} connection_url uses dialect '{parsed.drivername}' but its DBAPI "
51
+ f"driver package is not installed ({exc}). Install the driver package for this "
52
+ "dialect (e.g. 'psycopg2-binary' for 'postgresql+psycopg2', 'pymysql' for "
53
+ "'mysql+pymysql') before constructing this resource."
54
+ ) from exc
43
55
  return parsed
44
56
 
45
57
 
@@ -116,8 +116,7 @@ _FALLBACK_DTYPE_BYTES: int = 16
116
116
  def _estimate_bytes_per_row(meta_dtypes: dict[str, str]) -> int:
117
117
  """Estimate the in-memory footprint of one result row from its column dtypes."""
118
118
  total = sum(
119
- _DTYPE_BYTE_WEIGHTS.get(dtype, _FALLBACK_DTYPE_BYTES)
120
- for dtype in meta_dtypes.values()
119
+ _DTYPE_BYTE_WEIGHTS.get(dtype, _FALLBACK_DTYPE_BYTES) for dtype in meta_dtypes.values()
121
120
  )
122
121
  return max(1, total)
123
122
 
@@ -388,7 +387,12 @@ class SqlAlchemyStrategy(BackendStrategy):
388
387
  cfg = dict(cfg)
389
388
  raw_url = cfg.pop("connection_url", None)
390
389
  if raw_url is None:
391
- raise ValueError("from_config requires 'connection_url'.")
390
+ raise ValueError(
391
+ "from_config requires 'connection_url' for the 'sqlalchemy' backend "
392
+ "(the default used when no 'backend' key is present in config). "
393
+ "If you intended a different backend, pass an explicit 'backend' key "
394
+ "(e.g. 'parquet', 'datacube')."
395
+ )
392
396
  connection_url = SecretStr(raw_url) if isinstance(raw_url, str) else raw_url
393
397
  return SqlDatabaseConfig(connection_url=connection_url, **cfg)
394
398
 
@@ -620,7 +624,8 @@ class SqlAlchemyStrategy(BackendStrategy):
620
624
  t_coerce = perf_counter()
621
625
  if len(df) <= single_fetch_ceiling:
622
626
  df = SqlPartitionExecutor.align_and_coerce_partition(
623
- df, meta_dtypes,
627
+ df,
628
+ meta_dtypes,
624
629
  )
625
630
  t_dask = perf_counter()
626
631
  result = dd.from_pandas(df, npartitions=1)
@@ -808,10 +813,7 @@ class SqlAlchemyStrategy(BackendStrategy):
808
813
  )
809
814
  t1 = perf_counter()
810
815
  if diagnostics:
811
- resource.logger.info(
812
- "Configured async select "
813
- f"reflect_select={t1 - t0:.3f}s"
814
- )
816
+ resource.logger.info(f"Configured async select reflect_select={t1 - t0:.3f}s")
815
817
  if ctx.execution_mode == "lazy":
816
818
  t2 = perf_counter()
817
819
  partitioned_options = build_partitioned_load_options(
@@ -826,10 +828,7 @@ class SqlAlchemyStrategy(BackendStrategy):
826
828
  request = build_sql_partitioned_request(options_dict)
827
829
  t3 = perf_counter()
828
830
  if diagnostics:
829
- resource.logger.info(
830
- "Build partitioned request "
831
- f"elapsed={t3 - t2:.3f}s"
832
- )
831
+ resource.logger.info(f"Build partitioned request elapsed={t3 - t2:.3f}s")
833
832
  return await self._run_async_partitioned_request(ctx, resource, request)
834
833
 
835
834
  df = await read_sql_async(
@@ -139,7 +139,9 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
139
139
  self._return_type: ReturnType = self._df_params.return_type
140
140
  self._execution_mode: ExecutionMode = self._df_params.execution_mode
141
141
  self._configured_select_cache: OrderedDict[tuple[str, ...], tuple[Any, Any]] = OrderedDict()
142
- self._configured_async_select_cache: OrderedDict[tuple[str, ...], tuple[Any, Any]] = OrderedDict()
142
+ self._configured_async_select_cache: OrderedDict[tuple[str, ...], tuple[Any, Any]] = (
143
+ OrderedDict()
144
+ )
143
145
  self._raw_sql_policy = raw_sql_policy
144
146
  self._policies = policies or GatewayPolicies()
145
147
  self._strict_filter_validation = strict_filter_validation
@@ -225,7 +227,6 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
225
227
  """
226
228
  return self._logger
227
229
 
228
-
229
230
  def _load_planner(self) -> LoadPlanner:
230
231
  return LoadPlanner(
231
232
  configured=self._configured,
@@ -354,7 +355,12 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
354
355
  provided.
355
356
  """
356
357
  cfg = dict(config)
357
- cfg.update(overrides)
358
+ # Wrapper classes (e.g. ParquetReader -> DataHelper) always forward
359
+ # fs=None/fs_factory=None when the caller didn't explicitly pass them,
360
+ # rather than omitting the keys. A blind cfg.update(overrides) would let
361
+ # that incidental None clobber a real fs embedded directly in the config
362
+ # mapping, so only non-None overrides are allowed to win.
363
+ cfg.update({key: value for key, value in overrides.items() if value is not None})
358
364
 
359
365
  backend, common = _factory.extract_config_common_options(cfg)
360
366
  strategy = get_strategy(backend)
@@ -389,7 +395,9 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
389
395
  # Load API — structured mode
390
396
  # ------------------------------------------------------------------
391
397
 
392
- def preview(self, statement: Any, model: Any, n: int = 5, npartitions: int = 1, **options: Any) -> pd.DataFrame:
398
+ def preview(
399
+ self, statement: Any, model: Any, n: int = 5, npartitions: int = 1, **options: Any
400
+ ) -> pd.DataFrame:
393
401
  """Load a preview (first *n* rows) as a pandas DataFrame.
394
402
 
395
403
  Leverages lazy loading + ``safe_head`` so a distributed client is used
@@ -404,7 +412,9 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
404
412
  )
405
413
  return safe_head(frame, n=n, npartitions=npartitions)
406
414
 
407
- async def apreview(self, statement: Any, model: Any, n: int = 5, npartitions: int = 1, **options: Any) -> pd.DataFrame:
415
+ async def apreview(
416
+ self, statement: Any, model: Any, n: int = 5, npartitions: int = 1, **options: Any
417
+ ) -> pd.DataFrame:
408
418
  """Async version of :meth:`preview`."""
409
419
  frame = await self.aload(
410
420
  statement=statement,
@@ -436,7 +446,6 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
436
446
  f"Allowed fields: {sorted(self._allowed_filter_fields)}"
437
447
  )
438
448
 
439
-
440
449
  @staticmethod
441
450
  def _perform_load_kwargs(*, request: GatewayLoadRequest | None, plan: Any) -> dict[str, Any]:
442
451
  """Shared kwargs for _perform_load_sync/_perform_load_async, derived
@@ -648,9 +657,7 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
648
657
 
649
658
  # Resolve any Series values before chunked dispatch so the chunker
650
659
  # sees plain lists (which it already knows how to split).
651
- loader_options = await _series_filters.resolve_series_filters_async(
652
- loader_options
653
- )
660
+ loader_options = await _series_filters.resolve_series_filters_async(loader_options)
654
661
  in_chunk_size, in_chunk_concurrency = self._resolve_in_chunk_controls_for_plan(
655
662
  loader_options, plan
656
663
  )
@@ -874,7 +881,3 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
874
881
  )
875
882
  kwargs[f"{on}__in"] = join_series
876
883
  return await self.aload(**kwargs)
877
-
878
-
879
-
880
-
@@ -27,7 +27,9 @@ from .requests import (
27
27
  )
28
28
 
29
29
 
30
- def _prepare_sql_statement(request: SqlLoadRequest, *, logger: Any, debug: bool) -> tuple[Any, dict[str, Any] | None]:
30
+ def _prepare_sql_statement(
31
+ request: SqlLoadRequest, *, logger: Any, debug: bool
32
+ ) -> tuple[Any, dict[str, Any] | None]:
31
33
  if request.statement is not None:
32
34
  statement = request.statement
33
35
  if request.columns:
@@ -61,9 +63,7 @@ def _arrow_table_from_sql_result(
61
63
  if statement is None:
62
64
  if not rows:
63
65
  return pa.table({name: [] for name in columns})
64
- return pa.Table.from_pylist(
65
- [dict(zip(columns, row)) for row in rows]
66
- )
66
+ return pa.Table.from_pylist([dict(zip(columns, row)) for row in rows])
67
67
 
68
68
  sql_types = [getattr(selected, "type", None) for selected in statement.selected_columns]
69
69
  schema = build_arrow_schema_from_sqlalchemy_types(columns, sql_types)
@@ -106,9 +106,7 @@ def build_sql_partitioned_request(options: dict[str, Any]) -> SqlPartitionedLoad
106
106
  # Discard any extra fields that SqlPartitionedLoadRequest does not accept
107
107
  # (e.g. dry_run, resilient — control keys passed through from the gateway).
108
108
  partitioned_options = {
109
- k: v
110
- for k, v in partitioned_options.items()
111
- if k in SqlPartitionedLoadRequest.model_fields
109
+ k: v for k, v in partitioned_options.items() if k in SqlPartitionedLoadRequest.model_fields
112
110
  }
113
111
  partitioned_options.setdefault("as_pandas", False)
114
112
  partitioned_options.setdefault("partitioned", True)
@@ -164,8 +162,11 @@ def load_sql_partitioned(
164
162
  resource: SqlDatabaseResource,
165
163
  request: SqlPartitionedLoadRequest,
166
164
  ):
167
- loader = SqlPartitionedLoader(config, resource=resource, use_arrow=request.use_arrow)
168
- return loader.load_request(request)
165
+ # resource is caller-owned (SqlPartitionedLoader._owns_resource is False here),
166
+ # so closing the loader only releases its own planner/executor state and does
167
+ # not touch resource — safe to close deterministically instead of relying on GC.
168
+ with SqlPartitionedLoader(config, resource=resource, use_arrow=request.use_arrow) as loader:
169
+ return loader.load_request(request)
169
170
 
170
171
 
171
172
  async def read_sql_async(
@@ -495,9 +495,9 @@ class ParquetDataResource(SecureResource):
495
495
  if modified_at is None:
496
496
  return False
497
497
 
498
- now = dt.datetime.now(dt.timezone.utc)
498
+ now = dt.datetime.now(dt.UTC)
499
499
  if modified_at.tzinfo is None:
500
- modified_at = modified_at.replace(tzinfo=dt.timezone.utc)
500
+ modified_at = modified_at.replace(tzinfo=dt.UTC)
501
501
  return (now - modified_at) <= dt.timedelta(minutes=self.config.parquet_max_age_minutes)
502
502
 
503
503
  def scan_summary(self, *, max_files: int) -> tuple[int | None, int | None]:
@@ -562,7 +562,7 @@ class ParquetDataResource(SecureResource):
562
562
  if isinstance(modified, dt.datetime):
563
563
  return modified
564
564
  if isinstance(modified, (int, float)):
565
- return dt.datetime.fromtimestamp(modified, tz=dt.timezone.utc)
565
+ return dt.datetime.fromtimestamp(modified, tz=dt.UTC)
566
566
  if isinstance(modified, str):
567
567
  try:
568
568
  return dt.datetime.fromisoformat(modified)
@@ -3,7 +3,7 @@ from __future__ import annotations
3
3
  import inspect
4
4
  from collections.abc import Mapping, Sequence
5
5
  from dataclasses import dataclass
6
- from typing import Any, Protocol, TypeAlias, Union, cast
6
+ from typing import Any, Protocol, Union, cast
7
7
 
8
8
  from boti.core.lifecycle import LifecycleCore
9
9
  from boti.core.lifecycle_pickle import PicklableLifecycleCoreMixin
@@ -24,8 +24,8 @@ from boti_data.watermark import (
24
24
  advance_watermark,
25
25
  )
26
26
 
27
- PipelineSource: TypeAlias = Union[DataHelper, HybridDataset]
28
- PipelineDestination: TypeAlias = ParquetDestination
27
+ type PipelineSource = Union[DataHelper, HybridDataset]
28
+ type PipelineDestination = ParquetDestination
29
29
 
30
30
 
31
31
  class FrameEnricher(Protocol):
@@ -3,7 +3,7 @@ from __future__ import annotations
3
3
  import asyncio
4
4
  from collections.abc import Callable, Mapping, Sequence
5
5
  from dataclasses import dataclass
6
- from typing import Any, Protocol, TypeAlias, Union, cast
6
+ from typing import Any, Protocol, Union, cast
7
7
  from urllib.parse import urlparse
8
8
 
9
9
  import dask.dataframe as dd
@@ -19,8 +19,8 @@ from pydantic import Field, field_validator
19
19
 
20
20
  from boti_data.parquet import ParquetDataConfig, ParquetReader
21
21
 
22
- FrameResult: TypeAlias = Union[pd.DataFrame, dd.DataFrame, pa.Table, pl.DataFrame]
23
- ParquetDestination: TypeAlias = Union[ParquetReader, ParquetDataConfig, Mapping[str, Any]]
22
+ type FrameResult = Union[pd.DataFrame, dd.DataFrame, pa.Table, pl.DataFrame]
23
+ type ParquetDestination = Union[ParquetReader, ParquetDataConfig, Mapping[str, Any]]
24
24
 
25
25
 
26
26
  def to_dask_frame(frame: FrameResult) -> dd.DataFrame:
@@ -57,9 +57,21 @@ def prepare_partitioned_frame(
57
57
  f"{sink_name} expected date_field={date_field!r} in loaded frame columns "
58
58
  f"but only found {list(frame.columns)!r}."
59
59
  )
60
+ pre_assign_dtypes = frame.dtypes
60
61
  frame = frame.assign(
61
62
  partition_date=dd.to_datetime(frame[date_field], errors="coerce").dt.date.astype(str)
62
63
  )
64
+ # dask-expr's query optimizer can re-derive _meta for the whole frame when
65
+ # .assign() adds a column, silently dropping explicit per-column dtype hints
66
+ # (e.g. bool cast via map_partitions upstream) in favor of generic `object`.
67
+ # Restore any dtype that changed as an unintended side effect of the assign.
68
+ changed_dtypes = {
69
+ column: dtype
70
+ for column, dtype in pre_assign_dtypes.items()
71
+ if column in frame.columns and frame.dtypes[column] != dtype
72
+ }
73
+ if changed_dtypes:
74
+ frame = frame.astype(changed_dtypes)
63
75
  missing = [name for name in partition_columns if name not in frame.columns]
64
76
 
65
77
  if missing:
@@ -217,6 +229,13 @@ class CsvSink(SecureResource):
217
229
  overwrite: bool = True,
218
230
  persist: bool = False,
219
231
  ) -> SinkWriteResult:
232
+ """Write *frame* to the configured CSV destination.
233
+
234
+ ``overwrite=True`` replaces the *entire* target directory (all
235
+ partitions/dates), not just the partition(s) present in *frame* — see
236
+ :func:`_write_with_staging`. For incremental per-date writes, do not
237
+ rely on ``overwrite=True`` to mean "overwrite this date only".
238
+ """
220
239
  ddf = to_dask_frame(frame)
221
240
  ddf = prepare_partitioned_frame(
222
241
  ddf,
@@ -381,6 +400,13 @@ class JsonlSink(SecureResource):
381
400
  overwrite: bool = True,
382
401
  persist: bool = False,
383
402
  ) -> SinkWriteResult:
403
+ """Write *frame* to the configured JSONL destination.
404
+
405
+ ``overwrite=True`` replaces the *entire* target directory (all
406
+ partitions/dates), not just the partition(s) present in *frame* — see
407
+ :func:`_write_with_staging`. For incremental per-date writes, do not
408
+ rely on ``overwrite=True`` to mean "overwrite this date only".
409
+ """
384
410
  ddf = to_dask_frame(frame)
385
411
  ddf = prepare_partitioned_frame(
386
412
  ddf,
@@ -558,6 +584,13 @@ class ParquetSink(PicklableLifecycleCoreMixin, LifecycleCore):
558
584
  overwrite: bool = True,
559
585
  persist: bool = False,
560
586
  ) -> SinkWriteResult:
587
+ """Write *frame* to the configured parquet destination.
588
+
589
+ ``overwrite=True`` replaces the *entire* target directory (all
590
+ partitions/dates), not just the partition(s) present in *frame* — see
591
+ :func:`_write_with_staging`. For incremental per-date writes, do not
592
+ rely on ``overwrite=True`` to mean "overwrite this date only".
593
+ """
561
594
  ddf = to_dask_frame(frame)
562
595
  ddf = prepare_partitioned_frame(
563
596
  ddf,
@@ -1,8 +1,9 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: boti-data
3
- Version: 1.0.2
3
+ Version: 1.0.6
4
4
  Summary: Data infrastructure for the Boti ecosystem
5
- Author-email: Luis Valverde <lvalverdeb@gmail.com>
5
+ Author: Luis Valverde
6
+ Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>
6
7
  License-Expression: MIT
7
8
  Project-URL: Homepage, https://github.com/lvalverdeb/boti-data
8
9
  Project-URL: Repository, https://github.com/lvalverdeb/boti-data
@@ -11,12 +12,10 @@ Project-URL: Issues, https://github.com/lvalverdeb/boti-data/issues
11
12
  Classifier: Development Status :: 3 - Alpha
12
13
  Classifier: Intended Audience :: Developers
13
14
  Classifier: Programming Language :: Python :: 3
14
- Classifier: Programming Language :: Python :: 3.10
15
- Classifier: Programming Language :: Python :: 3.11
16
- Classifier: Programming Language :: Python :: 3.12
17
15
  Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Python :: 3 :: Only
18
17
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
19
- Requires-Python: >=3.10
18
+ Requires-Python: >=3.13
20
19
  Description-Content-Type: text/markdown
21
20
  Requires-Dist: asyncmy<0.3.0,>=0.2.11
22
21
  Requires-Dist: boti<2.0.0,>=1.0.2
File without changes
File without changes