boti-data 1.3.2__tar.gz → 1.3.4__tar.gz

{boti_data-1.3.2 → boti_data-1.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boti-data
-Version: 1.3.2
+Version: 1.3.4
 Summary: Data infrastructure for the Boti ecosystem
 Author: Luis Valverde
 Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>

{boti_data-1.3.2 → boti_data-1.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "boti-data"
-version = "1.3.2"
+version = "1.3.4"
 description = "Data infrastructure for the Boti ecosystem"
 readme = "README.md"
 requires-python = ">=3.13"

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/__init__.py

@@ -63,6 +63,7 @@ __all__ = [
     "SqlModelRegistry",
     "SqlRepository",
     "SqlUnitOfWork",
+    "TableDescription",
     "TrueExpr",
     "VectorMetric",
     "WatermarkStore",
@@ -70,6 +71,7 @@ __all__ = [
     "align_frames_for_join",
     "apply_schema_map",
     "available_sinks",
+    "awrite_parquet",
     "build_incremental_filters",
     "create_sink",
     "ensure_greenlet_available",
@@ -83,6 +85,7 @@ __all__ = [
     "register_sink",
     "validate_schema",
     "vector_distance",
+    "write_parquet",
 ]
 
 # Every public name is loaded lazily (PEP 562) so that importing boti_data (or
@@ -132,6 +135,7 @@ _LAZY = {
     "SqlLoadRequest": ("boti_data.gateway", "SqlLoadRequest"),
     "DataFrameOptions": ("boti_data.gateway", "DataFrameOptions"),
     "DataFrameParams": ("boti_data.gateway", "DataFrameParams"),
+    "TableDescription": ("boti_data.gateway", "TableDescription"),
     "DataHelper": ("boti_data.helper", "DataHelper"),
     "indexed_left_join": ("boti_data.joins", "indexed_left_join"),
     "left_join_frames": ("boti_data.joins", "left_join_frames"),
@@ -161,8 +165,10 @@ _LAZY = {
     "SinkRegistry": ("boti_data.pipelines", "SinkRegistry"),
     "SinkWriteResult": ("boti_data.pipelines", "SinkWriteResult"),
     "available_sinks": ("boti_data.pipelines", "available_sinks"),
+    "awrite_parquet": ("boti_data.pipelines", "awrite_parquet"),
     "create_sink": ("boti_data.pipelines", "create_sink"),
     "register_sink": ("boti_data.pipelines", "register_sink"),
+    "write_parquet": ("boti_data.pipelines", "write_parquet"),
     "FileWatermarkStore": ("boti_data.watermark", "FileWatermarkStore"),
     "FsspecWatermarkStore": ("boti_data.watermark", "FsspecWatermarkStore"),
     "IncrementalResult": ("boti_data.watermark", "IncrementalResult"),
@@ -172,6 +178,12 @@ _LAZY = {
 }
 
 
+# Deliberately identical to the __getattr__ in boti_data/db/__init__.py and
+# boti_data/parquet/__init__.py -- this is PEP 562's own documented
+# module-level lazy-import boilerplate. Each copy needs its own private
+# _LAZY dict/globals()/__name__, so sharing an implementation would need an
+# awkward globals()-passing helper for no real gain over these 6 lines.
+# spaghetti-ignore[duplicate-function-body]: see above
 def __getattr__(name: str) -> Any:
     if name in _LAZY:
         module_name, attr = _LAZY[name]

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/db/__init__.py

@@ -63,6 +63,12 @@ _LAZY = {
 }
 
 
+# Deliberately identical to the __getattr__ in boti_data/__init__.py and
+# boti_data/parquet/__init__.py -- this is PEP 562's own documented
+# module-level lazy-import boilerplate. Each copy needs its own private
+# _LAZY dict/globals()/__name__, so sharing an implementation would need an
+# awkward globals()-passing helper for no real gain over these 6 lines.
+# spaghetti-ignore[duplicate-function-body]: see above
 def __getattr__(name: str) -> Any:
     if name in _LAZY:
         module_name, attr = _LAZY[name]

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/db/arrow_schema_mapper.py

@@ -274,6 +274,37 @@ def build_empty_arrow_table(schema: pa.Schema) -> pa.Table:
     return pa.Table.from_arrays(arrays, schema=schema)
 
 
+def arrow_table_from_pylist_with_fallback(
+    rows: list[dict[str, Any]],
+    columns: list[str],
+) -> pa.Table:
+    """Build a Table from raw dict rows with no SQLAlchemy schema to guide it.
+
+    Used for raw-SQL-text loads, where there is no ``Select`` statement to
+    derive column types from, so pyarrow must infer everything from the
+    Python values themselves. A column of otherwise-uniform values pyarrow's
+    inference can't place (``uuid.UUID``, ``decimal.Decimal``, ...) fails the
+    *entire* table with an opaque ``ArrowInvalid`` naming the value, never the
+    column. Isolate any such column and fall back to stringifying just that
+    column instead of failing the whole load.
+    """
+    try:
+        return pa.Table.from_pylist(rows)
+    except (pa.ArrowInvalid, pa.ArrowTypeError):
+        _logger.debug("Whole-table inference failed, falling back to per-column coercion")
+
+    arrays: dict[str, pa.Array] = {}
+    for column in columns:
+        values = [row.get(column) for row in rows]
+        try:
+            arrays[column] = pa.array(values)
+        except (pa.ArrowInvalid, pa.ArrowTypeError):
+            arrays[column] = pa.array(
+                [str(value) if value is not None else None for value in values]
+            )
+    return pa.Table.from_pydict(arrays)
+
+
 # ---------------------------------------------------------------------------
 # Arrow → pandas conversion with type preservation
 # ---------------------------------------------------------------------------

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/db/sql_config.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
-import warnings
 from pathlib import Path
 from typing import Any, Union
 
+from boti.core.logger import Logger
 from boti.core.models import ResourceConfig
 from boti.core.security import is_valid_env_var_name
 from boti.core.settings import SqlDatabaseSettings, load_prefixed_model
@@ -162,11 +162,9 @@ class WorkerSqlConfig(BaseModel):
         if config.worker_connection_env_var is not None:
             payload["connection_env_var"] = config.worker_connection_env_var
         else:
-            warnings.warn(
+            Logger.default_logger(logger_name=cls.__name__).warning(
                 "worker_connection_env_var is not set; falling back to raw DSN. "
-                "Credentials may be visible in scheduler logs.",
-                UserWarning,
-                stacklevel=2,
+                "Credentials may be visible in scheduler logs."
             )
             payload["connection_url"] = config.connection_url
         return cls(**payload)

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/gateway/__init__.py

@@ -2,7 +2,13 @@ from boti_data.db import AsyncSqlDatabaseResource
 
 from .core import DataGateway
 from .policies import GatewayPolicies
-from .requests import DataFrameOptions, DataFrameParams, ParquetLoadRequest, SqlLoadRequest
+from .requests import (
+    DataFrameOptions,
+    DataFrameParams,
+    ParquetLoadRequest,
+    SqlLoadRequest,
+    TableDescription,
+)
 
 __all__ = [
     "AsyncSqlDatabaseResource",
@@ -12,4 +18,5 @@ __all__ = [
     "GatewayPolicies",
     "ParquetLoadRequest",
     "SqlLoadRequest",
+    "TableDescription",
 ]

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/gateway/_backend_strategies.py

@@ -50,6 +50,7 @@ from .requests import (
     BackendResource,
     ResolvedExecutionMode,
     ReturnType,
+    TableDescription,
 )
 
 _log = logging.getLogger(__name__)
@@ -244,6 +245,33 @@ class BackendStrategy(ABC):
         """
         return self.estimate_result_size(ctx)
 
+    # -- Schema/row-count discovery ------------------------------------------
+
+    def describe(
+        self,
+        resource: BackendResource | None,
+        table: str,
+        *,
+        row_count_limit: int,
+    ) -> TableDescription:
+        """Cheaply inspect *table*'s schema and an approximate row count.
+
+        Raises ``NotImplementedError`` by default — only backends with a
+        cheap, bounded introspection path (no full-table load) should
+        override this.
+        """
+        raise NotImplementedError(f"{self.name} backend does not support describe().")
+
+    async def describe_async(
+        self,
+        resource: BackendResource | None,
+        table: str,
+        *,
+        row_count_limit: int,
+    ) -> TableDescription:
+        """Async variant of :meth:`describe`. Default delegates to the sync method."""
+        return self.describe(resource, table, row_count_limit=row_count_limit)
+
 
 # ---------------------------------------------------------------------------
 # Registry

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/gateway/core.py

@@ -14,7 +14,7 @@ from boti.core.lifecycle import LifecycleCore
 from boti.core.lifecycle_pickle import PicklableLifecycleCoreMixin
 from boti_dask import async_safe_head, safe_head
 
-from . import core_load
+from . import core_describe, core_load
 from ._gateway_init import (
     GatewayInitOptions,
     build_gateway_from_backend,
@@ -31,6 +31,7 @@ from .requests import (
     ExecutionMode,
     ResolvedReturnType,
     ReturnType,
+    TableDescription,
 )
 
 
@@ -246,6 +247,31 @@ class DataGateway(PicklableLifecycleCoreMixin, LifecycleCore):
         )
         return await async_safe_head(frame, n=n, npartitions=npartitions)
 
+    def describe(
+        self, table: str | None = None, *, row_count_limit: int = 10_000
+    ) -> TableDescription:
+        """Cheaply inspect a table's schema and an approximate row count.
+
+        Runs a bounded introspection query — reflecting column dtypes without
+        loading any rows, then counting up to *row_count_limit* rows — instead
+        of requiring callers to know to self-impose ``limit=`` on a full
+        :meth:`load` before exploring an unfamiliar table.
+
+        Args:
+            table: DB table name. Defaults to the table configured at
+                construction time (configured mode).
+            row_count_limit: Row count is exact up to this many rows; beyond
+                it, :attr:`TableDescription.row_count_is_exact` is ``False``
+                and :attr:`TableDescription.row_count` reports this cap.
+        """
+        return core_describe.describe(self, table, row_count_limit=row_count_limit)
+
+    async def adescribe(
+        self, table: str | None = None, *, row_count_limit: int = 10_000
+    ) -> TableDescription:
+        """Async variant of :meth:`describe`."""
+        return await core_describe.describe_async(self, table, row_count_limit=row_count_limit)
+
     def load(self, **options: Any) -> FrameResult:
         """Load data from the configured backend.
 

boti_data-1.3.4/src/boti_data/gateway/core_describe.py

@@ -0,0 +1,45 @@
+"""DataGateway.describe()/adescribe() implementations.
+
+Split out of core.py purely for line-count headroom, mirroring core_load.py's
+split: DataGateway's public methods keep their docstrings and stay callable
+exactly as before, but their bodies move here as free functions taking the
+gateway instance explicitly.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .requests import TableDescription
+
+if TYPE_CHECKING:
+    from .core import DataGateway
+
+
+def resolve_describe_table(gateway: DataGateway, table: str | None) -> str:
+    resolved = table or gateway._table
+    if resolved is None:
+        raise ValueError(
+            "describe()/adescribe() require table= (or a table configured at construction time)."
+        )
+    return resolved
+
+
+# Not a copy-pasted twin: both are thin delegates to
+# gateway._strategy.describe()/describe_async() after the identical
+# table-resolution step; the real logic lives one layer down (sql_describe.py).
+# spaghetti-ignore[sync-async-duplication]: see above
+def describe(gateway: DataGateway, table: str | None, *, row_count_limit: int) -> TableDescription:
+    resolved_table = resolve_describe_table(gateway, table)
+    return gateway._strategy.describe(
+        gateway.resource, resolved_table, row_count_limit=row_count_limit
+    )
+
+
+async def describe_async(
+    gateway: DataGateway, table: str | None, *, row_count_limit: int
+) -> TableDescription:
+    resolved_table = resolve_describe_table(gateway, table)
+    return await gateway._strategy.describe_async(
+        gateway.resource, resolved_table, row_count_limit=row_count_limit
+    )

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/gateway/loaders.py

@@ -14,6 +14,7 @@ from boti_data.db import (
     SqlPartitionedLoadRequest,
 )
 from boti_data.db.arrow_schema_mapper import (
+    arrow_table_from_pylist_with_fallback,
     arrow_table_to_pandas,
     build_arrow_schema_from_sqlalchemy_types,
     rows_to_arrow_table,
@@ -61,10 +62,15 @@ def _arrow_table_from_sql_result(
     *,
     statement: Any | None,
 ) -> pa.Table:
-    if statement is None:
+    # A raw-SQL-text load's statement is a SQLAlchemy TextClause, which has no
+    # .selected_columns (that's Select-only) -- treat it the same as no
+    # statement at all rather than letting the attribute lookup below crash.
+    if statement is None or not hasattr(statement, "selected_columns"):
         if not rows:
             return pa.table({name: [] for name in columns})
-        return pa.Table.from_pylist([dict(zip(columns, row)) for row in rows])
+        return arrow_table_from_pylist_with_fallback(
+            [dict(zip(columns, row)) for row in rows], columns
+        )
 
     sql_types = [getattr(selected, "type", None) for selected in statement.selected_columns]
     schema = build_arrow_schema_from_sqlalchemy_types(columns, sql_types)

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/gateway/requests.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from dataclasses import dataclass
 from typing import Any, Literal, Union
 
 from pydantic import BaseModel, ConfigDict, Field, model_validator
@@ -207,3 +208,18 @@ class ParquetLoadRequest(BaseModel):
                 "Use as_pandas=True, return_type='arrow', or apply Dask head()/partitions explicitly."
             )
         return self
+
+
+@dataclass(frozen=True, slots=True)
+class TableDescription:
+    """Result of :meth:`~boti_data.gateway.core.DataGateway.describe`.
+
+    ``row_count`` is exact when ``row_count_is_exact`` is ``True``; otherwise
+    it reports the ``row_count_limit`` cap that was hit, meaning "at least
+    this many rows" rather than the real total.
+    """
+
+    table: str
+    columns: dict[str, str]
+    row_count: int
+    row_count_is_exact: bool

boti_data-1.3.4/src/boti_data/gateway/sql_describe.py

@@ -0,0 +1,41 @@
+"""Cheap SQL table introspection backing DataGateway.describe()/adescribe().
+
+Split out of sql_strategy.py purely for line-count headroom, mirroring the
+sql_size_estimation.py module split.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from boti_data.db.partitioned_planner import SqlPartitionPlanner
+from boti_data.db.partitioned_statements import infer_meta_dtypes
+
+from .loaders import reflect_and_select
+from .requests import TableDescription
+
+
+def describe_table(
+    resource: Any,
+    table: str,
+    *,
+    row_count_limit: int,
+) -> TableDescription:
+    """Reflect *table*'s columns and count rows up to *row_count_limit*.
+
+    Never loads any actual row data: column dtypes come from SQLAlchemy
+    reflection (no query executed), and the row count is a single bounded
+    ``COUNT(*)`` subquery capped at *row_count_limit* rows — safe to run
+    against an unfamiliar table without risking an accidental unbounded
+    query against a large production table.
+    """
+    _model, statement = reflect_and_select(resource, table)
+    columns = infer_meta_dtypes(statement)
+    counted = SqlPartitionPlanner(resource).count_rows_up_to(statement, row_count_limit)
+    is_exact = counted <= row_count_limit
+    return TableDescription(
+        table=table,
+        columns=columns,
+        row_count=counted if is_exact else row_count_limit,
+        row_count_is_exact=is_exact,
+    )

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/gateway/sql_strategy.py

@@ -34,7 +34,8 @@ from .frame_strategies import FrameResult
 from .loaders import build_sql_partitioned_request, read_sql_async
 from .normalization import build_partitioned_load_options
 from .planning import InChunkPolicy
-from .requests import BackendConfig, BackendName, BackendResource, SqlLoadRequest
+from .requests import BackendConfig, BackendName, BackendResource, SqlLoadRequest, TableDescription
+from .sql_describe import describe_table
 from .sql_partitioned_exec import run_async_partitioned_request
 from .sql_size_estimation import (
     _DEFAULT_PARTITION_CHUNK_SIZE,
@@ -376,3 +377,32 @@ class SqlAlchemyStrategy(BackendStrategy):
                 return await asyncio.to_thread(self.estimate_result_size, ctx)
             return await estimate_configured_async(ctx)
         return await estimate_structured_async(ctx)
+
+    # -- Schema/row-count discovery ------------------------------------------
+
+    def describe(
+        self,
+        resource: BackendResource | None,
+        table: str,
+        *,
+        row_count_limit: int,
+    ) -> TableDescription:
+        if resource is None:
+            raise ValueError("describe() requires an active SQL resource.")
+        return describe_table(resource, table, row_count_limit=row_count_limit)
+
+    # describe() is already a single quick, bounded query -- offloading it to
+    # a thread (like estimate_result_size_async's sync-resource branch above)
+    # is enough; no separate async reflection/count path is worth the
+    # duplication for something this cheap.
+    # spaghetti-ignore[sync-async-duplication]: see above
+    async def describe_async(
+        self,
+        resource: BackendResource | None,
+        table: str,
+        *,
+        row_count_limit: int,
+    ) -> TableDescription:
+        return await asyncio.to_thread(
+            self.describe, resource, table, row_count_limit=row_count_limit
+        )

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/helper.py

@@ -10,7 +10,7 @@ from boti_dask import DaskSession, dask_session
 
 from boti_data.gateway import DataGateway
 from boti_data.gateway.incremental import IncrementalLoadService
-from boti_data.gateway.requests import BackendConfig
+from boti_data.gateway.requests import BackendConfig, TableDescription
 from boti_data.joins import indexed_left_join, left_join_frames
 from boti_data.schema import DataFrameLike
 from boti_data.watermark.incremental import IncrementalResult
@@ -273,6 +273,20 @@ class DataHelper(PicklableLifecycleCoreMixin, LifecycleCore):
     async def apreview(self, statement: Any, model: Any, n: int = 5, **options: Any) -> Any:
         return await self.gateway.aload(**self._build_preview_options(statement, model, n, options))
 
+    # Not a copy-pasted twin: pure delegate to self.gateway.describe()/
+    # adescribe() with identical kwargs, matching this class's other
+    # unflagged proxy pairs (load/aload, preview/apreview).
+    # spaghetti-ignore[sync-async-duplication]: see above
+    def describe(
+        self, table: str | None = None, *, row_count_limit: int = 10_000
+    ) -> TableDescription:
+        return self.gateway.describe(table, row_count_limit=row_count_limit)
+
+    async def adescribe(
+        self, table: str | None = None, *, row_count_limit: int = 10_000
+    ) -> TableDescription:
+        return await self.gateway.adescribe(table, row_count_limit=row_count_limit)
+
     def load_period(self, dt_field: str, start: str, end: str, **kwargs: Any) -> Any:
         return self.gateway.load_period(dt_field, start, end, **kwargs)
 

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/parquet/__init__.py

@@ -19,6 +19,12 @@ __all__ = ["ParquetDataConfig", "ParquetDataResource", "ParquetReader"]
 _LAZY = {"ParquetReader": ("boti_data.parquet.reader", "ParquetReader")}
 
 
+# Deliberately identical to the __getattr__ in boti_data/__init__.py and
+# boti_data/db/__init__.py -- this is PEP 562's own documented module-level
+# lazy-import boilerplate. Each copy needs its own private
+# _LAZY dict/globals()/__name__, so sharing an implementation would need an
+# awkward globals()-passing helper for no real gain over these 6 lines.
+# spaghetti-ignore[duplicate-function-body]: see above
 def __getattr__(name: str) -> Any:
     if name in _LAZY:
         module_name, attr = _LAZY[name]

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/pipelines/__init__.py

@@ -3,7 +3,16 @@ from __future__ import annotations
 from .base import SinkPipeline
 from .parquet_pipeline import ParquetMaterializationResult, ParquetPipeline
 from .registry import SinkRegistry, available_sinks, create_sink, register_sink
-from .sinks import CsvSink, CsvSinkConfig, JsonlSink, JsonlSinkConfig, ParquetSink, SinkWriteResult
+from .sinks import (
+    CsvSink,
+    CsvSinkConfig,
+    JsonlSink,
+    JsonlSinkConfig,
+    ParquetSink,
+    SinkWriteResult,
+    awrite_parquet,
+    write_parquet,
+)
 
 __all__ = [
     "CsvSink",
@@ -17,6 +26,8 @@ __all__ = [
     "SinkPipeline",
     "SinkWriteResult",
     "available_sinks",
+    "awrite_parquet",
     "create_sink",
     "register_sink",
+    "write_parquet",
 ]

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/pipelines/sinks.py

@@ -16,7 +16,7 @@ from boti_data.pipelines.sinks_common import (
     prepare_partitioned_frame,
     to_dask_frame,
 )
-from boti_data.pipelines.sinks_parquet import ParquetSink
+from boti_data.pipelines.sinks_parquet import ParquetSink, awrite_parquet, write_parquet
 from boti_data.pipelines.sinks_text import CsvSink, CsvSinkConfig, JsonlSink, JsonlSinkConfig
 
 __all__ = [
@@ -29,6 +29,8 @@ __all__ = [
     "ParquetSink",
     "PipelineSink",
     "SinkWriteResult",
+    "awrite_parquet",
     "prepare_partitioned_frame",
     "to_dask_frame",
+    "write_parquet",
 ]

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/pipelines/sinks_common.py

@@ -71,6 +71,43 @@ def _derive_partition_date_column(
     return frame
 
 
+def _validate_meta_matches_real_dtypes(frame: dd.DataFrame, *, sink_name: str) -> None:
+    """Guard against dask-expr's meta/real dtype divergence (wishlist #2/#7).
+
+    A bare `dd.to_datetime()`/`pd.to_numeric()` reassignment or an unannotated
+    `map_partitions()` call anywhere upstream in a pipeline -- not just inside
+    this module's own partition_date derivation -- can silently corrupt an
+    *unrelated* column's synthetic `_meta_nonempty` sample without touching
+    the real per-partition data. Left unchecked, that divergence doesn't fail
+    here: it surfaces later as an opaque pyarrow `ArrowInvalid` at write time
+    with nothing pointing back to the real cause. Catch it against one real
+    computed partition before any sink commits to a write.
+    """
+    meta_dtypes = frame.dtypes
+    try:
+        real_dtypes = frame.head(1, npartitions=-1, compute=True).dtypes
+    except ValueError:
+        return  # empty frame -- nothing real to validate meta against
+    mismatched = {
+        column: (meta_dtypes[column], real_dtypes[column])
+        for column in real_dtypes.index
+        if meta_dtypes[column] != real_dtypes[column]
+    }
+    if not mismatched:
+        return
+    details = ", ".join(
+        f"{column!r} (meta={meta!s}, real={real!s})" for column, (meta, real) in mismatched.items()
+    )
+    raise ValueError(
+        f"{sink_name} detected a dask meta/real dtype mismatch before write: {details}. "
+        "This usually means a column elsewhere in the pipeline was reassigned via a bare "
+        "dd.to_datetime()/pd.to_numeric() call or an unannotated map_partitions(), which can "
+        "silently corrupt _meta_nonempty for unrelated columns. Pass an explicit meta= to "
+        "map_partitions() (or route the coercion through it) instead of reassigning via a "
+        "bare top-level pandas/dask function."
+    )
+
+
 def prepare_partitioned_frame(
     frame: dd.DataFrame,
     *,
@@ -78,20 +115,20 @@ def prepare_partitioned_frame(
     date_field: str | None,
     sink_name: str,
 ) -> dd.DataFrame:
-    if not partition_on:
-        return frame
-
-    partition_columns = list(partition_on)
-    missing = [name for name in partition_columns if name not in frame.columns]
-    if "partition_date" in missing:
-        frame = _derive_partition_date_column(frame, date_field, sink_name)
+    if partition_on:
+        partition_columns = list(partition_on)
         missing = [name for name in partition_columns if name not in frame.columns]
+        if "partition_date" in missing:
+            frame = _derive_partition_date_column(frame, date_field, sink_name)
+            missing = [name for name in partition_columns if name not in frame.columns]
 
-    if missing:
-        raise ValueError(
-            f"{sink_name} partition columns must already exist in the loaded frame or be derivable "
-            f"from date_field. Missing columns: {missing!r}."
-        )
+        if missing:
+            raise ValueError(
+                f"{sink_name} partition columns must already exist in the loaded frame or be "
+                f"derivable from date_field. Missing columns: {missing!r}."
+            )
+
+    _validate_meta_matches_real_dtypes(frame, sink_name=sink_name)
     return frame
 
 
@@ -109,6 +146,60 @@ def _rm_recursive(fs: fsspec.AbstractFileSystem, path: str) -> None:
         fs.rm(path)
 
 
+def _rebase_staged_path(
+    fs: fsspec.AbstractFileSystem,
+    path: str,
+    staging_path: str,
+    target_root: str,
+) -> str:
+    """Rewrite *path* from the staging tree to the target tree.
+
+    ``glob()``/``find()`` on most fsspec backends (S3, memory, ...) return
+    protocol-stripped paths (``bucket/key``) even when queried with a
+    scheme-prefixed directory (``s3://bucket/key``) -- but *staging_path*/
+    *target_root* here are whatever the caller originally passed in, which
+    for a ``Datasources``-resolved destination is scheme-prefixed. A naive
+    ``path.replace(staging_path, target_root)`` then silently no-ops (no
+    substring match, string returned unchanged) instead of raising, so both
+    sides are normalized through ``_strip_protocol()`` first to guarantee the
+    replace actually matches regardless of which form *path* arrived in.
+    """
+    normalized_path = fs._strip_protocol(path)
+    normalized_staging = fs._strip_protocol(staging_path)
+    normalized_target = fs._strip_protocol(target_root)
+    return normalized_path.replace(normalized_staging, normalized_target, 1)
+
+
+def _move_staged_files(
+    fs: fsspec.AbstractFileSystem,
+    staged_files: Sequence[str],
+    staging_path: str,
+    target_root: str,
+) -> None:
+    """Move each enumerated staged file individually into *target_root*.
+
+    Deliberately avoids a single recursive ``fs.mv(staging_path, target_root,
+    recursive=True)``. On prefix-only S3-compatible backends (e.g. MinIO),
+    ``expand_path(recursive=True)`` can return the bare directory prefix
+    itself as a phantom "file" alongside the real objects -- no object
+    actually exists at that literal key -- and ``mv()`` forces
+    ``on_error="raise"`` on the resulting copy, so that one phantom entry
+    404s the entire swap even though every real file would have copied fine.
+    Moving the already-known file list one by one never expands a directory,
+    so the phantom entry never gets a chance to appear.
+    """
+    for staged_file in staged_files:
+        destination = _rebase_staged_path(fs, staged_file, staging_path, target_root)
+        parent = destination.rsplit("/", 1)[0]
+        if parent and parent != destination:
+            fs.makedirs(parent, exist_ok=True)
+        fs.mv(staged_file, destination)
+    if fs.exists(staging_path):
+        # Empty leftover directory structure (local-disk backends only --
+        # prefix-only stores have nothing left here once every file moved).
+        _rm_recursive(fs, staging_path)
+
+
 def _write_with_staging(
     *,
     fs: fsspec.AbstractFileSystem,
@@ -134,20 +225,18 @@ def _write_with_staging(
         _rm_recursive(fs, staging_path)
 
     files = write_fn(staging_path)
+    target_root = target_path.rstrip("/")
 
     try:
         _rm_recursive(fs, target_path)
-        try:
-            fs.mv(staging_path, target_path, recursive=True)
-        except TypeError:
-            fs.mv(staging_path, target_path)
+        _move_staged_files(fs, files, staging_path, target_root)
     except Exception as exc:
         raise RuntimeError(
             f"Failed to swap staged sink output into {target_path!r}. "
             f"The newly written data is intact at {staging_path!r}."
         ) from exc
 
-    return [file.replace(staging_path, target_path.rstrip("/"), 1) for file in files]
+    return [_rebase_staged_path(fs, file, staging_path, target_root) for file in files]
 
 
 @dataclass(slots=True)

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data/pipelines/sinks_parquet.py

@@ -125,3 +125,42 @@ class ParquetSink(_AsyncWriteViaThreadMixin, PicklableLifecycleCoreMixin, Lifecy
     @staticmethod
     def _restore_protocol(path: str) -> str:
         return str(path)
+
+
+# Not a copy-pasted twin: `with`/`async with` + sink.write()/await sink.awrite()
+# is the irreducible sync/async difference; there's no further shared logic to
+# hoist for what's just a construct+write+close convenience.
+# spaghetti-ignore[sync-async-duplication]: see above
+def write_parquet(
+    destination: ParquetDestination,
+    frame: FrameResult,
+    *,
+    partition_on: Sequence[str] | None = ("partition_date",),
+    **kwargs: Any,
+) -> SinkWriteResult:
+    """One-shot convenience: construct a ParquetSink, write *frame*, and close it.
+
+    A ``ParquetSink`` is typically built, used for exactly one ``write()``,
+    and discarded — forgetting ``with ParquetSink(...) as sink:`` leaks the
+    underlying ``DataGateway``/``ParquetDataResource``, surfaced only as a
+    background GC warning well after the fact, not at the call site. Callers
+    who genuinely reuse one ``ParquetSink`` instance across multiple writes
+    should keep using the class directly instead.
+
+    ``**kwargs`` are forwarded to :meth:`ParquetSink.write` (``date_field``,
+    ``write_index``, ``overwrite``, ``persist``).
+    """
+    with ParquetSink(destination, partition_on=partition_on) as sink:
+        return sink.write(frame, **kwargs)
+
+
+async def awrite_parquet(
+    destination: ParquetDestination,
+    frame: FrameResult,
+    *,
+    partition_on: Sequence[str] | None = ("partition_date",),
+    **kwargs: Any,
+) -> SinkWriteResult:
+    """Async variant of :func:`write_parquet`."""
+    async with ParquetSink(destination, partition_on=partition_on) as sink:
+        return await sink.awrite(frame, **kwargs)

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boti-data
-Version: 1.3.2
+Version: 1.3.4
 Summary: Data infrastructure for the Boti ecosystem
 Author: Luis Valverde
 Maintainer-email: Luis Valverde <lvalverdeb@gmail.com>

{boti_data-1.3.2 → boti_data-1.3.4}/src/boti_data.egg-info/SOURCES.txt

@@ -72,6 +72,7 @@ src/boti_data/gateway/build_context.py
 src/boti_data/gateway/chunking.py
 src/boti_data/gateway/configured_load.py
 src/boti_data/gateway/core.py
+src/boti_data/gateway/core_describe.py
 src/boti_data/gateway/core_load.py
 src/boti_data/gateway/datacube_strategy.py
 src/boti_data/gateway/frame_convert.py
@@ -89,6 +90,7 @@ src/boti_data/gateway/requests.py
 src/boti_data/gateway/return_type.py
 src/boti_data/gateway/select_cache.py
 src/boti_data/gateway/semi_join.py
+src/boti_data/gateway/sql_describe.py
 src/boti_data/gateway/sql_guard.py
 src/boti_data/gateway/sql_partitioned_exec.py
 src/boti_data/gateway/sql_size_estimation.py