sibi-dst 2025.8.9__py3-none-any.whl → 2025.9.2__py3-none-any.whl

sibi_dst/utils/boilerplate/__init__.py

@@ -1,6 +1,11 @@
-from .base_data_artifact import BaseDataArtifact
+from .base_parquet_artifact import BaseParquetArtifact
 from .base_data_cube import BaseDataCube
+from .base_attacher import make_attacher
+from .base_parquet_reader import BaseParquetReader
+__all__ = [
+    "BaseDataCube",
+    "BaseParquetArtifact",
+    "make_attacher",
+    "BaseParquetReader"
+]
 
-__all__ = ["BaseDataCube",
-           "BaseDataArtifact"
-           ]

sibi_dst/utils/boilerplate/base_attacher.py

@@ -0,0 +1,25 @@
+from typing import Any, Awaitable, Callable, Sequence, Type
+
+def make_attacher(
+    cube_cls: Type,
+    fieldnames: Sequence[str],
+    column_names: Sequence[str],
+) -> Callable[..., Awaitable[Any]]:
+    """
+    Factory for async attachers.
+    Skips work if any param value is falsy ([], None, {}, etc.).
+    """
+
+    async def attach(*, logger=None, debug: bool = False, **params: Any):
+        if any(not v for v in params.values()):
+            return None
+        call_params = {
+            "fieldnames": tuple(fieldnames),
+            "column_names": list(column_names),
+            **params,
+        }
+        return await cube_cls(logger=logger, debug=debug).aload(**call_params)
+
+    return attach
+
+__all__ = ['make_attacher']

sibi_dst/utils/boilerplate/{base_data_artifact.py → base_parquet_artifact.py}

@@ -30,7 +30,7 @@ def _validate_and_format_date(name: str, value: DateLike) -> Optional[str]:
     raise TypeError(f"{name} must be str, date, datetime, or None; got {type(value)}")
 
 
-class BaseDataArtifact(ParquetArtifact):
+class BaseParquetArtifact(ParquetArtifact):
     """
     Base class for Parquet artifacts with optional date window.
 

sibi_dst/utils/boilerplate/base_parquet_reader.py

@@ -0,0 +1,21 @@
+from sibi_dst.df_helper import ParquetReader
+
+class BaseParquetReader(ParquetReader):
+    """
+    Base class for Parquet readers that merges configuration parameters and handles
+    debug and logger initialization.
+    """
+    config = {
+        'backend': 'parquet'
+    }
+    def __init__(self, parquet_start_date, parquet_end_date, **kwargs):
+        # Merge the class-level config with any additional keyword arguments,
+        # and include debug and logger.
+        kwargs = {**self.config,**kwargs}
+        super().__init__(
+            parquet_start_date=parquet_start_date,
+            parquet_end_date=parquet_end_date,
+            **kwargs
+        )
+
+__all__ = ['BaseParquetReader']

sibi_dst/utils/clickhouse_writer.py

@@ -10,6 +10,14 @@ import clickhouse_connect
 
 from . import ManagedResource
 
+def _to_bool(val: Any) -> bool:
+    if isinstance(val, bool):
+        return val
+    if isinstance(val, (int, float)):
+        return bool(val)
+    if isinstance(val, str):
+        return val.strip().lower() in ("1", "true", "yes", "on")
+    return False
 
 class ClickHouseWriter(ManagedResource):
     """
@@ -47,6 +55,11 @@ class ClickHouseWriter(ManagedResource):
         database: str = "sibi_data",
         user: str = "default",
         password: str = "",
+        secure: bool = False,
+        verify: bool = False,
+        ca_cert: str = "",
+        client_cert: str = "",
+        compression: str = "",
         table: str = "test_sibi_table",
         order_by: str = "id",
         engine: Optional[str] = None,  # e.g. "ENGINE MergeTree ORDER BY (`id`)"
@@ -61,6 +74,11 @@ class ClickHouseWriter(ManagedResource):
         self.database = database
         self.user = user
         self.password = password
+        self.secure = _to_bool(secure)
+        self.verify = _to_bool(verify)
+        self.ca_cert = ca_cert
+        self.client_cert = client_cert
+        self.compression = compression  # e.g. 'lz4', 'zstd',
         self.table = table
         self.order_by = order_by
         self.engine = engine  # if None → default MergeTree ORDER BY
@@ -224,6 +242,7 @@ class ClickHouseWriter(ManagedResource):
     # ------------- low-level helpers -------------
 
     def _get_client(self):
+        print(self.secure, " ", self.verify)
         cli = getattr(self._tlocal, "client", None)
         if cli is not None:
             return cli
@@ -233,6 +252,11 @@ class ClickHouseWriter(ManagedResource):
             database=self.database,
             username=self.user,  # clickhouse-connect uses 'username'
             password=self.password,
+            secure=self.secure,
+            verify=self.verify,
+            ca_cert=self.ca_cert or None,
+            client_cert=self.client_cert or None,
+            compression=self.compression or None,
         )
         self._tlocal.client = cli
         return cli

sibi_dst/utils/dask_utils.py

@@ -0,0 +1,61 @@
+import asyncio
+from typing import List, Any, Dict
+
+import dask
+import dask.dataframe as dd
+
+def _to_int_safe(x) -> int:
+    """
+    Convert scalar-like to int safely.
+    Handles numpy scalars, pandas Series/DataFrame outputs.
+    """
+    if hasattr(x, "item"):        # numpy scalar, pandas scalar
+        return int(x.item())
+    if hasattr(x, "iloc"):        # Series-like
+        return int(x.iloc[0])
+    return int(x)
+
+def dask_is_probably_empty(ddf: dd.DataFrame) -> bool:
+    return getattr(ddf, "npartitions", 0) == 0 or len(ddf._meta.columns) == 0
+
+
+def dask_is_empty_truthful(ddf: dd.DataFrame) -> bool:
+    n = ddf.map_partitions(len).sum().compute()
+    return int(n) == 0
+
+
+def dask_is_empty(ddf: dd.DataFrame, *, sample: int = 4) -> bool:
+    if dask_is_probably_empty(ddf):
+        return True
+
+    k = min(max(sample, 1), ddf.npartitions)
+    probes = dask.compute(*[
+        ddf.get_partition(i).map_partitions(len) for i in range(k)
+    ])
+
+    if any(_to_int_safe(n) > 0 for n in probes):
+        return False
+    if k == ddf.npartitions and all(_to_int_safe(n) == 0 for n in probes):
+        return True
+
+    return dask_is_empty_truthful(ddf)
+
+class UniqueValuesExtractor:
+    @staticmethod
+    def _compute_to_list_sync(series) -> List[Any]:
+        """Run in a worker thread when Dask-backed."""
+        if hasattr(series, "compute"):
+            return series.compute().tolist()
+        return series.tolist()
+
+    async def compute_to_list(self, series) -> List[Any]:
+        # Offload potential Dask .compute() to a thread to avoid blocking the event loop
+        return await asyncio.to_thread(self._compute_to_list_sync, series)
+
+    async def extract_unique_values(self, df, *columns: str) -> Dict[str, List[Any]]:
+        async def one(col: str):
+            ser = df[col].dropna().unique()
+            return col, await self.compute_to_list(ser)
+
+        pairs = await asyncio.gather(*(one(c) for c in columns))
+        return dict(pairs)

sibi_dst/utils/progress/sse_runner.py

@@ -49,6 +49,8 @@ class SSERunner:
         async def handler(request: Request):  # <-- only Request
             queue: asyncio.Queue = asyncio.Queue()
             task_id = str(asyncio.get_running_loop().time()).replace(".", "")
+            self.logger.debug(
+                f"SSE {task_id}: new request client={request.client} path={request.url.path} q={dict(request.query_params)}")
 
             ctx: Dict[str, Any] = {
                 "path": dict(request.path_params),          # <-- pull path params here

{sibi_dst-2025.8.9.dist-info → sibi_dst-2025.9.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.8.9
+Version: 2025.9.2
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -21,6 +21,7 @@ Requires-Dist: pyarrow (>=20.0.0,<21.0.0)
 Requires-Dist: pydantic (>=2.11.7,<3.0.0)
 Requires-Dist: pyiceberg[hive,s3fs] (>=0.9.1,<0.10.0)
 Requires-Dist: pymysql (>=1.1.1,<2.0.0)
+Requires-Dist: pyrosm (>=0.6.2,<0.7.0)
 Requires-Dist: s3fs (>=2025.5.1,<2026.0.0)
 Requires-Dist: sqlalchemy (>=2.0.41,<3.0.0)
 Requires-Dist: sse-starlette (>=3.0.2,<4.0.0)

{sibi_dst-2025.8.9.dist-info → sibi_dst-2025.9.2.dist-info}/RECORD

@@ -2,19 +2,19 @@ sibi_dst/__init__.py,sha256=D01Z2Ds4zES8uz5Zp7qOWD0EcfCllWgew7AWt2X1SQg,445
 sibi_dst/df_helper/__init__.py,sha256=CyDXtFhRnMrycktxNO8jGGkP0938QiScl56kMZS1Sf8,578
 sibi_dst/df_helper/_artifact_updater_async.py,sha256=0lUwel-IkmKewRnmMv9GtuT-P6SivkIKtgOHvKchHlc,8462
 sibi_dst/df_helper/_artifact_updater_threaded.py,sha256=M5GNZismOqMmBrcyfolP1DPv87VILQf_P18is_epn50,7238
-sibi_dst/df_helper/_df_helper.py,sha256=tDBpiDpYW9xn_AGPCJ_sXxuAwvfyeS5Wt79f5PCmy4w,15477
+sibi_dst/df_helper/_df_helper.py,sha256=rgVP4ggiCW6tTHmUz2UqUvLznwOtY5IyoVS3WSlg73U,17005
 sibi_dst/df_helper/_parquet_artifact.py,sha256=Lse0wlgHMEnyOfQTGD2OeT8U1ZK9aP93_42JkDk46r4,12636
 sibi_dst/df_helper/_parquet_reader.py,sha256=SKLpCeZdBEO86IRGNEp5IegE6lZtmNoXzjpGBoO-AZo,3215
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/df_helper/backends/http/__init__.py,sha256=d1pfgYxbiYg7E0Iw8RbJ7xfqIfJShqqTBQQGU_S6OOo,105
 sibi_dst/df_helper/backends/http/_http_config.py,sha256=eGPFdqZ5M3Tscqx2P93B6XoBEEzlmdt7yNg7PXUQnNQ,4726
 sibi_dst/df_helper/backends/parquet/__init__.py,sha256=0A6BGHZLwiLBmuBBaUvEHfeWTcInvy2NbymlrI_nuXE,104
-sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=BDJwBP8IJrtfccdt0v2ElQFUp8YSHv3lX8pRTdSNTKM,25231
+sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=L0GBvPXRAL_2PpaqyGabva6B99uNYrSVPiwEYfZWsvk,25308
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=LjWm9B7CweTvlvFOgB90XjSe0lVLILAIYMWKPkFXFm8,265
 sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=6705rABdh0RY0JisxD7sE62m6890hMCAv_cpyHOMSvM,8729
 sibi_dst/df_helper/backends/sqlalchemy/_db_gatekeeper.py,sha256=GQwDy2JwPUx37vpwxPM5hg4ZydilPIP824y5C_clsl0,383
 sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=Ur1V7J89nULdtvtFTr2nkKuCcIS-6tVBt5NWO87WyCc,7662
-sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=K_YRTxLqCSOPztG49X0w87tF8aRinB9b8Lnp4WmsHz0,2257
+sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=urMT7f1WWieVdCYKjfzyhiEoNIgAlXcMx0rVnv2vMAk,2259
 sibi_dst/df_helper/backends/sqlalchemy/_model_registry.py,sha256=MHk64f5WDOKHQ_L4mM8L-I-Uep_y1dczAodxA9fDJHs,6667
 sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py,sha256=yc5ij1oLOZvMN_mFWFUwuOoLnOOCxSerYpbmrHlWxzE,1480
 sibi_dst/df_helper/core/__init__.py,sha256=LfmTqFh6GUZup-g95bcXgAxX7J5Hkve7ftLE_CJg_AE,409
@@ -33,18 +33,21 @@ sibi_dst/osmnx_helper/basemaps/calendar_html.py,sha256=UArt6FDgoCgoRte45Xo3IHqd-
 sibi_dst/osmnx_helper/basemaps/route_map_plotter.py,sha256=rsJidieojcqIoe0kBanZbrxcelrS6nWoAyWoQXWdPiQ,11849
 sibi_dst/osmnx_helper/basemaps/router_plotter.py,sha256=UAiijn-J-jjX4YnL0_P9SFqTadrxMx-YK4djYhqPqfQ,10941
 sibi_dst/osmnx_helper/route_path_builder.py,sha256=XJJyu4YXegAkCRjE-knyQncwXaxDVXZhalYacLcb7e0,3557
-sibi_dst/osmnx_helper/utils.py,sha256=HfxrmXVPq3akf68SiwncbAp7XI1ER-zp8YN_doh7YaY,20679
+sibi_dst/osmnx_helper/utils.py,sha256=7-lFVhGn4rHjGz6FvpXtC2jY8UzGIVyKR3MVyEfB7nw,14407
 sibi_dst/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/tests/test_data_wrapper_class.py,sha256=6uFmZR2DxnxQz49L5jT2ehlKvlLnpUHMLFB_PqqUq7k,3336
 sibi_dst/utils/__init__.py,sha256=vShNCOMPw8KKwlb4tq5XGrpjqakJ_OE8YDc_xDAWAxI,1302
 sibi_dst/utils/async_utils.py,sha256=53aywfgq1Q6-0OVr9qR1Sf6g7Qv3I9qunAAR4fjFXBE,351
-sibi_dst/utils/base.py,sha256=IyObjZ7AaE-YjVU0RLIXNCnQKWwzi5NH2I6D1KfcIyk,8716
-sibi_dst/utils/boilerplate/__init__.py,sha256=RYERDvBTIoaTtKLyVAmfcB4HyImqd-S8wr605iiB-TQ,161
-sibi_dst/utils/boilerplate/base_data_artifact.py,sha256=rlUc3iOOhuiiqXczp3nEiXCVCxDn2CVzlmSPGPD0BzA,3750
+sibi_dst/utils/base.py,sha256=W501bJFjpgElPBo9Xp7SkgFj-oGPXXfFE25Br0dZqxc,25470
+sibi_dst/utils/boilerplate/__init__.py,sha256=998ptGqawJl79WZA-UEeTyBhvc-ClENzXrMaCSWsrL4,295
+sibi_dst/utils/boilerplate/base_attacher.py,sha256=JRAyvfljQjKVD5BJDDd09cBY9pGPIe8LQp0aUv_xJs0,736
 sibi_dst/utils/boilerplate/base_data_cube.py,sha256=ErKTM2kT8LsSXADcyYvT436O_Mp0J2hm8xs1IUircb4,2760
+sibi_dst/utils/boilerplate/base_parquet_artifact.py,sha256=oqPbjHFfChA9j1WL-eDAh7XLA3zmf-Rq7s_kzITVniA,3753
+sibi_dst/utils/boilerplate/base_parquet_reader.py,sha256=3kN9_bbxyX-WuJLMBWejeApW2V_BDArSljhSUOAOhVU,692
 sibi_dst/utils/business_days.py,sha256=dP0Xj4FhTBIvZZrZYLOHZl5zOpDAgWkD4p_1a7BOT7I,8461
-sibi_dst/utils/clickhouse_writer.py,sha256=NngJyJpx2PjUQWsX0YmwCuGdeViK77Wi3HmYqHz3jTc,9544
+sibi_dst/utils/clickhouse_writer.py,sha256=XjOxPirylcYkxT3U9wu4gleZLVR1Fmir75eeBWiXrsw,10409
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
+sibi_dst/utils/dask_utils.py,sha256=FURwrNqij6ptxFhI4v7yaGkyOIIyW9lSPpMfE9-kxHY,1970
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=7bLidEjppieNoozDFb6OuRY0W995cxg4tiGAlkGfePI,7768
 sibi_dst/utils/data_wrapper.py,sha256=090s2odlgS77mSw150V6m8-pEpD4sJ7OvjeMKNjbXxg,11604
@@ -61,7 +64,7 @@ sibi_dst/utils/periods.py,sha256=8eTGi-bToa6_a8Vwyg4fkBPryyzft9Nzy-3ToxjqC8c,143
 sibi_dst/utils/phone_formatter.py,sha256=oeM22nLjhObENrpItCNeVpkYS4pXRm5hSxdk0M4nvwU,4580
 sibi_dst/utils/progress/__init__.py,sha256=VELVxzo2cePN_-LL0veel8-F3po6tokY5MOOpu6pz1A,92
 sibi_dst/utils/progress/jobs.py,sha256=nE58ng9GPCPZhnaCDltr1tQgu3AJVqBJ1dWbGcCH4xo,3089
-sibi_dst/utils/progress/sse_runner.py,sha256=wx6-wGcG5Lktr5NnT_u0vTs1yGvNWOqsWfYg2xeH-YM,3569
+sibi_dst/utils/progress/sse_runner.py,sha256=PySHBXcpxd_eqLqZRBU1t8Ys7Df3SM-iz5R9P_vthfE,3726
 sibi_dst/utils/storage_config.py,sha256=DLtP5jKVM0mdFdgRw6LQfRqyavMjJcCVU7GhsUCRH78,4427
 sibi_dst/utils/storage_hive.py,sha256=eZ3nq2YWLUUG-06iJubSC15cwSHEbKKdKIwoVhD_I_E,8568
 sibi_dst/utils/storage_manager.py,sha256=La1NY79bhRAmHWXp7QcXJZtbHoRboJMgoXOSXbIl1SA,6643
@@ -87,6 +90,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-2025.8.9.dist-info/METADATA,sha256=euFCqxH_OCKd1_56ino8Dg9k9FfZszwrNf-8wnJSX4s,2671
-sibi_dst-2025.8.9.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-2025.8.9.dist-info/RECORD,,
+sibi_dst-2025.9.2.dist-info/METADATA,sha256=YhNbMyjgWVHGl4gQiTs0QdKrV35tCdT2hcis0m76UwY,2710
+sibi_dst-2025.9.2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-2025.9.2.dist-info/RECORD,,