sibi-dst 2025.1.5__py3-none-any.whl → 2025.1.7__py3-none-any.whl

sibi_dst/df_helper/__init__.py

@@ -3,11 +3,12 @@ from __future__ import annotations
 from ._df_helper import DfHelper
 from ._parquet_artifact import ParquetArtifact
 from ._parquet_reader import ParquetReader
-from ._artifact_updater_multi_wrapper import ArtifactUpdaterMultiWrapperThreaded
+from ._artifact_updater_multi_wrapper import ArtifactUpdaterMultiWrapperThreaded, ArtifactUpdaterMultiWrapperAsync
 
 __all__ = [
     'DfHelper',
     'ParquetArtifact',
     'ParquetReader',
     'ArtifactUpdaterMultiWrapperThreaded',
+    'ArtifactUpdaterMultiWrapperAsync',
 ]

sibi_dst/df_helper/_artifact_updater_multi_wrapper.py

@@ -1,11 +1,8 @@
-import datetime
 import time
-import random
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any, Callable, Dict, List, Optional, Type
 
-from sibi_dst.utils import  ManagedResource
-
+from sibi_dst.utils import ManagedResource
 
 class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
     """
@@ -14,7 +11,7 @@ class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
     This version is refactored for a pure multi-threaded environment, aligning
     the orchestration model with the underlying threaded workers (DataWrapper).
     """
-
+    wrapped_classes: Dict[str, List[Type]]
     def __init__(
             self,
             wrapped_classes: Dict[str, List[Type]],
@@ -26,7 +23,7 @@ class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
             backoff_jitter: float = 0.1,
             priority_fn: Optional[Callable[[Type], int]] = None,
             artifact_class_kwargs: Optional[Dict[str, Any]] = None,
-            **kwargs: Any,
+            **kwargs: Dict[str, Any]
     ) -> None:
         super().__init__(**kwargs)
         self.wrapped_classes = wrapped_classes
@@ -75,14 +72,14 @@ class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
             except Exception as e:
                 self.logger.warning(f"priority_fn error for {name}: {e}")
 
-        # Fallback to size estimate if available
-        if hasattr(artifact_cls, 'get_size_estimate'):
-            try:
-                # This performs blocking I/O
-                return artifact_cls(**self.artifact_class_kwargs).get_size_estimate()
-
-            except Exception as e:
-                self.logger.warning(f"get_size_estimate failed for {name}: {e}")
+        # # Fallback to size estimate if available
+        # if hasattr(artifact_cls, 'get_size_estimate'):
+        #     try:
+        #         # This performs blocking I/O
+        #         return artifact_cls(**self.artifact_class_kwargs).get_size_estimate()
+        #
+        #     except Exception as e:
+        #         self.logger.warning(f"get_size_estimate failed for {name}: {e}")
 
         # Default priority
         return 999
@@ -115,7 +112,7 @@ class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
         # If all retries fail, raise an exception to be caught by the main loop
         raise RuntimeError(f"{name} failed after {self.retry_attempts} attempts.")
 
-    def update_data(self, data_type: str, **kwargs: Any) -> None:
+    async def update_data(self, data_type: str, **kwargs: Any) -> None:
         """
         Entry point to update all artifacts of a given type using a ThreadPoolExecutor.
         """
@@ -190,242 +187,234 @@ class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
         return "\n".join(lines)
 
 
-# import asyncio
-# import logging
-# import datetime
-# import random
-# from typing import Any, Callable, Dict, List, Optional, Type
-#
-# from sibi_dst.utils import Logger
-#
-#
-# class ArtifactUpdaterMultiWrapperAsync:
-#     """
-#     Simplified wrapper that updates artifacts concurrently using an asyncio.Semaphore.
-#
-#     Features:
-#     - Caps concurrency at max_workers via semaphore
-#     - Optionally prioritises tasks via a priority function or static method on artifact classes
-#     - Tracks per-artifact completion times
-#     - Configurable retry/backoff strategy
-#     - Optional metrics integration
-#     - Thread-safe within a single asyncio loop
-#
-#     Usage:
-#         wrapper = ArtifactUpdaterMultiWrapper(
-#             wrapped_classes={
-#                 'mydata': [DataArtifactA, DataArtifactB],
-#             },
-#             max_workers=4,
-#             retry_attempts=3,
-#             update_timeout_seconds=600,
-#             backoff_base=2,
-#             backoff_max=60,
-#             backoff_jitter=0.1,
-#             priority_fn=None,  # or custom
-#             metrics_client=None,
-#             debug=True,
-#             logger=None,
-#             artifact_class_kwargs={
-#                 'fs': my_fs,
-#                 'parquet_storage_path': 's3://bucket/data',
-#                 'logger': my_logger,
-#                 'debug': True,
-#             }
-#         )
-#         await wrapper.update_data('mydata', period='ytd', overwrite=True)
-#     """
-#     def __init__(
-#         self,
-#         wrapped_classes: Dict[str, List[Type]],
-#         *,
-#         max_workers: int = 3,
-#         retry_attempts: int = 3,
-#         update_timeout_seconds: int = 600,
-#         backoff_base: int = 2,
-#         backoff_max: Optional[int] = 60,
-#         backoff_jitter: float = 0.1,
-#         priority_fn: Optional[Callable[[Type], int]] = None,
-#         metrics_client: Any = None,
-#         debug: bool = False,
-#         logger: Optional[logging.Logger] = None,
-#         artifact_class_kwargs: Optional[Dict[str, Any]] = None,
-#     ) -> None:
-#         self.wrapped_classes = wrapped_classes
-#         self.max_workers = max_workers
-#         self.retry_attempts = retry_attempts
-#         self.update_timeout_seconds = update_timeout_seconds
-#         self.backoff_base = backoff_base
-#         self.backoff_max = backoff_max
-#         self.backoff_jitter = backoff_jitter
-#         self.priority_fn = priority_fn
-#         self.metrics_client = metrics_client
-#
-#         self.debug = debug
-#         self.logger = logger or Logger.default_logger(
-#             logger_name=self.__class__.__name__,
-#             log_level=Logger.DEBUG if debug else Logger.INFO
-#         )
-#
-#         # Default artifact init kwargs
-#         today = datetime.datetime.today() + datetime.timedelta(days=1)
-#         default_kwargs = {
-#             'parquet_start_date': today.strftime('%Y-%m-%d'),
-#             'parquet_end_date':   today.strftime('%Y-%m-%d'),
-#             'logger':             self.logger,
-#             'debug':              self.debug,
-#         }
-#         self.artifact_class_kwargs = artifact_class_kwargs or default_kwargs.copy()
-#
-#         # State
-#         self.completion_times: Dict[str, float] = {}
-#         self.failed: List[str] = []
-#         self.original_classes: List[Type] = []
-#
-#     def get_artifact_classes(self, data_type: str) -> List[Type]:
-#         """
-#         Retrieve artifact classes by data type.
-#         """
-#         self.logger.info(f"Fetching artifact classes for '{data_type}'")
-#         if data_type not in self.wrapped_classes:
-#             raise ValueError(f"Unsupported data type: {data_type}")
-#         classes = self.wrapped_classes[data_type]
-#         self.logger.info(f"Found {len(classes)} artifact classes for '{data_type}'")
-#         return classes
-#
-#     def estimate_priority(self, artifact_cls: Type) -> int:
-#         """
-#         Determine task priority for ordering. Lower values run first.
-#         """
-#         name = artifact_cls.__name__
-#         if self.priority_fn:
-#             try:
-#                 pr = self.priority_fn(artifact_cls)
-#                 self.logger.debug(f"priority_fn for {name}: {pr}")
-#                 return pr
-#             except Exception as e:
-#                 self.logger.warning(f"priority_fn error for {name}: {e}")
-#         try:
-#             fs = self.artifact_class_kwargs.get('fs')
-#             path = self.artifact_class_kwargs.get('parquet_storage_path')
-#             pr=1
-#             if hasattr(artifact_cls, 'get_size_estimate'):
-#                 pr = artifact_cls.get_size_estimate(fs, path)
-#             self.logger.debug(f"Estimated priority for {name}: {pr}")
-#             return pr
-#         except Exception:
-#             return 1
-#
-#     async def _bounded_update(self, artifact_cls: Type, sem: asyncio.Semaphore, **update_kwargs) -> None:
-#         """
-#         Wrap update_artifact in a semaphore slot to limit concurrency.
-#         """
-#         async with sem:
-#             name = artifact_cls.__name__
-#             start = asyncio.get_event_loop().time()
-#             self.logger.info(f"Starting update for {name}")
-#             try:
-#                 for attempt in range(1, self.retry_attempts + 1):
-#                     try:
-#                         artifact = await asyncio.to_thread(
-#                             artifact_cls, **self.artifact_class_kwargs
-#                         )
-#                         await asyncio.wait_for(
-#                             asyncio.to_thread(
-#                                 artifact.update_parquet, **update_kwargs
-#                             ),
-#                             timeout=self.update_timeout_seconds
-#                         )
-#                         duration = asyncio.get_event_loop().time() - start
-#                         self.completion_times[name] = duration
-#                         self.logger.info(f"✅ {name} updated in {duration:.2f}s (attempt {attempt})")
-#                         if self.metrics_client:
-#                             self.metrics_client.increment('task_succeeded')
-#                         return
-#                     except asyncio.TimeoutError:
-#                         self.logger.warning(f"Timeout on {name}, attempt {attempt}")
-#                     except Exception as e:
-#                         self.logger.error(f"Error on {name} attempt {attempt}: {e}")
-#
-#                     delay = min(self.backoff_base ** (attempt - 1), self.backoff_max)
-#                     delay *= 1 + random.uniform(0, self.backoff_jitter)
-#                     self.logger.info(f"Sleeping {delay:.1f}s before retrying {name}")
-#                     await asyncio.sleep(delay)
-#
-#             except asyncio.CancelledError:
-#                 self.logger.warning(f"{name} update cancelled")
-#                 raise
-#
-#             # permanent failure
-#             self.logger.error(f"✖️  {name} permanently failed after {self.retry_attempts} attempts")
-#             if self.metrics_client:
-#                 self.metrics_client.increment('task_failed')
-#             self.failed.append(name)
-#
-#     async def update_data(self, data_type: str, **kwargs: Any) -> None:
-#         """
-#         Entry point to update all artifacts of a given type concurrently.
-#         """
-#         self.logger.info(f"Starting update_data for '{data_type}' with kwargs={kwargs}")
-#
-#         # RESET STATE
-#         self.completion_times.clear()
-#         self.failed.clear()
-#         self.original_classes = self.get_artifact_classes(data_type)
-#
-#         # NON-DESTRUCTIVE SORTING
-#         ordered = sorted(self.original_classes, key=self.estimate_priority)
-#
-#         sem = asyncio.Semaphore(self.max_workers)
-#         tasks = [
-#             asyncio.create_task(self._bounded_update(cls, sem, **kwargs))
-#             for cls in ordered
-#         ]
-#
-#         try:
-#             for coro in asyncio.as_completed(tasks):
-#                 await coro
-#         except asyncio.CancelledError:
-#             self.logger.warning("update_data was cancelled—aborting remaining retries")
-#             for t in tasks:
-#                 t.cancel()
-#             raise
-#         finally:
-#             total = len(self.original_classes)
-#             completed = len(self.completion_times)
-#             failed = len(self.failed)
-#             self.logger.info(f"All artifacts processed: total={total}, completed={completed}, failed={failed}")
-#
-#     def get_update_status(self) -> Dict[str, Any]:
-#         """
-#         Returns summary status including completion times.
-#         """
-#         total = len(self.original_classes)
-#         completed = set(self.completion_times.keys())
-#         failed = set(self.failed)
-#         pending = {cls.__name__ for cls in self.original_classes} - completed - failed
-#
-#         return {
-#             'total': total,
-#             'completed': list(completed),
-#             'failed':    list(failed),
-#             'pending':   list(pending),
-#             'completion_times': self.completion_times,
-#         }
-#
-#     @staticmethod
-#     def format_status_table(status: Dict[str, Any]) -> str:
-#         """
-#         Formats the status dict into a readable table.
-#         """
-#         lines = [
-#             f"Total: {status['total']}",
-#             f"Completed: {len(status['completed'])}  {status['completed']}",
-#             f"Failed:    {len(status['failed'])}  {status['failed']}",
-#             f"Pending:   {len(status['pending'])}  {status['pending']}",
-#             "",
-#             "Per-artifact timings:"
-#         ]
-#         for name, dur in status['completion_times'].items():
-#             lines.append(f"  {name}: {dur:.2f}s")
-#         return "\n".join(lines)
+import asyncio
+import datetime
+import random
+from typing import Any, Callable, Dict, List, Optional, Type
+
+class ArtifactUpdaterMultiWrapperAsync(ManagedResource):
+    """
+    Simplified wrapper that updates artifacts concurrently using an asyncio.Semaphore.
+
+    Features:
+    - Caps concurrency at max_workers via semaphore
+    - Optionally prioritises tasks via a priority function or static method on artifact classes
+    - Tracks per-artifact completion times
+    - Configurable retry/backoff strategy
+    - Optional metrics integration
+    - Thread-safe within a single asyncio loop
+
+    Usage:
+        wrapper = ArtifactUpdaterMultiWrapper(
+            wrapped_classes={
+                'mydata': [DataArtifactA, DataArtifactB],
+            },
+            max_workers=4,
+            retry_attempts=3,
+            update_timeout_seconds=600,
+            backoff_base=2,
+            backoff_max=60,
+            backoff_jitter=0.1,
+            priority_fn=None,  # or custom
+            metrics_client=None,
+            debug=True,
+            logger=None,
+            artifact_class_kwargs={
+                'fs': my_fs,
+                'parquet_storage_path': 's3://bucket/data',
+                'logger': my_logger,
+                'debug': True,
+            }
+        )
+        await wrapper.update_data('mydata', period='ytd', overwrite=True)
+    """
+    def __init__(
+        self,
+        wrapped_classes: Dict[str, List[Type]],
+        *,
+        max_workers: int = 3,
+        retry_attempts: int = 3,
+        update_timeout_seconds: int = 600,
+        backoff_base: int = 2,
+        backoff_max: Optional[int] = 60,
+        backoff_jitter: float = 0.1,
+        priority_fn: Optional[Callable[[Type], int]] = None,
+        metrics_client: Any = None,
+        artifact_class_kwargs: Optional[Dict[str, Any]] = None,
+        **kwargs: Dict[str, Any]
+    ) -> None:
+        super().__init__(**kwargs)
+        self.wrapped_classes = wrapped_classes
+        self.max_workers = max_workers
+        self.retry_attempts = retry_attempts
+        self.update_timeout_seconds = update_timeout_seconds
+        self.backoff_base = backoff_base
+        self.backoff_max = backoff_max
+        self.backoff_jitter = backoff_jitter
+        self.priority_fn = priority_fn
+        self.metrics_client = metrics_client
+
+        # Default artifact init kwargs
+        today = datetime.datetime.today() + datetime.timedelta(days=1)
+        default_kwargs = {
+            'parquet_start_date': today.strftime('%Y-%m-%d'),
+            'parquet_end_date': today.strftime('%Y-%m-%d'),
+            'logger': self.logger,
+            'debug': self.debug,
+            'fs': self.fs,
+            'verbose': self.verbose,
+        }
+        self.artifact_class_kwargs = artifact_class_kwargs or default_kwargs.copy()
+
+        # State
+        self.completion_times: Dict[str, float] = {}
+        self.failed: List[str] = []
+        self.original_classes: List[Type] = []
+
+    def get_artifact_classes(self, data_type: str) -> List[Type]:
+        """
+        Retrieve artifact classes by data type.
+        """
+        self.logger.info(f"Fetching artifact classes for '{data_type}'")
+        if data_type not in self.wrapped_classes:
+            raise ValueError(f"Unsupported data type: {data_type}")
+        classes = self.wrapped_classes[data_type]
+        self.logger.info(f"Found {len(classes)} artifact classes for '{data_type}'")
+        return classes
+
+    def estimate_priority(self, artifact_cls: Type) -> int:
+        """
+        Determine task priority for ordering. Lower values run first.
+        """
+        name = artifact_cls.__name__
+        if self.priority_fn:
+            try:
+                pr = self.priority_fn(artifact_cls)
+                self.logger.debug(f"priority_fn for {name}: {pr}")
+                return pr
+            except Exception as e:
+                self.logger.warning(f"priority_fn error for {name}: {e}")
+        try:
+            fs = self.artifact_class_kwargs.get('fs')
+            path = self.artifact_class_kwargs.get('parquet_storage_path')
+            pr=1
+            if hasattr(artifact_cls, 'get_size_estimate'):
+                pr = artifact_cls.get_size_estimate(fs, path)
+            self.logger.debug(f"Estimated priority for {name}: {pr}")
+            return pr
+        except Exception:
+            return 1
+
+    async def _bounded_update(self, artifact_cls: Type, sem: asyncio.Semaphore, **update_kwargs) -> None:
+        """
+        Wrap update_artifact in a semaphore slot to limit concurrency.
+        """
+        async with sem:
+            name = artifact_cls.__name__
+            start = asyncio.get_event_loop().time()
+            self.logger.info(f"Starting update for {name}")
+            try:
+                for attempt in range(1, self.retry_attempts + 1):
+                    try:
+                        artifact = await asyncio.to_thread(
+                            artifact_cls, **self.artifact_class_kwargs
+                        )
+                        await asyncio.wait_for(
+                            asyncio.to_thread(
+                                artifact.update_parquet, **update_kwargs
+                            ),
+                            timeout=self.update_timeout_seconds
+                        )
+                        duration = asyncio.get_event_loop().time() - start
+                        self.completion_times[name] = duration
+                        self.logger.info(f"✅ {name} updated in {duration:.2f}s (attempt {attempt})")
+                        if self.metrics_client:
+                            self.metrics_client.increment('task_succeeded')
+                        return
+                    except asyncio.TimeoutError:
+                        self.logger.warning(f"Timeout on {name}, attempt {attempt}")
+                    except Exception as e:
+                        self.logger.error(f"Error on {name} attempt {attempt}: {e}")
+
+                    delay = min(self.backoff_base ** (attempt - 1), self.backoff_max)
+                    delay *= 1 + random.uniform(0, self.backoff_jitter)
+                    self.logger.info(f"Sleeping {delay:.1f}s before retrying {name}")
+                    await asyncio.sleep(delay)
+
+            except asyncio.CancelledError:
+                self.logger.warning(f"{name} update cancelled")
+                raise
+
+            # permanent failure
+            self.logger.error(f"✖️  {name} permanently failed after {self.retry_attempts} attempts")
+            if self.metrics_client:
+                self.metrics_client.increment('task_failed')
+            self.failed.append(name)
+
+    async def update_data(self, data_type: str, **kwargs: Any) -> None:
+        """
+        Entry point to update all artifacts of a given type concurrently.
+        """
+        self.logger.info(f"Starting update_data for '{data_type}' with kwargs={kwargs}")
+
+        # RESET STATE
+        self.completion_times.clear()
+        self.failed.clear()
+        self.original_classes = self.get_artifact_classes(data_type)
+
+        # NON-DESTRUCTIVE SORTING
+        ordered = sorted(self.original_classes, key=self.estimate_priority)
+
+        sem = asyncio.Semaphore(self.max_workers)
+        tasks = [
+            asyncio.create_task(self._bounded_update(cls, sem, **kwargs))
+            for cls in ordered
+        ]
+
+        try:
+            for coro in asyncio.as_completed(tasks):
+                await coro
+        except asyncio.CancelledError:
+            self.logger.warning("update_data was cancelled—aborting remaining retries")
+            for t in tasks:
+                t.cancel()
+            raise
+        finally:
+            total = len(self.original_classes)
+            completed = len(self.completion_times)
+            failed = len(self.failed)
+            self.logger.info(f"All artifacts processed: total={total}, completed={completed}, failed={failed}")
+
+    def get_update_status(self) -> Dict[str, Any]:
+        """
+        Returns summary status including completion times.
+        """
+        total = len(self.original_classes)
+        completed = set(self.completion_times.keys())
+        failed = set(self.failed)
+        pending = {cls.__name__ for cls in self.original_classes} - completed - failed
+
+        return {
+            'total': total,
+            'completed': list(completed),
+            'failed':    list(failed),
+            'pending':   list(pending),
+            'completion_times': self.completion_times,
+        }
+
+    @staticmethod
+    def format_status_table(status: Dict[str, Any]) -> str:
+        """
+        Formats the status dict into a readable table.
+        """
+        lines = [
+            f"Total: {status['total']}",
+            f"Completed: {len(status['completed'])}  {status['completed']}",
+            f"Failed:    {len(status['failed'])}  {status['failed']}",
+            f"Pending:   {len(status['pending'])}  {status['pending']}",
+            "",
+            "Per-artifact timings:"
+        ]
+        for name, dur in status['completion_times'].items():
+            lines.append(f"  {name}: {dur:.2f}s")
+        return "\n".join(lines)

sibi_dst/df_helper/_parquet_artifact.py

@@ -1,8 +1,7 @@
 from __future__ import annotations
 import datetime
-import logging
 import threading
-from typing import Optional, Any, Dict
+from typing import Optional, Any, Dict, ClassVar
 
 import dask.dataframe as dd
 import fsspec
@@ -55,7 +54,7 @@ class ParquetArtifact(DfHelper):
     :ivar fs: Filesystem object used for storage operations.
     :type fs: fsspec.AbstractFileSystem
     """
-    DEFAULT_CONFIG = {
+    DEFAULT_CONFIG: ClassVar[Dict[str, str]] = {
         'backend': 'parquet'
     }
 
@@ -91,8 +90,6 @@ class ParquetArtifact(DfHelper):
         }
         self.df: Optional[dd.DataFrame] = None
         super().__init__(**self.config)
-        #self._own_logger = False
-        #self._setup_logging()
         self.data_wrapper_class = data_wrapper_class
 
         self.date_field = self._validate_required('date_field')
@@ -101,16 +98,6 @@ class ParquetArtifact(DfHelper):
         self.parquet_start_date = self._validate_required('parquet_start_date')
         self.parquet_end_date = self._validate_required('parquet_end_date')
 
-        # Filesystem setup
-        #self.filesystem_type = self.config.setdefault('filesystem_type', 'file')
-        #self.filesystem_options = self.config.setdefault('filesystem_options', {})
-        #self.fs = self.config.setdefault('fs', None)
-        #self._own_fs = self.fs is None
-        #if self.fs is None:
-        #    self.fs = fsspec.filesystem(self.filesystem_type, **self.filesystem_options)
-        #    self._own_fs = True
-        #self.config.setdefault('fs', self.fs)
-        ## Populate to parameters to pass to data_wrapper_class
         self.class_params = self.config.pop('class_params', {
             'debug': self.debug,
             'logger': self.logger,
@@ -125,15 +112,6 @@ class ParquetArtifact(DfHelper):
         self.update_planner_params = {}
         self.datawrapper_params = {}
 
-    #def _setup_logging(self):
-    #    """Initialize logger and debug settings."""
-    #    self.debug = self.config.get('debug', False)
-    #    logger = self.config.get('logger', None)
-    #    self._own_logger = logger is None
-    #    self.logger = logger or Logger.default_logger(
-    #                                      logger_name=f'Parquet_Artifact_InstanceOf_{self.__class__.__name__}')
-    #    self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
-
     def _validate_required(self, key: str) -> Any:
         """Validate required configuration fields."""
         value = self.config.setdefault(key, None)
@@ -211,28 +189,28 @@ class ParquetArtifact(DfHelper):
         """
         Synchronously estimates artifact size for use in multi-threaded environments.
 
-        This method uses the filesystem's own .sync() method to safely execute
-        asynchronous I/O operations from a synchronous context, preventing
-        event loop conflicts.
+        This method safely executes asynchronous I/O operations from a synchronous
+        context, handling variations in fsspec filesystem implementations.
         """
 
         async def _get_total_bytes_async():
             """A helper async coroutine to perform the I/O."""
             import asyncio
 
-            # Use the async versions of fsspec methods (e.g., _glob, _size)
             files = await self.fs._glob(f"{self.parquet_storage_path}/*.parquet")
             if not files:
                 return 0
 
-            # Concurrently gather the size of all files for performance
             size_tasks = [self.fs._size(f) for f in files]
             sizes = await asyncio.gather(*size_tasks)
             return sum(s for s in sizes if s is not None)
 
-        # Use the filesystem's own built-in sync method. This is the most
-        # reliable way to bridge the sync/async gap for fsspec.
-        total_bytes = self.fs.sync(_get_total_bytes_async())
+        try:
+            # Attempt the standard fsspec method first
+            total_bytes = self.fs.sync(_get_total_bytes_async())
+        except AttributeError:
+            #  fallback for filesystems like s3fs that lack .sync()
+            total_bytes = self.fs.loop.run_until_complete(_get_total_bytes_async())
 
         # Convert to megabytes, ensuring a minimum of 1
         return max(1, int(total_bytes / (1024 ** 2)))

sibi_dst/df_helper/_parquet_reader.py

@@ -1,11 +1,9 @@
-import logging
-from typing import Optional
+from typing import Optional, ClassVar, Dict
 
 import dask.dataframe as dd
 import fsspec
 
 from sibi_dst.df_helper import DfHelper
-from sibi_dst.utils import Logger
 
 class ParquetReader(DfHelper):
     """
@@ -44,19 +42,17 @@ class ParquetReader(DfHelper):
         Parquet storage.
     :type fs: fsspec.AbstractFileSystem
     """
-    DEFAULT_CONFIG = {
+    DEFAULT_CONFIG: ClassVar[Dict[str, int]] = {
         'backend': 'parquet'
     }
 
-    def __init__(self, filesystem_type="file", filesystem_options=None, **kwargs):
+    def __init__(self,  **kwargs):
         self.config = {
             **self.DEFAULT_CONFIG,
             **kwargs,
         }
-        self.df: Optional[dd.DataFrame] = None
-        #self.debug = self.config.setdefault('debug', False)
-        #self.logger = self.config.setdefault('logger', Logger.default_logger(logger_name=self.__class__.__name__))
-        #self.logger.set_level(logging.DEBUG if self.debug else logging.INFO)
+        super().__init__(**self.config)
+
         self.parquet_storage_path = self.config.setdefault('parquet_storage_path', None)
         if self.parquet_storage_path is None:
             raise ValueError('parquet_storage_path must be set')
@@ -68,19 +64,9 @@ class ParquetReader(DfHelper):
         if self.parquet_end_date is None:
             raise ValueError('parquet_end_date must be set')
 
-        # Filesystem setup
-        #self.filesystem_type = filesystem_type
-        #self.filesystem_options = filesystem_options or {}
-        #self.fs = self.config.setdefault('fs', None)
-        #if self.fs is None:
-        #    self.fs = fsspec.filesystem(self.filesystem_type, **self.filesystem_options)
-        #self.config.setdefault('fs', self.fs)
-
         if not self.directory_exists():
             raise ValueError(f"{self.parquet_storage_path} does not exist")
 
-        super().__init__(**self.config)
-
     def load(self, **kwargs):
         self.df = super().load(**kwargs)
         return self.df
@@ -90,5 +76,4 @@ class ParquetReader(DfHelper):
             info = self.fs.info(self.parquet_storage_path)
             return info['type'] == 'directory'
         except FileNotFoundError:
-            return False
-
+            return False

sibi_dst/df_helper/backends/parquet/_parquet_options.py

@@ -85,7 +85,8 @@ class ParquetConfig(BaseModel):
         if self.logger is None:
             self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
         self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
-
+        if self.fs is None:
+            raise ValueError('Parquet Options: File system (fs) must be specified')
 
         if self.parquet_storage_path is None:
             raise ValueError('Parquet storage path must be specified')

sibi_dst/utils/base.py

@@ -1,3 +1,4 @@
+import asyncio
 from .log_utils import Logger
 
 class ManagedResource:
@@ -61,7 +62,7 @@ class ManagedResource:
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         """Exit the runtime context and trigger cleanup for 'async with' statements."""
-        self.cleanup()
+        await self.acleanup()
         return False  # Propagate exceptions
 
     def __repr__(self) -> str:
@@ -80,7 +81,7 @@ class ManagedResource:
 
     def cleanup(self):
         """
-        Clean up resources managed by this instance.
+        Cleanup resources managed by this instance.
         """
         if self._own_fs and hasattr(self.fs, "clear_instance_cache"):
             if self.logger:
@@ -88,10 +89,29 @@ class ManagedResource:
             self.fs.clear_instance_cache()
 
         if self._own_logger and hasattr(self.logger, "shutdown"):
-            # Ensure logger exists before trying to use or shut it down
+            # Ensure the logger exists before trying to use or shut it down
             if self.logger:
                 self.logger.debug(f"'{self.__class__.__name__}' is shutting down its own logger.")
                 self.logger.shutdown()
             self.logger = None  # Set to None after shutdown
 
         self._entered = False
+
+    async def acleanup(self):
+        """
+        Async Cleanup resources managed by this instance.
+        """
+        if self._own_fs and hasattr(self.fs, "clear_instance_cache"):
+            if self.logger:
+                self.logger.debug(f"'{self.__class__.__name__}' is clearing its own filesystem cache.")
+            self.fs.clear_instance_cache()
+
+        if self._own_logger and hasattr(self.logger, "shutdown"):
+            # Ensure the logger exists before trying to use or shut it down
+            if self.logger:
+                self.logger.debug(f"'{self.__class__.__name__}' is shutting down its own logger.")
+                self.logger.shutdown()
+            self.logger = None  # Set to None after shutdown
+
+        self._entered = False
+

sibi_dst/utils/clickhouse_writer.py

@@ -1,4 +1,5 @@
 from concurrent.futures import ThreadPoolExecutor
+from typing import ClassVar, Dict
 
 import clickhouse_connect
 import pandas as pd
@@ -36,7 +37,7 @@ class ClickHouseWriter(ManagedResource):
     :ivar order_by: Field or column name to use for table ordering.
     :type order_by: str
     """
-    dtype_to_clickhouse = {
+    dtype_to_clickhouse:  ClassVar[Dict[str, str]] = {
         'int64': 'Int64',
         'int32': 'Int32',
         'float64': 'Float64',

sibi_dst/utils/data_wrapper.py

@@ -3,7 +3,7 @@ import logging
 import threading
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Type, Any, Dict, Optional, Union, List
+from typing import Type, Any, Dict, Optional, Union, List, ClassVar
 
 import fsspec
 import pandas as pd
@@ -15,15 +15,15 @@ from .parquet_saver import ParquetSaver
 
 
 class DataWrapper(ManagedResource):
-    DEFAULT_PRIORITY_MAP = {
+    DEFAULT_PRIORITY_MAP: ClassVar[Dict[str, int]] = {
         "overwrite": 1,
         "missing_in_history": 2,
         "existing_but_stale": 3,
         "missing_outside_history": 4,
         "file_is_recent": 0
     }
-    DEFAULT_MAX_AGE_MINUTES = 1440
-    DEFAULT_HISTORY_DAYS_THRESHOLD = 30
+    DEFAULT_MAX_AGE_MINUTES: int = 1440
+    DEFAULT_HISTORY_DAYS_THRESHOLD: int = 30
 
     def __init__(
             self,
@@ -31,12 +31,8 @@ class DataWrapper(ManagedResource):
             date_field: str,
             data_path: str,
             parquet_filename: str,
-            #fs: Optional[fsspec.AbstractFileSystem] = None,
-            #debug: bool = False,
-            #verbose: bool = False,
             class_params: Optional[Dict] = None,
             load_params: Optional[Dict] = None,
-            #logger: Logger = None,
             show_progress: bool = False,
             timeout: float = 30,
             max_threads: int = 3,
@@ -47,14 +43,8 @@ class DataWrapper(ManagedResource):
         self.date_field = date_field
         self.data_path = self._ensure_forward_slash(data_path)
         self.parquet_filename = parquet_filename
-        #self.fs = fs or None
         if self.fs is None:
             raise ValueError("Datawrapper requires a File system (fs) to be provided .")
-        #self.debug = debug
-        #self.verbose = verbose
-        #self._own_logger = logger is None
-        #self.logger = logger or Logger.default_logger(logger_name=self.dataclass.__name__)
-        #self.logger.set_level(logging.DEBUG if debug else logging.INFO)
         self.show_progress = show_progress
         self.timeout = timeout
         self.max_threads = max_threads
@@ -71,7 +61,6 @@ class DataWrapper(ManagedResource):
         self.benchmarks: Dict[datetime.date, Dict[str, float]] = {}
         self.mmanifest = kwargs.get("mmanifest", None)
         self.update_planner=kwargs.get("update_planner", None)
-        # self.datacls = self.dataclass(**self.class_params)
 
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -174,7 +163,7 @@ class DataWrapper(ManagedResource):
             load_time = time.perf_counter() - load_start
 
             if hasattr(local_class_instance, "total_records"):
-                self.logger.debug(f"Total records loaded by {local_class_instance}: {local_class_instance.total_records}")
+                self.logger.debug(f"Total records loaded by {local_class_instance.__class__.__name__}: {local_class_instance.total_records}")
                 if int(local_class_instance.total_records) == 0:  # If no records were loaded but not due to an error
                     if self.mmanifest:
                         self.mmanifest.record(
@@ -183,12 +172,13 @@ class DataWrapper(ManagedResource):
                     self.logger.info(f"No data found for {date}. Logged to missing manifest.")
                     return
             save_start = time.perf_counter()
-            ParquetSaver(
+            with ParquetSaver(
                 df_result=df,
                 parquet_storage_path=path,
                 fs=self.fs,
                 logger=self.logger
-            ).save_to_parquet(self.parquet_filename, overwrite=True)
+            ) as ps:
+                ps.save_to_parquet(self.parquet_filename, overwrite=True)
             save_time = time.perf_counter() - save_start
 
             total_time = time.perf_counter() - overall_start
@@ -218,4 +208,4 @@ class DataWrapper(ManagedResource):
             return
         df_bench = pd.DataFrame.from_records([{"date": d, **m} for d, m in self.benchmarks.items()])
         df_bench = df_bench.set_index("date").sort_index(ascending=not self.update_planner.reverse_order)
-        self.logger.info("Benchmark Summary:\n" + df_bench.to_string())
+        self.logger.info(f"Benchmark Summary:\n {self.dataclass.__name__}\n" + df_bench.to_string())

sibi_dst/utils/update_planner.py

@@ -1,6 +1,6 @@
 import datetime
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import List, Optional, Dict, Union, Tuple, Set, Iterator
+from typing import List, Optional, Dict, Union, Tuple, Set, Iterator, ClassVar
 import pandas as pd
 from .date_utils import FileAgeChecker
 from pydantic import BaseModel, Field
@@ -55,7 +55,7 @@ class UpdatePlanner(ManagedResource):
         generate_plan() will overwrite self.plan and self.df_req, and returns a DataFrame of required updates.
     """
 
-    DEFAULT_PRIORITY_MAP = {
+    DEFAULT_PRIORITY_MAP: ClassVar[Dict[str, int]]={
         "file_is_recent": 0,
         "missing_ignored": 0,
         "overwrite_forced": 1,
@@ -64,8 +64,8 @@ class UpdatePlanner(ManagedResource):
         "stale_in_history": 4,
     }
 
-    DEFAULT_MAX_AGE_MINUTES = 1440
-    DEFAULT_HISTORY_DAYS_THRESHOLD = 30
+    DEFAULT_MAX_AGE_MINUTES: int = 1440
+    DEFAULT_HISTORY_DAYS_THRESHOLD: int = 30
 
     def __init__(
             self,
@@ -217,7 +217,7 @@ class UpdatePlanner(ManagedResource):
         for priority in sorted(required_updates["update_priority"].unique()):
             dates_df = required_updates[required_updates["update_priority"] == priority]
             # Sort dates within the priority group
-            sorted_dates = dates_df.sort_values(by="date", ascending=not self.reverse_order)
+            sorted_dates = dates_df.sort_values(by=["date"], ascending=not self.reverse_order)
             dates = sorted_dates["date"].tolist()
             if dates:
                 yield priority, dates

{sibi_dst-2025.1.5.dist-info → sibi_dst-2025.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.1.5
+Version: 2025.1.7
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -13,6 +13,8 @@ Requires-Dist: clickhouse-connect (>=0.8.18,<0.9.0)
 Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
 Requires-Dist: dask[complete] (>=2025.5.1,<2026.0.0)
 Requires-Dist: mysqlclient (>=2.2.7,<3.0.0)
+Requires-Dist: opentelemetry-exporter-otlp (>=1.35.0,<2.0.0)
+Requires-Dist: opentelemetry-sdk (>=1.35.0,<2.0.0)
 Requires-Dist: pandas (>=2.3.1,<3.0.0)
 Requires-Dist: psycopg2 (>=2.9.10,<3.0.0)
 Requires-Dist: pyarrow (>=20.0.0,<21.0.0)

{sibi_dst-2025.1.5.dist-info → sibi_dst-2025.1.7.dist-info}/RECORD

@@ -1,15 +1,15 @@
 sibi_dst/__init__.py,sha256=j8lZpGCJlxlLgEgeIMxZnWdqJ0g3MCs7-gsnbvPn_KY,285
-sibi_dst/df_helper/__init__.py,sha256=VJE1qvKO-7QsFADZxSY5s4LVoWnPKfz0rP3nYO2ljhA,358
-sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=LvFCNr4VKFV-b-NS_TeRkaoKsM4tdsPtAgSIwMvKgGA,18043
+sibi_dst/df_helper/__init__.py,sha256=Jur_MO8RGPkVw0CS3XH5YIWv-d922DC_FwRDTvHHV6Y,432
+sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=10EkCYEfoWwTQbS-ahYWo6TvbtNXM8p0UqqDu0gTuyI,17426
 sibi_dst/df_helper/_df_helper.py,sha256=iBoWz2iVgLzQ3hA1EwllL62dkraKamRx2sXseu30FVI,11914
-sibi_dst/df_helper/_parquet_artifact.py,sha256=vDZOtSugBuWuZ3W6l2Y7IBO6RohIrA_sVNuPHp8e8h0,15438
-sibi_dst/df_helper/_parquet_reader.py,sha256=o5ijxHtD1EMzUUD9e6PIoGMeuSLHDItvZIouGfVZhgA,3817
+sibi_dst/df_helper/_parquet_artifact.py,sha256=dCvUA2bytv0wY0pFI8lxbcLwXlgGpHndS36iKfEmjLw,14310
+sibi_dst/df_helper/_parquet_reader.py,sha256=m98C0TZRroOXvVc2LpEuElrJnquGlR81E1gjI7v1hi4,3102
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/df_helper/backends/http/__init__.py,sha256=d1pfgYxbiYg7E0Iw8RbJ7xfqIfJShqqTBQQGU_S6OOo,105
 sibi_dst/df_helper/backends/http/_http_config.py,sha256=eGPFdqZ5M3Tscqx2P93B6XoBEEzlmdt7yNg7PXUQnNQ,4726
 sibi_dst/df_helper/backends/parquet/__init__.py,sha256=esWJ9aSuYC26d-T01z9dPrJ1uqJzvdaPNTYRb5qXTlQ,182
 sibi_dst/df_helper/backends/parquet/_filter_handler.py,sha256=TvDf0RXta7mwJv11GNQttYJsXgFf2XDj4oLIjt4xTzA,5219
-sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=TaU5_wG1Y3lQC8DVCItVvMnc6ZJmECLu3avssVEMbaM,10591
+sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=lrDn2-BbgxDor5g71LAu5LDg2g3ApGAPiQfbFTB2xNA,10702
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=LjWm9B7CweTvlvFOgB90XjSe0lVLILAIYMWKPkFXFm8,265
 sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=Rsvh1nfVtqzfMhv968vNTYYIqVxYsEs4PB-O5CTSYdk,10935
 sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=NqBSHqeYv_1vHt6J0tez0GdMwKrP_sIRcXYXu869ZkY,13313
@@ -33,12 +33,12 @@ sibi_dst/osmnx_helper/utils.py,sha256=BzuY8CtYnBAAO8UAr_M7EOk6CP1zcifNLs8pkdFZEF
 sibi_dst/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/tests/test_data_wrapper_class.py,sha256=6uFmZR2DxnxQz49L5jT2ehlKvlLnpUHMLFB_PqqUq7k,3336
 sibi_dst/utils/__init__.py,sha256=PQsG188_lnqgSFljkCc15Nyv933HnvmQ7XYs02m77Vc,1217
-sibi_dst/utils/base.py,sha256=MBshlQKg-WNeTeuE_aMLQjcBRaa4O_TFED7bVKDhJ-o,3783
-sibi_dst/utils/clickhouse_writer.py,sha256=5XDRjXU9d0Vhb9GFdCiRoNXrucJvTm6h8auCAQbEwW0,9917
+sibi_dst/utils/base.py,sha256=RGLcCpGeWTKxsAl9wcxicaS6nfq3lFdsgE9XiOOD_a8,4568
+sibi_dst/utils/clickhouse_writer.py,sha256=mNUJoYOreIdRrEFv2mQ6pdtLi1Iz_2rALDyO6ARTxhs,9978
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
-sibi_dst/utils/data_wrapper.py,sha256=Vx3At8SlAoMCTaXmVsTiTGynfjV2isB9W6yL0cWZ7g4,9346
+sibi_dst/utils/data_wrapper.py,sha256=8lbKe85XJB0VhIR9GRo7-R_9DBhRZvNv8MsqSbsupeA,8945
 sibi_dst/utils/date_utils.py,sha256=8fwPpOYqSdM3nHeNykh7Ftk-uPdFa44cEAy5S8iUNw4,18667
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
@@ -49,7 +49,7 @@ sibi_dst/utils/parquet_saver.py,sha256=zau_s0Mn2ccz9ivVtRbTkBmCghUgCofI1LsCdy1df
 sibi_dst/utils/phone_formatter.py,sha256=tsVTDamuthFYgy4-5UwmQkPQ-FGTGH7MjZyH8utAkIY,4945
 sibi_dst/utils/storage_config.py,sha256=uaCBF8rgCeYkk-lxVSCjsic8O8HJKAu455MR-OBliCo,4325
 sibi_dst/utils/storage_manager.py,sha256=yyZqT8XjTf4MKFrfznCmxXxOYz_TiWgtQhzqPoXR9So,6569
-sibi_dst/utils/update_planner.py,sha256=2ZVsuZlghKDRv7IhqaraS-7GRIY6nGRpFnpBdXYo7Io,11538
+sibi_dst/utils/update_planner.py,sha256=UH14HVFUtvWtCY9jzmYbQbe7lUHRrpnNDn2h0JT7OvI,11584
 sibi_dst/utils/webdav_client.py,sha256=pYF1UsGOuxYeGLq7aBfwZFvkvD4meOcbbaiZ4d6GW9I,7107
 sibi_dst/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/v2/df_helper/__init__.py,sha256=XuH6jKYAPg2DdRbsxxBSxp9X3x-ARyaT0xe27uILrVo,99
@@ -71,6 +71,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-2025.1.5.dist-info/METADATA,sha256=T0zFKtNSQ7if1S590EwTZ_CN96oiDe8t559-xFQ-XWM,2498
-sibi_dst-2025.1.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-2025.1.5.dist-info/RECORD,,
+sibi_dst-2025.1.7.dist-info/METADATA,sha256=AaJunhF_PdvxT9KpA0mVzQhoY1sAZoe5HTA9ClabYPs,2610
+sibi_dst-2025.1.7.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-2025.1.7.dist-info/RECORD,,