sibi-dst 0.3.58__tar.gz → 0.3.59__tar.gz

{sibi_dst-0.3.58 → sibi_dst-0.3.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.58
+Version: 0.3.59
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.58 → sibi_dst-0.3.59}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.58"
+version = "0.3.59"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

sibi_dst-0.3.59/sibi_dst/df_helper/_artifact_updater_multi_wrapper.py

@@ -0,0 +1,239 @@
+import asyncio
+import logging
+import datetime
+import random
+from typing import Any, Callable, Dict, List, Optional, Type
+
+from sibi_dst.utils import Logger
+
+
+class ArtifactUpdaterMultiWrapper:
+    """
+    Simplified wrapper that updates artifacts concurrently using an asyncio.Semaphore.
+
+    Features:
+    - Caps concurrency at max_workers via semaphore
+    - Optionally prioritises tasks via a priority function or static method on artifact classes
+    - Tracks per-artifact completion times
+    - Configurable retry/backoff strategy
+    - Optional metrics integration
+    - Thread-safe within a single asyncio loop
+
+    Usage:
+        wrapper = ArtifactUpdaterMultiWrapper(
+            wrapped_classes={
+                'mydata': [DataArtifactA, DataArtifactB],
+            },
+            max_workers=4,
+            retry_attempts=3,
+            update_timeout_seconds=600,
+            backoff_base=2,
+            backoff_max=60,
+            backoff_jitter=0.1,
+            priority_fn=None,  # or custom
+            metrics_client=None,
+            debug=True,
+            logger=None,
+            artifact_class_kwargs={
+                'fs': my_fs,
+                'parquet_storage_path': 's3://bucket/data',
+                'logger': my_logger,
+                'debug': True,
+            }
+        )
+        await wrapper.update_data('mydata', period='ytd', overwrite=True)
+    """
+    def __init__(
+        self,
+        wrapped_classes: Dict[str, List[Type]],
+        *,
+        max_workers: int = 3,
+        retry_attempts: int = 3,
+        update_timeout_seconds: int = 600,
+        backoff_base: int = 2,
+        backoff_max: Optional[int] = 60,
+        backoff_jitter: float = 0.1,
+        priority_fn: Optional[Callable[[Type], int]] = None,
+        metrics_client: Any = None,
+        debug: bool = False,
+        logger: Optional[logging.Logger] = None,
+        artifact_class_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        self.wrapped_classes = wrapped_classes
+        self.max_workers = max_workers
+        self.retry_attempts = retry_attempts
+        self.update_timeout_seconds = update_timeout_seconds
+        self.backoff_base = backoff_base
+        self.backoff_max = backoff_max
+        self.backoff_jitter = backoff_jitter
+        self.priority_fn = priority_fn
+        self.metrics_client = metrics_client
+
+        self.debug = debug
+        self.logger = logger or Logger.default_logger(
+            logger_name=self.__class__.__name__,
+            log_level=Logger.DEBUG if debug else Logger.INFO
+        )
+
+        # Default artifact init kwargs
+        today = datetime.datetime.today() + datetime.timedelta(days=1)
+        default_kwargs = {
+            'parquet_start_date': today.strftime('%Y-%m-%d'),
+            'parquet_end_date':   today.strftime('%Y-%m-%d'),
+            'logger':             self.logger,
+            'debug':              self.debug,
+        }
+        self.artifact_class_kwargs = artifact_class_kwargs or default_kwargs.copy()
+
+        # State
+        self.completion_times: Dict[str, float] = {}
+        self.failed: List[str] = []
+        self.original_classes: List[Type] = []
+
+    def get_artifact_classes(self, data_type: str) -> List[Type]:
+        """
+        Retrieve artifact classes by data type.
+        """
+        self.logger.info(f"Fetching artifact classes for '{data_type}'")
+        if data_type not in self.wrapped_classes:
+            raise ValueError(f"Unsupported data type: {data_type}")
+        classes = self.wrapped_classes[data_type]
+        self.logger.info(f"Found {len(classes)} artifact classes for '{data_type}'")
+        return classes
+
+    def estimate_priority(self, artifact_cls: Type) -> int:
+        """
+        Determine task priority for ordering. Lower values run first.
+        """
+        name = artifact_cls.__name__
+        if self.priority_fn:
+            try:
+                pr = self.priority_fn(artifact_cls)
+                self.logger.debug(f"priority_fn for {name}: {pr}")
+                return pr
+            except Exception as e:
+                self.logger.warning(f"priority_fn error for {name}: {e}")
+        try:
+            fs = self.artifact_class_kwargs.get('fs')
+            path = self.artifact_class_kwargs.get('parquet_storage_path')
+            pr=1
+            if hasattr(artifact_cls, 'get_size_estimate'):
+                pr = artifact_cls.get_size_estimate(fs, path)
+            self.logger.debug(f"Estimated priority for {name}: {pr}")
+            return pr
+        except Exception:
+            return 1
+
+    async def _bounded_update(self, artifact_cls: Type, sem: asyncio.Semaphore, **update_kwargs) -> None:
+        """
+        Wrap update_artifact in a semaphore slot to limit concurrency.
+        """
+        async with sem:
+            name = artifact_cls.__name__
+            start = asyncio.get_event_loop().time()
+            self.logger.info(f"Starting update for {name}")
+            try:
+                for attempt in range(1, self.retry_attempts + 1):
+                    try:
+                        artifact = await asyncio.to_thread(
+                            artifact_cls, **self.artifact_class_kwargs
+                        )
+                        await asyncio.wait_for(
+                            asyncio.to_thread(
+                                artifact.update_parquet, **update_kwargs
+                            ),
+                            timeout=self.update_timeout_seconds
+                        )
+                        duration = asyncio.get_event_loop().time() - start
+                        self.completion_times[name] = duration
+                        self.logger.info(f"✅ {name} updated in {duration:.2f}s (attempt {attempt})")
+                        if self.metrics_client:
+                            self.metrics_client.increment('task_succeeded')
+                        return
+                    except asyncio.TimeoutError:
+                        self.logger.warning(f"Timeout on {name}, attempt {attempt}")
+                    except Exception as e:
+                        self.logger.error(f"Error on {name} attempt {attempt}: {e}")
+
+                    delay = min(self.backoff_base ** (attempt - 1), self.backoff_max)
+                    delay *= 1 + random.uniform(0, self.backoff_jitter)
+                    self.logger.info(f"Sleeping {delay:.1f}s before retrying {name}")
+                    await asyncio.sleep(delay)
+
+            except asyncio.CancelledError:
+                self.logger.warning(f"{name} update cancelled")
+                raise
+
+            # permanent failure
+            self.logger.error(f"✖️  {name} permanently failed after {self.retry_attempts} attempts")
+            if self.metrics_client:
+                self.metrics_client.increment('task_failed')
+            self.failed.append(name)
+
+    async def update_data(self, data_type: str, **kwargs: Any) -> None:
+        """
+        Entry point to update all artifacts of a given type concurrently.
+        """
+        self.logger.info(f"Starting update_data for '{data_type}' with kwargs={kwargs}")
+
+        # RESET STATE
+        self.completion_times.clear()
+        self.failed.clear()
+        self.original_classes = self.get_artifact_classes(data_type)
+
+        # NON-DESTRUCTIVE SORTING
+        ordered = sorted(self.original_classes, key=self.estimate_priority)
+
+        sem = asyncio.Semaphore(self.max_workers)
+        tasks = [
+            asyncio.create_task(self._bounded_update(cls, sem, **kwargs))
+            for cls in ordered
+        ]
+
+        try:
+            for coro in asyncio.as_completed(tasks):
+                await coro
+        except asyncio.CancelledError:
+            self.logger.warning("update_data was cancelled—aborting remaining retries")
+            for t in tasks:
+                t.cancel()
+            raise
+        finally:
+            total = len(self.original_classes)
+            completed = len(self.completion_times)
+            failed = len(self.failed)
+            self.logger.info(f"All artifacts processed: total={total}, completed={completed}, failed={failed}")
+
+    def get_update_status(self) -> Dict[str, Any]:
+        """
+        Returns summary status including completion times.
+        """
+        total = len(self.original_classes)
+        completed = set(self.completion_times.keys())
+        failed = set(self.failed)
+        pending = {cls.__name__ for cls in self.original_classes} - completed - failed
+
+        return {
+            'total': total,
+            'completed': list(completed),
+            'failed':    list(failed),
+            'pending':   list(pending),
+            'completion_times': self.completion_times,
+        }
+
+    @staticmethod
+    def format_status_table(status: Dict[str, Any]) -> str:
+        """
+        Formats the status dict into a readable table.
+        """
+        lines = [
+            f"Total: {status['total']}",
+            f"Completed: {len(status['completed'])}  {status['completed']}",
+            f"Failed:    {len(status['failed'])}  {status['failed']}",
+            f"Pending:   {len(status['pending'])}  {status['pending']}",
+            "",
+            "Per-artifact timings:"
+        ]
+        for name, dur in status['completion_times'].items():
+            lines.append(f"  {name}: {dur:.2f}s")
+        return "\n".join(lines)

{sibi_dst-0.3.58 → sibi_dst-0.3.59}/sibi_dst/df_helper/_df_helper.py

@@ -46,8 +46,7 @@ class DfHelper:
 
     :ivar df: The DataFrame currently being processed or loaded.
     :type df: Union[dd.DataFrame, pd.DataFrame]
-    :ivar backend_django: Configuration for interacting with Django database backends.
-    :type backend_connection: Optional[DjangoConnectionConfig]
+    :type backend_connection: Optional[DjangoConnectionConfig | SqlAlchemyConnectionConfig]
     :ivar _backend_query: Internal configuration for query handling.
     :type _backend_query: Optional[QueryConfig]
     :ivar _backend_params: Internal parameters configuration for DataFrame handling.
@@ -81,9 +80,10 @@ class DfHelper:
         self.debug = kwargs.setdefault("debug", False)
         self.logger = kwargs.get("logger", Logger.default_logger(logger_name=self.__class__.__name__))
         # Configure logger level
-        self.logger.set_level(logging.DEBUG if self.debug else logging.INFO)
+        self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
         self.logger.debug("Logger initialized in DEBUG mode.")
         self.parquet_storage_path = kwargs.setdefault("parquet_storage_path", None)
+        self.parquet_filename = kwargs.setdefault("parquet_filename", None)
         self.dt_field = kwargs.setdefault("dt_field", None)
         self.as_pandas = kwargs.setdefault("as_pandas", False)
         self.filesystem = kwargs.pop('filesystem', 'file')
@@ -429,7 +429,11 @@ class DfHelper:
                          the instance's attribute for storage path.
         :return: None
         """
+        if self.df.map_partitions(len).compute().sum() == 0:
+            self.logger.debug("Cannot save to parquet since DataFrame is empty")
+            return
         fs = kwargs.pop('fs', self.fs)
+        parquet_filename = parquet_filename or self.parquet_filename
         parquet_storage_path = kwargs.pop('parquet_storage_path', self.parquet_storage_path)
         ps = ParquetSaver(df_result=self.df, parquet_storage_path=parquet_storage_path, logger=self.logger, fs=fs)
         ps.save_to_parquet(parquet_filename)

{sibi_dst-0.3.58 → sibi_dst-0.3.59}/sibi_dst/df_helper/_parquet_artifact.py

@@ -7,7 +7,8 @@ import dask.dataframe as dd
 import fsspec
 
 from sibi_dst.df_helper import DfHelper
-from sibi_dst.utils import DataWrapper, DateUtils, Logger
+from sibi_dst.utils import DataWrapper, DateUtils, Logger, ParquetSaver, UpdatePlanner
+from sibi_dst.utils import MissingManifestManager
 
 
 class ParquetArtifact(DfHelper):
@@ -57,7 +58,16 @@ class ParquetArtifact(DfHelper):
         'backend': 'parquet'
     }
 
-    def __init__(self, data_wrapper_class,  **kwargs):
+    # DEFAULT_UPDATE_PLANNER_CONFIG = {
+    #     'reverse_order': True,
+    #     'overwrite': False,
+    #     'ignore_missing': True,
+    #     'history_days_threshold': 30,
+    #     'max_age_minutes': 10,
+    #     'show_progress': False
+    # }
+
+    def __init__(self, data_wrapper_class, **kwargs):
         """
         Initializes an instance of the class with given configuration and validates
         required parameters. Sets up the filesystem to handle storage, ensuring
@@ -79,46 +89,89 @@ class ParquetArtifact(DfHelper):
                             `parquet_filename`, `parquet_start_date`,
                             or `parquet_end_date`) are missing or not set properly.
         """
+
+        """Initialize with config, validate required fields, and setup filesystem."""
         self._lock = threading.Lock()
         self.config = {
             **self.DEFAULT_CONFIG,
             **kwargs,
         }
         self.df: Optional[dd.DataFrame] = None
-        self.debug = self.config.setdefault('debug', False)
-        self.logger = self.config.setdefault('logger',Logger.default_logger(logger_name=f'parquet_artifact_{__class__.__name__}'))
-        self.logger.set_level(logging.DEBUG if self.debug else logging.INFO)
+        self._setup_logging()
         self.data_wrapper_class = data_wrapper_class
-        self.class_params = self.config.setdefault('class_params', None)
-        self.load_params = self.config.setdefault('load_params', None)
-        self.date_field = self.config.setdefault('date_field', None)
-        if self.date_field is None:
-            raise ValueError('date_field must be set')
-        self.parquet_storage_path = self.config.setdefault('parquet_storage_path', None)
-        if self.parquet_storage_path is None:
-            raise ValueError('parquet_storage_path must be set')
-
-        self.parquet_filename = self.config.setdefault('parquet_filename', None)
-        if self.parquet_filename is None:
-            raise ValueError('parquet_filename must be set')
-        self.parquet_start_date = self.config.setdefault('parquet_start_date', None)
-        if self.parquet_start_date is None:
-            raise ValueError('parquet_start_date must be set')
-
-        self.parquet_end_date = self.config.setdefault('parquet_end_date', None)
-        if self.parquet_end_date is None:
-            raise ValueError('parquet_end_date must be set')
+
+        self.date_field = self._validate_required('date_field')
+        self.parquet_storage_path = self._validate_required('parquet_storage_path')
+        self.parquet_filename = self._validate_required('parquet_filename')
+        self.parquet_start_date = self._validate_required('parquet_start_date')
+        self.parquet_end_date = self._validate_required('parquet_end_date')
 
         # Filesystem setup
         self.filesystem_type = self.config.setdefault('filesystem_type', 'file')
         self.filesystem_options = self.config.setdefault('filesystem_options', {})
         self.fs = self.config.setdefault('fs', None)
+        self._own_fs = self.fs is None
         if self.fs is None:
             self.fs = fsspec.filesystem(self.filesystem_type, **self.filesystem_options)
+            self._own_fs = True
         self.config.setdefault('fs', self.fs)
+        ## Populate to parameters to pass to data_wrapper_class
+        self.class_params = self.config.pop('class_params', {
+            'debug': self.debug,
+            'logger': self.logger,
+            'fs': self.fs,
+        })
+        # Populate parameters to pass to load method of DataWrapper class
+        self.load_params = self.config.setdefault('load_params', {})
         # Ensure the directory exists
         self.ensure_directory_exists(self.parquet_storage_path)
         super().__init__(**self.config)
+        self.update_planner_params = {}
+        self.datawrapper_params = {}
+
+    def _setup_logging(self):
+        """Initialize logger and debug settings."""
+        self.debug = self.config.get('debug', False)
+        self.logger = self.config.get('logger',
+                                      Logger.default_logger(
+                                          logger_name=f'Parquet_Artifact_InstanceOf_{self.__class__.__name__}'))
+        self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
+
+    def _validate_required(self, key: str) -> Any:
+        """Validate required configuration fields."""
+        value = self.config.setdefault(key, None)
+        if value is None:
+            raise ValueError(f'{key} must be set')
+        return value
+
+    def _setup_manifest(self, overwrite: bool = False, ignore_missing: bool = False):
+        self.skipped = []
+        self.missing_manifest_path = f"{self.parquet_storage_path}_manifests/missing.parquet"
+        self.mmanifest = MissingManifestManager(
+            fs=self.fs,
+            manifest_path=self.missing_manifest_path,
+            clear_existing=overwrite
+        )
+
+        # Initialize skipped files
+        manifest_exists = self.mmanifest._safe_exists(self.missing_manifest_path)
+        if not manifest_exists:
+            self.logger.info(f"Creating new manifest at {self.missing_manifest_path}")
+            self.mmanifest.save()
+        else:
+            self.logger.info(f"Manifest already exists at {self.missing_manifest_path}")
+
+        # Load skipped files if manifest exists and ignore_missing is True
+        self.skipped = self.mmanifest.load_existing()  # if ignore_missing and manifest_exists else []
+        self.logger.info(f"Skipped: {self.skipped}")
+        if overwrite:
+            self.skipped = []
+            self.ignore_missing = False
+
+    def _setup_update_planner(self, **kwargs) -> None:
+        self._prepare_update_params(**kwargs)
+        self.update_planner = UpdatePlanner(**self.update_planner_params)
+        self.update_planner.generate_plan(self.start_date, self.end_date)
 
     def load(self, **kwargs):
         with self._lock:
@@ -130,20 +183,30 @@ class ParquetArtifact(DfHelper):
         Generate a Parquet file using the configured DataWrapper class.
         """
         with self._lock:
-            params = self._prepare_params(kwargs)
-            dw = DataWrapper(self.data_wrapper_class, **params)
-            dw.process()
+            overwrite = kwargs.get('overwrite', False)
+            ignore_missing = kwargs.get('ignore_missing', False)
+            self._setup_manifest(overwrite, ignore_missing)
+            self._setup_update_planner(**kwargs)
+            params = self.datawrapper_params.copy()
+            params.update({
+                'mmanifest': self.mmanifest,
+                'update_planner': self.update_planner
+            })
+
+            with DataWrapper(self.data_wrapper_class, **params) as dw:
+                dw.process()
 
     def __enter__(self):
         if getattr(self, "_entered", False):
             return self
         self._entered = True
-        self.ensure_directory_exists(self.parquet_storage_path)
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
         try:
-            if getattr(self, "_entered", False) and self.fs:
+            if self.mmanifest and self.mmanifest._new_records:
+                self.mmanifest.save()
+            if getattr(self, "_entered", False) and self.fs and self._own_fs:
                 self.fs.close()
         except Exception as e:
             self.logger.warning(f"Error closing filesystem: {e}")
@@ -152,6 +215,18 @@ class ParquetArtifact(DfHelper):
         # return False so exceptions aren’t suppressed
         return False
 
+    @classmethod
+    def get_size_estimate(cls, parquet_path: str, **kwargs) -> int:
+        """
+        Estimate complexity as total bytes of all .parquet files under parquet_path.
+        Returns size in megabytes (so you can cap or scale priority sensibly).
+        """
+        fs, _, paths = cls.fs.get_fs_token_paths(parquet_path)
+        files = fs.glob(f"{parquet_path}/*.parquet")
+        total_bytes = sum(fs.size(f) for f in files)
+        # convert to “units” (e.g. MB) so priorities stay in a reasonable range
+        return max(1, int(total_bytes / (1024 ** 2)))
+
     def update_parquet(self, period: str = 'today', **kwargs) -> None:
         """Update the Parquet file with data from a specific period."""
 
@@ -206,36 +281,52 @@ class ParquetArtifact(DfHelper):
             'end_date': kwargs.get('parquet_end_date', self.parquet_end_date),
         }
 
-    def _prepare_params(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
-        """Prepare the parameters for generating the Parquet file."""
-        kwargs = {**self.config, **kwargs}
-        return {
-            'class_params': kwargs.pop('class_params', None),
-            'date_field': kwargs.pop('date_field', self.date_field),
+    def _prepare_update_params(self, **kwargs) -> Dict[str, Any]:
+        self.reverse_order = kwargs.pop('reverse_order', True)
+        self.overwrite = kwargs.pop('overwrite', False)
+        self.ignore_missing = kwargs.pop('ignore_missing', False)
+        self.history_days_threshold = kwargs.pop('history_days_threshold', 30)
+        self.max_age_minutes = kwargs.pop('max_age_minutes', 10)
+        self.show_progress = kwargs.pop('show_progress', False)
+        self.start_date = kwargs.pop('parquet_start_date', self.parquet_start_date)
+        self.end_date = kwargs.pop('parquet_end_date', self.parquet_end_date)
+        self.parquet_filename = kwargs.pop('parquet_filename', self.parquet_filename)
+        self.verbose = kwargs.pop('verbose', False)
+
+        self.update_planner_params.update({
+            'filename': self.parquet_filename,
             'data_path': self.parquet_storage_path,
-            'parquet_filename': kwargs.pop('parquet_filename', self.parquet_filename),
-            'start_date': kwargs.pop('parquet_start_date', self.parquet_start_date),
-            'end_date': kwargs.pop('parquet_end_date', self.parquet_end_date),
-            'verbose': kwargs.pop('verbose', False),
-            'load_params': kwargs.pop('load_params', None),
-            'reverse_order': kwargs.pop('reverse_order', True),
-            'overwrite': kwargs.pop('overwrite', False),
-            'ignore_missing': kwargs.pop('ignore_missing', False),
+            'fs': self.fs,
+            'debug': self.debug,
             'logger': self.logger,
-            'history_days_threshold': kwargs.pop('history_days_threshold', 30),
-            'max_age_minutes': kwargs.pop('max_age_minutes', 10),
-            'show_progress': kwargs.pop('show_progress', False),
+            'reverse_order': self.reverse_order,
+            'overwrite': self.overwrite,
+            'ignore_missing': self.ignore_missing,
+            'history_days_threshold': self.history_days_threshold,
+            'max_age_minutes': self.max_age_minutes,
+            'show_progress': self.show_progress,
+            'description': f"{self.data_wrapper_class.__name__}",
+            'skipped': self.skipped,
+            'verbose': self.verbose,
+        })
+
+        self.datawrapper_params = {
+            'parquet_filename': self.parquet_filename,
+            'data_path': self.parquet_storage_path,
             'fs': self.fs,
-            'filesystem_type': self.filesystem_type,
-            'filesystem_options': self.filesystem_options,
+            'class_params': self.class_params,
+            'date_field': self.date_field,
+            'load_params': self.load_params,
+            'verbose': self.verbose
         }
 
-    @staticmethod
-    def parse_parquet_period(**kwargs):
+    def parse_parquet_period(self, **kwargs):
         start_date, end_date = DateUtils.parse_period(**kwargs)
+        self.parquet_start_date = start_date.strftime('%Y-%m-%d')
+        self.parquet_end_date = end_date.strftime('%Y-%m-%d')
         return {
-            'parquet_start_date': start_date.strftime('%Y-%m-%d'),
-            'parquet_end_date': end_date.strftime('%Y-%m-%d'),
+            'parquet_start_date': self.parquet_start_date,
+            'parquet_end_date': self.parquet_end_date,
         }
 
     def ensure_directory_exists(self, path: str) -> None:

{sibi_dst-0.3.58 → sibi_dst-0.3.59}/sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py

@@ -155,7 +155,7 @@ class SqlAlchemyConnectionConfig(BaseModel):
         key = self._engine_key()
         with self._registry_lock:
             if self._engine_registry.get(key) is not self.engine:
-                self.logger.warning("Engine changed")
+                self.logger.debug("Engine changed")
                 return 0
             pool = self.engine.pool
             if isinstance(pool, QueuePool):
@@ -191,10 +191,10 @@ class SqlAlchemyConnectionConfig(BaseModel):
         with self._registry_lock:
             key = self._engine_key()
             if not self._owns_engine:
-                self.logger.warning("Not owner, skipping close")
+                self.logger.debug("Not owner, skipping close")
                 return
             if self._engine_registry.get(key) != self.engine:
-                self.logger.warning("Engine not in registry")
+                self.logger.debug("Engine not in registry")
                 return
             self.engine.dispose()
             del self._engine_registry[key]

{sibi_dst-0.3.58 → sibi_dst-0.3.59}/sibi_dst/utils/__init__.py

@@ -17,7 +17,7 @@ from .data_wrapper import DataWrapper
 from .storage_config import StorageConfig
 from .data_from_http_source import DataFromHttpSource
 from .webdav_client import WebDAVClient
-
+from .manifest_manager import MissingManifestManager
 
 __all__ = [
     "Logger",
@@ -38,5 +38,6 @@ __all__ = [
     "AirflowDAGManager",
     "StorageConfig",
     "DataFromHttpSource",
-    "WebDAVClient"
+    "WebDAVClient",
+    "MissingManifestManager",
 ]