sibi-dst 0.3.58__py3-none-any.whl → 0.3.59__py3-none-any.whl

sibi_dst/utils/update_planner.py

@@ -1,14 +1,9 @@
 import datetime
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import List, Optional, Dict, Union, Tuple
-import logging
-from sibi_dst.utils import Logger
-
+from typing import List, Optional, Dict, Union, Tuple, Set
 import pandas as pd
 import fsspec
-from IPython.display import display
-from tqdm import tqdm
-
+from sibi_dst.utils import Logger
 from .date_utils import FileAgeChecker
 
 
@@ -28,41 +23,45 @@ class UpdatePlanner:
         overwrite:             If True, forces updates for all dates.
         ignore_missing:        If True, skips missing files outside history.
         reverse_order:         If True, sorts dates descending in output.
-        priority_map:          Maps category names to numeric priorities.
         show_progress:         If True, displays a tqdm progress bar.
         logger:                Logger for informational messages.
 
     Note:
-        generate_plan() will overwrite self.plan and self.df_req.
+        generate_plan() will overwrite self.plan and self.df_req, and returns a DataFrame of required updates.
     """
 
     DEFAULT_PRIORITY_MAP = {
-        "overwrite forced":        1,
-        "missing_in_history":      2,
-        "existing_but_stale":      3,
-        "missing_outside_history": 4,
-        "missing_ignored":         0,
-        "file_is_recent":          0
+        "file_is_recent": 0,
+        "missing_ignored": 0,
+        "overwrite_forced": 1,
+        "create_missing": 2,
+        "missing_in_history": 3,
+        "stale_in_history": 4,
     }
 
+    DEFAULT_MAX_AGE_MINUTES = 1440
+    DEFAULT_HISTORY_DAYS_THRESHOLD = 30
+
     def __init__(
-        self,
-        data_path: str,
-        filename: str,
-        description: str = "Update Planner",
-        fs: Optional[fsspec.AbstractFileSystem] = None,
-        filesystem_type: str = "file",
-        filesystem_options: Optional[Dict] = None,
-        reference_date: Union[str, datetime.date] = None,
-        history_days_threshold: int = 30,
-        max_age_minutes: int = 1440,
-        overwrite: bool = False,
-        ignore_missing: bool = False,
-        custom_priority_map: Optional[Dict[str, int]] = None,
-        reverse_order: bool = False,
-        show_progress: bool = False,
-        debug: bool = False,
-        logger: Optional[Logger] = None
+            self,
+            data_path: str,
+            filename: str,
+            description: str = "Update Planner",
+            fs: Optional[fsspec.AbstractFileSystem] = None,
+            filesystem_type: str = "file",
+            filesystem_options: Optional[Dict] = None,
+            reference_date: Union[str, datetime.date] = None,
+            history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
+            max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
+            overwrite: bool = False,
+            ignore_missing: bool = False,
+            custom_priority_map: Optional[Dict[str, int]] = None,
+            reverse_order: bool = False,
+            show_progress: bool = False,
+            verbose: bool = False,
+            debug: bool = False,
+            logger: Optional[Logger] = None,
+            skipped: Optional[List[str]] = None,
     ):
         # Initialize state
         self.plan: pd.DataFrame = pd.DataFrame()
@@ -77,7 +76,7 @@ class UpdatePlanner:
 
         # Filesystem and age helper
         self.fs = fs or fsspec.filesystem(filesystem_type, **(filesystem_options or {}))
-        self.age_checker = FileAgeChecker(logger=self.logger)
+        self.age_checker = FileAgeChecker(debug=debug, logger=self.logger)
 
         # Normalize reference date
         if reference_date is None:
@@ -91,6 +90,7 @@ class UpdatePlanner:
         self.overwrite = overwrite
         self.ignore_missing = ignore_missing
         self.priority_map = custom_priority_map or self.DEFAULT_PRIORITY_MAP
+        self.skipped = skipped or []
 
     @staticmethod
     def _ensure_trailing_slash(path: str) -> str:
@@ -98,15 +98,14 @@ class UpdatePlanner:
         return path.rstrip('/') + '/'
 
     def _generate_plan(
-        self,
-        start: datetime.date,
-        end:   datetime.date,
-        freq:  str = "D"
+            self,
+            start: datetime.date,
+            end: datetime.date,
+            freq: str = "D"
     ) -> None:
         """
-        Internal: populates self.plan and self.df_req with all dates and required subset.
+        Internal: populates self.plan with all dates, and self.df_req with only those needing update.
         """
-        # Generate list of dates
         dates = pd.date_range(start=start, end=end, freq=freq).date.tolist()
         history_start = self.reference_date - datetime.timedelta(days=self.history_days_threshold)
         rows: List[Dict] = []
@@ -116,10 +115,11 @@ class UpdatePlanner:
             futures = {executor.submit(self._get_file_status, d): d for d in dates}
             iterator = as_completed(futures)
             if self.show_progress:
+                from tqdm import tqdm
                 iterator = tqdm(
                     iterator,
                     total=len(futures),
-                    desc="Scanning dates",
+                    desc=f"Scanning dates for {self.description}",
                     unit="date",
                     leave=False
                 )
@@ -128,7 +128,6 @@ class UpdatePlanner:
                 exists, age = future.result()
                 rows.append(self._make_row(d, history_start, exists, age))
 
-        # Build DataFrame and filtered subset
         df = pd.DataFrame(rows)
         df = df.sort_values(
             by=["update_priority", "date"],
@@ -139,44 +138,27 @@ class UpdatePlanner:
         self.df_req = df[df.update_required].copy()
 
     def generate_plan(
-        self,
-        start: Union[str, datetime.date],
-        end:   Union[str, datetime.date]
-    ) -> List[Dict[str, Union[str, int]]]:
+            self,
+            start: Union[str, datetime.date],
+            end: Union[str, datetime.date]
+    ) -> pd.DataFrame:
         """
-        Generate and return the update plan for dates between start and end.
-
-        Returns:
-            A list of dicts for dates requiring updates, each with:
-              - date: str 'YYYY-MM-DD'
-              - update_priority: int
-        The list is sorted by update_priority ascending, then by date
-        (descending if reverse_order=True).
-
-        Raises:
-            ValueError: if start > end.
+        Generate and return a DataFrame of dates requiring updates between start and end,
+        sorted by update_priority and date (descending if reverse_order=True).
         """
-        # Normalize and validate inputs
         sd = pd.to_datetime(start).date()
         ed = pd.to_datetime(end).date()
         if sd > ed:
             raise ValueError(f"Start date ({sd}) must be on or before end date ({ed}).")
 
-        if self.logger:
-            self.logger.info(f"Generating update plan for {self.description} from {sd} to {ed}")
-
-        # Populate plan
+        self.logger.info(f"Generating update plan for {self.description} from {sd} to {ed}")
         self._generate_plan(sd, ed)
+        self.logger.info(
+            f"Plan built for {self.description}: {len(self.plan)} dates evaluated, "
+            f"{len(self.df_req)} require update"
+        )
 
-        if self.logger:
-            self.logger.info(f"Plan built for {self.description}: {len(self.df_req)} dates require updates")
-
-        # Format output
-        output = [
-            {"date": d.strftime("%Y-%m-%d"), "update_priority": int(p)}
-            for d, p in zip(self.df_req.date, self.df_req.update_priority)
-        ]
-        return self.df_req[['date','update_priority','description']]
+        return self.df_req
 
     def show_update_plan(self) -> None:
         """
@@ -185,16 +167,21 @@ class UpdatePlanner:
         if self.plan.empty:
             self.logger.warning("No update plan available. Call generate_plan() first.")
             return
+        from IPython.display import display
         display(self.plan)
 
     def _get_file_status(
-        self,
-        date: datetime.date
+            self,
+            date: datetime.date
     ) -> Tuple[bool, Optional[float]]:
         """
         Check file existence and age for the given date.
         """
-        path = f"{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/{self.filename}"
+        just_path = f"{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/"
+        if just_path in self.skipped:
+            self.logger.debug(f"Update plan is skipping date {date} as it is in the skipped list.")
+            return False, None
+        path = f"{just_path}{self.filename}"
         try:
             exists = self.fs.exists(path)
             age = self.age_checker.get_file_or_dir_age_minutes(path, self.fs) if exists else None
@@ -203,28 +190,39 @@ class UpdatePlanner:
             return False, None
 
     def _make_row(
-        self,
-        date: datetime.date,
-        history_start: datetime.date,
-        file_exists: bool,
-        file_age: Optional[float]
+            self,
+            date: datetime.date,
+            history_start: datetime.date,
+            file_exists: bool,
+            file_age: Optional[float]
     ) -> Dict:
         """
         Build a single plan row based on flags and thresholds.
         """
-        """Create a row for the update plan DataFrame"""
         within_history = history_start <= date <= self.reference_date
-        category, update_required = "file_is_recent", False
+        update_required = False
 
+        # 1. Overwrite mode forces update
         if self.overwrite:
-            category, update_required = "overwrite", True
+            category = "overwrite_forced"
+            update_required = True
+        # 2. Within history window: missing or stale
         elif within_history:
             if not file_exists:
-                category, update_required = "missing_in_history", True
-            elif file_age > self.max_age_minutes:
-                category, update_required = "existing_but_stale", True
+                category = "missing_in_history"
+                update_required = True
+            elif file_age is not None and file_age > self.max_age_minutes:
+                category = "stale_in_history"
+                update_required = True
+            else:
+                category = "file_is_recent"
+        # 3. Outside history, missing file
         elif not file_exists and not self.ignore_missing:
-            category, update_required = "missing_outside_history", True
+            category = "create_missing"
+            update_required = True
+        # 4. Everything else (existing files outside history, or ignored missing)
+        else:
+            category = "missing_ignored" if not file_exists else "file_is_recent"
 
         return {
             "date": date,
@@ -235,3 +233,16 @@ class UpdatePlanner:
             "update_required": update_required,
             "description": self.description,
         }
+
+    def exclude_dates(self, dates: Set[datetime.date]) -> None:
+        """
+        Exclude specific dates from the update plan.
+        """
+        if not isinstance(dates, set):
+            raise ValueError("dates must be a set of datetime.date objects.")
+        if self.plan.empty:
+            self.logger.warning("No update plan available. Call generate_plan() first.")
+            return
+        self.plan = self.plan[~self.plan['date'].isin(dates)]
+        self.df_req = self.plan[self.plan["update_required"]]
+        self.logger.info(f"Excluded {len(dates)} dates from the update plan.")

{sibi_dst-0.3.58.dist-info → sibi_dst-0.3.59.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.58
+Version: 0.3.59
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.58.dist-info → sibi_dst-0.3.59.dist-info}/RECORD

@@ -1,8 +1,8 @@
 sibi_dst/__init__.py,sha256=3pbriM7Ym5f9gew7n9cO4G_p9n-0bnxdmQ0hwBdJjr4,253
 sibi_dst/df_helper/__init__.py,sha256=McYrw2N0MsMgtawLrONXTGdyHfQWVOBUvIDbklfjb54,342
-sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=2iQ4bQLCPKnqfKMwmttJFLdNon0bc8Lz7UfDcVRG8pQ,10175
-sibi_dst/df_helper/_df_helper.py,sha256=WFqEyNhmE4hAXgMcqT8a7iYAhsmM3f_Z_DSQdlZu3P4,30968
-sibi_dst/df_helper/_parquet_artifact.py,sha256=i6QhQhGz6jyt7MWQk7CocQxUUZwG_50oSk9eyB42_kQ,11248
+sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=-Y4i5KAxKY2BNkmoVeMEZxjTFD7zaM9oQ0aRsvUbQrs,9340
+sibi_dst/df_helper/_df_helper.py,sha256=uKP6i-7dasZQ5zViD8-VJU0lNHumrdZG6IXvDFijZ18,31214
+sibi_dst/df_helper/_parquet_artifact.py,sha256=6y8nJ-HDAdmy3XNSvnEdA2zBXDhUIVoUeKgXLmVMGCo,14879
 sibi_dst/df_helper/_parquet_reader.py,sha256=L6mr2FeKtTeIn37G9EGpvOx8PwMqXb6qnEECqBaiwxo,3954
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/df_helper/backends/django/__init__.py,sha256=uWHi-DtQX5re7b2HcqoXUH3_FZWOw1VTmDf552FAkNs,256
@@ -16,7 +16,7 @@ sibi_dst/df_helper/backends/parquet/__init__.py,sha256=esWJ9aSuYC26d-T01z9dPrJ1u
 sibi_dst/df_helper/backends/parquet/_filter_handler.py,sha256=TvDf0RXta7mwJv11GNQttYJsXgFf2XDj4oLIjt4xTzA,5219
 sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=TaU5_wG1Y3lQC8DVCItVvMnc6ZJmECLu3avssVEMbaM,10591
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=TuVp8Ce49dCIIxtyrtFGRblarQUl8QGcS-TDZd515IE,348
-sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=PgMbjmEBSu5g5Vi0DL6Gd6871j_KuK5DNxKH9WDbWGE,7986
+sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=JID-urZLbWjMd2dXt7onp6cPxAWQ3jnsY88s_lCscn8,7980
 sibi_dst/df_helper/backends/sqlalchemy/_filter_handler.py,sha256=58RCda1Hg_nsuJw-2V36IstsT8O84IQFgsdE7FnqvMk,4655
 sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=BtiRSYA4kFIM-mBCdrwE20vzByfq8_Biv_jPLUCDv58,5466
 sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=ltqB5814PMecxwZgmsJL6nDhQf72V-w71YWFAf7aYZ8,6490
@@ -38,23 +38,24 @@ sibi_dst/osmnx_helper/basemaps/router_plotter.py,sha256=UAiijn-J-jjX4YnL0_P9SFqT
 sibi_dst/osmnx_helper/utils.py,sha256=BzuY8CtYnBAAO8UAr_M7EOk6CP1zcifNLs8pkdFZEFg,20577
 sibi_dst/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/tests/test_data_wrapper_class.py,sha256=6uFmZR2DxnxQz49L5jT2ehlKvlLnpUHMLFB_PqqUq7k,3336
-sibi_dst/utils/__init__.py,sha256=RYVJ1f_bE3l-awgCpa-eV24HnBQC26hT4AFlwYghnPg,1120
+sibi_dst/utils/__init__.py,sha256=I-LeSCEnnOKunblmCrv_KTBzbsr7Rx1MEWM9TypDqWM,1203
 sibi_dst/utils/airflow_manager.py,sha256=-d44EKUZNYJyp4wuNwRvilRQktunArPOB5fZuWdQv10,7526
 sibi_dst/utils/clickhouse_writer.py,sha256=iAUe4_Kn2WR1xZjpLW2FOWCWfOTw6fCGMTUcWxIQJ60,9877
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
-sibi_dst/utils/data_wrapper.py,sha256=npOVesxPBdvAUQVGhFc_3B9TgBQSZlKfNWOSm-FNdRQ,9779
-sibi_dst/utils/date_utils.py,sha256=OCJqkWl5e8fE7z11Ufz4206DUeuLMd_Gf_JGZu914Pg,18539
+sibi_dst/utils/data_wrapper.py,sha256=DFkqi84DIGxcrf36FfbgmeF9Hu7PZjMO9otNerV8ZYk,10546
+sibi_dst/utils/date_utils.py,sha256=T3ij-WOQu3cIfmNAweSVMWWr-hVtuBcTGjEY-cMJIvU,18627
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
 sibi_dst/utils/filepath_generator.py,sha256=-HHO0U-PR8fysDDFwnWdHRlgqksh_RkmgBZLWv9hM7s,6669
-sibi_dst/utils/log_utils.py,sha256=eSAbi_jmMpJ8RpycakzT4S4zNkqVZDj3FY8WwnxpdXc,4623
+sibi_dst/utils/log_utils.py,sha256=77xACRagKU83H9vn7aVeBzkQjxWlbe4dg4KuxPRCgvw,4635
+sibi_dst/utils/manifest_manager.py,sha256=abm97TuWgJqNViPXMbpl5W7ttrg1BeiJkf2SMGc4hd8,5512
 sibi_dst/utils/parquet_saver.py,sha256=O62xwPfphOpKgEiHqnts20CPSU96pxs49Cg7PVetLK0,8193
 sibi_dst/utils/phone_formatter.py,sha256=tsVTDamuthFYgy4-5UwmQkPQ-FGTGH7MjZyH8utAkIY,4945
 sibi_dst/utils/storage_config.py,sha256=Cg8EOGLZ_5v9sunaQHZLYHdp5FDkgPrCVVNHF-ys5sQ,2181
 sibi_dst/utils/storage_manager.py,sha256=btecX7ggNb7rfu5EK9Xuu2q_FZA7r_rB_tfhQ8V96qc,6567
-sibi_dst/utils/update_planner.py,sha256=AaprHgUsKeyZNvA3nRHrCqnxy8GXwySkaV27X_k54Xc,8799
+sibi_dst/utils/update_planner.py,sha256=dJXLC-KdbWrCs-MFe7Xa8F-ZhlNJq8P1szjLAzMJZk0,9684
 sibi_dst/utils/webdav_client.py,sha256=pYF1UsGOuxYeGLq7aBfwZFvkvD4meOcbbaiZ4d6GW9I,7107
 sibi_dst/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/v2/df_helper/__init__.py,sha256=XuH6jKYAPg2DdRbsxxBSxp9X3x-ARyaT0xe27uILrVo,99
@@ -76,6 +77,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-0.3.58.dist-info/METADATA,sha256=DAQ2TfVwh0aB5uiZ-tnvlPzYe6hdB16M5NXswXZr9ZE,4292
-sibi_dst-0.3.58.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.58.dist-info/RECORD,,
+sibi_dst-0.3.59.dist-info/METADATA,sha256=zYb_0a1ImTPUxB4raHM8cvip0_oZSot52N8okr3r_ZY,4292
+sibi_dst-0.3.59.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.59.dist-info/RECORD,,