sibi-dst 0.3.55__py3-none-any.whl → 0.3.57__py3-none-any.whl

sibi_dst/utils/update_planner.py

@@ -0,0 +1,237 @@
+import datetime
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import List, Optional, Dict, Union, Tuple
+import logging
+from sibi_dst.utils import Logger
+
+import pandas as pd
+import fsspec
+from IPython.display import display
+from tqdm import tqdm
+
+from .date_utils import FileAgeChecker
+
+
+class UpdatePlanner:
+    """
+    A utility class to scan a date-partitioned filesystem and
+    generate an update plan indicating which dates need processing.
+
+    Attributes:
+        data_path:             Base path (always ends with '/').
+        filename:              Filename inside each date folder.
+        fs:                    fsspec filesystem instance.
+        age_checker:           FileAgeChecker for computing file ages.
+        reference_date:        The "today" date used for history windows (date or ISO string).
+        history_days_threshold: Number of days considered "in history".
+        max_age_minutes:       File staleness threshold in minutes.
+        overwrite:             If True, forces updates for all dates.
+        ignore_missing:        If True, skips missing files outside history.
+        reverse_order:         If True, sorts dates descending in output.
+        priority_map:          Maps category names to numeric priorities.
+        show_progress:         If True, displays a tqdm progress bar.
+        logger:                Logger for informational messages.
+
+    Note:
+        generate_plan() will overwrite self.plan and self.df_req.
+    """
+
+    DEFAULT_PRIORITY_MAP = {
+        "overwrite forced":        1,
+        "missing_in_history":      2,
+        "existing_but_stale":      3,
+        "missing_outside_history": 4,
+        "missing_ignored":         0,
+        "file_is_recent":          0
+    }
+
+    def __init__(
+        self,
+        data_path: str,
+        filename: str,
+        description: str = "Update Planner",
+        fs: Optional[fsspec.AbstractFileSystem] = None,
+        filesystem_type: str = "file",
+        filesystem_options: Optional[Dict] = None,
+        reference_date: Union[str, datetime.date] = None,
+        history_days_threshold: int = 30,
+        max_age_minutes: int = 1440,
+        overwrite: bool = False,
+        ignore_missing: bool = False,
+        custom_priority_map: Optional[Dict[str, int]] = None,
+        reverse_order: bool = False,
+        show_progress: bool = False,
+        debug: bool = False,
+        logger: Optional[Logger] = None
+    ):
+        # Initialize state
+        self.plan: pd.DataFrame = pd.DataFrame()
+        self.df_req: pd.DataFrame = pd.DataFrame()
+        self.description = description
+        self.data_path = self._ensure_trailing_slash(data_path)
+        self.filename = filename
+        self.reverse_order = reverse_order
+        self.show_progress = show_progress
+        self.logger = logger or Logger.default_logger(logger_name="update_planner")
+        self.logger.set_level(Logger.DEBUG if debug else Logger.INFO)
+
+        # Filesystem and age helper
+        self.fs = fs or fsspec.filesystem(filesystem_type, **(filesystem_options or {}))
+        self.age_checker = FileAgeChecker(logger=self.logger)
+
+        # Normalize reference date
+        if reference_date is None:
+            self.reference_date = datetime.date.today()
+        else:
+            self.reference_date = pd.to_datetime(reference_date).date()
+
+        # Thresholds and flags
+        self.history_days_threshold = history_days_threshold
+        self.max_age_minutes = max_age_minutes
+        self.overwrite = overwrite
+        self.ignore_missing = ignore_missing
+        self.priority_map = custom_priority_map or self.DEFAULT_PRIORITY_MAP
+
+    @staticmethod
+    def _ensure_trailing_slash(path: str) -> str:
+        """Ensure that the provided path ends with a single '/'."""
+        return path.rstrip('/') + '/'
+
+    def _generate_plan(
+        self,
+        start: datetime.date,
+        end:   datetime.date,
+        freq:  str = "D"
+    ) -> None:
+        """
+        Internal: populates self.plan and self.df_req with all dates and required subset.
+        """
+        # Generate list of dates
+        dates = pd.date_range(start=start, end=end, freq=freq).date.tolist()
+        history_start = self.reference_date - datetime.timedelta(days=self.history_days_threshold)
+        rows: List[Dict] = []
+
+        # Parallel file status checks
+        with ThreadPoolExecutor() as executor:
+            futures = {executor.submit(self._get_file_status, d): d for d in dates}
+            iterator = as_completed(futures)
+            if self.show_progress:
+                iterator = tqdm(
+                    iterator,
+                    total=len(futures),
+                    desc="Scanning dates",
+                    unit="date",
+                    leave=False
+                )
+            for future in iterator:
+                d = futures[future]
+                exists, age = future.result()
+                rows.append(self._make_row(d, history_start, exists, age))
+
+        # Build DataFrame and filtered subset
+        df = pd.DataFrame(rows)
+        df = df.sort_values(
+            by=["update_priority", "date"],
+            ascending=[True, not self.reverse_order]
+        ).reset_index(drop=True)
+
+        self.plan = df
+        self.df_req = df[df.update_required].copy()
+
+    def generate_plan(
+        self,
+        start: Union[str, datetime.date],
+        end:   Union[str, datetime.date]
+    ) -> List[Dict[str, Union[str, int]]]:
+        """
+        Generate and return the update plan for dates between start and end.
+
+        Returns:
+            A list of dicts for dates requiring updates, each with:
+              - date: str 'YYYY-MM-DD'
+              - update_priority: int
+        The list is sorted by update_priority ascending, then by date
+        (descending if reverse_order=True).
+
+        Raises:
+            ValueError: if start > end.
+        """
+        # Normalize and validate inputs
+        sd = pd.to_datetime(start).date()
+        ed = pd.to_datetime(end).date()
+        if sd > ed:
+            raise ValueError(f"Start date ({sd}) must be on or before end date ({ed}).")
+
+        if self.logger:
+            self.logger.info(f"Generating update plan for {self.description} from {sd} to {ed}")
+
+        # Populate plan
+        self._generate_plan(sd, ed)
+
+        if self.logger:
+            self.logger.info(f"Plan built for {self.description}: {len(self.df_req)} dates require updates")
+
+        # Format output
+        output = [
+            {"date": d.strftime("%Y-%m-%d"), "update_priority": int(p)}
+            for d, p in zip(self.df_req.date, self.df_req.update_priority)
+        ]
+        return self.df_req[['date','update_priority','description']]
+
+    def show_update_plan(self) -> None:
+        """
+        Display the full update plan as a styled DataFrame.
+        """
+        if self.plan.empty:
+            self.logger.warning("No update plan available. Call generate_plan() first.")
+            return
+        display(self.plan)
+
+    def _get_file_status(
+        self,
+        date: datetime.date
+    ) -> Tuple[bool, Optional[float]]:
+        """
+        Check file existence and age for the given date.
+        """
+        path = f"{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/{self.filename}"
+        try:
+            exists = self.fs.exists(path)
+            age = self.age_checker.get_file_or_dir_age_minutes(path, self.fs) if exists else None
+            return exists, age
+        except Exception:
+            return False, None
+
+    def _make_row(
+        self,
+        date: datetime.date,
+        history_start: datetime.date,
+        file_exists: bool,
+        file_age: Optional[float]
+    ) -> Dict:
+        """
+        Build a single plan row based on flags and thresholds.
+        """
+        """Create a row for the update plan DataFrame"""
+        within_history = history_start <= date <= self.reference_date
+        category, update_required = "file_is_recent", False
+
+        if self.overwrite:
+            category, update_required = "overwrite", True
+        elif within_history:
+            if not file_exists:
+                category, update_required = "missing_in_history", True
+            elif file_age > self.max_age_minutes:
+                category, update_required = "existing_but_stale", True
+        elif not file_exists and not self.ignore_missing:
+            category, update_required = "missing_outside_history", True
+
+        return {
+            "date": date,
+            "file_exists": file_exists,
+            "file_age_minutes": file_age,
+            "update_category": category,
+            "update_priority": self.priority_map.get(category, 99),
+            "update_required": update_required,
+            "description": self.description,
+        }

{sibi_dst-0.3.55.dist-info → sibi_dst-0.3.57.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.55
+Version: 0.3.57
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.55.dist-info → sibi_dst-0.3.57.dist-info}/RECORD

@@ -1,8 +1,8 @@
 sibi_dst/__init__.py,sha256=3pbriM7Ym5f9gew7n9cO4G_p9n-0bnxdmQ0hwBdJjr4,253
 sibi_dst/df_helper/__init__.py,sha256=McYrw2N0MsMgtawLrONXTGdyHfQWVOBUvIDbklfjb54,342
-sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=toH2QvNF-CQNJ4Bc8xreytuWr37G0EWz4ciWVdFMVqU,11646
-sibi_dst/df_helper/_df_helper.py,sha256=D85n4oUdu92IN2QaPc6k9uJJ_Vm197me1aoHojuWEYs,29833
-sibi_dst/df_helper/_parquet_artifact.py,sha256=dDoR5Tq0EJViW52tD9XW_vno7hkOfA8WeAilR1mAb_g,10636
+sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=f0yz5L1WE2a_N5Kkjr06ZwyJktdXy9xBLMDWJXQAaOI,10145
+sibi_dst/df_helper/_df_helper.py,sha256=Th5IakQjF22dfxDF5mC7DJOQm1ZAxM09AMW9uDH83oc,30962
+sibi_dst/df_helper/_parquet_artifact.py,sha256=i6QhQhGz6jyt7MWQk7CocQxUUZwG_50oSk9eyB42_kQ,11248
 sibi_dst/df_helper/_parquet_reader.py,sha256=L6mr2FeKtTeIn37G9EGpvOx8PwMqXb6qnEECqBaiwxo,3954
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/df_helper/backends/django/__init__.py,sha256=uWHi-DtQX5re7b2HcqoXUH3_FZWOw1VTmDf552FAkNs,256
@@ -16,10 +16,10 @@ sibi_dst/df_helper/backends/parquet/__init__.py,sha256=esWJ9aSuYC26d-T01z9dPrJ1u
 sibi_dst/df_helper/backends/parquet/_filter_handler.py,sha256=TvDf0RXta7mwJv11GNQttYJsXgFf2XDj4oLIjt4xTzA,5219
 sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=TaU5_wG1Y3lQC8DVCItVvMnc6ZJmECLu3avssVEMbaM,10591
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=TuVp8Ce49dCIIxtyrtFGRblarQUl8QGcS-TDZd515IE,348
-sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=LxuTh4U1rN-h921QO5Owck4SQcrABpcWUi4EjEBPf-c,2846
+sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=PgMbjmEBSu5g5Vi0DL6Gd6871j_KuK5DNxKH9WDbWGE,7986
 sibi_dst/df_helper/backends/sqlalchemy/_filter_handler.py,sha256=58RCda1Hg_nsuJw-2V36IstsT8O84IQFgsdE7FnqvMk,4655
 sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=BtiRSYA4kFIM-mBCdrwE20vzByfq8_Biv_jPLUCDv58,5466
-sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=I2Us3RrxHci561yyZYBuUCrLVOhB0F3KBnae78m_ARw,6259
+sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=ltqB5814PMecxwZgmsJL6nDhQf72V-w71YWFAf7aYZ8,6490
 sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py,sha256=ksvJ0EvktrVsoJ9DTMIQHzHe8ghw2mzDIBD_YgWytgw,8402
 sibi_dst/df_helper/core/__init__.py,sha256=o4zDwgVmaijde3oix0ezb6KLxI5QFy-SGUhFTDVFLT4,569
 sibi_dst/df_helper/core/_defaults.py,sha256=eNpHD2sZxir-2xO0b3_V16ryw8YP_5FfpIKK0HNuiN4,7011
@@ -38,22 +38,23 @@ sibi_dst/osmnx_helper/basemaps/router_plotter.py,sha256=UAiijn-J-jjX4YnL0_P9SFqT
 sibi_dst/osmnx_helper/utils.py,sha256=BzuY8CtYnBAAO8UAr_M7EOk6CP1zcifNLs8pkdFZEFg,20577
 sibi_dst/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/tests/test_data_wrapper_class.py,sha256=6uFmZR2DxnxQz49L5jT2ehlKvlLnpUHMLFB_PqqUq7k,3336
-sibi_dst/utils/__init__.py,sha256=_4kuTzjCfbRF9927ywXqi-JKoHNec8wf05LYh4DMbPI,1077
+sibi_dst/utils/__init__.py,sha256=RYVJ1f_bE3l-awgCpa-eV24HnBQC26hT4AFlwYghnPg,1120
 sibi_dst/utils/airflow_manager.py,sha256=-d44EKUZNYJyp4wuNwRvilRQktunArPOB5fZuWdQv10,7526
 sibi_dst/utils/clickhouse_writer.py,sha256=iAUe4_Kn2WR1xZjpLW2FOWCWfOTw6fCGMTUcWxIQJ60,9877
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
-sibi_dst/utils/data_wrapper.py,sha256=-RqK_sU3uuc6U9dFfnICFZkaIetk3JOvJain6_lTWlo,12446
+sibi_dst/utils/data_wrapper.py,sha256=npOVesxPBdvAUQVGhFc_3B9TgBQSZlKfNWOSm-FNdRQ,9779
 sibi_dst/utils/date_utils.py,sha256=OCJqkWl5e8fE7z11Ufz4206DUeuLMd_Gf_JGZu914Pg,18539
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
 sibi_dst/utils/filepath_generator.py,sha256=-HHO0U-PR8fysDDFwnWdHRlgqksh_RkmgBZLWv9hM7s,6669
 sibi_dst/utils/log_utils.py,sha256=eSAbi_jmMpJ8RpycakzT4S4zNkqVZDj3FY8WwnxpdXc,4623
-sibi_dst/utils/parquet_saver.py,sha256=EBtd9blzk7Wb65aDBVVU0ZMHFtqjfWi_fCUt1LvyAC4,8069
+sibi_dst/utils/parquet_saver.py,sha256=O62xwPfphOpKgEiHqnts20CPSU96pxs49Cg7PVetLK0,8193
 sibi_dst/utils/phone_formatter.py,sha256=tsVTDamuthFYgy4-5UwmQkPQ-FGTGH7MjZyH8utAkIY,4945
 sibi_dst/utils/storage_config.py,sha256=Cg8EOGLZ_5v9sunaQHZLYHdp5FDkgPrCVVNHF-ys5sQ,2181
 sibi_dst/utils/storage_manager.py,sha256=btecX7ggNb7rfu5EK9Xuu2q_FZA7r_rB_tfhQ8V96qc,6567
+sibi_dst/utils/update_planner.py,sha256=AaprHgUsKeyZNvA3nRHrCqnxy8GXwySkaV27X_k54Xc,8799
 sibi_dst/utils/webdav_client.py,sha256=pYF1UsGOuxYeGLq7aBfwZFvkvD4meOcbbaiZ4d6GW9I,7107
 sibi_dst/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/v2/df_helper/__init__.py,sha256=XuH6jKYAPg2DdRbsxxBSxp9X3x-ARyaT0xe27uILrVo,99
@@ -75,6 +76,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-0.3.55.dist-info/METADATA,sha256=EHj1miIF5IHvHATok4rYdLhSo2kD8cSrAs9P1fER1_8,4292
-sibi_dst-0.3.55.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.55.dist-info/RECORD,,
+sibi_dst-0.3.57.dist-info/METADATA,sha256=U6Ra-N2vtA8yJOdLmj1jNZ9skrrhlvQR3wauhkAbKgk,4292
+sibi_dst-0.3.57.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.57.dist-info/RECORD,,