sibi-dst 0.3.25__py3-none-any.whl → 0.3.27__py3-none-any.whl

sibi_dst/df_helper/core/_filter_handler.py

@@ -185,6 +185,15 @@ class FilterHandler:
             "startswith": lambda col, val: col.like(f"{val}%"),
             "endswith": lambda col, val: col.like(f"%{val}"),
             "isnull": lambda col, val: col.is_(None) if val else col.isnot(None),
+            "not_exact": lambda col, val: col != val,
+            "not_contains": lambda col, val: ~col.like(f"%{val}%"),
+            "not_in": lambda col, val: ~col.in_(val),  # Custom operation
+            "regex": lambda col, val: col.op("~")(val),  # Custom operation
+            "icontains": lambda col, val: col.ilike(f"%{val}%"),  # Custom operation
+            "istartswith": lambda col, val: col.ilike(f"{val}%"),  # Custom operation
+            "iendswith": lambda col, val: col.ilike(f"%{val}"),  # Custom operation
+            "iexact": lambda col, val: col.ilike(val),  # Added iexact
+            "iregex": lambda col, val: col.op("~*")(val),  # Added iregex
         }
 
     @staticmethod
@@ -201,6 +210,15 @@ class FilterHandler:
             "startswith": lambda col, val: col.str.startswith(val),
             "endswith": lambda col, val: col.str.endswith(val),
             "isnull": lambda col, val: col.isnull() if val else col.notnull(),
+            "not_exact": lambda col, val: col != val,
+            "not_contains": lambda col, val: ~col.str.contains(val, regex=True),
+            "not_in": lambda col, val: ~col.isin(val),  # Custom operation
+            "regex": lambda col, val: col.str.contains(val, regex=True),  # Custom operation
+            "icontains": lambda col, val: col.str.contains(val, case=False, regex=True),  # Custom operation
+            "istartswith": lambda col, val: col.str.startswith(val, case=False),  # Custom operation
+            "iendswith": lambda col, val: col.str.endswith(val, case=False),  # Custom operation
+            "iexact": lambda col, val: col.str.contains(f"^{val}$", case=False, regex=True),  # Added iexact
+            "iregex": lambda col, val: col.str.contains(val, case=False, regex=True),  # Added iregex
         }
 
     @staticmethod
@@ -216,4 +234,7 @@ class FilterHandler:
         return [
             "gte", "lte", "gt", "lt", "exact", "in", "range",
             "contains", "startswith", "endswith", "isnull",
+            "not_exact", "not_contains", "not_in",
+            "regex", "icontains", "istartswith", "iendswith",
+            "iexact", "iregex"
         ]

sibi_dst/df_helper/core/_params_config.py

@@ -29,8 +29,8 @@ LOOKUP_SEP = "__"
 class ParamsConfig(BaseModel):
     field_map: Optional[Dict] = Field(default_factory=dict)
     legacy_filters: bool = False
-    sticky_filters: Dict[str, Union[str, bool, int, float]] = Field(default_factory=dict)
-    filters: Dict[str, Union[str, Dict, bool, int, float]] = Field(default_factory=dict)
+    sticky_filters: Dict[str, Union[str, bool, int, float, list, tuple]] = Field(default_factory=dict)
+    filters: Dict[str, Union[str, Dict, bool, int, float, list, tuple]] = Field(default_factory=dict)
     df_params: Dict[str, Union[tuple, str, bool, None]] = Field(default_factory=dict)
     df_options: Dict[str, Union[bool, str, None]] = Field(default_factory=dict)
     params: Dict[str, Union[str, bool, int, float, List[Union[str, int, bool, float]]]] = Field(default_factory=dict)

sibi_dst/utils/__init__.py

@@ -1,17 +1,17 @@
 from __future__ import annotations
 
-from ._log_utils import Logger
-from ._date_utils import *
-from ._data_utils import DataUtils
-from ._file_utils import FileUtils
-from ._filepath_generator import FilePathGenerator
-from ._df_utils import DfUtils
-from ._storage_manager import StorageManager
-from ._parquet_saver import ParquetSaver
-from ._clickhouse_writer import ClickHouseWriter
-from ._airflow_manager import AirflowDAGManager
-from ._credentials import *
-from ._data_wrapper import DataWrapper
+from .log_utils import Logger
+from .date_utils import *
+from .data_utils import DataUtils
+from .file_utils import FileUtils
+from .filepath_generator import FilePathGenerator
+from .df_utils import DfUtils
+from .storage_manager import StorageManager
+from .parquet_saver import ParquetSaver
+from .clickhouse_writer import ClickHouseWriter
+from .airflow_manager import AirflowDAGManager
+from .credentials import *
+from .data_wrapper import DataWrapper
 
 __all__ = [
     "Logger",

sibi_dst/utils/{_clickhouse_writer.py → clickhouse_writer.py}

@@ -125,7 +125,7 @@ class ClickHouseWriter:
         """
         Writes the Dask DataFrame to a ClickHouse table partition by partition.
         """
-        if len(self.df.head().index) == 0:
+        if len(self.df.index) == 0:
             self.logger.debug("No data found. Nothing written.")
             return
 

sibi_dst/utils/{_data_utils.py → data_utils.py}

@@ -95,7 +95,7 @@ class DataUtils:
         # Get unique IDs from source column
         ids = df[source_col].dropna().unique()
         # Compute if it's a Dask Series
-        if isinstance(ids, dd.core.Series):
+        if isinstance(ids, dd.Series):
             ids = ids.compute()
 
         # Check if any IDs are found

sibi_dst/utils/{_data_wrapper.py → data_wrapper.py}

@@ -126,32 +126,6 @@ class DataWrapper:
                     futures[new_future] = priority
                     self.logger.info(f"Resubmitted task for priority {priority} after timeout.")
 
-    # def process(self):
-    #     """Execute the update plan following the specified hierarchy."""
-    #     update_plan_table = self.generate_update_plan_with_conditions()
-    #
-    #     # Display the update plan table to the user if show_progress is True
-    #     if self.show_progress:
-    #         display(update_plan_table)
-    #
-    #     # Process files according to the hierarchy, considering only `update_required` dates
-    #     for category, description in [
-    #         ("overwrite", "Processing files due to overwrite=True"),
-    #         ("history_days", "Processing files within history_days_threshold"),
-    #         ("missing_files", "Processing missing files")
-    #     ]:
-    #         # Filter dates in the category where `update_required` is True
-    #         dates_to_process = update_plan_table[
-    #             (update_plan_table["update_category"] == category) & (update_plan_table["update_required"])
-    #             ]["date"].tolist()
-    #
-    #         date_iterator = dates_to_process
-    #         if self.show_progress:
-    #             date_iterator = tqdm(date_iterator, desc=f"{description}:{self.dataclass.__name__}", unit="date")
-    #
-    #         for current_date in date_iterator:
-    #             self.process_date(current_date)
-
     def is_file_older_than(self, file_path: str) -> bool:
         """
         Check if a file is older than the specified max_age_minutes.

sibi_dst/utils/{_df_utils.py → df_utils.py}

@@ -1,7 +1,7 @@
 import dask.dataframe as dd
 import pandas as pd
 
-from ._log_utils import Logger
+from .log_utils import Logger
 
 
 class DfUtils:

{sibi_dst-0.3.25.dist-info → sibi_dst-0.3.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.25
+Version: 0.3.27
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -32,6 +32,7 @@ Requires-Dist: pymysql (>=1.1.1,<2.0.0)
 Requires-Dist: pytest (>=8.3.3,<9.0.0)
 Requires-Dist: pytest-mock (>=3.14.0,<4.0.0)
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: s3fs (>=2024.12.0,<2025.0.0)
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0)
 Requires-Dist: tornado (>=6.4.1,<7.0.0)
 Requires-Dist: tqdm (>=4.67.0,<5.0.0)

{sibi_dst-0.3.25.dist-info → sibi_dst-0.3.27.dist-info}/RECORD

@@ -22,22 +22,22 @@ sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=ML-m_WeTR1_UMgiDR
 sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py,sha256=Bmhh6VvmBfNfBA2JpuEdsYD_193yJ768Si2TvkY9HmU,4405
 sibi_dst/df_helper/core/__init__.py,sha256=o4zDwgVmaijde3oix0ezb6KLxI5QFy-SGUhFTDVFLT4,569
 sibi_dst/df_helper/core/_defaults.py,sha256=eNpHD2sZxir-2xO0b3_V16ryw8YP_5FfpIKK0HNuiN4,7011
-sibi_dst/df_helper/core/_filter_handler.py,sha256=1-IdviSYi5Hc28KckO4dkYHDfQ8X9SUb6kwfobm16_E,8580
-sibi_dst/df_helper/core/_params_config.py,sha256=mM1CnF29zls5LXx7rpKY8uix_GyOG5smO4ry_OX31IU,3477
+sibi_dst/df_helper/core/_filter_handler.py,sha256=g9FMcB_koT724ggcWt98jow2XgUnmupK_fNhF95W5bQ,10217
+sibi_dst/df_helper/core/_params_config.py,sha256=Og3GYth0GVWpcOYWZWRy7CZ5PDsg63Nmqo-W7TUrA_0,3503
 sibi_dst/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
-sibi_dst/utils/__init__.py,sha256=-7gtDN58v5f09bd2OGKAqMyccEyDP8tANWdw3RwI408,787
-sibi_dst/utils/_airflow_manager.py,sha256=-d44EKUZNYJyp4wuNwRvilRQktunArPOB5fZuWdQv10,7526
-sibi_dst/utils/_clickhouse_writer.py,sha256=dL5pixjn4cj0Rwpc3POfCcY2D-aQCMbPSECX0dKATyE,8587
-sibi_dst/utils/_credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
-sibi_dst/utils/_data_utils.py,sha256=ch4j5FEs8ZnniUzpbeLO-b4Yco_6nwCu71xHaVqMGi4,7050
-sibi_dst/utils/_data_wrapper.py,sha256=4U0sKVXK7qDTObhufO19jxTzJa6ohs2VOh3WAhhzLCU,11982
-sibi_dst/utils/_date_utils.py,sha256=CMAZBNwVj7cvERcNiTA8Pf7_5EjV9By9yxkYJpkqz1g,10656
-sibi_dst/utils/_df_utils.py,sha256=UraCQ7qyprVW_oFcZ2w3UBz2mFvSzxvcJ5pp50tQzIE,10990
-sibi_dst/utils/_file_utils.py,sha256=JpsybYj3XvVJisSBeVU6YSaZnYRm4_6YWTI3TLnnY4Y,1257
-sibi_dst/utils/_filepath_generator.py,sha256=hjI7gQwfwRToPeuzoUQDayHKQrr4Ivhi4Chl1J4Phlk,6689
-sibi_dst/utils/_log_utils.py,sha256=4eLmoV8VC7wDwPr1mRfDKP24_-laGO6ogE4U0u3DUuA,2315
-sibi_dst/utils/_parquet_saver.py,sha256=hLrWr1G132y94eLopDPPGQGDsAiR1lQ8id4QQtGYPE4,4349
-sibi_dst/utils/_storage_manager.py,sha256=7nkfeBW_2xlF59pGj7V2aY5TLwpJnPQuPVclqjavJOA,3856
-sibi_dst-0.3.25.dist-info/METADATA,sha256=KKpRMeVyREhvcvvVY8ZsR_uXhD9SJLblSK5PH7U6R5k,2221
-sibi_dst-0.3.25.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-sibi_dst-0.3.25.dist-info/RECORD,,
+sibi_dst/utils/__init__.py,sha256=z51o5sjIo_gTjnDXk5SBniCxWJIrDBMS7df0dTs8VMk,775
+sibi_dst/utils/airflow_manager.py,sha256=-d44EKUZNYJyp4wuNwRvilRQktunArPOB5fZuWdQv10,7526
+sibi_dst/utils/clickhouse_writer.py,sha256=xUhFDOuZt0eZDpVJNuLb7pfTHUV06NCYrNUx_a7qrSM,8580
+sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
+sibi_dst/utils/data_utils.py,sha256=Kv87Br78EXlH_MSVzRspqLwrf6sqHIRQc0t3LDI0dSM,7045
+sibi_dst/utils/data_wrapper.py,sha256=DTK4hd_GUUi5lxcbEbMraKwzpmPh2IwX6WNWA4t-vx0,10693
+sibi_dst/utils/date_utils.py,sha256=CMAZBNwVj7cvERcNiTA8Pf7_5EjV9By9yxkYJpkqz1g,10656
+sibi_dst/utils/df_utils.py,sha256=OFEtcwVKIilvf9qVf-IfIOHp4jcFAHX5l2IDGudhPZg,10989
+sibi_dst/utils/file_utils.py,sha256=JpsybYj3XvVJisSBeVU6YSaZnYRm4_6YWTI3TLnnY4Y,1257
+sibi_dst/utils/filepath_generator.py,sha256=hjI7gQwfwRToPeuzoUQDayHKQrr4Ivhi4Chl1J4Phlk,6689
+sibi_dst/utils/log_utils.py,sha256=4eLmoV8VC7wDwPr1mRfDKP24_-laGO6ogE4U0u3DUuA,2315
+sibi_dst/utils/parquet_saver.py,sha256=hLrWr1G132y94eLopDPPGQGDsAiR1lQ8id4QQtGYPE4,4349
+sibi_dst/utils/storage_manager.py,sha256=7nkfeBW_2xlF59pGj7V2aY5TLwpJnPQuPVclqjavJOA,3856
+sibi_dst-0.3.27.dist-info/METADATA,sha256=YFb0ZGbz2m0-aczvItyKK4Iqf1wn6pSVGE41ZUQ6YI8,2265
+sibi_dst-0.3.27.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+sibi_dst-0.3.27.dist-info/RECORD,,