sibi-dst 0.3.25__tar.gz → 0.3.27__tar.gz

{sibi_dst-0.3.25 → sibi_dst-0.3.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.25
+Version: 0.3.27
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -32,6 +32,7 @@ Requires-Dist: pymysql (>=1.1.1,<2.0.0)
 Requires-Dist: pytest (>=8.3.3,<9.0.0)
 Requires-Dist: pytest-mock (>=3.14.0,<4.0.0)
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: s3fs (>=2024.12.0,<2025.0.0)
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0)
 Requires-Dist: tornado (>=6.4.1,<7.0.0)
 Requires-Dist: tqdm (>=4.67.0,<5.0.0)

{sibi_dst-0.3.25 → sibi_dst-0.3.27}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.25"
+version = "0.3.27"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"
@@ -36,6 +36,7 @@ dask-expr = "^1.1.20"
 psycopg2 = "^2.9.10"
 uvicorn = "^0.34.0"
 pytest-mock = "^3.14.0"
+s3fs = "^2024.12.0"
 
 
 [build-system]

{sibi_dst-0.3.25 → sibi_dst-0.3.27}/sibi_dst/df_helper/core/_filter_handler.py

@@ -185,6 +185,15 @@ class FilterHandler:
             "startswith": lambda col, val: col.like(f"{val}%"),
             "endswith": lambda col, val: col.like(f"%{val}"),
             "isnull": lambda col, val: col.is_(None) if val else col.isnot(None),
+            "not_exact": lambda col, val: col != val,
+            "not_contains": lambda col, val: ~col.like(f"%{val}%"),
+            "not_in": lambda col, val: ~col.in_(val),  # Custom operation
+            "regex": lambda col, val: col.op("~")(val),  # Custom operation
+            "icontains": lambda col, val: col.ilike(f"%{val}%"),  # Custom operation
+            "istartswith": lambda col, val: col.ilike(f"{val}%"),  # Custom operation
+            "iendswith": lambda col, val: col.ilike(f"%{val}"),  # Custom operation
+            "iexact": lambda col, val: col.ilike(val),  # Added iexact
+            "iregex": lambda col, val: col.op("~*")(val),  # Added iregex
         }
 
     @staticmethod
@@ -201,6 +210,15 @@ class FilterHandler:
             "startswith": lambda col, val: col.str.startswith(val),
             "endswith": lambda col, val: col.str.endswith(val),
             "isnull": lambda col, val: col.isnull() if val else col.notnull(),
+            "not_exact": lambda col, val: col != val,
+            "not_contains": lambda col, val: ~col.str.contains(val, regex=True),
+            "not_in": lambda col, val: ~col.isin(val),  # Custom operation
+            "regex": lambda col, val: col.str.contains(val, regex=True),  # Custom operation
+            "icontains": lambda col, val: col.str.contains(val, case=False, regex=True),  # Custom operation
+            "istartswith": lambda col, val: col.str.startswith(val, case=False),  # Custom operation
+            "iendswith": lambda col, val: col.str.endswith(val, case=False),  # Custom operation
+            "iexact": lambda col, val: col.str.contains(f"^{val}$", case=False, regex=True),  # Added iexact
+            "iregex": lambda col, val: col.str.contains(val, case=False, regex=True),  # Added iregex
         }
 
     @staticmethod
@@ -216,4 +234,7 @@ class FilterHandler:
         return [
             "gte", "lte", "gt", "lt", "exact", "in", "range",
             "contains", "startswith", "endswith", "isnull",
+            "not_exact", "not_contains", "not_in",
+            "regex", "icontains", "istartswith", "iendswith",
+            "iexact", "iregex"
         ]

{sibi_dst-0.3.25 → sibi_dst-0.3.27}/sibi_dst/df_helper/core/_params_config.py

@@ -29,8 +29,8 @@ LOOKUP_SEP = "__"
 class ParamsConfig(BaseModel):
     field_map: Optional[Dict] = Field(default_factory=dict)
     legacy_filters: bool = False
-    sticky_filters: Dict[str, Union[str, bool, int, float]] = Field(default_factory=dict)
-    filters: Dict[str, Union[str, Dict, bool, int, float]] = Field(default_factory=dict)
+    sticky_filters: Dict[str, Union[str, bool, int, float, list, tuple]] = Field(default_factory=dict)
+    filters: Dict[str, Union[str, Dict, bool, int, float, list, tuple]] = Field(default_factory=dict)
     df_params: Dict[str, Union[tuple, str, bool, None]] = Field(default_factory=dict)
     df_options: Dict[str, Union[bool, str, None]] = Field(default_factory=dict)
     params: Dict[str, Union[str, bool, int, float, List[Union[str, int, bool, float]]]] = Field(default_factory=dict)

sibi_dst-0.3.27/sibi_dst/utils/__init__.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from .log_utils import Logger
+from .date_utils import *
+from .data_utils import DataUtils
+from .file_utils import FileUtils
+from .filepath_generator import FilePathGenerator
+from .df_utils import DfUtils
+from .storage_manager import StorageManager
+from .parquet_saver import ParquetSaver
+from .clickhouse_writer import ClickHouseWriter
+from .airflow_manager import AirflowDAGManager
+from .credentials import *
+from .data_wrapper import DataWrapper
+
+__all__ = [
+    "Logger",
+    "ConfigManager",
+    "ConfigLoader",
+    "DateUtils",
+    "BusinessDays",
+    "FileUtils",
+    "DataWrapper",
+    "DataUtils",
+    "FilePathGenerator",
+    "ParquetSaver",
+    "StorageManager",
+    "DfUtils",
+    "ClickHouseWriter",
+    "AirflowDAGManager",
+]

sibi_dst-0.3.25/sibi_dst/utils/_clickhouse_writer.py → sibi_dst-0.3.27/sibi_dst/utils/clickhouse_writer.py

@@ -125,7 +125,7 @@ class ClickHouseWriter:
         """
         Writes the Dask DataFrame to a ClickHouse table partition by partition.
         """
-        if len(self.df.head().index) == 0:
+        if len(self.df.index) == 0:
             self.logger.debug("No data found. Nothing written.")
             return
 

sibi_dst-0.3.25/sibi_dst/utils/_data_utils.py → sibi_dst-0.3.27/sibi_dst/utils/data_utils.py

@@ -95,7 +95,7 @@ class DataUtils:
         # Get unique IDs from source column
         ids = df[source_col].dropna().unique()
         # Compute if it's a Dask Series
-        if isinstance(ids, dd.core.Series):
+        if isinstance(ids, dd.Series):
             ids = ids.compute()
 
         # Check if any IDs are found

sibi_dst-0.3.25/sibi_dst/utils/_data_wrapper.py → sibi_dst-0.3.27/sibi_dst/utils/data_wrapper.py

@@ -126,32 +126,6 @@ class DataWrapper:
                     futures[new_future] = priority
                     self.logger.info(f"Resubmitted task for priority {priority} after timeout.")
 
-    # def process(self):
-    #     """Execute the update plan following the specified hierarchy."""
-    #     update_plan_table = self.generate_update_plan_with_conditions()
-    #
-    #     # Display the update plan table to the user if show_progress is True
-    #     if self.show_progress:
-    #         display(update_plan_table)
-    #
-    #     # Process files according to the hierarchy, considering only `update_required` dates
-    #     for category, description in [
-    #         ("overwrite", "Processing files due to overwrite=True"),
-    #         ("history_days", "Processing files within history_days_threshold"),
-    #         ("missing_files", "Processing missing files")
-    #     ]:
-    #         # Filter dates in the category where `update_required` is True
-    #         dates_to_process = update_plan_table[
-    #             (update_plan_table["update_category"] == category) & (update_plan_table["update_required"])
-    #             ]["date"].tolist()
-    #
-    #         date_iterator = dates_to_process
-    #         if self.show_progress:
-    #             date_iterator = tqdm(date_iterator, desc=f"{description}:{self.dataclass.__name__}", unit="date")
-    #
-    #         for current_date in date_iterator:
-    #             self.process_date(current_date)
-
     def is_file_older_than(self, file_path: str) -> bool:
         """
         Check if a file is older than the specified max_age_minutes.

sibi_dst-0.3.25/sibi_dst/utils/_df_utils.py → sibi_dst-0.3.27/sibi_dst/utils/df_utils.py

@@ -1,7 +1,7 @@
 import dask.dataframe as dd
 import pandas as pd
 
-from ._log_utils import Logger
+from .log_utils import Logger
 
 
 class DfUtils:

sibi_dst-0.3.25/sibi_dst/utils/__init__.py

@@ -1,31 +0,0 @@
-from __future__ import annotations
-
-from ._log_utils import Logger
-from ._date_utils import *
-from ._data_utils import DataUtils
-from ._file_utils import FileUtils
-from ._filepath_generator import FilePathGenerator
-from ._df_utils import DfUtils
-from ._storage_manager import StorageManager
-from ._parquet_saver import ParquetSaver
-from ._clickhouse_writer import ClickHouseWriter
-from ._airflow_manager import AirflowDAGManager
-from ._credentials import *
-from ._data_wrapper import DataWrapper
-
-__all__ = [
-    "Logger",
-    "ConfigManager",
-    "ConfigLoader",
-    "DateUtils",
-    "BusinessDays",
-    "FileUtils",
-    "DataWrapper",
-    "DataUtils",
-    "FilePathGenerator",
-    "ParquetSaver",
-    "StorageManager",
-    "DfUtils",
-    "ClickHouseWriter",
-    "AirflowDAGManager",
-]