sibi-dst 0.3.43__py3-none-any.whl → 0.3.45__py3-none-any.whl

sibi_dst/df_helper/_parquet_artifact.py

@@ -1,3 +1,4 @@
+import datetime
 import logging
 from typing import Optional, Any, Dict
 
@@ -136,7 +137,30 @@ class ParquetArtifact(DfHelper):
 
     def update_parquet(self, period: str = 'today', **kwargs) -> None:
         """Update the Parquet file with data from a specific period."""
-        kwargs.update(self.parse_parquet_period(period=period))
+
+        def itd_config():
+            try:
+                start_date = kwargs.pop('history_begins_on')
+            except KeyError:
+                raise ValueError("For period 'itd', you must provide 'history_begins_on' in kwargs.")
+            return {'parquet_start_date': start_date, 'parquet_end_date': datetime.date.today().strftime('%Y-%m-%d')}
+
+        def ytd_config():
+            return {
+                'parquet_start_date': datetime.date(datetime.date.today().year, 1, 1).strftime('%Y-%m-%d'),
+                'parquet_end_date': datetime.date.today().strftime('%Y-%m-%d')
+            }
+
+        config_map = {
+            'itd': itd_config,
+            'ytd': ytd_config
+        }
+
+        if period in config_map:
+            kwargs.update(config_map[period]())
+        else:
+            kwargs.update(self.parse_parquet_period(period=period))
+        self.logger.debug(f"kwargs passed to update parquet: {kwargs}")
         self.generate_parquet(**kwargs)
 
     def rebuild_parquet(self, **kwargs) -> None:

sibi_dst/df_helper/backends/django/_load_from_db.py

@@ -64,6 +64,7 @@ class DjangoLoadFromDb:
         self.connection_config = db_connection
         self.debug = kwargs.pop('debug', False)
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+        self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
         if self.connection_config.model is None:
             if self.debug:
                 self.logger.debug('Model must be specified')

sibi_dst/df_helper/backends/parquet/_filter_handler.py

@@ -17,8 +17,9 @@ class ParquetFilterHandler(object):
     :ivar logger: Logger object to handle logging within the class. Defaults to the class-level logger.
     :type logger: Logger
     """
-    def __init__(self, logger=None):
+    def __init__(self, logger=None, debug=False):
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+        self.logger.set_level(Logger.DEBUG if debug else Logger.INFO)
 
     @staticmethod
     def apply_filters_dask(df, filters):

sibi_dst/df_helper/backends/parquet/_parquet_options.py

@@ -62,6 +62,7 @@ class ParquetConfig(BaseModel):
     parquet_end_date: Optional[str] = None
     fs: Optional[fsspec.spec.AbstractFileSystem] = None  # Your fsspec filesystem object
     logger: Optional[Logger] = None
+    debug: bool = False
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     @model_validator(mode='after')
@@ -83,9 +84,7 @@ class ParquetConfig(BaseModel):
         # Configure paths based on fsspec
         if self.logger is None:
             self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
-        #self.fs = fsspec.filesystem("file") if "://" not in str(self.parquet_storage_path) else fsspec.filesystem(
-        #    str(self.parquet_storage_path).split("://")[0])
-        # Validation for parquet path
+        self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
 
 
         if self.parquet_storage_path is None:

sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py

@@ -29,6 +29,7 @@ class SQLAlchemyDask:
         self.engine = create_engine(engine_url)
         self.Session = sessionmaker(bind=self.engine)
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+        self.logger.set_level(logger.DEBUG if debug else logger.INFO)
 
     @staticmethod
     def infer_dtypes_from_model(model):
@@ -70,11 +71,7 @@ class SQLAlchemyDask:
                 # Build query
                 self.query = select(self.model)
                 if self.filters:
-                    """
-                    deprecated specific filter handling to a generic one
-                    #self.query = SqlAlchemyFilterHandler.apply_filters_sqlalchemy(self.query, self.model, self.filters)
-                    """
-                    self.query = FilterHandler(backend="sqlalchemy", logger=self.logger).apply_filters(self.query,
+                    self.query = FilterHandler(backend="sqlalchemy", logger=self.logger, debug=self.debug).apply_filters(self.query,
                                                                                                        model=self.model,
                                                                                                        filters=self.filters)
                 else:

sibi_dst/df_helper/core/_filter_handler.py

@@ -25,7 +25,7 @@ class FilterHandler:
     :ivar backend_methods: A dictionary mapping backend-specific methods for column retrieval and operation application.
     :type backend_methods: dict
     """
-    def __init__(self, backend, logger=None):
+    def __init__(self, backend, logger=None, debug=False):
         """
         Initialize the FilterHandler.
 
@@ -36,6 +36,7 @@ class FilterHandler:
         self.backend = backend
         self.logger = logger or Logger.default_logger(
             logger_name=self.__class__.__name__)  # No-op logger if none provided
+        self.logger.set_level(Logger.DEBUG if debug else Logger.INFO)
         self.backend_methods = self._get_backend_methods(backend)
 
     def apply_filters(self, query_or_df, model=None, filters=None):

sibi_dst/utils/log_utils.py

@@ -26,6 +26,12 @@ class Logger:
     :type logger: logging.Logger
     """
 
+    DEBUG = logging.DEBUG
+    INFO = logging.INFO
+    WARNING = logging.WARNING
+    ERROR = logging.ERROR
+    CRITICAL = logging.CRITICAL
+
     def __init__(self, log_dir: str, logger_name: str, log_file: str, log_level: int = logging.DEBUG):
         """
         Initialize the Logger instance.

{sibi_dst-0.3.43.dist-info → sibi_dst-0.3.45.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.43
+Version: 0.3.45
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.43.dist-info → sibi_dst-0.3.45.dist-info}/RECORD

@@ -2,28 +2,28 @@ sibi_dst/__init__.py,sha256=CLHfzrFNqklNx5uMKAPtbZfkbBbVYR5qsiMro0RTfmA,252
 sibi_dst/df_helper/__init__.py,sha256=A-f5cCBy949HHxgiPt0T4MG3qdLAnDpGOpRvP-2dXWc,400
 sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=toH2QvNF-CQNJ4Bc8xreytuWr37G0EWz4ciWVdFMVqU,11646
 sibi_dst/df_helper/_df_helper.py,sha256=IS1m9r9U-eJ7EVMBqmITmre6S0JfIl6nJtPIwNI3xKY,29771
-sibi_dst/df_helper/_parquet_artifact.py,sha256=-9VdTigmHPpF1itzLdCqagjdFLZLZO_ArdTV0t_huHw,9310
+sibi_dst/df_helper/_parquet_artifact.py,sha256=qt3WZXrE2EZs3KI0biGzm3znIZazzTh8fgiipCVr_Ic,10196
 sibi_dst/df_helper/_parquet_reader.py,sha256=L6mr2FeKtTeIn37G9EGpvOx8PwMqXb6qnEECqBaiwxo,3954
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/df_helper/backends/django/__init__.py,sha256=uWHi-DtQX5re7b2HcqoXUH3_FZWOw1VTmDf552FAkNs,256
 sibi_dst/df_helper/backends/django/_db_connection.py,sha256=AGbqCnmiX4toMaFPE5ne5h7QCkImjnBKvzGtUD6Ge8Q,3698
 sibi_dst/df_helper/backends/django/_io_dask.py,sha256=NjvJg6y9qKKCRiNrJL4f_A03iKDKEcjCi7LGbr9DgtM,19555
-sibi_dst/df_helper/backends/django/_load_from_db.py,sha256=jQGIeviggHmSfK28Z47nHz3cXs78gHsE8Op7Ov1kFCo,10562
+sibi_dst/df_helper/backends/django/_load_from_db.py,sha256=htG9ec4ix371ClEHQVpx4r3mhBdQaSykeHUCCRhN7L4,10637
 sibi_dst/df_helper/backends/django/_sql_model_builder.py,sha256=at9J7ecGkZbOOYba85uofe9C-ic4wwOqVgJcHpQNiYQ,21449
 sibi_dst/df_helper/backends/http/__init__.py,sha256=d1pfgYxbiYg7E0Iw8RbJ7xfqIfJShqqTBQQGU_S6OOo,105
 sibi_dst/df_helper/backends/http/_http_config.py,sha256=eGPFdqZ5M3Tscqx2P93B6XoBEEzlmdt7yNg7PXUQnNQ,4726
 sibi_dst/df_helper/backends/parquet/__init__.py,sha256=esWJ9aSuYC26d-T01z9dPrJ1uqJzvdaPNTYRb5qXTlQ,182
-sibi_dst/df_helper/backends/parquet/_filter_handler.py,sha256=VrT79Cr1HXu0VLVTuwAYOwSo52p8B1Q5lT3tTIk4uAA,5136
-sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=RZ5JopJVv9G_j0S9Vw66ohNvNcZHmeBfncMwSsLjM6o,10707
+sibi_dst/df_helper/backends/parquet/_filter_handler.py,sha256=TvDf0RXta7mwJv11GNQttYJsXgFf2XDj4oLIjt4xTzA,5219
+sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=TaU5_wG1Y3lQC8DVCItVvMnc6ZJmECLu3avssVEMbaM,10591
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=TuVp8Ce49dCIIxtyrtFGRblarQUl8QGcS-TDZd515IE,348
 sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=Kli83IEg5SFVqkhsK4w45cV6PbZnfdGanfsyiW6Xw00,2502
 sibi_dst/df_helper/backends/sqlalchemy/_filter_handler.py,sha256=58RCda1Hg_nsuJw-2V36IstsT8O84IQFgsdE7FnqvMk,4655
-sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=UuAHzZWBADsTwGhwZTJzR66Xdh189OR81C1IITwzls0,5620
+sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=BtiRSYA4kFIM-mBCdrwE20vzByfq8_Biv_jPLUCDv58,5466
 sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=I2Us3RrxHci561yyZYBuUCrLVOhB0F3KBnae78m_ARw,6259
 sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py,sha256=9oOOGrqAj9yL0FNWR1Fm9PdN7GoFi03ktCOyMjxAKLY,8402
 sibi_dst/df_helper/core/__init__.py,sha256=o4zDwgVmaijde3oix0ezb6KLxI5QFy-SGUhFTDVFLT4,569
 sibi_dst/df_helper/core/_defaults.py,sha256=eNpHD2sZxir-2xO0b3_V16ryw8YP_5FfpIKK0HNuiN4,7011
-sibi_dst/df_helper/core/_filter_handler.py,sha256=t3uLLJX5hWO_dWKCCz8Dwpc9RZ5PMHBIWkHSELCpFXI,11131
+sibi_dst/df_helper/core/_filter_handler.py,sha256=Pmbzygry2mpkNPVS7DBMulHpAb1yYZNFqUU0bJTWJF0,11214
 sibi_dst/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxRrQKE5FQRxcEWsac,6736
 sibi_dst/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/df_helper/data_cleaner.py,sha256=lkxQoXLvGzXCicFUimnA5nen5qkrO1oxgl_p2Be2o8w,5183
@@ -53,10 +53,10 @@ sibi_dst/utils/date_utils.py,sha256=QSsL_O9ZqIdNjhppyiVOjCFLmhJEKH6F2TeiGBZe9m4,
 sibi_dst/utils/df_utils.py,sha256=GAX0lthULTmGaDvYzuLmo0op7YKaCM5uot403QpztoM,11278
 sibi_dst/utils/file_utils.py,sha256=JpsybYj3XvVJisSBeVU6YSaZnYRm4_6YWTI3TLnnY4Y,1257
 sibi_dst/utils/filepath_generator.py,sha256=volVm0SSlBrtZp1RpTHxyui5rj5asNcVsWEBRY5FOUQ,6673
-sibi_dst/utils/log_utils.py,sha256=DtUAcY1aJzlIi9ItWDRz8Yyrz68A908JTm3XEd1HKBo,4484
+sibi_dst/utils/log_utils.py,sha256=eSAbi_jmMpJ8RpycakzT4S4zNkqVZDj3FY8WwnxpdXc,4623
 sibi_dst/utils/parquet_saver.py,sha256=_BFeD0HDXceBaBR0Y48t9MR_fY9G1Y8HrZCkj2X1yJs,8173
 sibi_dst/utils/phone_formatter.py,sha256=tsVTDamuthFYgy4-5UwmQkPQ-FGTGH7MjZyH8utAkIY,4945
 sibi_dst/utils/storage_manager.py,sha256=H_itUFJv9nP0BfXYYQDsw4RzB0YWfgVOAHNWAiMpZ_w,4443
-sibi_dst-0.3.43.dist-info/METADATA,sha256=k8F8a6kw9xKz0dRgqGOiM48hIoBR9MJUj5N04HSLhYk,6514
-sibi_dst-0.3.43.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.43.dist-info/RECORD,,
+sibi_dst-0.3.45.dist-info/METADATA,sha256=rH9MmLHB02ZhPVAs7gG5iYn6KvAnZMLHiIQeYYv6Rrg,6514
+sibi_dst-0.3.45.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.45.dist-info/RECORD,,