sibi-dst 0.3.18__py3-none-any.whl → 0.3.19__py3-none-any.whl

sibi_dst/df_helper/__init__.py

@@ -2,8 +2,10 @@ from __future__ import annotations
 
 from ._df_helper import DfHelper
 from ._parquet_artifact import ParquetArtifact
+from ._parquet_reader import ParquetReader
 
 __all__=[
     'DfHelper',
     'ParquetArtifact',
+    'ParquetReader',
 ]

sibi_dst/df_helper/_df_helper.py

@@ -56,6 +56,9 @@ class DfHelper:
         kwargs.setdefault("logger", self.logger)
         self.post_init(**kwargs)
 
+    def __str__(self):
+        return self.__class__.__name__
+
     def post_init(self, **kwargs):
         self.logger.debug(f"backend used: {self.backend}")
         self.backend_query = self.__get_config(QueryConfig, kwargs)

sibi_dst/df_helper/_parquet_reader.py

@@ -0,0 +1,49 @@
+from typing import Optional
+
+import dask.dataframe as dd
+import fsspec
+
+from sibi_dst.df_helper import DfHelper
+
+class ParquetReader(DfHelper):
+    DEFAULT_CONFIG = {
+        'backend': 'parquet'
+    }
+
+    def __init__(self, filesystem_type="file", filesystem_options=None, **kwargs):
+        self.config = {
+            **self.DEFAULT_CONFIG,
+            **kwargs,
+        }
+        self.df: Optional[dd.DataFrame] = None
+        self.parquet_storage_path = self.config.setdefault('parquet_storage_path', None)
+        if self.parquet_storage_path is None:
+            raise ValueError('parquet_storage_path must be set')
+        self.parquet_start_date = self.config.setdefault('parquet_start_date', None)
+        if self.parquet_start_date is None:
+            raise ValueError('parquet_start_date must be set')
+
+        self.parquet_end_date = self.config.setdefault('parquet_end_date', None)
+        if self.parquet_end_date is None:
+            raise ValueError('parquet_end_date must be set')
+
+        # Filesystem setup
+        self.filesystem_type = filesystem_type
+        self.filesystem_options = filesystem_options or {}
+        self.fs = fsspec.filesystem(self.filesystem_type, **self.filesystem_options)
+
+        if not self.directory_exists():
+            raise ValueError(f"{self.parquet_storage_path} does not exist")
+
+        super().__init__(**self.config)
+
+    def load(self, **kwargs):
+        self.df = super().load(**kwargs)
+        return self.df
+
+    def directory_exists(self):
+        try:
+            info = self.fs.info(self.parquet_storage_path)
+            return info['type'] == 'directory'
+        except FileNotFoundError:
+            return False

sibi_dst/df_helper/backends/parquet/_parquet_options.py

@@ -52,7 +52,7 @@ class ParquetConfig(BaseModel):
                 raise ValueError('Parquet end date must be greater than start date')
 
             # Saving to parquet is disabled when start and end dates are provided, as we will load parquet files
-            self.parquet_folder_list = FilePathGenerator(str(self.parquet_storage_path)).generate_file_paths(start_date, end_date)
+            self.parquet_folder_list = FilePathGenerator(str(self.parquet_storage_path), logger=self.logger).generate_file_paths(start_date, end_date)
             self.parquet_size_bytes = self.get_parquet_size_bytes()
             self.load_parquet = True
             #self.load_parquet = all([self.fs.exists(folder) for folder in self.parquet_folder_list]) and self.parquet_size_bytes > 0

sibi_dst/df_helper/backends/sql_alchemy/_sqlalchemy_model_builder.py

@@ -9,6 +9,7 @@ Base = declarative_base()
 apps_label = "datacubes"
 
 class SqlAlchemyModelBuilder:
+    _model_cache = {}  # Local cache for model classes
     def __init__(self, engine, table_name):
         """
         Initialize the model builder with a database engine and specific table.
@@ -21,28 +22,21 @@ class SqlAlchemyModelBuilder:
         self.table_name = table_name
         self.metadata = MetaData()
         self.table = None  # Placeholder for the specific table
+        self.class_name = self.normalize_class_name(self.table_name)
 
     def build_model(self) -> type:
-        """
-        Build a SQLAlchemy ORM model for the specified table.
-
-        Returns:
-            type: Dynamically generated SQLAlchemy ORM model class.
-        """
-        # Check if the class is already registered
-        class_name = self.normalize_class_name(self.table_name)
-        mapper_registry = Base.registry
-        if class_name in mapper_registry._class_registry:
-            return mapper_registry._class_registry[class_name]
+        # Check if the model is already registered
+        model = Base.registry._class_registry.get(self.class_name)
+        if model:
+            return model
 
-        # Reflect only the specified table
         self.metadata.reflect(only=[self.table_name], bind=self.engine)
         self.table = self.metadata.tables.get(self.table_name)
-
         if self.table is None:
             raise ValueError(f"Table '{self.table_name}' does not exist in the database.")
 
-        return self.create_model()
+        model = self.create_model()
+        return model
 
     def create_model(self) -> type:
         """
@@ -52,7 +46,6 @@ class SqlAlchemyModelBuilder:
             type: Dynamically generated SQLAlchemy ORM model class.
         """
         # Normalize the class name from the table name
-        class_name = self.normalize_class_name(self.table_name)
         columns = self.get_columns(self.table)
 
         # Define attributes for the model class
@@ -66,9 +59,11 @@ class SqlAlchemyModelBuilder:
         # Add columns and relationships to the model
         attrs.update(columns)
         #self.add_relationships(attrs, self.table)
-
-        # Dynamically create the model class
-        model = type(class_name, (Base,), attrs)
+        model = Base.registry._class_registry.get(self.class_name)
+        if not model:
+            model = type(self.class_name, (Base,), attrs)
+            # Add the class to Base.registry so it is registered
+            Base.registry._class_registry[self.class_name] = model
         return model
 
     def get_columns(self, table: Table):

sibi_dst/utils/_data_wrapper.py

@@ -1,15 +1,12 @@
 import datetime
 from typing import Type, Any, Dict, Optional
-
 import fsspec
 import pandas as pd
 from IPython.display import display
-from tqdm import tqdm
-
 from sibi_dst.utils import Logger
+from tqdm import tqdm
 from sibi_dst.utils import ParquetSaver
 
-
 class DataWrapper:
     DEFAULT_MAX_AGE_MINUTES = 1440
     DEFAULT_HISTORY_DAYS_THRESHOLD = 30
@@ -46,7 +43,7 @@ class DataWrapper:
         self.reverse_order = reverse_order
         self.overwrite = overwrite
         self.ignore_missing = ignore_missing
-        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+        self.logger = logger or Logger.default_logger(logger_name=self.dataclass.__name__)
         self.max_age_minutes = max_age_minutes
         self.history_days_threshold = history_days_threshold
         self.show_progress = show_progress
@@ -96,7 +93,7 @@ class DataWrapper:
 
             date_iterator = dates_to_process
             if self.show_progress:
-                date_iterator = tqdm(date_iterator, desc=description, unit="date")
+                date_iterator = tqdm(date_iterator, desc=f"{description}:{self.dataclass.__name__}", unit="date")
 
             for current_date in date_iterator:
                 self.process_date(current_date)
@@ -113,16 +110,14 @@ class DataWrapper:
             )
             current_time = datetime.datetime.now(datetime.timezone.utc)
             file_age_minutes = (current_time - file_modification_datetime).total_seconds() / 60
-
-            if self.verbose:
-                self.logger.debug(
-                    f"File {file_path} is {round(file_age_minutes, 2)} minutes old "
-                    f"(threshold: {self.max_age_minutes} minutes)"
-                )
+            self.logger.info(
+                f"File {file_path} is {round(file_age_minutes, 2)} minutes old "
+                f"(threshold: {self.max_age_minutes} minutes)"
+            )
 
             return file_age_minutes > self.max_age_minutes
         except FileNotFoundError:
-            return True  # Treat missing files as old
+            return True
 
     def process_date(self, date: datetime.date):
         """Process a specific date by regenerating data as necessary."""
@@ -130,16 +125,13 @@ class DataWrapper:
         full_parquet_filename = f"{folder}{self.parquet_filename}"
 
         start_time = datetime.datetime.now()
-
-        if self.verbose:
-            self.logger.debug(f"Processing {full_parquet_filename}...")
+        self.logger.info(f"Processing {full_parquet_filename}...")
 
         data_object = self.dataclass(**self.class_params)
         df = data_object.load_period(dt_field=self.date_field, start=date, end=date)
 
         if len(df.index)==0:
-            if self.verbose:
-                self.logger.debug("No data found for the specified date.")
+            self.logger.error("No data found for the specified date.")
             return
 
         parquet_saver = ParquetSaver(df, folder, self.logger)
@@ -147,11 +139,9 @@ class DataWrapper:
 
         end_time = datetime.datetime.now()
         duration_seconds = (end_time - start_time).total_seconds()
-
-        if self.verbose:
-            self.logger.debug(
-                f"Data saved to {full_parquet_filename}. Processing time: {duration_seconds:.2f} seconds"
-            )
+        self.logger.info(
+            f"Data saved to {full_parquet_filename}. Processing time: {duration_seconds:.2f} seconds"
+        )
 
     def generate_update_plan_with_conditions(self):
         """
@@ -167,7 +157,7 @@ class DataWrapper:
 
         date_range = self.generate_date_range()
         if self.show_progress:
-            date_range = tqdm(date_range, desc=f"Evaluating update plan {self.__class__.__name__}", unit="date")
+            date_range = tqdm(date_range, desc=f"Evaluating update plan:{self.dataclass.__name__}", unit="date")
 
         for current_date in date_range:
             folder = f'{self.data_path}{current_date.year}/{current_date.month:02d}/{current_date.day:02d}/'
@@ -203,25 +193,13 @@ class DataWrapper:
                 "within_history": within_history,
                 "missing_file": missing_file,
                 "update_required": update_required,
-                "update_category": category
+                "update_category": category,
+                "datawrapper class":self.dataclass.__name__
             })
 
         update_plan_table = pd.DataFrame(rows)
         return update_plan_table
 
-
-
-# # Usage:
-# # wrapper = DataWrapper(
-# #     dataclass=YourDataClass,
-# #     date_field="created_at",
-# #     data_path="/path/to/data",
-# #     parquet_filename="data.parquet",
-# #     start_date="2022-01-01",
-# #     end_date="2022-12-31",
-# #     filesystem_type="file",
-# #     verbose=True
-# # )
 # # wrapper.process()
 # # wrapper = DataWrapper(
 # #    dataclass=YourDataClass,

{sibi_dst-0.3.18.dist-info → sibi_dst-0.3.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.18
+Version: 0.3.19
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -40,13 +40,13 @@ Description-Content-Type: text/markdown
 
 Data Science Toolkit
 ---------------------
-Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, Scikit-Learn, XGBOOST, Django ORM, DjangoRestFrameWork
+Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, Scikit-Learn, XGBOOST, Django ORM, SQLAlchemy, DjangoRestFrameWork
 
 Major Functionality
 --------------------
 1) Build DataCubes, DataSets and DataObjects from different datasources. These include relational databases, parquet files, xlsx, delimited tables, json, json API REST.
-2) Rich set of common dataframe management utilities.
-3) Share Data with client applications by write to Datawarehouses in local filesystems as well as other supported platforms.
+2) Common dataframe management utilities.
+3) Share Data with client applications by write to Data Warehouses in local filesystems as well as other supported platforms.
 4) Build microservices to communicate/share data via API-REST, gRPC.
 
 

{sibi_dst-0.3.18.dist-info → sibi_dst-0.3.19.dist-info}/RECORD

@@ -1,7 +1,8 @@
 sibi_dst/__init__.py,sha256=1KaC0LYTHxjpENq-NXI325WcEYZ8GCBrHGkLoFxEcu0,251
-sibi_dst/df_helper/__init__.py,sha256=JXJBY47G6wOYhzNI646OBl3pSGWIy4282-3qPGYHU7w,167
-sibi_dst/df_helper/_df_helper.py,sha256=CRlHREaakaInigqsswK8FqGqHgHgzhFEGR-usWNWWiU,13854
+sibi_dst/df_helper/__init__.py,sha256=rbTr9CqwbJhu8pbZabwfcOqhm-5hm2iXk0vVBtK01bA,231
+sibi_dst/df_helper/_df_helper.py,sha256=e6e32CRTCKjFVvYMytWTuBVpwB1VcnVQ1T4Rg8KXWvY,13917
 sibi_dst/df_helper/_parquet_artifact.py,sha256=ctISmwxP9icFCXsELBjbPiz-FK3CEojN7yNIlStdOWw,4974
+sibi_dst/df_helper/_parquet_reader.py,sha256=A8qWuWQiaiS7pk4sD5EDAvGs-qz7VfziINXpSA7o00U,1683
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/df_helper/backends/django/__init__.py,sha256=VkvYql-PUyCKOfoKx5aGdUAki8X-kULfhnCvgSQzHDQ,311
 sibi_dst/df_helper/backends/django/_django_db_connection.py,sha256=9fGvXXgqPk_TC7BvaSljxTgNDWoXi_LZxKabEgExznM,1654
@@ -13,13 +14,13 @@ sibi_dst/df_helper/backends/http/__init__.py,sha256=AG9JSDRyVna2r1yxCQ9HcY32EaGn
 sibi_dst/df_helper/backends/http/_http_config.py,sha256=NN3bol7NgBTDv70yOX7hJkazt1-dAAdFWVkYyHdIXsI,2128
 sibi_dst/df_helper/backends/parquet/__init__.py,sha256=ClkyIsIh_ovEwqm0dTrkXImbPjLDTVHW2NQqqfQwWAw,187
 sibi_dst/df_helper/backends/parquet/_parquet_filter_handler.py,sha256=6iFvblnVq0qj89QvieQuYxe_2RPX5ArKfq5zBcEIj90,3660
-sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=suJC7LfNEWAo-7_R62YTMSRku3k8orysft83VxRUems,4394
+sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=09b9yLPREvx6ebs62B9qEqJt1cCKJz97plGW82i4630,4414
 sibi_dst/df_helper/backends/sql_alchemy/__init__.py,sha256=FHorj40SbHc0OBzQ_ieG6MG-HLbf0tw6I_5eoIjJkOI,369
 sibi_dst/df_helper/backends/sql_alchemy/_io_sqlalchemy_dask.py,sha256=YYhjt5rL1yomcrby4i4bD5wPVDzRJpZZbxHp5CM40tQ,5414
 sibi_dst/df_helper/backends/sql_alchemy/_sqlachemy_filter_handler.py,sha256=KShsLJYGVxN0ps9Wot7fF0nR0wW9WzcPIcWZ9f5vdBo,4654
 sibi_dst/df_helper/backends/sql_alchemy/_sqlalchemy_db_connection.py,sha256=HtMsfH5com4dLVJxh3wdMUpQI3mz0cKDJz0CmFS2S8U,1648
 sibi_dst/df_helper/backends/sql_alchemy/_sqlalchemy_load_from_db.py,sha256=QkR-_S4zqJpwH9dJ5cqXW8iy9XoAFUXmcsgUSm3PbLo,2251
-sibi_dst/df_helper/backends/sql_alchemy/_sqlalchemy_model_builder.py,sha256=IQK2jOXMNJRQOSD0VQ0p11BeDGlvxD8NfFRilw9Go80,4466
+sibi_dst/df_helper/backends/sql_alchemy/_sqlalchemy_model_builder.py,sha256=RjtKEk-i8EmX98rwqkq1Bg7IgPwYDduL967gsl9T73c,4401
 sibi_dst/df_helper/backends/sql_model/__init__.py,sha256=MXd4OOdTqR4cENSV733SGodPO6eQMCexANs-3w0qL5U,226
 sibi_dst/df_helper/backends/sql_model/_sqlmodel_db_connection.py,sha256=6jmMjKIv5Btysj3kZMaXQ98IqKQkhnOC-JWtb1B8rus,4265
 sibi_dst/df_helper/backends/sql_model/_sqlmodel_load_from_db.py,sha256=jYwkIz7_E9Z6Mqw1a9TCWKWD146Tbx7mcQFxIpmKgKU,3686
@@ -33,7 +34,7 @@ sibi_dst/utils/_airflow_manager.py,sha256=rlt3eolR5QvtxWhAtBTCpHXvxftnKM-ibPMv3f
 sibi_dst/utils/_clickhouse_writer.py,sha256=JcnWN2635ATCOaFiB6NYglNXDwqKw0jC7Urs9WOZE20,8571
 sibi_dst/utils/_credentials.py,sha256=8i6z7y3y5S-6mSk4xrT2AwhzCA32mTn1n1iYX9IVyHk,1724
 sibi_dst/utils/_data_utils.py,sha256=ch4j5FEs8ZnniUzpbeLO-b4Yco_6nwCu71xHaVqMGi4,7050
-sibi_dst/utils/_data_wrapper.py,sha256=_hLZhKqSxcfXe8IyWM2paBxtW2JlOCq2jYhNGcInPi4,9406
+sibi_dst/utils/_data_wrapper.py,sha256=cvUkGRiPfCyLD4XcoX7FWLYzM8gnHBGR1pJ08PMneCk,9010
 sibi_dst/utils/_date_utils.py,sha256=KYB07puKDrSG8tOm_i1HGX0TjLNUtSWjwfsCYBmW9co,10619
 sibi_dst/utils/_df_utils.py,sha256=pjEfkof9hggXQgYerG0p4DXrwBeIRynJFg4IX3Yrb4c,10919
 sibi_dst/utils/_file_utils.py,sha256=5EN90c8N1n9d-_xwz2RzaYcXRMQY_rws2Q3EA3pNAog,1254
@@ -41,6 +42,6 @@ sibi_dst/utils/_filepath_generator.py,sha256=ytPSZ9GYOnnSP25zwA-0NjFHupPRZyXwixW
 sibi_dst/utils/_log_utils.py,sha256=rPp8z1UglwvqzBOOAvMOct0syQZ-54gGYafnJDRYZN4,2313
 sibi_dst/utils/_parquet_saver.py,sha256=3BK0XXgMOOAdIw4OzbwMxmDrzDw3_MKi8RTpulIVUe0,4367
 sibi_dst/utils/_storage_manager.py,sha256=KP2HBXnLUMMquqcO30ecfuoU7g1z8RtaV3Dv0TvEXoY,3856
-sibi_dst-0.3.18.dist-info/METADATA,sha256=FOkOpqGX3IXCQMULhwbXYwTCGMDULJrp-OYtYlZYdzw,2133
-sibi_dst-0.3.18.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-sibi_dst-0.3.18.dist-info/RECORD,,
+sibi_dst-0.3.19.dist-info/METADATA,sha256=IDeMqZZHRsAV-v5TngSTKaB7y7SQhMjfEduHozqhOsk,2134
+sibi_dst-0.3.19.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+sibi_dst-0.3.19.dist-info/RECORD,,