sibi-dst 0.3.18__tar.gz → 0.3.19__tar.gz

{sibi_dst-0.3.18 → sibi_dst-0.3.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.18
+Version: 0.3.19
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -40,13 +40,13 @@ Description-Content-Type: text/markdown
 
 Data Science Toolkit
 ---------------------
-Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, Scikit-Learn, XGBOOST, Django ORM, DjangoRestFrameWork
+Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, Scikit-Learn, XGBOOST, Django ORM, SQLAlchemy, DjangoRestFrameWork
 
 Major Functionality
 --------------------
 1) Build DataCubes, DataSets and DataObjects from different datasources. These include relational databases, parquet files, xlsx, delimited tables, json, json API REST.
-2) Rich set of common dataframe management utilities.
-3) Share Data with client applications by write to Datawarehouses in local filesystems as well as other supported platforms.
+2) Common dataframe management utilities.
+3) Share Data with client applications by write to Data Warehouses in local filesystems as well as other supported platforms.
 4) Build microservices to communicate/share data via API-REST, gRPC.
 
 

{sibi_dst-0.3.18 → sibi_dst-0.3.19}/README.md

@@ -2,12 +2,12 @@
 
 Data Science Toolkit
 ---------------------
-Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, Scikit-Learn, XGBOOST, Django ORM, DjangoRestFrameWork
+Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, Scikit-Learn, XGBOOST, Django ORM, SQLAlchemy, DjangoRestFrameWork
 
 Major Functionality
 --------------------
 1) Build DataCubes, DataSets and DataObjects from different datasources. These include relational databases, parquet files, xlsx, delimited tables, json, json API REST.
-2) Rich set of common dataframe management utilities.
-3) Share Data with client applications by write to Datawarehouses in local filesystems as well as other supported platforms.
+2) Common dataframe management utilities.
+3) Share Data with client applications by write to Data Warehouses in local filesystems as well as other supported platforms.
 4) Build microservices to communicate/share data via API-REST, gRPC.
 

{sibi_dst-0.3.18 → sibi_dst-0.3.19}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.18"
+version = "0.3.19"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-0.3.18 → sibi_dst-0.3.19}/sibi_dst/df_helper/__init__.py

@@ -2,8 +2,10 @@ from __future__ import annotations
 
 from ._df_helper import DfHelper
 from ._parquet_artifact import ParquetArtifact
+from ._parquet_reader import ParquetReader
 
 __all__=[
     'DfHelper',
     'ParquetArtifact',
+    'ParquetReader',
 ]

{sibi_dst-0.3.18 → sibi_dst-0.3.19}/sibi_dst/df_helper/_df_helper.py

@@ -56,6 +56,9 @@ class DfHelper:
         kwargs.setdefault("logger", self.logger)
         self.post_init(**kwargs)
 
+    def __str__(self):
+        return self.__class__.__name__
+
     def post_init(self, **kwargs):
         self.logger.debug(f"backend used: {self.backend}")
         self.backend_query = self.__get_config(QueryConfig, kwargs)

sibi_dst-0.3.19/sibi_dst/df_helper/_parquet_reader.py

@@ -0,0 +1,49 @@
+from typing import Optional
+
+import dask.dataframe as dd
+import fsspec
+
+from sibi_dst.df_helper import DfHelper
+
+class ParquetReader(DfHelper):
+    DEFAULT_CONFIG = {
+        'backend': 'parquet'
+    }
+
+    def __init__(self, filesystem_type="file", filesystem_options=None, **kwargs):
+        self.config = {
+            **self.DEFAULT_CONFIG,
+            **kwargs,
+        }
+        self.df: Optional[dd.DataFrame] = None
+        self.parquet_storage_path = self.config.setdefault('parquet_storage_path', None)
+        if self.parquet_storage_path is None:
+            raise ValueError('parquet_storage_path must be set')
+        self.parquet_start_date = self.config.setdefault('parquet_start_date', None)
+        if self.parquet_start_date is None:
+            raise ValueError('parquet_start_date must be set')
+
+        self.parquet_end_date = self.config.setdefault('parquet_end_date', None)
+        if self.parquet_end_date is None:
+            raise ValueError('parquet_end_date must be set')
+
+        # Filesystem setup
+        self.filesystem_type = filesystem_type
+        self.filesystem_options = filesystem_options or {}
+        self.fs = fsspec.filesystem(self.filesystem_type, **self.filesystem_options)
+
+        if not self.directory_exists():
+            raise ValueError(f"{self.parquet_storage_path} does not exist")
+
+        super().__init__(**self.config)
+
+    def load(self, **kwargs):
+        self.df = super().load(**kwargs)
+        return self.df
+
+    def directory_exists(self):
+        try:
+            info = self.fs.info(self.parquet_storage_path)
+            return info['type'] == 'directory'
+        except FileNotFoundError:
+            return False

{sibi_dst-0.3.18 → sibi_dst-0.3.19}/sibi_dst/df_helper/backends/parquet/_parquet_options.py

@@ -52,7 +52,7 @@ class ParquetConfig(BaseModel):
                 raise ValueError('Parquet end date must be greater than start date')
 
             # Saving to parquet is disabled when start and end dates are provided, as we will load parquet files
-            self.parquet_folder_list = FilePathGenerator(str(self.parquet_storage_path)).generate_file_paths(start_date, end_date)
+            self.parquet_folder_list = FilePathGenerator(str(self.parquet_storage_path), logger=self.logger).generate_file_paths(start_date, end_date)
             self.parquet_size_bytes = self.get_parquet_size_bytes()
             self.load_parquet = True
             #self.load_parquet = all([self.fs.exists(folder) for folder in self.parquet_folder_list]) and self.parquet_size_bytes > 0

{sibi_dst-0.3.18 → sibi_dst-0.3.19}/sibi_dst/df_helper/backends/sql_alchemy/_sqlalchemy_model_builder.py

@@ -9,6 +9,7 @@ Base = declarative_base()
 apps_label = "datacubes"
 
 class SqlAlchemyModelBuilder:
+    _model_cache = {}  # Local cache for model classes
     def __init__(self, engine, table_name):
         """
         Initialize the model builder with a database engine and specific table.
@@ -21,28 +22,21 @@ class SqlAlchemyModelBuilder:
         self.table_name = table_name
         self.metadata = MetaData()
         self.table = None  # Placeholder for the specific table
+        self.class_name = self.normalize_class_name(self.table_name)
 
     def build_model(self) -> type:
-        """
-        Build a SQLAlchemy ORM model for the specified table.
-
-        Returns:
-            type: Dynamically generated SQLAlchemy ORM model class.
-        """
-        # Check if the class is already registered
-        class_name = self.normalize_class_name(self.table_name)
-        mapper_registry = Base.registry
-        if class_name in mapper_registry._class_registry:
-            return mapper_registry._class_registry[class_name]
+        # Check if the model is already registered
+        model = Base.registry._class_registry.get(self.class_name)
+        if model:
+            return model
 
-        # Reflect only the specified table
         self.metadata.reflect(only=[self.table_name], bind=self.engine)
         self.table = self.metadata.tables.get(self.table_name)
-
         if self.table is None:
             raise ValueError(f"Table '{self.table_name}' does not exist in the database.")
 
-        return self.create_model()
+        model = self.create_model()
+        return model
 
     def create_model(self) -> type:
         """
@@ -52,7 +46,6 @@ class SqlAlchemyModelBuilder:
             type: Dynamically generated SQLAlchemy ORM model class.
         """
         # Normalize the class name from the table name
-        class_name = self.normalize_class_name(self.table_name)
         columns = self.get_columns(self.table)
 
         # Define attributes for the model class
@@ -66,9 +59,11 @@ class SqlAlchemyModelBuilder:
         # Add columns and relationships to the model
         attrs.update(columns)
         #self.add_relationships(attrs, self.table)
-
-        # Dynamically create the model class
-        model = type(class_name, (Base,), attrs)
+        model = Base.registry._class_registry.get(self.class_name)
+        if not model:
+            model = type(self.class_name, (Base,), attrs)
+            # Add the class to Base.registry so it is registered
+            Base.registry._class_registry[self.class_name] = model
         return model
 
     def get_columns(self, table: Table):

{sibi_dst-0.3.18 → sibi_dst-0.3.19}/sibi_dst/utils/_data_wrapper.py

@@ -1,15 +1,12 @@
 import datetime
 from typing import Type, Any, Dict, Optional
-
 import fsspec
 import pandas as pd
 from IPython.display import display
-from tqdm import tqdm
-
 from sibi_dst.utils import Logger
+from tqdm import tqdm
 from sibi_dst.utils import ParquetSaver
 
-
 class DataWrapper:
     DEFAULT_MAX_AGE_MINUTES = 1440
     DEFAULT_HISTORY_DAYS_THRESHOLD = 30
@@ -46,7 +43,7 @@ class DataWrapper:
         self.reverse_order = reverse_order
         self.overwrite = overwrite
         self.ignore_missing = ignore_missing
-        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+        self.logger = logger or Logger.default_logger(logger_name=self.dataclass.__name__)
         self.max_age_minutes = max_age_minutes
         self.history_days_threshold = history_days_threshold
         self.show_progress = show_progress
@@ -96,7 +93,7 @@ class DataWrapper:
 
             date_iterator = dates_to_process
             if self.show_progress:
-                date_iterator = tqdm(date_iterator, desc=description, unit="date")
+                date_iterator = tqdm(date_iterator, desc=f"{description}:{self.dataclass.__name__}", unit="date")
 
             for current_date in date_iterator:
                 self.process_date(current_date)
@@ -113,16 +110,14 @@ class DataWrapper:
             )
             current_time = datetime.datetime.now(datetime.timezone.utc)
             file_age_minutes = (current_time - file_modification_datetime).total_seconds() / 60
-
-            if self.verbose:
-                self.logger.debug(
-                    f"File {file_path} is {round(file_age_minutes, 2)} minutes old "
-                    f"(threshold: {self.max_age_minutes} minutes)"
-                )
+            self.logger.info(
+                f"File {file_path} is {round(file_age_minutes, 2)} minutes old "
+                f"(threshold: {self.max_age_minutes} minutes)"
+            )
 
             return file_age_minutes > self.max_age_minutes
         except FileNotFoundError:
-            return True  # Treat missing files as old
+            return True
 
     def process_date(self, date: datetime.date):
         """Process a specific date by regenerating data as necessary."""
@@ -130,16 +125,13 @@ class DataWrapper:
         full_parquet_filename = f"{folder}{self.parquet_filename}"
 
         start_time = datetime.datetime.now()
-
-        if self.verbose:
-            self.logger.debug(f"Processing {full_parquet_filename}...")
+        self.logger.info(f"Processing {full_parquet_filename}...")
 
         data_object = self.dataclass(**self.class_params)
         df = data_object.load_period(dt_field=self.date_field, start=date, end=date)
 
         if len(df.index)==0:
-            if self.verbose:
-                self.logger.debug("No data found for the specified date.")
+            self.logger.error("No data found for the specified date.")
             return
 
         parquet_saver = ParquetSaver(df, folder, self.logger)
@@ -147,11 +139,9 @@ class DataWrapper:
 
         end_time = datetime.datetime.now()
         duration_seconds = (end_time - start_time).total_seconds()
-
-        if self.verbose:
-            self.logger.debug(
-                f"Data saved to {full_parquet_filename}. Processing time: {duration_seconds:.2f} seconds"
-            )
+        self.logger.info(
+            f"Data saved to {full_parquet_filename}. Processing time: {duration_seconds:.2f} seconds"
+        )
 
     def generate_update_plan_with_conditions(self):
         """
@@ -167,7 +157,7 @@ class DataWrapper:
 
         date_range = self.generate_date_range()
         if self.show_progress:
-            date_range = tqdm(date_range, desc=f"Evaluating update plan {self.__class__.__name__}", unit="date")
+            date_range = tqdm(date_range, desc=f"Evaluating update plan:{self.dataclass.__name__}", unit="date")
 
         for current_date in date_range:
             folder = f'{self.data_path}{current_date.year}/{current_date.month:02d}/{current_date.day:02d}/'
@@ -203,25 +193,13 @@ class DataWrapper:
                 "within_history": within_history,
                 "missing_file": missing_file,
                 "update_required": update_required,
-                "update_category": category
+                "update_category": category,
+                "datawrapper class":self.dataclass.__name__
             })
 
         update_plan_table = pd.DataFrame(rows)
         return update_plan_table
 
-
-
-# # Usage:
-# # wrapper = DataWrapper(
-# #     dataclass=YourDataClass,
-# #     date_field="created_at",
-# #     data_path="/path/to/data",
-# #     parquet_filename="data.parquet",
-# #     start_date="2022-01-01",
-# #     end_date="2022-12-31",
-# #     filesystem_type="file",
-# #     verbose=True
-# # )
 # # wrapper.process()
 # # wrapper = DataWrapper(
 # #    dataclass=YourDataClass,