sibi-dst 0.3.21__tar.gz → 0.3.23__tar.gz

{sibi_dst-0.3.21 → sibi_dst-0.3.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.21
+Version: 0.3.23
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.21 → sibi_dst-0.3.23}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.21"
+version = "0.3.23"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-0.3.21 → sibi_dst-0.3.23}/sibi_dst/utils/_data_wrapper.py

@@ -1,4 +1,5 @@
 import datetime
+from concurrent.futures import ThreadPoolExecutor
 from typing import Type, Any, Dict, Optional
 
 import fsspec
@@ -32,7 +33,8 @@ class DataWrapper:
                  logger: Optional[Logger] = None,
                  max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
                  history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
-                 show_progress: bool = False):
+                 show_progress: bool = False,
+                 timeout: Optional[int] = 300):
         self.dataclass = dataclass
         self.date_field = date_field
         self.data_path = self.ensure_forward_slash(data_path)
@@ -50,6 +52,7 @@ class DataWrapper:
         self.max_age_minutes = max_age_minutes
         self.history_days_threshold = history_days_threshold
         self.show_progress = show_progress
+        self.timeout = timeout
 
         self.start_date = self.convert_to_date(start_date)
         self.end_date = self.convert_to_date(end_date)
@@ -76,31 +79,79 @@ class DataWrapper:
             yield date.date()
 
     def process(self):
-        """Execute the update plan following the specified hierarchy."""
+        """Execute the update plan using 'update_priority' to determine processing order."""
         update_plan_table = self.generate_update_plan_with_conditions()
 
-        # Display the update plan table to the user if show_progress is True
+        # Display the update plan table to the user if requested
         if self.show_progress:
             display(update_plan_table)
 
-        # Process files according to the hierarchy, considering only `update_required` dates
-        for category, description in [
-            ("overwrite", "Processing files due to overwrite=True"),
-            ("history_days", "Processing files within history_days_threshold"),
-            ("missing_files", "Processing missing files")
-        ]:
-            # Filter dates in the category where `update_required` is True
+        # Filter out rows that do not require updates (priority 0 means skip)
+        update_plan_table = update_plan_table[
+            (update_plan_table["update_required"] == True) & (update_plan_table["update_priority"] != 0)
+            ]
+
+        # Group by priority
+        priorities = sorted(update_plan_table["update_priority"].unique())
+
+        # We will process each priority level in its own thread.
+        # Each thread will handle all dates associated with that priority.
+        def process_priority(priority):
+            # Extract dates for the current priority
             dates_to_process = update_plan_table[
-                (update_plan_table["update_category"] == category) & (update_plan_table["update_required"])
+                update_plan_table["update_priority"] == priority
                 ]["date"].tolist()
 
+            # If show_progress is True, wrap in a progress bar
             date_iterator = dates_to_process
             if self.show_progress:
-                date_iterator = tqdm(date_iterator, desc=f"{description}:{self.dataclass.__name__}", unit="date")
+                date_iterator = tqdm(date_iterator, desc=f"Processing priority {priority}:{self.dataclass.__name__}",
+                                     unit="date")
 
+            # Process each date for this priority
             for current_date in date_iterator:
                 self.process_date(current_date)
 
+        # Launch a separate thread for each priority
+        with ThreadPoolExecutor(max_workers=len(priorities)) as executor:
+            futures = {executor.submit(process_priority, p): p for p in priorities}
+            for future in futures:
+                try:
+                    future.result(timeout=self.timeout)
+                except TimeoutError:
+                    self.logger.error(f"Thread for {self.dataclass.__name__} timed out. Thread cancelled.")
+                    future.cancel()
+                    priority = futures[future]
+                    new_future = executor.submit(process_priority, priority)
+                    futures[new_future] = priority
+                    self.logger.info(f"Resubmitted task for priority {priority} after timeout.")
+
+    # def process(self):
+    #     """Execute the update plan following the specified hierarchy."""
+    #     update_plan_table = self.generate_update_plan_with_conditions()
+    #
+    #     # Display the update plan table to the user if show_progress is True
+    #     if self.show_progress:
+    #         display(update_plan_table)
+    #
+    #     # Process files according to the hierarchy, considering only `update_required` dates
+    #     for category, description in [
+    #         ("overwrite", "Processing files due to overwrite=True"),
+    #         ("history_days", "Processing files within history_days_threshold"),
+    #         ("missing_files", "Processing missing files")
+    #     ]:
+    #         # Filter dates in the category where `update_required` is True
+    #         dates_to_process = update_plan_table[
+    #             (update_plan_table["update_category"] == category) & (update_plan_table["update_required"])
+    #             ]["date"].tolist()
+    #
+    #         date_iterator = dates_to_process
+    #         if self.show_progress:
+    #             date_iterator = tqdm(date_iterator, desc=f"{description}:{self.dataclass.__name__}", unit="date")
+    #
+    #         for current_date in date_iterator:
+    #             self.process_date(current_date)
+
     def is_file_older_than(self, file_path: str) -> bool:
         """
         Check if a file is older than the specified max_age_minutes.
@@ -181,12 +232,14 @@ class DataWrapper:
                     category = "history_days"
                     update_required = True
                 else:
+                    category = "file age is recent"
                     update_required = False
             # Hierarchy 3: Missing files
             elif missing_file and current_date <= today:
                 category = "missing_files"
                 update_required = True
             else:
+                category = "No Update Required"
                 update_required = False
 
             # Collect condition descriptions for the update plan table
@@ -199,6 +252,16 @@ class DataWrapper:
                 "update_category": category,
                 "datawrapper class": self.dataclass.__name__
             })
+            priority_map = {
+                "overwrite": 1,
+                "history_days": 2,
+                "missing_files": 3
+            }
+
+            for row in rows:
+                category = row.get("update_category")
+                # Default to None if no category assigned (no update required)
+                row["update_priority"] = priority_map.get(category, 0)
 
         update_plan_table = pd.DataFrame(rows)
         return update_plan_table

sibi_dst-0.3.21/sibi_dst/df_helper/backends/django/_io_dask_alt.py

@@ -1,189 +0,0 @@
-import itertools
-
-import dask.dataframe as dd
-import django
-import pandas as pd
-from django.core.cache import cache
-from django.core.exceptions import FieldDoesNotExist
-from django.db import models
-from django.db.models import Field
-from django.utils.encoding import force_str as force_text
-
-
-class ReadFrameDask:
-    FieldDoesNotExist = (
-        django.core.exceptions.FieldDoesNotExist
-        if django.VERSION < (1, 8)
-        else django.core.exceptions.FieldDoesNotExist
-    )
-
-    def __init__(
-            self,
-            qs,
-            **kwargs,
-    ):
-        self.qs = qs
-        self.coerce_float = kwargs.setdefault("coerce_float", False)
-        self.chunk_size = kwargs.setdefault("chunk_size", 1000)
-        self.verbose = kwargs.setdefault("verbose", True)
-
-    @staticmethod
-    def get_model_name(model):
-        return model._meta.model_name
-
-    @staticmethod
-    def get_related_model(field):
-        model = None
-        if hasattr(field, "related_model") and field.related_model:
-            model = field.related_model
-        elif hasattr(field, "rel") and field.rel:
-            model = field.rel.to
-        return model
-
-    @classmethod
-    def get_base_cache_key(cls, model):
-        return (
-            f"dask_{model._meta.app_label}_{cls.get_model_name(model)}_%s_rendering"
-        )
-
-    @classmethod
-    def replace_pk(cls, model):
-        base_cache_key = cls.get_base_cache_key(model)
-
-        def get_cache_key_from_pk(pk):
-            return None if pk is None else base_cache_key % str(pk)
-
-        def inner(pk_series):
-            pk_series = pk_series.astype(object).where(pk_series.notnull(), None)
-            cache_keys = pk_series.apply(get_cache_key_from_pk, convert_dtype=False)
-            unique_cache_keys = list(filter(None, cache_keys.unique()))
-            if not unique_cache_keys:
-                return pk_series
-
-            out_dict = cache.get_many(unique_cache_keys)
-            if len(out_dict) < len(unique_cache_keys):
-                out_dict = dict(
-                    [
-                        (base_cache_key % obj.pk, force_text(obj))
-                        for obj in model.objects.filter(
-                        pk__in=list(filter(None, pk_series.unique()))
-                    )
-                    ]
-                )
-                cache.set_many(out_dict)
-            return list(map(out_dict.get, cache_keys))
-
-        return inner
-
-    @staticmethod
-    def replace_from_choices(choices):
-        def inner(values):
-            return [choices.get(v, v) for v in values]
-
-        return inner
-
-    @classmethod
-    def build_update_functions(cls, fieldnames, fields):
-        for fieldname, field in zip(fieldnames, fields):
-            if not isinstance(field, Field):
-                yield fieldname, None
-            else:
-                if field.choices:
-                    choices = dict([(k, force_text(v)) for k, v in field.flatchoices])
-                    yield fieldname, cls.replace_from_choices(choices)
-                elif field.get_internal_type() == "ForeignKey":
-                    yield fieldname, cls.replace_pk(cls.get_related_model(field))
-
-    @classmethod
-    def update_with_verbose(cls, df, fieldnames, fields):
-        for fieldname, function in cls.build_update_functions(fieldnames, fields):
-            if function is not None:
-                df[fieldname] = df[fieldname].map_partitions(lambda x: function(x))
-
-    @staticmethod
-    def infer_dtypes_from_django(qs):
-        """Infers Dask data types based on Django queryset model fields, with support for nullable integers."""
-        django_to_dask_dtype = {
-            'AutoField': 'Int64',  # Use nullable integer
-            'BigAutoField': 'Int64',
-            'BigIntegerField': 'Int64',
-            'BooleanField': 'bool',
-            'CharField': 'object',
-            'DateField': 'datetime64[ns]',
-            'DateTimeField': 'datetime64[ns]',
-            'DecimalField': 'float64',
-            'FloatField': 'float64',
-            'IntegerField': 'Int64',  # Use nullable integer
-            'PositiveIntegerField': 'Int64',
-            'SmallIntegerField': 'Int64',
-            'TextField': 'object',
-            'TimeField': 'object',
-            'UUIDField': 'object',
-            'ForeignKey': 'Int64',  # Use nullable integer for FK fields
-        }
-
-        dtypes = {}
-        # Handle model fields
-        for field in qs.model._meta.get_fields():
-            # Skip reverse relationships and non-concrete fields
-            if not getattr(field, 'concrete', False):
-                continue
-
-            # Check for AutoField or BigAutoField explicitly
-            if isinstance(field, (models.AutoField, models.BigAutoField)):
-                dtypes[field.name] = 'Int64'  # Nullable integer for autoincremented fields
-            else:
-                # Use field type to infer dtype
-                field_type = field.get_internal_type()
-                dtypes[field.name] = django_to_dask_dtype.get(field_type, 'object')
-
-        # Handle annotated fields
-        for annotation_name, annotation in qs.query.annotation_select.items():
-            if hasattr(annotation, 'output_field'):
-                field_type = annotation.output_field.get_internal_type()
-                dtype = django_to_dask_dtype.get(field_type, 'object')
-            else:
-                dtype = 'object'  # Default to object for untyped annotations
-            dtypes[annotation_name] = dtype
-
-        return dtypes
-
-    def read_frame(self, fillna_value=None):
-        qs = self.qs
-        fieldnames = tuple(qs.model._meta.get_fields())
-        dtypes = self.infer_dtypes_from_django(qs)
-        chunk_size = self.chunk_size
-        verbose = self.verbose
-
-        # Use values to directly fetch required fields
-        qs = qs.values(*fieldnames)
-
-        # Create partitions for Dask
-        partitions = []
-        iterator = qs.iterator(chunk_size=chunk_size)
-        for chunk in itertools.islice(iterator, chunk_size):
-            df = pd.DataFrame.from_records(chunk, columns=fieldnames)
-
-            # Handle NaN values
-            if fillna_value:
-                df = df.fillna(fillna_value)
-
-            # Optimize timezone conversions
-            for col in df.columns:
-                if isinstance(df[col].dtype, pd.DatetimeTZDtype):
-                    df[col] = df[col].dt.tz_localize(None)
-
-            # Optimize dtype conversion
-            df = df.convert_dtypes()
-
-            # Convert to Dask DataFrame
-            partitions.append(dd.from_pandas(df, npartitions=1))
-
-        # Combine all partitions
-        dask_df = dd.concat(partitions, axis=0, ignore_index=True)
-
-        # Apply verbose updates
-        if verbose:
-            self.update_with_verbose(dask_df, fieldnames, qs.model._meta.fields)
-
-        return dask_df

sibi_dst-0.3.21/sibi_dst/df_helper/backends/sql_model/__init__.py

@@ -1,9 +0,0 @@
-from __future__ import annotations
-
-from ._sqlmodel_db_connection import SQLModelConnectionConfig
-from ._sqlmodel_load_from_db import SQLModelLoadFromDb
-
-__all__ = [
-    "SQLModelLoadFromDb",
-    "SQLModelConnectionConfig",
-]

sibi_dst-0.3.21/sibi_dst/df_helper/backends/sql_model/_sqlmodel_db_connection.py

@@ -1,134 +0,0 @@
-import datetime
-from typing import Any, Optional, Dict, Type
-
-from pydantic import BaseModel, model_validator
-from sqlalchemy import inspect
-from sqlalchemy.exc import OperationalError
-from sqlalchemy.sql import text
-from sqlalchemy.sql.sqltypes import (
-    Integer,
-    String,
-    Float,
-    Boolean,
-    DateTime,
-    Date,
-    Time,
-    Numeric,
-)
-from sqlmodel import SQLModel, Field, create_engine
-
-
-class SQLModelConnectionConfig(BaseModel):
-    live: bool = False
-    connection_url: str
-    table: Optional[str] = None
-    model: Optional[Any] = None
-    engine: Optional[Any] = None  # Save engine to reuse it
-
-    class Config:
-        arbitrary_types_allowed = True
-
-    @model_validator(mode="after")
-    def validate_and_initialize(self):
-        """
-        Validate connection parameters, initialize the engine, and build the dynamic model if necessary.
-        """
-        # Validate `connection_url`
-        if not self.connection_url:
-            raise ValueError("`connection_url` must be provided.")
-
-        # Initialize the engine
-        self.engine = create_engine(self.connection_url)
-
-        # Validate the connection
-        self.validate_connection()
-
-        # If table is provided, set `live=False`
-        if self.table:
-            self.live = False
-
-        # If model is not provided, build dynamically
-        if not self.model:
-            if not self.table:
-                raise ValueError("`table_name` must be provided to build the model.")
-            try:
-                self.model = self.build_model()
-            except Exception as e:
-                raise ValueError(f"Failed to build model for table '{self.table}': {e}")
-        else:
-            self.live = True
-
-        return self
-
-    def validate_connection(self):
-        """
-        Test the database connection by executing a simple query.
-        """
-        try:
-            with self.engine.connect() as connection:
-                connection.execute(text("SELECT 1"))
-        except OperationalError as e:
-            raise ValueError(f"Failed to connect to the database: {e}")
-
-    def build_model(self) -> Type[SQLModel]:
-        """
-        Dynamically build a SQLModel class based on the table schema.
-        """
-        inspector = inspect(self.engine)
-
-        # Validate table existence
-        if self.table not in inspector.get_table_names():
-            raise ValueError(f"Table '{self.table}' does not exist in the database.")
-
-        columns = inspector.get_columns(self.table)
-        if not columns:
-            raise ValueError(f"No columns found for table '{self.table}'.")
-
-        type_mapping = {
-            Integer: int,
-            String: str,
-            Float: float,
-            Boolean: bool,
-            DateTime: datetime.datetime,
-            Date: datetime.date,
-            Time: datetime.time,
-            Numeric: float,
-        }
-
-        annotations: Dict[str, Type] = {}
-        model_fields = {}
-
-        for column in columns:
-            name = column["name"]
-            sa_type = column["type"]
-            nullable = column["nullable"]
-            default = column.get("default", None)
-            primary_key = column.get("primary_key", False)
-
-            py_type = None
-            for sa_base_type, py_base_type in type_mapping.items():
-                if isinstance(sa_type, sa_base_type):
-                    py_type = py_base_type
-                    break
-
-            if py_type is None:
-                raise ValueError(f"Unsupported SQLAlchemy type for column '{name}': {sa_type}")
-
-            # Define field type and attributes
-            annotations[name] = py_type
-            model_fields[name] = Field(
-                default=default,
-                nullable=nullable,
-                primary_key=primary_key,
-                sa_column_args={"type_": sa_type},
-            )
-
-        model_fields["__annotations__"] = annotations
-        model_fields["__table__"] = self.table
-        model_name = self._table2model(self.table)
-        return type(model_name, (SQLModel,), model_fields)
-
-    @staticmethod
-    def _table2model(table_name: str) -> str:
-        """Convert table name to PascalCase model name."""
-        return "".join(word.capitalize() for word in table_name.split("_"))

sibi_dst-0.3.21/sibi_dst/df_helper/backends/sql_model/_sqlmodel_load_from_db.py

@@ -1,101 +0,0 @@
-import logging
-from typing import Any, Dict, Optional
-
-import dask.dataframe as dd
-import pandas as pd
-from sqlmodel import Session, select, text
-
-
-class SQLModelLoadFromDb:
-    df: dd.DataFrame
-
-    def __init__(
-            self,
-            db_connection,
-            db_query: Optional[Dict[str, Any]] = None,
-            db_params: Optional[Dict[str, Any]] = None,
-            logger=None,
-            **kwargs,
-    ):
-        """
-        Initialize the loader with database connection, query, and parameters.
-        """
-        self.db_connection = db_connection
-        self.table_name = self.db_connection.table
-        self.model = self.db_connection.model
-        self.engine = self.db_connection.engine
-        self.logger = logger or self._default_logger()
-        self.query_config = db_query or {}
-        self.params_config = db_params or {}
-        self.debug = kwargs.pop("debug", False)
-
-    def _default_logger(self):
-        """Create a default logger."""
-        logging.basicConfig(level=logging.INFO)
-        return logging.getLogger("SQLModelLoadFromDb")
-
-    def build_and_load(self) -> dd.DataFrame:
-        """
-        Load data into a Dask DataFrame based on the query and parameters.
-        """
-        self.df = self._build_and_load()
-        if not self.df.empty:
-            self._process_loaded_data()
-        return self.df
-
-    def _build_and_load(self) -> dd.DataFrame:
-        """
-        Query the database and load results into a Dask DataFrame.
-        """
-        print(self.model.__name__)
-        with Session(self.engine) as session:
-            try:
-                query = select(text(self.model.__table__))
-                print("query:", query)
-
-                # Apply filters if provided
-                filters = self.params_config.df_params.get("filters")
-                if filters:
-                    # Apply ORM filters (simple equality conditions)
-                    for column_name, value in filters.items():
-                        column = getattr(self.model, column_name, None)
-                        if column is not None:
-                            query = query.filter(column == value)
-                        else:
-                            self.logger.warning(f"Filter column '{column_name}' not found in model.")
-
-                # Apply limit if provided in query_config
-                n_records = self.query_config.n_records
-                if n_records:
-                    query = query.limit(n_records)
-
-                # Debug: Log the SQL query
-                self.logger.debug(f"Executing query: {str(query)}")
-
-                # Execute the query
-                results = session.exec(query).fetchall()
-
-                # Convert query results to a Dask DataFrame
-                print("results:", results)
-                if results:
-                    df = dd.from_pandas(pd.DataFrame([r.dict() for r in results]), npartitions=1)
-                else:
-                    self.logger.debug("Query returned no results.")
-                    df = dd.from_pandas(pd.DataFrame(), npartitions=1)
-
-            except Exception as e:
-                print(e)
-                self.logger.error(f"Error loading data: {e}")
-                df = dd.from_pandas(pd.DataFrame(), npartitions=1)
-
-        return df
-
-    def _process_loaded_data(self):
-        """
-        Process and clean the loaded data.
-        """
-        field_map = self.params_config.get("field_map", {})
-        if field_map:
-            rename_mapping = {k: v for k, v in field_map.items() if k in self.df.columns}
-            if rename_mapping:
-                self.df = self.df.rename(columns=rename_mapping, meta={v: "object" for v in rename_mapping.values()})

{sibi_dst-0.3.21 → sibi_dst-0.3.23}/sibi_dst/df_helper/backends/django/__init__.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
+from ._io_dask import ReadFrameDask
 from ._django_db_connection import DjangoConnectionConfig
 from ._django_load_from_db import DjangoLoadFromDb
-from ._io_dask import ReadFrameDask
 
 __all__ = [
     "DjangoConnectionConfig",

{sibi_dst-0.3.21 → sibi_dst-0.3.23}/sibi_dst/utils/__init__.py

@@ -1,22 +1,22 @@
 from __future__ import annotations
 
-from ._airflow_manager import AirflowDAGManager
-from ._clickhouse_writer import ClickHouseWriter
-from ._credentials import *
-from ._data_utils import DataUtils
-from ._data_wrapper import DataWrapper
+from ._log_utils import Logger
 from ._date_utils import *
-from ._df_utils import DfUtils
+from ._data_utils import DataUtils
 from ._file_utils import FileUtils
 from ._filepath_generator import FilePathGenerator
-from ._log_utils import Logger
-from ._parquet_saver import ParquetSaver
+from ._df_utils import DfUtils
 from ._storage_manager import StorageManager
+from ._parquet_saver import ParquetSaver
+from ._clickhouse_writer import ClickHouseWriter
+from ._airflow_manager import AirflowDAGManager
+from ._credentials import *
+from ._data_wrapper import DataWrapper
 
 __all__ = [
+    "Logger",
     "ConfigManager",
     "ConfigLoader",
-    "Logger",
     "DateUtils",
     "BusinessDays",
     "FileUtils",