sibi-dst 0.3.22__tar.gz → 0.3.24__tar.gz

{sibi_dst-0.3.22 → sibi_dst-0.3.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.22
+Version: 0.3.24
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -25,6 +25,7 @@ Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: paramiko (>=3.5.0,<4.0.0)
 Requires-Dist: psutil (>=6.1.0,<7.0.0)
+Requires-Dist: psycopg2 (>=2.9.10,<3.0.0)
 Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
 Requires-Dist: pydantic (>=2.9.2,<3.0.0)
 Requires-Dist: pymysql (>=1.1.1,<2.0.0)
@@ -33,7 +34,7 @@ Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0)
 Requires-Dist: tornado (>=6.4.1,<7.0.0)
 Requires-Dist: tqdm (>=4.67.0,<5.0.0)
-Requires-Dist: uvicorn (>=0.32.1,<0.33.0)
+Requires-Dist: uvicorn (>=0.34.0,<0.35.0)
 Description-Content-Type: text/markdown
 
 # sibi-dst

{sibi_dst-0.3.22 → sibi_dst-0.3.24}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.22"
+version = "0.3.24"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"
@@ -30,10 +30,11 @@ clickhouse-driver = "^0.2.9"
 paramiko = "^3.5.0"
 chardet = "^5.2.0"
 charset-normalizer = "^3.4.0"
-uvicorn = "^0.32.1"
 sqlalchemy = "^2.0.36"
 djangorestframework = "^3.15.2"
 dask-expr = "^1.1.20"
+psycopg2 = "^2.9.10"
+uvicorn = "^0.34.0"
 
 
 [build-system]

sibi_dst-0.3.22/sibi_dst/df_helper/backends/django/_io_dask_alt.py

@@ -1,189 +0,0 @@
-import itertools
-
-import dask.dataframe as dd
-import django
-import pandas as pd
-from django.core.cache import cache
-from django.core.exceptions import FieldDoesNotExist
-from django.db import models
-from django.db.models import Field
-from django.utils.encoding import force_str as force_text
-
-
-class ReadFrameDask:
-    FieldDoesNotExist = (
-        django.core.exceptions.FieldDoesNotExist
-        if django.VERSION < (1, 8)
-        else django.core.exceptions.FieldDoesNotExist
-    )
-
-    def __init__(
-            self,
-            qs,
-            **kwargs,
-    ):
-        self.qs = qs
-        self.coerce_float = kwargs.setdefault("coerce_float", False)
-        self.chunk_size = kwargs.setdefault("chunk_size", 1000)
-        self.verbose = kwargs.setdefault("verbose", True)
-
-    @staticmethod
-    def get_model_name(model):
-        return model._meta.model_name
-
-    @staticmethod
-    def get_related_model(field):
-        model = None
-        if hasattr(field, "related_model") and field.related_model:
-            model = field.related_model
-        elif hasattr(field, "rel") and field.rel:
-            model = field.rel.to
-        return model
-
-    @classmethod
-    def get_base_cache_key(cls, model):
-        return (
-            f"dask_{model._meta.app_label}_{cls.get_model_name(model)}_%s_rendering"
-        )
-
-    @classmethod
-    def replace_pk(cls, model):
-        base_cache_key = cls.get_base_cache_key(model)
-
-        def get_cache_key_from_pk(pk):
-            return None if pk is None else base_cache_key % str(pk)
-
-        def inner(pk_series):
-            pk_series = pk_series.astype(object).where(pk_series.notnull(), None)
-            cache_keys = pk_series.apply(get_cache_key_from_pk, convert_dtype=False)
-            unique_cache_keys = list(filter(None, cache_keys.unique()))
-            if not unique_cache_keys:
-                return pk_series
-
-            out_dict = cache.get_many(unique_cache_keys)
-            if len(out_dict) < len(unique_cache_keys):
-                out_dict = dict(
-                    [
-                        (base_cache_key % obj.pk, force_text(obj))
-                        for obj in model.objects.filter(
-                        pk__in=list(filter(None, pk_series.unique()))
-                    )
-                    ]
-                )
-                cache.set_many(out_dict)
-            return list(map(out_dict.get, cache_keys))
-
-        return inner
-
-    @staticmethod
-    def replace_from_choices(choices):
-        def inner(values):
-            return [choices.get(v, v) for v in values]
-
-        return inner
-
-    @classmethod
-    def build_update_functions(cls, fieldnames, fields):
-        for fieldname, field in zip(fieldnames, fields):
-            if not isinstance(field, Field):
-                yield fieldname, None
-            else:
-                if field.choices:
-                    choices = dict([(k, force_text(v)) for k, v in field.flatchoices])
-                    yield fieldname, cls.replace_from_choices(choices)
-                elif field.get_internal_type() == "ForeignKey":
-                    yield fieldname, cls.replace_pk(cls.get_related_model(field))
-
-    @classmethod
-    def update_with_verbose(cls, df, fieldnames, fields):
-        for fieldname, function in cls.build_update_functions(fieldnames, fields):
-            if function is not None:
-                df[fieldname] = df[fieldname].map_partitions(lambda x: function(x))
-
-    @staticmethod
-    def infer_dtypes_from_django(qs):
-        """Infers Dask data types based on Django queryset model fields, with support for nullable integers."""
-        django_to_dask_dtype = {
-            'AutoField': 'Int64',  # Use nullable integer
-            'BigAutoField': 'Int64',
-            'BigIntegerField': 'Int64',
-            'BooleanField': 'bool',
-            'CharField': 'object',
-            'DateField': 'datetime64[ns]',
-            'DateTimeField': 'datetime64[ns]',
-            'DecimalField': 'float64',
-            'FloatField': 'float64',
-            'IntegerField': 'Int64',  # Use nullable integer
-            'PositiveIntegerField': 'Int64',
-            'SmallIntegerField': 'Int64',
-            'TextField': 'object',
-            'TimeField': 'object',
-            'UUIDField': 'object',
-            'ForeignKey': 'Int64',  # Use nullable integer for FK fields
-        }
-
-        dtypes = {}
-        # Handle model fields
-        for field in qs.model._meta.get_fields():
-            # Skip reverse relationships and non-concrete fields
-            if not getattr(field, 'concrete', False):
-                continue
-
-            # Check for AutoField or BigAutoField explicitly
-            if isinstance(field, (models.AutoField, models.BigAutoField)):
-                dtypes[field.name] = 'Int64'  # Nullable integer for autoincremented fields
-            else:
-                # Use field type to infer dtype
-                field_type = field.get_internal_type()
-                dtypes[field.name] = django_to_dask_dtype.get(field_type, 'object')
-
-        # Handle annotated fields
-        for annotation_name, annotation in qs.query.annotation_select.items():
-            if hasattr(annotation, 'output_field'):
-                field_type = annotation.output_field.get_internal_type()
-                dtype = django_to_dask_dtype.get(field_type, 'object')
-            else:
-                dtype = 'object'  # Default to object for untyped annotations
-            dtypes[annotation_name] = dtype
-
-        return dtypes
-
-    def read_frame(self, fillna_value=None):
-        qs = self.qs
-        fieldnames = tuple(qs.model._meta.get_fields())
-        dtypes = self.infer_dtypes_from_django(qs)
-        chunk_size = self.chunk_size
-        verbose = self.verbose
-
-        # Use values to directly fetch required fields
-        qs = qs.values(*fieldnames)
-
-        # Create partitions for Dask
-        partitions = []
-        iterator = qs.iterator(chunk_size=chunk_size)
-        for chunk in itertools.islice(iterator, chunk_size):
-            df = pd.DataFrame.from_records(chunk, columns=fieldnames)
-
-            # Handle NaN values
-            if fillna_value:
-                df = df.fillna(fillna_value)
-
-            # Optimize timezone conversions
-            for col in df.columns:
-                if isinstance(df[col].dtype, pd.DatetimeTZDtype):
-                    df[col] = df[col].dt.tz_localize(None)
-
-            # Optimize dtype conversion
-            df = df.convert_dtypes()
-
-            # Convert to Dask DataFrame
-            partitions.append(dd.from_pandas(df, npartitions=1))
-
-        # Combine all partitions
-        dask_df = dd.concat(partitions, axis=0, ignore_index=True)
-
-        # Apply verbose updates
-        if verbose:
-            self.update_with_verbose(dask_df, fieldnames, qs.model._meta.fields)
-
-        return dask_df

sibi_dst-0.3.22/sibi_dst/df_helper/backends/sql_model/__init__.py

@@ -1,9 +0,0 @@
-from __future__ import annotations
-
-from ._sqlmodel_db_connection import SQLModelConnectionConfig
-from ._sqlmodel_load_from_db import SQLModelLoadFromDb
-
-__all__ = [
-    "SQLModelLoadFromDb",
-    "SQLModelConnectionConfig",
-]

sibi_dst-0.3.22/sibi_dst/df_helper/backends/sql_model/_sqlmodel_db_connection.py

@@ -1,134 +0,0 @@
-import datetime
-from typing import Any, Optional, Dict, Type
-
-from pydantic import BaseModel, model_validator
-from sqlalchemy import inspect
-from sqlalchemy.exc import OperationalError
-from sqlalchemy.sql import text
-from sqlalchemy.sql.sqltypes import (
-    Integer,
-    String,
-    Float,
-    Boolean,
-    DateTime,
-    Date,
-    Time,
-    Numeric,
-)
-from sqlmodel import SQLModel, Field, create_engine
-
-
-class SQLModelConnectionConfig(BaseModel):
-    live: bool = False
-    connection_url: str
-    table: Optional[str] = None
-    model: Optional[Any] = None
-    engine: Optional[Any] = None  # Save engine to reuse it
-
-    class Config:
-        arbitrary_types_allowed = True
-
-    @model_validator(mode="after")
-    def validate_and_initialize(self):
-        """
-        Validate connection parameters, initialize the engine, and build the dynamic model if necessary.
-        """
-        # Validate `connection_url`
-        if not self.connection_url:
-            raise ValueError("`connection_url` must be provided.")
-
-        # Initialize the engine
-        self.engine = create_engine(self.connection_url)
-
-        # Validate the connection
-        self.validate_connection()
-
-        # If table is provided, set `live=False`
-        if self.table:
-            self.live = False
-
-        # If model is not provided, build dynamically
-        if not self.model:
-            if not self.table:
-                raise ValueError("`table_name` must be provided to build the model.")
-            try:
-                self.model = self.build_model()
-            except Exception as e:
-                raise ValueError(f"Failed to build model for table '{self.table}': {e}")
-        else:
-            self.live = True
-
-        return self
-
-    def validate_connection(self):
-        """
-        Test the database connection by executing a simple query.
-        """
-        try:
-            with self.engine.connect() as connection:
-                connection.execute(text("SELECT 1"))
-        except OperationalError as e:
-            raise ValueError(f"Failed to connect to the database: {e}")
-
-    def build_model(self) -> Type[SQLModel]:
-        """
-        Dynamically build a SQLModel class based on the table schema.
-        """
-        inspector = inspect(self.engine)
-
-        # Validate table existence
-        if self.table not in inspector.get_table_names():
-            raise ValueError(f"Table '{self.table}' does not exist in the database.")
-
-        columns = inspector.get_columns(self.table)
-        if not columns:
-            raise ValueError(f"No columns found for table '{self.table}'.")
-
-        type_mapping = {
-            Integer: int,
-            String: str,
-            Float: float,
-            Boolean: bool,
-            DateTime: datetime.datetime,
-            Date: datetime.date,
-            Time: datetime.time,
-            Numeric: float,
-        }
-
-        annotations: Dict[str, Type] = {}
-        model_fields = {}
-
-        for column in columns:
-            name = column["name"]
-            sa_type = column["type"]
-            nullable = column["nullable"]
-            default = column.get("default", None)
-            primary_key = column.get("primary_key", False)
-
-            py_type = None
-            for sa_base_type, py_base_type in type_mapping.items():
-                if isinstance(sa_type, sa_base_type):
-                    py_type = py_base_type
-                    break
-
-            if py_type is None:
-                raise ValueError(f"Unsupported SQLAlchemy type for column '{name}': {sa_type}")
-
-            # Define field type and attributes
-            annotations[name] = py_type
-            model_fields[name] = Field(
-                default=default,
-                nullable=nullable,
-                primary_key=primary_key,
-                sa_column_args={"type_": sa_type},
-            )
-
-        model_fields["__annotations__"] = annotations
-        model_fields["__table__"] = self.table
-        model_name = self._table2model(self.table)
-        return type(model_name, (SQLModel,), model_fields)
-
-    @staticmethod
-    def _table2model(table_name: str) -> str:
-        """Convert table name to PascalCase model name."""
-        return "".join(word.capitalize() for word in table_name.split("_"))

sibi_dst-0.3.22/sibi_dst/df_helper/backends/sql_model/_sqlmodel_load_from_db.py

@@ -1,101 +0,0 @@
-import logging
-from typing import Any, Dict, Optional
-
-import dask.dataframe as dd
-import pandas as pd
-from sqlmodel import Session, select, text
-
-
-class SQLModelLoadFromDb:
-    df: dd.DataFrame
-
-    def __init__(
-            self,
-            db_connection,
-            db_query: Optional[Dict[str, Any]] = None,
-            db_params: Optional[Dict[str, Any]] = None,
-            logger=None,
-            **kwargs,
-    ):
-        """
-        Initialize the loader with database connection, query, and parameters.
-        """
-        self.db_connection = db_connection
-        self.table_name = self.db_connection.table
-        self.model = self.db_connection.model
-        self.engine = self.db_connection.engine
-        self.logger = logger or self._default_logger()
-        self.query_config = db_query or {}
-        self.params_config = db_params or {}
-        self.debug = kwargs.pop("debug", False)
-
-    def _default_logger(self):
-        """Create a default logger."""
-        logging.basicConfig(level=logging.INFO)
-        return logging.getLogger("SQLModelLoadFromDb")
-
-    def build_and_load(self) -> dd.DataFrame:
-        """
-        Load data into a Dask DataFrame based on the query and parameters.
-        """
-        self.df = self._build_and_load()
-        if not self.df.empty:
-            self._process_loaded_data()
-        return self.df
-
-    def _build_and_load(self) -> dd.DataFrame:
-        """
-        Query the database and load results into a Dask DataFrame.
-        """
-        print(self.model.__name__)
-        with Session(self.engine) as session:
-            try:
-                query = select(text(self.model.__table__))
-                print("query:", query)
-
-                # Apply filters if provided
-                filters = self.params_config.df_params.get("filters")
-                if filters:
-                    # Apply ORM filters (simple equality conditions)
-                    for column_name, value in filters.items():
-                        column = getattr(self.model, column_name, None)
-                        if column is not None:
-                            query = query.filter(column == value)
-                        else:
-                            self.logger.warning(f"Filter column '{column_name}' not found in model.")
-
-                # Apply limit if provided in query_config
-                n_records = self.query_config.n_records
-                if n_records:
-                    query = query.limit(n_records)
-
-                # Debug: Log the SQL query
-                self.logger.debug(f"Executing query: {str(query)}")
-
-                # Execute the query
-                results = session.exec(query).fetchall()
-
-                # Convert query results to a Dask DataFrame
-                print("results:", results)
-                if results:
-                    df = dd.from_pandas(pd.DataFrame([r.dict() for r in results]), npartitions=1)
-                else:
-                    self.logger.debug("Query returned no results.")
-                    df = dd.from_pandas(pd.DataFrame(), npartitions=1)
-
-            except Exception as e:
-                print(e)
-                self.logger.error(f"Error loading data: {e}")
-                df = dd.from_pandas(pd.DataFrame(), npartitions=1)
-
-        return df
-
-    def _process_loaded_data(self):
-        """
-        Process and clean the loaded data.
-        """
-        field_map = self.params_config.get("field_map", {})
-        if field_map:
-            rename_mapping = {k: v for k, v in field_map.items() if k in self.df.columns}
-            if rename_mapping:
-                self.df = self.df.rename(columns=rename_mapping, meta={v: "object" for v in rename_mapping.values()})