ecodev-core 0.0.42__tar.gz → 0.0.44__tar.gz

{ecodev_core-0.0.42 → ecodev_core-0.0.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ecodev-core
-Version: 0.0.42
+Version: 0.0.44
 Summary: Low level sqlmodel/fastapi/pydantic building blocks
 License: MIT
 Author: Thomas Epelbaum

{ecodev_core-0.0.42 → ecodev_core-0.0.44}/ecodev_core/__init__.py

@@ -40,6 +40,12 @@ from ecodev_core.db_insertion import get_raw_df
 from ecodev_core.db_retrieval import count_rows
 from ecodev_core.db_retrieval import get_rows
 from ecodev_core.db_retrieval import ServerSideField
+from ecodev_core.db_upsertion import field
+from ecodev_core.db_upsertion import sfield
+from ecodev_core.db_upsertion import upsert_data
+from ecodev_core.db_upsertion import upsert_deletor
+from ecodev_core.db_upsertion import upsert_dict_data
+from ecodev_core.db_upsertion import upsert_selector
 from ecodev_core.deployment import Deployment
 from ecodev_core.email_sender import send_email
 from ecodev_core.enum_utils import enum_converter
@@ -78,7 +84,10 @@ from ecodev_core.safe_utils import SafeTestCase
 from ecodev_core.safe_utils import SimpleReturn
 from ecodev_core.safe_utils import stringify
 from ecodev_core.settings import Settings
-
+from ecodev_core.version import db_to_value
+from ecodev_core.version import get_row_versions
+from ecodev_core.version import get_versions
+from ecodev_core.version import Version
 __all__ = [
     'AUTH', 'Token', 'get_app_services', 'attempt_to_log', 'get_current_user', 'is_admin_user',
     'write_json_file', 'load_json_file', 'make_dir', 'check_dependencies', 'compute_dependencies',
@@ -92,4 +101,6 @@ __all__ = [
     'fastapi_monitor', 'dash_monitor', 'is_monitoring_user', 'get_recent_activities', 'select_user',
     'get_access_token', 'safe_get_user', 'backup', 'group_by', 'get_excelfile', 'upsert_new_user',
     'datify', 'safe_drop_columns', 'get_value', 'is_null', 'send_email', 'first_func_or_default',
-    'sort_by_keys', 'sort_by_values', 'Settings', 'load_yaml_file', 'Deployment']
+    'sort_by_keys', 'sort_by_values', 'Settings', 'load_yaml_file', 'Deployment', 'Version',
+    'sfield', 'field', 'upsert_data', 'upsert_deletor', 'get_row_versions', 'get_versions',
+    'db_to_value', 'upsert_dict_data', 'upsert_selector']

ecodev_core-0.0.44/ecodev_core/db_upsertion.py

@@ -0,0 +1,108 @@
+"""
+Module handling CRUD and version operations
+"""
+from functools import partial
+from typing import Union
+
+import pandas as pd
+import progressbar
+from sqlmodel import and_
+from sqlmodel import Field
+from sqlmodel import inspect
+from sqlmodel import select
+from sqlmodel import Session
+from sqlmodel import SQLModel
+from sqlmodel import update
+from sqlmodel.main import SQLModelMetaclass
+from sqlmodel.sql.expression import SelectOfScalar
+
+from ecodev_core.version import get_row_versions
+from ecodev_core.version import Version
+
+BATCH_SIZE = 5000
+FILTER_ON = 'filter_on'
+INFO = 'info'
+
+
+def sfield(**kwargs):
+    """
+    Field constructor for columns not to be versioned. Those are the columns on which to select.
+    They morally are a sort of unique identifier of a row (like id but more business meaningful)
+    """
+    return Field(**kwargs, sa_column_kwargs={INFO: {FILTER_ON: True}})
+
+
+def field(**kwargs):
+    """
+    Field constructor for columns to be versioned.
+    """
+    return Field(**kwargs, sa_column_kwargs={INFO: {FILTER_ON: False}})
+
+
+def upsert_selector(values: SQLModel, db_schema: SQLModelMetaclass) -> SelectOfScalar:
+    """
+    Return the query allowing to select on column not to be versioned values.
+    """
+    conditions = [getattr(db_schema, x.name) == getattr(values, x.name)
+                  for x in inspect(db_schema).c if x.info.get(FILTER_ON) is True]
+    return select(db_schema).where(and_(*conditions))
+
+
+def upsert_updator(values: SQLModel,
+                   row_id: int,
+                   session: Session,
+                   db_schema: SQLModelMetaclass
+                   ) -> None:
+    """
+    Update the passed row_id from db_schema db with passed new_values.
+     Only update columns to be versioned.
+
+    At the same time, store previous (column, row_id) versions for all columns that changed values.
+    """
+    to_update = {col.name: getattr(values, col.name)
+                 for col in inspect(db_schema).c if col.info.get(FILTER_ON) is False}
+    db = session.exec(select(db_schema).where(db_schema.id == row_id)).first().model_dump()
+    col_types = {x: y.annotation for x, y in db_schema.__fields__.items()}
+    table = db_schema.__tablename__
+
+    for col, val in {k: v for k, v in db.items() if k in to_update and v != to_update[k]}.items():
+        session.add(Version.from_table_row(table, col, row_id, col_types[col], val))
+
+    return update(db_schema).where(db_schema.id == row_id).values(**to_update)
+
+
+def upsert_deletor(values: SQLModel, session: Session):
+    """
+    Delete row in db corresponding to the passed values, selecting on columns not to be versioned.
+    """
+    db_schema = values.__class__
+    if in_db := session.exec(upsert_selector(values, db_schema=db_schema)).first():
+        for version in get_row_versions(db_schema.__tablename__, in_db.id, session):
+            session.delete(version)
+        session.delete(in_db)
+        session.commit()
+
+
+def upsert_data(df: Union[pd.DataFrame], db_schema: SQLModelMetaclass, session: Session) -> None:
+    """
+    Upsert the passed df into db_schema db.
+    """
+    upsert_dict_data([x.to_dict() for _, x in df.iterrows()], db_schema, session)
+
+
+def upsert_dict_data(data: list[dict], db_schema: SQLModelMetaclass, session: Session) -> None:
+    """
+    Upsert the passed list of dicts (corresponding to db_schema) into db_schema db.
+    """
+    selector = partial(upsert_selector, db_schema=db_schema)
+    updator = partial(upsert_updator, db_schema=db_schema)
+    batches = [data[i:i + BATCH_SIZE] for i in range(0, len(data), BATCH_SIZE)]
+
+    for batch in progressbar.progressbar(batches, redirect_stdout=False):
+        for row in batch:
+            new_object = db_schema(**row)
+            if in_db := session.exec(selector(new_object)).first():
+                session.exec(updator(new_object, in_db.id, session))
+            else:
+                session.add(new_object)
+        session.commit()

ecodev_core-0.0.44/ecodev_core/version.py

@@ -0,0 +1,144 @@
+"""
+Module implementing the Version table
+"""
+from datetime import datetime
+from enum import Enum
+from enum import EnumType
+from enum import unique
+from typing import Optional
+
+from sqlmodel import desc
+from sqlmodel import Field
+from sqlmodel import Index
+from sqlmodel import select
+from sqlmodel import Session
+from sqlmodel import SQLModel
+
+
+@unique
+class ColType(str, Enum):
+    """
+    Enum listing all col types allowed to be stored in version
+    """
+    STR = 'str'
+    BOOL = 'bool'
+    INT = 'int'
+    FLOAT = 'float'
+    DATE = 'datetime'
+    ENUM = 'enum'
+
+
+COL_TYPES = str | int | bool | float | datetime | Enum | None
+
+
+class Version(SQLModel, table=True):  # type: ignore
+    """
+    Table handling versioning.
+
+    Attributes are:
+        - table: the table to version
+        - column: the column of table to version
+        - row_id: the the row id of the column of table to version
+        - col_type: the column type
+        - value: the value to store (previous row/column/table version)
+    """
+    __tablename__ = 'version'
+    id: Optional[int] = Field(default=None, primary_key=True)
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+    table: str = Field(index=True)
+    column: str = Field(index=True)
+    row_id: int = Field(index=True)
+    col_type: ColType
+    value: str | None = Field(index=True)
+
+    __table_args__ = (
+        Index(
+            'version_filter_id',
+            'table',
+            'row_id',
+            'column'
+        ),
+    )
+
+    @classmethod
+    def from_table_row(cls, table: str,
+                       column: str,
+                       row_id: int,
+                       raw_type: type | EnumType,
+                       raw_val: COL_TYPES
+                       ) -> 'Version':
+        """
+        Create a new Version out of the passed information
+        """
+        col_type = _col_type_to_db(raw_type)
+        value = _value_to_db(raw_val, col_type)
+        return cls(table=table, column=column, row_id=row_id, col_type=col_type, value=value)
+
+
+def get_versions(table: str, column: str, row_id: int, session: Session) -> list[Version]:
+    """
+    Retrieve all previous versions of a (table, column, row_id) triplet
+    """
+    query = select(Version).where(Version.table == table, Version.row_id == row_id,
+                                  Version.column == column).order_by(desc(Version.created_at))
+    return session.exec(query).all()
+
+
+def get_row_versions(table: str, row_id: int, session: Session) -> list[Version]:
+    """
+    Retrieve all previous versions of a (table, row_id) couples. Hence all columns previous versions
+    """
+    query = select(Version).where(Version.table == table, Version.row_id == row_id
+                                  ).order_by(desc(Version.created_at))
+    return session.exec(query).all()
+
+
+def _col_type_to_db(col_type: type | EnumType) -> ColType:
+    """
+    Forge ColType out of passed col_type
+    """
+    if col_type == int:
+        return ColType.INT
+    if col_type == bool:
+        return ColType.BOOL
+    if col_type == float:
+        return ColType.FLOAT
+    if col_type == str:
+        return ColType.STR
+    if col_type == datetime:
+        return ColType.DATE
+    return ColType.ENUM
+
+
+def _value_to_db(value: COL_TYPES, col_type: ColType) -> str | None:
+    """
+    Convert a value to version to it's str version
+    """
+    if value is None:
+        return None
+
+    match col_type:
+        case ColType.BOOL | ColType.STR | ColType.INT | ColType.FLOAT:
+            return str(value)
+        case ColType.DATE:
+            return value.strftime('%Y-%m-%d %H:%M:%S.%f')  # type: ignore[union-attr]
+        case ColType.ENUM:
+            return value.name  # type: ignore[union-attr]
+        case _:
+            return None
+
+
+def db_to_value(db_value: str | None, col_type: type | EnumType) -> COL_TYPES:
+    """
+    Convert back a str version value stored to a real value (types hancled listed in ColType)
+    NB: assumption that if the type is not known, this is an enum type.
+    """
+    if db_value is None:
+        return None
+    if col_type in [int, str, float]:
+        return col_type(db_value)
+    if col_type == bool:
+        return True if db_value == 'True' else False
+    if col_type == datetime:
+        return datetime.strptime(db_value, '%Y-%m-%d %H:%M:%S.%f')
+    return col_type[db_value]   # type: ignore[index]

{ecodev_core-0.0.42 → ecodev_core-0.0.44}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ecodev-core"
-version = "0.0.42"
+version = "0.0.44"
 description = "Low level sqlmodel/fastapi/pydantic building blocks"
 authors = ["Thomas Epelbaum <tomepel@gmail.com>",
 					 "Olivier Gabriel <olivier.gabriel.geom@gmail.com>",