ecodev-core 0.0.67__py3-none-any.whl

ecodev_core/db_connection.py

@@ -0,0 +1,94 @@
+"""
+Module implementing postgresql connection
+"""
+from typing import Callable
+from typing import List
+from typing import Optional
+from urllib.parse import quote
+
+from pydantic_settings import BaseSettings
+from pydantic_settings import SettingsConfigDict
+from sqlalchemy import delete
+from sqlalchemy import text
+from sqlmodel import create_engine
+from sqlmodel import Session
+from sqlmodel import SQLModel
+
+from ecodev_core.logger import logger_get
+from ecodev_core.settings import SETTINGS
+
+log = logger_get(__name__)
+
+
+class DbSettings(BaseSettings):
+    """
+    Settings class used to connect to the postgresql database
+    """
+    db_host: str = ''
+    db_port: str = ''
+    db_name: str = ''
+    db_username: str = ''
+    db_password: str = ''
+    db_test_name: str = ''
+    model_config = SettingsConfigDict(env_file='.env')
+
+
+DB, SETTINGS_DB = DbSettings(), SETTINGS.database  # type: ignore[attr-defined]
+_PASSWORD = quote(SETTINGS_DB.db_password or DB.db_password, safe='')
+_USER, _HOST = SETTINGS_DB.db_username or DB.db_username, SETTINGS_DB.db_host or DB.db_host
+_PORT, _NAME = SETTINGS_DB.db_port or DB.db_port, SETTINGS_DB.db_name or DB.db_name
+DB_URL = f'postgresql://{_USER}:{_PASSWORD}@{_HOST}:{_PORT}/{_NAME}'
+TEST_DB = SETTINGS_DB.db_test_name or DB.db_test_name
+TEST_DB_URL = f'postgresql://{_USER}:{_PASSWORD}@{_HOST}:{_PORT}/{TEST_DB}'
+_ADMIN_DB_URL = f'postgresql://{_USER}:{_PASSWORD}@{_HOST}:{_PORT}/postgres'
+engine = create_engine(DB_URL, pool_pre_ping=True)
+
+
+def exec_admin_queries(queries: list[str]) -> None:
+    """
+    execute sequentially queries from the admin db
+    """
+    admin_engine = create_engine(_ADMIN_DB_URL, isolation_level='AUTOCOMMIT')
+    with admin_engine.connect() as conn:
+        for query in queries:
+            conn.execute(text(query))
+    admin_engine.dispose()
+
+
+def create_db_and_tables(model: Callable, excluded_tables: Optional[List[str]] = None) -> None:
+    """
+    Create all tables based on the declared schemas in core/models thanks to sqlmodel
+    Does not create the tables which are passed in the optional list of excluded tables,
+    must be the table names
+    """
+    log.info(f'Inserting on the fly {model} and all other domain tables')
+    SQLModel.metadata.tables = {table: meta_data for table, meta_data in
+                                SQLModel.metadata.__dict__.get('tables').items()
+                                if not excluded_tables or table
+                                not in excluded_tables}
+    SQLModel.metadata.create_all(engine)
+
+
+def delete_table(model: Callable) -> None:
+    """
+    Delete all rows of the passed model from db
+    """
+    with Session(engine) as session:
+        result = session.execute(delete(model))
+        session.commit()
+        log.info(f'Deleted {result.rowcount} rows')
+
+
+def get_session():
+    """
+    Retrieves the session, used in Depends() attributes of fastapi routes
+    """
+    with Session(engine) as session:
+        yield session
+
+
+def info_message(model: Callable):
+    """
+    Create all tables based on the declared schemas in core/models thanks to sqlmodel
+    """
+    log.info(f'hack to get all models imported in an alembic env.py. {model}')

ecodev_core/db_filters.py

@@ -0,0 +1,142 @@
+"""
+Low level db filtering methods
+"""
+from datetime import datetime
+from enum import Enum
+from enum import unique
+from typing import Callable
+from typing import Dict
+
+from sqlalchemy import func
+from sqlalchemy.orm.attributes import InstrumentedAttribute
+from sqlmodel import col
+from sqlmodel.sql.expression import Select
+from sqlmodel.sql.expression import SelectOfScalar
+
+SelectOfScalar.inherit_cache = True  # type: ignore
+Select.inherit_cache = True  # type: ignore
+OPERATORS = ['>=', '<=', '!=', '=', '<', '>', 'contains ']
+
+
+@unique
+class ServerSideFilter(str, Enum):
+    """
+    All possible server side filtering mechanisms
+    """
+    STARTSTR = 'start_str'
+    ILIKESTR = 'ilike_str'
+    STRICTSTR = 'strict_str'
+    LIKESTR = 'like_str'
+    BOOL = 'bool'
+    NUM = 'num'
+
+
+def _filter_start_str_field(field: InstrumentedAttribute,
+                            query: SelectOfScalar,
+                            operator: str,
+                            value: str
+                            ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a str like field. The filtering is done by checking if
+     the passed value starts the field.
+
+    NB: case-insensitive!
+    """
+    return query.where(func.lower(col(field)).startswith(value.lower())) if value else query
+
+
+def _filter_str_ilike_field(field: InstrumentedAttribute,
+                            query: SelectOfScalar,
+                            operator: str,
+                            value: str
+                            ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a str like field. The filtering is done by checking if
+     the passed value is contained in db values
+
+    NB: case-insensitive!
+    """
+    return query.where(col(field).ilike(f'%{value}%')) if value else query
+
+
+def _filter_str_like_field(field: InstrumentedAttribute,
+                           query: SelectOfScalar,
+                           operator: str,
+                           value: str
+                           ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a str like field. The filtering is done by checking if
+     the passed value is contained in db values
+    """
+    return query.where(col(field).contains(value)) if value else query
+
+
+def _filter_strict_str_field(field: InstrumentedAttribute,
+                             query: SelectOfScalar,
+                             operator: str,
+                             value: str
+                             ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a strict str equality matching.
+    The filtering is done by checking if the passed value is equal to one of the db values
+    """
+    return query.where(col(field) == value) if value else query
+
+
+def _filter_bool_like_field(field: InstrumentedAttribute,
+                            query: SelectOfScalar,
+                            operator: str,
+                            value: str
+                            ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a bool like field. The filtering is done by comparing
+     the passed value to db values
+    """
+    return query.where(col(field) == value) if value else query
+
+
+def _filter_num_like_field(field: InstrumentedAttribute,
+                           query: SelectOfScalar,
+                           operator: str,
+                           value: str,
+                           is_date: bool = False
+                           ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a num like (even datetime in case where is_date
+    is set to True) field. The filtering is done by comparing the passed value to db values
+    with the passed operator.
+    """
+    if not operator or not value:
+        return query
+
+    if operator == '>=':
+        query = query.where(col(field) >= (_date(value) if is_date else float(value)))
+    elif operator == '<=':
+        query = query.where(col(field) <= (_date(value) if is_date else float(value)))
+    elif operator == '!=':
+        query = query.where(col(field) != (_date(value) if is_date else float(value)))
+    elif operator == '=':
+        query = query.where(col(field) == (_date(value) if is_date else float(value)))
+    elif operator == '>':
+        query = query.where(col(field) > (_date(value) if is_date else float(value)))
+    elif operator == '<':
+        query = query.where(col(field) < (_date(value) if is_date else float(value)))
+
+    return query
+
+
+SERVER_SIDE_FILTERS: Dict[ServerSideFilter, Callable] = {
+    ServerSideFilter.STARTSTR: _filter_start_str_field,
+    ServerSideFilter.STRICTSTR: _filter_strict_str_field,
+    ServerSideFilter.LIKESTR: _filter_str_like_field,
+    ServerSideFilter.ILIKESTR: _filter_str_ilike_field,
+    ServerSideFilter.BOOL: _filter_bool_like_field,
+    ServerSideFilter.NUM: _filter_num_like_field
+}
+
+
+def _date(year: str) -> datetime:
+    """
+    Convert the passed str year to a datetime to allow filtering on datetime years.
+    """
+    return datetime(year=int(year), month=1, day=1)

ecodev_core/db_i18n.py

@@ -0,0 +1,211 @@
+"""
+Module implementing internationalization (i18n) for sqlmodel
+"""
+import contextvars
+from enum import Enum
+from typing import Optional
+
+from sqlalchemy import Label
+from sqlalchemy import label
+from sqlmodel import func
+from sqlmodel.main import SQLModelMetaclass
+
+
+class Lang(str, Enum):
+    """
+    Enum of the languages available for localization.
+    """
+    EN = 'en'
+    FR = 'fr'
+
+
+DB_Lang = 'db_Lang'
+CONTEXT_DB_Lang = contextvars.ContextVar(DB_Lang, default=Lang.EN)
+"""Context variables for storing the active database language, defaults to Lang.EN"""
+
+
+def set_lang(lang: Lang) -> None:
+    """
+    Sets the `CONTEXT_DB_Lang` context var.
+
+    Args:
+        lang (Lang): The language to assign to the `CONTEXT_DB_Lang` context var.
+    """
+    CONTEXT_DB_Lang.set(lang)
+
+
+def get_lang() -> Lang:
+    """
+    Fetches the value of `CONTEXT_DB_Lang` context var.
+
+    Returns:
+        lang (Lang): The value of `CONTEXT_DB_Lang` context var
+    """
+    return Lang(CONTEXT_DB_Lang.get())
+
+
+class I18nMixin:
+    """
+    I18n (localization) mixin for string attributes of pydantic BaseModel classes.
+
+    Maps arbitrary string attributes of the class to their localized values. Localized fields 
+    should be defined following the rules below :
+        - The field name must be defined as a key of the private attribute `__localized_fields__`
+        - Each field defined in `__localized_fields__` must be present as an attribute for \
+            each of its localized versions in the following format <field>_<lang>.
+            For example :
+            ```
+            __localized_fields__ = {
+                'name':[Lang.EN, Lang.FR]
+                }
+            ```
+            assumes that the attributes `name_en` and `name_fr` are attribute of the class.
+            These attributes must have a type `str`.
+        - All localized field must have a localized version using `__fallback_lang__`
+
+    Args:
+        __localized_fields__ (dict[str, list[Lang]]): Mapping between localized fields and a \
+            list of their available localized versions. Defaults to {}.
+        __fallback_lang__ (Lang): Fallback locale if the requested localized version of a \
+            field is None. Defaults to Lang.EN.
+    """
+    __localized_fields__: dict[str, list[Lang]] = {}
+    __fallback_lang__: Lang = Lang.EN
+
+    @classmethod
+    def _get_lang_chain(cls, field: str, lang: Optional[Lang] = None) -> list[Lang]:
+        """
+        Returns the chain of localized versions of the requested field with the priority given
+        to the `lang` argument, followed by the lang returned by
+        [get_lang][ecodev_core.db_i18n.get_lang] and finally the lang defined in
+        `cls.__fallback_lang__`.
+
+        Args:
+            field (str): Name of the attribute/field to localize
+            lang (Optional[Lang]): The requested locale language. If none, then uses that
+            returned by [get_lang][ecodev_core.db_i18n.get_lang]. Defaults to None.
+
+        Returns:
+            list[Lang]: List of Lang enums to use for generating the name of the localized \
+                fields.
+        """
+        if field not in cls.__localized_fields__:
+            raise AttributeError(f'Field {field!r} is not internationalized.')
+        
+        available_langs = cls.__localized_fields__[field]
+        
+        if cls.__fallback_lang__ not in available_langs:
+            raise AttributeError(
+                f'Fallback language {cls.__fallback_lang__!r} not available for field {field!r}. '
+                f'Available: {available_langs}'
+            )
+        
+        lang = lang or get_lang()
+        if lang not in cls.__localized_fields__[field]:
+            raise AttributeError(f'Field {field!r} is not localized to {lang!r}')
+
+        return [lang] if cls.__fallback_lang__ == lang else [lang, cls.__fallback_lang__]
+
+    @classmethod
+    def _get_localized_field_name(cls, field: str, lang: Lang) -> str:
+        """
+        Returns the name of the localized version of `field` for the requested `lang` in the
+        following format <field>_<lang>.
+
+        Args:
+            field (str): Name of the attribute/field to localize
+            lang (Optional[Lang]): The requested locale language.
+        Returns:
+            str: the name of the localized version of `field` for the requested `lang`
+        """
+        return f'{field}_{lang.value}'
+
+    @classmethod
+    def get_localized_field_chain(cls, field: str, lang: Optional[Lang] = None) -> list[str]:
+        """
+        Returns a chain of the localized versions of the requested field with the priority given
+        to the `lang` argument, followed by the lang returned by
+        [get_lang][ecodev_core.db_i18n.get_lang] and finally
+        the lang defined in `cls.__fallback_lang__`
+
+        Args:
+            field (str): Name of the attribute/field to localize
+            lang (Optional[Lang]): The requested locale language. If none, then uses that
+            returned by [get_lang][ecodev_core.db_i18n.get_lang]. Defaults to None.
+
+        Returns:
+            list[str]: chain of the localized versions of the requested field.
+
+        """
+        return [cls._get_localized_field_name(field, lang) 
+                for lang in cls._get_lang_chain(field, lang)]
+
+    def _get_localized(self, field: str, lang: Optional[Lang] = None) -> Optional[str]:
+        """
+        Returns the localized version of a field.
+
+        The localized version is returned following the rules defined below :
+            - If the requested localized version is not available then the an attempt \
+                will be made to localize the field using `__fallback_lang__`
+            - The specified language can be passed to `_get_localized`. If it is not passed, \
+                the value returned by [get_lang][ecodev_core.db_i18n.get_lang]
+                is  used instead (Defaults to Lang.EN)
+            - If None is returned using the format <field>_<lang> for the language defined in \
+                `__localized_fields__` & `__fallback_lang__` is found then returns None
+
+        Args:
+            field (str): Name of the attribute/field to localize
+            lang (Optional[Lang]): Requested locale. If None, then fetched from \
+                [get_lang][ecodev_core.db_i18n.get_lang]. Defaults to None.
+
+        Return:
+            localized_field (Optional[str]): localized version of the field
+        """
+        lang_chain = self._get_lang_chain(field=field, lang=lang)
+
+        for lang in lang_chain:
+            attr = self._get_localized_field_name(field=field, lang=lang)
+            value = getattr(self, attr, None)
+            if value:
+                return value
+
+        return None
+
+    def __getattr__(self, item: str) -> Optional[str]:
+        """
+        Overrides __getattr__ to get the localized value of a item if it figures in
+        `__localized_fields__`.
+        """
+        if item in self.__localized_fields__:
+            return self._get_localized(item)
+        raise AttributeError(f'{self.__class__.__name__!r} object has no attribute {item!r}')
+
+
+def localized_col(
+    field: str,
+    db_schema: SQLModelMetaclass,
+    lang: Optional[Lang] = None,
+) -> Label:
+    """
+    Returns the localized version of `field` for the requested `lang` of a
+    given SqlModel class. If `lang` is not specified, then fetches the active
+    locale from [get_lang][ecodev_core.db_i18n.get_lang].
+
+    Args:
+        field (str): Name of the field to localize
+        db_schema (SQLModelMetaclass): SQLModelMetaclass instance from which the localized \
+            fields will be fetched
+        lang (Optional[Lang]): Requested locale language. If None, then fetches language \
+            from [get_lang][ecodev_core.db_i18n.get_lang] Defaults to None.
+
+    Returns:
+        localized_field (Label): Localized version of the requested `field` wrapped in label \
+            with the name of `field`.
+    """
+    if not issubclass(db_schema, I18nMixin):
+        raise TypeError(f"{db_schema.__name__} does not inherit from I18nMixin")
+    
+    localized_fields_chain = db_schema.get_localized_field_chain(field, lang)
+    coalesce_fields = [getattr(db_schema, field_name) for field_name in localized_fields_chain]
+
+    return label(field, func.coalesce(*coalesce_fields))

ecodev_core/db_insertion.py

@@ -0,0 +1,128 @@
+"""
+Module implementing functions to insert data within the db
+"""
+from io import BytesIO
+from pathlib import Path
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import List
+from typing import Union
+
+import pandas as pd
+import progressbar
+from fastapi import BackgroundTasks
+from fastapi import UploadFile
+from pandas import ExcelFile
+from sqlmodel import Session
+from sqlmodel import SQLModel
+from sqlmodel.main import SQLModelMetaclass
+from sqlmodel.sql.expression import SelectOfScalar
+
+from ecodev_core.db_upsertion import BATCH_SIZE
+from ecodev_core.logger import log_critical
+from ecodev_core.logger import logger_get
+from ecodev_core.pydantic_utils import CustomFrozen
+from ecodev_core.safe_utils import SimpleReturn
+
+log = logger_get(__name__)
+
+
+class Insertor(CustomFrozen):
+    """
+    Configuration class to insert date into the postgresql sb.
+
+    Attributes are:
+
+        - reductor: how to create or update a row in db
+        - db_schema: the default constructor of the sqlmodel based class defining the db table
+        - selector: the criteria on which to decide whether to create or update (example: only add
+          a user if a user with the same name is not already present in the db)
+        - convertor: how to convert the raw csv/excel passed by the user to json like db rows
+        - read_excel_file: whether to insert data based on an xlsx (if true) or a csv (if false)
+    """
+    reductor: Callable[[Any, Any], Any]
+    db_schema: Callable
+    selector: Callable[[Any], SelectOfScalar]
+    convertor: Callable[[Union[pd.DataFrame, ExcelFile]], List[Dict]]
+    read_excel_file: bool = True
+
+
+def generic_insertion(df_or_xl: Union[pd.DataFrame, ExcelFile, Path],
+                      session: Session,
+                      insertor: Callable[[Union[pd.DataFrame, pd.ExcelFile], Session], None],
+                      background_tasks: Union[BackgroundTasks, None] = None):
+    """
+    Generic insertion of temperature tool related csv into db
+    """
+    try:
+        if background_tasks:
+            background_tasks.add_task(insertor, df_or_xl, session)
+        else:
+            insertor(df_or_xl, session)
+        return SimpleReturn.route_success()
+    except Exception as error:
+        log_critical(f'Something wrong happened: {error}', log)
+        return SimpleReturn.route_failure(str(error))
+
+
+async def insert_file(file: UploadFile, insertor: Insertor, session: Session) -> None:
+    """
+    Inserts an uploaded file into a database
+    """
+    df_raw = await get_raw_df(file, insertor.read_excel_file)
+    insert_data(df_raw, insertor, session)
+
+
+def insert_data(df: Union[pd.DataFrame, ExcelFile], insertor: Insertor, session: Session) -> None:
+    """
+    Inserts a csv/df into a database
+    """
+    for row in insertor.convertor(df):
+        session.add(create_or_update(session, row, insertor))
+    session.commit()
+
+
+def insert_batch_data(data: list[dict | SQLModelMetaclass],
+                      session: Session,
+                      raw_db_schema: SQLModelMetaclass | None = None) -> None:
+    """
+    Insert the passed list of dicts (corresponding to db_schema) into db_schema db.
+    Warning: this only inserts data, without checking for pre-existence.
+    Ensure deleting the data before using it to avoid duplicates.
+    """
+    db_schema = raw_db_schema or data[0].__class__
+    batches = [data[i:i + BATCH_SIZE] for i in range(0, len(data), BATCH_SIZE)]
+
+    for batch in progressbar.progressbar(batches, redirect_stdout=False):
+        for row in batch:
+            new_object = db_schema(**row) if isinstance(row, dict) else row
+            session.add(new_object)
+        session.commit()
+
+
+def create_or_update(session: Session, row: Dict, insertor: Insertor) -> SQLModel:
+    """
+    Create a new row in db if the selector insertor does not find existing row in db. Update the row
+    otherwise.
+    """
+    db_row = insertor.db_schema(**row)
+    if in_db_row := session.exec(insertor.selector(db_row)).first():
+        return insertor.reductor(in_db_row, db_row)
+    return db_row
+
+
+async def get_raw_df(file: UploadFile,
+                     read_excel_file: bool,
+                     sep: str = ',') -> Union[pd.DataFrame, ExcelFile]:
+    """
+    Retrieves the raw data from the uploaded file at pandas format
+    """
+    contents = await file.read()
+    if read_excel_file:
+        return pd.ExcelFile(contents)
+
+    buffer = BytesIO(contents)
+    raw_content = pd.read_csv(buffer, sep=sep)
+    buffer.close()
+    return raw_content