ecodev-core 0.0.1__py3-none-any.whl

ecodev_core/db_retrieval.py

@@ -0,0 +1,194 @@
+"""
+Low level methods to retrieve data from db in a paginated way
+"""
+from math import ceil
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+import pandas as pd
+from sqlmodel import col
+from sqlmodel import or_
+from sqlmodel import select
+from sqlmodel import Session
+from sqlmodel.sql.expression import Select
+from sqlmodel.sql.expression import SelectOfScalar
+
+from ecodev_core.db_connection import engine
+from ecodev_core.db_filters import SERVER_SIDE_FILTERS
+from ecodev_core.db_filters import ServerSideFilter
+from ecodev_core.list_utils import first_or_default
+from ecodev_core.pydantic_utils import Frozen
+
+SelectOfScalar.inherit_cache = True  # type: ignore
+Select.inherit_cache = True  # type: ignore
+OPERATORS = ['>=', '<=', '!=', '=', '<', '>', 'contains ']
+
+
+class ServerSideField(Frozen):
+    """
+    Simple class used for sever side data retrieval
+
+    Attributes are:
+        - col_name: the name as it will appear on the frontend interface
+        - field_name: the SQLModel attribute name associated with this field
+        - field: the SQLModel attribute associated with this field
+        - filter: the filtering mechanism to use for this field
+    """
+    col_name: str
+    field_name: str
+    field: Any
+    filter: ServerSideFilter
+
+
+def count_rows(fields: List[ServerSideField],
+               model: Any,
+               limit: Union[int, None] = None,
+               filter_str: str = '',
+               search_str: str = '',
+               search_cols: Optional[List] = None) -> int:
+    """
+    Count the total number of rows in the db model, with statically defined field_filters fed with
+    dynamically set frontend filters. Divide this total number by limit to account for pagination.
+    """
+    with Session(engine) as session:
+        count = _get_full_query(fields, model, filter_str, session, True,
+                                search_str, search_cols).count()
+
+        return ceil(count / limit) if limit else count
+
+
+def get_rows(fields: List[ServerSideField],
+             model: Any,
+             limit: Union[int, None] = None,
+             offset: Union[int, None] = None,
+             filter_str: str = '',
+             search_str: str = '',
+             search_cols: Optional[List] = None,
+             fields_order: Optional[Callable] = None
+             ) -> pd.DataFrame:
+    """
+    Select relevant row lines from model db. Select the whole db if no limit or offset is provided.
+    Convert the rows to a dataframe in order to show the result in a dash data_table.
+
+    NB:
+    * 'fields_order' specify how to order the result rows
+    * 'limit' and 'offset' correspond to the pagination of the results.
+    * 'search_str' corresponds to the search string from the search input.
+    """
+    with Session(engine) as session:
+        rows = _paginate_db_lines(fields, model, session, limit, offset, filter_str,
+                                  search_str, search_cols, fields_order)
+    if len(raw_df := pd.DataFrame.from_records([row.dict() for row in rows])) > 0:
+        return raw_df.rename(columns={field.field_name: field.col_name for field in fields}
+                             )[[field.col_name for field in fields]]
+    return pd.DataFrame(columns=[field.col_name for field in fields])
+
+
+def _paginate_db_lines(fields: List[ServerSideField],
+                       model: Any,
+                       session: Session,
+                       limit: Union[int, None],
+                       offset: Union[int, None],
+                       filter_str: str,
+                       search_str: str = '',
+                       search_cols: Optional[List] = None,
+                       fields_order: Optional[Callable] = None,
+                       ) -> List:
+    """
+    Select relevant row lines from model db. Select the whole db if no limit or offset is provided.
+    """
+    if fields_order is None:
+        fields_order = _get_default_field_order(fields)
+
+    query = fields_order(_get_full_query(fields, model, filter_str, session, count=False,
+                                         search_str=search_str, search_cols=search_cols))
+    if limit is not None and offset is not None:
+        return list(session.exec(query.offset(offset * limit).limit(limit)))
+    return list(session.exec(query).all())
+
+
+def _get_full_query(fields: List[ServerSideField],
+                    model: Any,
+                    filter_str: str,
+                    session: Session,
+                    count: bool = False,
+                    search_str: str = '',
+                    search_cols: Optional[List] = None
+                    ) -> SelectOfScalar:
+    """
+    Forge a complete select query given both search and filter strings
+
+    NB:
+    * This relies on the passed statically defined field_filters corresponding to the model.
+    * The field_filters are used jointly with the dynamically set frontend filters.
+
+    """
+    filter_query = _get_filter_query(fields, model, _get_frontend_filters(filter_str), session,
+                                     count)
+
+    if not search_str or not search_cols:
+        return filter_query
+
+    return filter_query.where(or_(col(field).ilike(f'%{search_str.strip()}%')
+                                  for field in search_cols))
+
+
+def _get_frontend_filters(raw_filters: str) -> Dict[str, Tuple[str, str]]:
+    """
+    Forge a dictionary of field keys, (operator, value) values in order to filter a db model.
+    """
+    split_filters = raw_filters.split(' && ')
+    return {elt[elt.find('{') + 1: elt.rfind('}')]: _forge_filter(elt) for elt in split_filters}
+
+
+def _forge_filter(elt: str) -> Tuple[str, str]:
+    """
+    Forge the operator and value associated to the passed element. Do so by scanning the ordered
+    sequence of OPERATORS and returning the first matching (value is on the right of it).
+    """
+    return next(((key, elt.split(key)[-1]) for key in OPERATORS if key in elt), ('', ''))
+
+
+def _get_filter_query(fields: List[ServerSideField],
+                      model: Any,
+                      frontend_filters: Dict[str, Tuple[str, str]],
+                      session: Session,
+                      count: bool = False
+                      ) -> SelectOfScalar:
+    """
+    Filter a model given backend static field_filters called with dynamically set frontend_filters.
+
+    Returns:
+        * either the query fetching the filtered rows (count = False)
+        * or the filter row count.
+    """
+    query = session.query(model) if count else select(model)
+    if not frontend_filters or not all(frontend_filters.keys()):
+        return query
+
+    for key, (operator, value) in frontend_filters.items():
+        if field := first_or_default(fields, lambda x: x.col_name == key):
+            query = SERVER_SIDE_FILTERS[field.filter](query=query, operator=operator,
+                                                      value=value, field=field.field)
+
+    return query
+
+
+def _get_default_field_order(fields: List[ServerSideField]) -> Callable:
+    """
+    Recover default field order from list of fields
+    """
+    def fields_order(query):
+        """
+        Default field ordering
+
+        Take the initial query as input and specify the order to use.
+        """
+        return query.order_by(*[field.field for field in fields])
+
+    return fields_order

ecodev_core/enum_utils.py

@@ -0,0 +1,21 @@
+"""
+Module implementing helper methods working on lists
+"""
+from enum import Enum
+from typing import Type
+from typing import Union
+
+from ecodev_core.safe_utils import stringify
+
+
+def enum_converter(field: Union[str, float],
+                   enum_type: Type,
+                   default: Union[Enum, None] = None
+                   ) -> Union[Enum, None]:
+    """
+    Convert possibly None field to an enum_type if possible, return default otherwise
+    """
+    try:
+        return enum_type(stringify(field))
+    except ValueError:
+        return default

ecodev_core/list_utils.py

@@ -0,0 +1,65 @@
+"""
+Module implementing helper methods working on lists
+"""
+from collections import defaultdict
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Union
+
+
+def group_by_value(list_to_group: List[Any]) -> Dict[Any, List[int]]:
+    """
+    Given a list, group together all equal values by storing them in a dictionary.
+    The keys are the unique list values (think about overriding the class equals if you pass
+    to this method your custom classes) and the values are list of ints, corresponding to the
+    position of the current key in the original list.
+
+    See https://towardsdatascience.com/explaining-the-settingwithcopywarning-in-pandas-ebc19d799d25
+    for why not to use df['base_year'][values] for instance
+    """
+
+    indices: Dict[Any, List[int]] = defaultdict(list)
+    for i in range(len(list_to_group)):
+        indices[list_to_group[i]].append(i)
+    return indices
+
+
+def first_or_default(sequence: Union[List[Any], None],
+                     condition: Union[Callable, None] = None,
+                     default: Optional[Any] = None
+                     ) -> Union[Any, None]:
+    """
+    Returns the first element of a sequence, or default value if the sequence contains no elements.
+    """
+    if not sequence:
+        return default
+
+    if condition is None:
+        return next(iter(sequence), default)
+    return next((elt for elt in sequence if condition(elt)), default)
+
+
+def lselect(sequence: List[Any], condition: Union[Callable, None] = None) -> List[Any]:
+    """
+    Filter the passed sequence according to the passed condition
+    """
+    return list(filter(condition, sequence))
+
+
+def lselectfirst(sequence: List[Any], condition: Union[Callable, None] = None) -> Union[Any, None]:
+    """
+    Select the filtered element of the passed sequence according to the passed condition
+    """
+
+    return filtered_list[0] if (filtered_list := list(filter(condition, sequence))) else None
+
+
+def first_transformed_or_default(sequence: List[Any], transformation: Callable) -> Union[Any, None]:
+    """
+    Returns the first non-trivial transformed element of a sequence,
+     or default value if no non-trivial transformed elements are found.
+    """
+    return next((fx for elt in sequence if (fx := transformation(elt)) is not None), None)

ecodev_core/logger.py

@@ -0,0 +1,106 @@
+"""
+Helpers for pretty logging
+"""
+import logging
+import sys
+import traceback
+
+
+def log_critical(message: str, logger):
+    """
+    Traceback enabled for unintended serious errors
+    """
+    logger.error(message)
+    logger.error(traceback.format_exc())
+
+
+def logger_get(name, level=logging.DEBUG):
+    """
+    Main method called by all other modules to log
+    """
+    logger = logging.getLogger(name)
+    config_log(logger, level, MyFormatter())
+    return logger
+
+
+class MyFormatter(logging.Formatter):
+    """
+    Formatter to print %(filename)s:%(funcName)s:%(lineno)d on 24 characters
+
+    Typical format :
+    2016-10-26 14:20:21,379 | DEBUG    | logger:log_me:57         : This is a log
+    """
+    message_width = 110
+    cpath_width = 32
+    date_fmt = '%Y-%m-%d %H:%M:%S'
+
+    pink = '\x1b[35m'
+    green = '\x1b[32m'
+    yellow = '\x1b[33m'
+    red = '\x1b[31m'
+    bold_red = '\x1b[31;1m'
+    reset = '\x1b[0m'
+
+    FORMATS = {
+        logging.DEBUG: pink,
+        logging.INFO: green,
+        logging.WARNING: yellow,
+        logging.ERROR: red,
+        logging.CRITICAL: bold_red,
+    }
+
+    def format(self, record):
+        """
+        Format logs
+        """
+        initial_record = f'{record.module}:{record.funcName}:{ record.lineno}'
+        cpath = initial_record[-self.cpath_width:].ljust(self.cpath_width)
+        time = self.formatTime(record, self.date_fmt)
+        prefix = f'{time} | {record.levelname} | {record.process} | {cpath}'
+
+        # fixing max length
+        limited_lines = []
+        for line in record.getMessage().split(str('\n')):
+            while len(line) > self.message_width:
+                if (last_space_position := line[:self.message_width - 1].rfind(' ')) > 0:
+                    splitting_position = last_space_position
+                else:
+                    splitting_position = self.message_width
+                limited_lines.append(line[:splitting_position])
+                line = line[splitting_position:]
+
+            # don't forget end of line
+            limited_lines.append(line)
+
+        # formatting final message
+        final_message = ''.join(f'{prefix} | {line}\n' for line in limited_lines).rstrip()
+
+        return f'{self.FORMATS[record.levelno]}{final_message}{self.reset}'
+
+
+def config_log(logger, level, formatter):
+    """ Configures the logging.
+
+    This function defines the root logger. It needs to be called only once.
+    Then, all modules should log like this:
+    '''
+    from logger.logger import get as logger_get
+    log = logger_get(__name__)
+    '''
+    If the function is called more than once, duplicate handlers are ignored
+    to avoid duplicate logging.
+
+    Args:
+        logger: logging object
+        level: Logging level
+        formatter: Logging format
+
+    """
+    # Get the root logger (because no name is specified in getLogger())
+    logger.setLevel(level)
+    logger.propagate = False
+
+    console_handler = logging.StreamHandler(stream=sys.stdout)
+    if all(handler.stream.name != console_handler.stream.name for handler in logger.handlers):
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)

ecodev_core/pandas_utils.py

@@ -0,0 +1,30 @@
+"""
+Module implementing some utilitary methods on pandas types
+"""
+import tempfile
+from pathlib import Path
+from typing import Dict
+
+import numpy as np
+import pandas as pd
+
+
+def pd_equals(prediction: pd.DataFrame, gt_path=Path):
+    """
+    Since some Nones are serialized as Nans by pandas (heavy type inference),
+     we store the prediction at a temporary location in order to reload it on the fly and compare it
+     to a pre-store ground truth, in order that both gt and prediction benefited from the same
+     type inferences.
+    """
+    with tempfile.TemporaryDirectory() as folder:
+        prediction.to_csv(Path(folder) / 'tmp.csv', index=False)
+        reloaded_prediction = pd.read_csv(Path(folder) / 'tmp.csv')
+    pd.testing.assert_frame_equal(reloaded_prediction, pd.read_csv(gt_path))
+
+
+def jsonify_series(row: pd.Series) -> Dict:
+    """
+    Convert a serie into a json compliant dictionary (replacing np.nans by Nones)
+    """
+    return {key: None if isinstance(value, float) and np.isnan(value) else value for key, value in
+            row.to_dict().items()}

ecodev_core/permissions.py

@@ -0,0 +1,15 @@
+"""
+Module implementing all permission levels an application user can have
+"""
+from enum import Enum
+from enum import unique
+
+
+@unique
+class Permission(str, Enum):
+    """
+    Enum listing all permission levels an application user can have
+    """
+    ADMIN = 'Admin'
+    Consultant = 'Consultant'
+    Client = 'Client'

ecodev_core/pydantic_utils.py

@@ -0,0 +1,52 @@
+"""
+Simple Pydantic wrapper classes around BaseModel to accommodate for orm and frozen cases
+"""
+from pydantic import BaseModel
+
+
+class Basic(BaseModel):
+    """
+    Basic pydantic configuration
+    """
+
+    class Config:
+        """
+        Allow mutation in inheriting classes
+        """
+        allow_mutation = True
+        arbitrary_types_allowed = True
+
+
+class Frozen(BaseModel):
+    """
+    Frozen pydantic configuration
+    """
+
+    class Config:
+        """
+        Forbid mutation in order to freeze the inheriting classes
+        """
+        allow_mutation = False
+
+
+class CustomFrozen(Frozen):
+    """
+    Frozen pydantic configuration for custom types
+    """
+    class Config:
+        """
+        Allow arbitrary custom types
+        """
+        arbitrary_types_allowed = True
+
+
+class OrmFrozen(CustomFrozen):
+    """
+    Frozen pydantic configuration for orm like object
+    """
+
+    class Config:
+        """
+        Allow to create object from orm one
+        """
+        orm_mode = True

ecodev_core/read_write.py

@@ -0,0 +1,40 @@
+"""
+Module regrouping low level reading and writing helper methods
+"""
+import json
+import os
+from pathlib import Path
+from typing import Dict
+from typing import List
+from typing import Union
+
+
+def write_json_file(json_data: Union[Dict, List], file_path: Path):
+    """
+    Write json_data at file_path location
+    """
+    os.umask(0)
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(json.dumps(json_data, indent=4))
+
+
+def load_json_file(file_path: Path):
+    """
+    Load a json file at file_path location
+    """
+    with open(file_path, 'r', encoding='utf-8') as f:
+        loaded_json = json.load(f)
+
+    return loaded_json
+
+
+def make_dir(directory: Path):
+    """
+    Helper that create the directory "directory" if it doesn't exist yet
+    """
+    try:
+        os.umask(0)
+        os.makedirs(directory)
+    except OSError as error:
+        if not directory.is_dir():
+            raise OSError(f'directory={directory!r} should exist but does not.: {error}') from error