ecodev-core 0.0.67__py3-none-any.whl

ecodev_core/enum_utils.py

@@ -0,0 +1,21 @@
+"""
+Module implementing helper methods working on lists
+"""
+from enum import Enum
+from typing import Type
+from typing import Union
+
+from ecodev_core.safe_utils import stringify
+
+
+def enum_converter(field: Union[str, float],
+                   enum_type: Type,
+                   default: Union[Enum, None] = None
+                   ) -> Union[Enum, None]:
+    """
+    Convert possibly None field to an enum_type if possible, return default otherwise
+    """
+    try:
+        return enum_type(stringify(field))
+    except ValueError:
+        return default

ecodev_core/es_connection.py

@@ -0,0 +1,79 @@
+"""
+Module implementing a connection to an elastic search instance, and basic insertion/retrieval.
+"""
+from typing import Any
+from typing import Union
+
+import progressbar
+from elasticsearch import Elasticsearch
+from elasticsearch import helpers
+from pydantic_settings import BaseSettings
+from pydantic_settings import SettingsConfigDict
+
+from ecodev_core.logger import logger_get
+from ecodev_core.settings import SETTINGS
+
+ES_CLIENT: Union[Elasticsearch, None] = None
+log = logger_get(__name__)
+ES_BATCH_SIZE = 5000
+
+
+class ESAuth(BaseSettings):
+    """
+    Simple ES authentication configuration class
+    """
+    host: str = ''
+    user: str = ''
+    password: str = ''
+    port: int = 9200
+    index: str = ''
+    model_config = SettingsConfigDict(env_file='.env', env_prefix='ES_')
+
+
+ES_AUTH, ES_SETTINGS = ESAuth(), SETTINGS.elastic_search  # type: ignore[attr-defined]
+_HOST, _PORT = ES_SETTINGS.host or ES_AUTH.host,  ES_SETTINGS.port or ES_AUTH.port
+_USER, _PASSWD = ES_SETTINGS.user or ES_AUTH.user, ES_SETTINGS.password or ES_AUTH.password
+_INDEX = ES_SETTINGS.index or ES_AUTH.index
+
+
+def get_es_client():
+    """
+    Get the elasticsearch client
+    """
+    global ES_CLIENT
+
+    if ES_CLIENT is None:
+        ES_CLIENT = Elasticsearch(f'http://{_HOST}:{_PORT}/', basic_auth=[_USER, _PASSWD])
+
+    return ES_CLIENT
+
+
+def create_es_index(body: dict) -> None:
+    """
+    create an es index
+    """
+    client = get_es_client()
+    try:
+        client.indices.delete(index=_INDEX)
+    except Exception:
+        pass
+    client.indices.create(index=_INDEX, body=body)
+    log.info(f'index {_INDEX} created')
+
+
+def insert_es_fields(operations: list[dict], batch_size: int = ES_BATCH_SIZE) -> None:
+    """
+    Generic es insertion
+    """
+    client = get_es_client()
+    batches = [list(operations)[i:i + batch_size] for i in range(0, len(operations), batch_size)]
+    log.info('indexing fields')
+    for batch in progressbar.progressbar(batches, redirect_stdout=False):
+        helpers.bulk(client, batch, index=_INDEX)
+
+
+def retrieve_es_fields(body: dict[str, Any]) -> list[dict]:
+    """
+    Core call to the elasticsearch index
+    """
+    return get_es_client().search(index=_INDEX, body=body)

ecodev_core/list_utils.py

@@ -0,0 +1,134 @@
+"""
+Module implementing helper methods working on lists
+"""
+from collections import defaultdict
+from itertools import groupby
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import Iterator
+from typing import List
+from typing import Optional
+from typing import Tuple
+from typing import Union
+
+
+def group_by_value(list_to_group: List[Any]) -> Dict[Any, List[int]]:
+    """
+    Given a list, group together all equal values by storing them in a dictionary.
+    The keys are the unique list values (think about overriding the class equals if you pass
+    to this method your custom classes) and the values are list of ints, corresponding to the
+    position of the current key in the original list.
+
+    See https://towardsdatascience.com/explaining-the-settingwithcopywarning-in-pandas-ebc19d799d25
+    for why not to use df['base_year'][values] for instance
+    """
+
+    indices: Dict[Any, List[int]] = defaultdict(list)
+    for i in range(len(list_to_group)):
+        indices[list_to_group[i]].append(i)
+    return indices
+
+
+def first_or_default(sequence: Union[List[Any], None],
+                     condition: Union[Callable, None] = None,
+                     default: Optional[Any] = None
+                     ) -> Union[Any, None]:
+    """
+    Returns the first element of a sequence, or default value if the sequence contains no elements.
+    """
+    if not sequence:
+        return default
+
+    if condition is None:
+        return next(iter(sequence), default)
+    return next((elt for elt in sequence if condition(elt)), default)
+
+
+def sort_by_keys(unsorted_dict: dict, reverse: bool = False) -> dict:
+    """
+    Returns a sorted dictionary out of the passed unsorted_dict.
+    Sorting is done on unsorted_dict keys.
+    If reverse is True, reverse sorting
+    """
+    return dict(sorted(unsorted_dict.items(), reverse=reverse))
+
+
+def sort_by_values(unsorted_dict: dict, reverse: bool = False) -> dict:
+    """
+    Returns a sorted dictionary out of the passed unsorted_dict.
+    Sorting is done on unsorted_dict values.
+    If reverse is True, reverse sorting
+    """
+    return dict(sorted(unsorted_dict.items(), key=lambda item: item[1], reverse=reverse))
+
+
+def first_func_or_default(sequence: list[Callable] | None,
+                          elt: Any,
+                          condition: Callable | None = None,
+                          default: Any | None = None
+                          ) -> Any | None:
+    """
+    Returns the first element of a functional sequence if a certain criteria is met
+    or default value if the criteria is never met.
+    The criteria is like so:
+    - If no condition is provided, then
+      just check that func applied on elt is not None
+    - If a condition is provided, then
+       check that condition applied on func(elt) is not None
+    """
+    if not sequence:
+        return default
+
+    return next((func(elt) for func in sequence if (condition or (lambda x: x))(func(elt))),
+                default)
+
+
+def group_by(sequence: List[Any], key: Union[Callable, None]) -> Iterator[Tuple[Any, List[Any]]]:
+    """
+    Extension of itertools groupby method.
+
+    Reasons of existence:
+        - do the sorting before the grouping to avoid the usual mistake of forgetting the sorting
+        - convert the group Iterator to a list. More convenient that the default groupby behaviour
+           in all cases where you need to iterate more than once on the group
+    """
+    for key, group in groupby(sorted(sequence, key=key), key=key):
+        yield key, list(group)
+
+
+def lselect(sequence: List[Any], condition: Union[Callable, None] = None) -> List[Any]:
+    """
+    Filter the passed sequence according to the passed condition
+    """
+    return list(filter(condition, sequence))
+
+
+def lselectfirst(sequence: List[Any], condition: Union[Callable, None] = None) -> Union[Any, None]:
+    """
+    Select the filtered element of the passed sequence according to the passed condition
+    """
+
+    return filtered_list[0] if (filtered_list := list(filter(condition, sequence))) else None
+
+
+def first_transformed_or_default(sequence: List[Any], transformation: Callable) -> Union[Any, None]:
+    """
+    Returns the first non-trivial transformed element of a sequence,
+     or default value if no non-trivial transformed elements are found.
+    """
+    return next((fx for elt in sequence if (fx := transformation(elt)) is not None), None)
+
+
+def dict_to_class(data: dict):
+    """
+    Convert a (possibly nested) dictionary to a class.
+    """
+    return {k: type(k, (), dict_to_class(v)) if isinstance(v, dict) else v for k, v in data.items()}
+
+
+def list_tuple_to_dict(data: list[tuple]) -> list[dict[str, Any]] | None:
+    """
+    Transforms the result of a sqlmodel query into a list of Dict
+    """
+    return [x._asdict() for x in data] if data else None  # type: ignore[attr-defined]

ecodev_core/logger.py

@@ -0,0 +1,122 @@
+"""
+Helpers for pretty logging
+"""
+import logging
+import sys
+import traceback
+
+LIBS = ['azure', 'passlib', 'trimesh', 'fiona',
+        'urllib3', 'botocore', 'boto', 'boto3', 's3transfer']
+
+
+def log_critical(message: str, logger):
+    """
+    Traceback enabled for unintended serious errors
+    """
+    logger.error(message)
+    logger.error(traceback.format_exc())
+
+
+def logger_get(name, level=logging.DEBUG):
+    """
+    Main method called by all other modules to log
+    """
+    logging.basicConfig(level=level, stream=sys.stdout)
+    for lib in LIBS:
+        _safe_log_setter(lib)
+    logger = logging.getLogger(name)
+    config_log(logger, level, MyFormatter())
+    return logger
+
+
+def _safe_log_setter(lib: str) -> None:
+    """
+    Safe logger. ERROR level not to be swamped by verbose library info.
+    """
+    try:
+        logging.getLogger(lib).setLevel(logging.ERROR)
+    except Exception:
+        pass
+
+
+class MyFormatter(logging.Formatter):
+    """
+    Formatter to print %(filename)s:%(funcName)s:%(lineno)d on 24 characters
+
+    Typical format :
+    2016-10-26 14:20:21,379 | DEBUG    | logger:log_me:57         : This is a log
+    """
+    message_width = 110
+    cpath_width = 32
+    date_fmt = '%Y-%m-%d %H:%M:%S'
+
+    pink = '\x1b[35m'
+    green = '\x1b[32m'
+    yellow = '\x1b[33m'
+    red = '\x1b[31m'
+    bold_red = '\x1b[31;1m'
+    reset = '\x1b[0m'
+
+    FORMATS = {
+        logging.DEBUG: pink,
+        logging.INFO: green,
+        logging.WARNING: yellow,
+        logging.ERROR: red,
+        logging.CRITICAL: bold_red,
+    }
+
+    def format(self, record):
+        """
+        Format logs
+        """
+        initial_record = f'{record.module}:{record.funcName}:{ record.lineno}'
+        cpath = initial_record[-self.cpath_width:].ljust(self.cpath_width)
+        time = self.formatTime(record, self.date_fmt)
+        prefix = f'{time} | {record.levelname} | {record.process} | {cpath}'
+
+        # fixing max length
+        limited_lines = []
+        for line in record.getMessage().split(str('\n')):
+            while len(line) > self.message_width:
+                if (last_space_position := line[:self.message_width - 1].rfind(' ')) > 0:
+                    splitting_position = last_space_position
+                else:
+                    splitting_position = self.message_width
+                limited_lines.append(line[:splitting_position])
+                line = line[splitting_position:]
+
+            # don't forget end of line
+            limited_lines.append(line)
+
+        # formatting final message
+        final_message = ''.join(f'{prefix} | {line}\n' for line in limited_lines).rstrip()
+
+        return f'{self.FORMATS[record.levelno]}{final_message}{self.reset}'
+
+
+def config_log(logger, level, formatter):
+    """ Configures the logging.
+
+    This function defines the root logger. It needs to be called only once.
+    Then, all modules should log like this:
+    '''
+    from logger.logger import get as logger_get
+    log = logger_get(__name__)
+    '''
+    If the function is called more than once, duplicate handlers are ignored
+    to avoid duplicate logging.
+
+    Args:
+        logger: logging object
+        level: Logging level
+        formatter: Logging format
+
+    """
+    # Get the root logger (because no name is specified in getLogger())
+    logger.setLevel(level)
+    logger.propagate = False
+
+    console_handler = logging.StreamHandler(stream=sys.stdout)
+    if all(handler.stream.name != console_handler.stream.name for handler in logger.handlers):
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)

ecodev_core/pandas_utils.py

@@ -0,0 +1,69 @@
+"""
+Module implementing some utilitary methods on pandas types
+"""
+import tempfile
+from base64 import b64decode
+from pathlib import Path
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+
+
+def pd_equals(prediction: pd.DataFrame, gt_path=Path):
+    """
+    Since some Nones are serialized as Nans by pandas (heavy type inference),
+     we store the prediction at a temporary location in order to reload it on the fly and compare it
+     to a pre-store ground truth, in order that both gt and prediction benefited from the same
+     type inferences.
+    """
+    with tempfile.TemporaryDirectory() as folder:
+        prediction.to_csv(Path(folder) / 'tmp.csv', index=False)
+        reloaded_prediction = pd.read_csv(Path(folder) / 'tmp.csv')
+    pd.testing.assert_frame_equal(reloaded_prediction, pd.read_csv(gt_path))
+
+
+def jsonify_series(row: pd.Series) -> Dict:
+    """
+    Convert a serie into a json compliant dictionary (replacing np.nans by Nones)
+    """
+    return {key: None if isinstance(value, float) and np.isnan(value) else value for key, value in
+            row.to_dict().items()}
+
+
+def get_excelfile(contents: str) -> pd.ExcelFile:
+    """
+    Function which converts user xlsx file upload into a pd.ExcelFile
+    """
+    content_type, content_string = contents.split(',')
+    xl = b64decode(content_string)
+    return pd.ExcelFile(xl)
+
+
+def safe_drop_columns(df: pd.DataFrame, columns: list[str]) -> pd.DataFrame:
+    """
+    Returns a DataFrame without a list of columns, with a prior check on the existence of these
+    columns in the DataFrame
+    """
+
+    return df.drop(columns=[col for col in columns if col in df.columns])
+
+
+def is_null(value: Any) -> bool:
+    """
+    Checks if a value is null or not
+    """
+    return value is None or isinstance(value, float) and np.isnan(value)
+
+
+def get_value(column: str, method: Callable, row: pd.Series) -> Optional[Any]:
+    """
+    Function which performs a method on a value if the column name is in the row index
+    """
+    if column not in row.index or is_null(row[column]):
+        return None
+
+    return method(row[column])

ecodev_core/permissions.py

@@ -0,0 +1,21 @@
+"""
+Module implementing all permission levels an application user can have
+"""
+from enum import Enum
+from enum import unique
+
+
+@unique
+class Permission(str, Enum):
+    """
+    Enum listing all permission levels an application user can have
+    """
+    ADMIN = 'Admin'
+    Consultant = 'Consultant'
+    Client = 'Client'
+    CLIENT_ADMIN = 'Client Admin'
+    DENIED_PERMISSION = 'Denied Permission'
+    FORM_ADMIN = 'Form Admin'
+    VALIDATOR = 'Validator'
+    APPRAISER = 'Appraiser'
+    USER = 'User'

ecodev_core/pydantic_utils.py

@@ -0,0 +1,33 @@
+"""
+Simple Pydantic wrapper classes around BaseModel to accommodate for orm and frozen cases
+"""
+from pydantic import BaseModel
+from pydantic import ConfigDict
+
+
+class Basic(BaseModel):
+    """
+    Basic pydantic configuration
+    """
+    model_config = ConfigDict(frozen=False, arbitrary_types_allowed=True)
+
+
+class Frozen(BaseModel):
+    """
+    Frozen pydantic configuration
+    """
+    model_config = ConfigDict(frozen=True)
+
+
+class CustomFrozen(Frozen):
+    """
+    Frozen pydantic configuration for custom types
+    """
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class OrmFrozen(CustomFrozen):
+    """
+    Frozen pydantic configuration for orm like object
+    """
+    model_config = ConfigDict(from_attributes=True)

ecodev_core/read_write.py

@@ -0,0 +1,52 @@
+"""
+Module regrouping low level reading and writing helper methods
+"""
+import json
+import os
+from pathlib import Path
+from typing import Dict
+from typing import List
+from typing import Union
+
+import yaml
+
+
+def write_json_file(json_data: Union[Dict, List], file_path: Path):
+    """
+    Write json_data at file_path location
+    """
+    os.umask(0)
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(json.dumps(json_data, indent=4))
+
+
+def load_json_file(file_path: Path):
+    """
+    Load a json file at file_path location
+    """
+    with open(file_path, 'r', encoding='utf-8') as f:
+        loaded_json = json.load(f)
+
+    return loaded_json
+
+
+def load_yaml_file(file_path: Path):
+    """
+    Load a yaml file at file_path location
+    """
+    with open(file_path) as file:
+        loaded_yaml = yaml.safe_load(file)
+
+    return loaded_yaml
+
+
+def make_dir(directory: Path):
+    """
+    Helper that create the directory "directory" if it doesn't exist yet
+    """
+    try:
+        os.umask(0)
+        os.makedirs(directory)
+    except OSError as error:
+        if not directory.is_dir():
+            raise OSError(f'directory={directory!r} should exist but does not.: {error}') from error