ecodev-core 0.0.1__py3-none-any.whl

ecodev_core/check_dependencies.py

@@ -0,0 +1,179 @@
+"""
+Module computing and checking high level dependencies in the coe (based on pydeps)
+"""
+from pathlib import Path
+from subprocess import run
+from typing import Dict
+from typing import Iterator
+from typing import List
+
+from ecodev_core.logger import logger_get
+
+
+CONF_FILE = 'dependencies.json'
+Dependency = Dict[str, List[str]]
+log = logger_get(__name__)
+
+
+def check_dependencies(code_base: Path,  theoretical_deps: Path):
+    """
+    hook for preserving the pre established solution dependencies.
+    Compare regroupment of module dependencies matrix to a pre-computed matrix stored
+     in theoretical_deps. Computation done on code_base.
+    """
+    dependencies = _get_current_dependencies(_valid_modules(code_base), code_base,  code_base.name)
+    allowed_dependencies = _get_allowed_dependencies(theoretical_deps)
+    if not (ok_deps := _test_dependency(allowed_dependencies, dependencies)):
+        log.error('you changed high level solution dependencies. Intended?')
+    return ok_deps
+
+
+def compute_dependencies(code_base: Path, output_folder: Path, plot: bool = True):
+    """
+    Given a code base, compute the dependencies between its high level modules.
+     Store in output_folder the dependency matrix in txt format and the png of the dependencies
+    """
+    code_folder = code_base.name
+    modules = _valid_modules(code_base)
+
+    deps: Dependency = _get_current_dependencies(modules, code_base, code_folder)
+
+    for mod, mod_deps in deps.items():
+        with open(output_folder / f'{mod}.py', 'w') as f_stream:
+            f_stream.writelines([f'from .{other_module} import to\n' for other_module in mod_deps])
+
+    with open(output_folder / '__init__.py', 'w') as f_stream:
+        f_stream.writelines([])
+
+    with open(output_folder / f'dependencies_{code_folder}.txt', 'w') as f_stream:
+        f_stream.writelines([f'{dependency}\n' for dependency in _get_dep_matrix(modules, deps)])
+
+    if plot:
+        run(['pydeps', '.', '-T', 'png', '--no-show', '--rmprefix',
+             f'{output_folder.name}.'], cwd=output_folder)
+
+
+def _test_dependency(allowed_deps: Dependency, dependencies: Dependency) -> bool:
+    """
+    For each modules stored in a dependencies.json file, check whether the current
+    module dependencies are the same as the config ones.
+    """
+    for module in dependencies:
+        for dep in allowed_deps[module]:
+            if dep not in dependencies[module]:
+                log.error(f'{module} no longer imported in {dep}. Intended ?')
+        for dep in dependencies[module]:
+            if dep not in allowed_deps[module]:
+                log.error(f'{dep} now imported in {module}. Intended ?')
+        for dep in dependencies[module]:
+            if module in dependencies[dep] and dep != module:
+                log.error(f'Circular ref created between {module} and {dep}.')
+    return dependencies == allowed_deps
+
+
+def _get_allowed_dependencies(config_path: Path) -> Dependency:
+    """
+    Given the pre established dependency file path, compute
+    the pre established modules and their dependencies seen as an adjacency dict.
+    All the values of a given key are its dependencies.
+    The keys and the values of the dictionary take their labels in
+    the pre established module list.
+    """
+    raw_lines = list(_safe_read_lines(config_path))
+    raw_matrix = [raw_dependency.split(',') for raw_dependency in raw_lines][1:]
+    modules = [raw_dependency[0] for raw_dependency in raw_matrix]
+    module_dependencies: Dict[str, List[str]] = {
+        module: [modules[idx_other_module] for idx_other_module in range(len(modules))
+                 if raw_matrix[idx_module][idx_other_module + 1] == 'Yes']
+        for idx_module, module in enumerate(modules)
+    }
+
+    return module_dependencies
+
+
+def _get_current_dependencies(modules: List[str],
+                              code_base: Path,
+                              code_folder: str) -> Dependency:
+    """
+    Given the pre established modules, the code_base directory and the relative path of the code
+    directory wrt code_folder, compute the pre current dependencies as an adjacency dict.
+    All the values of a given key are its dependencies. The keys and the values of
+    the dictionary take their labels in the pre established module list.
+    """
+    module_dependencies: Dependency = {}
+    for module in modules:
+        module_dependencies[module] = []
+        module_dir = code_base / module
+        for other_module in modules:
+            if other_module in module_dependencies[module]:
+                break
+            for py_file in _get_recursively_all_files_in_dir(module_dir, 'py'):
+                if _depends_on_module(module_dir / py_file, other_module, code_folder):
+                    module_dependencies[module].append(other_module)
+                    break
+
+    return module_dependencies
+
+
+def _depends_on_module(file: Path, module: str, code_folder: str) -> bool:
+    """
+     check if a reference to module is in the imports of python_file
+    """
+    return any(
+        (f'from {code_folder}.{module}' in line and 'import' in line)
+        or (line.startswith(f'import {code_folder}.{module}.'))
+        for line in _safe_read_lines(file)
+    )
+
+
+def _safe_read_lines(filename: Path) -> Iterator[str]:
+    """
+    read all lines in file, erase the final special \n character
+    """
+    with open(filename, 'r') as f:
+        lines = f.readlines()
+    yield from [line.strip() for line in lines]
+
+
+def _get_recursively_all_files_in_dir(code_folder: Path, extension: str) -> Iterator[Path]:
+    """
+    Given a folder, recursively return all files of the given extension in the folder
+    """
+    yield from code_folder.glob(f'**/*.{extension}')
+
+
+def _valid_folder(folder: Path):
+    """
+    Return True if folder is a python regroupment of module to be considered for dependency analysis
+    """
+    return (
+        folder.is_dir()
+        and not folder.name.startswith('.')
+        and not folder.name.startswith('_')
+        and folder.name != 'data'
+    )
+
+
+def _valid_modules(root_folder: Path):
+    """
+    Retrieve valid solution module (found at the base level of root_folder)
+    """
+    return sorted([folder.name for folder in root_folder.iterdir() if _valid_folder(folder)])
+
+
+def _get_dep_matrix(modules: List[str], deps: Dependency) -> List[str]:
+    """
+    Retrieve the dependency matrix of the inspected solution in txt format
+    """
+    dependencies = [f'module x depends on,{",".join(modules)}']
+    dependencies.extend(f'{module},' + ','.join([_depends_on(module, other_module, deps)
+                                                 for other_module in modules])for module in modules)
+
+    return dependencies
+
+
+def _depends_on(module, other_module, deps):
+    """
+    Write correct input in the dependency matrix ("Yes" if other_module is in deps of module key)
+    """
+    return 'Yes' if other_module in deps[module] else 'No'

ecodev_core/custom_equal.py

@@ -0,0 +1,27 @@
+"""
+Module comparing whether two elements are both None or both not None and equals
+"""
+from typing import Optional
+
+import numpy as np
+
+
+def custom_equal(element_1: Optional[object], element_2: Optional[object], element_type: type):
+    """
+    Compare whether two elements are both None or both not None and equals (same type/same value)
+
+    Args:
+        element_1: the first element of the comparison
+        element_2:  the second element of the comparison
+        element_type:  the expected element type for both elements
+
+    Returns:
+        True if both None or both not None and equals (same type/same value)
+    """
+    if element_1 is None:
+        return element_2 is None
+
+    if not isinstance(element_1, element_type) or not isinstance(element_2, element_type):
+        return False
+
+    return np.isclose(element_1, element_2) if element_type == float else element_1 == element_2

ecodev_core/db_connection.py

@@ -0,0 +1,68 @@
+"""
+Module implementing postgresql connection
+"""
+from typing import Callable
+
+from pydantic import BaseSettings
+from sqlalchemy import delete
+from sqlmodel import create_engine
+from sqlmodel import Session
+from sqlmodel import SQLModel
+
+from ecodev_core.logger import logger_get
+log = logger_get(__name__)
+
+
+class DbSettings(BaseSettings):
+    """
+    Settings class used to connect to the postgresql database
+    """
+    db_host: str
+    db_port: str
+    db_name: str
+    db_username: str
+    db_password: str
+
+    class Config:
+        """
+        Config class specifying the name of the environment file to read
+        """
+        env_file = '.env'
+
+
+DB = DbSettings()
+DB_URL = f'postgresql://{DB.db_username}:{DB.db_password}@{DB.db_host}:{DB.db_port}/{DB.db_name}'
+engine = create_engine(DB_URL)
+
+
+def create_db_and_tables(model: Callable) -> None:
+    """
+    Create all tables based on the declared schemas in core/models thanks to sqlmodel
+    """
+    log.info(f'Inserting on the fly {model} and all other domain tables')
+    SQLModel.metadata.create_all(engine)
+
+
+def delete_table(model: Callable) -> None:
+    """
+    Delete all rows of the passed model from db
+    """
+    with Session(engine) as session:
+        result = session.execute(delete(model))
+        session.commit()
+        log.info(f'Deleted {result.rowcount} rows')
+
+
+def get_session():
+    """
+    Retrieves the session, used in Depends() attributes of fastapi routes
+    """
+    with Session(engine) as session:
+        yield session
+
+
+def info_message(model: Callable):
+    """
+    Create all tables based on the declared schemas in core/models thanks to sqlmodel
+    """
+    log.info(f'hack to get all models imported in an alembic env.py. {model}')

ecodev_core/db_filters.py

@@ -0,0 +1,142 @@
+"""
+Low level db filtering methods
+"""
+from datetime import datetime
+from enum import Enum
+from enum import unique
+from typing import Callable
+from typing import Dict
+
+from sqlalchemy import func
+from sqlalchemy.orm.attributes import InstrumentedAttribute
+from sqlmodel import col
+from sqlmodel.sql.expression import Select
+from sqlmodel.sql.expression import SelectOfScalar
+
+SelectOfScalar.inherit_cache = True  # type: ignore
+Select.inherit_cache = True  # type: ignore
+OPERATORS = ['>=', '<=', '!=', '=', '<', '>', 'contains ']
+
+
+@unique
+class ServerSideFilter(str, Enum):
+    """
+    All possible server side filtering mechanisms
+    """
+    STARTSTR = 'start_str'
+    ILIKESTR = 'ilike_str'
+    STRICTSTR = 'strict_str'
+    LIKESTR = 'like_str'
+    BOOL = 'bool'
+    NUM = 'num'
+
+
+def _filter_start_str_field(field: InstrumentedAttribute,
+                            query: SelectOfScalar,
+                            operator: str,
+                            value: str
+                            ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a str like field. The filtering is done by checking if
+     the passed value starts the field.
+
+    NB: case-insensitive!
+    """
+    return query.where(func.lower(col(field)).startswith(value.lower())) if value else query
+
+
+def _filter_str_ilike_field(field: InstrumentedAttribute,
+                            query: SelectOfScalar,
+                            operator: str,
+                            value: str
+                            ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a str like field. The filtering is done by checking if
+     the passed value is contained in db values
+
+    NB: case-insensitive!
+    """
+    return query.where(col(field).ilike(f'%{value}%')) if value else query
+
+
+def _filter_str_like_field(field: InstrumentedAttribute,
+                           query: SelectOfScalar,
+                           operator: str,
+                           value: str
+                           ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a str like field. The filtering is done by checking if
+     the passed value is contained in db values
+    """
+    return query.where(col(field).contains(value)) if value else query
+
+
+def _filter_strict_str_field(field: InstrumentedAttribute,
+                             query: SelectOfScalar,
+                             operator: str,
+                             value: str
+                             ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a strict str equality matching.
+    The filtering is done by checking if the passed value is equal to one of the db values
+    """
+    return query.where(col(field) == value) if value else query
+
+
+def _filter_bool_like_field(field: InstrumentedAttribute,
+                            query: SelectOfScalar,
+                            operator: str,
+                            value: str
+                            ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a bool like field. The filtering is done by comparing
+     the passed value to db values
+    """
+    return query.where(col(field) == value) if value else query
+
+
+def _filter_num_like_field(field: InstrumentedAttribute,
+                           query: SelectOfScalar,
+                           operator: str,
+                           value: str,
+                           is_date: bool = False
+                           ) -> SelectOfScalar:
+    """
+    Add filter to the passed query for a num like (even datetime in case where is_date
+    is set to True) field. The filtering is done by comparing the passed value to db values
+    with the passed operator.
+    """
+    if not operator or not value:
+        return query
+
+    if operator == '>=':
+        query = query.where(col(field) >= (_date(value) if is_date else float(value)))
+    elif operator == '<=':
+        query = query.where(col(field) <= (_date(value) if is_date else float(value)))
+    elif operator == '!=':
+        query = query.where(col(field) != (_date(value) if is_date else float(value)))
+    elif operator == '=':
+        query = query.where(col(field) == (_date(value) if is_date else float(value)))
+    elif operator == '>':
+        query = query.where(col(field) > (_date(value) if is_date else float(value)))
+    elif operator == '<':
+        query = query.where(col(field) < (_date(value) if is_date else float(value)))
+
+    return query
+
+
+SERVER_SIDE_FILTERS: Dict[ServerSideFilter, Callable] = {
+    ServerSideFilter.STARTSTR: _filter_start_str_field,
+    ServerSideFilter.STRICTSTR: _filter_strict_str_field,
+    ServerSideFilter.LIKESTR: _filter_str_like_field,
+    ServerSideFilter.ILIKESTR: _filter_str_ilike_field,
+    ServerSideFilter.BOOL: _filter_bool_like_field,
+    ServerSideFilter.NUM: _filter_num_like_field
+}
+
+
+def _date(year: str) -> datetime:
+    """
+    Convert the passed str year to a datetime to allow filtering on datetime years.
+    """
+    return datetime(year=int(year), month=1, day=1)

ecodev_core/db_insertion.py

@@ -0,0 +1,108 @@
+"""
+Module implementing functions to insert data within the db
+"""
+from io import BytesIO
+from pathlib import Path
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import List
+from typing import Union
+
+import pandas as pd
+from fastapi import BackgroundTasks
+from fastapi import UploadFile
+from pandas import ExcelFile
+from sqlmodel import Session
+from sqlmodel import SQLModel
+from sqlmodel.sql.expression import SelectOfScalar
+
+from ecodev_core.logger import log_critical
+from ecodev_core.logger import logger_get
+from ecodev_core.pydantic_utils import CustomFrozen
+from ecodev_core.safe_utils import SimpleReturn
+
+
+log = logger_get(__name__)
+
+
+class Insertor(CustomFrozen):
+    """
+    Configuration class to insert date into the postgresql sb.
+
+    Attributes are:
+
+        - reductor: how to create or update a row in db
+        - db_schema: the default constructor of the sqlmodel based class defining the db table
+        - selector: the criteria on which to decide whether to create or update (example: only add
+          a user if a user with the same name is not already present in the db)
+        - convertor: how to convert the raw csv/excel passed by the user to json like db rows
+        - whether to insert data based on an xlsx (if true) or a csv (if false)
+    """
+    reductor: Callable[[Any, Any], Any]
+    db_schema: Callable
+    selector: Callable[[Any], SelectOfScalar]
+    convertor: Callable[[Union[pd.DataFrame, ExcelFile]], List[Dict]]
+    read_excel_file: bool = True
+
+
+def generic_insertion(df_or_xl: Union[pd.DataFrame, ExcelFile, Path],
+                      session: Session,
+                      insertor: Callable[[Union[pd.DataFrame, pd.ExcelFile], Session], None],
+                      background_tasks: Union[BackgroundTasks, None] = None):
+    """
+    Generic insertion of temperature tool related csv into db
+    """
+    try:
+        if background_tasks:
+            background_tasks.add_task(insertor, df_or_xl, session)
+        else:
+            insertor(df_or_xl, session)
+        return SimpleReturn.route_success()
+    except Exception as error:
+        log_critical(f'Something wrong happened: {error}', log)
+        return SimpleReturn.route_failure(str(error))
+
+
+async def insert_file(file: UploadFile, insertor: Insertor, session: Session) -> None:
+    """
+    Inserts an uploaded file into a database
+    """
+    df_raw = await get_raw_df(file, insertor.read_excel_file)
+    insert_data(df_raw, insertor, session)
+
+
+def insert_data(df:  Union[pd.DataFrame, ExcelFile], insertor: Insertor, session: Session) -> None:
+    """
+    Inserts a csv/df into a database
+    """
+    for row in insertor.convertor(df):
+        session.add(create_or_update(session, row, insertor))
+    session.commit()
+
+
+def create_or_update(session: Session, row: Dict, insertor: Insertor) -> SQLModel:
+    """
+    Create a new row in db if the selector insertor does not find existing row in db. Update the row
+    otherwise.
+    """
+    db_row = insertor.db_schema(**row)
+    if in_db_row := session.exec(insertor.selector(db_row)).first():
+        return insertor.reductor(in_db_row, db_row)
+    return db_row
+
+
+async def get_raw_df(file: UploadFile,
+                     read_excel_file: bool,
+                     sep: str = ',') -> Union[pd.DataFrame, ExcelFile]:
+    """
+    Retrieves the raw data from the uploaded file at pandas format
+    """
+    contents = await file.read()
+    if read_excel_file:
+        return pd.ExcelFile(contents)
+
+    buffer = BytesIO(contents)
+    raw_content = pd.read_csv(buffer, sep=sep)
+    buffer.close()
+    return raw_content