ecodev-core 0.0.67__py3-none-any.whl

ecodev_core/authentication.py

@@ -0,0 +1,316 @@
+"""
+Module implementing all jwt security logic
+"""
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Union
+
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import status
+from fastapi.security import OAuth2PasswordBearer
+from jose import jwt
+from jose import JWTError
+from passlib.context import CryptContext
+from sqladmin.authentication import AuthenticationBackend
+from sqlmodel import col
+from sqlmodel import select
+from sqlmodel import Session
+from starlette.requests import Request
+from starlette.responses import RedirectResponse
+
+from ecodev_core.app_user import AppUser
+from ecodev_core.auth_configuration import ALGO
+from ecodev_core.auth_configuration import EXPIRATION_LENGTH
+from ecodev_core.auth_configuration import SECRET_KEY
+from ecodev_core.db_connection import engine
+from ecodev_core.logger import logger_get
+from ecodev_core.permissions import Permission
+from ecodev_core.pydantic_utils import Frozen
+from ecodev_core.token_banlist import TokenBanlist
+
+SCHEME = OAuth2PasswordBearer(tokenUrl='login')
+auth_router = APIRouter(tags=['authentication'])
+CONTEXT = CryptContext(schemes=['bcrypt'], deprecated='auto')
+MONITORING = 'monitoring'
+MONITORING_ERROR = 'Could not validate credentials. You need to be the monitoring user to call this'
+INVALID_USER = 'Invalid User'
+INVALID_TFA = 'Invalid TFA code'
+ADMIN_ERROR = 'Could not validate credentials. You need admin rights to call this'
+INVALID_CREDENTIALS = 'Invalid Credentials'
+REVOKED_TOKEN = 'This token has been revoked (by a logout action), please login again.'
+log = logger_get(__name__)
+
+
+class Token(Frozen):
+    """
+    Simple class for storing token value and type
+    """
+    access_token: str
+    token_type: str
+
+
+class TokenData(Frozen):
+    """
+    Simple class storing token id information
+    """
+    id: int
+
+
+def get_access_token(token: Dict[str, Any]) -> str | None:
+    """
+    Robust method to return access token or None
+    """
+    try:
+        return token.get('token', {}).get('access_token')
+    except AttributeError:
+        return None
+
+
+def get_app_services(user: AppUser, session: Session) -> List[str]:
+    """
+    Retrieve all app services the passed user has access to
+    """
+    if db_user := session.exec(select(AppUser).where(col(AppUser.id) == user.id)).first():
+        return [right.app_service for right in db_user.rights]
+    return []
+
+
+class JwtAuth(AuthenticationBackend):
+    """
+    Sqladmin security class. Implement login/logout procedure as well as the authentication check.
+    """
+
+    async def login(self, request: Request) -> bool:
+        """
+        Login procedure: factorized with the fastapi jwt logic
+        """
+        form = await request.form()
+        if token := self.authorized(form):
+            request.session.update(token)
+        return True if token else False
+
+    def authorized(self, form: Any):
+        """
+        Check that the user information contained in the form corresponds to an admin user
+        """
+        with Session(engine) as session:
+            try:
+                return self.admin_token(form, session)
+            except HTTPException:
+                return None
+
+    def admin_token(self, form: Any, session: Session) -> Union[Dict[str, str], None]:
+        """
+        Unsafe attempt to retrieve the token, only return it if admin rights
+        """
+        token = attempt_to_log(form.get('username', ''), form.get('password', ''), session)
+        return token if is_admin_user(token['access_token']) else None
+
+    async def logout(self, request: Request) -> bool:
+        """
+        Logout procedure: clears the cache
+        """
+        request.session.clear()
+        return True
+
+    async def authenticate(self, request: Request) -> Optional[RedirectResponse]:
+        """
+        Authentication procedure
+        """
+        return (token := request.session.get('access_token')) and is_admin_user(token)
+
+
+def attempt_to_log(user: str,
+                   password: str,
+                   session: Session,
+                   tfa_value: Optional[str] = None
+                   ) -> Union[Dict, HTTPException]:
+    """
+    Factorized security logic. Ensure that the user is a legit one with a valid password.
+    If so, generate a token (with or without encoded tfa_value depending on whether this argument is
+    passed as an argument). If not, returns an HTTP exception with an intelligible error message.
+
+    Attributes are:
+        user: the user as expected to be found in the AppUser db
+        password: the plain password, to be compared with the hashed one in the AppUser db
+        session: db connection
+        tfa_value: if filled, add it encoded to the generated token
+    """
+    selector = select(AppUser).where(col(AppUser.user) == user)
+    if not (db_user := session.exec(selector).first()):
+        log.warning('unauthorized user')
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=INVALID_USER)
+    if not _check_password(password, db_user.password):
+        log.warning('invalid user')
+        raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=INVALID_CREDENTIALS)
+
+    return {'access_token': _create_access_token(data={'user_id': db_user.id}, tfa_value=tfa_value),
+            'token_type': 'bearer'}
+
+
+def is_authorized_user(token: str = Depends(SCHEME)) -> bool:
+    """
+    Check if the passed token corresponds to an authorized user
+    """
+    if is_banned(token):
+        return False
+
+    try:
+        return get_current_user(token) is not None
+    except Exception:
+        return False
+
+
+def safe_get_user(token: Dict, tfa_check: bool = False) -> Union[AppUser, None]:
+    """
+    Safe method returning a user if one found given the passed token
+    """
+    try:
+        return get_user(get_access_token(token), token['tfa'] if tfa_check else None, tfa_check)
+    except (HTTPException, AttributeError):
+        return None
+
+
+def get_user(token: str = Depends(SCHEME),
+             tfa_value: Optional[str] = None,
+             tfa_check: bool = False) -> AppUser:
+    """
+    Retrieves (if it exists) the db user corresponding to the passed token
+    """
+    if is_banned(token):
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=REVOKED_TOKEN,
+                            headers={'WWW-Authenticate': 'Bearer'})
+    if user := get_current_user(token, tfa_value, tfa_check):
+        return user
+    raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_CREDENTIALS,
+                        headers={'WWW-Authenticate': 'Bearer'})
+
+
+def ban_token(token: str, session: Session) -> None:
+    """
+    Ban the passed token
+    """
+    session.add(TokenBanlist(token=token))
+    session.commit()
+
+
+def is_banned(token: str) -> bool:
+    """
+    Check if the passed token is banned.
+
+    NB: Clean the TokenBanlist table (deleting old entries) on the fly
+    """
+    with Session(engine) as session:
+        threshold = datetime.now() - timedelta(minutes=EXPIRATION_LENGTH)
+        for token_banned in session.exec(
+                select(TokenBanlist).where(TokenBanlist.created_at <= threshold)).all():
+            session.delete(token_banned)
+        session.commit()
+        return token in session.exec(select(TokenBanlist.token)).all()
+
+
+def get_current_user(token: str,
+                     tfa_value: Optional[str] = None,
+                     tfa_check: bool = False
+                     ) -> Union[AppUser, None]:
+    """
+    Retrieves (if it exists) a valid (meaning who has valid credentials) user from the db
+    """
+    token = _verify_access_token(token, tfa_value, tfa_check)
+    with Session(engine) as session:
+        return session.exec(select(AppUser).where(col(AppUser.id) == token.id)).first()
+
+
+def is_admin_user(token: str = Depends(SCHEME)) -> AppUser:
+    """
+    Retrieves (if it exists) the admin (meaning who has valid credentials) user from the db
+    """
+    if is_banned(token):
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=REVOKED_TOKEN,
+                            headers={'WWW-Authenticate': 'Bearer'})
+
+    if (user := get_current_user(token)) and user.permission == Permission.ADMIN:
+        return user
+    raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=ADMIN_ERROR,
+                        headers={'WWW-Authenticate': 'Bearer'})
+
+
+def is_monitoring_user(token: str = Depends(SCHEME)) -> AppUser:
+    """
+    Retrieves (if it exists) the monitoring user from the db
+    """
+    if is_banned(token):
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=REVOKED_TOKEN,
+                            headers={'WWW-Authenticate': 'Bearer'})
+
+    if (user := get_current_user(token)) and user.user == MONITORING:
+        return user
+    raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED,
+                        detail=MONITORING_ERROR, headers={'WWW-Authenticate': 'Bearer'})
+
+
+def upsert_new_user(token: str, user: str, password: str = '') -> None:
+    """
+    Upsert a new user if not already present in db.
+
+    NB: this method RAISES a http error if he token is invalid
+    """
+    user_id = _verify_access_token(token).id
+    with Session(engine) as session:
+        if not session.exec(select(AppUser).where(col(AppUser.id) == user_id)).first():
+            session.add(AppUser(user=user, password=password, permission=Permission.Consultant,
+                                id=user_id))
+            session.commit()
+
+
+def _create_access_token(data: Dict, tfa_value: Optional[str] = None) -> str:
+    """
+    Create an access token out of the passed data. Only called if credentials are valid
+    """
+    to_encode = data.copy()
+    expire = datetime.now(timezone.utc) + timedelta(minutes=EXPIRATION_LENGTH)
+    to_encode['exp'] = expire
+    if tfa_value:
+        to_encode['tfa'] = _hash_password(tfa_value)
+    return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGO)
+
+
+def _verify_access_token(token: str,
+                         tfa_value: Optional[str] = None,
+                         tfa_check: bool = False) -> TokenData:
+    """
+    Retrieves the token data associated to the passed token if it contains valid credential info.
+    """
+    try:
+        payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGO])
+        if tfa_check and (not tfa_value or not _check_password(tfa_value, payload.get('tfa'))):
+            raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_TFA,
+                                headers={'WWW-Authenticate': 'Bearer'})
+        if (user_id := payload.get('user_id')) is None:
+            raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_USER,
+                                headers={'WWW-Authenticate': 'Bearer'})
+        return TokenData(id=user_id)
+    except JWTError as e:
+        raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_CREDENTIALS,
+                            headers={'WWW-Authenticate': 'Bearer'}) from e
+
+
+def _hash_password(password: str) -> str:
+    """
+    Hashes the passed password (encoding).
+    """
+    return CONTEXT.hash(password)
+
+
+def _check_password(plain_password: Optional[str], hashed_password: str) -> bool:
+    """
+    Check the passed password (compare it to the passed encoded one).
+    """
+    return CONTEXT.verify(plain_password, hashed_password)

ecodev_core/backup.py

@@ -0,0 +1,105 @@
+"""
+Module implementing backup mechanism on a ftp server.
+"""
+import tarfile
+from datetime import datetime
+from pathlib import Path
+from subprocess import PIPE
+from subprocess import Popen
+from subprocess import run
+from subprocess import STDOUT
+from typing import List
+
+from pydantic_settings import BaseSettings
+from pydantic_settings import SettingsConfigDict
+
+from ecodev_core.db_connection import DB_URL
+from ecodev_core.logger import logger_get
+from ecodev_core.settings import SETTINGS
+
+log = logger_get(__name__)
+
+
+class BackUpSettings(BaseSettings):
+    """
+    Settings class used to connect to the ftp server backup
+    """
+    backup_username: str = ''
+    backup_password: str = ''
+    backup_url: str = ''
+    model_config = SettingsConfigDict(env_file='.env')
+
+
+BCK, SETTINGS_BCK = BackUpSettings(), SETTINGS.backup  # type: ignore[attr-defined]
+_USER = SETTINGS_BCK.backup_username or BCK.backup_username
+_PASSWD = SETTINGS_BCK.backup_password or BCK.backup_password
+_URL = SETTINGS_BCK.backup_url or BCK.backup_url
+BACKUP_URL = f'ftp://{_USER}:{_PASSWD}@{_URL}'
+
+
+def backup(backed_folder: Path, nb_saves: int = 5, additional_id: str = 'default') -> None:
+    """
+    Backup db and backed_folder: write the dump/tar on the backup server and erase old copies
+    """
+    timestamp = datetime.now().strftime('%Y_%m_%d_%Hh_%Mmn_%Ss')
+    _backup_db(Path.cwd() / f'{additional_id}_db.{timestamp}.dump', nb_saves)
+    _backup_files(backed_folder, Path.cwd() / f'{additional_id}_files.{timestamp}.tgz', nb_saves)
+
+
+def retrieve_most_recent_backup(name: str = 'default_files') -> None:
+    """
+    Retrieve from backup server the most recent backup of the name family.
+    """
+    output = run(['lftp', '-c', f'open {BACKUP_URL}; ls'], capture_output=True, text=True)
+    all_backups = sorted([x.split(' ')[-1] for x in output.stdout.splitlines() if name in x])
+    log.info(f'most recent backup {all_backups[-1]}')
+    run(['lftp', '-c', f'open {BACKUP_URL}; get {all_backups[-1]}'])
+    return None
+
+
+def _backup_db(db_dump_path: Path, nb_saves: int) -> None:
+    """
+    Pg_dump of DB_URL db andwrite on the backup server
+    """
+    process = Popen(['pg_dump', f'--dbname={DB_URL}', '-f', db_dump_path.name],
+                    stdout=PIPE, stderr=STDOUT, cwd=db_dump_path.parent)
+    if not (process_output := process.communicate()[0]):
+        _backup_content(db_dump_path, nb_saves)
+    else:
+        log.critical(f'something went wrong : {process_output}')
+
+
+
+def _backup_files(backed_folder: Path, backup_file: Path, nb_saves: int) -> None:
+    """
+    Zip backed_folder and write on the backup server
+    """
+    with tarfile.open(backup_file, 'w:gz') as tar:
+        tar.add(backed_folder, arcname=backed_folder.name)
+    _backup_content(backup_file, nb_saves)
+
+
+def _backup_content(file_to_backup: Path, nb_saves: int) -> None:
+    """
+    Write file_to_backup on the backup server and delete versions so as to keep only nb_saves copies
+    """
+    log.warning(f'Transferring backup to server: {file_to_backup}')
+    run(['lftp', '-c', f'open {BACKUP_URL}; put {file_to_backup}'])
+    backups_to_delete = _get_old_backups(file_to_backup, nb_saves)
+    log.info(f'deleting remote backups {backups_to_delete}')
+    for to_rm in backups_to_delete:
+        run(['lftp', '-c', f'open {BACKUP_URL}; rm {to_rm}'], capture_output=True, text=True)
+    log.info(f'deleting local {file_to_backup}')
+    file_to_backup.unlink()
+
+
+def _get_old_backups(file_to_backup: Path, nb_saves: int) -> List[str]:
+    """
+    Retrieve old versions of file_to_backup in order to erase them (more than nb_saves ago)
+    """
+    output = run(['lftp', '-c', f'open {BACKUP_URL}; ls'], capture_output=True, text=True)
+    filename_base = file_to_backup.name.split('.')[0]
+    all_backups = sorted([x.split(' ')[-1]
+                         for x in output.stdout.splitlines() if filename_base in x])
+    log.info(f'existing remote backups {all_backups}')
+    return all_backups[:-nb_saves]

ecodev_core/check_dependencies.py

@@ -0,0 +1,179 @@
+"""
+Module computing and checking high level dependencies in the coe (based on pydeps)
+"""
+from pathlib import Path
+from subprocess import run
+from typing import Dict
+from typing import Iterator
+from typing import List
+
+from ecodev_core.logger import logger_get
+
+
+CONF_FILE = 'dependencies.json'
+Dependency = Dict[str, List[str]]
+log = logger_get(__name__)
+
+
+def check_dependencies(code_base: Path,  theoretical_deps: Path):
+    """
+    hook for preserving the pre established solution dependencies.
+    Compare regroupment of module dependencies matrix to a pre-computed matrix stored
+     in theoretical_deps. Computation done on code_base.
+    """
+    dependencies = _get_current_dependencies(_valid_modules(code_base), code_base,  code_base.name)
+    allowed_dependencies = _get_allowed_dependencies(theoretical_deps)
+    if not (ok_deps := _test_dependency(allowed_dependencies, dependencies)):
+        log.error('you changed high level solution dependencies. Intended?')
+    return ok_deps
+
+
+def compute_dependencies(code_base: Path, output_folder: Path, plot: bool = True):
+    """
+    Given a code base, compute the dependencies between its high level modules.
+     Store in output_folder the dependency matrix in txt format and the png of the dependencies
+    """
+    code_folder = code_base.name
+    modules = _valid_modules(code_base)
+
+    deps: Dependency = _get_current_dependencies(modules, code_base, code_folder)
+
+    for mod, mod_deps in deps.items():
+        with open(output_folder / f'{mod}.py', 'w') as f_stream:
+            f_stream.writelines([f'from .{other_module} import to\n' for other_module in mod_deps])
+
+    with open(output_folder / '__init__.py', 'w') as f_stream:
+        f_stream.writelines([])
+
+    with open(output_folder / f'dependencies_{code_folder}.txt', 'w') as f_stream:
+        f_stream.writelines([f'{dependency}\n' for dependency in _get_dep_matrix(modules, deps)])
+
+    if plot:
+        run(['pydeps', '.', '-T', 'png', '--no-show', '--rmprefix',
+             f'{output_folder.name}.'], cwd=output_folder)
+
+
+def _test_dependency(allowed_deps: Dependency, dependencies: Dependency) -> bool:
+    """
+    For each modules stored in a dependencies.json file, check whether the current
+    module dependencies are the same as the config ones.
+    """
+    for module in dependencies:
+        for dep in allowed_deps[module]:
+            if dep not in dependencies[module]:
+                log.error(f'{module} no longer imported in {dep}. Intended ?')
+        for dep in dependencies[module]:
+            if dep not in allowed_deps[module]:
+                log.error(f'{dep} now imported in {module}. Intended ?')
+        for dep in dependencies[module]:
+            if module in dependencies[dep] and dep != module:
+                log.error(f'Circular ref created between {module} and {dep}.')
+    return dependencies == allowed_deps
+
+
+def _get_allowed_dependencies(config_path: Path) -> Dependency:
+    """
+    Given the pre established dependency file path, compute
+    the pre established modules and their dependencies seen as an adjacency dict.
+    All the values of a given key are its dependencies.
+    The keys and the values of the dictionary take their labels in
+    the pre established module list.
+    """
+    raw_lines = list(_safe_read_lines(config_path))
+    raw_matrix = [raw_dependency.split(',') for raw_dependency in raw_lines][1:]
+    modules = [raw_dependency[0] for raw_dependency in raw_matrix]
+    module_dependencies: Dict[str, List[str]] = {
+        module: [modules[idx_other_module] for idx_other_module in range(len(modules))
+                 if raw_matrix[idx_module][idx_other_module + 1] == 'Yes']
+        for idx_module, module in enumerate(modules)
+    }
+
+    return module_dependencies
+
+
+def _get_current_dependencies(modules: List[str],
+                              code_base: Path,
+                              code_folder: str) -> Dependency:
+    """
+    Given the pre established modules, the code_base directory and the relative path of the code
+    directory wrt code_folder, compute the pre current dependencies as an adjacency dict.
+    All the values of a given key are its dependencies. The keys and the values of
+    the dictionary take their labels in the pre established module list.
+    """
+    module_dependencies: Dependency = {}
+    for module in modules:
+        module_dependencies[module] = []
+        module_dir = code_base / module
+        for other_module in modules:
+            if other_module in module_dependencies[module]:
+                break
+            for py_file in _get_recursively_all_files_in_dir(module_dir, 'py'):
+                if _depends_on_module(module_dir / py_file, other_module, code_folder):
+                    module_dependencies[module].append(other_module)
+                    break
+
+    return module_dependencies
+
+
+def _depends_on_module(file: Path, module: str, code_folder: str) -> bool:
+    """
+     check if a reference to module is in the imports of python_file
+    """
+    return any(
+        (f'from {code_folder}.{module}' in line and 'import' in line)
+        or (line.startswith(f'import {code_folder}.{module}.'))
+        for line in _safe_read_lines(file)
+    )
+
+
+def _safe_read_lines(filename: Path) -> Iterator[str]:
+    """
+    read all lines in file, erase the final special \n character
+    """
+    with open(filename, 'r') as f:
+        lines = f.readlines()
+    yield from [line.strip() for line in lines]
+
+
+def _get_recursively_all_files_in_dir(code_folder: Path, extension: str) -> Iterator[Path]:
+    """
+    Given a folder, recursively return all files of the given extension in the folder
+    """
+    yield from code_folder.glob(f'**/*.{extension}')
+
+
+def _valid_folder(folder: Path):
+    """
+    Return True if folder is a python regroupment of module to be considered for dependency analysis
+    """
+    return (
+        folder.is_dir()
+        and not folder.name.startswith('.')
+        and not folder.name.startswith('_')
+        and folder.name != 'data'
+    )
+
+
+def _valid_modules(root_folder: Path):
+    """
+    Retrieve valid solution module (found at the base level of root_folder)
+    """
+    return sorted([folder.name for folder in root_folder.iterdir() if _valid_folder(folder)])
+
+
+def _get_dep_matrix(modules: List[str], deps: Dependency) -> List[str]:
+    """
+    Retrieve the dependency matrix of the inspected solution in txt format
+    """
+    dependencies = [f'module x depends on,{",".join(modules)}']
+    dependencies.extend(f'{module},' + ','.join([_depends_on(module, other_module, deps)
+                                                 for other_module in modules])for module in modules)
+
+    return dependencies
+
+
+def _depends_on(module, other_module, deps):
+    """
+    Write correct input in the dependency matrix ("Yes" if other_module is in deps of module key)
+    """
+    return 'Yes' if other_module in deps[module] else 'No'

ecodev_core/custom_equal.py

@@ -0,0 +1,27 @@
+"""
+Module comparing whether two elements are both None or both not None and equals
+"""
+from typing import Optional
+
+import numpy as np
+
+
+def custom_equal(element_1: Optional[object], element_2: Optional[object], element_type: type):
+    """
+    Compare whether two elements are both None or both not None and equals (same type/same value)
+
+    Args:
+        element_1: the first element of the comparison
+        element_2:  the second element of the comparison
+        element_type:  the expected element type for both elements
+
+    Returns:
+        True if both None or both not None and equals (same type/same value)
+    """
+    if element_1 is None:
+        return element_2 is None
+
+    if not isinstance(element_1, element_type) or not isinstance(element_2, element_type):
+        return False
+
+    return np.isclose(element_1, element_2) if element_type == float else element_1 == element_2