ecodev-core 0.0.49__tar.gz → 0.0.51__tar.gz

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ecodev-core
-Version: 0.0.49
+Version: 0.0.51
 Summary: Low level sqlmodel/fastapi/pydantic building blocks
 License: MIT
 Author: Thomas Epelbaum

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/ecodev_core/__init__.py

@@ -46,6 +46,8 @@ from ecodev_core.db_upsertion import upsert_data
 from ecodev_core.db_upsertion import upsert_deletor
 from ecodev_core.db_upsertion import upsert_df_data
 from ecodev_core.db_upsertion import upsert_selector
+from ecodev_core.db_upsertion import get_sfield_columns
+from ecodev_core.db_upsertion import filter_to_sfield_dict
 from ecodev_core.deployment import Deployment
 from ecodev_core.email_sender import send_email
 from ecodev_core.enum_utils import enum_converter
@@ -103,4 +105,4 @@ __all__ = [
     'datify', 'safe_drop_columns', 'get_value', 'is_null', 'send_email', 'first_func_or_default',
     'sort_by_keys', 'sort_by_values', 'Settings', 'load_yaml_file', 'Deployment', 'Version',
     'sfield', 'field', 'upsert_df_data', 'upsert_deletor', 'get_row_versions', 'get_versions',
-    'db_to_value', 'upsert_data', 'upsert_selector']
+    'db_to_value', 'upsert_data', 'upsert_selector', 'get_sfield_columns', 'filter_to_sfield_dict']

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/ecodev_core/auth_configuration.py

@@ -4,6 +4,8 @@ Module implementing authentication configuration.
 from pydantic_settings import BaseSettings
 from pydantic_settings import SettingsConfigDict
 
+from ecodev_core.settings import SETTINGS
+
 
 class AuthenticationConfiguration(BaseSettings):
     """
@@ -16,3 +18,7 @@ class AuthenticationConfiguration(BaseSettings):
 
 
 AUTH = AuthenticationConfiguration()
+SETTINGS_AUTH = SETTINGS.authentication  # type: ignore[attr-defined]
+SECRET_KEY = SETTINGS_AUTH.secret_key or AUTH.secret_key
+ALGO = SETTINGS_AUTH.algorithm or AUTH.algorithm
+EXPIRATION_LENGTH = SETTINGS_AUTH.access_token_expire_minutes or AUTH.access_token_expire_minutes

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/ecodev_core/authentication.py

@@ -26,7 +26,9 @@ from starlette.requests import Request
 from starlette.responses import RedirectResponse
 
 from ecodev_core.app_user import AppUser
-from ecodev_core.auth_configuration import AUTH
+from ecodev_core.auth_configuration import ALGO
+from ecodev_core.auth_configuration import EXPIRATION_LENGTH
+from ecodev_core.auth_configuration import SECRET_KEY
 from ecodev_core.db_connection import engine
 from ecodev_core.logger import logger_get
 from ecodev_core.permissions import Permission
@@ -235,11 +237,11 @@ def _create_access_token(data: Dict, tfa_value: Optional[str] = None) -> str:
     Create an access token out of the passed data. Only called if credentials are valid
     """
     to_encode = data.copy()
-    expire = datetime.now(timezone.utc) + timedelta(minutes=AUTH.access_token_expire_minutes)
+    expire = datetime.now(timezone.utc) + timedelta(minutes=EXPIRATION_LENGTH)
     to_encode['exp'] = expire
     if tfa_value:
         to_encode['tfa'] = _hash_password(tfa_value)
-    return jwt.encode(to_encode, AUTH.secret_key, algorithm=AUTH.algorithm)
+    return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGO)
 
 
 def _verify_access_token(token: str,
@@ -249,7 +251,7 @@ def _verify_access_token(token: str,
     Retrieves the token data associated to the passed token if it contains valid credential info.
     """
     try:
-        payload = jwt.decode(token, AUTH.secret_key, algorithms=[AUTH.algorithm])
+        payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGO])
         if tfa_check and (not tfa_value or not _check_password(tfa_value, payload.get('tfa'))):
             raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_TFA,
                                 headers={'WWW-Authenticate': 'Bearer'})

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/ecodev_core/backup.py

@@ -15,7 +15,7 @@ from pydantic_settings import SettingsConfigDict
 
 from ecodev_core.db_connection import DB_URL
 from ecodev_core.logger import logger_get
-
+from ecodev_core.settings import SETTINGS
 
 log = logger_get(__name__)
 
@@ -30,8 +30,11 @@ class BackUpSettings(BaseSettings):
     model_config = SettingsConfigDict(env_file='.env')
 
 
-BCK = BackUpSettings()
-BACKUP_URL = f'ftp://{BCK.backup_username}:{BCK.backup_password}@{BCK.backup_url}'
+BCK, SETTINGS_BCK = BackUpSettings(), SETTINGS.backup  # type: ignore[attr-defined]
+_USER = SETTINGS_BCK.backup_username or BCK.backup_username
+_PASSWD = SETTINGS_BCK.backup_password or BCK.backup_password
+_URL = SETTINGS_BCK.backup_url or BCK.backup_url
+BACKUP_URL = f'ftp://{_USER}:{_PASSWD}@{_URL}'
 
 
 def backup(backed_folder: Path, nb_saves: int = 5, additional_id: str = 'default') -> None:

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/ecodev_core/db_connection.py

@@ -14,6 +14,7 @@ from sqlmodel import Session
 from sqlmodel import SQLModel
 
 from ecodev_core.logger import logger_get
+from ecodev_core.settings import SETTINGS
 
 log = logger_get(__name__)
 
@@ -30,9 +31,11 @@ class DbSettings(BaseSettings):
     model_config = SettingsConfigDict(env_file='.env')
 
 
-DB = DbSettings()
-_PASSWORD = quote(DB.db_password, safe='')
-DB_URL = f'postgresql://{DB.db_username}:{_PASSWORD}@{DB.db_host}:{DB.db_port}/{DB.db_name}'
+DB, SETTINGS_DB = DbSettings(), SETTINGS.database  # type: ignore[attr-defined]
+_PASSWORD = quote(SETTINGS_DB.db_password or DB.db_password, safe='')
+_USER, _HOST = SETTINGS_DB.db_username or DB.db_username, SETTINGS_DB.db_host or DB.db_host
+_PORT, _NAME = SETTINGS_DB.db_port or DB.db_port, SETTINGS_DB.db_name or DB.db_name
+DB_URL = f'postgresql://{_USER}:{_PASSWORD}@{_HOST}:{_PORT}/{_NAME}'
 engine = create_engine(DB_URL)
 
 

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/ecodev_core/db_insertion.py

@@ -10,19 +10,21 @@ from typing import List
 from typing import Union
 
 import pandas as pd
+import progressbar
 from fastapi import BackgroundTasks
 from fastapi import UploadFile
 from pandas import ExcelFile
 from sqlmodel import Session
 from sqlmodel import SQLModel
+from sqlmodel.main import SQLModelMetaclass
 from sqlmodel.sql.expression import SelectOfScalar
 
+from ecodev_core.db_upsertion import BATCH_SIZE
 from ecodev_core.logger import log_critical
 from ecodev_core.logger import logger_get
 from ecodev_core.pydantic_utils import CustomFrozen
 from ecodev_core.safe_utils import SimpleReturn
 
-
 log = logger_get(__name__)
 
 
@@ -72,7 +74,7 @@ async def insert_file(file: UploadFile, insertor: Insertor, session: Session) ->
     insert_data(df_raw, insertor, session)
 
 
-def insert_data(df:  Union[pd.DataFrame, ExcelFile], insertor: Insertor, session: Session) -> None:
+def insert_data(df: Union[pd.DataFrame, ExcelFile], insertor: Insertor, session: Session) -> None:
     """
     Inserts a csv/df into a database
     """
@@ -81,6 +83,24 @@ def insert_data(df:  Union[pd.DataFrame, ExcelFile], insertor: Insertor, session
     session.commit()
 
 
+def insert_batch_data(data: list[dict | SQLModelMetaclass],
+                      session: Session,
+                      raw_db_schema: SQLModelMetaclass | None = None) -> None:
+    """
+    Insert the passed list of dicts (corresponding to db_schema) into db_schema db.
+    Warning: this only inserts data, without checking for pre-existence.
+    Ensure deleting the data before using it to avoid duplicates.
+    """
+    db_schema = raw_db_schema or data[0].__class__
+    batches = [data[i:i + BATCH_SIZE] for i in range(0, len(data), BATCH_SIZE)]
+
+    for batch in progressbar.progressbar(batches, redirect_stdout=False):
+        for row in batch:
+            new_object = db_schema(**row) if isinstance(row, dict) else row
+            session.add(new_object)
+        session.commit()
+
+
 def create_or_update(session: Session, row: Dict, insertor: Insertor) -> SQLModel:
     """
     Create a new row in db if the selector insertor does not find existing row in db. Update the row

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/ecodev_core/db_retrieval.py

@@ -22,6 +22,7 @@ from sqlmodel.sql.expression import SelectOfScalar
 from ecodev_core.db_connection import engine
 from ecodev_core.db_filters import SERVER_SIDE_FILTERS
 from ecodev_core.db_filters import ServerSideFilter
+from ecodev_core.db_upsertion import FILTER_ON
 from ecodev_core.list_utils import first_or_default
 from ecodev_core.pydantic_utils import Frozen
 

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/ecodev_core/db_upsertion.py

@@ -132,3 +132,34 @@ def upsert_data(data: list[dict | SQLModelMetaclass],
             else:
                 session.add(new_object)
         session.commit()
+
+
+def get_sfield_columns(db_model: SQLModelMetaclass) -> list[str]:
+    """
+    get all the columsn flagged as sfields from schema
+    Args:
+        db_model (SQLModelMetaclass): db_model
+    Returns:
+        list of str with the names of the columns
+    """
+    return [
+        x.name
+        for x in inspect(db_model).c
+        if x.info.get(FILTER_ON) is True
+    ]
+    
+    
+def filter_to_sfield_dict(row: dict | SQLModelMetaclass, 
+                           db_schema: SQLModelMetaclass | None = None) \
+                              -> dict[str, dict | SQLModelMetaclass]:
+    """
+    Returns a dict with only sfields from object
+    Args:
+        row: any object with ecodev_core field and sfield
+        db_schema (SQLModelMetaclass): db_schema. Use the schema of row if not specified
+    Returns:
+        dict
+    """
+    return {pk: getattr(row, pk)
+            for pk in get_sfield_columns(db_schema or row.__class__)}
+    

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/ecodev_core/email_sender.py

@@ -11,6 +11,8 @@ from ssl import create_default_context
 from pydantic_settings import BaseSettings
 from pydantic_settings import SettingsConfigDict
 
+from ecodev_core.settings import SETTINGS
+
 
 class EmailAuth(BaseSettings):
     """
@@ -19,10 +21,15 @@ class EmailAuth(BaseSettings):
     email_smtp: str = ''
     email_sender: str = ''
     email_password: str = ''
+    email_port: int = 587
     model_config = SettingsConfigDict(env_file='.env')
 
 
-EMAIL_AUTH = EmailAuth()
+EMAIL_AUTH, EMAIL_SETTINGS = EmailAuth(), SETTINGS.smtp  # type: ignore[attr-defined]
+_SENDER = EMAIL_SETTINGS.email_sender or EMAIL_AUTH.email_sender
+_SMTP = EMAIL_SETTINGS.email_smtp or EMAIL_AUTH.email_smtp
+_PASSWD = EMAIL_SETTINGS.email_password or EMAIL_AUTH.email_password
+_PORT = EMAIL_SETTINGS.email_port or EMAIL_AUTH.email_port
 
 
 def send_email(email: str, body: str, topic: str, images: dict[str, Path] | None = None) -> None:
@@ -36,7 +43,7 @@ def send_email(email: str, body: str, topic: str, images: dict[str, Path] | None
         - images: if any, the Dict of image tags:image paths to incorporate in the email
     """
     em = MIMEMultipart('related')
-    em['From'] = EMAIL_AUTH.email_sender
+    em['From'] = _SENDER
     em['To'] = email
     em['Subject'] = topic
     em.attach(MIMEText(body, 'html'))
@@ -46,8 +53,8 @@ def send_email(email: str, body: str, topic: str, images: dict[str, Path] | None
         img.add_header('Content-ID', f'<{tag}>')
         em.attach(img)
 
-    with SMTP(EMAIL_AUTH.email_smtp, 587) as server:
+    with SMTP(_SMTP, _PORT) as server:
         server.ehlo()
         server.starttls(context=create_default_context())
-        server.login(EMAIL_AUTH.email_sender, EMAIL_AUTH.email_password)
-        server.sendmail(EMAIL_AUTH.email_sender, email, em.as_string())
+        server.login(_SENDER, _PASSWD)
+        server.sendmail(_SENDER, email, em.as_string())

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/ecodev_core/es_connection.py

@@ -11,6 +11,7 @@ from pydantic_settings import BaseSettings
 from pydantic_settings import SettingsConfigDict
 
 from ecodev_core.logger import logger_get
+from ecodev_core.settings import SETTINGS
 
 ES_CLIENT: Union[Elasticsearch, None] = None
 log = logger_get(__name__)
@@ -21,15 +22,18 @@ class ESAuth(BaseSettings):
     """
     Simple ES authentication configuration class
     """
-    host: str
-    user: str
-    password: str
-    port: int
-    index: str
+    host: str = ''
+    user: str = ''
+    password: str = ''
+    port: int = 9200
+    index: str = ''
     model_config = SettingsConfigDict(env_file='.env', env_prefix='ES_')
 
 
-ES_AUTH = ESAuth()  # type: ignore
+ES_AUTH, ES_SETTINGS = ESAuth(), SETTINGS.elastic_search  # type: ignore[attr-defined]
+_HOST, _PORT = ES_SETTINGS.host or ES_AUTH.host,  ES_SETTINGS.port or ES_AUTH.port
+_USER, _PASSWD = ES_SETTINGS.user or ES_AUTH.user, ES_SETTINGS.password or ES_AUTH.password
+_INDEX = ES_SETTINGS.index or ES_AUTH.index
 
 
 def get_es_client():
@@ -39,8 +43,7 @@ def get_es_client():
     global ES_CLIENT
 
     if ES_CLIENT is None:
-        ES_CLIENT = Elasticsearch(f'http://{ES_AUTH.host}:{ES_AUTH.port}/',
-                                  basic_auth=[ES_AUTH.user, ES_AUTH.password])
+        ES_CLIENT = Elasticsearch(f'http://{_HOST}:{_PORT}/', basic_auth=[_USER, _PASSWD])
 
     return ES_CLIENT
 
@@ -51,11 +54,11 @@ def create_es_index(body: dict) -> None:
     """
     client = get_es_client()
     try:
-        client.indices.delete(index=ES_AUTH.index)
+        client.indices.delete(index=_INDEX)
     except Exception:
         pass
-    client.indices.create(index=ES_AUTH.index, body=body)
-    log.info(f'index {ES_AUTH.index} created')
+    client.indices.create(index=_INDEX, body=body)
+    log.info(f'index {_INDEX} created')
 
 
 def insert_es_fields(operations: list[dict], batch_size: int = ES_BATCH_SIZE) -> None:
@@ -66,11 +69,11 @@ def insert_es_fields(operations: list[dict], batch_size: int = ES_BATCH_SIZE) ->
     batches = [list(operations)[i:i + batch_size] for i in range(0, len(operations), batch_size)]
     log.info('indexing fields')
     for batch in progressbar.progressbar(batches, redirect_stdout=False):
-        helpers.bulk(client, batch, index=ES_AUTH.index)
+        helpers.bulk(client, batch, index=_INDEX)
 
 
 def retrieve_es_fields(body: dict[str, Any]) -> list[dict]:
     """
     Core call to the elasticsearch index
     """
-    return get_es_client().search(index=ES_AUTH.index, body=body)
+    return get_es_client().search(index=_INDEX, body=body)

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/ecodev_core/settings.py

@@ -1,6 +1,7 @@
 """
 Module defining a dynamic setting class
 """
+from contextlib import suppress
 from pathlib import Path
 
 from pydantic.v1.utils import deep_update
@@ -9,18 +10,24 @@ from pydantic_settings import SettingsConfigDict
 
 from ecodev_core.deployment import Deployment
 from ecodev_core.list_utils import dict_to_class
+from ecodev_core.logger import logger_get
 from ecodev_core.read_write import load_yaml_file
 
+log = logger_get(__name__)
+
 
 class DeploymentSetting(BaseSettings):
     """
     Settings class used to load the deployment type from environment variables.
     """
     environment: str = 'local'
+    base_path: str = '/app'
     model_config = SettingsConfigDict(env_file='.env')
 
 
-DEPLOYMENT = Deployment(DeploymentSetting().environment.lower())
+DEPLOYMENT_SETTINGS = DeploymentSetting()
+DEPLOYMENT = Deployment(DEPLOYMENT_SETTINGS.environment.lower())
+BASE_PATH = Path(DEPLOYMENT_SETTINGS.base_path)
 
 
 class Settings:
@@ -29,15 +36,21 @@ class Settings:
     this configuration with additional information coming from a secret file.
     """
 
-    def __init__(self, base_path: Path = Path('/app'), deployment: Deployment = DEPLOYMENT):
+    def __init__(self, base_path: Path = BASE_PATH, deployment: Deployment = DEPLOYMENT):
         """
         Dynamically setting Settings attributes, doing so recursively. Attributes are loaded
          from config file, possibly overwriting some of this configuration with additional
          information coming from a secret file.
         """
         self.deployment = deployment
-        data = load_yaml_file(base_path / 'config' / f'{deployment.value}.yaml')
-        if (secrets_file := base_path / 'secrets' / f'{deployment.value}.yaml').exists():
-            data = deep_update(data, load_yaml_file(secrets_file))
-        for k, v in dict_to_class(data).items():
-            setattr(self, k, v)
+
+        with suppress(FileNotFoundError):
+            log.info((base_path / 'config' / f'{deployment.value}.yaml').exists())
+            data = load_yaml_file(base_path / 'config' / f'{deployment.value}.yaml')
+            if (secrets_file := base_path / 'secrets' / f'{deployment.value}.yaml').exists():
+                data = deep_update(data, load_yaml_file(secrets_file))
+            for k, v in dict_to_class(data).items():
+                setattr(self, k, v)
+
+
+SETTINGS = Settings()

{ecodev_core-0.0.49 → ecodev_core-0.0.51}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ecodev-core"
-version = "0.0.49"
+version = "0.0.51"
 description = "Low level sqlmodel/fastapi/pydantic building blocks"
 authors = ["Thomas Epelbaum <tomepel@gmail.com>",
 					 "Olivier Gabriel <olivier.gabriel.geom@gmail.com>",