ecodev-core 0.0.35__py3-none-any.whl → 0.0.36__py3-none-any.whl

ecodev_core/__init__.py

@@ -40,6 +40,7 @@ from ecodev_core.db_insertion import get_raw_df
 from ecodev_core.db_retrieval import count_rows
 from ecodev_core.db_retrieval import get_rows
 from ecodev_core.db_retrieval import ServerSideField
+from ecodev_core.deployment import Deployment
 from ecodev_core.email_sender import send_email
 from ecodev_core.enum_utils import enum_converter
 from ecodev_core.list_utils import first_func_or_default
@@ -65,6 +66,7 @@ from ecodev_core.pydantic_utils import CustomFrozen
 from ecodev_core.pydantic_utils import Frozen
 from ecodev_core.pydantic_utils import OrmFrozen
 from ecodev_core.read_write import load_json_file
+from ecodev_core.read_write import load_yaml_file
 from ecodev_core.read_write import make_dir
 from ecodev_core.read_write import write_json_file
 from ecodev_core.safe_utils import boolify
@@ -75,6 +77,7 @@ from ecodev_core.safe_utils import safe_clt
 from ecodev_core.safe_utils import SafeTestCase
 from ecodev_core.safe_utils import SimpleReturn
 from ecodev_core.safe_utils import stringify
+from ecodev_core.settings import Settings
 
 __all__ = [
     'AUTH', 'Token', 'get_app_services', 'attempt_to_log', 'get_current_user', 'is_admin_user',
@@ -89,4 +92,4 @@ __all__ = [
     'fastapi_monitor', 'dash_monitor', 'is_monitoring_user', 'get_recent_activities', 'select_user',
     'get_access_token', 'safe_get_user', 'backup', 'group_by', 'get_excelfile', 'upsert_new_user',
     'datify', 'safe_drop_columns', 'get_value', 'is_null', 'send_email', 'first_func_or_default',
-    'sort_by_keys', 'sort_by_values']
+    'sort_by_keys', 'sort_by_values', 'Settings', 'load_yaml_file', 'Deployment']

ecodev_core/deployment.py

@@ -0,0 +1,16 @@
+"""
+Module implementing all types of deployment
+"""
+from enum import Enum
+from enum import unique
+
+
+@unique
+class Deployment(str, Enum):
+    """
+    Enum listing all types of deployment
+    """
+    LOCAL = 'local'
+    NON_PROD = 'nonprod'
+    PREPROD = 'preprod'
+    PROD = 'prod'

ecodev_core/es_connection.py

@@ -0,0 +1,76 @@
+"""
+Module implementing a connection to an elastic search instance, and basic insertion/retrieval.
+"""
+from typing import Any
+from typing import Union
+
+import progressbar
+from elasticsearch import Elasticsearch
+from elasticsearch import helpers
+from pydantic_settings import BaseSettings
+from pydantic_settings import SettingsConfigDict
+
+from ecodev_core.logger import logger_get
+
+ES_CLIENT: Union[Elasticsearch, None] = None
+log = logger_get(__name__)
+ES_BATCH_SIZE = 5000
+
+
+class ESAuth(BaseSettings):
+    """
+    Simple ES authentication configuration class
+    """
+    host: str
+    user: str
+    password: str
+    port: int
+    index: str
+    model_config = SettingsConfigDict(env_file='.env', env_prefix='ES_')
+
+
+ES_AUTH = ESAuth()  # type: ignore
+
+
+def get_es_client():
+    """
+    Get the elasticsearch client
+    """
+    global ES_CLIENT
+
+    if ES_CLIENT is None:
+        ES_CLIENT = Elasticsearch(f'http://{ES_AUTH.host}:{ES_AUTH.port}/',
+                                  basic_auth=[ES_AUTH.user, ES_AUTH.password])
+
+    return ES_CLIENT
+
+
+def create_es_index(body: dict) -> None:
+    """
+    create an es index
+    """
+    client = get_es_client()
+    try:
+        client.indices.delete(index=ES_AUTH.index)
+    except Exception:
+        pass
+    client.indices.create(index=ES_AUTH.index, body=body)
+    log.info(f'index {ES_AUTH.index} created')
+
+
+def insert_es_fields(operations: list[dict], batch_size: int = ES_BATCH_SIZE) -> None:
+    """
+    Generic es insertion
+    """
+    client = get_es_client()
+    batches = [list(operations)[i:i + batch_size] for i in range(0, len(operations), batch_size)]
+    log.info('indexing fields')
+    for batch in progressbar.progressbar(batches, redirect_stdout=False):
+        helpers.bulk(client, batch, index=ES_AUTH.index)
+
+
+def retrieve_es_fields(body: dict[str, Any]) -> list[dict]:
+    """
+    Core call to the elasticsearch index
+    """
+    return get_es_client().search(index=ES_AUTH.index, body=body)

ecodev_core/list_utils.py

@@ -118,3 +118,10 @@ def first_transformed_or_default(sequence: List[Any], transformation: Callable)
      or default value if no non-trivial transformed elements are found.
     """
     return next((fx for elt in sequence if (fx := transformation(elt)) is not None), None)
+
+
+def dict_to_class(data: dict):
+    """
+    Convert a (possibly nested) dictionary to a class.
+    """
+    return {k: type(k, (), dict_to_class(v)) if isinstance(v, dict) else v for k, v in data.items()}

ecodev_core/logger.py

@@ -5,6 +5,9 @@ import logging
 import sys
 import traceback
 
+LIBS = ['azure', 'passlib', 'trimesh', 'fiona',
+        'urllib3', 'botocore', 'boto', 'boto3', 's3transfer']
+
 
 def log_critical(message: str, logger):
     """
@@ -18,11 +21,24 @@ def logger_get(name, level=logging.DEBUG):
     """
     Main method called by all other modules to log
     """
+    logging.basicConfig(level=level, stream=sys.stdout)
+    for lib in LIBS:
+        _safe_log_setter(lib)
     logger = logging.getLogger(name)
     config_log(logger, level, MyFormatter())
     return logger
 
 
+def _safe_log_setter(lib: str) -> None:
+    """
+    Safe logger. ERROR level not to be swamped by verbose library info.
+    """
+    try:
+        logging.getLogger(lib).setLevel(logging.ERROR)
+    except Exception:
+        pass
+
+
 class MyFormatter(logging.Formatter):
     """
     Formatter to print %(filename)s:%(funcName)s:%(lineno)d on 24 characters

ecodev_core/read_write.py

@@ -8,6 +8,8 @@ from typing import Dict
 from typing import List
 from typing import Union
 
+import yaml
+
 
 def write_json_file(json_data: Union[Dict, List], file_path: Path):
     """
@@ -28,6 +30,16 @@ def load_json_file(file_path: Path):
     return loaded_json
 
 
+def load_yaml_file(file_path: Path):
+    """
+    Load a yaml file at file_path location
+    """
+    with open(file_path) as file:
+        loaded_yaml = yaml.safe_load(file)
+
+    return loaded_yaml
+
+
 def make_dir(directory: Path):
     """
     Helper that create the directory "directory" if it doesn't exist yet

ecodev_core/settings.py

@@ -0,0 +1,43 @@
+"""
+Module defining a dynamic setting class
+"""
+from pathlib import Path
+
+from pydantic.utils import deep_update
+from pydantic_settings import BaseSettings
+from pydantic_settings import SettingsConfigDict
+
+from ecodev_core.deployment import Deployment
+from ecodev_core.list_utils import dict_to_class
+from ecodev_core.read_write import load_yaml_file
+
+
+class DeploymentSetting(BaseSettings):
+    """
+    Settings class used to load the deployment type from environment variables.
+    """
+    environment: str = 'local'
+    model_config = SettingsConfigDict(env_file='.env')
+
+
+DEPLOYMENT = Deployment(DeploymentSetting().environment.lower())
+
+
+class Settings:
+    """
+    Dynami setting class, loading yaml configuration from config file, possibly overwriting some of
+    this configuration with additional information coming from a secret file.
+    """
+
+    def __init__(self, base_path: Path = Path('/app'), deployment: Deployment = DEPLOYMENT):
+        """
+        Dynamically setting Settings attributes, doing so recursively. Attributes are loaded
+         from config file, possibly overwriting some of this configuration with additional
+         information coming from a secret file.
+        """
+        self.deployment = deployment
+        data = load_yaml_file(base_path / 'config' / f'{deployment.value}.yaml')
+        if (secrets_file := base_path / 'secrets' / f'{deployment.value}.yaml').exists():
+            data = deep_update(data, load_yaml_file(secrets_file))
+        for k, v in dict_to_class(data).items():
+            setattr(self, k, v)

{ecodev_core-0.0.35.dist-info → ecodev_core-0.0.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ecodev-core
-Version: 0.0.35
+Version: 0.0.36
 Summary: Low level sqlmodel/fastapi/pydantic building blocks
 License: MIT
 Author: Thomas Epelbaum
@@ -33,6 +33,7 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
+Requires-Dist: elasticsearch (>=8.14.0,<9.0.0)
 Requires-Dist: fastapi (>=0,<1)
 Requires-Dist: httpx (>=0,<1)
 Requires-Dist: numpy (>=1,<2)
@@ -40,6 +41,7 @@ Requires-Dist: openpyxl (>=3,<4)
 Requires-Dist: orjson (>=3.10.5,<4.0.0)
 Requires-Dist: pandas (>=2,<3)
 Requires-Dist: passlib[bcyrypt] (>=1,<2)
+Requires-Dist: progressbar2 (>=4.4.2,<5.0.0)
 Requires-Dist: psycopg2-binary (>=2,<3)
 Requires-Dist: pydantic (==2.9.2)
 Requires-Dist: pydantic-settings (>=2,<3)

{ecodev_core-0.0.35.dist-info → ecodev_core-0.0.36.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ecodev_core/__init__.py,sha256=BcZLnkIVxOlVVXubWqKAAMRbUyFQt8v6JgfawC_6cPY,4970
+ecodev_core/__init__.py,sha256=gIm_-wiXh8YyM-7xSQ07uNFNlPyAngmw45Sd062CT-k,5152
 ecodev_core/app_activity.py,sha256=_rU5uPfttHxXX5IaCuTA7K9We5w2qluJ3Xpf6i12HhY,3763
 ecodev_core/app_rights.py,sha256=RZPdDtydFqc_nFj96huKAc56BS0qS6ScKv4Kghqd6lc,726
 ecodev_core/app_user.py,sha256=r1bqA4H08x53XmxmjwyGKl_PFjYQazzBbVErdkztqeE,2947
@@ -11,17 +11,20 @@ ecodev_core/db_connection.py,sha256=bc5MujZ57f204wTsuNVdn1JdP-zBzkDJxHmdxBDTiNs,
 ecodev_core/db_filters.py,sha256=T_5JVF27UEu7sC6NOm7-W3_Y0GLfbWQO_EeTXcD2cv8,5041
 ecodev_core/db_insertion.py,sha256=RSCyAlUObbBlWJuMRX-YFY4VgtWqYLdwRqMWw--x95Y,3646
 ecodev_core/db_retrieval.py,sha256=IxyF3ZtKgACLiNFggK7boKggvMRKYDRD2IimxU4dap4,7345
+ecodev_core/deployment.py,sha256=z8ACI00EtKknXOB8xyPwYIXTvPjIDOH9z9cBGEU0YrA,281
 ecodev_core/email_sender.py,sha256=XD7jAVXhGzvbiHqMhK9_aTEIS70Lw_CmPeAxRZGji-Y,1610
 ecodev_core/enum_utils.py,sha256=BkQ4YQ97tXBYmMcQiSIi0mbioD5CgVU79myg1BBAXuA,556
-ecodev_core/list_utils.py,sha256=s56vvhylMFzi6A76YvOIi4d5QYml73n_TmYNq1wM_P4,4427
-ecodev_core/logger.py,sha256=AWGGNZz12Ql0JDq1wCJQxwxK2syiczEBftmuJkjbAPw,3149
+ecodev_core/es_connection.py,sha256=3z8KWF9yUzCW4xC9nhLlIcoW3Gw6m19MvH4Z6nxy7R4,1967
+ecodev_core/list_utils.py,sha256=OltBlhp0COYPD8tc5eruqQq5nlVA7vp7fPYhTwXu6Ww,4632
+ecodev_core/logger.py,sha256=_v63vq7jh72RRkzajHB9QkRfEPcwCMNcLh7maLeKwRg,3596
 ecodev_core/pandas_utils.py,sha256=Juc6gvPnoBiSVF2SR6_vfMi5W-QEkY3fnpo5ROB1L9s,2191
 ecodev_core/permissions.py,sha256=WAx-ilMu8LlQp2sjJVdkhNQieytEaEm8577ZF1HWeTY,502
 ecodev_core/pydantic_utils.py,sha256=e3GH50JmcpTmd2UgrB94QSwWOlOCW3WIlVdyX9C4T-U,741
-ecodev_core/read_write.py,sha256=auJ5bBJTVGkLRkiP_vZxVCX64B0Y-9qpsaDhovHmbas,996
+ecodev_core/read_write.py,sha256=YIGRERvFHU7vy-JIaCWAza4CPMysLRUHKJxN-ZgFmu0,1208
 ecodev_core/safe_utils.py,sha256=JCfxo6fcznjsL-XHNJ1TKo1UvfJB83WT5jpTFmtJwsE,6160
+ecodev_core/settings.py,sha256=lgC0GFi3nH9FxUlf2azzzKNT9-i2thl4UMKLxQHL9Xg,1559
 ecodev_core/sqlmodel_utils.py,sha256=t57H3QPtKRy4ujic1clMK_2L4p0yjGJLZbDjHPZ8M94,453
-ecodev_core-0.0.35.dist-info/LICENSE.md,sha256=jebQDe1ib9LAODuNvcSoo2CoqS6P0_q8--mMTICh_kI,1074
-ecodev_core-0.0.35.dist-info/METADATA,sha256=iiIe974766YSdMZpBmNJP3IPyPlszfeAM0mWr8qfzOI,3386
-ecodev_core-0.0.35.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-ecodev_core-0.0.35.dist-info/RECORD,,
+ecodev_core-0.0.36.dist-info/LICENSE.md,sha256=jebQDe1ib9LAODuNvcSoo2CoqS6P0_q8--mMTICh_kI,1074
+ecodev_core-0.0.36.dist-info/METADATA,sha256=9OpeiKjWbClyXKtL2jirg96XdzVZRH9wiOs_QL5SEvc,3478
+ecodev_core-0.0.36.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+ecodev_core-0.0.36.dist-info/RECORD,,