ecodev-core 0.0.48__py3-none-any.whl → 0.0.50__py3-none-any.whl

ecodev_core/__init__.py

@@ -46,6 +46,8 @@ from ecodev_core.db_upsertion import upsert_data
 from ecodev_core.db_upsertion import upsert_deletor
 from ecodev_core.db_upsertion import upsert_df_data
 from ecodev_core.db_upsertion import upsert_selector
+from ecodev_core.db_upsertion import get_sfield_columns
+from ecodev_core.db_upsertion import filter_to_sfield_dict
 from ecodev_core.deployment import Deployment
 from ecodev_core.email_sender import send_email
 from ecodev_core.enum_utils import enum_converter
@@ -103,4 +105,4 @@ __all__ = [
     'datify', 'safe_drop_columns', 'get_value', 'is_null', 'send_email', 'first_func_or_default',
     'sort_by_keys', 'sort_by_values', 'Settings', 'load_yaml_file', 'Deployment', 'Version',
     'sfield', 'field', 'upsert_df_data', 'upsert_deletor', 'get_row_versions', 'get_versions',
-    'db_to_value', 'upsert_data', 'upsert_selector']
+    'db_to_value', 'upsert_data', 'upsert_selector', 'get_sfield_columns', 'filter_to_sfield_dict']

ecodev_core/db_insertion.py

@@ -10,19 +10,21 @@ from typing import List
 from typing import Union
 
 import pandas as pd
+import progressbar
 from fastapi import BackgroundTasks
 from fastapi import UploadFile
 from pandas import ExcelFile
 from sqlmodel import Session
 from sqlmodel import SQLModel
+from sqlmodel.main import SQLModelMetaclass
 from sqlmodel.sql.expression import SelectOfScalar
 
+from ecodev_core.db_upsertion import BATCH_SIZE
 from ecodev_core.logger import log_critical
 from ecodev_core.logger import logger_get
 from ecodev_core.pydantic_utils import CustomFrozen
 from ecodev_core.safe_utils import SimpleReturn
 
-
 log = logger_get(__name__)
 
 
@@ -72,7 +74,7 @@ async def insert_file(file: UploadFile, insertor: Insertor, session: Session) ->
     insert_data(df_raw, insertor, session)
 
 
-def insert_data(df:  Union[pd.DataFrame, ExcelFile], insertor: Insertor, session: Session) -> None:
+def insert_data(df: Union[pd.DataFrame, ExcelFile], insertor: Insertor, session: Session) -> None:
     """
     Inserts a csv/df into a database
     """
@@ -81,6 +83,24 @@ def insert_data(df:  Union[pd.DataFrame, ExcelFile], insertor: Insertor, session
     session.commit()
 
 
+def insert_batch_data(data: list[dict | SQLModelMetaclass],
+                      session: Session,
+                      raw_db_schema: SQLModelMetaclass | None = None) -> None:
+    """
+    Insert the passed list of dicts (corresponding to db_schema) into db_schema db.
+    Warning: this only inserts data, without checking for pre-existence.
+    Ensure deleting the data before using it to avoid duplicates.
+    """
+    db_schema = raw_db_schema or data[0].__class__
+    batches = [data[i:i + BATCH_SIZE] for i in range(0, len(data), BATCH_SIZE)]
+
+    for batch in progressbar.progressbar(batches, redirect_stdout=False):
+        for row in batch:
+            new_object = db_schema(**row) if isinstance(row, dict) else row
+            session.add(new_object)
+        session.commit()
+
+
 def create_or_update(session: Session, row: Dict, insertor: Insertor) -> SQLModel:
     """
     Create a new row in db if the selector insertor does not find existing row in db. Update the row

ecodev_core/db_retrieval.py

@@ -22,6 +22,7 @@ from sqlmodel.sql.expression import SelectOfScalar
 from ecodev_core.db_connection import engine
 from ecodev_core.db_filters import SERVER_SIDE_FILTERS
 from ecodev_core.db_filters import ServerSideFilter
+from ecodev_core.db_upsertion import FILTER_ON
 from ecodev_core.list_utils import first_or_default
 from ecodev_core.pydantic_utils import Frozen
 

ecodev_core/db_upsertion.py

@@ -132,3 +132,34 @@ def upsert_data(data: list[dict | SQLModelMetaclass],
             else:
                 session.add(new_object)
         session.commit()
+
+
+def get_sfield_columns(db_model: SQLModelMetaclass) -> list[str]:
+    """
+    get all the columsn flagged as sfields from schema
+    Args:
+        db_model (SQLModelMetaclass): db_model
+    Returns:
+        list of str with the names of the columns
+    """
+    return [
+        x.name
+        for x in inspect(db_model).c
+        if x.info.get(FILTER_ON) is True
+    ]
+    
+    
+def filter_to_sfield_dict(row: dict | SQLModelMetaclass, 
+                           db_schema: SQLModelMetaclass | None = None) \
+                              -> dict[str, dict | SQLModelMetaclass]:
+    """
+    Returns a dict with only sfields from object
+    Args:
+        row: any object with ecodev_core field and sfield
+        db_schema (SQLModelMetaclass): db_schema. Use the schema of row if not specified
+    Returns:
+        dict
+    """
+    return {pk: getattr(row, pk)
+            for pk in get_sfield_columns(db_schema or row.__class__)}
+    

ecodev_core/version.py

@@ -130,15 +130,15 @@ def _value_to_db(value: COL_TYPES, col_type: ColType) -> str | None:
 
 def db_to_value(db_value: str | None, col_type: type | EnumType) -> COL_TYPES:
     """
-    Convert back a str version value stored to a real value (types hancled listed in ColType)
+    Convert back a str version value stored to a real value (types handled listed in ColType)
     NB: assumption that if the type is not known, this is an enum type.
     """
     if db_value is None:
         return None
-    if issubclass(int, col_type) or issubclass(str, col_type) or issubclass(float, col_type):
+    if col_type in [int, str, float]:
         return col_type(db_value)
-    if issubclass(bool, col_type):
+    if col_type == bool:
         return db_value == 'True'
-    if issubclass(datetime, col_type):
+    if col_type == datetime:
         return datetime.strptime(db_value, '%Y-%m-%d %H:%M:%S.%f')
-    return col_type[db_value]   # type: ignore[index]
+    return col_type[db_value]  # type: ignore[index]

{ecodev_core-0.0.48.dist-info → ecodev_core-0.0.50.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ecodev-core
-Version: 0.0.48
+Version: 0.0.50
 Summary: Low level sqlmodel/fastapi/pydantic building blocks
 License: MIT
 Author: Thomas Epelbaum

{ecodev_core-0.0.48.dist-info → ecodev_core-0.0.50.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ecodev_core/__init__.py,sha256=cIICQkHHfjmPIpLnpGlJ6GzGDvFT51bDeI8c1Kg3MH0,5781
+ecodev_core/__init__.py,sha256=YsvkU4bKXgYW_0jWVx0Z7_31UUQCxfADcVRxR-CcaxI,5943
 ecodev_core/app_activity.py,sha256=KBtI-35LBLPDppFB7xjxWthXQrY3Z_aGDnC-HrW8Ea0,4641
 ecodev_core/app_rights.py,sha256=RZPdDtydFqc_nFj96huKAc56BS0qS6ScKv4Kghqd6lc,726
 ecodev_core/app_user.py,sha256=r1bqA4H08x53XmxmjwyGKl_PFjYQazzBbVErdkztqeE,2947
@@ -9,9 +9,9 @@ ecodev_core/check_dependencies.py,sha256=aFn8GI4eBbuJT8RxsfhSSnlpNYYj_LPOH-tZF0E
 ecodev_core/custom_equal.py,sha256=2gRn0qpyJ8-Kw9GQSueu0nLngLrRrwyMPlP6zqPac0U,899
 ecodev_core/db_connection.py,sha256=bc5MujZ57f204wTsuNVdn1JdP-zBzkDJxHmdxBDTiNs,2286
 ecodev_core/db_filters.py,sha256=T_5JVF27UEu7sC6NOm7-W3_Y0GLfbWQO_EeTXcD2cv8,5041
-ecodev_core/db_insertion.py,sha256=RSCyAlUObbBlWJuMRX-YFY4VgtWqYLdwRqMWw--x95Y,3646
-ecodev_core/db_retrieval.py,sha256=IxyF3ZtKgACLiNFggK7boKggvMRKYDRD2IimxU4dap4,7345
-ecodev_core/db_upsertion.py,sha256=xL70gr3_NTpMDSGnakzE3mm_i3xm-dfgbD9zYCYjc64,4924
+ecodev_core/db_insertion.py,sha256=k-r798MMrW1sRb-gb8lQTxyJrb4QP5iZT8GDzCYYwlo,4544
+ecodev_core/db_retrieval.py,sha256=sCP7TDGIcTOK5gT3Inga91bE4S31HbQPw4yI22WJbss,7392
+ecodev_core/db_upsertion.py,sha256=_OdCILP1NEw1hkbvTn9ZOF5YkU9U02Fj_U3E7uY5AoI,5861
 ecodev_core/deployment.py,sha256=z8ACI00EtKknXOB8xyPwYIXTvPjIDOH9z9cBGEU0YrA,281
 ecodev_core/email_sender.py,sha256=XD7jAVXhGzvbiHqMhK9_aTEIS70Lw_CmPeAxRZGji-Y,1610
 ecodev_core/enum_utils.py,sha256=BkQ4YQ97tXBYmMcQiSIi0mbioD5CgVU79myg1BBAXuA,556
@@ -25,8 +25,8 @@ ecodev_core/read_write.py,sha256=YIGRERvFHU7vy-JIaCWAza4CPMysLRUHKJxN-ZgFmu0,120
 ecodev_core/safe_utils.py,sha256=Q8N15El1tSxZJJsy1i_1CCycuBN1_98QQoHmYJRcLIY,6904
 ecodev_core/settings.py,sha256=ARAPkXxggVUsYqSQIAgCK8C2DKSMPia1CekULn428bA,1562
 ecodev_core/sqlmodel_utils.py,sha256=t57H3QPtKRy4ujic1clMK_2L4p0yjGJLZbDjHPZ8M94,453
-ecodev_core/version.py,sha256=XvhEsbp380DF90CbGiVdepmJD2WZAIiF7fhEId0O3zQ,4480
-ecodev_core-0.0.48.dist-info/LICENSE.md,sha256=8dqVJEbwXjPWjjRKjdLMym5k9Gi8hwtrHh84sti6KIs,1068
-ecodev_core-0.0.48.dist-info/METADATA,sha256=w3-pJmtOrVNOs04Qn7TJaKUHqiYvSNhwlwz-JAhOgaw,3509
-ecodev_core-0.0.48.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-ecodev_core-0.0.48.dist-info/RECORD,,
+ecodev_core/version.py,sha256=eyIf8KkW_t-hMuYFIoy0cUlNaMewLe6i45m2HKZKh0Q,4403
+ecodev_core-0.0.50.dist-info/LICENSE.md,sha256=8dqVJEbwXjPWjjRKjdLMym5k9Gi8hwtrHh84sti6KIs,1068
+ecodev_core-0.0.50.dist-info/METADATA,sha256=JxveylReLzzqau_K4I-u8Wh4nKpE7-VgkcavjHYHX3w,3509
+ecodev_core-0.0.50.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+ecodev_core-0.0.50.dist-info/RECORD,,