ecodev-core 0.0.49__tar.gz → 0.0.50__tar.gz

{ecodev_core-0.0.49 → ecodev_core-0.0.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ecodev-core
-Version: 0.0.49
+Version: 0.0.50
 Summary: Low level sqlmodel/fastapi/pydantic building blocks
 License: MIT
 Author: Thomas Epelbaum

{ecodev_core-0.0.49 → ecodev_core-0.0.50}/ecodev_core/__init__.py

@@ -46,6 +46,8 @@ from ecodev_core.db_upsertion import upsert_data
 from ecodev_core.db_upsertion import upsert_deletor
 from ecodev_core.db_upsertion import upsert_df_data
 from ecodev_core.db_upsertion import upsert_selector
+from ecodev_core.db_upsertion import get_sfield_columns
+from ecodev_core.db_upsertion import filter_to_sfield_dict
 from ecodev_core.deployment import Deployment
 from ecodev_core.email_sender import send_email
 from ecodev_core.enum_utils import enum_converter
@@ -103,4 +105,4 @@ __all__ = [
     'datify', 'safe_drop_columns', 'get_value', 'is_null', 'send_email', 'first_func_or_default',
     'sort_by_keys', 'sort_by_values', 'Settings', 'load_yaml_file', 'Deployment', 'Version',
     'sfield', 'field', 'upsert_df_data', 'upsert_deletor', 'get_row_versions', 'get_versions',
-    'db_to_value', 'upsert_data', 'upsert_selector']
+    'db_to_value', 'upsert_data', 'upsert_selector', 'get_sfield_columns', 'filter_to_sfield_dict']

{ecodev_core-0.0.49 → ecodev_core-0.0.50}/ecodev_core/db_insertion.py

@@ -10,19 +10,21 @@ from typing import List
 from typing import Union
 
 import pandas as pd
+import progressbar
 from fastapi import BackgroundTasks
 from fastapi import UploadFile
 from pandas import ExcelFile
 from sqlmodel import Session
 from sqlmodel import SQLModel
+from sqlmodel.main import SQLModelMetaclass
 from sqlmodel.sql.expression import SelectOfScalar
 
+from ecodev_core.db_upsertion import BATCH_SIZE
 from ecodev_core.logger import log_critical
 from ecodev_core.logger import logger_get
 from ecodev_core.pydantic_utils import CustomFrozen
 from ecodev_core.safe_utils import SimpleReturn
 
-
 log = logger_get(__name__)
 
 
@@ -72,7 +74,7 @@ async def insert_file(file: UploadFile, insertor: Insertor, session: Session) ->
     insert_data(df_raw, insertor, session)
 
 
-def insert_data(df:  Union[pd.DataFrame, ExcelFile], insertor: Insertor, session: Session) -> None:
+def insert_data(df: Union[pd.DataFrame, ExcelFile], insertor: Insertor, session: Session) -> None:
     """
     Inserts a csv/df into a database
     """
@@ -81,6 +83,24 @@ def insert_data(df:  Union[pd.DataFrame, ExcelFile], insertor: Insertor, session
     session.commit()
 
 
+def insert_batch_data(data: list[dict | SQLModelMetaclass],
+                      session: Session,
+                      raw_db_schema: SQLModelMetaclass | None = None) -> None:
+    """
+    Insert the passed list of dicts (corresponding to db_schema) into db_schema db.
+    Warning: this only inserts data, without checking for pre-existence.
+    Ensure deleting the data before using it to avoid duplicates.
+    """
+    db_schema = raw_db_schema or data[0].__class__
+    batches = [data[i:i + BATCH_SIZE] for i in range(0, len(data), BATCH_SIZE)]
+
+    for batch in progressbar.progressbar(batches, redirect_stdout=False):
+        for row in batch:
+            new_object = db_schema(**row) if isinstance(row, dict) else row
+            session.add(new_object)
+        session.commit()
+
+
 def create_or_update(session: Session, row: Dict, insertor: Insertor) -> SQLModel:
     """
     Create a new row in db if the selector insertor does not find existing row in db. Update the row

{ecodev_core-0.0.49 → ecodev_core-0.0.50}/ecodev_core/db_retrieval.py

@@ -22,6 +22,7 @@ from sqlmodel.sql.expression import SelectOfScalar
 from ecodev_core.db_connection import engine
 from ecodev_core.db_filters import SERVER_SIDE_FILTERS
 from ecodev_core.db_filters import ServerSideFilter
+from ecodev_core.db_upsertion import FILTER_ON
 from ecodev_core.list_utils import first_or_default
 from ecodev_core.pydantic_utils import Frozen
 

{ecodev_core-0.0.49 → ecodev_core-0.0.50}/ecodev_core/db_upsertion.py

@@ -132,3 +132,34 @@ def upsert_data(data: list[dict | SQLModelMetaclass],
             else:
                 session.add(new_object)
         session.commit()
+
+
+def get_sfield_columns(db_model: SQLModelMetaclass) -> list[str]:
+    """
+    get all the columsn flagged as sfields from schema
+    Args:
+        db_model (SQLModelMetaclass): db_model
+    Returns:
+        list of str with the names of the columns
+    """
+    return [
+        x.name
+        for x in inspect(db_model).c
+        if x.info.get(FILTER_ON) is True
+    ]
+    
+    
+def filter_to_sfield_dict(row: dict | SQLModelMetaclass, 
+                           db_schema: SQLModelMetaclass | None = None) \
+                              -> dict[str, dict | SQLModelMetaclass]:
+    """
+    Returns a dict with only sfields from object
+    Args:
+        row: any object with ecodev_core field and sfield
+        db_schema (SQLModelMetaclass): db_schema. Use the schema of row if not specified
+    Returns:
+        dict
+    """
+    return {pk: getattr(row, pk)
+            for pk in get_sfield_columns(db_schema or row.__class__)}
+    

{ecodev_core-0.0.49 → ecodev_core-0.0.50}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ecodev-core"
-version = "0.0.49"
+version = "0.0.50"
 description = "Low level sqlmodel/fastapi/pydantic building blocks"
 authors = ["Thomas Epelbaum <tomepel@gmail.com>",
 					 "Olivier Gabriel <olivier.gabriel.geom@gmail.com>",