pjdev-sqlmodel 3.4.2a3__tar.gz → 3.5.0a1__tar.gz

{pjdev_sqlmodel-3.4.2a3 → pjdev_sqlmodel-3.5.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pjdev-sqlmodel
-Version: 3.4.2a3
+Version: 3.5.0a1
 Project-URL: Documentation, https://gitlab.purplejay.net/keystone/python
 Project-URL: Issues, https://gitlab.purplejay.net/keystone/python/issues
 Project-URL: Source, https://gitlab.purplejay.net/keystone/python

{pjdev_sqlmodel-3.4.2a3 → pjdev_sqlmodel-3.5.0a1}/src/pjdev_sqlmodel/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Chris O'Neill <chris@purplejay.io>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "3.4.2a3"
+__version__ = "3.5.0a1"

{pjdev_sqlmodel-3.4.2a3 → pjdev_sqlmodel-3.5.0a1}/src/pjdev_sqlmodel/service.py

@@ -42,6 +42,12 @@ def configure_single_context(settings: SqlModelSettings, tables: List[Type[Model
     __ctx.engine = initialize_engine(settings, tables)
 
 
+def get_engine() -> Engine:
+    if __ctx.engine is None:
+        raise ValueError("No engine has been initialized")
+    return __ctx.engine
+
+
 @contextmanager
 def session_context() -> SQLModelSession:
     with SQLModelSession(__ctx.engine) as session:

{pjdev_sqlmodel-3.4.2a3 → pjdev_sqlmodel-3.5.0a1}/src/pjdev_sqlmodel/utilities.py

@@ -7,7 +7,8 @@ from pydantic import ValidationError, BaseModel
 import pandas as pd
 from openpyxl.reader.excel import load_workbook
 from loguru import logger
-
+from sqlalchemy import Engine
+from pjdev_sqlmodel import service
 from pjdev_sqlmodel.db_models import ModelBase
 from pjdev_sqlmodel.service import session_context
 
@@ -123,13 +124,72 @@ def __convert_to_models(
     return data
 
 
-def __read_csv(file, cols) -> pd.DataFrame:
-    return pd.read_csv(file, engine="pyarrow", usecols=cols, na_filter=False)
+def load_raw_csv_data(
+    filename: Path,
+    table_name: str,
+    data_type_map: Optional[Dict[str, Type]] = None,
+    engine: Optional[Engine] = None,
+) -> None:
+    df = __read_csv(filename, data_type_map=data_type_map)
+
+    if df is None:
+        return
+
+    engine = service.get_engine() if engine is None else engine
+    df.to_sql(
+        table_name,
+        con=engine,
+        if_exists="replace",
+        index_label="row_id",
+        dtype=data_type_map,
+    )
+
+
+def load_raw_excel_data(
+    filename: Path,
+    table_name: str,
+    header_ndx: int = 0,
+    data_type_map: Optional[Dict[str, Type]] = None,
+    engine: Optional[Engine] = None,
+) -> None:
+    df = __read_excel(filename, header_ndx=header_ndx, data_type_map=data_type_map)
+
+    if df is None:
+        return
+
+    engine = service.get_engine() if engine is None else engine
+    df.to_sql(
+        table_name,
+        con=engine,
+        if_exists="replace",
+        index_label="row_id",
+        dtype=data_type_map,
+    )
+
+
+def __read_csv(
+    file: Path | str,
+    cols: Optional[List[str]] = None,
+    data_type_map: Optional[Dict[str, Type]] = None,
+) -> pd.DataFrame:
+    return pd.read_csv(
+        file, engine="pyarrow", usecols=cols, na_filter=False, dtype=data_type_map
+    )
 
 
-def __read_excel(file, cols, header_ndx) -> pd.DataFrame:
+def __read_excel(
+    file: Path | str,
+    cols: Optional[List[str]] = None,
+    header_ndx: int = 0,
+    data_type_map: Optional[Dict[str, Type]] = None,
+) -> pd.DataFrame:
     return pd.read_excel(
-        io=file, usecols=cols, na_filter=False, header=header_ndx, engine="calamine"
+        io=file,
+        usecols=cols,
+        na_filter=False,
+        header=header_ndx,
+        engine="calamine",
+        dtype=data_type_map,
     )