pjdev-sqlmodel 3.4.2__tar.gz → 3.4.2a3__tar.gz

{pjdev_sqlmodel-3.4.2 → pjdev_sqlmodel-3.4.2a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pjdev-sqlmodel
-Version: 3.4.2
+Version: 3.4.2a3
 Project-URL: Documentation, https://gitlab.purplejay.net/keystone/python
 Project-URL: Issues, https://gitlab.purplejay.net/keystone/python/issues
 Project-URL: Source, https://gitlab.purplejay.net/keystone/python
@@ -16,7 +16,9 @@ Requires-Python: >=3.12
 Requires-Dist: loguru
 Requires-Dist: openpyxl>=3.1.5
 Requires-Dist: pandas>=2.2.2
+Requires-Dist: pyarrow>=17.0.0
 Requires-Dist: pydantic-settings>=2.3.4
+Requires-Dist: python-calamine>=0.2.3
 Requires-Dist: sqlmodel>=0.0.19
 Description-Content-Type: text/markdown
 

{pjdev_sqlmodel-3.4.2 → pjdev_sqlmodel-3.4.2a3}/pyproject.toml

@@ -24,6 +24,8 @@ dependencies = [
   "pydantic-settings>=2.3.4",
   "sqlmodel>=0.0.19",
   "pandas>=2.2.2",
+  "pyarrow>=17.0.0",
+  "python-calamine>=0.2.3",
   "openpyxl>=3.1.5",
   "loguru"
 ]

{pjdev_sqlmodel-3.4.2 → pjdev_sqlmodel-3.4.2a3}/src/pjdev_sqlmodel/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Chris O'Neill <chris@purplejay.io>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "3.4.2"
+__version__ = "3.4.2a3"

{pjdev_sqlmodel-3.4.2 → pjdev_sqlmodel-3.4.2a3}/src/pjdev_sqlmodel/utilities.py

@@ -58,14 +58,8 @@ def load_csv_data(model_type: Type[T], data_files: List[Path]) -> None:
 
     data: List[model_type] = []
     for file in filtered_files:
-        df = pd.read_csv(file, usecols=cols, na_filter=False)
-
-        try:
-            for _, row in df.iterrows():
-                d = model_type.model_validate(row.to_dict())
-                data.append(d)
-        except ValidationError as e:
-            logger.error(f"Error when parsing {file.name}: {e}")
+        df = __read_csv(file, cols)
+        data.extend(__convert_to_models(file.name, df, model_type))
 
     with session_context() as session:
         session.add_all(data)
@@ -106,14 +100,8 @@ def load_excel_data(
         )
 
     for file in filtered_files:
-        df = pd.read_excel(io=file, usecols=cols, na_filter=False, header=header_ndx)
-
-        try:
-            for _, row in df.iterrows():
-                d = model_type.model_validate(row.to_dict())
-                data.append(d)
-        except ValidationError as e:
-            logger.error(f"Error when parsing {file.name}: {e}")
+        df = __read_excel(file, cols, header_ndx)
+        data.extend(__convert_to_models(file.name, df, model_type))
 
     with session_context() as session:
         session.add_all(data)
@@ -121,6 +109,30 @@ def load_excel_data(
     logger.info("Loaded {} rows for {} table".format(len(data), model_type.__name__))
 
 
+def __convert_to_models(
+    filename: str, df: pd.DataFrame, model_type: Type[T]
+) -> List[T]:
+    data: List[model_type] = []
+    try:
+        for _, row in df.iterrows():
+            d = model_type.model_validate(row.to_dict())
+            data.append(d)
+    except ValidationError as e:
+        logger.error(f"Error when parsing {filename}: {e}")
+
+    return data
+
+
+def __read_csv(file, cols) -> pd.DataFrame:
+    return pd.read_csv(file, engine="pyarrow", usecols=cols, na_filter=False)
+
+
+def __read_excel(file, cols, header_ndx) -> pd.DataFrame:
+    return pd.read_excel(
+        io=file, usecols=cols, na_filter=False, header=header_ndx, engine="calamine"
+    )
+
+
 def convert_to_csv(
     data: List[BaseModel],
     col_mapping_tuple: Tuple[List[str], Dict[str, str], List[str]],