pjdev-sqlmodel 4.6.3__tar.gz → 4.6.5__tar.gz

{pjdev_sqlmodel-4.6.3 → pjdev_sqlmodel-4.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pjdev-sqlmodel
-Version: 4.6.3
+Version: 4.6.5
 Project-URL: Documentation, https://gitlab.purplejay.net/keystone/python
 Project-URL: Issues, https://gitlab.purplejay.net/keystone/python/issues
 Project-URL: Source, https://gitlab.purplejay.net/keystone/python

{pjdev_sqlmodel-4.6.3 → pjdev_sqlmodel-4.6.5}/src/pjdev_sqlmodel/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2024-present Chris O'Neill <chris@purplejay.io>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "4.6.3"
+__version__ = "4.6.5"

{pjdev_sqlmodel-4.6.3 → pjdev_sqlmodel-4.6.5}/src/pjdev_sqlmodel/utilities.py

@@ -55,13 +55,16 @@ def get_excel_columns(file_path, header_ndx: int = 1, col_range: Optional[str] =
 
 def load_csv_data(model_type: Type[T], data_files: List[Path]) -> None:
     fields = model_type.model_fields.keys()
+    # Check validation alias first, then alias, and fallback to field name
     cols = [
-        f
-        if model_type.model_fields.get(f).alias is None
-        else model_type.model_fields.get(f).alias
+        model_type.model_fields[f].validation_alias
+        or model_type.model_fields[f].alias
+        or f
         for f in fields
         if f != "row_id"
     ]
+
+    # Filter only CSV files that match the required columns
     filtered_files = [
         f
         for f in data_files
@@ -80,41 +83,28 @@ def load_csv_data(model_type: Type[T], data_files: List[Path]) -> None:
     logger.info("Loaded {} rows for {} table".format(len(data), model_type.__name__))
 
 
-def load_excel_data(
-        model_type: Type[T],
-        data_files: List[Path],
-        header_ndx: int = 0,
-        sheet_name: str | int = 0,
-        col_range: Optional[str] = None,
-) -> None:
+def load_excel_data(model_type: Type[T], data_files: List[Path], sheet_name: str | int = 0) -> None:
     fields = model_type.model_fields.keys()
+    # Check validation alias first, then alias, and fallback to field name
     cols = [
-        f
-        if model_type.model_fields.get(f).alias is None
-        else model_type.model_fields.get(f).alias
+        model_type.model_fields[f].validation_alias
+        or model_type.model_fields[f].alias
+        or f
         for f in fields
         if f != "row_id"
     ]
 
+    # Filter only Excel files that match the required columns
     filtered_files = [
         f
         for f in data_files
-        if f.name.endswith(".xlsx")
-           and len(
-            set(cols).difference(set(get_excel_columns(f, header_ndx + 1, col_range, sheet_name=sheet_name)))
-        )
-           == 0
+        if f.name.endswith((".xls", ".xlsx"))
+           and len(set(cols).difference(set(get_excel_columns(f, sheet_name=sheet_name)))) == 0
     ]
 
     data: List[model_type] = []
-
-    if len(filtered_files) == 0:
-        raise Exception(
-            f"No files found that matched the schema for {model_type.__name__}"
-        )
-
     for file in filtered_files:
-        df = __read_excel(file=file, cols=cols, header_ndx=header_ndx, sheet_name=sheet_name)
+        df = __read_excel(file, cols, sheet_name=sheet_name)
         data.extend(__convert_to_models(file.name, df, model_type))
 
     with session_context() as session:
@@ -251,33 +241,53 @@ def export_to_sheet(InputTable: type[BaseModel], wb: Workbook, sheet_name: str,
         ws.column_dimensions[column_letter].width = adjusted_width
 
 
-def __read_csv(
-        file: Path | str,
-        cols: Optional[List[str]] = None,
-        data_type_map: Optional[Dict[str, Type]] = None,
-) -> pd.DataFrame:
-    return pd.read_csv(
-        file, engine="pyarrow", usecols=cols, na_filter=False, dtype=data_type_map
+def __read_csv(file: Path, cols: List[str]) -> pd.DataFrame:
+    df = pd.read_csv(
+        file,
+        usecols=cols,
+        na_filter=True,
+        na_values=[""],
+        keep_default_na=False,
     )
 
-
-def __read_excel(
-        file: Path | str,
-        sheet_name: str | int = 0,
-        cols: Optional[List[str]] = None,
-        header_ndx: int = 0,
-        data_type_map: Optional[Dict[str, Type]] = None,
-) -> pd.DataFrame:
-    return pd.read_excel(
-        io=file,
+    df = df.convert_dtypes()
+    for col in df.select_dtypes(include="number").columns:
+        s = df[col]
+        if s.isna().all():
+            continue
+        non_na = s.dropna()
+        if (non_na % 1 == 0).all():
+            df[col] = s.astype("Int64")
+        else:
+            df[col] = s.astype("Float64")
+    df = df.convert_dtypes()
+
+    return df
+
+def __read_excel(file: Path, cols: List[str], sheet_name: str | int = 0) -> pd.DataFrame:
+    df = pd.read_excel(
+        file,
         usecols=cols,
-        sheet_name=sheet_name,
-        na_filter=False,
-        header=header_ndx,
-        engine="calamine",
-        dtype=data_type_map,
+        na_filter=True,
+        na_values=[""],
+        keep_default_na=False,
+        sheet_name = sheet_name
     )
 
+    df = df.convert_dtypes()
+    for col in df.select_dtypes(include="number").columns:
+        s = df[col]
+        if s.isna().all():
+            continue
+        non_na = s.dropna()
+        if (non_na % 1 == 0).all():
+            df[col] = s.astype("Int64")
+        else:
+            df[col] = s.astype("Float64")
+    df = df.convert_dtypes()
+
+    return df
+
 
 def convert_to_csv(
         data: List[BaseModel],