datakraken 1.7.2__tar.gz → 1.7.4__tar.gz

{datakraken-1.7.2 → datakraken-1.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datakraken
-Version: 1.7.2
+Version: 1.7.4
 Summary: Kraken is a convenience package that orchestrates data extraction by integrating SQL with Python, while managing safe storage and recall of sensitive connection credentials.
 Project-URL: Repository, https://github.com/SETT-Centre-Data-and-AI/Kraken
 Author: Cai Davis, Michael George, Faizan Hemotra

{datakraken-1.7.2 → datakraken-1.7.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datakraken"
-version = "1.7.2"
+version = "1.7.4"
 description = "Kraken is a convenience package that orchestrates data extraction by integrating SQL with Python, while managing safe storage and recall of sensitive connection credentials."
 authors = [
   { name = "Cai Davis" },

{datakraken-1.7.2 → datakraken-1.7.4}/src/kraken/analysis/data_manipulation.py

@@ -4,7 +4,7 @@ import numpy as np
 import pandas as pd
 from IPython.display import display
 from pandas import DataFrame
-from pandas.api.types import is_numeric_dtype
+from pandas.api.types import is_numeric_dtype, is_object_dtype, is_string_dtype
 
 from kraken.classes.data_types import StatsPack
 from kraken.exceptions import DuplicateColumnError
@@ -100,7 +100,7 @@ def examine(
             if is_numeric_dtype(df[column])
             else False
         )
-        if df[column].dtype == "object":
+        if is_object_dtype(df[column]) or is_string_dtype(df[column]):
             max_length = df[column].dropna().astype(str).str.len().max()
             max_length = int(max_length) if pd.notna(max_length) else None
         else:
@@ -205,7 +205,7 @@ def examine(
 def date_converter_df(
     df: DataFrame, columns: str | list | None = None, readouts: bool = True
 ) -> DataFrame:
-    """Checks DataFrame for missed date columns identified as type 'object' by pandas, and converts those found in place.
+    """Checks object/string columns for missed dates and converts those found in place.
 
     Args:
         df (DataFrame): DataFrame
@@ -233,7 +233,11 @@ def date_converter_df(
         columns = [columns]
 
     for col in columns:
-        if col is None or isinstance(df[col], DataFrame) or df[col].dtype != "object":
+        if (
+            col is None
+            or isinstance(df[col], DataFrame)
+            or not (is_object_dtype(df[col]) or is_string_dtype(df[col]))
+        ):
             pass
         else:
             values = df[col].map(

{datakraken-1.7.2 → datakraken-1.7.4}/src/kraken/connection/connector.py

@@ -3,17 +3,17 @@ from abc import ABC, abstractmethod
 from collections.abc import Sequence
 from typing import Any, Generic, Literal, Protocol, Type, TypeVar, runtime_checkable
 
+import pandas as pd
 import pyodbc  # type: ignore[import-not-found]
 import sqlalchemy as sa
 from pandas import DataFrame
 from sqlalchemy.engine.url import URL
 
-from kraken.analysis.data_manipulation import check_df_integers as clean_dataframe
 from kraken.connection.support import (
+    DEFAULT_ARRAYSIZE,
+    _check_execution_error,
     _compile_pyodbc_connection_string,
     _split_pyodbc_connection_string,
-    _check_execution_error,
-    DEFAULT_ARRAYSIZE,
 )
 from kraken.credentials.credentials import Credentials
 from kraken.credentials.helpers import fetch_credentials
@@ -266,8 +266,9 @@ class Connector(ABC, Generic[TConn, TCursor]):
             -   close (bool): If True, closes the connection after executing. Defaults to None, using the Connector default (Connector.autoclose).
             -   batch_size (int): Downloads data in batches. Uses current Connector.batch_size if None. Set to 0 to
                 download all data without batching.
-            -   clean_df (bool): Cleans DataFrame after pandas generation, including converting float64 to Int64 if
-                applicable (recommended).
+            -   clean_df (bool): Preserves fetched scalar types during DataFrame construction,
+                using nullable integer/boolean dtypes where needed without a float intermediate.
+                Float values remain floats. False uses ordinary pandas inference.
             -   check_column_duplicates (bool): Checks DataFrame for duplicate column names and throws warning.
             -   header_indent (str): Prefix each readout with an indent (e.g. ' -> ')
             -   progress (bool): If True, activates query progress bar. Switching to False may save a little
@@ -432,7 +433,32 @@ class Connector(ABC, Generic[TConn, TCursor]):
         if df is None:
             return
 
-        df = clean_dataframe(df) if clean_df else df
+        if clean_df:
+            for position in range(len(df.columns)):
+                column = df.iloc[:, position]
+                raw_values = column.to_numpy()
+                present = pd.notna(raw_values)
+                nullable = not present.all()
+                values = raw_values[present] if nullable else raw_values
+                scalar_type = pd.api.types.infer_dtype(values, skipna=True)
+                converted: pd.Series[Any]
+                if scalar_type == "integer":
+                    dtype: Literal["Int64", "int64", "UInt64", "uint64"]
+                    minimum, maximum = values.min(), values.max()
+                    if -(2**63) <= minimum and maximum < 2**63:
+                        dtype = "Int64" if nullable else "int64"
+                    elif 0 <= minimum and maximum < 2**64:
+                        dtype = "UInt64" if nullable else "uint64"
+                    else:
+                        continue
+                    converted = column.astype(dtype)
+                elif scalar_type == "boolean":
+                    converted = column.astype("boolean" if nullable else "bool")
+                elif scalar_type in ("mixed-integer", "mixed-integer-float"):
+                    continue
+                else:
+                    converted = column.infer_objects()
+                df.isetitem(position, converted.array)
         if check_column_duplicates and df.columns.has_duplicates:
             dupe_cols = df.columns[df.columns.duplicated()].to_list()
             readout.warn(
@@ -495,7 +521,9 @@ class Connector(ABC, Generic[TConn, TCursor]):
                     progress.update_header(
                         header=f"{indent}Query: {query_name} Processing  |"
                     )
-                    df = DataFrame(data=rows, columns=columns)
+                    df = DataFrame(
+                        data=rows, columns=columns, dtype=object if clean_df else None
+                    )
                     df = self._process_dataframe(
                         df=df,
                         clean_df=clean_df,
@@ -626,8 +654,9 @@ class SaConnector(
             close (bool): If True, closes the connection after executing. Defaults to None, using the Connector default (Connector.autoclose).
             batch_size (int): Downloads data in batches. Uses current Connector.batch_size if None. Set to 0 to
                 download all data without batching.
-            clean_df (bool): Cleans DataFrame after pandas generation, including converting float64 to Int64 if
-                applicable (recommended).
+            clean_df (bool): Preserves fetched scalar types during DataFrame construction,
+                using nullable integer/boolean dtypes where needed without a float intermediate.
+                Float values remain floats. False uses ordinary pandas inference.
             check_column_duplicates (bool): Checks DataFrame for duplicate column names and throws warning.
             header_indent (str): Prefix each readout with an indent (e.g. ' -> ')
             progress (bool): If True, activates query progress bar. Switching to False may save a little

{datakraken-1.7.2 → datakraken-1.7.4}/src/kraken/exporting/result_export.py

@@ -42,10 +42,16 @@ def export_results(
     **kwargs: Any,
 ) -> None:
     """
-    Exports results or a dataframe to a supported extension type (csv or xlsx).
+    Exports results or a dataframe to CSV, XLSX or Parquet without its index.
     For filetypes supporting multiple dataframes (such as xlsx), the filename argument can be provided.
     For filetypes supporting single dataframes only (such as csv) the filename provided will be appended to the prefix (if given), with the dataframe name used instead.
 
+    CSV keeps datetime time components, including midnight, but carries no schema.
+    XLSX writes string values literally and uses distinct date/datetime display formats;
+    Excel numeric precision and timestamp/timezone limits still apply. Parquet is the
+    typed round-trip format for supported column types. Export does not clean or mutate
+    caller dataframes. Promote meaningful indexes to columns before exporting.
+
     Args:
         results (Result | ResultList | DataFrame): results to export.
         directory (str | Path): Output directory. Created if not found.

{datakraken-1.7.2 → datakraken-1.7.4}/src/kraken/exporting/result_export_extensions.py

@@ -14,6 +14,9 @@ def _prepare_results_xlsx(export_pack: dict[str, pd.DataFrame]) -> bytes:
         engine="xlsxwriter",
         date_format="DD/MM/YYYY",
         datetime_format="DD/MM/YYYY HH:MM:SS",
+        engine_kwargs={
+            "options": {"strings_to_formulas": False, "strings_to_urls": False}
+        },
     ) as writer:
         for df_name, dataframe in export_pack.items():
             readout.print(f"\n - writing sheet '{df_name}'...", end="")
@@ -28,7 +31,19 @@ def _prepare_results_csv(
 ) -> bytes:
     buffer = BytesIO()
     for dataframe in export_pack.values():
-        dataframe.to_csv(buffer, sep=delimiter, index=False)
+        csv_frame = dataframe.copy(deep=False)
+        for position in range(len(dataframe.columns)):
+            column = dataframe.iloc[:, position]
+            if pd.api.types.is_datetime64_any_dtype(column.dtype) and not isinstance(
+                column.dtype, pd.DatetimeTZDtype
+            ):
+                csv_frame.isetitem(
+                    position,
+                    column.map(
+                        lambda value: "" if pd.isna(value) else value.isoformat(sep=" ")
+                    ).array,
+                )
+        csv_frame.to_csv(buffer, sep=delimiter, index=False)
     return buffer.getvalue()
 
 

datakraken-1.7.4/src/kraken/importing/data_import.py

@@ -0,0 +1,162 @@
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from kraken.classes.pack_lists import ResultList
+from kraken.classes.packs import Result
+from kraken.support.readout import readout
+from kraken.support.support import _check_filetype, _load_filepaths, calculate_runtime
+
+
+def extract_spreadsheets(
+    filepaths: str | list[str] | Path | list[Path], clean_df: bool = True, **kwargs: Any
+) -> ResultList:
+    """Imports CSV, XLS, XLSX and Parquet files into a ResultList. Outputs results in
+    order of user directory/file input. Raises errors if a filepath does not point to
+    a supported file or a valid directory, or if no supported files are detected. Raises warnings
+    if any given directory filepath returns no files.
+
+    Args:
+        filepaths (str | list[str]): Path or list of paths to spreadsheet files or
+            directories of spreadsheet files
+        clean_df (bool): Uses pandas' numpy_nullable reader backend by default for
+            CSV/Excel, avoiding float intermediates for nullable signed integers.
+            False leaves backend selection to pandas. Explicit dtype/backend options
+            are honored. Parquet retains its stored schema regardless of this flag.
+        kwargs: Forwarded to the relevant pandas reader. CSV/Excel default to
+            keep_default_na=False and na_values=[""], preserving literal NA strings
+            but treating blanks as missing. Override these defaults as needed.
+            Use dtype for identifiers or unsigned integers, converters for exact
+            decimals, and parse_dates for CSV timestamps. These formats do not
+            retain a full column schema.
+            Options must be valid for every requested format when mixing file types.
+
+    Returns:
+        ResultList: Imported dataframes with filename, dataframe name and path metadata.
+    """
+    start = datetime.now()
+    supported_extensions = ["csv", "xlsx", "xls", "parquet"]
+    reader_kwargs = {"keep_default_na": False, "na_values": [""], **kwargs}
+    if clean_df:
+        reader_kwargs.setdefault("dtype_backend", "numpy_nullable")
+    parse_dates = reader_kwargs.get("parse_dates")
+    declared_dtypes = reader_kwargs.get("dtype")
+    inferred_date_columns: list[str | int] = []
+    if "dtype_backend" in reader_kwargs and isinstance(parse_dates, list):
+        date_dtypes: dict[str | int, Any]
+        if declared_dtypes is None:
+            date_dtypes = {}
+        elif isinstance(declared_dtypes, dict):
+            date_dtypes = declared_dtypes.copy()
+        else:
+            date_dtypes = defaultdict(lambda: declared_dtypes)
+        for column in parse_dates:
+            if isinstance(column, (str, int)) and column not in date_dtypes:
+                date_dtypes[column] = object
+                inferred_date_columns.append(column)
+        reader_kwargs["dtype"] = date_dtypes
+
+    # Load Filepaths
+    filepaths = _load_filepaths(filepaths, supported_extensions)
+    spreadsheet_packs = ResultList()
+    df_name_list = []
+    df_name_duplicates = set()
+    df_name_duplicate_count = 0
+
+    readout.print("Loading dataframes from spreadsheets...")
+    # Load spreadsheets
+    for filepath in filepaths:
+        # Load csv dataframes
+        if _check_filetype(filepath, "csv"):
+            filename = Path(filepath).name
+            df_name = Path(filepath).stem
+            readout.print(f" - From csv '{filename}'...", end="")
+            df = pd.read_csv(filepath, **reader_kwargs)
+            _restore_parsed_dates(df, inferred_date_columns)
+            spreadsheet_packs.append(
+                Result(
+                    filename=filename,
+                    df_name=df_name,
+                    df=df,
+                    filepath=filepath,
+                    db_alias="",
+                    platform="",
+                    sql="",
+                )
+            )
+            df_name_list.append(df_name)
+            readout.print(f" loaded dataframe '{df_name}'")
+
+        # Load xlsx dataframes
+        if _check_filetype(filepath, ["xlsx", "xls"]):
+            filename = Path(filepath).name
+            readout.print(f" - From xlsx '{filename}'...")
+            with pd.ExcelFile(filepath) as file:
+                for sheet_name in file.sheet_names:
+                    df_name = str(sheet_name)
+                    df = file.parse(df_name, **reader_kwargs)
+                    _restore_parsed_dates(df, inferred_date_columns)
+                    spreadsheet_packs.append(
+                        Result(
+                            filename=filename,
+                            df_name=df_name,
+                            df=df,
+                            filepath=filepath,
+                            db_alias="",
+                            platform="",
+                            sql="",
+                        )
+                    )
+                    df_name_list.append(df_name)
+                    readout.print(f"   - loaded dataframe '{df_name}'")
+
+        if _check_filetype(filepath, "parquet"):
+            filename = Path(filepath).name
+            df_name = Path(filepath).stem
+            readout.print(f" - From parquet '{filename}'...", end="")
+            df = pd.read_parquet(filepath, **kwargs)
+            spreadsheet_packs.append(
+                Result(
+                    filename=filename,
+                    df_name=df_name,
+                    df=df,
+                    filepath=filepath,
+                    db_alias="",
+                    platform="",
+                    sql="",
+                )
+            )
+            df_name_list.append(df_name)
+            readout.print(f" loaded dataframe '{df_name}'")
+
+    stop = datetime.now()
+    readout.print(
+        f"{len(spreadsheet_packs)} dataframes loaded from {len(filepaths)} files in {calculate_runtime(start, stop).message}"
+    )
+
+    # Check for df name duplicates
+    for df_name in df_name_list:
+        df_name_count = df_name_list.count(df_name)
+        if df_name_count != 1:
+            df_name_duplicate_count += df_name_count - 1
+            df_name_duplicates.add(df_name)
+
+    if df_name_duplicates:
+        s = "" if len(df_name_duplicates) == 1 else "s"
+        readout.print(
+            f"\nWARNING: {len(df_name_duplicates)} duplicate dataframe name{s}:"
+        )
+        for duplicate in df_name_duplicates:
+            readout.warn(f"' -> '{duplicate}")
+
+    readout.print("")
+    return spreadsheet_packs
+
+
+def _restore_parsed_dates(df: pd.DataFrame, columns: list[str | int]) -> None:
+    for position, name in enumerate(df.columns):
+        if name in columns or position in columns:
+            df.isetitem(position, df.iloc[:, position].infer_objects().array)

{datakraken-1.7.2 → datakraken-1.7.4}/src/kraken/ribosome/ribosome.py

@@ -56,8 +56,8 @@ def run(
         username (str, optional): If entered, downstream execution will attempt to use this username. If left
             blank, Kraken will use the default username saved for each given database alias in credential manager.
             Defaults to None.
-        clean_df (bool): Checks each DataFrame after pandas generation and applies cleaning, including converting
-            float64 to Int64 if applicable (recommended).
+        clean_df (bool): Preserves fetched scalar types during DataFrame construction, including
+            nullable integers/booleans without a float intermediate. False uses ordinary pandas inference.
         export_directory (str, optional): Sets a directory to export results to. If left blank, Kraken ignores
             exporting. Enter "" to export to directory of expecting script. Defaults to None.
         export_extension (str, optional): If exporting, Kraken will export as this extension if supported.

{datakraken-1.7.2 → datakraken-1.7.4}/src/kraken/ribosome/sql_execution.py

@@ -53,8 +53,8 @@ def execute(
         username (str, optional): If blank, Kraken will use the default username for the given alias. Otherwise,
             Kraken will fetch saved credentials for the given username/database alias pair. Defaults to None.
         batch_size (int): Downloads rows in batches. Use `batch_size=0` to fetch all data without batching.
-        clean_df (bool): Checks DataFrame after pandas generation and applies cleaning, including converting
-            float64 to Int64 if applicable (recommended).
+        clean_df (bool): Preserves fetched scalar types during DataFrame construction, including
+            nullable integers/booleans without a float intermediate. False uses ordinary pandas inference.
         isolation_level (Literal["SERIALIZABLE", "REPEATABLE READ", "READ COMMITTED", "READ UNCOMMITTED", "AUTOCOMMIT"] | None):
             SQL Alchemy isolation level. Defaults to None. If errors are raised related to not being able to perform
             queries within transactions, (for example as typical with Synapse databases), try using "AUTOCOMMIT".
@@ -152,8 +152,8 @@ def execute_sql(
         query_list (list[Query]): List of QueryList of Query objects prepared by extract_sql()
         username (str, optional): Overriding username to fetch database credentials. Defaults to None. WARNING:
             Kraken will attempt to use this username to execute all queries.
-        clean_df (bool): Checks each DataFrame after pandas generation and applies cleaning, including converting
-            float64 to Int64 if applicable (recommended).
+        clean_df (bool): Preserves fetched scalar types during DataFrame construction, including
+            nullable integers/booleans without a float intermediate. False uses ordinary pandas inference.
         batch_size (int): Downloads rows in batches. Use `batch_size=0` to fetch all data without batching.
         concurrent (bool): If True, executes all SQL files concurrently. Queries within each SQL file will still
             execute sequentially. Defaults to False.

datakraken-1.7.2/src/kraken/importing/data_import.py

@@ -1,125 +0,0 @@
-from datetime import datetime
-from pathlib import Path
-from typing import Any
-
-import pandas as pd
-
-from kraken.analysis.data_manipulation import check_df_integers as clean_dataframe
-from kraken.classes.pack_lists import ResultList
-from kraken.classes.packs import Result
-from kraken.support.readout import readout
-from kraken.support.support import _check_filetype, _load_filepaths, calculate_runtime
-
-
-def extract_spreadsheets(
-    filepaths: str | list[str] | Path | list[Path], clean_df: bool = True, **kwargs: Any
-) -> ResultList:
-    """Takes paths, or list of paths, to spreadsheet files or directories of spreadsheet
-    files and outputs a list of results with dataframes. Outputs list in order of user
-    directory/file input. Raises errors if a filepath does not point to a spreadsheet
-    file or a valid directory, or if no spreadsheet files are detected. Raises warnings
-    if any given directory filepath returns no files.
-
-    Args:
-        filepaths (str | list[str]): Path or list of paths to spreadsheet files or
-            directories of spreadsheet files
-        clean_df (bool): Checks DataFrame after pandas generation and applies cleaning,
-            including converting float64 to Int64 if applicable (recommended).
-
-    Returns:
-        list: List of results, consisting of tuples as: (filename, df name, dataframe, df, filepath)
-    """
-    start = datetime.now()
-    supported_extensions = ["csv", "xlsx", "xls"]
-    keep_default_na = False
-    na_values = [""]
-
-    # Load Filepaths
-    filepaths = _load_filepaths(filepaths, supported_extensions)
-    spreadsheet_packs = ResultList()
-    df_name_list = []
-    df_name_duplicates = set()
-    df_name_duplicate_count = 0
-
-    readout.print("Loading dataframes from spreadsheets...")
-    # Load spreadsheets
-    for filepath in filepaths:
-        # Load csv dataframes
-        if _check_filetype(filepath, "csv"):
-            filename = Path(filepath).name
-            df_name = Path(filepath).stem
-            readout.print(f" - From csv '{filename}'...", end="")
-            df = pd.read_csv(
-                filepath,
-                keep_default_na=keep_default_na,
-                na_values=na_values,
-                **kwargs,
-            )
-            spreadsheet_packs.append(
-                Result(
-                    filename=filename,
-                    df_name=df_name,
-                    df=df,
-                    filepath=filepath,
-                    db_alias="",
-                    platform="",
-                    sql="",
-                )
-            )
-            df_name_list.append(df_name)
-            readout.print(f" loaded dataframe '{df_name}'")
-
-        # Load xlsx dataframes
-        if _check_filetype(filepath, ["xlsx", "xls"]):
-            filename = Path(filepath).name
-            readout.print(f" - From xlsx '{filename}'...")
-            with pd.ExcelFile(filepath) as file:
-                for sheet_name in file.sheet_names:
-                    df_name = str(sheet_name)
-                    df = file.parse(
-                        df_name,
-                        keep_default_na=keep_default_na,
-                        na_values=na_values,
-                        **kwargs,
-                    )
-                    spreadsheet_packs.append(
-                        Result(
-                            filename=filename,
-                            df_name=df_name,
-                            df=df,
-                            filepath=filepath,
-                            db_alias="",
-                            platform="",
-                            sql="",
-                        )
-                    )
-                    df_name_list.append(df_name)
-                    readout.print(f"   - loaded dataframe '{df_name}'")
-
-    # Clean SpreadsheetPack DataFrames
-    if clean_df:
-        for pack in spreadsheet_packs:
-            pack.df = clean_dataframe(pack.df)
-
-    stop = datetime.now()
-    readout.print(
-        f"{len(spreadsheet_packs)} dataframes loaded from {len(filepaths)} files in {calculate_runtime(start, stop).message}"
-    )
-
-    # Check for df name duplicates
-    for df_name in df_name_list:
-        df_name_count = df_name_list.count(df_name)
-        if df_name_count != 1:
-            df_name_duplicate_count += df_name_count - 1
-            df_name_duplicates.add(df_name)
-
-    if df_name_duplicates:
-        s = "" if len(df_name_duplicates) == 1 else "s"
-        readout.print(
-            f"\nWARNING: {len(df_name_duplicates)} duplicate dataframe name{s}:"
-        )
-        for duplicate in df_name_duplicates:
-            readout.warn(f"' -> '{duplicate}")
-
-    readout.print("")
-    return spreadsheet_packs