datablade 0.0.0__py3-none-any.whl → 0.0.5__py3-none-any.whl

datablade/__init__.py

@@ -1 +1,41 @@
-#empty
+"""
+datablade - A suite of functions providing standard syntax across data engineering projects.
+
+The package is organized into four main modules:
+- dataframes: DataFrame operations, transformations, and memory-aware file reading
+- io: Input/output operations for external data
+- utils: General utility functions and logging
+- sql: Multi-dialect SQL generation, quoting, and bulk loading
+
+For backward compatibility, all functions are also available from datablade.core.
+"""
+
+# Also maintain core for backward compatibility
+# Import from new organized structure
+from . import core, dataframes, io, sql, utils
+from .blade import Blade
+from .dataframes import read_file_chunked, read_file_smart, read_file_to_parquets
+from .sql import Dialect, bulk_load, generate_create_table
+
+# Convenience re-exports for commonly used functions
+from .utils.logging import configure_logging, get_logger
+
+__version__ = "0.0.5"
+
+__all__ = [
+    "dataframes",
+    "io",
+    "utils",
+    "sql",
+    "core",  # Maintain backward compatibility
+    # Convenience re-exports
+    "configure_logging",
+    "get_logger",
+    "read_file_smart",
+    "read_file_chunked",
+    "read_file_to_parquets",
+    "Dialect",
+    "generate_create_table",
+    "bulk_load",
+    "Blade",
+]

datablade/blade.py

@@ -0,0 +1,153 @@
+"""Optional facade class for datablade.
+
+The canonical API is module-level functions (e.g., datablade.dataframes.read_file_iter).
+This module provides a small convenience wrapper for users who prefer an object-style
+entrypoint with shared defaults.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterator, Optional, Union
+
+import pandas as pd
+
+from .dataframes import (
+    clean_dataframe_columns,
+    read_file_iter,
+    read_file_smart,
+    read_file_to_parquets,
+    stream_to_parquets,
+    try_cast_string_columns_to_numeric,
+)
+from .sql import Dialect, generate_create_table, generate_create_table_from_parquet
+
+PathLike = Union[str, Path]
+
+
+@dataclass(frozen=True)
+class Blade:
+    """Convenience facade for common datablade workflows.
+
+    Stores default options that are threaded through to the underlying functions.
+    """
+
+    memory_fraction: float = 0.5
+    verbose: bool = False
+    convert_types: bool = True
+
+    def read(self, file_path: PathLike, **read_kwargs: Any) -> pd.DataFrame:
+        return read_file_smart(
+            file_path=file_path,
+            memory_fraction=self.memory_fraction,
+            verbose=self.verbose,
+            **read_kwargs,
+        )
+
+    def iter(
+        self,
+        file_path: PathLike,
+        *,
+        chunksize: Optional[int] = None,
+        **read_kwargs: Any,
+    ) -> Iterator[pd.DataFrame]:
+        return read_file_iter(
+            file_path=file_path,
+            chunksize=chunksize,
+            memory_fraction=self.memory_fraction,
+            verbose=self.verbose,
+            **read_kwargs,
+        )
+
+    def partition_to_parquets(
+        self,
+        file_path: PathLike,
+        output_dir: PathLike,
+        *,
+        output_prefix: str = "part",
+        rows_per_file: Optional[int] = None,
+        convert_types: Optional[bool] = None,
+        **read_kwargs: Any,
+    ):
+        return read_file_to_parquets(
+            file_path=file_path,
+            output_dir=output_dir,
+            output_prefix=output_prefix,
+            rows_per_file=rows_per_file,
+            memory_fraction=self.memory_fraction,
+            convert_types=(
+                self.convert_types if convert_types is None else convert_types
+            ),
+            verbose=self.verbose,
+            **read_kwargs,
+        )
+
+    def stream_to_parquets(
+        self,
+        file_path: PathLike,
+        output_dir: PathLike,
+        *,
+        output_prefix: str = "part",
+        rows_per_file: Optional[int] = None,
+        convert_types: Optional[bool] = None,
+        **read_kwargs: Any,
+    ):
+        return stream_to_parquets(
+            file_path=file_path,
+            output_dir=output_dir,
+            output_prefix=output_prefix,
+            rows_per_file=rows_per_file,
+            memory_fraction=self.memory_fraction,
+            convert_types=(
+                self.convert_types if convert_types is None else convert_types
+            ),
+            verbose=self.verbose,
+            **read_kwargs,
+        )
+
+    def clean(self, df: pd.DataFrame) -> pd.DataFrame:
+        return clean_dataframe_columns(df, verbose=self.verbose)
+
+    def cast_numeric(self, df: pd.DataFrame) -> pd.DataFrame:
+        return try_cast_string_columns_to_numeric(df, verbose=self.verbose)
+
+    def create_table_sql(
+        self,
+        df: pd.DataFrame,
+        *,
+        catalog: Optional[str] = None,
+        schema: Optional[str] = None,
+        table: str = "table",
+        drop_existing: bool = True,
+        dialect: Dialect = Dialect.SQLSERVER,
+    ) -> str:
+        return generate_create_table(
+            df=df,
+            catalog=catalog,
+            schema=schema,
+            table=table,
+            drop_existing=drop_existing,
+            dialect=dialect,
+            verbose=self.verbose,
+        )
+
+    def create_table_sql_from_parquet(
+        self,
+        parquet_path: str,
+        *,
+        catalog: Optional[str] = None,
+        schema: Optional[str] = None,
+        table: str = "table",
+        drop_existing: bool = True,
+        dialect: Dialect = Dialect.SQLSERVER,
+    ) -> str:
+        return generate_create_table_from_parquet(
+            parquet_path=parquet_path,
+            catalog=catalog,
+            schema=schema,
+            table=table,
+            drop_existing=drop_existing,
+            dialect=dialect,
+            verbose=self.verbose,
+        )

datablade/core/__init__.py

@@ -1,7 +1,28 @@
-import os, re 
-
-def find_python_files(path):
-    return [one_file_name.replace('.py','') for one_file_name in os.listdir(os.path.abspath(path)) if one_file_name != '__init__.py' and re.match(r'.*\.py$',one_file_name) is not None]
-
-for each_file in find_python_files(path=os.path.dirname(__file__)):
-    exec('from .'+each_file+' import *')
+"""Backward-compatible exports for the legacy datablade.core namespace.
+
+Historically, this package used dynamic imports to re-export everything.
+We keep the same runtime surface area but use explicit imports so that IDEs,
+type checkers, and static analysis tools can reason about the module.
+"""
+
+from . import frames as _frames
+from . import json as _json
+from . import lists as _lists
+from . import messages as _messages
+from . import strings as _strings
+from . import zip as _zip
+from .frames import *  # noqa: F401,F403
+from .json import *  # noqa: F401,F403
+from .lists import *  # noqa: F401,F403
+from .messages import *  # noqa: F401,F403
+from .strings import *  # noqa: F401,F403
+from .zip import *  # noqa: F401,F403
+
+__all__ = [
+    *_frames.__all__,
+    *_json.__all__,
+    *_lists.__all__,
+    *_messages.__all__,
+    *_strings.__all__,
+    *_zip.__all__,
+]

datablade/core/frames.py

@@ -1,236 +1,23 @@
-import pandas as pd 
-import pyarrow as pa
-import numpy as np 
-
-from .messages import print_verbose
-from .strings  import sql_quotename
-
-def try_cast_string_columns_to_numeric(df: pd.DataFrame=None, convert_partial: bool=False, verbose: bool=False) -> pd.DataFrame|None:
-    """
-    Attempt to cast DataFrame string columns to numeric values where possible.
-    
-    Parameters:
-        df (pd.DataFrame): The DataFrame to process.
-        convert_partial (bool): If True, columns with some values convertible to numeric types
-                                will be converted to numeric types with NaNs where conversion failed.
-                                If False, only columns where all values can be converted will be converted.
-    
-    Returns:
-        pd.DataFrame: DataFrame with string columns converted to numeric types where possible.
-    """
-    if df is None:
-        print_verbose("No DataFrame provided; exiting try_cast_string_columns_to_numeric.", verbose)
-        exit # Exit the function if no DataFrame is provided
-
-    for col in df.columns:
-        if df[col].dtype == 'object':
-            converted = pd.to_numeric(df[col], errors='coerce')
-            has_nan = converted.isnull().any()
-            if not has_nan:
-                df[col] = converted
-                print_verbose(f"Column '{col}' successfully converted to numeric.", verbose)
-            else:
-                if convert_partial:
-                    df[col] = converted
-                    print_verbose(f"Column '{col}' partially converted to numeric with NaNs where conversion failed.", verbose)
-                else:
-                    print_verbose(f"Column '{col}' could not be fully converted to numeric; leaving as is.", verbose)
-    return df
-
-def clean_dataframe_columns(df: pd.DataFrame=None, verbose: bool=False) -> pd.DataFrame|None:
-    """
-    Clean the DataFrame columns by:
-    - Flattening MultiIndex columns
-    - Converting non-string column names to strings
-    - Removing duplicate columns, keeping the first occurrence
-
-    Parameters:
-        df (pd.DataFrame): The DataFrame to clean.
-
-    Returns:
-        pd.DataFrame: The cleaned DataFrame.
-    """
-    if df is None:
-        print_verbose("No DataFrame provided; exiting clean_dataframe_columns.", verbose)
-        exit 
-    # Step 1: Flatten MultiIndex columns
-    if isinstance(df.columns, pd.MultiIndex):
-        df.columns = ['_'.join(map(str, col)).strip() for col in df.columns.values]
-        print_verbose("Flattened MultiIndex columns.", verbose)
-
-    # Step 2: Convert non-string column names to strings
-    df.columns = df.columns.map(str)
-    print_verbose("Converted column names to strings.", verbose)
-
-    # Step 3: Remove duplicate columns, keeping the first occurrence
-    duplicates = df.columns.duplicated()
-    if duplicates.any():
-        duplicate_cols = df.columns[duplicates]
-        print_verbose(f"Duplicate columns found: {list(duplicate_cols)}", verbose)
-        df = df.loc[:, ~duplicates]
-        print_verbose("Removed duplicate columns, keeping the first occurrence.", verbose)
-
-    return df
-
-def generate_parquet_schema(df: pd.DataFrame=None, verbose: bool=False) -> pa.Schema|None:
-    """
-    Generate a PyArrow Schema from a pandas DataFrame.
-    Parameters:
-        df (pandas.DataFrame): The DataFrame to generate the schema from.
-    Returns:
-        pyarrow.Schema: The PyArrow Schema object.
-    """
-    if df is None:
-        print_verbose("No DataFrame provided; exiting generate_parquet_schema.", verbose)
-        exit 
-    
-    fields = []
-    for column in df.columns:
-        col_data = df[column]
-        col_name = column
-        dtype = col_data.dtype
-
-        # Determine if the column contains any nulls
-        nullable = col_data.isnull().any()
-
-        # Map pandas dtype to PyArrow type
-        pa_type = None
-
-        if pd.api.types.is_integer_dtype(dtype):
-            # Check the range to determine the smallest integer type
-            min_value = col_data.min()
-            max_value = col_data.max()
-            if min_value >= np.iinfo(np.int8).min and max_value <= np.iinfo(np.int8).max:
-                pa_type = pa.int8()
-            elif min_value >= np.iinfo(np.int16).min and max_value <= np.iinfo(np.int16).max:
-                pa_type = pa.int16()
-            elif min_value >= np.iinfo(np.int32).min and max_value <= np.iinfo(np.int32).max:
-                pa_type = pa.int32()
-            else:
-                pa_type = pa.int64()
-
-        elif pd.api.types.is_float_dtype(dtype):
-            pa_type = pa.float64()
-
-        elif pd.api.types.is_bool_dtype(dtype):
-            pa_type = pa.bool_()
-
-        elif pd.api.types.is_datetime64_any_dtype(dtype):
-            pa_type = pa.timestamp('ms')
-
-        elif isinstance(dtype, pd.CategoricalDtype) or pd.api.types.is_object_dtype(dtype):
-            pa_type = pa.string()
-
-        else:
-            pa_type = pa.string()
-
-        # Create a field
-        field = pa.field(col_name, pa_type, nullable=nullable)
-        fields.append(field)
-
-    schema = pa.schema(fields)
-    return schema
-
-def pandas_to_parquet_table(df: pd.DataFrame=None, convert: bool=True, partial: bool=False, preserve_index: bool=False, verbose: bool=False) -> pa.Table|None:
-    """
-    Generate a PyArrow Table from a pandas DataFrame.
-
-    Parameters:
-        df (pandas.DataFrame): The DataFrame to generate the table from.
-        table (str): The name of the table.
-
-    Returns:
-        pyarrow.Table: The PyArrow Table object.
-    """
-    if df is None:
-        print_verbose("No DataFrame provided; exiting generate_parquet_table.", verbose)
-        exit 
-    
-    df     = clean_dataframe_columns(df=df, verbose=verbose)
-    
-    if convert:
-        df = try_cast_string_columns_to_numeric(df=df, convert_partial=partial, verbose=verbose)
-
-    schema = generate_parquet_schema(df=df, verbose=verbose)
-    try:
-        table = pa.Table.from_pandas(df, schema=schema, preserve_index=preserve_index)
-        return table
-    except Exception as e:
-        print_verbose(f"Error generating PyArrow Table: {e}", verbose)
-        exit 
-
-def generate_sql_server_create_table_string(df: pd.DataFrame=None, catalog: str='database', schema: str='dbo', table: str='table', dropexisting: bool=True, verbose: bool=False) -> str|None:
-    """
-    Generate a SQL Server CREATE TABLE string from a pandas DataFrame.
-
-    Parameters:
-        df (pandas.DataFrame): The DataFrame to generate the schema from.
-        table_name (str): The name of the SQL table.
-
-    Returns:
-        str: The SQL Server CREATE TABLE statement.
-    """
-    if df is None:
-        print_verbose("No DataFrame provided; exiting try_cast_string_columns_to_numeric.", verbose)
-        exit 
-    
-    table_name = f"{sql_quotename(catalog)}.{sql_quotename(schema)}.{sql_quotename(table)}"
-    drop_statement = f"use {sql_quotename(catalog)}\rgo\rif object_id('{table_name}') is not null drop table {table_name};\r" if dropexisting else ""
-    
-    create_statement = [f"{drop_statement};create table {table_name} ("]
-    indent = "    "
-    column_lines = []
-
-    for column in df.columns:
-        col_data = df[column]
-        col_name = column
-        dtype = col_data.dtype
-
-        # Determine if the column contains any nulls
-        nullable = col_data.isnull().any()
-        null_str = f"{'   ' if nullable else 'not'} null"
-
-        # Map pandas dtype to SQL Server type
-        sql_type = None
-
-        if pd.api.types.is_integer_dtype(dtype):
-            min_value = col_data.min()
-            max_value = col_data.max()
-            if min_value >= 0 and max_value <= 255:
-                sql_type = "tinyint"
-            elif min_value >= -32768 and max_value <= 32767:
-                sql_type = "smallint"
-            elif min_value >= -2147483648 and max_value <= 2147483647:
-                sql_type = "int"
-            else:
-                sql_type = "bigint"
-
-        elif pd.api.types.is_float_dtype(dtype):
-            sql_type = "float"
-
-        elif pd.api.types.is_bool_dtype(dtype):
-            sql_type = "bit"
-
-        elif pd.api.types.is_datetime64_any_dtype(dtype):
-            sql_type = "datetime2"
-
-        elif isinstance(dtype, pd.CategoricalDtype) or pd.api.types.is_object_dtype(dtype):
-            # Determine maximum length of string data
-            max_length = col_data.dropna().astype(str).map(len).max()
-            sql_type = f"nvarchar({str(max_length) if max_length <= 4000 else 'max'})"
-            
-        else:
-            sql_type = "nvarchar(max)"
-
-        # Build the column definition
-        column_line = f"{indent}{sql_quotename(col_name)} {sql_type} {null_str},"
-        column_lines.append(column_line)
-
-    # Remove the last comma from the last column definition
-    if column_lines:
-        column_lines[-1] = column_lines[-1].rstrip(',')
-
-    create_statement.extend(column_lines)
-    create_statement.append(");")
-    return_statement = "\r".join(create_statement)
-    return return_statement
+"""Backward-compatibility re-exports.
+
+This module intentionally contains no independent implementations.
+All functionality is provided by the newer modules in datablade.dataframes.
+"""
+
+from ..dataframes.frames import (  # noqa: F401
+    clean_dataframe_columns,
+    generate_parquet_schema,
+    generate_sql_server_create_table_string,
+    pandas_to_parquet_table,
+    try_cast_string_columns_to_numeric,
+    write_to_file_and_sql,
+)
+
+__all__ = [
+    "try_cast_string_columns_to_numeric",
+    "clean_dataframe_columns",
+    "generate_parquet_schema",
+    "pandas_to_parquet_table",
+    "generate_sql_server_create_table_string",
+    "write_to_file_and_sql",
+]

datablade/core/json.py

@@ -1,10 +1,5 @@
-import requests 
-from .messages import print_verbose
-
-def get(url: str, verbose: bool = False, **kwargs) -> dict:
-    """Get JSON data from a URL."""
-    try:
-        response = requests.get(url, **kwargs)
-        return response.json()
-    except requests.exceptions.RequestException as e:
-        print_verbose(f"Error: {e}", verbose=verbose)
+"""Backward-compatibility re-exports for IO JSON helpers."""
+
+from ..io.json import get  # noqa: F401
+
+__all__ = ["get"]

datablade/core/lists.py

@@ -1,10 +1,5 @@
-
-def flatten(nest: list) -> list:
-    """Flatten a nested list."""
-    result = []
-    for item in nest:
-        if isinstance(item, list):
-            result.extend(flatten(item))
-        else:
-            result.append(item)
-    return result
+"""Backward-compatibility re-exports for list utilities."""
+
+from ..utils.lists import flatten  # noqa: F401
+
+__all__ = ["flatten"]

datablade/core/messages.py

@@ -1,11 +1,23 @@
-
-def print_verbose(message: str, verbose: bool=True) -> None:
-    """
-    Print a message if verbose is True.
-    
-    Parameters:
-        message (str): The message to print.
-        verbose (bool): If True, the message will be printed.
-    """
-    if verbose:
-        print(str(message))
+"""Backward-compatibility re-exports for message/logging helpers."""
+
+from ..utils.messages import (  # noqa: F401
+    configure_logging,
+    get_logger,
+    log,
+    log_debug,
+    log_error,
+    log_info,
+    log_warning,
+    print_verbose,
+)
+
+__all__ = [
+    "print_verbose",
+    "log",
+    "log_debug",
+    "log_info",
+    "log_warning",
+    "log_error",
+    "get_logger",
+    "configure_logging",
+]

datablade/core/strings.py

@@ -1,43 +1,5 @@
-from .messages import print_verbose
-import pathlib
-
-def sql_quotename(name: str=None, brackets: bool=True, ticks: bool=False, verbose: bool=False) -> str|None:
-    """
-    Quote a SQL Server name string.
-    Parameters:
-        name (str): The name to quote.
-        brackets (bool): Whether to use brackets.
-    Returns:
-        str: The quoted name.
-    """
-    if name is None:
-        print_verbose("No name provided; exiting sql_quotename.", verbose)
-        exit
-    return_value = f"{name.replace('[','').replace(']','')}"
-    if brackets:
-        return_value = f"[{return_value}]"
-    if ticks or not brackets:
-        return_value = f"'{return_value}'"
-    return return_value
-
-def pathing(input: str | pathlib.Path, verbose: bool=False) -> pathlib.Path|None:
-    """
-    Standardize a path string.
-    Parameters:
-        path (str): The path to standardize.
-    Returns:
-        str: The standardized path.
-    """
-    if input is None:
-        print_verbose("No path provided; exiting pathing.", verbose)
-        exit
-    if isinstance(input, str):
-        input.replace('\\','/')
-        input = pathlib.Path(input)
-    else:
-        input = input
-    if input.exists():
-        return input
-    else:
-        print_verbose(f"Path {input} does not exist; exiting pathing.", verbose)
-        exit
+"""Backward-compatibility re-exports for string/path utilities."""
+
+from ..utils.strings import pathing, sql_quotename  # noqa: F401
+
+__all__ = ["sql_quotename", "pathing"]

datablade/core/zip.py

@@ -1,24 +1,5 @@
-import requests, zipfile, io, pathlib
-from .messages import print_verbose
-from .strings import pathing
-
-def get(url:str, path:str|pathlib.Path=None, verbose:bool=False, **kwargs) -> None|io.BytesIO:
-    """Download a file from a URL and save it to a path."""
-    try:
-        print_verbose(f"Downloading {url}", verbose=verbose)
-        data = requests.get(url, **kwargs).content 
-        zip_buffer = io.BytesIO(data)
-        if path is None:
-            return zip_buffer
-        else:
-            print_verbose(f"Saving data to {path}", verbose=verbose)
-            zip_buffer.seek(0)
-            with zipfile.ZipFile(zip_buffer, 'r') as zip_ref:
-                for zip_info in zip_ref.infolist():
-                    extract_path = pathing(path) / zip_info.filename
-                    extract_path.parent.mkdir(parents=True, exist_ok=True) 
-                    with open(extract_path, 'wb') as f:
-                        f.write(zip_ref.read(zip_info.filename))
-                        f.close()
-    except requests.exceptions.RequestException as e:
-        print_verbose(f"Error: {e}", verbose=verbose)
+"""Backward-compatibility re-exports for IO ZIP helpers."""
+
+from ..io.zip import get  # noqa: F401
+
+__all__ = ["get"]

datablade/dataframes/__init__.py

@@ -0,0 +1,43 @@
+"""
+DataFrame operations and utilities for data transformation.
+
+This module provides functions for:
+- DataFrame column cleaning and type conversion
+- Parquet schema generation and conversion
+- SQL Server schema generation
+- Memory-aware file reading with optional Polars support
+- Chunked file reading for large files
+- Partitioned Parquet writing
+"""
+
+from .frames import (
+    clean_dataframe_columns,
+    generate_parquet_schema,
+    generate_sql_server_create_table_string,
+    pandas_to_parquet_table,
+    try_cast_string_columns_to_numeric,
+    write_to_file_and_sql,
+)
+from .readers import (
+    read_file_chunked,
+    read_file_iter,
+    read_file_smart,
+    read_file_to_parquets,
+    stream_to_parquets,
+)
+
+__all__ = [
+    # DataFrame operations
+    "try_cast_string_columns_to_numeric",
+    "clean_dataframe_columns",
+    "generate_parquet_schema",
+    "pandas_to_parquet_table",
+    "generate_sql_server_create_table_string",
+    "write_to_file_and_sql",
+    # Memory-aware readers
+    "read_file_chunked",
+    "read_file_iter",
+    "read_file_to_parquets",
+    "stream_to_parquets",
+    "read_file_smart",
+]