datablade 0.0.0__py3-none-any.whl → 0.0.6__py3-none-any.whl

datablade/__init__.py

@@ -1 +1,49 @@
-#empty
+"""
+datablade - A suite of functions providing standard syntax across data engineering projects.
+
+The package is organized into four main modules:
+- dataframes: DataFrame operations, transformations, and memory-aware file reading
+- io: Input/output operations for external data
+- utils: General utility functions and logging
+- sql: Multi-dialect SQL generation, quoting, and bulk loading
+
+For backward compatibility, all functions are also available from datablade.core.
+"""
+
+# Also maintain core for backward compatibility
+# Import from new organized structure
+from . import core, dataframes, io, registry, sql, utils
+from .blade import Blade
+from .dataframes import read_file_chunked, read_file_smart, read_file_to_parquets
+from .registry import DialectSpec, ObjectNode, ObjectRef, ObjectRegistry
+from .sql import Dialect, bulk_load, generate_create_table
+
+# Convenience re-exports for commonly used functions
+from .utils.logging import configure_logging, get_logger
+from .utils.strings import configure_paths
+
+__version__ = "0.0.6"
+
+__all__ = [
+    "dataframes",
+    "io",
+    "utils",
+    "sql",
+    "registry",
+    "core",  # Maintain backward compatibility
+    # Convenience re-exports
+    "configure_logging",
+    "configure_paths",
+    "get_logger",
+    "read_file_smart",
+    "read_file_chunked",
+    "read_file_to_parquets",
+    "Dialect",
+    "generate_create_table",
+    "bulk_load",
+    "Blade",
+    "DialectSpec",
+    "ObjectRef",
+    "ObjectNode",
+    "ObjectRegistry",
+]

datablade/blade.py

@@ -0,0 +1,322 @@
+"""Optional facade class for datablade.
+
+The canonical API is module-level functions (e.g., datablade.dataframes.read_file_iter).
+This module provides a small convenience wrapper for users who prefer an object-style
+entrypoint with shared defaults.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterator, Optional, Sequence, Union
+
+import pandas as pd
+
+from .dataframes import (
+    clean_dataframe_columns,
+    read_file_iter,
+    read_file_smart,
+    read_file_to_parquets,
+    stream_to_parquets,
+    stream_to_sink,
+    try_cast_string_columns_to_numeric,
+)
+from .sql import (
+    Dialect,
+    ParquetDDLMetadata,
+    generate_create_table,
+    generate_create_table_from_parquet,
+    sqlserver_create_and_insert_from_parquet,
+    sqlserver_create_and_stage_from_parquets,
+    sqlserver_openrowset_parquet,
+)
+
+PathLike = Union[str, Path]
+
+
+@dataclass(frozen=True)
+class Blade:
+    """Convenience facade for common datablade workflows.
+
+    Stores default options that are threaded through to the underlying functions.
+    """
+
+    memory_fraction: float = 0.5
+    verbose: bool = False
+    convert_types: bool = True
+
+    def read(
+        self,
+        file_path: PathLike,
+        *,
+        return_type: str = "dataframe",
+        **read_kwargs: Any,
+    ):
+        return read_file_smart(
+            file_path=file_path,
+            memory_fraction=self.memory_fraction,
+            verbose=self.verbose,
+            return_type=return_type,
+            **read_kwargs,
+        )
+
+    def iter(
+        self,
+        file_path: PathLike,
+        *,
+        chunksize: Optional[int] = None,
+        **read_kwargs: Any,
+    ) -> Iterator[pd.DataFrame]:
+        """Yield DataFrame chunks from a file without materializing."""
+        return read_file_iter(
+            file_path=file_path,
+            chunksize=chunksize,
+            memory_fraction=self.memory_fraction,
+            verbose=self.verbose,
+            **read_kwargs,
+        )
+
+    def partition_to_parquets(
+        self,
+        file_path: PathLike,
+        output_dir: PathLike,
+        *,
+        output_prefix: str = "part",
+        rows_per_file: Optional[int] = None,
+        convert_types: Optional[bool] = None,
+        **read_kwargs: Any,
+    ):
+        """Read and materialize a file into Parquet partitions."""
+        return read_file_to_parquets(
+            file_path=file_path,
+            output_dir=output_dir,
+            output_prefix=output_prefix,
+            rows_per_file=rows_per_file,
+            memory_fraction=self.memory_fraction,
+            convert_types=(
+                self.convert_types if convert_types is None else convert_types
+            ),
+            verbose=self.verbose,
+            **read_kwargs,
+        )
+
+    def stream_to_parquets(
+        self,
+        file_path: PathLike,
+        output_dir: PathLike,
+        *,
+        output_prefix: str = "part",
+        rows_per_file: Optional[int] = None,
+        convert_types: Optional[bool] = None,
+        **read_kwargs: Any,
+    ):
+        """Stream input file chunks into Parquet partitions."""
+        return stream_to_parquets(
+            file_path=file_path,
+            output_dir=output_dir,
+            output_prefix=output_prefix,
+            rows_per_file=rows_per_file,
+            memory_fraction=self.memory_fraction,
+            convert_types=(
+                self.convert_types if convert_types is None else convert_types
+            ),
+            verbose=self.verbose,
+            **read_kwargs,
+        )
+
+    def stream_to_sink(
+        self,
+        chunks: Iterator[pd.DataFrame],
+        output_dir: PathLike,
+        *,
+        output_prefix: str = "part",
+        convert_types: Optional[bool] = None,
+    ):
+        return stream_to_sink(
+            chunks=chunks,
+            output_dir=output_dir,
+            output_prefix=output_prefix,
+            convert_types=(
+                self.convert_types if convert_types is None else convert_types
+            ),
+            verbose=self.verbose,
+        )
+
+    def clean(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Normalize column names and remove duplicate columns."""
+        return clean_dataframe_columns(df, verbose=self.verbose)
+
+    def cast_numeric(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Attempt numeric conversion on string columns."""
+        return try_cast_string_columns_to_numeric(df, verbose=self.verbose)
+
+    def create_table_sql(
+        self,
+        df: pd.DataFrame,
+        *,
+        catalog: Optional[str] = None,
+        schema: Optional[str] = None,
+        table: str = "table",
+        drop_existing: bool = True,
+        dialect: Dialect = Dialect.SQLSERVER,
+        use_go: bool = False,
+        schema_spec: Optional[dict] = None,
+    ) -> Union[str, tuple[str, ParquetDDLMetadata]]:
+        """Generate CREATE TABLE DDL from a DataFrame schema."""
+        return generate_create_table(
+            df=df,
+            catalog=catalog,
+            schema=schema,
+            table=table,
+            drop_existing=drop_existing,
+            dialect=dialect,
+            use_go=use_go,
+            schema_spec=schema_spec,
+            verbose=self.verbose,
+        )
+
+    def create_table_sql_from_parquet(
+        self,
+        parquet_path: PathLike,
+        *,
+        catalog: Optional[str] = None,
+        schema: Optional[str] = None,
+        table: str = "table",
+        drop_existing: bool = True,
+        dialect: Dialect = Dialect.SQLSERVER,
+        use_go: bool = False,
+        schema_spec: Optional[dict] = None,
+        fallback_to_json: bool = False,
+        return_metadata: bool = False,
+    ) -> str:
+        """Generate CREATE TABLE DDL from a Parquet schema."""
+        return generate_create_table_from_parquet(
+            parquet_path=parquet_path,
+            catalog=catalog,
+            schema=schema,
+            table=table,
+            drop_existing=drop_existing,
+            dialect=dialect,
+            use_go=use_go,
+            schema_spec=schema_spec,
+            verbose=self.verbose,
+            fallback_to_json=fallback_to_json,
+            return_metadata=return_metadata,
+        )
+
+    def sqlserver_openrowset_parquet(
+        self,
+        parquet_path: PathLike,
+        *,
+        data_source: Optional[str] = None,
+        table_alias: str = "rows",
+        select_columns: Optional[Sequence[str]] = None,
+        where: Optional[str] = None,
+        top: Optional[int] = None,
+    ) -> str:
+        """Generate a SQL Server OPENROWSET query over Parquet files."""
+        return sqlserver_openrowset_parquet(
+            parquet_path,
+            data_source=data_source,
+            table_alias=table_alias,
+            select_columns=select_columns,
+            where=where,
+            top=top,
+        )
+
+    def sqlserver_create_and_insert_from_parquet(
+        self,
+        parquet_path: PathLike,
+        output_dir: PathLike,
+        *,
+        table: str,
+        catalog: Optional[str] = None,
+        schema: Optional[str] = None,
+        drop_existing: bool = True,
+        use_go: bool = False,
+        schema_spec: Optional[dict] = None,
+        rows_per_file: Optional[int] = None,
+        memory_fraction: Optional[float] = None,
+        convert_types: Optional[bool] = None,
+        output_prefix: str = "part",
+        delimiter: str = ",",
+        include_header: bool = True,
+        line_terminator: str = "\n",
+        first_row: Optional[int] = None,
+        tablock: bool = True,
+        codepage: Optional[str] = None,
+        fallback_to_json: bool = False,
+    ) -> tuple[str, list[Path]]:
+        """Create SQL Server DDL + BULK INSERT statements from Parquet."""
+        return sqlserver_create_and_insert_from_parquet(
+            parquet_path=parquet_path,
+            output_dir=output_dir,
+            table=table,
+            catalog=catalog,
+            schema=schema,
+            drop_existing=drop_existing,
+            use_go=use_go,
+            schema_spec=schema_spec,
+            rows_per_file=rows_per_file,
+            memory_fraction=(
+                self.memory_fraction if memory_fraction is None else memory_fraction
+            ),
+            convert_types=(
+                self.convert_types if convert_types is None else convert_types
+            ),
+            output_prefix=output_prefix,
+            delimiter=delimiter,
+            include_header=include_header,
+            line_terminator=line_terminator,
+            first_row=first_row,
+            tablock=tablock,
+            codepage=codepage,
+            fallback_to_json=fallback_to_json,
+            verbose=self.verbose,
+        )
+
+    def sqlserver_create_and_stage_from_parquets(
+        self,
+        parquet_paths: Sequence[PathLike],
+        output_dir: PathLike,
+        *,
+        table: str,
+        catalog: Optional[str] = None,
+        schema: Optional[str] = None,
+        drop_existing: bool = True,
+        use_go: bool = False,
+        schema_spec: Optional[dict] = None,
+        rows_per_file: Optional[int] = None,
+        memory_fraction: Optional[float] = None,
+        convert_types: Optional[bool] = None,
+        output_prefix: str = "part",
+        delimiter: str = ",",
+        include_header: bool = True,
+        line_terminator: str = "\n",
+        fallback_to_json: bool = False,
+    ) -> tuple[str, list[Path]]:
+        """Generate SQL Server DDL and stage multiple Parquet files as CSVs."""
+        return sqlserver_create_and_stage_from_parquets(
+            parquet_paths=parquet_paths,
+            output_dir=output_dir,
+            table=table,
+            catalog=catalog,
+            schema=schema,
+            drop_existing=drop_existing,
+            use_go=use_go,
+            schema_spec=schema_spec,
+            rows_per_file=rows_per_file,
+            memory_fraction=(
+                self.memory_fraction if memory_fraction is None else memory_fraction
+            ),
+            convert_types=(
+                self.convert_types if convert_types is None else convert_types
+            ),
+            output_prefix=output_prefix,
+            delimiter=delimiter,
+            include_header=include_header,
+            line_terminator=line_terminator,
+            fallback_to_json=fallback_to_json,
+            verbose=self.verbose,
+        )

datablade/core/__init__.py

@@ -1,7 +1,28 @@
-import os, re 
-
-def find_python_files(path):
-    return [one_file_name.replace('.py','') for one_file_name in os.listdir(os.path.abspath(path)) if one_file_name != '__init__.py' and re.match(r'.*\.py$',one_file_name) is not None]
-
-for each_file in find_python_files(path=os.path.dirname(__file__)):
-    exec('from .'+each_file+' import *')
+"""Backward-compatible exports for the legacy datablade.core namespace.
+
+Historically, this package used dynamic imports to re-export everything.
+We keep the same runtime surface area but use explicit imports so that IDEs,
+type checkers, and static analysis tools can reason about the module.
+"""
+
+from . import frames as _frames
+from . import json as _json
+from . import lists as _lists
+from . import messages as _messages
+from . import strings as _strings
+from . import zip as _zip
+from .frames import *  # noqa: F401,F403
+from .json import *  # noqa: F401,F403
+from .lists import *  # noqa: F401,F403
+from .messages import *  # noqa: F401,F403
+from .strings import *  # noqa: F401,F403
+from .zip import *  # noqa: F401,F403
+
+__all__ = [
+    *_frames.__all__,
+    *_json.__all__,
+    *_lists.__all__,
+    *_messages.__all__,
+    *_strings.__all__,
+    *_zip.__all__,
+]

datablade/core/frames.py

@@ -1,236 +1,23 @@
-import pandas as pd 
-import pyarrow as pa
-import numpy as np 
-
-from .messages import print_verbose
-from .strings  import sql_quotename
-
-def try_cast_string_columns_to_numeric(df: pd.DataFrame=None, convert_partial: bool=False, verbose: bool=False) -> pd.DataFrame|None:
-    """
-    Attempt to cast DataFrame string columns to numeric values where possible.
-    
-    Parameters:
-        df (pd.DataFrame): The DataFrame to process.
-        convert_partial (bool): If True, columns with some values convertible to numeric types
-                                will be converted to numeric types with NaNs where conversion failed.
-                                If False, only columns where all values can be converted will be converted.
-    
-    Returns:
-        pd.DataFrame: DataFrame with string columns converted to numeric types where possible.
-    """
-    if df is None:
-        print_verbose("No DataFrame provided; exiting try_cast_string_columns_to_numeric.", verbose)
-        exit # Exit the function if no DataFrame is provided
-
-    for col in df.columns:
-        if df[col].dtype == 'object':
-            converted = pd.to_numeric(df[col], errors='coerce')
-            has_nan = converted.isnull().any()
-            if not has_nan:
-                df[col] = converted
-                print_verbose(f"Column '{col}' successfully converted to numeric.", verbose)
-            else:
-                if convert_partial:
-                    df[col] = converted
-                    print_verbose(f"Column '{col}' partially converted to numeric with NaNs where conversion failed.", verbose)
-                else:
-                    print_verbose(f"Column '{col}' could not be fully converted to numeric; leaving as is.", verbose)
-    return df
-
-def clean_dataframe_columns(df: pd.DataFrame=None, verbose: bool=False) -> pd.DataFrame|None:
-    """
-    Clean the DataFrame columns by:
-    - Flattening MultiIndex columns
-    - Converting non-string column names to strings
-    - Removing duplicate columns, keeping the first occurrence
-
-    Parameters:
-        df (pd.DataFrame): The DataFrame to clean.
-
-    Returns:
-        pd.DataFrame: The cleaned DataFrame.
-    """
-    if df is None:
-        print_verbose("No DataFrame provided; exiting clean_dataframe_columns.", verbose)
-        exit 
-    # Step 1: Flatten MultiIndex columns
-    if isinstance(df.columns, pd.MultiIndex):
-        df.columns = ['_'.join(map(str, col)).strip() for col in df.columns.values]
-        print_verbose("Flattened MultiIndex columns.", verbose)
-
-    # Step 2: Convert non-string column names to strings
-    df.columns = df.columns.map(str)
-    print_verbose("Converted column names to strings.", verbose)
-
-    # Step 3: Remove duplicate columns, keeping the first occurrence
-    duplicates = df.columns.duplicated()
-    if duplicates.any():
-        duplicate_cols = df.columns[duplicates]
-        print_verbose(f"Duplicate columns found: {list(duplicate_cols)}", verbose)
-        df = df.loc[:, ~duplicates]
-        print_verbose("Removed duplicate columns, keeping the first occurrence.", verbose)
-
-    return df
-
-def generate_parquet_schema(df: pd.DataFrame=None, verbose: bool=False) -> pa.Schema|None:
-    """
-    Generate a PyArrow Schema from a pandas DataFrame.
-    Parameters:
-        df (pandas.DataFrame): The DataFrame to generate the schema from.
-    Returns:
-        pyarrow.Schema: The PyArrow Schema object.
-    """
-    if df is None:
-        print_verbose("No DataFrame provided; exiting generate_parquet_schema.", verbose)
-        exit 
-    
-    fields = []
-    for column in df.columns:
-        col_data = df[column]
-        col_name = column
-        dtype = col_data.dtype
-
-        # Determine if the column contains any nulls
-        nullable = col_data.isnull().any()
-
-        # Map pandas dtype to PyArrow type
-        pa_type = None
-
-        if pd.api.types.is_integer_dtype(dtype):
-            # Check the range to determine the smallest integer type
-            min_value = col_data.min()
-            max_value = col_data.max()
-            if min_value >= np.iinfo(np.int8).min and max_value <= np.iinfo(np.int8).max:
-                pa_type = pa.int8()
-            elif min_value >= np.iinfo(np.int16).min and max_value <= np.iinfo(np.int16).max:
-                pa_type = pa.int16()
-            elif min_value >= np.iinfo(np.int32).min and max_value <= np.iinfo(np.int32).max:
-                pa_type = pa.int32()
-            else:
-                pa_type = pa.int64()
-
-        elif pd.api.types.is_float_dtype(dtype):
-            pa_type = pa.float64()
-
-        elif pd.api.types.is_bool_dtype(dtype):
-            pa_type = pa.bool_()
-
-        elif pd.api.types.is_datetime64_any_dtype(dtype):
-            pa_type = pa.timestamp('ms')
-
-        elif isinstance(dtype, pd.CategoricalDtype) or pd.api.types.is_object_dtype(dtype):
-            pa_type = pa.string()
-
-        else:
-            pa_type = pa.string()
-
-        # Create a field
-        field = pa.field(col_name, pa_type, nullable=nullable)
-        fields.append(field)
-
-    schema = pa.schema(fields)
-    return schema
-
-def pandas_to_parquet_table(df: pd.DataFrame=None, convert: bool=True, partial: bool=False, preserve_index: bool=False, verbose: bool=False) -> pa.Table|None:
-    """
-    Generate a PyArrow Table from a pandas DataFrame.
-
-    Parameters:
-        df (pandas.DataFrame): The DataFrame to generate the table from.
-        table (str): The name of the table.
-
-    Returns:
-        pyarrow.Table: The PyArrow Table object.
-    """
-    if df is None:
-        print_verbose("No DataFrame provided; exiting generate_parquet_table.", verbose)
-        exit 
-    
-    df     = clean_dataframe_columns(df=df, verbose=verbose)
-    
-    if convert:
-        df = try_cast_string_columns_to_numeric(df=df, convert_partial=partial, verbose=verbose)
-
-    schema = generate_parquet_schema(df=df, verbose=verbose)
-    try:
-        table = pa.Table.from_pandas(df, schema=schema, preserve_index=preserve_index)
-        return table
-    except Exception as e:
-        print_verbose(f"Error generating PyArrow Table: {e}", verbose)
-        exit 
-
-def generate_sql_server_create_table_string(df: pd.DataFrame=None, catalog: str='database', schema: str='dbo', table: str='table', dropexisting: bool=True, verbose: bool=False) -> str|None:
-    """
-    Generate a SQL Server CREATE TABLE string from a pandas DataFrame.
-
-    Parameters:
-        df (pandas.DataFrame): The DataFrame to generate the schema from.
-        table_name (str): The name of the SQL table.
-
-    Returns:
-        str: The SQL Server CREATE TABLE statement.
-    """
-    if df is None:
-        print_verbose("No DataFrame provided; exiting try_cast_string_columns_to_numeric.", verbose)
-        exit 
-    
-    table_name = f"{sql_quotename(catalog)}.{sql_quotename(schema)}.{sql_quotename(table)}"
-    drop_statement = f"use {sql_quotename(catalog)}\rgo\rif object_id('{table_name}') is not null drop table {table_name};\r" if dropexisting else ""
-    
-    create_statement = [f"{drop_statement};create table {table_name} ("]
-    indent = "    "
-    column_lines = []
-
-    for column in df.columns:
-        col_data = df[column]
-        col_name = column
-        dtype = col_data.dtype
-
-        # Determine if the column contains any nulls
-        nullable = col_data.isnull().any()
-        null_str = f"{'   ' if nullable else 'not'} null"
-
-        # Map pandas dtype to SQL Server type
-        sql_type = None
-
-        if pd.api.types.is_integer_dtype(dtype):
-            min_value = col_data.min()
-            max_value = col_data.max()
-            if min_value >= 0 and max_value <= 255:
-                sql_type = "tinyint"
-            elif min_value >= -32768 and max_value <= 32767:
-                sql_type = "smallint"
-            elif min_value >= -2147483648 and max_value <= 2147483647:
-                sql_type = "int"
-            else:
-                sql_type = "bigint"
-
-        elif pd.api.types.is_float_dtype(dtype):
-            sql_type = "float"
-
-        elif pd.api.types.is_bool_dtype(dtype):
-            sql_type = "bit"
-
-        elif pd.api.types.is_datetime64_any_dtype(dtype):
-            sql_type = "datetime2"
-
-        elif isinstance(dtype, pd.CategoricalDtype) or pd.api.types.is_object_dtype(dtype):
-            # Determine maximum length of string data
-            max_length = col_data.dropna().astype(str).map(len).max()
-            sql_type = f"nvarchar({str(max_length) if max_length <= 4000 else 'max'})"
-            
-        else:
-            sql_type = "nvarchar(max)"
-
-        # Build the column definition
-        column_line = f"{indent}{sql_quotename(col_name)} {sql_type} {null_str},"
-        column_lines.append(column_line)
-
-    # Remove the last comma from the last column definition
-    if column_lines:
-        column_lines[-1] = column_lines[-1].rstrip(',')
-
-    create_statement.extend(column_lines)
-    create_statement.append(");")
-    return_statement = "\r".join(create_statement)
-    return return_statement
+"""Backward-compatibility re-exports.
+
+This module intentionally contains no independent implementations.
+All functionality is provided by the newer modules in datablade.dataframes.
+"""
+
+from ..dataframes.frames import (  # noqa: F401
+    clean_dataframe_columns,
+    generate_parquet_schema,
+    generate_sql_server_create_table_string,
+    pandas_to_parquet_table,
+    try_cast_string_columns_to_numeric,
+    write_to_file_and_sql,
+)
+
+__all__ = [
+    "try_cast_string_columns_to_numeric",
+    "clean_dataframe_columns",
+    "generate_parquet_schema",
+    "pandas_to_parquet_table",
+    "generate_sql_server_create_table_string",
+    "write_to_file_and_sql",
+]

datablade/core/json.py

@@ -1,10 +1,5 @@
-import requests 
-from .messages import print_verbose
-
-def get(url: str, verbose: bool = False, **kwargs) -> dict:
-    """Get JSON data from a URL."""
-    try:
-        response = requests.get(url, **kwargs)
-        return response.json()
-    except requests.exceptions.RequestException as e:
-        print_verbose(f"Error: {e}", verbose=verbose)
+"""Backward-compatibility re-exports for IO JSON helpers."""
+
+from ..io.json import get  # noqa: F401
+
+__all__ = ["get"]