datablade 0.0.5__py3-none-any.whl → 0.0.6__py3-none-any.whl

datablade/__init__.py

@@ -12,24 +12,28 @@ For backward compatibility, all functions are also available from datablade.core
 
 # Also maintain core for backward compatibility
 # Import from new organized structure
-from . import core, dataframes, io, sql, utils
+from . import core, dataframes, io, registry, sql, utils
 from .blade import Blade
 from .dataframes import read_file_chunked, read_file_smart, read_file_to_parquets
+from .registry import DialectSpec, ObjectNode, ObjectRef, ObjectRegistry
 from .sql import Dialect, bulk_load, generate_create_table
 
 # Convenience re-exports for commonly used functions
 from .utils.logging import configure_logging, get_logger
+from .utils.strings import configure_paths
 
-__version__ = "0.0.5"
+__version__ = "0.0.6"
 
 __all__ = [
     "dataframes",
     "io",
     "utils",
     "sql",
+    "registry",
     "core",  # Maintain backward compatibility
     # Convenience re-exports
     "configure_logging",
+    "configure_paths",
     "get_logger",
     "read_file_smart",
     "read_file_chunked",
@@ -38,4 +42,8 @@ __all__ = [
     "generate_create_table",
     "bulk_load",
     "Blade",
+    "DialectSpec",
+    "ObjectRef",
+    "ObjectNode",
+    "ObjectRegistry",
 ]

datablade/blade.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Iterator, Optional, Union
+from typing import Any, Iterator, Optional, Sequence, Union
 
 import pandas as pd
 
@@ -19,9 +19,18 @@ from .dataframes import (
     read_file_smart,
     read_file_to_parquets,
     stream_to_parquets,
+    stream_to_sink,
     try_cast_string_columns_to_numeric,
 )
-from .sql import Dialect, generate_create_table, generate_create_table_from_parquet
+from .sql import (
+    Dialect,
+    ParquetDDLMetadata,
+    generate_create_table,
+    generate_create_table_from_parquet,
+    sqlserver_create_and_insert_from_parquet,
+    sqlserver_create_and_stage_from_parquets,
+    sqlserver_openrowset_parquet,
+)
 
 PathLike = Union[str, Path]
 
@@ -37,11 +46,18 @@ class Blade:
     verbose: bool = False
     convert_types: bool = True
 
-    def read(self, file_path: PathLike, **read_kwargs: Any) -> pd.DataFrame:
+    def read(
+        self,
+        file_path: PathLike,
+        *,
+        return_type: str = "dataframe",
+        **read_kwargs: Any,
+    ):
         return read_file_smart(
             file_path=file_path,
             memory_fraction=self.memory_fraction,
             verbose=self.verbose,
+            return_type=return_type,
             **read_kwargs,
         )
 
@@ -52,6 +68,7 @@ class Blade:
         chunksize: Optional[int] = None,
         **read_kwargs: Any,
     ) -> Iterator[pd.DataFrame]:
+        """Yield DataFrame chunks from a file without materializing."""
         return read_file_iter(
             file_path=file_path,
             chunksize=chunksize,
@@ -70,6 +87,7 @@ class Blade:
         convert_types: Optional[bool] = None,
         **read_kwargs: Any,
     ):
+        """Read and materialize a file into Parquet partitions."""
         return read_file_to_parquets(
             file_path=file_path,
             output_dir=output_dir,
@@ -93,6 +111,7 @@ class Blade:
         convert_types: Optional[bool] = None,
         **read_kwargs: Any,
     ):
+        """Stream input file chunks into Parquet partitions."""
         return stream_to_parquets(
             file_path=file_path,
             output_dir=output_dir,
@@ -106,10 +125,30 @@ class Blade:
             **read_kwargs,
         )
 
+    def stream_to_sink(
+        self,
+        chunks: Iterator[pd.DataFrame],
+        output_dir: PathLike,
+        *,
+        output_prefix: str = "part",
+        convert_types: Optional[bool] = None,
+    ):
+        return stream_to_sink(
+            chunks=chunks,
+            output_dir=output_dir,
+            output_prefix=output_prefix,
+            convert_types=(
+                self.convert_types if convert_types is None else convert_types
+            ),
+            verbose=self.verbose,
+        )
+
     def clean(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Normalize column names and remove duplicate columns."""
         return clean_dataframe_columns(df, verbose=self.verbose)
 
     def cast_numeric(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Attempt numeric conversion on string columns."""
         return try_cast_string_columns_to_numeric(df, verbose=self.verbose)
 
     def create_table_sql(
@@ -121,7 +160,10 @@ class Blade:
         table: str = "table",
         drop_existing: bool = True,
         dialect: Dialect = Dialect.SQLSERVER,
-    ) -> str:
+        use_go: bool = False,
+        schema_spec: Optional[dict] = None,
+    ) -> Union[str, tuple[str, ParquetDDLMetadata]]:
+        """Generate CREATE TABLE DDL from a DataFrame schema."""
         return generate_create_table(
             df=df,
             catalog=catalog,
@@ -129,19 +171,26 @@ class Blade:
             table=table,
             drop_existing=drop_existing,
             dialect=dialect,
+            use_go=use_go,
+            schema_spec=schema_spec,
             verbose=self.verbose,
         )
 
     def create_table_sql_from_parquet(
         self,
-        parquet_path: str,
+        parquet_path: PathLike,
         *,
         catalog: Optional[str] = None,
         schema: Optional[str] = None,
         table: str = "table",
         drop_existing: bool = True,
         dialect: Dialect = Dialect.SQLSERVER,
+        use_go: bool = False,
+        schema_spec: Optional[dict] = None,
+        fallback_to_json: bool = False,
+        return_metadata: bool = False,
     ) -> str:
+        """Generate CREATE TABLE DDL from a Parquet schema."""
         return generate_create_table_from_parquet(
             parquet_path=parquet_path,
             catalog=catalog,
@@ -149,5 +198,125 @@ class Blade:
             table=table,
             drop_existing=drop_existing,
             dialect=dialect,
+            use_go=use_go,
+            schema_spec=schema_spec,
+            verbose=self.verbose,
+            fallback_to_json=fallback_to_json,
+            return_metadata=return_metadata,
+        )
+
+    def sqlserver_openrowset_parquet(
+        self,
+        parquet_path: PathLike,
+        *,
+        data_source: Optional[str] = None,
+        table_alias: str = "rows",
+        select_columns: Optional[Sequence[str]] = None,
+        where: Optional[str] = None,
+        top: Optional[int] = None,
+    ) -> str:
+        """Generate a SQL Server OPENROWSET query over Parquet files."""
+        return sqlserver_openrowset_parquet(
+            parquet_path,
+            data_source=data_source,
+            table_alias=table_alias,
+            select_columns=select_columns,
+            where=where,
+            top=top,
+        )
+
+    def sqlserver_create_and_insert_from_parquet(
+        self,
+        parquet_path: PathLike,
+        output_dir: PathLike,
+        *,
+        table: str,
+        catalog: Optional[str] = None,
+        schema: Optional[str] = None,
+        drop_existing: bool = True,
+        use_go: bool = False,
+        schema_spec: Optional[dict] = None,
+        rows_per_file: Optional[int] = None,
+        memory_fraction: Optional[float] = None,
+        convert_types: Optional[bool] = None,
+        output_prefix: str = "part",
+        delimiter: str = ",",
+        include_header: bool = True,
+        line_terminator: str = "\n",
+        first_row: Optional[int] = None,
+        tablock: bool = True,
+        codepage: Optional[str] = None,
+        fallback_to_json: bool = False,
+    ) -> tuple[str, list[Path]]:
+        """Create SQL Server DDL + BULK INSERT statements from Parquet."""
+        return sqlserver_create_and_insert_from_parquet(
+            parquet_path=parquet_path,
+            output_dir=output_dir,
+            table=table,
+            catalog=catalog,
+            schema=schema,
+            drop_existing=drop_existing,
+            use_go=use_go,
+            schema_spec=schema_spec,
+            rows_per_file=rows_per_file,
+            memory_fraction=(
+                self.memory_fraction if memory_fraction is None else memory_fraction
+            ),
+            convert_types=(
+                self.convert_types if convert_types is None else convert_types
+            ),
+            output_prefix=output_prefix,
+            delimiter=delimiter,
+            include_header=include_header,
+            line_terminator=line_terminator,
+            first_row=first_row,
+            tablock=tablock,
+            codepage=codepage,
+            fallback_to_json=fallback_to_json,
+            verbose=self.verbose,
+        )
+
+    def sqlserver_create_and_stage_from_parquets(
+        self,
+        parquet_paths: Sequence[PathLike],
+        output_dir: PathLike,
+        *,
+        table: str,
+        catalog: Optional[str] = None,
+        schema: Optional[str] = None,
+        drop_existing: bool = True,
+        use_go: bool = False,
+        schema_spec: Optional[dict] = None,
+        rows_per_file: Optional[int] = None,
+        memory_fraction: Optional[float] = None,
+        convert_types: Optional[bool] = None,
+        output_prefix: str = "part",
+        delimiter: str = ",",
+        include_header: bool = True,
+        line_terminator: str = "\n",
+        fallback_to_json: bool = False,
+    ) -> tuple[str, list[Path]]:
+        """Generate SQL Server DDL and stage multiple Parquet files as CSVs."""
+        return sqlserver_create_and_stage_from_parquets(
+            parquet_paths=parquet_paths,
+            output_dir=output_dir,
+            table=table,
+            catalog=catalog,
+            schema=schema,
+            drop_existing=drop_existing,
+            use_go=use_go,
+            schema_spec=schema_spec,
+            rows_per_file=rows_per_file,
+            memory_fraction=(
+                self.memory_fraction if memory_fraction is None else memory_fraction
+            ),
+            convert_types=(
+                self.convert_types if convert_types is None else convert_types
+            ),
+            output_prefix=output_prefix,
+            delimiter=delimiter,
+            include_header=include_header,
+            line_terminator=line_terminator,
+            fallback_to_json=fallback_to_json,
             verbose=self.verbose,
         )

datablade/dataframes/__init__.py

@@ -19,11 +19,15 @@ from .frames import (
     write_to_file_and_sql,
 )
 from .readers import (
+    excel_to_parquets,
+    json_to_jsonl,
+    parquet_to_csv_partitions,
     read_file_chunked,
     read_file_iter,
     read_file_smart,
     read_file_to_parquets,
     stream_to_parquets,
+    stream_to_sink,
 )
 
 __all__ = [
@@ -35,9 +39,13 @@ __all__ = [
     "generate_sql_server_create_table_string",
     "write_to_file_and_sql",
     # Memory-aware readers
+    "excel_to_parquets",
+    "json_to_jsonl",
+    "parquet_to_csv_partitions",
     "read_file_chunked",
     "read_file_iter",
     "read_file_to_parquets",
+    "stream_to_sink",
     "stream_to_parquets",
     "read_file_smart",
 ]

datablade/dataframes/frames.py

@@ -1,25 +1,42 @@
+"""DataFrame transformation and export helpers.
+
+This module focuses on schema-aware DataFrame cleanup, type inference,
+and downstream serialization helpers (Parquet schema/table generation
+and SQL Server-compatible DDL helpers).
+"""
+
+import os
 import pathlib
+import shutil
 import subprocess
-from typing import Any, Optional
+from typing import Any, Optional, Union
 
 import numpy as np
 import pandas as pd
 import pyarrow as pa
 
 from ..utils.logging import log_debug, log_error, log_info, log_warning
+from ..utils.strings import coerce_path, ensure_directory
 
 _BYTES_LIKE_TYPES = (bytes, bytearray, memoryview)
 
 
 def _is_bytes_like(value: Any) -> bool:
+    """Return True when a value should be treated as binary data."""
     return isinstance(value, _BYTES_LIKE_TYPES)
 
 
 def _infer_object_pa_type(col_data: pd.Series) -> pa.DataType:
+    """Infer a PyArrow type for object-dtype Series.
+
+    We sample non-null values and prefer "narrower" types (binary, string,
+    bool, integer, float) before falling back to pyarrow's inference.
+    """
     non_null = col_data.dropna()
     if non_null.empty:
         return pa.string()
 
+    # Sample to avoid scanning very large object columns.
     sample = non_null.iloc[:100].tolist()
 
     if all(_is_bytes_like(v) for v in sample):
@@ -45,6 +62,8 @@ def _infer_object_pa_type(col_data: pd.Series) -> pa.DataType:
         return pa.float64()
 
     try:
+        # PyArrow can infer types for mixed object values; we normalize into
+        # the closest "primitive" type for DDL/Parquet stability.
         inferred = pa.infer_type(sample)
         if pa.types.is_binary(inferred) or pa.types.is_large_binary(inferred):
             return pa.binary()
@@ -93,9 +112,14 @@ def try_cast_string_columns_to_numeric(
         raise TypeError("df must be a pandas DataFrame")
 
     for col in df.columns:
-        if df[col].dtype == "object":
+        dtype = df[col].dtype
+        if pd.api.types.is_object_dtype(dtype) or pd.api.types.is_string_dtype(dtype):
             non_null = df[col].dropna()
-            if not non_null.empty and non_null.iloc[:100].map(_is_bytes_like).any():
+            if (
+                pd.api.types.is_object_dtype(dtype)
+                and not non_null.empty
+                and non_null.iloc[:100].map(_is_bytes_like).any()
+            ):
                 log_debug(
                     f"Column '{col}' contains bytes-like values; skipping numeric coercion.",
                     verbose,
@@ -144,16 +168,16 @@ def clean_dataframe_columns(
         raise ValueError("df must be provided")
     if not isinstance(df, pd.DataFrame):
         raise TypeError("df must be a pandas DataFrame")
-    # Step 1: Flatten MultiIndex columns
+    # Step 1: Flatten MultiIndex columns.
     if isinstance(df.columns, pd.MultiIndex):
         df.columns = ["_".join(map(str, col)).strip() for col in df.columns.values]
         log_info("Flattened MultiIndex columns.", verbose)
 
-    # Step 2: Convert non-string column names to strings
+    # Step 2: Convert non-string column names to strings.
     df.columns = df.columns.map(str)
     log_debug("Converted column names to strings.", verbose)
 
-    # Step 3: Remove duplicate columns, keeping the first occurrence
+    # Step 3: Remove duplicate columns, keeping the first occurrence.
     duplicates = df.columns.duplicated()
     if duplicates.any():
         duplicate_cols = df.columns[duplicates]
@@ -190,14 +214,14 @@ def generate_parquet_schema(
         col_name = column
         dtype = col_data.dtype
 
-        # Determine if the column contains any nulls
+        # Determine if the column contains any nulls.
         nullable = col_data.isnull().any()
 
-        # Map pandas dtype to PyArrow type
+        # Map pandas dtype to PyArrow type.
         pa_type = None
 
         if pd.api.types.is_integer_dtype(dtype):
-            # Check the range to determine the smallest integer type
+            # Check the range to determine the smallest integer type.
             non_null = col_data.dropna()
             if non_null.empty:
                 pa_type = pa.int64()
@@ -249,7 +273,7 @@ def generate_parquet_schema(
         else:
             pa_type = pa.string()
 
-        # Create a field
+        # Create a field.
         field = pa.field(col_name, pa_type, nullable=nullable)
         fields.append(field)
 
@@ -285,6 +309,7 @@ def pandas_to_parquet_table(
         raise TypeError("df must be a pandas DataFrame")
 
     def _unique_col_name(existing: set[str], desired: str) -> str:
+        """Return a column name that does not collide with existing names."""
         if desired not in existing:
             return desired
         i = 1
@@ -293,6 +318,7 @@ def pandas_to_parquet_table(
         return f"{desired}_{i}"
 
     def _materialize_index_columns(input_df: pd.DataFrame) -> pd.DataFrame:
+        """Convert Index/MultiIndex into explicit columns."""
         if isinstance(input_df.index, pd.MultiIndex):
             index_names: list[str] = []
             for i, name in enumerate(input_df.index.names):
@@ -319,16 +345,20 @@ def pandas_to_parquet_table(
         out_df = out_df.reset_index(drop=True)
         return out_df
 
+    # Clean columns before schema inference to avoid duplicate/invalid names.
     df = clean_dataframe_columns(df=df, verbose=verbose)
 
     if preserve_index:
+        # Preserve index by materializing it into real columns.
         df = _materialize_index_columns(df)
 
     if convert:
+        # Attempt numeric coercion so Parquet schema uses numeric types.
         df = try_cast_string_columns_to_numeric(
             df=df, convert_partial=partial, verbose=verbose
         )
 
+    # Build schema explicitly so the Parquet table uses stable, optimized types.
     schema = generate_parquet_schema(df=df, verbose=verbose)
     try:
         # We materialize index into regular columns above so that the schema can
@@ -347,6 +377,8 @@ def generate_sql_server_create_table_string(
     schema: str = "dbo",
     table: str = "table",
     dropexisting: bool = True,
+    use_go: bool = False,
+    schema_spec: Optional[dict] = None,
     verbose: bool = False,
 ) -> str:
     """
@@ -358,6 +390,7 @@ def generate_sql_server_create_table_string(
         schema: The schema name (default: 'dbo').
         table: The table name.
         dropexisting: If True, includes DROP TABLE IF EXISTS statement.
+        use_go: If True, inserts GO after USE when a catalog is supplied.
         verbose: If True, prints progress messages.
 
     Returns:
@@ -389,18 +422,23 @@ def generate_sql_server_create_table_string(
         table=table,
         drop_existing=dropexisting,
         dialect=Dialect.SQLSERVER,
+        use_go=use_go,
+        schema_spec=schema_spec,
         verbose=verbose,
     )
 
 
 def write_to_file_and_sql(
     df: pd.DataFrame,
-    file_path: str,
+    file_path: Union[str, pathlib.Path],
     table_name: str,
     sql_server: str,
     database: str,
-    username: str,
-    password: str,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    use_trusted_connection: bool = False,
+    use_azure_ad: bool = False,
+    use_env_credentials: bool = True,
     verbose: bool = False,
 ) -> None:
     """
@@ -414,6 +452,10 @@ def write_to_file_and_sql(
         database: Database name.
         username: SQL Server username.
         password: SQL Server password.
+        use_trusted_connection: If True, use integrated authentication (-T).
+        use_azure_ad: If True, use Azure AD authentication (-G).
+        use_env_credentials: If True, fall back to DATABLADE_SQLSERVER_USERNAME
+            and DATABLADE_SQLSERVER_PASSWORD when username/password not provided.
         verbose: If True, prints progress messages.
 
     Raises:
@@ -421,21 +463,34 @@ def write_to_file_and_sql(
     """
     if df is None or not isinstance(df, pd.DataFrame):
         raise TypeError("df must be a pandas DataFrame")
-    if not isinstance(file_path, str) or not file_path:
-        raise ValueError("file_path must be a non-empty string")
+    path_obj = coerce_path(
+        file_path,
+        must_exist=False,
+        verbose=verbose,
+        label="file_path",
+    )
     if not isinstance(table_name, str) or not table_name.strip():
         raise ValueError("table_name must be a non-empty string")
     if not isinstance(sql_server, str) or not sql_server.strip():
         raise ValueError("sql_server must be a non-empty string")
     if not isinstance(database, str) or not database.strip():
         raise ValueError("database must be a non-empty string")
-    if not isinstance(username, str) or not username.strip():
-        raise ValueError("username must be a non-empty string")
-    if not isinstance(password, str) or not password:
-        raise ValueError("password must be provided")
-
-    path_obj = pathlib.Path(file_path)
-    path_obj.parent.mkdir(parents=True, exist_ok=True)
+    if use_trusted_connection and use_azure_ad:
+        raise ValueError(
+            "use_trusted_connection and use_azure_ad are mutually exclusive"
+        )
+    if use_env_credentials:
+        if not username:
+            username = os.getenv("DATABLADE_SQLSERVER_USERNAME")
+        if not password:
+            password = os.getenv("DATABLADE_SQLSERVER_PASSWORD")
+    if not use_trusted_connection:
+        if not isinstance(username, str) or not username.strip():
+            raise ValueError("username must be a non-empty string")
+        if not password and not use_azure_ad:
+            raise ValueError("password must be provided")
+
+    ensure_directory(path_obj.parent, verbose=verbose, label="output_dir")
 
     df.to_csv(path_obj, index=False)
     log_info(f"DataFrame written to file {path_obj}.", verbose)
@@ -450,16 +505,57 @@ def write_to_file_and_sql(
         "-t,",
         "-S",
         sql_server,
-        "-U",
-        username,
-        "-P",
-        password,
     ]
+    bcp_preview = bcp_args[:-1] + ["***REDACTED***"]
+    bcp_path = shutil.which("bcp")
+    if not bcp_path:
+        install_steps = (
+            "Install the SQL Server command line utilities (bcp) and ensure the "
+            "binary is on PATH. For example: "
+            "macOS (Homebrew): brew install msodbcsql17 mssql-tools; "
+            "Linux (Debian/Ubuntu): install mssql-tools; "
+            "Windows: install SQL Server Command Line Utilities and restart your shell."
+        )
+        path_env = os.environ.get("PATH", "")
+        message = (
+            "BCP executable was not found on PATH. "
+            f"PATH={path_env}. {install_steps} "
+            f"Command preview: {bcp_preview}."
+        )
+        log_error(message, verbose)
+        raise FileNotFoundError(message)
+
+    if use_trusted_connection:
+        bcp_args.append("-T")
+    else:
+        if use_azure_ad:
+            bcp_args.append("-G")
+        if username:
+            bcp_args.extend(["-U", username])
+        if password:
+            bcp_args.extend(["-P", password])
 
     log_debug(
         f"Executing BCP load to {qualified_table} on server {sql_server} as user {username}.",
         verbose,
     )
+    redacted_args = []
+    redact_next = False
+    for arg in bcp_args:
+        if redact_next:
+            redacted_args.append("***REDACTED***")
+            redact_next = False
+            continue
+        redacted_args.append(arg)
+        if arg == "-P":
+            redact_next = True
+    if "-P" in bcp_args:
+        log_warning(
+            "BCP authentication uses -P with a plaintext password. "
+            "Consider using trusted connection (-T) or Azure AD (-G).",
+            verbose,
+        )
+    log_debug(f"BCP args: {redacted_args}", verbose)
     process = subprocess.run(
         bcp_args,
         shell=False,
@@ -474,7 +570,11 @@ def write_to_file_and_sql(
     else:
         error_msg = process.stderr.decode()
         log_error(
-            f"Error writing DataFrame to SQL Server table {table_name}: {error_msg}",
+            "Error writing DataFrame to SQL Server table "
+            f"{table_name}: {error_msg} "
+            f"PATH={os.environ.get('PATH', '')}. "
+            "Ensure BCP is installed (SQL Server command line utilities) and on PATH. "
+            f"Command preview: {bcp_preview}.",
             verbose,
         )
         raise subprocess.CalledProcessError(