datablade 0.0.0__py3-none-any.whl → 0.0.6__py3-none-any.whl

datablade/sql/sqlserver.py

@@ -0,0 +1,390 @@
+"""SQL Server-specific SQL helpers."""
+
+from __future__ import annotations
+
+import os
+import pathlib
+from typing import Any, Iterable, Mapping, Optional, Sequence
+
+from ..dataframes.readers import parquet_to_csv_partitions
+from ..utils.logging import log_warning
+from ..utils.strings import coerce_path
+from .ddl import _qualify_name
+from .ddl_pyarrow import generate_create_table_from_parquet
+from .dialects import Dialect
+from .quoting import quote_identifier
+
+
+def sqlserver_openrowset_parquet(
+    parquet_path: str | os.PathLike,
+    *,
+    data_source: Optional[str] = None,
+    table_alias: str = "rows",
+    select_columns: Optional[Sequence[str]] = None,
+    where: Optional[str] = None,
+    top: Optional[int] = None,
+) -> str:
+    """Generate a SQL Server OPENROWSET query over Parquet files.
+
+    Args:
+        parquet_path: Path or wildcard to Parquet files (directory or pattern).
+        data_source: Optional external data source name.
+        table_alias: Alias for the OPENROWSET rowset.
+        select_columns: Optional list of columns/expressions to select.
+        where: Optional WHERE clause (without the WHERE keyword).
+        top: Optional TOP limit.
+    """
+    try:
+        path_value = os.fspath(parquet_path)
+    except TypeError as exc:
+        raise TypeError("parquet_path must be a string or pathlib.Path") from exc
+
+    if not isinstance(path_value, str):
+        raise TypeError("parquet_path must be a string or pathlib.Path")
+    if not path_value.strip():
+        raise ValueError("parquet_path must be provided")
+
+    if data_source is not None and (
+        not isinstance(data_source, str) or not data_source.strip()
+    ):
+        raise ValueError("data_source, if provided, must be a non-empty string")
+    if not isinstance(table_alias, str) or not table_alias.strip():
+        raise ValueError("table_alias must be a non-empty string")
+    if top is not None:
+        if not isinstance(top, int) or top <= 0:
+            raise ValueError("top must be a positive integer")
+
+    path_literal = path_value.replace("'", "''")
+    options = [f"BULK '{path_literal}'", "FORMAT = 'PARQUET'"]
+    if data_source:
+        options.append(
+            f"DATA_SOURCE = {quote_identifier(data_source, Dialect.SQLSERVER)}"
+        )
+
+    select_list = "*" if not select_columns else ", ".join(select_columns)
+    top_clause = f"TOP ({top}) " if top is not None else ""
+    alias = quote_identifier(table_alias, Dialect.SQLSERVER)
+
+    statement = (
+        f"SELECT {top_clause}{select_list}\n"
+        "FROM OPENROWSET(\n"
+        f"    {', '.join(options)}\n"
+        f") AS {alias}"
+    )
+    if where:
+        statement = f"{statement}\nWHERE {where}"
+    return f"{statement};"
+
+
+def sqlserver_bulk_insert_statements(
+    csv_files: Iterable[str | os.PathLike],
+    *,
+    table: str,
+    catalog: Optional[str] = None,
+    schema: Optional[str] = None,
+    first_row: int = 2,
+    field_terminator: str = ",",
+    row_terminator: str = "0x0a",
+    tablock: bool = True,
+    codepage: Optional[str] = None,
+) -> str:
+    """Generate BULK INSERT statements for CSV files."""
+    if not isinstance(table, str) or not table.strip():
+        raise ValueError("table must be a non-empty string")
+    if catalog is not None and (not isinstance(catalog, str) or not catalog.strip()):
+        raise ValueError("catalog, if provided, must be a non-empty string")
+    if schema is not None and (not isinstance(schema, str) or not schema.strip()):
+        raise ValueError("schema, if provided, must be a non-empty string")
+    if not isinstance(first_row, int) or first_row <= 0:
+        raise ValueError("first_row must be a positive integer")
+    if not isinstance(field_terminator, str) or not field_terminator:
+        raise ValueError("field_terminator must be a non-empty string")
+    if not isinstance(row_terminator, str) or not row_terminator:
+        raise ValueError("row_terminator must be a non-empty string")
+    if codepage is not None and (not isinstance(codepage, str) or not codepage.strip()):
+        raise ValueError("codepage, if provided, must be a non-empty string")
+
+    qualified_table = _qualify_name(catalog, schema, table, Dialect.SQLSERVER)
+    statements: list[str] = []
+    for file_path in csv_files:
+        try:
+            path_value = os.fspath(file_path)
+        except TypeError as exc:
+            raise TypeError(
+                "csv_files must contain strings or pathlib.Path values"
+            ) from exc
+        if not isinstance(path_value, str) or not path_value.strip():
+            raise ValueError("csv_files must contain non-empty paths")
+
+        path_literal = path_value.replace("'", "''")
+        options = [
+            f"FIRSTROW = {first_row}",
+            f"FIELDTERMINATOR = '{field_terminator}'",
+            f"ROWTERMINATOR = '{row_terminator}'",
+        ]
+        if tablock:
+            options.append("TABLOCK")
+        if codepage:
+            options.append(f"CODEPAGE = '{codepage}'")
+        options_sql = ", ".join(options)
+        statements.append(
+            f"BULK INSERT {qualified_table} FROM '{path_literal}' WITH ({options_sql});"
+        )
+
+    if not statements:
+        raise ValueError("csv_files must contain at least one path")
+
+    return "\n".join(statements)
+
+
+def sqlserver_create_and_insert_from_parquet(
+    parquet_path: str | os.PathLike,
+    output_dir: str | os.PathLike,
+    *,
+    table: str,
+    catalog: Optional[str] = None,
+    schema: Optional[str] = None,
+    drop_existing: bool = True,
+    use_go: bool = False,
+    schema_spec: Optional[Mapping[str, Any]] = None,
+    rows_per_file: Optional[int] = None,
+    memory_fraction: float = 0.5,
+    convert_types: bool = True,
+    output_prefix: str = "part",
+    delimiter: str = ",",
+    include_header: bool = True,
+    line_terminator: str = "\n",
+    first_row: Optional[int] = None,
+    tablock: bool = True,
+    codepage: Optional[str] = None,
+    fallback_to_json: bool = False,
+    verbose: bool = False,
+) -> tuple[str, list[pathlib.Path]]:
+    """Create a SQL Server table from Parquet and generate CSV bulk insert SQL."""
+    import pyarrow.parquet as pq
+
+    path_obj = coerce_path(
+        parquet_path,
+        must_exist=True,
+        verbose=verbose,
+        label="parquet_path",
+    )
+    ddl, metadata = generate_create_table_from_parquet(
+        parquet_path=parquet_path,
+        catalog=catalog,
+        schema=schema,
+        table=table,
+        drop_existing=drop_existing,
+        use_go=use_go,
+        schema_spec=schema_spec,
+        dialect=Dialect.SQLSERVER,
+        verbose=verbose,
+        fallback_to_json=fallback_to_json,
+        return_metadata=True,
+    )
+
+    drop_columns: list[str] = []
+    if not fallback_to_json and metadata.dropped_columns:
+        drop_columns = [col.name for col in metadata.dropped_columns]
+
+    ref_schema = pq.ParquetFile(path_obj).schema_arrow
+    column_order = [
+        field.name for field in ref_schema if field.name not in drop_columns
+    ]
+
+    csv_files = parquet_to_csv_partitions(
+        file_path=parquet_path,
+        output_dir=output_dir,
+        output_prefix=output_prefix,
+        rows_per_file=rows_per_file,
+        memory_fraction=memory_fraction,
+        convert_types=convert_types,
+        verbose=verbose,
+        delimiter=delimiter,
+        include_header=include_header,
+        line_terminator=line_terminator,
+        drop_columns=drop_columns,
+        column_order=column_order,
+        drop_extra_columns=True,
+    )
+
+    if first_row is None:
+        first_row = 2 if include_header else 1
+
+    sql = sqlserver_bulk_insert_statements(
+        csv_files,
+        table=table,
+        catalog=catalog,
+        schema=schema,
+        first_row=first_row,
+        field_terminator=delimiter,
+        row_terminator=_sqlserver_row_terminator(line_terminator),
+        tablock=tablock,
+        codepage=codepage,
+    )
+
+    return f"{ddl}\n{sql}", csv_files
+
+
+def sqlserver_create_and_stage_from_parquets(
+    parquet_paths: Sequence[str | os.PathLike],
+    output_dir: str | os.PathLike,
+    *,
+    table: str,
+    catalog: Optional[str] = None,
+    schema: Optional[str] = None,
+    drop_existing: bool = True,
+    use_go: bool = False,
+    schema_spec: Optional[Mapping[str, Any]] = None,
+    rows_per_file: Optional[int] = None,
+    memory_fraction: float = 0.5,
+    convert_types: bool = True,
+    output_prefix: str = "part",
+    delimiter: str = ",",
+    include_header: bool = True,
+    line_terminator: str = "\n",
+    fallback_to_json: bool = False,
+    schema_strict: bool = True,
+    verbose: bool = False,
+) -> tuple[str, list[pathlib.Path]]:
+    """Generate SQL Server DDL and stage multiple Parquet files as CSV partitions.
+
+    Returns the CREATE TABLE DDL (derived from the first Parquet file) and a list
+    of CSV files produced from all Parquet inputs. Use bulk_load_sqlserver_many()
+    to load the returned CSV files via BCP.
+
+    Schema drift guard:
+      - Missing columns (vs. first file) raise a ValueError.
+      - Extra columns or type mismatches raise when schema_strict=True.
+      - When schema_strict=False, extra columns are dropped and type mismatches
+        are logged as warnings.
+    """
+    import pyarrow.parquet as pq
+
+    if parquet_paths is None:
+        raise ValueError("parquet_paths must be provided")
+    if not isinstance(parquet_paths, (list, tuple)):
+        raise TypeError("parquet_paths must be a list or tuple of paths")
+    if not parquet_paths:
+        raise ValueError("parquet_paths must contain at least one path")
+
+    first_path = coerce_path(
+        parquet_paths[0],
+        must_exist=True,
+        verbose=verbose,
+        label="parquet_paths[0]",
+    )
+    if first_path.suffix.lower() != ".parquet":
+        raise ValueError("parquet_paths must point to .parquet files")
+
+    if not isinstance(schema_strict, bool):
+        raise TypeError("schema_strict must be a boolean")
+
+    ddl, metadata = generate_create_table_from_parquet(
+        parquet_path=first_path,
+        catalog=catalog,
+        schema=schema,
+        table=table,
+        drop_existing=drop_existing,
+        use_go=use_go,
+        schema_spec=schema_spec,
+        dialect=Dialect.SQLSERVER,
+        verbose=verbose,
+        fallback_to_json=fallback_to_json,
+        return_metadata=True,
+    )
+
+    drop_columns: list[str] = []
+    if not fallback_to_json and metadata.dropped_columns:
+        drop_columns = [col.name for col in metadata.dropped_columns]
+
+    ref_schema = pq.ParquetFile(first_path).schema_arrow
+    ref_columns = [field.name for field in ref_schema if field.name not in drop_columns]
+    ref_types = {
+        field.name: field.type for field in ref_schema if field.name not in drop_columns
+    }
+
+    csv_files: list[pathlib.Path] = []
+    for index, parquet_path in enumerate(parquet_paths):
+        path_obj = coerce_path(
+            parquet_path,
+            must_exist=True,
+            verbose=verbose,
+            label=f"parquet_paths[{index}]",
+        )
+        if path_obj.suffix.lower() != ".parquet":
+            raise ValueError("parquet_paths must point to .parquet files")
+
+        current_schema = pq.ParquetFile(path_obj).schema_arrow
+        current_columns = [
+            field.name for field in current_schema if field.name not in drop_columns
+        ]
+        current_types = {
+            field.name: field.type
+            for field in current_schema
+            if field.name not in drop_columns
+        }
+
+        missing = [c for c in ref_columns if c not in current_columns]
+        if missing:
+            raise ValueError(
+                f"Schema drift detected in {path_obj}: missing columns {missing}."
+            )
+
+        extra = [c for c in current_columns if c not in ref_columns]
+        if extra:
+            message = f"Schema drift detected in {path_obj}: extra columns {extra}."
+            if schema_strict:
+                raise ValueError(message)
+            log_warning(message, verbose)
+
+        type_mismatches = [
+            (col, ref_types[col], current_types[col])
+            for col in ref_columns
+            if col in current_types and current_types[col] != ref_types[col]
+        ]
+        if type_mismatches:
+            details = ", ".join(
+                f"{col} (expected {expected}, got {actual})"
+                for col, expected, actual in type_mismatches
+            )
+            message = f"Schema drift detected in {path_obj}: type mismatches {details}."
+            if schema_strict:
+                raise ValueError(message)
+            log_warning(message, verbose)
+
+        if current_columns != ref_columns:
+            log_warning(
+                f"Column order mismatch detected in {path_obj}; "
+                "reordering to match the reference schema.",
+                verbose,
+            )
+
+        prefix = f"{output_prefix}_{index:05d}"
+        csv_files.extend(
+            parquet_to_csv_partitions(
+                file_path=path_obj,
+                output_dir=output_dir,
+                output_prefix=prefix,
+                rows_per_file=rows_per_file,
+                memory_fraction=memory_fraction,
+                convert_types=convert_types,
+                verbose=verbose,
+                delimiter=delimiter,
+                include_header=include_header,
+                line_terminator=line_terminator,
+                drop_columns=drop_columns,
+                column_order=ref_columns,
+                drop_extra_columns=True,
+            )
+        )
+
+    return ddl, csv_files
+
+
+def _sqlserver_row_terminator(line_terminator: str) -> str:
+    if line_terminator == "\n":
+        return "0x0a"
+    if line_terminator == "\r\n":
+        return "0x0d0a"
+    return line_terminator

datablade/utils/__init__.py

@@ -0,0 +1,38 @@
+"""
+General utility functions for common operations.
+
+This module provides functions for:
+- String manipulation and SQL name quoting
+- List operations (flattening)
+- Logging and messaging
+- Path standardization
+"""
+
+from .lists import flatten
+from .logging import print_verbose  # backward compatibility
+from .logging import (
+    configure_logging,
+    get_logger,
+    log,
+    log_debug,
+    log_error,
+    log_info,
+    log_warning,
+)
+from .strings import configure_paths, pathing, sql_quotename
+
+__all__ = [
+    "sql_quotename",
+    "pathing",
+    "configure_paths",
+    "flatten",
+    # Logging
+    "get_logger",
+    "configure_logging",
+    "log",
+    "log_debug",
+    "log_info",
+    "log_warning",
+    "log_error",
+    "print_verbose",
+]

datablade/utils/lists.py

@@ -0,0 +1,32 @@
+"""List helpers for common transformations."""
+
+from typing import Any, List
+
+
+def flatten(nest: List[Any]) -> List[Any]:
+    """
+    Flatten a nested list recursively to a single-level list.
+
+    Args:
+        nest: A potentially nested list structure.
+
+    Returns:
+        A flat list containing all elements from the nested structure.
+
+    Examples:
+        >>> flatten([1, [2, 3], [[4], 5]])
+        [1, 2, 3, 4, 5]
+        >>> flatten([1, 2, 3])
+        [1, 2, 3]
+    """
+    if not isinstance(nest, list):
+        raise TypeError("nest must be a list")
+
+    # Build a new list so the caller's input is untouched.
+    result = []
+    for item in nest:
+        if isinstance(item, list):
+            result.extend(flatten(item))
+        else:
+            result.append(item)
+    return result

datablade/utils/logging.py

@@ -0,0 +1,204 @@
+"""
+Logging utilities for datablade.
+
+Provides a configurable logger that can be used across all modules.
+By default, logs to console at INFO level. Users can configure
+handlers, levels, and formatters as needed.
+"""
+
+import logging
+import pathlib
+import time
+from contextlib import contextmanager
+from typing import Any, Iterator, Mapping, Optional
+
+# Create the datablade logger
+_logger = logging.getLogger("datablade")
+_logger.setLevel(logging.DEBUG)  # Allow all levels; handlers control output
+
+# Default console handler (can be replaced by user)
+_default_handler: Optional[logging.Handler] = None
+
+
+def _ensure_handler() -> None:
+    """Ensure at least one handler is configured."""
+    global _default_handler
+    if not _logger.handlers and _default_handler is None:
+        _default_handler = logging.StreamHandler()
+        _default_handler.setLevel(logging.INFO)
+        formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        )
+        _default_handler.setFormatter(formatter)
+        _logger.addHandler(_default_handler)
+
+
+def get_logger() -> logging.Logger:
+    """
+    Get the datablade logger instance.
+
+    Returns:
+        The configured datablade logger.
+    """
+    _ensure_handler()
+    return _logger
+
+
+def configure_logging(
+    level: int = logging.INFO,
+    handler: Optional[logging.Handler] = None,
+    format_string: Optional[str] = None,
+    *,
+    log_file: Optional[str | pathlib.Path] = None,
+    format: Optional[str] = None,
+) -> logging.Logger:
+    """
+    Configure the datablade logger.
+
+    Args:
+        level: Logging level (e.g., logging.DEBUG, logging.INFO).
+        handler: Optional custom handler. If None, uses StreamHandler.
+        format_string: Optional format string for log messages.
+
+    Returns:
+        The configured logger instance.
+    """
+    global _default_handler
+
+    if format is not None:
+        if format_string is not None:
+            raise ValueError("Provide only one of format_string or format")
+        format_string = format
+
+    # Remove existing handlers
+    for h in _logger.handlers[:]:
+        _logger.removeHandler(h)
+    _default_handler = None
+
+    # Add new handler
+    if handler is None:
+        if log_file is not None:
+            log_path = pathlib.Path(log_file)
+            if log_path.parent:
+                log_path.parent.mkdir(parents=True, exist_ok=True)
+            handler = logging.FileHandler(log_path, encoding="utf-8")
+        else:
+            handler = logging.StreamHandler()
+
+    handler.setLevel(level)
+
+    if format_string:
+        formatter = logging.Formatter(format_string)
+    else:
+        formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+        )
+    handler.setFormatter(formatter)
+
+    _logger.addHandler(handler)
+    _default_handler = handler
+
+    return _logger
+
+
+def log(
+    message: Any,
+    level: int = logging.INFO,
+    verbose: bool = True,
+) -> None:
+    """
+    Log a message at the specified level if verbose is True.
+
+    Args:
+        message: The message to log (converted to string).
+        level: Logging level (default: INFO).
+        verbose: If False, message is not logged.
+
+    Returns:
+        None
+    """
+    if not verbose:
+        return
+
+    _ensure_handler()
+    _logger.log(level, str(message))
+
+
+def log_debug(message: Any, verbose: bool = True) -> None:
+    """Log a DEBUG level message."""
+    log(message, logging.DEBUG, verbose)
+
+
+def log_info(message: Any, verbose: bool = True) -> None:
+    """Log an INFO level message."""
+    log(message, logging.INFO, verbose)
+
+
+def log_warning(message: Any, verbose: bool = True) -> None:
+    """Log a WARNING level message."""
+    log(message, logging.WARNING, verbose)
+
+
+def log_error(message: Any, verbose: bool = True) -> None:
+    """Log an ERROR level message."""
+    log(message, logging.ERROR, verbose)
+
+
+def build_log_context(
+    *,
+    file_path: Optional[str | pathlib.Path] = None,
+    chunk_index: Optional[int] = None,
+    **fields: Any,
+) -> dict[str, Any]:
+    """Build a logging context dict with common fields like file and chunk."""
+    context = dict(fields)
+    if file_path is not None:
+        context.setdefault("file", pathlib.Path(file_path).name)
+    if chunk_index is not None:
+        context.setdefault("chunk", chunk_index)
+    return context
+
+
+def format_log_context(context: Optional[Mapping[str, Any]]) -> str:
+    """Format a context mapping into a compact suffix for log messages."""
+    if not context:
+        return ""
+
+    parts: list[str] = []
+    for key, value in context.items():
+        if value is None:
+            continue
+        parts.append(f"{key}={value}")
+
+    if not parts:
+        return ""
+
+    return f" ({', '.join(parts)})"
+
+
+@contextmanager
+def timed_step(name: str, *, verbose: bool = True) -> Iterator[None]:
+    """Measure elapsed time for a block and log on exit."""
+    start = time.perf_counter()
+    try:
+        yield
+    finally:
+        duration = time.perf_counter() - start
+        log_info(f"{name} took {duration:.2f}s", verbose)
+
+
+# Backward compatibility alias
+def print_verbose(message: Any, verbose: bool = True) -> None:
+    """
+    Print a message if verbose is True.
+
+    This is a backward-compatible alias for log_info.
+
+    Args:
+        message: The message to print (converted to string).
+        verbose: If True, the message will be logged.
+
+    Returns:
+        None
+    """
+    log_info(message, verbose)

datablade/utils/messages.py

@@ -0,0 +1,29 @@
+"""
+Messaging utilities for datablade.
+
+This module provides backward-compatible message functions.
+For new code, prefer using datablade.utils.logging directly.
+"""
+
+# Re-export from logging module for backward compatibility
+from .logging import (
+    configure_logging,
+    get_logger,
+    log,
+    log_debug,
+    log_error,
+    log_info,
+    log_warning,
+    print_verbose,
+)
+
+__all__ = [
+    "print_verbose",
+    "log",
+    "log_debug",
+    "log_info",
+    "log_warning",
+    "log_error",
+    "get_logger",
+    "configure_logging",
+]