datablade 0.0.5__py3-none-any.whl → 0.0.6__py3-none-any.whl

datablade/sql/sqlserver.py

@@ -0,0 +1,390 @@
+"""SQL Server-specific SQL helpers."""
+
+from __future__ import annotations
+
+import os
+import pathlib
+from typing import Any, Iterable, Mapping, Optional, Sequence
+
+from ..dataframes.readers import parquet_to_csv_partitions
+from ..utils.logging import log_warning
+from ..utils.strings import coerce_path
+from .ddl import _qualify_name
+from .ddl_pyarrow import generate_create_table_from_parquet
+from .dialects import Dialect
+from .quoting import quote_identifier
+
+
+def sqlserver_openrowset_parquet(
+    parquet_path: str | os.PathLike,
+    *,
+    data_source: Optional[str] = None,
+    table_alias: str = "rows",
+    select_columns: Optional[Sequence[str]] = None,
+    where: Optional[str] = None,
+    top: Optional[int] = None,
+) -> str:
+    """Generate a SQL Server OPENROWSET query over Parquet files.
+
+    Args:
+        parquet_path: Path or wildcard to Parquet files (directory or pattern).
+        data_source: Optional external data source name.
+        table_alias: Alias for the OPENROWSET rowset.
+        select_columns: Optional list of columns/expressions to select.
+        where: Optional WHERE clause (without the WHERE keyword).
+        top: Optional TOP limit.
+    """
+    try:
+        path_value = os.fspath(parquet_path)
+    except TypeError as exc:
+        raise TypeError("parquet_path must be a string or pathlib.Path") from exc
+
+    if not isinstance(path_value, str):
+        raise TypeError("parquet_path must be a string or pathlib.Path")
+    if not path_value.strip():
+        raise ValueError("parquet_path must be provided")
+
+    if data_source is not None and (
+        not isinstance(data_source, str) or not data_source.strip()
+    ):
+        raise ValueError("data_source, if provided, must be a non-empty string")
+    if not isinstance(table_alias, str) or not table_alias.strip():
+        raise ValueError("table_alias must be a non-empty string")
+    if top is not None:
+        if not isinstance(top, int) or top <= 0:
+            raise ValueError("top must be a positive integer")
+
+    path_literal = path_value.replace("'", "''")
+    options = [f"BULK '{path_literal}'", "FORMAT = 'PARQUET'"]
+    if data_source:
+        options.append(
+            f"DATA_SOURCE = {quote_identifier(data_source, Dialect.SQLSERVER)}"
+        )
+
+    select_list = "*" if not select_columns else ", ".join(select_columns)
+    top_clause = f"TOP ({top}) " if top is not None else ""
+    alias = quote_identifier(table_alias, Dialect.SQLSERVER)
+
+    statement = (
+        f"SELECT {top_clause}{select_list}\n"
+        "FROM OPENROWSET(\n"
+        f"    {', '.join(options)}\n"
+        f") AS {alias}"
+    )
+    if where:
+        statement = f"{statement}\nWHERE {where}"
+    return f"{statement};"
+
+
+def sqlserver_bulk_insert_statements(
+    csv_files: Iterable[str | os.PathLike],
+    *,
+    table: str,
+    catalog: Optional[str] = None,
+    schema: Optional[str] = None,
+    first_row: int = 2,
+    field_terminator: str = ",",
+    row_terminator: str = "0x0a",
+    tablock: bool = True,
+    codepage: Optional[str] = None,
+) -> str:
+    """Generate BULK INSERT statements for CSV files."""
+    if not isinstance(table, str) or not table.strip():
+        raise ValueError("table must be a non-empty string")
+    if catalog is not None and (not isinstance(catalog, str) or not catalog.strip()):
+        raise ValueError("catalog, if provided, must be a non-empty string")
+    if schema is not None and (not isinstance(schema, str) or not schema.strip()):
+        raise ValueError("schema, if provided, must be a non-empty string")
+    if not isinstance(first_row, int) or first_row <= 0:
+        raise ValueError("first_row must be a positive integer")
+    if not isinstance(field_terminator, str) or not field_terminator:
+        raise ValueError("field_terminator must be a non-empty string")
+    if not isinstance(row_terminator, str) or not row_terminator:
+        raise ValueError("row_terminator must be a non-empty string")
+    if codepage is not None and (not isinstance(codepage, str) or not codepage.strip()):
+        raise ValueError("codepage, if provided, must be a non-empty string")
+
+    qualified_table = _qualify_name(catalog, schema, table, Dialect.SQLSERVER)
+    statements: list[str] = []
+    for file_path in csv_files:
+        try:
+            path_value = os.fspath(file_path)
+        except TypeError as exc:
+            raise TypeError(
+                "csv_files must contain strings or pathlib.Path values"
+            ) from exc
+        if not isinstance(path_value, str) or not path_value.strip():
+            raise ValueError("csv_files must contain non-empty paths")
+
+        path_literal = path_value.replace("'", "''")
+        options = [
+            f"FIRSTROW = {first_row}",
+            f"FIELDTERMINATOR = '{field_terminator}'",
+            f"ROWTERMINATOR = '{row_terminator}'",
+        ]
+        if tablock:
+            options.append("TABLOCK")
+        if codepage:
+            options.append(f"CODEPAGE = '{codepage}'")
+        options_sql = ", ".join(options)
+        statements.append(
+            f"BULK INSERT {qualified_table} FROM '{path_literal}' WITH ({options_sql});"
+        )
+
+    if not statements:
+        raise ValueError("csv_files must contain at least one path")
+
+    return "\n".join(statements)
+
+
+def sqlserver_create_and_insert_from_parquet(
+    parquet_path: str | os.PathLike,
+    output_dir: str | os.PathLike,
+    *,
+    table: str,
+    catalog: Optional[str] = None,
+    schema: Optional[str] = None,
+    drop_existing: bool = True,
+    use_go: bool = False,
+    schema_spec: Optional[Mapping[str, Any]] = None,
+    rows_per_file: Optional[int] = None,
+    memory_fraction: float = 0.5,
+    convert_types: bool = True,
+    output_prefix: str = "part",
+    delimiter: str = ",",
+    include_header: bool = True,
+    line_terminator: str = "\n",
+    first_row: Optional[int] = None,
+    tablock: bool = True,
+    codepage: Optional[str] = None,
+    fallback_to_json: bool = False,
+    verbose: bool = False,
+) -> tuple[str, list[pathlib.Path]]:
+    """Create a SQL Server table from Parquet and generate CSV bulk insert SQL."""
+    import pyarrow.parquet as pq
+
+    path_obj = coerce_path(
+        parquet_path,
+        must_exist=True,
+        verbose=verbose,
+        label="parquet_path",
+    )
+    ddl, metadata = generate_create_table_from_parquet(
+        parquet_path=parquet_path,
+        catalog=catalog,
+        schema=schema,
+        table=table,
+        drop_existing=drop_existing,
+        use_go=use_go,
+        schema_spec=schema_spec,
+        dialect=Dialect.SQLSERVER,
+        verbose=verbose,
+        fallback_to_json=fallback_to_json,
+        return_metadata=True,
+    )
+
+    drop_columns: list[str] = []
+    if not fallback_to_json and metadata.dropped_columns:
+        drop_columns = [col.name for col in metadata.dropped_columns]
+
+    ref_schema = pq.ParquetFile(path_obj).schema_arrow
+    column_order = [
+        field.name for field in ref_schema if field.name not in drop_columns
+    ]
+
+    csv_files = parquet_to_csv_partitions(
+        file_path=parquet_path,
+        output_dir=output_dir,
+        output_prefix=output_prefix,
+        rows_per_file=rows_per_file,
+        memory_fraction=memory_fraction,
+        convert_types=convert_types,
+        verbose=verbose,
+        delimiter=delimiter,
+        include_header=include_header,
+        line_terminator=line_terminator,
+        drop_columns=drop_columns,
+        column_order=column_order,
+        drop_extra_columns=True,
+    )
+
+    if first_row is None:
+        first_row = 2 if include_header else 1
+
+    sql = sqlserver_bulk_insert_statements(
+        csv_files,
+        table=table,
+        catalog=catalog,
+        schema=schema,
+        first_row=first_row,
+        field_terminator=delimiter,
+        row_terminator=_sqlserver_row_terminator(line_terminator),
+        tablock=tablock,
+        codepage=codepage,
+    )
+
+    return f"{ddl}\n{sql}", csv_files
+
+
+def sqlserver_create_and_stage_from_parquets(
+    parquet_paths: Sequence[str | os.PathLike],
+    output_dir: str | os.PathLike,
+    *,
+    table: str,
+    catalog: Optional[str] = None,
+    schema: Optional[str] = None,
+    drop_existing: bool = True,
+    use_go: bool = False,
+    schema_spec: Optional[Mapping[str, Any]] = None,
+    rows_per_file: Optional[int] = None,
+    memory_fraction: float = 0.5,
+    convert_types: bool = True,
+    output_prefix: str = "part",
+    delimiter: str = ",",
+    include_header: bool = True,
+    line_terminator: str = "\n",
+    fallback_to_json: bool = False,
+    schema_strict: bool = True,
+    verbose: bool = False,
+) -> tuple[str, list[pathlib.Path]]:
+    """Generate SQL Server DDL and stage multiple Parquet files as CSV partitions.
+
+    Returns the CREATE TABLE DDL (derived from the first Parquet file) and a list
+    of CSV files produced from all Parquet inputs. Use bulk_load_sqlserver_many()
+    to load the returned CSV files via BCP.
+
+    Schema drift guard:
+      - Missing columns (vs. first file) raise a ValueError.
+      - Extra columns or type mismatches raise when schema_strict=True.
+      - When schema_strict=False, extra columns are dropped and type mismatches
+        are logged as warnings.
+    """
+    import pyarrow.parquet as pq
+
+    if parquet_paths is None:
+        raise ValueError("parquet_paths must be provided")
+    if not isinstance(parquet_paths, (list, tuple)):
+        raise TypeError("parquet_paths must be a list or tuple of paths")
+    if not parquet_paths:
+        raise ValueError("parquet_paths must contain at least one path")
+
+    first_path = coerce_path(
+        parquet_paths[0],
+        must_exist=True,
+        verbose=verbose,
+        label="parquet_paths[0]",
+    )
+    if first_path.suffix.lower() != ".parquet":
+        raise ValueError("parquet_paths must point to .parquet files")
+
+    if not isinstance(schema_strict, bool):
+        raise TypeError("schema_strict must be a boolean")
+
+    ddl, metadata = generate_create_table_from_parquet(
+        parquet_path=first_path,
+        catalog=catalog,
+        schema=schema,
+        table=table,
+        drop_existing=drop_existing,
+        use_go=use_go,
+        schema_spec=schema_spec,
+        dialect=Dialect.SQLSERVER,
+        verbose=verbose,
+        fallback_to_json=fallback_to_json,
+        return_metadata=True,
+    )
+
+    drop_columns: list[str] = []
+    if not fallback_to_json and metadata.dropped_columns:
+        drop_columns = [col.name for col in metadata.dropped_columns]
+
+    ref_schema = pq.ParquetFile(first_path).schema_arrow
+    ref_columns = [field.name for field in ref_schema if field.name not in drop_columns]
+    ref_types = {
+        field.name: field.type for field in ref_schema if field.name not in drop_columns
+    }
+
+    csv_files: list[pathlib.Path] = []
+    for index, parquet_path in enumerate(parquet_paths):
+        path_obj = coerce_path(
+            parquet_path,
+            must_exist=True,
+            verbose=verbose,
+            label=f"parquet_paths[{index}]",
+        )
+        if path_obj.suffix.lower() != ".parquet":
+            raise ValueError("parquet_paths must point to .parquet files")
+
+        current_schema = pq.ParquetFile(path_obj).schema_arrow
+        current_columns = [
+            field.name for field in current_schema if field.name not in drop_columns
+        ]
+        current_types = {
+            field.name: field.type
+            for field in current_schema
+            if field.name not in drop_columns
+        }
+
+        missing = [c for c in ref_columns if c not in current_columns]
+        if missing:
+            raise ValueError(
+                f"Schema drift detected in {path_obj}: missing columns {missing}."
+            )
+
+        extra = [c for c in current_columns if c not in ref_columns]
+        if extra:
+            message = f"Schema drift detected in {path_obj}: extra columns {extra}."
+            if schema_strict:
+                raise ValueError(message)
+            log_warning(message, verbose)
+
+        type_mismatches = [
+            (col, ref_types[col], current_types[col])
+            for col in ref_columns
+            if col in current_types and current_types[col] != ref_types[col]
+        ]
+        if type_mismatches:
+            details = ", ".join(
+                f"{col} (expected {expected}, got {actual})"
+                for col, expected, actual in type_mismatches
+            )
+            message = f"Schema drift detected in {path_obj}: type mismatches {details}."
+            if schema_strict:
+                raise ValueError(message)
+            log_warning(message, verbose)
+
+        if current_columns != ref_columns:
+            log_warning(
+                f"Column order mismatch detected in {path_obj}; "
+                "reordering to match the reference schema.",
+                verbose,
+            )
+
+        prefix = f"{output_prefix}_{index:05d}"
+        csv_files.extend(
+            parquet_to_csv_partitions(
+                file_path=path_obj,
+                output_dir=output_dir,
+                output_prefix=prefix,
+                rows_per_file=rows_per_file,
+                memory_fraction=memory_fraction,
+                convert_types=convert_types,
+                verbose=verbose,
+                delimiter=delimiter,
+                include_header=include_header,
+                line_terminator=line_terminator,
+                drop_columns=drop_columns,
+                column_order=ref_columns,
+                drop_extra_columns=True,
+            )
+        )
+
+    return ddl, csv_files
+
+
+def _sqlserver_row_terminator(line_terminator: str) -> str:
+    if line_terminator == "\n":
+        return "0x0a"
+    if line_terminator == "\r\n":
+        return "0x0d0a"
+    return line_terminator

datablade/utils/__init__.py

@@ -19,11 +19,12 @@ from .logging import (
     log_info,
     log_warning,
 )
-from .strings import pathing, sql_quotename
+from .strings import configure_paths, pathing, sql_quotename
 
 __all__ = [
     "sql_quotename",
     "pathing",
+    "configure_paths",
     "flatten",
     # Logging
     "get_logger",

datablade/utils/lists.py

@@ -1,3 +1,5 @@
+"""List helpers for common transformations."""
+
 from typing import Any, List
 
 
@@ -20,6 +22,7 @@ def flatten(nest: List[Any]) -> List[Any]:
     if not isinstance(nest, list):
         raise TypeError("nest must be a list")
 
+    # Build a new list so the caller's input is untouched.
     result = []
     for item in nest:
         if isinstance(item, list):

datablade/utils/logging.py

@@ -8,7 +8,9 @@ handlers, levels, and formatters as needed.
 
 import logging
 import pathlib
-from typing import Any, Optional
+import time
+from contextlib import contextmanager
+from typing import Any, Iterator, Mapping, Optional
 
 # Create the datablade logger
 _logger = logging.getLogger("datablade")
@@ -142,6 +144,49 @@ def log_error(message: Any, verbose: bool = True) -> None:
     log(message, logging.ERROR, verbose)
 
 
+def build_log_context(
+    *,
+    file_path: Optional[str | pathlib.Path] = None,
+    chunk_index: Optional[int] = None,
+    **fields: Any,
+) -> dict[str, Any]:
+    """Build a logging context dict with common fields like file and chunk."""
+    context = dict(fields)
+    if file_path is not None:
+        context.setdefault("file", pathlib.Path(file_path).name)
+    if chunk_index is not None:
+        context.setdefault("chunk", chunk_index)
+    return context
+
+
+def format_log_context(context: Optional[Mapping[str, Any]]) -> str:
+    """Format a context mapping into a compact suffix for log messages."""
+    if not context:
+        return ""
+
+    parts: list[str] = []
+    for key, value in context.items():
+        if value is None:
+            continue
+        parts.append(f"{key}={value}")
+
+    if not parts:
+        return ""
+
+    return f" ({', '.join(parts)})"
+
+
+@contextmanager
+def timed_step(name: str, *, verbose: bool = True) -> Iterator[None]:
+    """Measure elapsed time for a block and log on exit."""
+    start = time.perf_counter()
+    try:
+        yield
+    finally:
+        duration = time.perf_counter() - start
+        log_info(f"{name} took {duration:.2f}s", verbose)
+
+
 # Backward compatibility alias
 def print_verbose(message: Any, verbose: bool = True) -> None:
     """

datablade/utils/strings.py

@@ -1,8 +1,181 @@
+"""String and path helpers used across datablade."""
+
+import os
 import pathlib
+from functools import singledispatch
 from typing import Optional, Union
 
+from .logging import log_warning
 from .messages import print_verbose
 
+PathInput = Union[str, pathlib.Path, os.PathLike]
+_PATH_STRICT_DEFAULT = False
+
+
+def configure_paths(*, path_strict: bool = False) -> None:
+    """Configure global path handling behavior."""
+    global _PATH_STRICT_DEFAULT
+    _PATH_STRICT_DEFAULT = bool(path_strict)
+
+
+def _resolve_path_strict(path_strict: Optional[bool]) -> bool:
+    if path_strict is None:
+        return _PATH_STRICT_DEFAULT
+    return bool(path_strict)
+
+
+@singledispatch
+def _coerce_path_input(value: object, type_label: str) -> str:
+    raise TypeError(f"{type_label} must be a string or pathlib.Path")
+
+
+@_coerce_path_input.register
+def _(value: str, type_label: str) -> str:
+    return value
+
+
+@_coerce_path_input.register
+def _(value: pathlib.Path, type_label: str) -> str:
+    return str(value)
+
+
+@_coerce_path_input.register
+def _(value: os.PathLike, type_label: str) -> str:
+    path_value = os.fspath(value)
+    if isinstance(path_value, bytes):
+        raise TypeError(f"{type_label} must be a string or pathlib.Path")
+    return path_value
+
+
+@_coerce_path_input.register
+def _(value: bytes, type_label: str) -> str:
+    raise TypeError(f"{type_label} must be a string or pathlib.Path")
+
+
+def _normalize_path_value(path_value: str) -> str:
+    if os.name == "nt":
+        return path_value.replace("\\", "/")
+    return path_value
+
+
+def _find_case_conflicts(
+    path_obj: pathlib.Path,
+    raw_path: str,
+) -> list[tuple[str, str, str]]:
+    """Return case mismatches as (provided, actual, parent) tuples."""
+    if not raw_path:
+        return []
+
+    try:
+        input_path = pathlib.Path(raw_path)
+    except Exception:
+        input_path = path_obj
+
+    if input_path.is_absolute():
+        anchor = input_path.anchor
+        if anchor:
+            current = pathlib.Path(anchor)
+            anchor_parts = pathlib.Path(anchor).parts
+            remaining_parts = input_path.parts[len(anchor_parts) :]
+        else:
+            current = pathlib.Path(anchor)
+            remaining_parts = input_path.parts
+    else:
+        current = pathlib.Path.cwd()
+        remaining_parts = input_path.parts
+
+    mismatches: list[tuple[str, str, str]] = []
+    for part in remaining_parts:
+        if part in ("", "."):
+            continue
+        if part == "..":
+            current = current.parent
+            continue
+        try:
+            with os.scandir(current) as entries:
+                actual = None
+                for entry in entries:
+                    if entry.name.casefold() == part.casefold():
+                        actual = entry.name
+                        break
+        except OSError:
+            break
+
+        if actual is None:
+            current = current / part
+            continue
+        if actual != part:
+            mismatches.append((part, actual, str(current)))
+        current = current / actual
+
+    return mismatches
+
+
+def coerce_path(
+    input: Optional[PathInput],
+    *,
+    must_exist: bool = False,
+    verbose: bool = False,
+    label: str = "path",
+    path_strict: Optional[bool] = None,
+    type_label: Optional[str] = None,
+) -> pathlib.Path:
+    """Normalize a path-like input and optionally validate existence and case."""
+    if input is None:
+        print_verbose(f"No {label} provided; exiting.", verbose)
+        raise ValueError(f"{label} must be provided")
+
+    type_label = type_label or label
+    path_value = _coerce_path_input(input, type_label)
+
+    if not path_value.strip():
+        print_verbose(f"No {label} provided; exiting.", verbose)
+        raise ValueError(f"{label} must be provided")
+
+    normalized = _normalize_path_value(path_value)
+    path_obj = pathlib.Path(normalized)
+    exists = path_obj.exists()
+
+    if must_exist and not exists:
+        print_verbose(f"Path {path_obj} does not exist; exiting.", verbose)
+        raise ValueError(f"Path does not exist: {path_obj}")
+
+    if exists:
+        strict = _resolve_path_strict(path_strict)
+        conflicts = _find_case_conflicts(path_obj, path_value)
+        if conflicts:
+            details = "; ".join(
+                f"{provided} -> {actual} in {parent}"
+                for provided, actual, parent in conflicts
+            )
+            message = f"Path case mismatch for {label}: {details}"
+            if strict:
+                raise ValueError(message)
+            log_warning(message, verbose)
+
+    return path_obj
+
+
+def ensure_directory(
+    input: Optional[PathInput],
+    *,
+    verbose: bool = False,
+    label: str = "path",
+    path_strict: Optional[bool] = None,
+    type_label: Optional[str] = None,
+) -> pathlib.Path:
+    """Ensure a directory exists and return the resolved path."""
+    path_obj = coerce_path(
+        input,
+        must_exist=False,
+        verbose=verbose,
+        label=label,
+        path_strict=path_strict,
+        type_label=type_label,
+    )
+    path_obj.mkdir(parents=True, exist_ok=True)
+    return path_obj
+
 
 def sql_quotename(
     name: Optional[str] = None,
@@ -67,20 +240,10 @@ def pathing(
         ValueError: If input is None or the path does not exist.
         TypeError: If input is not a string or pathlib.Path.
     """
-    if input is None:
-        print_verbose("No path provided; exiting pathing.", verbose)
-        raise ValueError("path input must be provided")
-
-    if isinstance(input, str):
-        normalized = input.replace("\\", "/")
-        path_obj = pathlib.Path(normalized)
-    elif isinstance(input, pathlib.Path):
-        path_obj = input
-    else:
-        raise TypeError("input must be a string or pathlib.Path")
-
-    if path_obj.exists():
-        return path_obj
-
-    print_verbose(f"Path {path_obj} does not exist; exiting pathing.", verbose)
-    raise ValueError(f"Path does not exist: {path_obj}")
+    return coerce_path(
+        input,
+        must_exist=True,
+        verbose=verbose,
+        label="path input",
+        type_label="input",
+    )