datablade 0.0.5__py3-none-any.whl → 0.0.6__py3-none-any.whl

datablade/sql/bulk_load.py

@@ -5,13 +5,16 @@ Provides dialect-aware bulk loading from files to database tables.
 Supports SQL Server (BCP), PostgreSQL (COPY), MySQL (LOAD DATA), and DuckDB.
 """
 
+import os
 import pathlib
+import shutil
 import subprocess
-from typing import Optional, Union
+from typing import Iterable, Optional, Union
 
 import pandas as pd
 
-from ..utils.logging import log_debug, log_error, log_info
+from ..utils.logging import log_debug, log_error, log_info, log_warning
+from ..utils.strings import coerce_path, ensure_directory
 from .dialects import Dialect
 from .quoting import quote_identifier
 
@@ -23,15 +26,18 @@ def _validate_bulk_load_params(
     server: Optional[str] = None,
     username: Optional[str] = None,
     password: Optional[str] = None,
+    use_trusted_connection: bool = False,
+    use_azure_ad: bool = False,
     dialect: Dialect = Dialect.SQLSERVER,
+    verbose: bool = False,
 ) -> pathlib.Path:
     """Validate bulk load parameters and return resolved path."""
-    if not file_path:
-        raise ValueError("file_path must be provided")
-
-    path_obj = pathlib.Path(file_path)
-    if not path_obj.exists():
-        raise ValueError(f"File does not exist: {path_obj}")
+    path_obj = coerce_path(
+        file_path,
+        must_exist=True,
+        verbose=verbose,
+        label="file_path",
+    )
 
     if not isinstance(table_name, str) or not table_name.strip():
         raise ValueError("table_name must be a non-empty string")
@@ -41,23 +47,106 @@ def _validate_bulk_load_params(
     if dialect == Dialect.SQLSERVER:
         if not server:
             raise ValueError("server is required for SQL Server")
+        if use_trusted_connection and use_azure_ad:
+            raise ValueError(
+                "use_trusted_connection and use_azure_ad are mutually exclusive"
+            )
+        if use_trusted_connection:
+            return path_obj
         if not username:
             raise ValueError("username is required for SQL Server")
-        if not password:
+        if not password and not use_azure_ad:
             raise ValueError("password is required for SQL Server")
+        if use_azure_ad and not username:
+            raise ValueError("username is required for Azure AD authentication")
 
     return path_obj
 
 
+def _build_bcp_args(
+    file_path: Union[str, pathlib.Path],
+    table_name: str,
+    database: str,
+    server: str,
+    username: Optional[str],
+    password: Optional[str],
+    schema: str,
+    delimiter: str,
+    use_trusted_connection: bool,
+    use_azure_ad: bool,
+    use_env_credentials: bool,
+    verbose: bool,
+) -> list[str]:
+    if use_env_credentials:
+        if not username:
+            username = os.getenv("DATABLADE_SQLSERVER_USERNAME")
+        if not password:
+            password = os.getenv("DATABLADE_SQLSERVER_PASSWORD")
+
+    path_obj = _validate_bulk_load_params(
+        file_path,
+        table_name,
+        database,
+        server,
+        username,
+        password,
+        use_trusted_connection=use_trusted_connection,
+        use_azure_ad=use_azure_ad,
+        dialect=Dialect.SQLSERVER,
+        verbose=verbose,
+    )
+
+    qualified_table = f"{database}.{schema}.{table_name}"
+    bcp_args = [
+        "bcp",
+        qualified_table,
+        "in",
+        str(path_obj),
+        "-c",
+        f"-t{delimiter}",
+        "-S",
+        server,
+    ]
+
+    if use_trusted_connection:
+        bcp_args.append("-T")
+    else:
+        if use_azure_ad:
+            bcp_args.append("-G")
+        if username:
+            bcp_args.extend(["-U", username])
+        if password:
+            bcp_args.extend(["-P", password])
+
+    return bcp_args
+
+
+def _redact_bcp_args(bcp_args: list[str]) -> list[str]:
+    redacted_args: list[str] = []
+    redact_next = False
+    for arg in bcp_args:
+        if redact_next:
+            redacted_args.append("***REDACTED***")
+            redact_next = False
+            continue
+        redacted_args.append(arg)
+        if arg == "-P":
+            redact_next = True
+    return redacted_args
+
+
 def bulk_load_sqlserver(
     file_path: Union[str, pathlib.Path],
     table_name: str,
     database: str,
     server: str,
-    username: str,
-    password: str,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
     schema: str = "dbo",
     delimiter: str = ",",
+    use_trusted_connection: bool = False,
+    use_azure_ad: bool = False,
+    use_env_credentials: bool = True,
     verbose: bool = False,
 ) -> None:
     """
@@ -72,38 +161,58 @@ def bulk_load_sqlserver(
         password: SQL Server password.
         schema: Schema name (default: dbo).
         delimiter: Field delimiter (default: comma).
+        use_trusted_connection: If True, use integrated authentication (-T).
+        use_azure_ad: If True, use Azure AD authentication (-G).
+        use_env_credentials: If True, fall back to DATABLADE_SQLSERVER_USERNAME
+            and DATABLADE_SQLSERVER_PASSWORD when username/password not provided.
         verbose: If True, logs progress messages.
 
     Raises:
         ValueError: On invalid inputs.
         subprocess.CalledProcessError: If BCP command fails.
     """
-    path_obj = _validate_bulk_load_params(
-        file_path, table_name, database, server, username, password, Dialect.SQLSERVER
+    bcp_args = _build_bcp_args(
+        file_path=file_path,
+        table_name=table_name,
+        database=database,
+        server=server,
+        username=username,
+        password=password,
+        schema=schema,
+        delimiter=delimiter,
+        use_trusted_connection=use_trusted_connection,
+        use_azure_ad=use_azure_ad,
+        use_env_credentials=use_env_credentials,
+        verbose=verbose,
     )
 
-    qualified_table = f"{database}.{schema}.{table_name}"
-
-    bcp_args = [
-        "bcp",
-        qualified_table,
-        "in",
-        str(path_obj),
-        "-c",
-        f"-t{delimiter}",
-        "-S",
-        server,
-        "-U",
-        username,
-        "-P",
-        password,
-    ]
-
-    log_info(f"Executing BCP load to {qualified_table}", verbose)
-    log_debug(
-        f"BCP args: {bcp_args[:-1] + ['***REDACTED***']}",
-        verbose,
-    )
+    bcp_preview = _redact_bcp_args(bcp_args)
+    bcp_path = shutil.which("bcp")
+    if not bcp_path:
+        install_steps = (
+            "Install the SQL Server command line utilities (bcp) and ensure the "
+            "binary is on PATH. For example: "
+            "macOS (Homebrew): brew install msodbcsql17 mssql-tools; "
+            "Linux (Debian/Ubuntu): install mssql-tools; "
+            "Windows: install SQL Server Command Line Utilities and restart your shell."
+        )
+        path_env = os.environ.get("PATH", "")
+        message = (
+            "BCP executable was not found on PATH. "
+            f"PATH={path_env}. {install_steps} "
+            f"Command preview: {bcp_preview}."
+        )
+        log_error(message, verbose)
+        raise FileNotFoundError(message)
+
+    log_info(f"Executing BCP load to {bcp_args[1]}", verbose)
+    if "-P" in bcp_args:
+        log_warning(
+            "BCP authentication uses -P with a plaintext password. "
+            "Consider using trusted connection (-T) or Azure AD (-G).",
+            verbose,
+        )
+    log_debug(f"BCP args: {bcp_preview}", verbose)
 
     try:
         process = subprocess.run(
@@ -113,15 +222,142 @@ def bulk_load_sqlserver(
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
-        log_info(f"Successfully loaded data to {qualified_table}", verbose)
+        log_info(f"Successfully loaded data to {bcp_args[1]}", verbose)
         if process.stdout:
             log_debug(f"BCP output: {process.stdout.decode()}", verbose)
     except subprocess.CalledProcessError as e:
         error_msg = e.stderr.decode() if e.stderr else str(e)
-        log_error(f"BCP load failed: {error_msg}", verbose)
+        log_error(
+            f"BCP load failed: {error_msg} "
+            f"PATH={os.environ.get('PATH', '')}. "
+            "Ensure BCP is installed (SQL Server command line utilities) and on PATH. "
+            f"Command preview: {bcp_preview}.",
+            verbose,
+        )
         raise
 
 
+def bulk_load_sqlserver_command(
+    file_path: Union[str, pathlib.Path],
+    table_name: str,
+    database: str,
+    server: str,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    schema: str = "dbo",
+    delimiter: str = ",",
+    use_trusted_connection: bool = False,
+    use_azure_ad: bool = False,
+    use_env_credentials: bool = True,
+    redact_password: bool = False,
+    verbose: bool = False,
+) -> str:
+    """Return a BCP command string for a SQL Server bulk load."""
+    bcp_args = _build_bcp_args(
+        file_path=file_path,
+        table_name=table_name,
+        database=database,
+        server=server,
+        username=username,
+        password=password,
+        schema=schema,
+        delimiter=delimiter,
+        use_trusted_connection=use_trusted_connection,
+        use_azure_ad=use_azure_ad,
+        use_env_credentials=use_env_credentials,
+        verbose=verbose,
+    )
+
+    if redact_password:
+        bcp_args = _redact_bcp_args(bcp_args)
+
+    return subprocess.list2cmdline(bcp_args)
+
+
+def bulk_load_sqlserver_many(
+    file_paths: list[Union[str, pathlib.Path]],
+    table_name: str,
+    database: str,
+    server: str,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    schema: str = "dbo",
+    delimiter: str = ",",
+    use_trusted_connection: bool = False,
+    use_azure_ad: bool = False,
+    use_env_credentials: bool = True,
+    verbose: bool = False,
+) -> None:
+    """Bulk load multiple files into SQL Server using BCP."""
+    if file_paths is None:
+        raise ValueError("file_paths must be provided")
+    if not isinstance(file_paths, list):
+        raise TypeError("file_paths must be a list of paths")
+    if not file_paths:
+        raise ValueError("file_paths must contain at least one path")
+
+    for file_path in file_paths:
+        bulk_load_sqlserver(
+            file_path=file_path,
+            table_name=table_name,
+            database=database,
+            server=server,
+            username=username,
+            password=password,
+            schema=schema,
+            delimiter=delimiter,
+            use_trusted_connection=use_trusted_connection,
+            use_azure_ad=use_azure_ad,
+            use_env_credentials=use_env_credentials,
+            verbose=verbose,
+        )
+
+
+def bulk_load_sqlserver_commands(
+    file_paths: Iterable[Union[str, pathlib.Path]],
+    table_name: str,
+    database: str,
+    server: str,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    schema: str = "dbo",
+    delimiter: str = ",",
+    use_trusted_connection: bool = False,
+    use_azure_ad: bool = False,
+    use_env_credentials: bool = True,
+    redact_password: bool = False,
+    verbose: bool = False,
+) -> list[str]:
+    """Return BCP command strings for multiple SQL Server bulk loads."""
+    if file_paths is None:
+        raise ValueError("file_paths must be provided")
+
+    commands: list[str] = []
+    for file_path in file_paths:
+        commands.append(
+            bulk_load_sqlserver_command(
+                file_path=file_path,
+                table_name=table_name,
+                database=database,
+                server=server,
+                username=username,
+                password=password,
+                schema=schema,
+                delimiter=delimiter,
+                use_trusted_connection=use_trusted_connection,
+                use_azure_ad=use_azure_ad,
+                use_env_credentials=use_env_credentials,
+                redact_password=redact_password,
+                verbose=verbose,
+            )
+        )
+
+    if not commands:
+        raise ValueError("file_paths must contain at least one path")
+
+    return commands
+
+
 def bulk_load_postgres(
     file_path: Union[str, pathlib.Path],
     table_name: str,
@@ -152,17 +388,25 @@ def bulk_load_postgres(
         ValueError: On invalid inputs.
     """
     path_obj = _validate_bulk_load_params(
-        file_path, table_name, database, dialect=Dialect.POSTGRES
+        file_path,
+        table_name,
+        database,
+        dialect=Dialect.POSTGRES,
+        verbose=verbose,
     )
 
-    qualified_table = f"{quote_identifier(schema, Dialect.POSTGRES)}.{quote_identifier(table_name, Dialect.POSTGRES)}"
-
-    header_clause = "HEADER" if header else ""
-    copy_cmd = (
-        f"\\COPY {qualified_table} FROM '{path_obj}' "
-        f"WITH (FORMAT csv, DELIMITER '{delimiter}', {header_clause})"
+    qualified_table = (
+        f"{quote_identifier(schema, Dialect.POSTGRES)}."
+        f"{quote_identifier(table_name, Dialect.POSTGRES)}"
     )
 
+    options = ["FORMAT csv", f"DELIMITER '{delimiter}'"]
+    if header:
+        options.append("HEADER")
+
+    options_sql = ", ".join(options)
+    copy_cmd = f"\\COPY {qualified_table} FROM '{path_obj}' WITH ({options_sql})"
+
     log_info(f"Generated COPY command for {qualified_table}", verbose)
     return copy_cmd
 
@@ -197,10 +441,17 @@ def bulk_load_mysql(
         ValueError: On invalid inputs.
     """
     path_obj = _validate_bulk_load_params(
-        file_path, table_name, database, dialect=Dialect.MYSQL
+        file_path,
+        table_name,
+        database,
+        dialect=Dialect.MYSQL,
+        verbose=verbose,
     )
 
-    qualified_table = f"{quote_identifier(database, Dialect.MYSQL)}.{quote_identifier(table_name, Dialect.MYSQL)}"
+    qualified_table = (
+        f"{quote_identifier(database, Dialect.MYSQL)}."
+        f"{quote_identifier(table_name, Dialect.MYSQL)}"
+    )
 
     load_cmd = (
         f"LOAD DATA LOCAL INFILE '{path_obj}' "
@@ -239,7 +490,11 @@ def bulk_load_duckdb(
         ValueError: On invalid inputs.
     """
     path_obj = _validate_bulk_load_params(
-        file_path, table_name, database, dialect=Dialect.DUCKDB
+        file_path,
+        table_name,
+        database,
+        dialect=Dialect.DUCKDB,
+        verbose=verbose,
     )
 
     qualified_table = f"{quote_identifier(schema, Dialect.DUCKDB)}.{quote_identifier(table_name, Dialect.DUCKDB)}"
@@ -377,8 +632,13 @@ def write_dataframe_and_load(
     if df is None or not isinstance(df, pd.DataFrame):
         raise TypeError("df must be a pandas DataFrame")
 
-    path_obj = pathlib.Path(file_path)
-    path_obj.parent.mkdir(parents=True, exist_ok=True)
+    path_obj = coerce_path(
+        file_path,
+        must_exist=False,
+        verbose=verbose,
+        label="file_path",
+    )
+    ensure_directory(path_obj.parent, verbose=verbose, label="output_dir")
 
     # Write based on file extension
     suffix = path_obj.suffix.lower()