sqlalchemy-mssql-bulkcopy 0.1.0__py3-none-any.whl

sqlalchemy_mssql_bulkcopy/__init__.py

@@ -0,0 +1,302 @@
+"""SQLAlchemy integration for mssql-python's ``cursor.bulkcopy()``.
+
+Uses SQL Server's native TDS bulk load protocol for 10-50x faster writes.
+All ``cursor.bulkcopy()`` kwargs pass through to the driver unchanged.
+
+Primary API::
+
+    from sqlalchemy_mssql_bulkcopy import bulkcopy
+
+    with engine.begin() as conn:
+        result = bulkcopy(conn, df, "dbo.users", batch_size=50_000)
+
+Event hook for SA Core/ORM::
+
+    from sqlalchemy_mssql_bulkcopy import register_bulkcopy
+
+    engine = create_engine("...", use_insertmanyvalues=False)
+    register_bulkcopy(engine)
+    conn.execute(
+        users.insert().execution_options(use_bulkcopy=True), rows
+    )
+
+See https://github.com/sqlalchemy/sqlalchemy/issues/13218
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable, Iterable
+from importlib.metadata import version as _pkg_version
+from typing import Any
+
+from sqlalchemy import event
+from sqlalchemy.engine import Connection, Engine
+
+__version__ = _pkg_version(__name__)
+__all__ = [
+    "bulkcopy",
+    "bulkcopy_insert_method",
+    "register_bulkcopy",
+]
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Primary API: direct bulkcopy
+# ---------------------------------------------------------------------------
+
+
+def bulkcopy(
+    conn: Connection,
+    data: Any,
+    table_name: str,
+    *,
+    columns: list[str] | None = None,
+    **kwargs: Any,
+) -> dict[str, Any]:
+    """Bulk-load data into SQL Server via the TDS bulk copy protocol.
+
+    Accesses the raw DBAPI cursor from *conn* and calls
+    ``cursor.bulkcopy()`` directly.  No engine configuration required.
+    All keyword arguments pass through to the driver unchanged.
+
+    Parameters
+    ----------
+    conn : sqlalchemy.engine.Connection
+        An active SA connection (from ``engine.connect()`` or
+        ``engine.begin()``).
+    data : DataFrame | Iterable[tuple | list]
+        Rows to load.  DataFrames are auto-converted; column names
+        are extracted unless *columns* is provided explicitly.
+    table_name : str
+        Target table, optionally schema-qualified (``"dbo.users"``).
+    columns : list[str] | None
+        Column names passed as ``column_mappings`` to the driver.
+        Auto-detected from DataFrames.
+    **kwargs
+        Forwarded to ``cursor.bulkcopy()``.  See mssql-python docs
+        for the full list: ``batch_size``, ``table_lock``,
+        ``keep_identity``, ``check_constraints``, ``fire_triggers``,
+        ``keep_nulls``, ``timeout``, ``use_internal_transaction``.
+
+    Returns
+    -------
+    dict[str, Any]
+        Driver result: ``rows_copied``, ``batch_count``,
+        ``elapsed_time``.
+
+    Raises
+    ------
+    AttributeError
+        If the cursor does not have a ``bulkcopy`` method (wrong
+        driver or driver version < 1.4).
+    """
+    dbapi_conn = conn.connection.dbapi_connection
+    rows, columns = _normalize_data(data, columns)
+
+    if not rows:
+        return {"rows_copied": 0, "batch_count": 0, "elapsed_time": 0.0}
+
+    if columns is not None:
+        kwargs["column_mappings"] = columns
+
+    cursor = dbapi_conn.cursor()
+    try:
+        result: dict[str, Any] = cursor.bulkcopy(table_name, rows, **kwargs)
+    finally:
+        cursor.close()
+
+    logger.info(
+        "bulkcopy: %d rows -> %s (%.3fs)",
+        result.get("rows_copied", len(rows)),
+        table_name,
+        result.get("elapsed_time", 0),
+    )
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Event hook for SA Core / ORM
+# ---------------------------------------------------------------------------
+
+
+def register_bulkcopy(
+    engine: Engine,
+    *,
+    on_complete: Callable[[str, dict[str, Any]], Any] | None = None,
+    **defaults: Any,
+) -> None:
+    """Register a ``do_executemany`` event hook that routes INSERT
+    batches through ``cursor.bulkcopy()`` when
+    ``execution_options(use_bulkcopy=True)`` is set.
+
+    Parameters
+    ----------
+    engine : Engine
+        **Must** be created with ``use_insertmanyvalues=False``.
+        SA 2.x defaults to multi-row ``INSERT … VALUES`` batching
+        which bypasses ``executemany()`` entirely; without this flag
+        the event hook will never fire.
+    on_complete : callable | None
+        ``on_complete(table_name, result_dict)`` is called after each
+        successful bulkcopy.  Errors in the callback propagate to the
+        caller (they do **not** trigger the executemany fallback).
+    **defaults
+        Default kwargs forwarded to ``cursor.bulkcopy()``.
+        Overridable per-statement via
+        ``execution_options(bulkcopy_<key>=value)``.
+
+    Raises
+    ------
+    ValueError
+        If the engine's dialect has ``use_insertmanyvalues`` enabled.
+    """
+    if getattr(engine.dialect, "use_insertmanyvalues", False):
+        raise ValueError(
+            "register_bulkcopy() requires use_insertmanyvalues=False. "
+            "SA 2.x defaults to multi-row INSERT batching which "
+            "bypasses executemany() — the event hook will never fire. "
+            "Create the engine with: "
+            "create_engine(..., use_insertmanyvalues=False)  "
+            "This does not affect bulkcopy(); only register_bulkcopy() "
+            "has this requirement."
+        )
+
+    @event.listens_for(engine, "do_executemany", retval=True)
+    def _do_bulkcopy(
+        cursor: Any,
+        statement: str,
+        parameters: Any,
+        context: Any,
+    ) -> bool:
+        opts = context.execution_options
+        if not opts.get("use_bulkcopy", False):
+            return False
+
+        if not hasattr(cursor, "bulkcopy"):
+            logger.debug(
+                "use_bulkcopy set but cursor has no bulkcopy method; falling back to executemany"
+            )
+            return False
+
+        try:
+            table = context.compiled.statement.table
+        except AttributeError:
+            logger.debug(
+                "use_bulkcopy set but no table metadata on compiled "
+                "statement; falling back to executemany"
+            )
+            return False
+
+        # Schema-qualified table name
+        schema = table.schema or context.dialect.default_schema_name
+        table_name = f"{schema}.{table.name}" if schema else table.name
+
+        # Map SA column keys → DB column names (key != name when
+        # Column("db_name", key="python_name") is used)
+        col_names = [table.c[key].name for key in context.compiled.column_keys]
+
+        # SA sends tuples for qmark paramstyle (mssql-python) and
+        # dicts for named paramstyle.  Handle both.
+        if parameters and isinstance(parameters[0], dict):
+            rows = [tuple(row[key] for key in context.compiled.column_keys) for row in parameters]
+        else:
+            rows = [tuple(row) for row in parameters]
+
+        # Engine defaults ← per-statement overrides (bulkcopy_ prefix)
+        bcp_kwargs: dict[str, Any] = {**defaults}
+        bcp_kwargs["column_mappings"] = col_names
+        for key, value in opts.items():
+            if key.startswith("bulkcopy_"):
+                bcp_kwargs[key.removeprefix("bulkcopy_")] = value
+
+        try:
+            result = cursor.bulkcopy(table_name, rows, **bcp_kwargs)
+        except Exception:
+            logger.warning(
+                "bulkcopy failed for %s; falling back to executemany",
+                table_name,
+                exc_info=True,
+            )
+            return False
+
+        logger.info(
+            "bulkcopy: %d rows -> %s (%d cols)",
+            len(rows),
+            table_name,
+            len(col_names),
+        )
+
+        # Callback runs outside try/except so errors propagate
+        # to the caller instead of silently triggering fallback.
+        if on_complete is not None:
+            on_complete(table_name, result)
+
+        return True
+
+
+# ---------------------------------------------------------------------------
+# Pandas helper
+# ---------------------------------------------------------------------------
+
+
+def bulkcopy_insert_method(
+    table: Any,
+    conn: Any,
+    keys: list[str],
+    data_iter: Iterable[tuple[Any, ...]],
+) -> None:
+    """``method`` callable for :meth:`~pandas.DataFrame.to_sql`.
+
+    Requires :func:`register_bulkcopy` on the engine::
+
+        df.to_sql("users", engine, method=bulkcopy_insert_method,
+                  if_exists="append", index=False)
+    """
+    from sqlalchemy import insert as sa_insert
+
+    data = [dict(zip(keys, row, strict=True)) for row in data_iter]
+    if not data:
+        return
+
+    sa_table = getattr(table, "table", table)
+    stmt = sa_insert(sa_table).execution_options(use_bulkcopy=True)
+    conn.execute(stmt, data)
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _normalize_data(
+    data: Any,
+    columns: list[str] | None,
+) -> tuple[list[tuple[Any, ...]], list[str] | None]:
+    """Convert input to ``(list_of_tuples, columns)``.
+
+    Handles DataFrames, generators, lists-of-lists, and plain tuples.
+    Extracts column names from DataFrames when *columns* is ``None``.
+    """
+    try:
+        import pandas as pd
+
+        if isinstance(data, pd.DataFrame):
+            if columns is None:
+                columns = list(data.columns)
+            return list(data.itertuples(index=False, name=None)), columns
+    except ImportError:
+        pass
+
+    if not isinstance(data, list):
+        data = list(data)
+
+    if not data:
+        return data, columns
+
+    if not isinstance(data[0], tuple):
+        return [tuple(row) for row in data], columns
+
+    return data, columns

sqlalchemy_mssql_bulkcopy/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.1.0'
+__version_tuple__ = version_tuple = (0, 1, 0)
+
+__commit_id__ = commit_id = None

sqlalchemy_mssql_bulkcopy-0.1.0.dist-info/METADATA

@@ -0,0 +1,165 @@
+Metadata-Version: 2.4
+Name: sqlalchemy-mssql-bulkcopy
+Version: 0.1.0
+Summary: Bulk copy for SQLAlchemy + mssql-python — 10-50x faster writes via SQL Server's native TDS protocol
+Author: Daniel Caspi
+License: MIT
+Project-URL: Homepage, https://github.com/dxdc/sqlalchemy-mssql-bulkcopy
+Project-URL: Issues, https://github.com/dxdc/sqlalchemy-mssql-bulkcopy/issues
+Project-URL: Changelog, https://github.com/dxdc/sqlalchemy-mssql-bulkcopy/releases
+Keywords: sqlalchemy,mssql,sql-server,bulk-copy,pandas,mssql-python,bulkcopy,tds,etl
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sqlalchemy>=2.1.0b1
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=6.0; extra == "dev"
+Requires-Dist: pandas>=2.0; extra == "dev"
+Requires-Dist: mssql-python>=1.4.0; extra == "dev"
+Requires-Dist: ruff>=0.9; extra == "dev"
+Dynamic: license-file
+
+# sqlalchemy-mssql-bulkcopy
+
+[![CI](https://github.com/dxdc/sqlalchemy-mssql-bulkcopy/actions/workflows/ci.yml/badge.svg)](https://github.com/dxdc/sqlalchemy-mssql-bulkcopy/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/sqlalchemy-mssql-bulkcopy)](https://pypi.org/project/sqlalchemy-mssql-bulkcopy/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+
+Bulk copy for SQLAlchemy + mssql-python — 10–50× faster writes via
+SQL Server's native TDS bulk load protocol.
+
+All `cursor.bulkcopy()` kwargs pass through to the driver. When
+mssql-python adds new options, this package supports them automatically.
+
+Based on the [event hook approach recommended by SQLAlchemy's maintainer](https://github.com/sqlalchemy/sqlalchemy/issues/13218).
+
+## Install
+
+The `mssql+mssqlpython` dialect requires SQLAlchemy ≥ 2.1.0b2,
+which is not yet on PyPI. Install SA from git until then:
+
+```bash
+pip install "sqlalchemy @ git+https://github.com/sqlalchemy/sqlalchemy.git@main"
+pip install sqlalchemy-mssql-bulkcopy
+```
+
+Once SA 2.1 is stable:
+
+```bash
+pip install sqlalchemy-mssql-bulkcopy
+```
+
+Requires Python ≥ 3.10, SQLAlchemy ≥ 2.1, mssql-python ≥ 1.4.
+
+## Usage
+
+### Direct (recommended)
+
+No engine configuration needed. Returns the driver result dict.
+
+```python
+from sqlalchemy import create_engine
+from sqlalchemy_mssql_bulkcopy import bulkcopy
+
+engine = create_engine("mssql+mssqlpython://user:pass@host/db")
+
+with engine.begin() as conn:
+    result = bulkcopy(conn, df, "dbo.users", batch_size=50_000)
+    print(f"{result['rows_copied']} rows in {result['elapsed_time']:.2f}s")
+```
+
+Accepts DataFrames (columns auto-detected), lists of tuples, generators,
+or any iterable:
+
+```python
+# All driver options pass through as kwargs
+result = bulkcopy(
+    conn, df, "staging.events",
+    batch_size=100_000,
+    table_lock=True,
+    keep_identity=True,
+    check_constraints=True,
+    fire_triggers=True,
+    keep_nulls=True,
+    timeout=300,
+    use_internal_transaction=True,
+)
+```
+
+### Event hook (SA Core / ORM)
+
+Hooks into SQLAlchemy's INSERT workflow. Per-statement opt-in via
+execution options:
+
+```python
+from sqlalchemy_mssql_bulkcopy import register_bulkcopy
+
+engine = create_engine(
+    "mssql+mssqlpython://user:pass@host/db",
+    use_insertmanyvalues=False,  # required — see note below
+)
+register_bulkcopy(engine, batch_size=10_000, table_lock=True)
+
+with engine.begin() as conn:
+    conn.execute(
+        users.insert().execution_options(use_bulkcopy=True),
+        rows,
+    )
+```
+
+Per-statement overrides use the `bulkcopy_` prefix:
+
+```python
+stmt = users.insert().execution_options(
+    use_bulkcopy=True,
+    bulkcopy_batch_size=100_000,
+    bulkcopy_fire_triggers=True,
+)
+```
+
+> **Note:** `register_bulkcopy()` raises `ValueError` if
+> `use_insertmanyvalues` is not disabled. SA 2.x defaults to multi-row
+> `INSERT … VALUES` batching which bypasses `executemany()` entirely —
+> the event hook never fires without this flag. This is a global engine
+> setting: non-bulkcopy inserts will use `executemany()` instead, which
+> is slightly slower for medium-sized batches. If that matters, use
+> `bulkcopy()` directly instead.
+
+### Pandas
+
+```python
+from sqlalchemy_mssql_bulkcopy import bulkcopy_insert_method
+
+df.to_sql("users", engine, method=bulkcopy_insert_method,
+          if_exists="append", index=False)
+```
+
+## Confirming bulkcopy was used
+
+```python
+# Direct — return value
+result = bulkcopy(conn, df, "dbo.users")
+assert result["rows_copied"] == len(df)
+
+# Event hook — on_complete callback
+results = []
+register_bulkcopy(engine, on_complete=lambda t, r: results.append(r))
+```
+
+Both approaches log at INFO level:
+`bulkcopy: 1000 rows -> dbo.users (0.050s)`
+
+## License
+
+MIT

sqlalchemy_mssql_bulkcopy-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+sqlalchemy_mssql_bulkcopy/__init__.py,sha256=xyv-uAB7SiU8999QGds56ZDasS0llGaPq7NN83ZSPkE,9723
+sqlalchemy_mssql_bulkcopy/_version.py,sha256=n_5vdJsPNu7wZ57LGuRL585uvll-hiuvZUBWzdG0RQU,520
+sqlalchemy_mssql_bulkcopy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sqlalchemy_mssql_bulkcopy-0.1.0.dist-info/licenses/LICENSE,sha256=b_V-8rBn8J8Kv6QmpyrZgFijN99-FtuK_JuGEL9Bf9s,1069
+sqlalchemy_mssql_bulkcopy-0.1.0.dist-info/METADATA,sha256=dFKt-_-wYkgE3vuCTdZzw8pV59BYYgt3S8ZzadWZaV0,5125
+sqlalchemy_mssql_bulkcopy-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+sqlalchemy_mssql_bulkcopy-0.1.0.dist-info/top_level.txt,sha256=hNYCeNLYd0056hxaQNC2B40n5rwPt8va-izV9KpYH-s,26
+sqlalchemy_mssql_bulkcopy-0.1.0.dist-info/RECORD,,

sqlalchemy_mssql_bulkcopy-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sqlalchemy_mssql_bulkcopy-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Daniel Caspi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sqlalchemy_mssql_bulkcopy-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sqlalchemy_mssql_bulkcopy