ferro-orm 0.16.0__tar.gz → 0.16.1__tar.gz

{ferro_orm-0.16.0 → ferro_orm-0.16.1}/CHANGELOG.md

@@ -1,6 +1,14 @@
 # CHANGELOG
 
 
+## v0.16.1 (2026-07-13)
+
+### Bug Fixes
+
+- Chunk bulk_create under backend bind-parameter limits
+  ([`38da62d`](https://github.com/syn54x/ferro-orm/commit/38da62d9ab9fab6b36a9eae0722c1d741b976d15))
+
+
 ## v0.16.0 (2026-07-13)
 
 ### Chores

{ferro_orm-0.16.0 → ferro_orm-0.16.1}/Cargo.lock

@@ -327,7 +327,7 @@ dependencies = [
 
 [[package]]
 name = "ferro"
-version = "0.16.0"
+version = "0.16.1"
 dependencies = [
  "chrono",
  "dashmap",

{ferro_orm-0.16.0 → ferro_orm-0.16.1}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/ferro-schema-ir", "crates/ferro-ddl-lowering", "crates/ferro-
 
 [package]
 name = "ferro"
-version = "0.16.0"
+version = "0.16.1"
 edition = "2024"
 readme = "README.md"
 

{ferro_orm-0.16.0 → ferro_orm-0.16.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ferro-orm
-Version: 0.16.0
+Version: 0.16.1
 Requires-Dist: pydantic>=2.0
 Requires-Dist: alembic>=1.18.1 ; extra == 'alembic'
 Requires-Dist: sqlalchemy>=2.0.46 ; extra == 'alembic'

{ferro_orm-0.16.0 → ferro_orm-0.16.1}/docs/pages/concepts/performance.md

@@ -6,7 +6,7 @@ Ferro moves SQL generation, parameter binding, and row hydration out of Python a
 
 The Rust core helps most where a traditional ORM spends significant CPU time in Python:
 
-**Bulk inserts.** `bulk_create` serializes and binds an entire batch in Rust and writes it as a single statement. The per-row Python overhead — building parameter lists, driver round-trips, object bookkeeping — largely disappears:
+**Bulk inserts.** `bulk_create` serializes and binds an entire batch in Rust and writes it in as few statements as the backend's bind-parameter limit allows. The per-row Python overhead — building parameter lists, driver round-trips, object bookkeeping — largely disappears:
 
 ```python
 users = [
@@ -43,7 +43,7 @@ for i in range(1000):
 await User.bulk_create([User(username=f"user_{i}") for i in range(1000)])
 ```
 
-Batches in the low thousands (roughly 1,000–5,000 rows) are a good unit of work — large enough to amortize overhead, small enough to keep statements and memory reasonable. Note that `bulk_create` skips the [identity map](identity-map.md) by design.
+Batch size is unbounded — Ferro splits large batches under the backend's bind-parameter limit automatically and keeps the whole call all-or-nothing — so size batches by what your application can hold in memory, not by statement limits. Note that `bulk_create` skips the [identity map](identity-map.md) by design.
 
 **Use batch update/delete instead of instance loops.** Push the work into one SQL statement:
 

{ferro_orm-0.16.0 → ferro_orm-0.16.1}/docs/pages/guide/mutations.md

@@ -14,12 +14,14 @@ Ferro's write surface is three verbs with three distinct intents — none of the
 
 ### create
 
-`Model.create(**fields)` validates, inserts, and returns the persisted instance in one call. For inserting many rows, `Model.bulk_create(instances)` batches them into a single statement and returns the inserted count:
+`Model.create(**fields)` validates, inserts, and returns the persisted instance in one call. For inserting many rows, `Model.bulk_create(instances)` writes the whole batch in bulk and returns the inserted count:
 
 ```python
 --8<-- "docs/examples/quickstart.py:create"
 ```
 
+Batch size is unbounded. Both backends cap how many parameters one statement may bind (SQLite 32,766; PostgreSQL 65,535), so Ferro splits a large batch across as many INSERT statements as the active backend requires — an implementation detail you never see. Atomicity is preserved either way: inside a [`transaction()`](transactions.md) block that transaction is the boundary, and a bare `bulk_create` call is all-or-nothing — if any row fails (say, a unique violation late in the batch), no rows from the batch are inserted.
+
 `create()` is a plain INSERT — it never updates an existing row. A duplicate primary key or unique value raises [`UniqueViolationError`](../api/exceptions.md), and the existing row is left untouched:
 
 ```python

{ferro_orm-0.16.0 → ferro_orm-0.16.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ferro-orm"
-version = "0.16.0"
+version = "0.16.1"
 description = "A high-performance, Rust-backed ORM for Python."
 readme = "README.md"
 authors = [

{ferro_orm-0.16.0 → ferro_orm-0.16.1}/src/backend.rs

@@ -415,6 +415,19 @@ impl EngineHandle {
         self.backend
     }
 
+    /// Maximum bind parameters one statement may carry on this backend.
+    ///
+    /// SQLite: `SQLITE_MAX_VARIABLE_NUMBER`, 32,766 by default since 3.32.
+    /// Postgres: the wire protocol's `Bind` message carries an `int16`
+    /// parameter count, so 65,535 is a hard protocol ceiling.
+    #[must_use]
+    pub fn max_bind_params(&self) -> usize {
+        match self.backend {
+            Dialect::Sqlite => 32_766,
+            Dialect::Postgres => 65_535,
+        }
+    }
+
     /// Record one catalog-introspection round-trip (called at the single
     /// choke point that executes catalog SQL).
     pub fn record_catalog_query(&self) {
@@ -611,7 +624,6 @@ impl EngineHandle {
         }
     }
 
-    #[allow(dead_code)]
     pub async fn begin_transaction_connection(&self) -> Result<EngineConnection, sqlx::Error> {
         match &self.pool_snapshot() {
             BackendPool::Sqlite(pool) => {

{ferro_orm-0.16.0 → ferro_orm-0.16.1}/src/ferro/models.py

@@ -639,6 +639,12 @@ class Model(BaseModel, metaclass=ModelMetaclass):
     ) -> int:
         """Persist multiple instances in a single bulk operation
 
+        Batch size is unbounded: batches larger than the backend's
+        bind-parameter limit allows in one statement are split internally.
+        The call is atomic either way — inside an ambient ``transaction()``
+        that transaction is the atomicity boundary; a bare call is
+        all-or-nothing (a failure on any row inserts none).
+
         Args:
             instances: Model instances to persist.
 

{ferro_orm-0.16.0 → ferro_orm-0.16.1}/src/operations.rs

@@ -2296,7 +2296,12 @@ pub fn update_record<'py>(
     })
 }
 
-/// Persist multiple model instances in a single batch `INSERT`.
+/// Persist multiple model instances in as few batch `INSERT`s as the
+/// backend's bind-parameter limit allows (#298).
+///
+/// Atomicity: with an ambient transaction every statement runs inside it;
+/// without one, a multi-statement batch runs in its own transaction so the
+/// call stays all-or-nothing.
 ///
 /// Args:
 ///     name (str): Model class name.
@@ -2340,24 +2345,32 @@ pub fn save_bulk_records<'py>(
 
         let catalog = postgres_table_catalog(&table_name, &engine, exec, backend).await?;
         let TableCatalog { enum_udt, uuid_columns, ts_cast } = &*catalog;
-        let (sql, bind_values) = {
+
+        // Columns come from the first row's order (skip a null auto-pk).
+        let mut column_names: Vec<String> = Vec::new();
+        for (key, input) in &record_inputs[0] {
+            let is_pk = pk_col.as_deref() == Some(key.as_str());
+            if is_pk && pk_is_auto && input.is_json_null() {
+                continue;
+            }
+            column_names.push(key.clone());
+        }
+
+        // One multi-row INSERT binds rows × columns parameters, and both
+        // backends cap binds per statement — split the batch so every
+        // statement stays within the active backend's budget (#298).
+        let rows_per_stmt =
+            bulk_insert_rows_per_statement(engine.max_bind_params(), column_names.len());
+
+        let mut statements: Vec<(String, sea_query::Values)> = Vec::new();
+        for chunk in record_inputs.chunks(rows_per_stmt) {
             let mut insert_stmt = InsertStatement::new()
                 .into_table(Alias::new(&table_name))
                 .to_owned();
-
-            // Columns come from the first row's order (skip a null auto-pk).
-            let mut column_names: Vec<String> = Vec::new();
-            for (key, input) in &record_inputs[0] {
-                let is_pk = pk_col.as_deref() == Some(key.as_str());
-                if is_pk && pk_is_auto && input.is_json_null() {
-                    continue;
-                }
-                column_names.push(key.clone());
-            }
             insert_stmt.columns(column_names.iter().map(|c| Alias::new(c)));
 
             let null_input = BindInput::Json(serde_json::Value::Null);
-            for row in &record_inputs {
+            for row in chunk {
                 let lookup: std::collections::HashMap<&str, &BindInput> =
                     row.iter().map(|(k, v)| (k.as_str(), v)).collect();
                 let mut row_values = Vec::with_capacity(column_names.len());
@@ -2377,20 +2390,94 @@ pub fn save_bulk_records<'py>(
             }
 
             let (s, values) = sea_query_build_for_backend!(insert_stmt, backend);
-            (s, values)
+            statements.push((s, values));
+        }
+
+        // A bare multi-statement call opens its own transaction: the single
+        // statement it replaces was atomic, and chunking must not regress
+        // that. With an ambient transaction (or a single statement) the
+        // existing execution path already gives the right boundary.
+        let own_tx: Option<TransactionConnection> = if tx_conn.is_none() && statements.len() > 1 {
+            let conn = engine.begin_transaction_connection().await.map_err(|e| {
+                crate::errors::map_db_error(&format!("Bulk save failed for '{}'", name), e)
+            })?;
+            Some(Arc::new(tokio::sync::Mutex::new(conn)))
+        } else {
+            None
         };
+        let exec_conn = tx_conn.as_ref().or(own_tx.as_ref());
+
+        let mut rows_affected: u64 = 0;
+        for (sql, bind_values) in &statements {
+            let executed =
+                execute_statement_with_optional_tx(&engine, exec_conn, sql, &bind_values.0).await;
+            match executed {
+                Ok(n) => rows_affected += n,
+                Err(e) => {
+                    if let Some(own_tx) = &own_tx {
+                        // Surface the insert failure even if ROLLBACK also
+                        // fails — a dead connection aborts the transaction
+                        // server-side, so the root cause is the honest error.
+                        let _ = execute_transaction_sql(own_tx, "ROLLBACK").await;
+                    }
+                    return Err(crate::errors::map_db_error(
+                        &format!("Bulk save failed for '{}'", name),
+                        e,
+                    ));
+                }
+            }
+        }
 
-        let rows_affected =
-            execute_statement_with_optional_tx(&engine, tx_conn.as_ref(), &sql, &bind_values.0)
+        if let Some(own_tx) = &own_tx {
+            execute_transaction_sql(own_tx, "COMMIT")
                 .await
-                .map_err(|e| {
-                    crate::errors::map_db_error(&format!("Bulk save failed for '{}'", name), e)
-                })?;
+                .map_err(|e| crate::errors::map_db_error("Failed to COMMIT", e))?;
+        }
 
         Ok(rows_affected)
     })
 }
 
+/// Rows per INSERT statement so `rows × columns` never exceeds the backend's
+/// bind-parameter budget. Always at least one row: a single row wider than
+/// the budget cannot be split further, so it passes through to the backend
+/// (whose error is then the honest answer). A zero-column statement binds
+/// nothing, so the whole batch fits in one statement.
+fn bulk_insert_rows_per_statement(max_bind_params: usize, n_columns: usize) -> usize {
+    if n_columns == 0 {
+        return usize::MAX;
+    }
+    (max_bind_params / n_columns).max(1)
+}
+
+#[cfg(test)]
+mod bulk_chunking_tests {
+    use super::bulk_insert_rows_per_statement;
+
+    #[test]
+    fn floors_to_the_bind_budget() {
+        // sqlite budget, 5 columns: 6,553 rows bind 32,765 of 32,766.
+        assert_eq!(bulk_insert_rows_per_statement(32_766, 5), 6_553);
+        // Postgres budget, 12 columns: 5,461 rows bind 65,532 of 65,535.
+        assert_eq!(bulk_insert_rows_per_statement(65_535, 12), 5_461);
+    }
+
+    #[test]
+    fn exact_division_uses_the_full_budget() {
+        assert_eq!(bulk_insert_rows_per_statement(65_535, 5), 13_107);
+    }
+
+    #[test]
+    fn one_row_minimum_even_when_a_single_row_exceeds_the_budget() {
+        assert_eq!(bulk_insert_rows_per_statement(10, 20), 1);
+    }
+
+    #[test]
+    fn zero_columns_never_split() {
+        assert_eq!(bulk_insert_rows_per_statement(32_766, 0), usize::MAX);
+    }
+}
+
 /// Fetches records for a given model class based on a QueryIR-defined query.
 ///
 /// The query's materialization plan (ADR-0007) selects the result shape:

ferro_orm-0.16.1/tests/test_bulk_create_chunking.py

@@ -0,0 +1,167 @@
+"""bulk_create internal chunking under backend bind-parameter limits (#298).
+
+All tests exercise the public seam — ``Model.bulk_create`` over the backend
+matrix. Chunking is an implementation detail: the only observable contract is
+that any batch size succeeds, the returned count is the total inserted, and
+atomicity is preserved with and without an ambient transaction.
+
+Bind budget arithmetic used throughout: ``ChunkedItem`` renders 5 columns per
+row (the unset autoincrement pk is skipped), so 13,200 rows bind 66,000
+parameters — over sqlite's 32,766 (SQLITE_MAX_VARIABLE_NUMBER) and Postgres's
+65,535 (int16 parameter count in the wire protocol's Bind message).
+"""
+
+import pytest
+from typing import Annotated
+
+from ferro import FerroField, Model, connect, engines, transaction
+from ferro.exceptions import IntegrityError
+
+pytestmark = pytest.mark.backend_matrix
+
+STRADDLE_ROWS = 13_200  # 13,200 rows x 5 columns = 66,000 binds > both limits
+
+
+class ChunkedItem(Model):
+    id: Annotated[int | None, FerroField(primary_key=True)] = None
+    label: str
+    quantity: int
+    price: float
+    is_active: bool
+    note: str
+
+
+def make_items(n: int) -> list[ChunkedItem]:
+    return [
+        ChunkedItem(
+            label=f"item-{i}",
+            quantity=i,
+            price=i * 0.5,
+            is_active=i % 2 == 0,
+            note=f"note-{i}",
+        )
+        for i in range(n)
+    ]
+
+
+@pytest.mark.asyncio
+async def test_bulk_create_succeeds_beyond_bind_limits(db_url):
+    """A batch binding more parameters than either backend allows inserts fully."""
+
+    await connect(db_url, auto_migrate=True)
+    async with engines.session():
+        inserted = await ChunkedItem.bulk_create(make_items(STRADDLE_ROWS))
+        assert inserted == STRADDLE_ROWS
+
+        assert (
+            await ChunkedItem.where(lambda item: item.quantity >= 0).count()
+            == STRADDLE_ROWS
+        )
+
+        # Spot-check a row from the final chunk survived with its values intact.
+        last = await ChunkedItem.where(
+            lambda item: item.label == f"item-{STRADDLE_ROWS - 1}"
+        ).first()
+        assert last is not None
+        assert last.quantity == STRADDLE_ROWS - 1
+        assert last.note == f"note-{STRADDLE_ROWS - 1}"
+
+
+class UniqueChunkedItem(Model):
+    id: Annotated[int | None, FerroField(primary_key=True)] = None
+    label: Annotated[str, FerroField(unique=True)]
+    quantity: int
+    price: float
+    is_active: bool
+    note: str
+
+
+def make_unique_items(n: int) -> list[UniqueChunkedItem]:
+    return [
+        UniqueChunkedItem(
+            label=f"item-{i}",
+            quantity=i,
+            price=i * 0.5,
+            is_active=i % 2 == 0,
+            note=f"note-{i}",
+        )
+        for i in range(n)
+    ]
+
+
+@pytest.mark.asyncio
+async def test_bare_bulk_create_is_all_or_nothing_across_chunks(db_url):
+    """A late-chunk failure in a bare call must leave no rows from the batch.
+
+    The collision sits at index 13,150 — past every chunk boundary either
+    backend produces for a 5-column row budget — so earlier chunks have
+    already executed when the violation fires.
+    """
+
+    await connect(db_url, auto_migrate=True)
+    async with engines.session():
+        await UniqueChunkedItem.create(
+            label="item-13150", quantity=0, price=0.0, is_active=True, note="seed"
+        )
+
+        with pytest.raises(IntegrityError):
+            await UniqueChunkedItem.bulk_create(make_unique_items(STRADDLE_ROWS))
+
+        survivors = await UniqueChunkedItem.all()
+        assert len(survivors) == 1
+        assert survivors[0].note == "seed"
+
+
+@pytest.mark.asyncio
+async def test_ambient_transaction_is_the_atomicity_boundary(db_url):
+    """An aborted ambient transaction() rolls back every internal chunk."""
+
+    await connect(db_url, auto_migrate=True)
+    async with engines.session():
+        try:
+            async with transaction():
+                inserted = await ChunkedItem.bulk_create(make_items(STRADDLE_ROWS))
+                assert inserted == STRADDLE_ROWS
+                raise RuntimeError("abort after chunked bulk_create")
+        except RuntimeError:
+            pass
+
+        assert await ChunkedItem.where(lambda item: item.quantity >= 0).count() == 0
+
+
+@pytest.mark.asyncio
+async def test_late_chunk_failure_inside_ambient_transaction_leaves_zero_rows(db_url):
+    """A mid-batch unique violation propagates and aborts the ambient transaction."""
+
+    await connect(db_url, auto_migrate=True)
+    async with engines.session():
+        with pytest.raises(IntegrityError):
+            async with transaction():
+                await UniqueChunkedItem.create(
+                    label="item-13150",
+                    quantity=0,
+                    price=0.0,
+                    is_active=True,
+                    note="seed",
+                )
+                await UniqueChunkedItem.bulk_create(make_unique_items(STRADDLE_ROWS))
+
+        assert await UniqueChunkedItem.all() == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "rows",
+    [
+        6_553,  # 32,765 binds — one under sqlite's 32,766 ceiling
+        6_554,  # 32,770 binds — the 0.16.0 sqlite failure threshold (#298)
+        13_107,  # 65,535 binds — exactly Postgres's ceiling
+    ],
+)
+async def test_bulk_create_at_exact_bind_limit_boundaries(db_url, rows):
+    """Row counts straddling each backend's exact bind ceiling insert fully."""
+
+    await connect(db_url, auto_migrate=True)
+    async with engines.session():
+        assert await ChunkedItem.bulk_create(make_items(rows)) == rows
+        assert await ChunkedItem.where(lambda item: item.quantity >= 0).count() == rows