sqlobjects 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlobjects/__init__.py +1 -1
- sqlobjects/cascade.py +22 -31
- sqlobjects/contrib/asgi.py +5 -3
- sqlobjects/database/manager.py +44 -30
- sqlobjects/expressions/aggregate.py +144 -83
- sqlobjects/expressions/scalar.py +5 -9
- sqlobjects/fields/__init__.py +2 -0
- sqlobjects/fields/core.py +3 -2
- sqlobjects/fields/proxies.py +78 -30
- sqlobjects/fields/relations/descriptors.py +4 -2
- sqlobjects/fields/relations/prefetch.py +62 -63
- sqlobjects/fields/relations/utils.py +50 -11
- sqlobjects/internal/results.py +1 -1
- sqlobjects/metadata.py +106 -30
- sqlobjects/mixins.py +24 -5
- sqlobjects/model.py +45 -72
- sqlobjects/objects/bulk.py +159 -245
- sqlobjects/objects/core.py +62 -58
- sqlobjects/objects/upsert.py +85 -8
- sqlobjects/queries/builder.py +77 -145
- sqlobjects/queries/dialect.py +4 -4
- sqlobjects/queries/executor.py +105 -209
- sqlobjects/queryset.py +64 -45
- sqlobjects/session.py +342 -38
- sqlobjects/signals.py +60 -33
- sqlobjects/sql_logging.py +100 -11
- sqlobjects/utils/naming.py +17 -1
- {sqlobjects-2.0.1.data → sqlobjects-2.1.0.data}/data/share/sqlobjects/rules/01-database-session-guide.md +44 -5
- {sqlobjects-2.0.1.data → sqlobjects-2.1.0.data}/data/share/sqlobjects/rules/02-model-definition-guide.md +25 -0
- {sqlobjects-2.0.1.data → sqlobjects-2.1.0.data}/data/share/sqlobjects/rules/03-query-operations-guide.md +33 -4
- {sqlobjects-2.0.1.data → sqlobjects-2.1.0.data}/data/share/sqlobjects/rules/05-relationships-guide.md +37 -15
- {sqlobjects-2.0.1.data → sqlobjects-2.1.0.data}/data/share/sqlobjects/rules/06-validation-signals-guide.md +32 -0
- {sqlobjects-2.0.1.dist-info → sqlobjects-2.1.0.dist-info}/METADATA +9 -2
- sqlobjects-2.1.0.dist-info/RECORD +75 -0
- sqlobjects-2.0.1.dist-info/RECORD +0 -75
- {sqlobjects-2.0.1.data → sqlobjects-2.1.0.data}/data/share/sqlobjects/rules/04-crud-operations-guide.md +0 -0
- {sqlobjects-2.0.1.data → sqlobjects-2.1.0.data}/data/share/sqlobjects/rules/07-performance-guide.md +0 -0
- {sqlobjects-2.0.1.data → sqlobjects-2.1.0.data}/data/share/sqlobjects/rules/README.md +0 -0
- {sqlobjects-2.0.1.dist-info → sqlobjects-2.1.0.dist-info}/WHEEL +0 -0
- {sqlobjects-2.0.1.dist-info → sqlobjects-2.1.0.dist-info}/entry_points.txt +0 -0
- {sqlobjects-2.0.1.dist-info → sqlobjects-2.1.0.dist-info}/licenses/LICENSE +0 -0
- {sqlobjects-2.0.1.dist-info → sqlobjects-2.1.0.dist-info}/top_level.txt +0 -0
sqlobjects/objects/bulk.py
CHANGED
|
@@ -14,6 +14,7 @@ from sqlalchemy import bindparam, insert, select, update
|
|
|
14
14
|
from ..cascade import has_cascade_delete_relations
|
|
15
15
|
from ..internal import ResultProcessor
|
|
16
16
|
from ..session import AsyncSession, ctx_session
|
|
17
|
+
from ..signals import Operation, SignalContext
|
|
17
18
|
from .upsert import ConflictResolution as UpsertConflictResolution
|
|
18
19
|
from .upsert import UpsertHandler
|
|
19
20
|
|
|
@@ -21,6 +22,13 @@ from .upsert import UpsertHandler
|
|
|
21
22
|
T = TypeVar("T")
|
|
22
23
|
|
|
23
24
|
|
|
25
|
+
class _BulkSignalError(Exception):
|
|
26
|
+
"""Keep application hook failures out of the record-error collection policy."""
|
|
27
|
+
|
|
28
|
+
def __init__(self, original: Exception):
|
|
29
|
+
self.original = original
|
|
30
|
+
|
|
31
|
+
|
|
24
32
|
def _get_error_code(exception: Exception) -> str:
|
|
25
33
|
"""Extract error code from database exception."""
|
|
26
34
|
error_str = str(exception).lower()
|
|
@@ -143,22 +151,12 @@ class BulkTransactionManager:
|
|
|
143
151
|
|
|
144
152
|
async def execute_batch(self, operation_func: Callable, batch_data: Any) -> Any:
|
|
145
153
|
"""Execute batch operation with appropriate transaction control."""
|
|
146
|
-
if self.mode == TransactionMode.
|
|
147
|
-
async with ctx_session() as new_session:
|
|
154
|
+
if self.mode == TransactionMode.BATCH:
|
|
155
|
+
async with ctx_session(self.session.db_name) as new_session:
|
|
148
156
|
return await operation_func(new_session, batch_data)
|
|
149
|
-
elif self.mode == TransactionMode.BATCH:
|
|
150
|
-
# For BATCH mode, use the existing session without creating new transaction
|
|
151
|
-
# The session should handle its own transaction boundaries
|
|
152
|
-
return await operation_func(self.session, batch_data)
|
|
153
157
|
elif self.mode == TransactionMode.SAVEPOINT:
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
result = await operation_func(self.session, batch_data)
|
|
157
|
-
await savepoint.commit()
|
|
158
|
-
return result
|
|
159
|
-
except Exception:
|
|
160
|
-
await savepoint.rollback()
|
|
161
|
-
raise
|
|
158
|
+
async with await self.session.begin_nested():
|
|
159
|
+
return await operation_func(self.session, batch_data)
|
|
162
160
|
else: # INHERIT
|
|
163
161
|
return await operation_func(self.session, batch_data)
|
|
164
162
|
|
|
@@ -173,6 +171,7 @@ class BulkOperationHandler:
|
|
|
173
171
|
self.dialect = session.bind.dialect
|
|
174
172
|
self.transaction_manager = BulkTransactionManager(session, transaction_mode)
|
|
175
173
|
self._current_return_fields: list[str] | None = None
|
|
174
|
+
self.accurate_count = True
|
|
176
175
|
|
|
177
176
|
async def execute_bulk_operation(
|
|
178
177
|
self,
|
|
@@ -181,33 +180,80 @@ class BulkOperationHandler:
|
|
|
181
180
|
operation_func,
|
|
182
181
|
transaction_mode: TransactionMode,
|
|
183
182
|
on_error: ErrorHandling,
|
|
183
|
+
operation: Operation,
|
|
184
184
|
):
|
|
185
185
|
"""Execute bulk operation with unified batch processing logic."""
|
|
186
|
+
if batch_size <= 0:
|
|
187
|
+
raise ValueError("batch_size must be positive")
|
|
188
|
+
if transaction_mode == TransactionMode.INDEPENDENT:
|
|
189
|
+
async with ctx_session(self.session.db_name) as transaction:
|
|
190
|
+
handler = BulkOperationHandler(transaction, self.table, self.model_class, TransactionMode.INHERIT)
|
|
191
|
+
return await handler.execute_bulk_operation(
|
|
192
|
+
data, batch_size, operation_func, TransactionMode.INHERIT, on_error, operation
|
|
193
|
+
)
|
|
194
|
+
if transaction_mode != TransactionMode.BATCH and self.session.auto_commit:
|
|
195
|
+
async with self.session.transaction() as transaction:
|
|
196
|
+
handler = BulkOperationHandler(transaction, self.table, self.model_class, transaction_mode)
|
|
197
|
+
return await handler.execute_bulk_operation(
|
|
198
|
+
data, batch_size, operation_func, transaction_mode, on_error, operation
|
|
199
|
+
)
|
|
200
|
+
|
|
186
201
|
all_successful_objects = []
|
|
187
202
|
all_failed_records = []
|
|
188
203
|
total_success_count = 0
|
|
189
204
|
batch_count = 0
|
|
190
205
|
failed_batches = 0
|
|
191
206
|
rollback_count = 0
|
|
207
|
+
batch_completed = False
|
|
208
|
+
|
|
209
|
+
async def emit(timing, context):
|
|
210
|
+
try:
|
|
211
|
+
await self.model_class._emit_bulk_signal(timing, context)
|
|
212
|
+
except Exception as error:
|
|
213
|
+
raise _BulkSignalError(error) from error
|
|
214
|
+
|
|
215
|
+
async def execute_signaled_batch(transaction, batch):
|
|
216
|
+
nonlocal batch_completed
|
|
217
|
+
context = SignalContext(
|
|
218
|
+
operation=Operation.DELETE if operation == Operation.DELETE else Operation.SAVE,
|
|
219
|
+
actual_operation=operation,
|
|
220
|
+
session=transaction,
|
|
221
|
+
model_class=self.model_class,
|
|
222
|
+
affected_count=len(batch),
|
|
223
|
+
)
|
|
224
|
+
await emit("before", context)
|
|
225
|
+
result = await operation_func(transaction, batch)
|
|
226
|
+
context.affected_count = result[1]
|
|
227
|
+
await emit("after", context)
|
|
228
|
+
batch_completed = True
|
|
229
|
+
return result
|
|
192
230
|
|
|
193
231
|
for i in range(0, len(data), batch_size):
|
|
232
|
+
batch_completed = False
|
|
194
233
|
batch = data[i : i + batch_size]
|
|
195
234
|
batch_count += 1
|
|
196
235
|
|
|
197
236
|
try:
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
237
|
+
successful_objects, success_count, failed_records = await self.transaction_manager.execute_batch(
|
|
238
|
+
execute_signaled_batch, batch
|
|
239
|
+
)
|
|
240
|
+
for failed in failed_records:
|
|
241
|
+
failed.index += i
|
|
242
|
+
failed.batch_index = batch_count - 1
|
|
204
243
|
|
|
205
244
|
all_successful_objects.extend(successful_objects)
|
|
206
245
|
total_success_count += success_count
|
|
207
246
|
all_failed_records.extend(failed_records)
|
|
208
247
|
|
|
248
|
+
except _BulkSignalError as error:
|
|
249
|
+
raise error.original from error
|
|
209
250
|
except Exception as e:
|
|
251
|
+
# Commit failures and post-commit callback errors are not row
|
|
252
|
+
# failures. In particular, committed data cannot be rolled back.
|
|
253
|
+
if batch_completed:
|
|
254
|
+
raise
|
|
210
255
|
failed_batches += 1
|
|
256
|
+
rollback_count += 1
|
|
211
257
|
if on_error == ErrorHandling.FAIL_FAST:
|
|
212
258
|
raise
|
|
213
259
|
elif on_error == ErrorHandling.IGNORE:
|
|
@@ -271,7 +317,11 @@ class BulkOperationHandler:
|
|
|
271
317
|
|
|
272
318
|
def create_objects_from_rows(self, rows, return_fields: list[str] | None = None) -> list:
|
|
273
319
|
"""Create model objects or dictionaries from database rows using ResultProcessor."""
|
|
274
|
-
|
|
320
|
+
objects = ResultProcessor.rows_to_objects(rows, self.model_class, return_fields)
|
|
321
|
+
for obj in objects:
|
|
322
|
+
if not isinstance(obj, dict):
|
|
323
|
+
obj.using(self.session.db_name)
|
|
324
|
+
return objects
|
|
275
325
|
|
|
276
326
|
async def execute_with_returning(
|
|
277
327
|
self,
|
|
@@ -300,7 +350,7 @@ class BulkOperationHandler:
|
|
|
300
350
|
else:
|
|
301
351
|
result = await exec_session.execute(stmt_with_returning)
|
|
302
352
|
objects = self.create_objects_from_rows(result.fetchall(), return_fields)
|
|
303
|
-
return objects,
|
|
353
|
+
return objects, len(objects), True
|
|
304
354
|
|
|
305
355
|
# Regular execution without RETURNING
|
|
306
356
|
if parameters is not None and isinstance(parameters, list) and len(parameters) > 1:
|
|
@@ -312,7 +362,11 @@ class BulkOperationHandler:
|
|
|
312
362
|
|
|
313
363
|
# Handle rowcount: -1 means unknown, use parameter count as fallback
|
|
314
364
|
rowcount = result.rowcount if result.rowcount >= 0 else 0 # noqa
|
|
315
|
-
if
|
|
365
|
+
if (
|
|
366
|
+
result.rowcount < 0
|
|
367
|
+
and isinstance(parameters, list)
|
|
368
|
+
and (operation == "insert" or (operation == "update" and not self.accurate_count))
|
|
369
|
+
):
|
|
316
370
|
# For INSERT/UPDATE without proper rowcount, use parameter count
|
|
317
371
|
rowcount = len(parameters)
|
|
318
372
|
|
|
@@ -354,58 +408,47 @@ class BulkOperationHandler:
|
|
|
354
408
|
successful_objects, success_count = await self._execute_batch_operation(
|
|
355
409
|
stmt, operation, data_batch, return_columns, exec_session
|
|
356
410
|
)
|
|
357
|
-
|
|
358
|
-
#
|
|
359
|
-
for _, data in enumerate(data_batch):
|
|
360
|
-
try:
|
|
361
|
-
single_stmt = stmt.values([data]) if hasattr(stmt, "values") else stmt
|
|
362
|
-
if return_columns and self.supports_returning(operation):
|
|
363
|
-
single_stmt = single_stmt.returning(*return_columns)
|
|
364
|
-
result = await exec_session.execute(single_stmt)
|
|
365
|
-
rows = result.fetchall()
|
|
366
|
-
if rows:
|
|
367
|
-
current_return_fields = getattr(self, "_current_return_fields", None)
|
|
368
|
-
successful_objects.extend(self.create_objects_from_rows(rows, current_return_fields))
|
|
369
|
-
success_count += 1
|
|
370
|
-
else:
|
|
371
|
-
result = await exec_session.execute(single_stmt)
|
|
372
|
-
if result.rowcount > 0: # noqa
|
|
373
|
-
success_count += 1
|
|
374
|
-
except Exception: # noqa
|
|
375
|
-
# Silently ignore errors
|
|
376
|
-
pass
|
|
377
|
-
|
|
378
|
-
elif error_handling == ErrorHandling.COLLECT:
|
|
379
|
-
# Process individually to collect detailed errors
|
|
411
|
+
else:
|
|
412
|
+
# Each recoverable error is isolated from successful rows and the outer transaction.
|
|
380
413
|
for i, data in enumerate(data_batch):
|
|
381
414
|
try:
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
else:
|
|
392
|
-
result = await exec_session.execute(single_stmt)
|
|
393
|
-
if result.rowcount > 0: # noqa
|
|
394
|
-
success_count += 1
|
|
395
|
-
except Exception as e:
|
|
396
|
-
failed_records.append(FailedRecord(i, data, e, _get_error_code(e)))
|
|
415
|
+
async with await exec_session.begin_nested():
|
|
416
|
+
objects, count = await self._execute_batch_operation(
|
|
417
|
+
stmt, operation, [data], return_columns, exec_session
|
|
418
|
+
)
|
|
419
|
+
successful_objects.extend(objects)
|
|
420
|
+
success_count += count
|
|
421
|
+
except Exception as error:
|
|
422
|
+
if error_handling == ErrorHandling.COLLECT:
|
|
423
|
+
failed_records.append(FailedRecord(i, data, error, _get_error_code(error)))
|
|
397
424
|
|
|
398
425
|
return successful_objects, success_count, failed_records
|
|
399
426
|
|
|
427
|
+
async def execute_upsert(self, data_batch, resolution, match_fields, on_error, session):
|
|
428
|
+
"""Apply the same recoverable-error boundaries to conflict-aware writes."""
|
|
429
|
+
handler = UpsertHandler(session)
|
|
430
|
+
if on_error == ErrorHandling.FAIL_FAST:
|
|
431
|
+
return await handler.execute_upsert_with_returning(self.table, data_batch, resolution, match_fields), []
|
|
432
|
+
rows, failures = [], []
|
|
433
|
+
for index, data in enumerate(data_batch):
|
|
434
|
+
try:
|
|
435
|
+
async with await session.begin_nested():
|
|
436
|
+
stored = await handler.execute_upsert_with_returning(self.table, [data], resolution, match_fields)
|
|
437
|
+
rows.extend(stored)
|
|
438
|
+
except Exception as error:
|
|
439
|
+
if on_error == ErrorHandling.COLLECT:
|
|
440
|
+
failures.append(FailedRecord(index, data, error, _get_error_code(error)))
|
|
441
|
+
return rows, failures
|
|
442
|
+
|
|
400
443
|
async def handle_insert_fallback(self, insert_result, return_columns, session: AsyncSession | None = None):
|
|
401
444
|
"""Handle INSERT fallback for single record."""
|
|
402
445
|
exec_session = session or self.session
|
|
403
446
|
if hasattr(insert_result, "inserted_primary_key") and insert_result.inserted_primary_key:
|
|
404
447
|
pk_columns = list(self.table.primary_key.columns)
|
|
405
448
|
if pk_columns:
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
449
|
+
select_stmt = select(*return_columns).where(
|
|
450
|
+
*(col == value for col, value in zip(pk_columns, insert_result.inserted_primary_key, strict=True))
|
|
451
|
+
)
|
|
409
452
|
result = await exec_session.execute(select_stmt)
|
|
410
453
|
current_return_fields = getattr(self, "_current_return_fields", None)
|
|
411
454
|
return self.create_objects_from_rows(result.fetchall(), current_return_fields)
|
|
@@ -420,131 +463,49 @@ class BulkOperationHandler:
|
|
|
420
463
|
current_return_fields = getattr(self, "_current_return_fields", None)
|
|
421
464
|
return self.create_objects_from_rows(rows, current_return_fields)
|
|
422
465
|
|
|
423
|
-
async def _handle_insert_fallback_batch(
|
|
424
|
-
self, data_batch: list, return_columns, session: AsyncSession | None = None
|
|
425
|
-
):
|
|
426
|
-
"""Handle INSERT fallback for batch operations to get accurate data including DB-generated fields."""
|
|
427
|
-
exec_session = session or self.session
|
|
428
|
-
objects = []
|
|
429
|
-
|
|
430
|
-
# For each inserted record, try to find it in the database
|
|
431
|
-
# This is expensive but ensures data accuracy
|
|
432
|
-
for data in data_batch:
|
|
433
|
-
# Build WHERE conditions based on the input data to find the inserted record
|
|
434
|
-
where_conditions = []
|
|
435
|
-
for key, value in data.items():
|
|
436
|
-
if key in self.table.columns and value is not None:
|
|
437
|
-
where_conditions.append(self.table.c[key] == value)
|
|
438
|
-
|
|
439
|
-
if where_conditions:
|
|
440
|
-
# Try to find the record using the input data
|
|
441
|
-
select_stmt = select(*return_columns).where(*where_conditions)
|
|
442
|
-
result = await exec_session.execute(select_stmt)
|
|
443
|
-
rows = result.fetchall()
|
|
444
|
-
|
|
445
|
-
if rows:
|
|
446
|
-
# Take the first match (there might be multiple if data isn't unique)
|
|
447
|
-
current_return_fields = getattr(self, "_current_return_fields", None)
|
|
448
|
-
objects.extend(self.create_objects_from_rows([rows[0]], current_return_fields))
|
|
449
|
-
else:
|
|
450
|
-
# Fallback: create object from input data (missing DB-generated fields)
|
|
451
|
-
objects.append(self.model_class.from_dict(data, validate=False))
|
|
452
|
-
else:
|
|
453
|
-
# No usable WHERE conditions, fallback to input data
|
|
454
|
-
objects.append(self.model_class.from_dict(data, validate=False))
|
|
455
|
-
|
|
456
|
-
return objects
|
|
457
|
-
|
|
458
466
|
async def _execute_batch_operation(
|
|
459
467
|
self, stmt, operation: str, data_batch: list, return_columns, session: AsyncSession
|
|
460
468
|
) -> tuple[list, int]:
|
|
461
469
|
"""Execute batch operation with unified logic for FAIL_FAST and IGNORE modes."""
|
|
462
|
-
|
|
463
|
-
|
|
470
|
+
if operation == "update" and (
|
|
471
|
+
return_columns or (self.accurate_count and not self.dialect.supports_sane_multi_rowcount)
|
|
472
|
+
):
|
|
464
473
|
return await self._handle_update_without_returning(stmt, data_batch, return_columns, session)
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
)
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
return successful_objects, success_count
|
|
474
|
+
if operation == "insert" and return_columns and not self.supports_returning(operation):
|
|
475
|
+
objects = []
|
|
476
|
+
for data in data_batch:
|
|
477
|
+
result = await session.execute(stmt, data)
|
|
478
|
+
objects.extend(await self.handle_insert_fallback(result, return_columns, session))
|
|
479
|
+
return objects, len(objects)
|
|
480
|
+
current_return_fields = self._current_return_fields
|
|
481
|
+
objects, count, _ = await self.execute_with_returning(
|
|
482
|
+
stmt, operation, return_columns, data_batch, session, current_return_fields
|
|
483
|
+
)
|
|
484
|
+
return objects, count
|
|
477
485
|
|
|
478
486
|
async def _handle_update_without_returning(
|
|
479
487
|
self, stmt, data_batch: list, return_columns, session: AsyncSession
|
|
480
488
|
) -> tuple[list, int]:
|
|
481
|
-
"""
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
for col in return_columns:
|
|
502
|
-
obj_data[col.name] = None
|
|
503
|
-
|
|
504
|
-
# For UPDATE operations, convert parameter names back to field names
|
|
505
|
-
for key, value in data.items():
|
|
506
|
-
if key.startswith("match::"):
|
|
507
|
-
field_name = key[7:] # Remove "match::" prefix
|
|
508
|
-
obj_data[field_name] = value
|
|
509
|
-
elif key.startswith("update::"):
|
|
510
|
-
field_name = key[8:] # Remove "update::" prefix
|
|
511
|
-
obj_data[field_name] = value
|
|
512
|
-
else:
|
|
513
|
-
obj_data[key] = value
|
|
514
|
-
|
|
515
|
-
current_return_fields = getattr(self, "_current_return_fields", None)
|
|
516
|
-
if current_return_fields:
|
|
517
|
-
# Return only requested fields as dict
|
|
518
|
-
filtered_dict = {f: obj_data.get(f) for f in current_return_fields}
|
|
519
|
-
successful_objects.append(filtered_dict)
|
|
520
|
-
else:
|
|
521
|
-
# Return full model object
|
|
522
|
-
successful_objects.append(self.model_class.from_dict(obj_data, validate=False))
|
|
523
|
-
else:
|
|
524
|
-
successful_objects = []
|
|
525
|
-
|
|
526
|
-
return successful_objects, success_count
|
|
527
|
-
|
|
528
|
-
async def _handle_fallback_without_returning(
|
|
529
|
-
self, operation: str, data_batch: list, return_columns, success_count: int, session: AsyncSession
|
|
530
|
-
) -> list:
|
|
531
|
-
"""Handle fallback for operations without RETURNING support."""
|
|
532
|
-
import warnings
|
|
533
|
-
|
|
534
|
-
warnings.warn(
|
|
535
|
-
f"Database does not support {operation.upper()} RETURNING. "
|
|
536
|
-
f"return_objects=True will have significant performance impact. "
|
|
537
|
-
f"Consider using a database that supports RETURNING for better performance.",
|
|
538
|
-
UserWarning,
|
|
539
|
-
stacklevel=4,
|
|
540
|
-
)
|
|
541
|
-
|
|
542
|
-
if operation == "insert":
|
|
543
|
-
# For INSERT operations, use fallback to get accurate data including DB-generated fields
|
|
544
|
-
return await self._handle_insert_fallback_batch(data_batch[:success_count], return_columns, session)
|
|
545
|
-
else:
|
|
546
|
-
# For other operations, create objects from input data as approximation
|
|
547
|
-
return [self.model_class.from_dict(data, validate=False) for data in data_batch[:success_count]]
|
|
489
|
+
"""Execute each update once and return only rows that actually matched."""
|
|
490
|
+
objects = []
|
|
491
|
+
count = 0
|
|
492
|
+
for data in data_batch:
|
|
493
|
+
if return_columns and self.dialect.update_returning:
|
|
494
|
+
result = await session.execute(stmt.returning(*return_columns), data)
|
|
495
|
+
rows = result.fetchall()
|
|
496
|
+
count += len(rows)
|
|
497
|
+
objects.extend(self.create_objects_from_rows(rows, self._current_return_fields))
|
|
498
|
+
else:
|
|
499
|
+
result = await session.execute(stmt, data)
|
|
500
|
+
matched = max(result.rowcount, 0)
|
|
501
|
+
count += matched
|
|
502
|
+
if return_columns and matched:
|
|
503
|
+
conditions = [
|
|
504
|
+
self.table.c[key[7:]] == value for key, value in data.items() if key.startswith("match::")
|
|
505
|
+
]
|
|
506
|
+
stored = await session.execute(select(*return_columns).where(*conditions))
|
|
507
|
+
objects.extend(self.create_objects_from_rows(stored.fetchall(), self._current_return_fields))
|
|
508
|
+
return objects, count
|
|
548
509
|
|
|
549
510
|
|
|
550
511
|
# ========================================
|
|
@@ -615,7 +576,6 @@ async def bulk_create(
|
|
|
615
576
|
async def operation_func(session_, batch_data):
|
|
616
577
|
# Use unified UPSERT handler for conflict resolution
|
|
617
578
|
if on_conflict != ConflictResolution.ERROR:
|
|
618
|
-
upsert_handler = UpsertHandler(session_)
|
|
619
579
|
upsert_conflict = (
|
|
620
580
|
UpsertConflictResolution.IGNORE
|
|
621
581
|
if on_conflict == ConflictResolution.IGNORE
|
|
@@ -626,11 +586,8 @@ async def bulk_create(
|
|
|
626
586
|
handler._current_return_fields = return_fields
|
|
627
587
|
|
|
628
588
|
# Execute UPSERT with returning
|
|
629
|
-
batch_results = await
|
|
630
|
-
|
|
631
|
-
batch_data,
|
|
632
|
-
upsert_conflict,
|
|
633
|
-
conflict_fields, # noqa
|
|
589
|
+
batch_results, failures = await handler.execute_upsert(
|
|
590
|
+
batch_data, upsert_conflict, conflict_fields, on_error, session_
|
|
634
591
|
)
|
|
635
592
|
|
|
636
593
|
# Convert results to model instances only if return_objects is True
|
|
@@ -641,10 +598,10 @@ async def bulk_create(
|
|
|
641
598
|
filtered_dict = {f: result_data.get(f) for f in return_fields}
|
|
642
599
|
operator_successful_objects.append(filtered_dict)
|
|
643
600
|
else:
|
|
644
|
-
instance = manager._model_class.
|
|
601
|
+
instance = manager._model_class._from_database(result_data, session_.db_name) # noqa
|
|
645
602
|
operator_successful_objects.append(instance)
|
|
646
603
|
|
|
647
|
-
return operator_successful_objects, len(batch_results),
|
|
604
|
+
return operator_successful_objects, len(batch_results), failures
|
|
648
605
|
else:
|
|
649
606
|
# Regular insert without conflict resolution
|
|
650
607
|
stmt = insert(manager._table) # noqa
|
|
@@ -660,7 +617,9 @@ async def bulk_create(
|
|
|
660
617
|
batch_count,
|
|
661
618
|
failed_batches,
|
|
662
619
|
rollback_count,
|
|
663
|
-
) = await handler.execute_bulk_operation(
|
|
620
|
+
) = await handler.execute_bulk_operation(
|
|
621
|
+
processed_objects, batch_size, operation_func, transaction_mode, on_error, Operation.CREATE
|
|
622
|
+
)
|
|
664
623
|
|
|
665
624
|
return handler.build_result(
|
|
666
625
|
successful_objects,
|
|
@@ -727,9 +686,7 @@ async def bulk_update(
|
|
|
727
686
|
session = manager._get_session(readonly=False) # noqa
|
|
728
687
|
handler = BulkOperationHandler(session, manager._table, manager._model_class, transaction_mode) # noqa
|
|
729
688
|
|
|
730
|
-
|
|
731
|
-
if not accurate_count:
|
|
732
|
-
return await _bulk_update_fast(manager, mappings, match_fields, batch_size, session)
|
|
689
|
+
handler.accurate_count = accurate_count
|
|
733
690
|
|
|
734
691
|
# Build base statement
|
|
735
692
|
where_conditions = [manager._table.c[field] == bindparam(f"match::{field}") for field in match_fields] # noqa
|
|
@@ -738,18 +695,14 @@ async def bulk_update(
|
|
|
738
695
|
async def operation_func(session_, batch_data):
|
|
739
696
|
# Use unified UPSERT handler for bulk update with conflict resolution
|
|
740
697
|
if on_conflict != ConflictResolution.ERROR:
|
|
741
|
-
upsert_handler = UpsertHandler(session_)
|
|
742
698
|
upsert_conflict = UpsertConflictResolution.UPDATE
|
|
743
699
|
|
|
744
700
|
# Store return_fields in handler for use in create_objects_from_rows
|
|
745
701
|
handler._current_return_fields = return_fields
|
|
746
702
|
|
|
747
703
|
# Execute UPSERT with returning
|
|
748
|
-
batch_results = await
|
|
749
|
-
|
|
750
|
-
batch_data,
|
|
751
|
-
upsert_conflict,
|
|
752
|
-
match_fields, # noqa
|
|
704
|
+
batch_results, failures = await handler.execute_upsert(
|
|
705
|
+
batch_data, upsert_conflict, match_fields, on_error, session_
|
|
753
706
|
)
|
|
754
707
|
|
|
755
708
|
# Convert results to model instances only if return_objects is True
|
|
@@ -760,10 +713,10 @@ async def bulk_update(
|
|
|
760
713
|
filtered_dict = {field: result_data.get(field) for field in return_fields}
|
|
761
714
|
oper_successful_objects.append(filtered_dict)
|
|
762
715
|
else:
|
|
763
|
-
instance = manager._model_class.
|
|
716
|
+
instance = manager._model_class._from_database(result_data, session_.db_name) # noqa
|
|
764
717
|
oper_successful_objects.append(instance)
|
|
765
718
|
|
|
766
|
-
return oper_successful_objects, len(batch_results),
|
|
719
|
+
return oper_successful_objects, len(batch_results), failures
|
|
767
720
|
else:
|
|
768
721
|
# Regular update without conflict resolution
|
|
769
722
|
update_values = {
|
|
@@ -798,7 +751,9 @@ async def bulk_update(
|
|
|
798
751
|
batch_count,
|
|
799
752
|
failed_batches,
|
|
800
753
|
rollback_count,
|
|
801
|
-
) = await handler.execute_bulk_operation(
|
|
754
|
+
) = await handler.execute_bulk_operation(
|
|
755
|
+
mappings, batch_size, operation_func, transaction_mode, on_error, Operation.UPDATE
|
|
756
|
+
)
|
|
802
757
|
|
|
803
758
|
return handler.build_result(
|
|
804
759
|
successful_objects,
|
|
@@ -915,7 +870,9 @@ async def bulk_delete(
|
|
|
915
870
|
batch_count,
|
|
916
871
|
failed_batches,
|
|
917
872
|
rollback_count,
|
|
918
|
-
) = await handler.execute_bulk_operation(
|
|
873
|
+
) = await handler.execute_bulk_operation(
|
|
874
|
+
ids, batch_size, operation_func, transaction_mode, on_error, Operation.DELETE
|
|
875
|
+
)
|
|
919
876
|
|
|
920
877
|
return handler.build_result(
|
|
921
878
|
successful_objects,
|
|
@@ -928,46 +885,3 @@ async def bulk_delete(
|
|
|
928
885
|
rollback_count,
|
|
929
886
|
return_objects,
|
|
930
887
|
)
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
async def _bulk_update_fast(
|
|
934
|
-
manager, mappings: list[dict[str, Any]], match_fields: list[str], batch_size: int, session: AsyncSession
|
|
935
|
-
) -> int:
|
|
936
|
-
"""Fast bulk update using executemany (may return approximate count for PostgreSQL)."""
|
|
937
|
-
if not mappings:
|
|
938
|
-
return 0
|
|
939
|
-
|
|
940
|
-
# Build UPDATE statement
|
|
941
|
-
where_conditions = [manager._table.c[f] == bindparam(f"match::{f}") for f in match_fields] # noqa
|
|
942
|
-
stmt = update(manager._table).where(*where_conditions) # noqa
|
|
943
|
-
|
|
944
|
-
# Get update fields (exclude match fields)
|
|
945
|
-
update_values = {key: bindparam(f"update::{key}") for key in mappings[0].keys() if key not in match_fields}
|
|
946
|
-
if not update_values:
|
|
947
|
-
return 0
|
|
948
|
-
|
|
949
|
-
stmt = stmt.values(**update_values)
|
|
950
|
-
|
|
951
|
-
# Process in batches
|
|
952
|
-
total = 0
|
|
953
|
-
for i in range(0, len(mappings), batch_size):
|
|
954
|
-
batch = mappings[i : i + batch_size]
|
|
955
|
-
|
|
956
|
-
# Prepare parameter mappings
|
|
957
|
-
param_mappings = []
|
|
958
|
-
for mapping in batch:
|
|
959
|
-
param_dict = {}
|
|
960
|
-
for f in match_fields:
|
|
961
|
-
param_dict[f"match::{f}"] = mapping[f]
|
|
962
|
-
for key, value in mapping.items():
|
|
963
|
-
if key not in match_fields:
|
|
964
|
-
param_dict[f"update::{key}"] = value
|
|
965
|
-
param_mappings.append(param_dict)
|
|
966
|
-
|
|
967
|
-
# Execute with executemany
|
|
968
|
-
result = await session.execute(stmt, param_mappings)
|
|
969
|
-
# PostgreSQL returns -1, use batch length as approximation
|
|
970
|
-
count = result.rowcount if result.rowcount >= 0 else len(batch) # noqa
|
|
971
|
-
total += count
|
|
972
|
-
|
|
973
|
-
return total
|