django-bulk-hooks 0.1.83__py3-none-any.whl → 0.2.100__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of django-bulk-hooks might be problematic. Click here for more details.
- django_bulk_hooks/__init__.py +53 -50
- django_bulk_hooks/changeset.py +214 -0
- django_bulk_hooks/conditions.py +230 -351
- django_bulk_hooks/constants.py +4 -0
- django_bulk_hooks/context.py +49 -9
- django_bulk_hooks/decorators.py +219 -96
- django_bulk_hooks/dispatcher.py +588 -0
- django_bulk_hooks/factory.py +541 -0
- django_bulk_hooks/handler.py +106 -167
- django_bulk_hooks/helpers.py +258 -0
- django_bulk_hooks/manager.py +134 -208
- django_bulk_hooks/models.py +89 -101
- django_bulk_hooks/operations/__init__.py +18 -0
- django_bulk_hooks/operations/analyzer.py +466 -0
- django_bulk_hooks/operations/bulk_executor.py +757 -0
- django_bulk_hooks/operations/coordinator.py +928 -0
- django_bulk_hooks/operations/field_utils.py +341 -0
- django_bulk_hooks/operations/mti_handler.py +696 -0
- django_bulk_hooks/operations/mti_plans.py +103 -0
- django_bulk_hooks/operations/record_classifier.py +196 -0
- django_bulk_hooks/queryset.py +233 -43
- django_bulk_hooks/registry.py +276 -25
- django_bulk_hooks-0.2.100.dist-info/METADATA +320 -0
- django_bulk_hooks-0.2.100.dist-info/RECORD +27 -0
- django_bulk_hooks/engine.py +0 -53
- django_bulk_hooks-0.1.83.dist-info/METADATA +0 -228
- django_bulk_hooks-0.1.83.dist-info/RECORD +0 -16
- {django_bulk_hooks-0.1.83.dist-info → django_bulk_hooks-0.2.100.dist-info}/LICENSE +0 -0
- {django_bulk_hooks-0.1.83.dist-info → django_bulk_hooks-0.2.100.dist-info}/WHEEL +0 -0
|
@@ -0,0 +1,757 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bulk executor service for database operations.
|
|
3
|
+
|
|
4
|
+
Coordinates bulk database operations with validation and MTI handling.
|
|
5
|
+
This service is the only component that directly calls Django ORM methods.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
from typing import Any
|
|
10
|
+
from typing import Dict
|
|
11
|
+
from typing import List
|
|
12
|
+
from typing import Optional
|
|
13
|
+
from typing import Set
|
|
14
|
+
from typing import Tuple
|
|
15
|
+
|
|
16
|
+
from django.db import transaction
|
|
17
|
+
from django.db.models import AutoField
|
|
18
|
+
from django.db.models import Case
|
|
19
|
+
from django.db.models import ForeignKey
|
|
20
|
+
from django.db.models import Model
|
|
21
|
+
from django.db.models import QuerySet
|
|
22
|
+
from django.db.models import Value
|
|
23
|
+
from django.db.models import When
|
|
24
|
+
from django.db.models.constants import OnConflict
|
|
25
|
+
from django.db.models.functions import Cast
|
|
26
|
+
|
|
27
|
+
from django_bulk_hooks.helpers import tag_upsert_metadata
|
|
28
|
+
from django_bulk_hooks.operations.field_utils import get_field_value_for_db
|
|
29
|
+
from django_bulk_hooks.operations.field_utils import handle_auto_now_fields_for_inheritance_chain
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class BulkExecutor:
|
|
35
|
+
"""
|
|
36
|
+
Executes bulk database operations.
|
|
37
|
+
|
|
38
|
+
Coordinates validation, MTI handling, and database operations.
|
|
39
|
+
This is the only service that directly calls Django ORM methods.
|
|
40
|
+
|
|
41
|
+
All dependencies are explicitly injected via constructor for testability.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
queryset: QuerySet,
|
|
47
|
+
analyzer: Any,
|
|
48
|
+
mti_handler: Any,
|
|
49
|
+
record_classifier: Any,
|
|
50
|
+
) -> None:
|
|
51
|
+
"""
|
|
52
|
+
Initialize bulk executor with explicit dependencies.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
queryset: Django QuerySet instance
|
|
56
|
+
analyzer: ModelAnalyzer instance (validation and field tracking)
|
|
57
|
+
mti_handler: MTIHandler instance
|
|
58
|
+
record_classifier: RecordClassifier instance
|
|
59
|
+
"""
|
|
60
|
+
self.queryset = queryset
|
|
61
|
+
self.analyzer = analyzer
|
|
62
|
+
self.mti_handler = mti_handler
|
|
63
|
+
self.record_classifier = record_classifier
|
|
64
|
+
self.model_cls = queryset.model
|
|
65
|
+
|
|
66
|
+
def bulk_create(
|
|
67
|
+
self,
|
|
68
|
+
objs: List[Model],
|
|
69
|
+
batch_size: Optional[int] = None,
|
|
70
|
+
ignore_conflicts: bool = False,
|
|
71
|
+
update_conflicts: bool = False,
|
|
72
|
+
update_fields: Optional[List[str]] = None,
|
|
73
|
+
unique_fields: Optional[List[str]] = None,
|
|
74
|
+
existing_record_ids: Optional[Set[int]] = None,
|
|
75
|
+
existing_pks_map: Optional[Dict[int, int]] = None,
|
|
76
|
+
**kwargs: Any,
|
|
77
|
+
) -> List[Model]:
|
|
78
|
+
"""
|
|
79
|
+
Execute bulk create operation.
|
|
80
|
+
|
|
81
|
+
NOTE: Coordinator validates inputs before calling this method.
|
|
82
|
+
This executor trusts that inputs are pre-validated.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
objs: Model instances to create (pre-validated)
|
|
86
|
+
batch_size: Objects per batch
|
|
87
|
+
ignore_conflicts: Whether to ignore conflicts
|
|
88
|
+
update_conflicts: Whether to update on conflict
|
|
89
|
+
update_fields: Fields to update on conflict
|
|
90
|
+
unique_fields: Fields for conflict detection
|
|
91
|
+
existing_record_ids: Pre-classified existing record IDs
|
|
92
|
+
existing_pks_map: Pre-classified existing PK mapping
|
|
93
|
+
**kwargs: Additional arguments
|
|
94
|
+
|
|
95
|
+
Returns:
|
|
96
|
+
List of created/updated objects
|
|
97
|
+
"""
|
|
98
|
+
if not objs:
|
|
99
|
+
return objs
|
|
100
|
+
|
|
101
|
+
# Route to appropriate handler
|
|
102
|
+
if self.mti_handler.is_mti_model():
|
|
103
|
+
result = self._handle_mti_create(
|
|
104
|
+
objs=objs,
|
|
105
|
+
batch_size=batch_size,
|
|
106
|
+
update_conflicts=update_conflicts,
|
|
107
|
+
update_fields=update_fields,
|
|
108
|
+
unique_fields=unique_fields,
|
|
109
|
+
existing_record_ids=existing_record_ids,
|
|
110
|
+
existing_pks_map=existing_pks_map,
|
|
111
|
+
)
|
|
112
|
+
else:
|
|
113
|
+
result = self._execute_standard_bulk_create(
|
|
114
|
+
objs=objs,
|
|
115
|
+
batch_size=batch_size,
|
|
116
|
+
ignore_conflicts=ignore_conflicts,
|
|
117
|
+
update_conflicts=update_conflicts,
|
|
118
|
+
update_fields=update_fields,
|
|
119
|
+
unique_fields=unique_fields,
|
|
120
|
+
**kwargs,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
# Tag upsert metadata
|
|
124
|
+
self._handle_upsert_metadata_tagging(
|
|
125
|
+
result_objects=result,
|
|
126
|
+
objs=objs,
|
|
127
|
+
update_conflicts=update_conflicts,
|
|
128
|
+
unique_fields=unique_fields,
|
|
129
|
+
existing_record_ids=existing_record_ids,
|
|
130
|
+
existing_pks_map=existing_pks_map,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
return result
|
|
134
|
+
|
|
135
|
+
def bulk_update(self, objs: List[Model], fields: List[str], batch_size: Optional[int] = None) -> int:
|
|
136
|
+
"""
|
|
137
|
+
Execute bulk update operation.
|
|
138
|
+
|
|
139
|
+
NOTE: Coordinator validates inputs before calling this method.
|
|
140
|
+
This executor trusts that inputs are pre-validated.
|
|
141
|
+
|
|
142
|
+
Args:
|
|
143
|
+
objs: Model instances to update (pre-validated)
|
|
144
|
+
fields: Field names to update
|
|
145
|
+
batch_size: Objects per batch
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
Number of objects updated
|
|
149
|
+
"""
|
|
150
|
+
if not objs:
|
|
151
|
+
return 0
|
|
152
|
+
|
|
153
|
+
# Debug: Check FK values at bulk_update entry point
|
|
154
|
+
for obj in objs:
|
|
155
|
+
logger.debug("🚀 BULK_UPDATE_ENTRY: obj.pk=%s, business_id in __dict__=%s, value=%s",
|
|
156
|
+
getattr(obj, 'pk', 'None'),
|
|
157
|
+
'business_id' in obj.__dict__,
|
|
158
|
+
obj.__dict__.get('business_id', 'NOT_IN_DICT'))
|
|
159
|
+
|
|
160
|
+
# Ensure auto_now fields are included
|
|
161
|
+
fields = self._add_auto_now_fields(fields, objs)
|
|
162
|
+
|
|
163
|
+
# Route to appropriate handler
|
|
164
|
+
if self.mti_handler.is_mti_model():
|
|
165
|
+
logger.info(f"Using MTI bulk update for {self.model_cls.__name__}")
|
|
166
|
+
plan = self.mti_handler.build_update_plan(objs, fields, batch_size=batch_size)
|
|
167
|
+
return self._execute_mti_update_plan(plan)
|
|
168
|
+
|
|
169
|
+
# Standard bulk update
|
|
170
|
+
base_qs = self._get_base_queryset()
|
|
171
|
+
return base_qs.bulk_update(objs, fields, batch_size=batch_size)
|
|
172
|
+
|
|
173
|
+
def delete_queryset(self) -> Tuple[int, Dict[str, int]]:
|
|
174
|
+
"""
|
|
175
|
+
Execute delete on the queryset.
|
|
176
|
+
|
|
177
|
+
NOTE: Coordinator validates inputs before calling this method.
|
|
178
|
+
|
|
179
|
+
Returns:
|
|
180
|
+
Tuple of (count, details dict)
|
|
181
|
+
"""
|
|
182
|
+
if not self.queryset:
|
|
183
|
+
return 0, {}
|
|
184
|
+
|
|
185
|
+
return QuerySet.delete(self.queryset)
|
|
186
|
+
|
|
187
|
+
# ==================== Private: Create Helpers ====================
|
|
188
|
+
|
|
189
|
+
def _handle_mti_create(
|
|
190
|
+
self,
|
|
191
|
+
objs: List[Model],
|
|
192
|
+
batch_size: Optional[int],
|
|
193
|
+
update_conflicts: bool,
|
|
194
|
+
update_fields: Optional[List[str]],
|
|
195
|
+
unique_fields: Optional[List[str]],
|
|
196
|
+
existing_record_ids: Optional[Set[int]],
|
|
197
|
+
existing_pks_map: Optional[Dict[int, int]],
|
|
198
|
+
) -> List[Model]:
|
|
199
|
+
"""Handle MTI model creation with classification and planning."""
|
|
200
|
+
# Classify records if not pre-classified
|
|
201
|
+
if existing_record_ids is None or existing_pks_map is None:
|
|
202
|
+
existing_record_ids, existing_pks_map = self._classify_mti_records(objs, update_conflicts, unique_fields)
|
|
203
|
+
|
|
204
|
+
# Build and execute plan
|
|
205
|
+
plan = self.mti_handler.build_create_plan(
|
|
206
|
+
objs=objs,
|
|
207
|
+
batch_size=batch_size,
|
|
208
|
+
update_conflicts=update_conflicts,
|
|
209
|
+
update_fields=update_fields,
|
|
210
|
+
unique_fields=unique_fields,
|
|
211
|
+
existing_record_ids=existing_record_ids,
|
|
212
|
+
existing_pks_map=existing_pks_map,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
return self._execute_mti_create_plan(plan)
|
|
216
|
+
|
|
217
|
+
def _classify_mti_records(
|
|
218
|
+
self,
|
|
219
|
+
objs: List[Model],
|
|
220
|
+
update_conflicts: bool,
|
|
221
|
+
unique_fields: Optional[List[str]],
|
|
222
|
+
) -> Tuple[Set[int], Dict[int, int]]:
|
|
223
|
+
"""Classify MTI records for upsert operations."""
|
|
224
|
+
if not update_conflicts or not unique_fields:
|
|
225
|
+
return set(), {}
|
|
226
|
+
|
|
227
|
+
# Find correct model to query
|
|
228
|
+
query_model = self.mti_handler.find_model_with_unique_fields(unique_fields)
|
|
229
|
+
logger.info(f"MTI upsert: querying {query_model.__name__} for unique fields {unique_fields}")
|
|
230
|
+
|
|
231
|
+
existing_record_ids, existing_pks_map = self.record_classifier.classify_for_upsert(objs, unique_fields, query_model=query_model)
|
|
232
|
+
|
|
233
|
+
logger.info(f"MTI classification: {len(existing_record_ids)} existing, {len(objs) - len(existing_record_ids)} new")
|
|
234
|
+
|
|
235
|
+
return existing_record_ids, existing_pks_map
|
|
236
|
+
|
|
237
|
+
def _execute_standard_bulk_create(
|
|
238
|
+
self,
|
|
239
|
+
objs: List[Model],
|
|
240
|
+
batch_size: Optional[int],
|
|
241
|
+
ignore_conflicts: bool,
|
|
242
|
+
update_conflicts: bool,
|
|
243
|
+
update_fields: Optional[List[str]],
|
|
244
|
+
unique_fields: Optional[List[str]],
|
|
245
|
+
**kwargs: Any,
|
|
246
|
+
) -> List[Model]:
|
|
247
|
+
"""Execute Django's native bulk_create for non-MTI models."""
|
|
248
|
+
base_qs = self._get_base_queryset()
|
|
249
|
+
|
|
250
|
+
return base_qs.bulk_create(
|
|
251
|
+
objs,
|
|
252
|
+
batch_size=batch_size,
|
|
253
|
+
ignore_conflicts=ignore_conflicts,
|
|
254
|
+
update_conflicts=update_conflicts,
|
|
255
|
+
update_fields=update_fields,
|
|
256
|
+
unique_fields=unique_fields,
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
def _handle_upsert_metadata_tagging(
|
|
260
|
+
self,
|
|
261
|
+
result_objects: List[Model],
|
|
262
|
+
objs: List[Model],
|
|
263
|
+
update_conflicts: bool,
|
|
264
|
+
unique_fields: Optional[List[str]],
|
|
265
|
+
existing_record_ids: Optional[Set[int]],
|
|
266
|
+
existing_pks_map: Optional[Dict[int, int]],
|
|
267
|
+
) -> None:
|
|
268
|
+
"""
|
|
269
|
+
Tag upsert metadata on result objects.
|
|
270
|
+
|
|
271
|
+
Centralizes metadata tagging logic for both MTI and non-MTI paths.
|
|
272
|
+
|
|
273
|
+
Args:
|
|
274
|
+
result_objects: Objects returned from bulk operation
|
|
275
|
+
objs: Original objects passed to bulk_create
|
|
276
|
+
update_conflicts: Whether this was an upsert operation
|
|
277
|
+
unique_fields: Fields used for conflict detection
|
|
278
|
+
existing_record_ids: Pre-classified existing record IDs
|
|
279
|
+
existing_pks_map: Pre-classified existing PK mapping
|
|
280
|
+
"""
|
|
281
|
+
if not (update_conflicts and unique_fields):
|
|
282
|
+
return
|
|
283
|
+
|
|
284
|
+
# Classify if needed
|
|
285
|
+
if existing_record_ids is None or existing_pks_map is None:
|
|
286
|
+
existing_record_ids, existing_pks_map = self.record_classifier.classify_for_upsert(objs, unique_fields)
|
|
287
|
+
|
|
288
|
+
tag_upsert_metadata(result_objects, existing_record_ids, existing_pks_map)
|
|
289
|
+
|
|
290
|
+
# ==================== Private: Update Helpers ====================
|
|
291
|
+
|
|
292
|
+
def _add_auto_now_fields(self, fields: List[str], objs: List[Model]) -> List[str]:
|
|
293
|
+
"""
|
|
294
|
+
Add auto_now fields to update list for all models in chain.
|
|
295
|
+
|
|
296
|
+
Handles both MTI and non-MTI models uniformly.
|
|
297
|
+
|
|
298
|
+
Args:
|
|
299
|
+
fields: Original field list
|
|
300
|
+
objs: Objects being updated
|
|
301
|
+
|
|
302
|
+
Returns:
|
|
303
|
+
Field list with auto_now fields included
|
|
304
|
+
"""
|
|
305
|
+
fields = list(fields) # Copy to avoid mutation
|
|
306
|
+
|
|
307
|
+
# Get models to check
|
|
308
|
+
if self.mti_handler.is_mti_model():
|
|
309
|
+
models_to_check = self.mti_handler.get_inheritance_chain()
|
|
310
|
+
else:
|
|
311
|
+
models_to_check = [self.model_cls]
|
|
312
|
+
|
|
313
|
+
# Handle auto_now fields uniformly
|
|
314
|
+
auto_now_fields = handle_auto_now_fields_for_inheritance_chain(models_to_check, objs, for_update=True)
|
|
315
|
+
|
|
316
|
+
# Add to fields list if not present
|
|
317
|
+
for auto_now_field in auto_now_fields:
|
|
318
|
+
if auto_now_field not in fields:
|
|
319
|
+
fields.append(auto_now_field)
|
|
320
|
+
|
|
321
|
+
return fields
|
|
322
|
+
|
|
323
|
+
# ==================== Private: MTI Create Execution ====================
|
|
324
|
+
|
|
325
|
+
def _execute_mti_create_plan(self, plan: Any) -> List[Model]:
|
|
326
|
+
"""
|
|
327
|
+
Execute an MTI create plan.
|
|
328
|
+
|
|
329
|
+
Handles INSERT and UPDATE for upsert operations.
|
|
330
|
+
|
|
331
|
+
Args:
|
|
332
|
+
plan: MTICreatePlan from MTIHandler
|
|
333
|
+
|
|
334
|
+
Returns:
|
|
335
|
+
List of created/updated objects with PKs assigned
|
|
336
|
+
"""
|
|
337
|
+
if not plan:
|
|
338
|
+
return []
|
|
339
|
+
|
|
340
|
+
with transaction.atomic(using=self.queryset.db, savepoint=False):
|
|
341
|
+
# Step 1: Upsert all parent levels
|
|
342
|
+
parent_instances_map = self._upsert_parent_levels(plan)
|
|
343
|
+
|
|
344
|
+
# Step 2: Link children to parents
|
|
345
|
+
self._link_children_to_parents(plan, parent_instances_map)
|
|
346
|
+
|
|
347
|
+
# Step 3: Handle child objects (insert new, update existing)
|
|
348
|
+
self._handle_child_objects(plan)
|
|
349
|
+
|
|
350
|
+
# Step 4: Copy PKs and auto-fields back to original objects
|
|
351
|
+
self._copy_fields_to_original_objects(plan, parent_instances_map)
|
|
352
|
+
|
|
353
|
+
return plan.original_objects
|
|
354
|
+
|
|
355
|
+
def _upsert_parent_levels(self, plan: Any) -> Dict[int, Dict[type, Model]]:
|
|
356
|
+
"""
|
|
357
|
+
Upsert all parent objects level by level.
|
|
358
|
+
|
|
359
|
+
Returns:
|
|
360
|
+
Mapping of original obj id() -> {model: parent_instance}
|
|
361
|
+
"""
|
|
362
|
+
parent_instances_map: Dict[int, Dict[type, Model]] = {}
|
|
363
|
+
|
|
364
|
+
for parent_level in plan.parent_levels:
|
|
365
|
+
base_qs = QuerySet(model=parent_level.model_class, using=self.queryset.db)
|
|
366
|
+
|
|
367
|
+
# Build bulk_create kwargs
|
|
368
|
+
bulk_kwargs = {"batch_size": len(parent_level.objects)}
|
|
369
|
+
|
|
370
|
+
if parent_level.update_conflicts:
|
|
371
|
+
self._add_upsert_kwargs(bulk_kwargs, parent_level)
|
|
372
|
+
|
|
373
|
+
# Execute upsert
|
|
374
|
+
upserted_parents = base_qs.bulk_create(parent_level.objects, **bulk_kwargs)
|
|
375
|
+
|
|
376
|
+
# Copy generated fields back
|
|
377
|
+
self._copy_generated_fields(upserted_parents, parent_level.objects, parent_level.model_class)
|
|
378
|
+
|
|
379
|
+
# Map parents to original objects
|
|
380
|
+
self._map_parents_to_originals(parent_level, parent_instances_map)
|
|
381
|
+
|
|
382
|
+
return parent_instances_map
|
|
383
|
+
|
|
384
|
+
def _add_upsert_kwargs(self, bulk_kwargs: Dict[str, Any], parent_level: Any) -> None:
|
|
385
|
+
"""Add upsert parameters to bulk_create kwargs."""
|
|
386
|
+
bulk_kwargs["update_conflicts"] = True
|
|
387
|
+
bulk_kwargs["unique_fields"] = parent_level.unique_fields
|
|
388
|
+
|
|
389
|
+
# Filter update fields
|
|
390
|
+
parent_model_fields = {field.name for field in parent_level.model_class._meta.local_fields}
|
|
391
|
+
filtered_update_fields = [field for field in parent_level.update_fields if field in parent_model_fields]
|
|
392
|
+
|
|
393
|
+
if filtered_update_fields:
|
|
394
|
+
bulk_kwargs["update_fields"] = filtered_update_fields
|
|
395
|
+
|
|
396
|
+
def _copy_generated_fields(
|
|
397
|
+
self,
|
|
398
|
+
upserted_parents: List[Model],
|
|
399
|
+
parent_objs: List[Model],
|
|
400
|
+
model_class: type[Model],
|
|
401
|
+
) -> None:
|
|
402
|
+
"""Copy generated fields from upserted objects back to parent objects."""
|
|
403
|
+
for upserted_parent, parent_obj in zip(upserted_parents, parent_objs):
|
|
404
|
+
for field in model_class._meta.local_fields:
|
|
405
|
+
# Use attname for FK fields to avoid queries
|
|
406
|
+
field_attr = field.attname if isinstance(field, ForeignKey) else field.name
|
|
407
|
+
upserted_value = getattr(upserted_parent, field_attr, None)
|
|
408
|
+
if upserted_value is not None:
|
|
409
|
+
setattr(parent_obj, field_attr, upserted_value)
|
|
410
|
+
|
|
411
|
+
parent_obj._state.adding = False
|
|
412
|
+
parent_obj._state.db = self.queryset.db
|
|
413
|
+
|
|
414
|
+
def _map_parents_to_originals(self, parent_level: Any, parent_instances_map: Dict[int, Dict[type, Model]]) -> None:
|
|
415
|
+
"""Map parent instances back to original objects."""
|
|
416
|
+
for parent_obj in parent_level.objects:
|
|
417
|
+
orig_obj_id = parent_level.original_object_map[id(parent_obj)]
|
|
418
|
+
if orig_obj_id not in parent_instances_map:
|
|
419
|
+
parent_instances_map[orig_obj_id] = {}
|
|
420
|
+
parent_instances_map[orig_obj_id][parent_level.model_class] = parent_obj
|
|
421
|
+
|
|
422
|
+
def _link_children_to_parents(self, plan: Any, parent_instances_map: Dict[int, Dict[type, Model]]) -> None:
|
|
423
|
+
"""Link child objects to their parent objects and set PKs."""
|
|
424
|
+
for child_obj, orig_obj in zip(plan.child_objects, plan.original_objects):
|
|
425
|
+
parent_instances = parent_instances_map.get(id(orig_obj), {})
|
|
426
|
+
|
|
427
|
+
for parent_model, parent_instance in parent_instances.items():
|
|
428
|
+
parent_link = plan.child_model._meta.get_ancestor_link(parent_model)
|
|
429
|
+
|
|
430
|
+
if parent_link:
|
|
431
|
+
parent_pk = parent_instance.pk
|
|
432
|
+
setattr(child_obj, parent_link.attname, parent_pk)
|
|
433
|
+
setattr(child_obj, parent_link.name, parent_instance)
|
|
434
|
+
# In MTI, child PK equals parent PK
|
|
435
|
+
child_obj.pk = parent_pk
|
|
436
|
+
child_obj.id = parent_pk
|
|
437
|
+
else:
|
|
438
|
+
logger.warning(f"No parent link found for {parent_model} in {plan.child_model}")
|
|
439
|
+
|
|
440
|
+
def _handle_child_objects(self, plan: Any) -> None:
|
|
441
|
+
"""Handle child object insertion and updates."""
|
|
442
|
+
base_qs = QuerySet(model=plan.child_model, using=self.queryset.db)
|
|
443
|
+
|
|
444
|
+
# Split objects: new vs existing
|
|
445
|
+
objs_without_pk, objs_with_pk = self._split_child_objects(plan, base_qs)
|
|
446
|
+
|
|
447
|
+
# Update existing children
|
|
448
|
+
if objs_with_pk and plan.update_fields:
|
|
449
|
+
self._update_existing_children(base_qs, objs_with_pk, plan)
|
|
450
|
+
|
|
451
|
+
# Insert new children
|
|
452
|
+
if objs_without_pk:
|
|
453
|
+
self._insert_new_children(base_qs, objs_without_pk, plan)
|
|
454
|
+
|
|
455
|
+
def _split_child_objects(self, plan: Any, base_qs: QuerySet) -> Tuple[List[Model], List[Model]]:
|
|
456
|
+
"""Split child objects into new and existing."""
|
|
457
|
+
if not plan.update_conflicts:
|
|
458
|
+
return plan.child_objects, []
|
|
459
|
+
|
|
460
|
+
# Check which child records exist
|
|
461
|
+
parent_pks = [
|
|
462
|
+
getattr(child_obj, plan.child_model._meta.pk.attname, None)
|
|
463
|
+
for child_obj in plan.child_objects
|
|
464
|
+
if getattr(child_obj, plan.child_model._meta.pk.attname, None)
|
|
465
|
+
]
|
|
466
|
+
|
|
467
|
+
existing_child_pks = set()
|
|
468
|
+
if parent_pks:
|
|
469
|
+
existing_child_pks = set(base_qs.filter(pk__in=parent_pks).values_list("pk", flat=True))
|
|
470
|
+
|
|
471
|
+
objs_without_pk = []
|
|
472
|
+
objs_with_pk = []
|
|
473
|
+
|
|
474
|
+
for child_obj in plan.child_objects:
|
|
475
|
+
child_pk = getattr(child_obj, plan.child_model._meta.pk.attname, None)
|
|
476
|
+
if child_pk and child_pk in existing_child_pks:
|
|
477
|
+
objs_with_pk.append(child_obj)
|
|
478
|
+
else:
|
|
479
|
+
objs_without_pk.append(child_obj)
|
|
480
|
+
|
|
481
|
+
return objs_without_pk, objs_with_pk
|
|
482
|
+
|
|
483
|
+
def _update_existing_children(self, base_qs: QuerySet, objs_with_pk: List[Model], plan: Any) -> None:
|
|
484
|
+
"""Update existing child records."""
|
|
485
|
+
child_model_fields = {field.name for field in plan.child_model._meta.local_fields}
|
|
486
|
+
filtered_child_update_fields = [field for field in plan.update_fields if field in child_model_fields]
|
|
487
|
+
|
|
488
|
+
if filtered_child_update_fields:
|
|
489
|
+
base_qs.bulk_update(objs_with_pk, filtered_child_update_fields)
|
|
490
|
+
|
|
491
|
+
for obj in objs_with_pk:
|
|
492
|
+
obj._state.adding = False
|
|
493
|
+
obj._state.db = self.queryset.db
|
|
494
|
+
|
|
495
|
+
def _insert_new_children(self, base_qs: QuerySet, objs_without_pk: List[Model], plan: Any) -> None:
|
|
496
|
+
"""Insert new child records using _batched_insert."""
|
|
497
|
+
base_qs._prepare_for_bulk_create(objs_without_pk)
|
|
498
|
+
opts = plan.child_model._meta
|
|
499
|
+
|
|
500
|
+
# Get fields for insertion
|
|
501
|
+
filtered_fields = [f for f in opts.local_fields if not f.generated]
|
|
502
|
+
|
|
503
|
+
# Build upsert kwargs
|
|
504
|
+
kwargs = self._build_batched_insert_kwargs(plan, len(objs_without_pk))
|
|
505
|
+
|
|
506
|
+
# Execute insert
|
|
507
|
+
returned_columns = base_qs._batched_insert(objs_without_pk, filtered_fields, **kwargs)
|
|
508
|
+
|
|
509
|
+
# Process returned columns
|
|
510
|
+
self._process_returned_columns(objs_without_pk, returned_columns, opts)
|
|
511
|
+
|
|
512
|
+
def _build_batched_insert_kwargs(self, plan: Any, batch_size: int) -> Dict[str, Any]:
|
|
513
|
+
"""Build kwargs for _batched_insert call."""
|
|
514
|
+
kwargs = {"batch_size": batch_size}
|
|
515
|
+
|
|
516
|
+
if not (plan.update_conflicts and plan.child_unique_fields):
|
|
517
|
+
return kwargs
|
|
518
|
+
|
|
519
|
+
batched_unique_fields = plan.child_unique_fields
|
|
520
|
+
batched_update_fields = plan.child_update_fields
|
|
521
|
+
|
|
522
|
+
if batched_update_fields:
|
|
523
|
+
on_conflict = OnConflict.UPDATE
|
|
524
|
+
else:
|
|
525
|
+
# No update fields on child - use IGNORE
|
|
526
|
+
on_conflict = OnConflict.IGNORE
|
|
527
|
+
batched_update_fields = None
|
|
528
|
+
|
|
529
|
+
kwargs.update(
|
|
530
|
+
{
|
|
531
|
+
"on_conflict": on_conflict,
|
|
532
|
+
"update_fields": batched_update_fields,
|
|
533
|
+
"unique_fields": batched_unique_fields,
|
|
534
|
+
}
|
|
535
|
+
)
|
|
536
|
+
|
|
537
|
+
return kwargs
|
|
538
|
+
|
|
539
|
+
def _process_returned_columns(self, objs: List[Model], returned_columns: Any, opts: Any) -> None:
|
|
540
|
+
"""Process returned columns from _batched_insert."""
|
|
541
|
+
if returned_columns:
|
|
542
|
+
for obj, results in zip(objs, returned_columns):
|
|
543
|
+
if hasattr(opts, "db_returning_fields"):
|
|
544
|
+
for result, field in zip(results, opts.db_returning_fields):
|
|
545
|
+
setattr(obj, field.attname, result)
|
|
546
|
+
obj._state.adding = False
|
|
547
|
+
obj._state.db = self.queryset.db
|
|
548
|
+
else:
|
|
549
|
+
for obj in objs:
|
|
550
|
+
obj._state.adding = False
|
|
551
|
+
obj._state.db = self.queryset.db
|
|
552
|
+
|
|
553
|
+
def _copy_fields_to_original_objects(self, plan: Any, parent_instances_map: Dict[int, Dict[type, Model]]) -> None:
|
|
554
|
+
"""Copy PKs and auto-generated fields to original objects."""
|
|
555
|
+
pk_field_name = plan.child_model._meta.pk.name
|
|
556
|
+
|
|
557
|
+
for orig_obj, child_obj in zip(plan.original_objects, plan.child_objects):
|
|
558
|
+
# Copy PK
|
|
559
|
+
child_pk = getattr(child_obj, pk_field_name)
|
|
560
|
+
setattr(orig_obj, pk_field_name, child_pk)
|
|
561
|
+
|
|
562
|
+
# Copy auto-generated fields from all levels
|
|
563
|
+
self._copy_auto_generated_fields(orig_obj, child_obj, plan, parent_instances_map, pk_field_name)
|
|
564
|
+
|
|
565
|
+
# Update state
|
|
566
|
+
orig_obj._state.adding = False
|
|
567
|
+
orig_obj._state.db = self.queryset.db
|
|
568
|
+
|
|
569
|
+
def _copy_auto_generated_fields(
|
|
570
|
+
self,
|
|
571
|
+
orig_obj: Model,
|
|
572
|
+
child_obj: Model,
|
|
573
|
+
plan: Any,
|
|
574
|
+
parent_instances_map: Dict[int, Dict[type, Model]],
|
|
575
|
+
pk_field_name: str,
|
|
576
|
+
) -> None:
|
|
577
|
+
"""Copy auto-generated fields from all inheritance levels."""
|
|
578
|
+
parent_instances = parent_instances_map.get(id(orig_obj), {})
|
|
579
|
+
|
|
580
|
+
for model_class in plan.inheritance_chain:
|
|
581
|
+
# Get source object
|
|
582
|
+
if model_class in parent_instances:
|
|
583
|
+
source_obj = parent_instances[model_class]
|
|
584
|
+
elif model_class == plan.child_model:
|
|
585
|
+
source_obj = child_obj
|
|
586
|
+
else:
|
|
587
|
+
continue
|
|
588
|
+
|
|
589
|
+
# Copy auto-generated fields
|
|
590
|
+
for field in model_class._meta.local_fields:
|
|
591
|
+
if field.name == pk_field_name:
|
|
592
|
+
continue
|
|
593
|
+
|
|
594
|
+
# Skip parent link fields
|
|
595
|
+
if self._is_parent_link_field(field, plan.child_model, model_class):
|
|
596
|
+
continue
|
|
597
|
+
|
|
598
|
+
# Copy auto_now, auto_now_add, and db_returning fields
|
|
599
|
+
if self._is_auto_generated_field(field):
|
|
600
|
+
source_value = getattr(source_obj, field.name, None)
|
|
601
|
+
if source_value is not None:
|
|
602
|
+
setattr(orig_obj, field.name, source_value)
|
|
603
|
+
|
|
604
|
+
def _is_parent_link_field(self, field: Any, child_model: type[Model], model_class: type[Model]) -> bool:
|
|
605
|
+
"""Check if field is a parent link field."""
|
|
606
|
+
if not (hasattr(field, "remote_field") and field.remote_field):
|
|
607
|
+
return False
|
|
608
|
+
|
|
609
|
+
parent_link = child_model._meta.get_ancestor_link(model_class)
|
|
610
|
+
return parent_link and field.name == parent_link.name
|
|
611
|
+
|
|
612
|
+
def _is_auto_generated_field(self, field: Any) -> bool:
|
|
613
|
+
"""Check if field is auto-generated."""
|
|
614
|
+
return getattr(field, "auto_now_add", False) or getattr(field, "auto_now", False) or getattr(field, "db_returning", False)
|
|
615
|
+
|
|
616
|
+
# ==================== Private: MTI Update Execution ====================
|
|
617
|
+
|
|
618
|
+
def _execute_mti_update_plan(self, plan: Any) -> int:
|
|
619
|
+
"""
|
|
620
|
+
Execute an MTI update plan.
|
|
621
|
+
|
|
622
|
+
Updates each table in the inheritance chain using CASE/WHEN.
|
|
623
|
+
|
|
624
|
+
Args:
|
|
625
|
+
plan: MTIUpdatePlan from MTIHandler
|
|
626
|
+
|
|
627
|
+
Returns:
|
|
628
|
+
Number of objects updated
|
|
629
|
+
"""
|
|
630
|
+
if not plan:
|
|
631
|
+
return 0
|
|
632
|
+
|
|
633
|
+
root_pks = self._get_root_pks(plan.objects)
|
|
634
|
+
if not root_pks:
|
|
635
|
+
return 0
|
|
636
|
+
|
|
637
|
+
total_updated = 0
|
|
638
|
+
|
|
639
|
+
with transaction.atomic(using=self.queryset.db, savepoint=False):
|
|
640
|
+
for field_group in plan.field_groups:
|
|
641
|
+
if not field_group.fields:
|
|
642
|
+
continue
|
|
643
|
+
|
|
644
|
+
updated_count = self._update_field_group(field_group, root_pks, plan.objects)
|
|
645
|
+
total_updated += updated_count
|
|
646
|
+
|
|
647
|
+
return total_updated
|
|
648
|
+
|
|
649
|
+
def _get_root_pks(self, objs: List[Model]) -> List[Any]:
|
|
650
|
+
"""Extract primary keys from objects."""
|
|
651
|
+
return [
|
|
652
|
+
getattr(obj, "pk", None) or getattr(obj, "id", None) for obj in objs if getattr(obj, "pk", None) or getattr(obj, "id", None)
|
|
653
|
+
]
|
|
654
|
+
|
|
655
|
+
def _update_field_group(self, field_group: Any, root_pks: List[Any], objs: List[Model]) -> int:
|
|
656
|
+
"""Update a single field group."""
|
|
657
|
+
base_qs = QuerySet(model=field_group.model_class, using=self.queryset.db)
|
|
658
|
+
|
|
659
|
+
# Check if records exist
|
|
660
|
+
if not self._check_records_exist(base_qs, field_group, root_pks):
|
|
661
|
+
return 0
|
|
662
|
+
|
|
663
|
+
# Build CASE statements
|
|
664
|
+
case_statements = self._build_case_statements(field_group, root_pks, objs)
|
|
665
|
+
|
|
666
|
+
if not case_statements:
|
|
667
|
+
logger.debug(f"No CASE statements for {field_group.model_class.__name__}")
|
|
668
|
+
return 0
|
|
669
|
+
|
|
670
|
+
# Execute update
|
|
671
|
+
return self._execute_field_group_update(base_qs, field_group, root_pks, case_statements)
|
|
672
|
+
|
|
673
|
+
def _check_records_exist(self, base_qs: QuerySet, field_group: Any, root_pks: List[Any]) -> bool:
|
|
674
|
+
"""Check if any records exist for update."""
|
|
675
|
+
existing_count = base_qs.filter(**{f"{field_group.filter_field}__in": root_pks}).count()
|
|
676
|
+
return existing_count > 0
|
|
677
|
+
|
|
678
|
+
def _build_case_statements(self, field_group: Any, root_pks: List[Any], objs: List[Model]) -> Dict[str, Case]:
|
|
679
|
+
"""Build CASE statements for all fields in the group."""
|
|
680
|
+
case_statements = {}
|
|
681
|
+
|
|
682
|
+
logger.debug(f"Building CASE statements for {field_group.model_class.__name__} with {len(field_group.fields)} fields")
|
|
683
|
+
|
|
684
|
+
# Debug: Check if business_id is still in __dict__ before field extraction
|
|
685
|
+
for obj in objs:
|
|
686
|
+
if 'business_id' in obj.__dict__ or 'business' in field_group.fields:
|
|
687
|
+
logger.debug("🏗️ CASE_BUILD_START: obj.pk=%s, business_id in __dict__=%s, value=%s",
|
|
688
|
+
getattr(obj, 'pk', 'None'),
|
|
689
|
+
'business_id' in obj.__dict__,
|
|
690
|
+
obj.__dict__.get('business_id', 'NOT_IN_DICT'))
|
|
691
|
+
|
|
692
|
+
for field_name in field_group.fields:
|
|
693
|
+
case_stmt = self._build_field_case_statement(field_name, field_group, root_pks, objs)
|
|
694
|
+
if case_stmt:
|
|
695
|
+
case_statements[field_name] = case_stmt
|
|
696
|
+
|
|
697
|
+
return case_statements
|
|
698
|
+
|
|
699
|
+
def _build_field_case_statement(
|
|
700
|
+
self,
|
|
701
|
+
field_name: str,
|
|
702
|
+
field_group: Any,
|
|
703
|
+
root_pks: List[Any],
|
|
704
|
+
objs: List[Model],
|
|
705
|
+
) -> Optional[Case]:
|
|
706
|
+
"""Build CASE statement for a single field."""
|
|
707
|
+
field = field_group.model_class._meta.get_field(field_name)
|
|
708
|
+
when_statements = []
|
|
709
|
+
|
|
710
|
+
for pk, obj in zip(root_pks, objs):
|
|
711
|
+
obj_pk = getattr(obj, "pk", None) or getattr(obj, "id", None)
|
|
712
|
+
if obj_pk is None:
|
|
713
|
+
continue
|
|
714
|
+
|
|
715
|
+
# Get and convert field value
|
|
716
|
+
value = get_field_value_for_db(obj, field_name, field_group.model_class)
|
|
717
|
+
value = field.to_python(value)
|
|
718
|
+
|
|
719
|
+
# Create WHEN with type casting
|
|
720
|
+
when_statement = When(
|
|
721
|
+
**{field_group.filter_field: pk},
|
|
722
|
+
then=Cast(Value(value), output_field=field),
|
|
723
|
+
)
|
|
724
|
+
when_statements.append(when_statement)
|
|
725
|
+
|
|
726
|
+
if when_statements:
|
|
727
|
+
return Case(*when_statements, output_field=field)
|
|
728
|
+
|
|
729
|
+
return None
|
|
730
|
+
|
|
731
|
+
def _execute_field_group_update(
|
|
732
|
+
self,
|
|
733
|
+
base_qs: QuerySet,
|
|
734
|
+
field_group: Any,
|
|
735
|
+
root_pks: List[Any],
|
|
736
|
+
case_statements: Dict[str, Case],
|
|
737
|
+
) -> int:
|
|
738
|
+
"""Execute the actual update query."""
|
|
739
|
+
logger.debug(f"Executing update for {field_group.model_class.__name__} with {len(case_statements)} fields")
|
|
740
|
+
|
|
741
|
+
try:
|
|
742
|
+
query_qs = base_qs.filter(**{f"{field_group.filter_field}__in": root_pks})
|
|
743
|
+
updated_count = query_qs.update(**case_statements)
|
|
744
|
+
|
|
745
|
+
logger.debug(f"Updated {updated_count} records in {field_group.model_class.__name__}")
|
|
746
|
+
|
|
747
|
+
return updated_count
|
|
748
|
+
|
|
749
|
+
except Exception as e:
|
|
750
|
+
logger.error(f"MTI bulk update failed for {field_group.model_class.__name__}: {e}")
|
|
751
|
+
raise
|
|
752
|
+
|
|
753
|
+
# ==================== Private: Utilities ====================
|
|
754
|
+
|
|
755
|
+
def _get_base_queryset(self) -> QuerySet:
|
|
756
|
+
"""Get base Django QuerySet to avoid recursion."""
|
|
757
|
+
return QuerySet(model=self.model_cls, using=self.queryset.db)
|