django-bulk-hooks 0.2.60__py3-none-any.whl → 0.2.61__py3-none-any.whl

django_bulk_hooks/operations/bulk_executor.py

@@ -11,7 +11,12 @@ from django.db.models import AutoField, ForeignKey, Case, When, Value
 from django.db.models.constants import OnConflict
 from django.db.models.functions import Cast
 
-from django_bulk_hooks.operations.field_utils import get_field_value_for_db
+from django_bulk_hooks.operations.field_utils import (
+    get_field_value_for_db,
+    collect_auto_now_fields_for_inheritance_chain,
+    pre_save_auto_now_fields,
+)
+from django_bulk_hooks.helpers import tag_upsert_metadata
 
 logger = logging.getLogger(__name__)
 
@@ -42,6 +47,57 @@ class BulkExecutor:
         self.record_classifier = record_classifier
         self.model_cls = queryset.model
 
+    def _handle_upsert_metadata_tagging(self, result_objects, objs, update_conflicts, unique_fields, existing_record_ids=None, existing_pks_map=None):
+
+        """
+
+        Handle classification and metadata tagging for upsert operations.
+
+
+
+        This centralizes the logic that was duplicated between MTI and non-MTI paths.
+
+
+
+        Args:
+
+            result_objects: List of objects returned from the bulk operation
+
+            objs: Original list of objects passed to bulk_create
+
+            update_conflicts: Whether this was an upsert operation
+
+            unique_fields: Fields used for conflict detection
+
+            existing_record_ids: Pre-classified existing record IDs (optional)
+
+            existing_pks_map: Pre-classified existing PK mapping (optional)
+
+
+
+        Returns:
+
+            None - modifies result_objects in place with metadata
+
+        """
+
+        if not (update_conflicts and unique_fields):
+
+            return
+
+
+
+        # Classify records if not already done
+
+        if existing_record_ids is None or existing_pks_map is None:
+
+            existing_record_ids, existing_pks_map = self.record_classifier.classify_for_upsert(objs, unique_fields)
+
+
+
+        # Tag the metadata
+        tag_upsert_metadata(result_objects, existing_record_ids, existing_pks_map)
+
     def bulk_create(
         self,
         objs,
@@ -77,7 +133,6 @@ class BulkExecutor:
 
         # Check if this is an MTI model and route accordingly
         if self.mti_handler.is_mti_model():
-
             # Use pre-classified records if provided, otherwise classify now
             if existing_record_ids is None or existing_pks_map is None:
                 existing_record_ids = set()
@@ -108,30 +163,24 @@ class BulkExecutor:
             # Execute the plan
             result = self._execute_mti_create_plan(plan)
 
-            # Tag objects with upsert metadata for hook dispatching
-            if update_conflicts and unique_fields:
-                self._tag_upsert_metadata(result, existing_record_ids, existing_pks_map)
-
-            return result
+        else:
+            # Non-MTI model - use Django's native bulk_create
+            result = self._execute_bulk_create(
+                objs,
+                batch_size,
+                ignore_conflicts,
+                update_conflicts,
+                update_fields,
+                unique_fields,
+                **kwargs,
+            )
 
-        # Non-MTI model - use Django's native bulk_create
-        result = self._execute_bulk_create(
-            objs,
-            batch_size,
-            ignore_conflicts,
-            update_conflicts,
-            update_fields,
-            unique_fields,
-            **kwargs,
+        # Unified upsert metadata handling for both paths
+        self._handle_upsert_metadata_tagging(
+            result, objs, update_conflicts, unique_fields, 
+            existing_record_ids, existing_pks_map
         )
 
-        # Tag objects with upsert metadata for hook dispatching
-        if update_conflicts and unique_fields:
-            # Use pre-classified results if available, otherwise classify now
-            if existing_record_ids is None:
-                existing_record_ids, existing_pks_map = self.record_classifier.classify_for_upsert(objs, unique_fields)
-            self._tag_upsert_metadata(result, existing_record_ids, existing_pks_map)
-
         return result
 
     def _execute_bulk_create(
@@ -185,23 +234,19 @@ class BulkExecutor:
         # Ensure auto_now fields are included and pre-saved for all models
         # This handles both MTI and non-MTI models uniformly (SOC & DRY)
         fields = list(fields)  # Make a copy so we can modify it
-        
+
         # Get models to check - for MTI, check entire inheritance chain
         if self.mti_handler.is_mti_model():
             models_to_check = self.mti_handler.get_inheritance_chain()
         else:
             models_to_check = [self.model_cls]
-        
-        # Collect all auto_now fields and pre-save them
-        auto_now_fields = set()
-        for model in models_to_check:
-            for field in model._meta.local_fields:
-                if getattr(field, "auto_now", False) and not getattr(field, "auto_now_add", False):
-                    auto_now_fields.add(field.name)
-                    # Pre-save the field to set the value on instances
-                    for obj in objs:
-                        field.pre_save(obj, add=False)
-        
+
+        # Use unified auto-now field handling
+        from django_bulk_hooks.operations.field_utils import handle_auto_now_fields_for_inheritance_chain
+        auto_now_fields = handle_auto_now_fields_for_inheritance_chain(
+            models_to_check, objs, for_update=True
+        )
+
         # Add auto_now fields to the update list if not already present
         for auto_now_field in auto_now_fields:
             if auto_now_field not in fields:
@@ -364,59 +409,41 @@ class BulkExecutor:
                 # For MTI, we need to include the parent link (which is the PK)
                 filtered_fields = [f for f in opts.local_fields if not f.generated]
 
-                # FIX: Pass conflict resolution parameters to _batched_insert for MTI child tables
-                # Previously, _batched_insert was called without on_conflict/unique_fields/update_fields,
-                # causing IntegrityError when child tables have unique constraints during upsert operations.
-                # See: https://github.com/user/repo/issues/XXX
-                # Prepare conflict resolution parameters for upsert
+                # Prepare conflict resolution parameters for upsert using pre-computed fields
                 on_conflict = None
                 batched_unique_fields = None
                 batched_update_fields = None
 
-                if plan.update_conflicts:
-                    # Filter unique_fields and update_fields to only those on child model
-                    # Django's _batched_insert expects field objects, not field names
-                    child_model_fields_dict = {field.name: field for field in plan.child_model._meta.local_fields}
-
-                    # Unique fields may be on parent or child - filter to child only for child table insert
-                    # Convert field names to field objects
-                    if plan.unique_fields:
-                        batched_unique_fields = [
-                            child_model_fields_dict[fname]
-                            for fname in plan.unique_fields
-                            if fname in child_model_fields_dict
-                        ]
-
-                    # Update fields - filter to child only
-                    # Keep as strings - Django's _batched_insert accepts field name strings for update_fields
-                    if plan.update_fields:
-                        batched_update_fields = [
-                            fname
-                            for fname in plan.update_fields
-                            if fname in child_model_fields_dict
-                        ]
-
-                    # Only set on_conflict if we have unique fields for this table
-                    # Note: If unique_fields are all on parent, batched_unique_fields will be empty,
-                    # meaning no conflict resolution needed for child table
-                    if batched_unique_fields:
-                        if batched_update_fields:
-                            # We have both unique fields and update fields on child - use UPDATE
-                            on_conflict = OnConflict.UPDATE
-                        else:
-                            # We have unique fields on child but no update fields - use IGNORE
-                            # This handles the case where all update fields are on parent tables
-                            on_conflict = OnConflict.IGNORE
-                            # Clear batched_update_fields to avoid issues
-                            batched_update_fields = None
+                # Only set up upsert logic if we have child-specific unique fields
+                if plan.update_conflicts and plan.child_unique_fields:
+                    batched_unique_fields = plan.child_unique_fields
+                    batched_update_fields = plan.child_update_fields
+
+                    if batched_update_fields:
+                        # We have both unique fields and update fields on child - use UPDATE
+                        on_conflict = OnConflict.UPDATE
+                    else:
+                        # We have unique fields on child but no update fields - use IGNORE
+                        # This handles the case where all update fields are on parent tables
+                        on_conflict = OnConflict.IGNORE
+                        batched_update_fields = None
+
+                # Build kwargs for _batched_insert call
+                kwargs = {
+                    'batch_size': len(objs_without_pk),
+                }
+                # Only pass conflict resolution parameters if we have unique fields for this table
+                if batched_unique_fields:
+                    kwargs.update({
+                        'on_conflict': on_conflict,
+                        'update_fields': batched_update_fields,
+                        'unique_fields': batched_unique_fields,
+                    })
 
                 returned_columns = base_qs._batched_insert(
                     objs_without_pk,
                     filtered_fields,
-                    batch_size=len(objs_without_pk),
-                    on_conflict=on_conflict,
-                    update_fields=batched_update_fields,
-                    unique_fields=batched_unique_fields,
+                    **kwargs,
                 )
                 if returned_columns:
                     for obj, results in zip(objs_without_pk, returned_columns):
@@ -597,38 +624,3 @@ class BulkExecutor:
 
         return QuerySet.delete(self.queryset)
 
-    def _tag_upsert_metadata(self, result_objects, existing_record_ids, existing_pks_map):
-        """
-        Tag objects with metadata indicating whether they were created or updated.
-
-        This metadata is used by the coordinator to determine which hooks to fire.
-        The metadata is temporary and will be cleaned up after hook execution.
-
-        Args:
-            result_objects: List of objects returned from bulk operation
-            existing_record_ids: Set of id() for objects that existed before the operation
-            existing_pks_map: Dict mapping id(obj) -> pk for existing records
-        """
-        created_count = 0
-        updated_count = 0
-
-        # Create a set of PKs that existed before the operation
-        existing_pks = set(existing_pks_map.values())
-
-        for obj in result_objects:
-            # Use PK to determine if this record was created or updated
-            # If the PK was in the existing_pks_map, it was updated; otherwise created
-            was_created = obj.pk not in existing_pks
-            obj._bulk_hooks_was_created = was_created
-            obj._bulk_hooks_upsert_metadata = True
-
-            if was_created:
-                created_count += 1
-            else:
-                updated_count += 1
-
-        logger.info(
-            f"Tagged upsert metadata: {created_count} created, {updated_count} updated "
-            f"(total={len(result_objects)}, existing_pks={len(existing_pks)})"
-        )
-

django_bulk_hooks/operations/coordinator.py

@@ -14,6 +14,7 @@ from django.db.models import QuerySet
 from django_bulk_hooks.helpers import build_changeset_for_create
 from django_bulk_hooks.helpers import build_changeset_for_delete
 from django_bulk_hooks.helpers import build_changeset_for_update
+from django_bulk_hooks.helpers import extract_pks
 
 logger = logging.getLogger(__name__)
 
@@ -46,55 +47,99 @@ class BulkOperationCoordinator:
         self._executor = None
         self._dispatcher = None
 
+    def _get_or_create_service(self, service_name, service_class, *args, **kwargs):
+        """
+        Generic lazy service initialization.
+
+        Args:
+            service_name: Name of the service attribute (e.g., 'analyzer')
+            service_class: The class to instantiate
+            *args, **kwargs: Arguments to pass to the service constructor
+
+        Returns:
+            The service instance
+        """
+        attr_name = f"_{service_name}"
+        if getattr(self, attr_name) is None:
+            setattr(self, attr_name, service_class(*args, **kwargs))
+        return getattr(self, attr_name)
+
     @property
     def analyzer(self):
         """Get or create ModelAnalyzer"""
-        if self._analyzer is None:
-            from django_bulk_hooks.operations.analyzer import ModelAnalyzer
-
-            self._analyzer = ModelAnalyzer(self.model_cls)
-        return self._analyzer
+        from django_bulk_hooks.operations.analyzer import ModelAnalyzer
+        return self._get_or_create_service("analyzer", ModelAnalyzer, self.model_cls)
 
     @property
     def mti_handler(self):
         """Get or create MTIHandler"""
-        if self._mti_handler is None:
-            from django_bulk_hooks.operations.mti_handler import MTIHandler
-
-            self._mti_handler = MTIHandler(self.model_cls)
-        return self._mti_handler
+        from django_bulk_hooks.operations.mti_handler import MTIHandler
+        return self._get_or_create_service("mti_handler", MTIHandler, self.model_cls)
 
     @property
     def record_classifier(self):
         """Get or create RecordClassifier"""
-        if self._record_classifier is None:
-            from django_bulk_hooks.operations.record_classifier import RecordClassifier
-
-            self._record_classifier = RecordClassifier(self.model_cls)
-        return self._record_classifier
+        from django_bulk_hooks.operations.record_classifier import RecordClassifier
+        return self._get_or_create_service("record_classifier", RecordClassifier, self.model_cls)
 
     @property
     def executor(self):
         """Get or create BulkExecutor"""
-        if self._executor is None:
-            from django_bulk_hooks.operations.bulk_executor import BulkExecutor
-
-            self._executor = BulkExecutor(
-                queryset=self.queryset,
-                analyzer=self.analyzer,
-                mti_handler=self.mti_handler,
-                record_classifier=self.record_classifier,
-            )
-        return self._executor
+        from django_bulk_hooks.operations.bulk_executor import BulkExecutor
+        return self._get_or_create_service(
+            "executor",
+            BulkExecutor,
+            queryset=self.queryset,
+            analyzer=self.analyzer,
+            mti_handler=self.mti_handler,
+            record_classifier=self.record_classifier,
+        )
 
     @property
     def dispatcher(self):
         """Get or create Dispatcher"""
-        if self._dispatcher is None:
-            from django_bulk_hooks.dispatcher import get_dispatcher
+        from django_bulk_hooks.dispatcher import get_dispatcher
+        return self._get_or_create_service("dispatcher", get_dispatcher)
 
-            self._dispatcher = get_dispatcher()
-        return self._dispatcher
+    @property
+    def inheritance_chain(self):
+        """Single source of truth for inheritance chain"""
+        return self.mti_handler.get_inheritance_chain()
+
+    def _validate_objects_for_operation(self, objs, operation_type):
+        """
+        Validate objects exist and return appropriate empty result.
+
+        Args:
+            objs: List of objects to validate
+            operation_type: 'create', 'update', 'delete', or 'validate'
+
+        Returns:
+            Appropriate empty result for the operation type, or None if objects exist
+        """
+        if not objs:
+            empty_results = {
+                "create": objs,
+                "update": 0,
+                "delete": (0, {}),
+                "validate": None,
+            }
+            return empty_results[operation_type]
+        return None  # Continue with operation
+
+    def _dispatch_hooks_for_models(self, models_in_chain, changeset, event_suffix, bypass_hooks=False):
+        """
+        Dispatch hooks for all models in inheritance chain.
+
+        Args:
+            models_in_chain: List of model classes in MTI inheritance chain
+            changeset: The changeset to use as base
+            event_suffix: Event name suffix (e.g., 'before_create', 'validate_update')
+            bypass_hooks: Whether to skip hook execution
+        """
+        for model_cls in models_in_chain:
+            model_changeset = self._build_changeset_for_model(changeset, model_cls)
+            self.dispatcher.dispatch(model_changeset, event_suffix, bypass_hooks=bypass_hooks)
 
     # ==================== PUBLIC API ====================
 
@@ -126,8 +171,9 @@ class BulkOperationCoordinator:
         Returns:
             List of created objects
         """
-        if not objs:
-            return objs
+        empty_result = self._validate_objects_for_operation(objs, 'create')
+        if empty_result is not None:
+            return empty_result
 
         # Validate
         self.analyzer.validate_for_create(objs)
@@ -141,7 +187,7 @@ class BulkOperationCoordinator:
             if self.mti_handler.is_mti_model():
                 query_model = self.mti_handler.find_model_with_unique_fields(unique_fields)
                 logger.info(f"MTI model detected: querying {query_model.__name__} for unique fields {unique_fields}")
-            
+
             existing_record_ids, existing_pks_map = self.record_classifier.classify_for_upsert(
                 objs, unique_fields, query_model=query_model
             )
@@ -203,8 +249,9 @@ class BulkOperationCoordinator:
         Returns:
             Number of objects updated
         """
-        if not objs:
-            return 0
+        empty_result = self._validate_objects_for_operation(objs, 'update')
+        if empty_result is not None:
+            return empty_result
 
         # Validate
         self.analyzer.validate_for_update(objs)
@@ -349,19 +396,11 @@ class BulkOperationCoordinator:
         changeset.operation_meta["allows_modifications"] = True
 
         # Step 5: Get MTI inheritance chain
-        models_in_chain = [self.model_cls]
-        if self.mti_handler.is_mti_model():
-            models_in_chain.extend(self.mti_handler.get_parent_models())
+        models_in_chain = self._get_models_in_chain(self.model_cls)
 
         # Step 6: Run VALIDATE hooks (if not bypassed)
         if not bypass_validation:
-            for model_cls in models_in_chain:
-                model_changeset = self._build_changeset_for_model(changeset, model_cls)
-                self.dispatcher.dispatch(
-                    model_changeset,
-                    "validate_update",
-                    bypass_hooks=False,
-                )
+            self._dispatch_hooks_for_models(models_in_chain, changeset, "validate_update", bypass_hooks=False)
 
         # Step 7: Run BEFORE_UPDATE hooks with modification tracking
         modified_fields = self._run_before_update_hooks_with_tracking(
@@ -378,13 +417,7 @@ class BulkOperationCoordinator:
         pre_after_hook_state = self._snapshot_instance_state(new_instances)
 
         # Step 10: Run AFTER_UPDATE hooks (read-only side effects)
-        for model_cls in models_in_chain:
-            model_changeset = self._build_changeset_for_model(changeset, model_cls)
-            self.dispatcher.dispatch(
-                model_changeset,
-                "after_update",
-                bypass_hooks=False,
-            )
+        self._dispatch_hooks_for_models(models_in_chain, changeset, "after_update", bypass_hooks=False)
 
         # Step 11: Auto-persist AFTER_UPDATE modifications (if any)
         after_modified_fields = self._detect_modifications(new_instances, pre_after_hook_state)
@@ -408,13 +441,7 @@ class BulkOperationCoordinator:
         pre_hook_state = self._snapshot_instance_state(instances)
 
         # Run BEFORE_UPDATE hooks
-        for model_cls in models_in_chain:
-            model_changeset = self._build_changeset_for_model(changeset, model_cls)
-            self.dispatcher.dispatch(
-                model_changeset,
-                "before_update",
-                bypass_hooks=False,
-            )
+        self._dispatch_hooks_for_models(models_in_chain, changeset, "before_update", bypass_hooks=False)
 
         # Detect modifications
         return self._detect_modifications(instances, pre_hook_state)
@@ -517,8 +544,9 @@ class BulkOperationCoordinator:
         """
         # Get objects
         objs = list(self.queryset)
-        if not objs:
-            return 0, {}
+        empty_result = self._validate_objects_for_operation(objs, 'delete')
+        if empty_result is not None:
+            return empty_result
 
         # Validate
         self.analyzer.validate_for_delete(objs)
@@ -553,13 +581,20 @@ class BulkOperationCoordinator:
         Returns:
             None
         """
-        if not objs:
+        empty_result = self._validate_objects_for_operation(objs, 'validate')
+        if empty_result is not None:
             return
 
         # Auto-detect if is_create not specified
         if is_create is None:
             is_create = objs[0].pk is None
 
+        # Use centralized validation logic (consistent with other operations)
+        if is_create:
+            self.analyzer.validate_for_create(objs)
+        else:
+            self.analyzer.validate_for_update(objs)
+
         # Build changeset based on operation type
         if is_create:
             changeset = build_changeset_for_create(self.model_cls, objs)
@@ -569,8 +604,9 @@ class BulkOperationCoordinator:
             changeset = build_changeset_for_update(self.model_cls, objs, {})
             event = "validate_update"
 
-        # Dispatch validation event only
-        self.dispatcher.dispatch(changeset, event, bypass_hooks=False)
+        # Dispatch validation event for entire inheritance chain
+        models_in_chain = self._get_models_in_chain(self.model_cls)
+        self._dispatch_hooks_for_models(models_in_chain, changeset, event)
 
     # ==================== MTI PARENT HOOK SUPPORT ====================
 
@@ -598,6 +634,21 @@ class BulkOperationCoordinator:
             operation_meta=original_changeset.operation_meta,
         )
 
+    def _get_models_in_chain(self, model_cls):
+        """
+        Get all models in the inheritance chain for hook dispatching.
+
+        DEPRECATED: Use self.inheritance_chain property instead for consistency.
+        This method is kept for backward compatibility.
+
+        Args:
+            model_cls: The model class to start from
+
+        Returns:
+            List of model classes in inheritance order [child, parent1, parent2, ...]
+        """
+        return self.inheritance_chain
+
     def _execute_with_mti_hooks(
         self,
         changeset,
@@ -627,21 +678,14 @@ class BulkOperationCoordinator:
             return operation()
 
         # Get all models in inheritance chain
-        models_in_chain = [changeset.model_cls]
-        if self.mti_handler.is_mti_model():
-            parent_models = self.mti_handler.get_parent_models()
-            models_in_chain.extend(parent_models)
+        models_in_chain = self._get_models_in_chain(changeset.model_cls)
 
         # VALIDATE phase - for all models in chain
         if not bypass_validation:
-            for model_cls in models_in_chain:
-                model_changeset = self._build_changeset_for_model(changeset, model_cls)
-                self.dispatcher.dispatch(model_changeset, f"validate_{event_prefix}", bypass_hooks=False)
+            self._dispatch_hooks_for_models(models_in_chain, changeset, f"validate_{event_prefix}")
 
         # BEFORE phase - for all models in chain
-        for model_cls in models_in_chain:
-            model_changeset = self._build_changeset_for_model(changeset, model_cls)
-            self.dispatcher.dispatch(model_changeset, f"before_{event_prefix}", bypass_hooks=False)
+        self._dispatch_hooks_for_models(models_in_chain, changeset, f"before_{event_prefix}")
 
         # Execute the actual operation
         result = operation()
@@ -660,14 +704,10 @@ class BulkOperationCoordinator:
 
                 changeset = build_changeset_for_create(changeset.model_cls, result)
 
-                for model_cls in models_in_chain:
-                    model_changeset = self._build_changeset_for_model(changeset, model_cls)
-                    self.dispatcher.dispatch(model_changeset, f"after_{event_prefix}", bypass_hooks=False)
+                self._dispatch_hooks_for_models(models_in_chain, changeset, f"after_{event_prefix}")
         else:
             # Non-create operations (update, delete)
-            for model_cls in models_in_chain:
-                model_changeset = self._build_changeset_for_model(changeset, model_cls)
-                self.dispatcher.dispatch(model_changeset, f"after_{event_prefix}", bypass_hooks=False)
+            self._dispatch_hooks_for_models(models_in_chain, changeset, f"after_{event_prefix}")
 
         return result
 
@@ -768,7 +808,7 @@ class BulkOperationCoordinator:
             for model_cls, objs in objects_by_model.items():
                 if hasattr(model_cls, "created_at") and hasattr(model_cls, "updated_at"):
                     # Bulk fetch timestamps for all objects of this model
-                    pks = [obj.pk for obj in objs if obj.pk is not None]
+                    pks = extract_pks(objs)
                     if pks:
                         timestamp_map = {
                             record["pk"]: (record["created_at"], record["updated_at"])
@@ -806,9 +846,7 @@ class BulkOperationCoordinator:
 
             create_changeset = build_changeset_for_create(self.model_cls, created_objects)
 
-            for model_cls in models_in_chain:
-                model_changeset = self._build_changeset_for_model(create_changeset, model_cls)
-                self.dispatcher.dispatch(model_changeset, "after_create", bypass_hooks=False)
+            self._dispatch_hooks_for_models(models_in_chain, create_changeset, "after_create", bypass_hooks=False)
 
         # Dispatch after_update hooks for updated objects
         if updated_objects:
@@ -824,9 +862,7 @@ class BulkOperationCoordinator:
                 old_records_map=old_records_map,
             )
 
-            for model_cls in models_in_chain:
-                model_changeset = self._build_changeset_for_model(update_changeset, model_cls)
-                self.dispatcher.dispatch(model_changeset, "after_update", bypass_hooks=False)
+            self._dispatch_hooks_for_models(models_in_chain, update_changeset, "after_update", bypass_hooks=False)
 
         # Clean up temporary metadata
         self._cleanup_upsert_metadata(result_objects)