karrio-server-events 2025.5rc16__py3-none-any.whl → 2025.5rc17__py3-none-any.whl

karrio/server/events/migrations/0010_event_event_created_at_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.2.5 on 2025-08-23 00:22
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("events", "0009_alter_webhook_enabled_events"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="event",
+            index=models.Index(fields=["created_at"], name="event_created_at_idx"),
+        ),
+    ]

karrio/server/events/models.py

@@ -64,6 +64,8 @@ class Event(core.OwnedEntity):
                 condition=models.Q(data__id__isnull=False),
                 name="event_object_idx",
             ),
+            # Index for archiving queries based on creation date
+            models.Index(fields=["created_at"], name="event_created_at_idx"),
         ]
 
     id = models.CharField(

karrio/server/events/task_definitions/base/archiving.py

@@ -24,35 +24,147 @@ def run_data_archiving(*args, **kwargs):
         days=conf.settings.TRACKER_DATA_RETENTION
     )
 
-    shipping_data = manager.Shipment.objects.filter(created_at__lt=shipment_retention)
-    tracking_Data = manager.Tracking.objects.filter(created_at__lt=tracker_retention)
-    tracing_data = tracing.TracingRecord.objects.filter(created_at__lt=log_retention)
-    api_log_data = core.APILog.objects.filter(requested_at__lt=log_retention)
-    order_data = orders.Order.objects.filter(created_at__lt=order_retention)
-    event_data = events.Event.objects.filter(created_at__lt=log_retention)
-
-    if any(tracing_data):
+    # Use efficient existence checks and batch processing
+    BATCH_SIZE = 1000  # Process in batches to avoid memory issues
+    
+    # Optimize querysets by only selecting required fields for existence checks
+    tracing_exists = tracing.TracingRecord.objects.filter(created_at__lt=log_retention).exists()
+    event_exists = events.Event.objects.filter(created_at__lt=log_retention).exists()
+    api_log_exists = core.APILog.objects.filter(requested_at__lt=log_retention).exists()
+    tracking_exists = manager.Tracking.objects.filter(created_at__lt=tracker_retention).exists()
+    shipping_exists = manager.Shipment.objects.filter(created_at__lt=shipment_retention).exists()
+    order_exists = orders.Order.objects.filter(created_at__lt=order_retention).exists()
+
+    if tracing_exists:
         logger.info(">> archiving SDK tracing backlog...")
-        utils.failsafe(lambda: tracing_data.delete())
+        tracing_data = tracing.TracingRecord.objects.filter(created_at__lt=log_retention)
+        utils.failsafe(lambda: _bulk_delete_tracing_data(tracing_data))
 
-    if any(event_data):
+    if event_exists:
         logger.info(">> archiving events backlog...")
+        event_data = events.Event.objects.filter(created_at__lt=log_retention)
         utils.failsafe(lambda: event_data.delete())
 
-    if any(api_log_data):
+    if api_log_exists:
         logger.info(">> archiving API request logs backlog...")
+        api_log_data = core.APILog.objects.filter(requested_at__lt=log_retention)
         utils.failsafe(lambda: api_log_data.delete())
 
-    if any(tracking_Data):
+    if tracking_exists:
         logger.info(">> archiving tracking data backlog...")
-        utils.failsafe(lambda: tracking_Data.delete())
+        tracking_data = manager.Tracking.objects.filter(created_at__lt=tracker_retention)
+        utils.failsafe(lambda: _bulk_delete_tracking_data(tracking_data))
 
-    if any(shipping_data):
+    if shipping_exists:
         logger.info(">> archiving shipping data backlog...")
-        utils.failsafe(lambda: shipping_data.delete())
+        shipping_data = manager.Shipment.objects.filter(created_at__lt=shipment_retention)
+        utils.failsafe(lambda: _bulk_delete_shipment_data(shipping_data))
 
-    if any(order_data):
+    if order_exists:
         logger.info(">> archiving order data backlog...")
-        utils.failsafe(lambda: order_data.delete())
+        order_data = orders.Order.objects.filter(created_at__lt=order_retention)
+        utils.failsafe(lambda: _bulk_delete_order_data(order_data))
 
     logger.info("> ending scheduled backlog archiving!")
+
+
+def _bulk_delete_tracing_data(tracing_queryset):
+    """Bulk delete tracing data to avoid N+1 queries with organization links."""
+    BATCH_SIZE = 1000
+    
+    try:
+        from karrio.server.orgs.models import TracingRecordLink
+        
+        # Process in batches to avoid memory issues
+        total_deleted = 0
+        while True:
+            # Get a batch of IDs to delete
+            batch_ids = list(tracing_queryset.values_list('id', flat=True)[:BATCH_SIZE])
+            
+            if not batch_ids:
+                break
+                
+            # Bulk delete TracingRecordLink entries first to avoid CASCADE N+1 queries
+            TracingRecordLink.objects.filter(item_id__in=batch_ids).delete()
+            
+            # Delete the tracing records in this batch
+            deleted_count = tracing_queryset.filter(id__in=batch_ids).delete()[0]
+            total_deleted += deleted_count
+            
+            logger.info(f"Deleted {deleted_count} tracing records (total: {total_deleted})")
+            
+    except ImportError:
+        # Organizations module not installed, just delete in batches
+        total_deleted = 0
+        while True:
+            # Get a batch to delete
+            batch_ids = list(tracing_queryset.values_list('id', flat=True)[:BATCH_SIZE])
+            
+            if not batch_ids:
+                break
+                
+            deleted_count = tracing_queryset.filter(id__in=batch_ids).delete()[0]
+            total_deleted += deleted_count
+            
+            logger.info(f"Deleted {deleted_count} tracing records (total: {total_deleted})")
+
+
+def _bulk_delete_tracking_data(tracking_queryset):
+    """Bulk delete tracking data to avoid N+1 queries with organization links."""
+    try:
+        from karrio.server.orgs.models import TrackingLink
+        
+        # Get the tracking record IDs that will be deleted
+        tracking_ids = list(tracking_queryset.values_list('id', flat=True))
+        
+        if tracking_ids:
+            # Bulk delete TrackingLink entries first to avoid CASCADE N+1 queries
+            TrackingLink.objects.filter(item_id__in=tracking_ids).delete()
+            
+        # Now delete the tracking records themselves
+        tracking_queryset.delete()
+        
+    except ImportError:
+        # Organizations module not installed, just delete normally
+        tracking_queryset.delete()
+
+
+def _bulk_delete_shipment_data(shipment_queryset):
+    """Bulk delete shipment data to avoid N+1 queries with organization links."""
+    try:
+        from karrio.server.orgs.models import ShipmentLink
+        
+        # Get the shipment record IDs that will be deleted
+        shipment_ids = list(shipment_queryset.values_list('id', flat=True))
+        
+        if shipment_ids:
+            # Bulk delete ShipmentLink entries first to avoid CASCADE N+1 queries
+            ShipmentLink.objects.filter(item_id__in=shipment_ids).delete()
+            
+        # Now delete the shipment records themselves
+        shipment_queryset.delete()
+        
+    except ImportError:
+        # Organizations module not installed, just delete normally
+        shipment_queryset.delete()
+
+
+def _bulk_delete_order_data(order_queryset):
+    """Bulk delete order data to avoid N+1 queries with organization links."""
+    try:
+        from karrio.server.orgs.models import OrderLink
+        
+        # Get the order record IDs that will be deleted
+        order_ids = list(order_queryset.values_list('id', flat=True))
+        
+        if order_ids:
+            # Bulk delete OrderLink entries first to avoid CASCADE N+1 queries
+            OrderLink.objects.filter(item_id__in=order_ids).delete()
+            
+        # Now delete the order records themselves
+        order_queryset.delete()
+        
+    except ImportError:
+        # Organizations module not installed, just delete normally
+        order_queryset.delete()
+

{karrio_server_events-2025.5rc16.dist-info → karrio_server_events-2025.5rc17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: karrio_server_events
-Version: 2025.5rc16
+Version: 2025.5rc17
 Summary: Multi-carrier shipping API Events module
 Author-email: karrio <hello@karrio.io>
 License-Expression: Apache-2.0

{karrio_server_events-2025.5rc16.dist-info → karrio_server_events-2025.5rc17.dist-info}/RECORD

@@ -2,7 +2,7 @@ karrio/server/events/__init__.py,sha256=tvia3JfJUbr8axL76i0ic308Q0623SVKOyg0FXP3
 karrio/server/events/admin.py,sha256=suMo4x8I3JBxAFBVIdE-5qnqZ6JAZV0FESABHOSc-vg,63
 karrio/server/events/apps.py,sha256=DIV8JdPB_3hD4DwhcQTevhph3PaceMkFSEGM1lOcdVY,301
 karrio/server/events/filters.py,sha256=vMeY42IsoFl482-yX1BI17sw-9x9euOnzTl-860W3sg,2230
-karrio/server/events/models.py,sha256=Ze4ITStRy_eb7qtujYbq7IISxvo7V2SKcmdCQ4fQO2I,2439
+karrio/server/events/models.py,sha256=WQQKpDnbUW97beFmCBJDTckSoE-h-HtxeRfjoNWepQY,2582
 karrio/server/events/router.py,sha256=IBUR7rfBkdEHQzWxYOPcVSM8NBp3fte9G6Q5BVTUNNw,95
 karrio/server/events/signals.py,sha256=gQ1FHwd-LXpg9OcV0inaqNBxSkqQVaztOgCOFNbo7I8,4549
 karrio/server/events/tasks.py,sha256=RVLJ2cNEQnT-3bUoitw9C_ST6g7tlLyzuJvu8omLOHc,712
@@ -17,6 +17,7 @@ karrio/server/events/migrations/0006_webhook_events_alter_event_data.py,sha256=N
 karrio/server/events/migrations/0007_auto_20221130_0255.py,sha256=o5tkTj6vzrqpOyX6mXAYS3GOYlDRMkMY4tcPxzOzvOc,780
 karrio/server/events/migrations/0008_alter_event_type.py,sha256=DhUAI-yG_ugEv4f9oigdhn1lH_6A2nV8GyS-N63cmdQ,397
 karrio/server/events/migrations/0009_alter_webhook_enabled_events.py,sha256=0bhn3Ab_4cW2Zjptw7yfKJT1iTnsrBl_7goCCquNzcQ,1825
+karrio/server/events/migrations/0010_event_event_created_at_idx.py,sha256=9vLPYVIId5Ni4W_EnmaCfL-F29YzoQ070963D3q6KgA,498
 karrio/server/events/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 karrio/server/events/serializers/__init__.py,sha256=S5KJONM1VFbB_Wyl__F2YbdSV7R_5RPNFos0ERkfDko,52
 karrio/server/events/serializers/base.py,sha256=Civqqod7X2u8wUfQpx7NejFpQfqQD8UWjztSOX4iWzg,2158
@@ -24,7 +25,7 @@ karrio/server/events/serializers/event.py,sha256=dKYqldfSEjKMY7uB1rBFfEfSZ_dfEp8
 karrio/server/events/serializers/webhook.py,sha256=Ri5JS1SXpSsPUvQH989T_7Amu1p46y5TP54MEEhK8j8,889
 karrio/server/events/task_definitions/__init__.py,sha256=bpT73UG7mZL_JjEqMwbYx6q69jA8J5Jcoul1LcDokhA,81
 karrio/server/events/task_definitions/base/__init__.py,sha256=3vP52gFdCo9hctLRKnYvEi9iz1UK8GL_a47Nw44h3Nc,1834
-karrio/server/events/task_definitions/base/archiving.py,sha256=flbcxmD1pKlzCrYvekbwJ7Dvz4TaLdudAAsWH8JAjzI,2257
+karrio/server/events/task_definitions/base/archiving.py,sha256=MtQoW5y4TwoFphg96hXrz_3wBoKxrmI5yQRiHm98pL4,6987
 karrio/server/events/task_definitions/base/tracking.py,sha256=UL7-z1nVp3Scz-Nm6xuJB8S6obfaiHd-IyBXXIcrwOo,8533
 karrio/server/events/task_definitions/base/webhook.py,sha256=NgXMJw_55pwZ5z4bu49VUekFNkUJ_cwvs01PNnLxhI4,3506
 karrio/server/events/tests/__init__.py,sha256=5szv9hDvu0R0jtm3XWktY19rCaE8bZ2LdjRm4oJ0pYA,220
@@ -38,7 +39,7 @@ karrio/server/graph/schemas/events/__init__.py,sha256=Yg1rWew5DgSb_OnLy8n7qqwPLf
 karrio/server/graph/schemas/events/inputs.py,sha256=Mgy9qCUrbL-sMI5dNXsnbW8K5_PnGKM13brzZrc5OWk,1526
 karrio/server/graph/schemas/events/mutations.py,sha256=930g2D5TnAEgEjfEfLml1Q9OGCyxe2D592BhzSGDQAQ,1862
 karrio/server/graph/schemas/events/types.py,sha256=TPlrDwQV4LZXu-1t4gFl_eYayU39THmgYT3uv7VXnnw,2759
-karrio_server_events-2025.5rc16.dist-info/METADATA,sha256=22gin3KztbVtme9fD8Z7hiHw2xJhS25zTK6md7volYc,705
-karrio_server_events-2025.5rc16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-karrio_server_events-2025.5rc16.dist-info/top_level.txt,sha256=D1D7x8R3cTfjF_15mfiO7wCQ5QMtuM4x8GaPr7z5i78,12
-karrio_server_events-2025.5rc16.dist-info/RECORD,,
+karrio_server_events-2025.5rc17.dist-info/METADATA,sha256=aOk6AOFvrGnJ0XbpjSTtZblpifv0cZchGzb-cr6nl60,705
+karrio_server_events-2025.5rc17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+karrio_server_events-2025.5rc17.dist-info/top_level.txt,sha256=D1D7x8R3cTfjF_15mfiO7wCQ5QMtuM4x8GaPr7z5i78,12
+karrio_server_events-2025.5rc17.dist-info/RECORD,,