karrio-server-events 2025.5rc1__py3-none-any.whl

karrio/server/events/signals.py

@@ -0,0 +1,138 @@
+import logging
+from django.db.models import signals
+
+from karrio.server.core import utils
+from karrio.server.conf import settings
+from karrio.server.events.serializers import EventTypes
+import karrio.server.core.serializers as serializers
+import karrio.server.manager.models as models
+import karrio.server.events.tasks as tasks
+
+logger = logging.getLogger(__name__)
+
+
+def register_signals():
+    signals.post_save.connect(shipment_updated, sender=models.Shipment)
+    signals.post_delete.connect(shipment_cancelled, sender=models.Shipment)
+    signals.post_save.connect(tracker_updated, sender=models.Tracking)
+
+    logger.info("karrio.events signals registered...")
+
+
+@utils.disable_for_loaddata
+@utils.error_wrapper
+def shipment_updated(
+    sender, instance, created, raw, using, update_fields, *args, **kwargs
+):
+    """Shipment related events:
+    - shipment purchased (label purchased)
+    - shipment fulfilled (shipped)
+    """
+    is_bound = "created_at" in (update_fields or [])
+    status_updated = "status" in (update_fields or [])
+
+    if created:
+        return
+    if is_bound and instance.status == serializers.ShipmentStatus.purchased.value:
+        event = EventTypes.shipment_purchased.value
+    elif (
+        status_updated and instance.status == serializers.ShipmentStatus.purchased.value
+    ):
+        event = EventTypes.shipment_purchased.value
+    elif (
+        status_updated
+        and instance.status == serializers.ShipmentStatus.in_transit.value
+    ):
+        event = EventTypes.shipment_fulfilled.value
+    elif (
+        status_updated and instance.status == serializers.ShipmentStatus.cancelled.value
+    ):
+        event = EventTypes.shipment_cancelled.value
+    elif (
+        status_updated
+        and instance.status == serializers.ShipmentStatus.out_for_delivery.value
+    ):
+        event = EventTypes.shipment_out_for_delivery.value
+    elif (
+        status_updated
+        and instance.status == serializers.ShipmentStatus.needs_attention.value
+    ):
+        event = EventTypes.shipment_needs_attention.value
+    elif (
+        status_updated
+        and instance.status == serializers.ShipmentStatus.delivery_failed.value
+    ):
+        event = EventTypes.shipment_delivery_failed.value
+    else:
+        return
+
+    data = serializers.Shipment(instance).data
+    event_at = instance.updated_at
+    context = dict(
+        test_mode=instance.test_mode,
+        user_id=utils.failsafe(lambda: instance.created_by.id),
+        org_id=utils.failsafe(
+            lambda: instance.org.first().id if hasattr(instance, "org") else None
+        ),
+    )
+
+    if settings.MULTI_ORGANIZATIONS and context["org_id"] is None:
+        return
+
+    tasks.notify_webhooks(event, data, event_at, context, schema=settings.schema)
+
+
+@utils.disable_for_loaddata
+def shipment_cancelled(sender, instance, *args, **kwargs):
+    """Shipment related events:
+    - shipment cancelled/deleted (label voided)
+    """
+    event = EventTypes.shipment_cancelled.value
+    data = serializers.Shipment(instance)
+    event_at = instance.updated_at
+    context = dict(
+        user_id=utils.failsafe(lambda: instance.created_by.id),
+        test_mode=instance.test_mode,
+        org_id=utils.failsafe(
+            lambda: instance.org.first().id if hasattr(instance, "org") else None
+        ),
+    )
+
+    if settings.MULTI_ORGANIZATIONS and context["org_id"] is None:
+        return
+
+    tasks.notify_webhooks(event, data, event_at, context, schema=settings.schema)
+
+
+@utils.disable_for_loaddata
+def tracker_updated(
+    sender, instance, created, raw, using, update_fields, *args, **kwargs
+):
+    """Tracking related events:
+    - tracker created (pending)
+    - tracker status changed (in_transit, delivered or blocked)
+    """
+    changes = update_fields or []
+    post_create = created or "created_at" in changes
+
+    if post_create:
+        event = EventTypes.tracker_created.value
+    elif any(field in changes for field in ["status", "events"]):
+        event = EventTypes.tracker_updated.value
+    else:
+        return
+
+    data = serializers.TrackingStatus(instance).data
+    event_at = instance.updated_at
+    context = dict(
+        user_id=utils.failsafe(lambda: instance.created_by.id),
+        test_mode=instance.test_mode,
+        org_id=utils.failsafe(
+            lambda: instance.org.first().id if hasattr(instance, "org") else None
+        ),
+    )
+
+    if settings.MULTI_ORGANIZATIONS and context["org_id"] is None:
+        return
+
+    tasks.notify_webhooks(event, data, event_at, context, schema=settings.schema)

karrio/server/events/task_definitions/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore

karrio/server/events/task_definitions/base/__init__.py

@@ -0,0 +1,64 @@
+import time
+import logging
+from django.conf import settings
+from huey import crontab
+from huey.contrib.djhuey import db_task, db_periodic_task
+
+import karrio.server.core.utils as utils
+
+logger = logging.getLogger(__name__)
+DATA_ARCHIVING_SCHEDULE = int(getattr(settings, "DATA_ARCHIVING_SCHEDULE", 168))
+DEFAULT_TRACKERS_UPDATE_INTERVAL = int(
+    getattr(settings, "DEFAULT_TRACKERS_UPDATE_INTERVAL", 7200) / 60
+)
+
+
+@db_periodic_task(crontab(minute=f"*/{DEFAULT_TRACKERS_UPDATE_INTERVAL}"))
+def background_trackers_update():
+    from karrio.server.events.task_definitions.base import tracking
+
+    @utils.run_on_all_tenants
+    def _run(**kwargs):
+        try:
+            tracking.update_trackers()
+        except Exception as e:
+            logger.error(f"An error occured during tracking statuses update: {e}")
+
+    _run()
+
+
+@db_task(retries=5, retry_delay=60)
+@utils.tenant_aware
+def notify_webhooks(*args, **kwargs):
+    try:
+        from karrio.server.events.task_definitions.base import webhook
+
+        webhook.notify_webhook_subscribers(*args, **kwargs)
+
+    except Exception as e:
+        logger.error(f"An error occured during webhook notification: {e}")
+        raise e
+
+
+@db_periodic_task(crontab(hour=f"*/{DATA_ARCHIVING_SCHEDULE}"))
+def periodic_data_archiving(*args, **kwargs):
+    from karrio.server.events.task_definitions.base import archiving
+
+    @utils.run_on_all_tenants
+    def _run(**kwargs):
+        try:
+            utils.failsafe(
+                lambda: archiving.run_data_archiving(*args, **kwargs),
+                "An error occured during data archiving: $error",
+            )
+        except Exception as e:
+            logger.error(f"An error occured during data archiving: {e}")
+
+    _run()
+
+
+TASK_DEFINITIONS = [
+    background_trackers_update,
+    periodic_data_archiving,
+    notify_webhooks,
+]

karrio/server/events/task_definitions/base/archiving.py

@@ -0,0 +1,58 @@
+import logging
+import datetime
+import django.utils.timezone as timezone
+
+import karrio.server.conf as conf
+import karrio.server.core.utils as utils
+import karrio.server.core.models as core
+import karrio.server.events.models as events
+import karrio.server.orders.models as orders
+import karrio.server.tracing.models as tracing
+import karrio.server.manager.models as manager
+
+logger = logging.getLogger(__name__)
+
+
+def run_data_archiving(*args, **kwargs):
+    now = timezone.now()
+    log_retention = now - datetime.timedelta(days=conf.settings.API_LOGS_DATA_RETENTION)
+    order_retention = now - datetime.timedelta(days=conf.settings.ORDER_DATA_RETENTION)
+    shipment_retention = now - datetime.timedelta(
+        days=conf.settings.SHIPMENT_DATA_RETENTION
+    )
+    tracker_retention = now - datetime.timedelta(
+        days=conf.settings.TRACKER_DATA_RETENTION
+    )
+
+    shipping_data = manager.Shipment.objects.filter(created_at__lt=shipment_retention)
+    tracking_Data = manager.Tracking.objects.filter(created_at__lt=tracker_retention)
+    tracing_data = tracing.TracingRecord.objects.filter(created_at__lt=log_retention)
+    api_log_data = core.APILog.objects.filter(requested_at__lt=log_retention)
+    order_data = orders.Order.objects.filter(created_at__lt=order_retention)
+    event_data = events.Event.objects.filter(created_at__lt=log_retention)
+
+    if any(tracing_data):
+        logger.info(">> archiving SDK tracing backlog...")
+        utils.failsafe(lambda: tracing_data.delete())
+
+    if any(event_data):
+        logger.info(">> archiving events backlog...")
+        utils.failsafe(lambda: event_data.delete())
+
+    if any(api_log_data):
+        logger.info(">> archiving API request logs backlog...")
+        utils.failsafe(lambda: api_log_data.delete())
+
+    if any(tracking_Data):
+        logger.info(">> archiving tracking data backlog...")
+        utils.failsafe(lambda: tracking_Data.delete())
+
+    if any(shipping_data):
+        logger.info(">> archiving shipping data backlog...")
+        utils.failsafe(lambda: shipping_data.delete())
+
+    if any(order_data):
+        logger.info(">> archiving order data backlog...")
+        utils.failsafe(lambda: order_data.delete())
+
+    logger.info("> ending scheduled backlog archiving!")

karrio/server/events/task_definitions/base/tracking.py

@@ -0,0 +1,227 @@
+import time
+import typing
+import logging
+import datetime
+import functools
+from itertools import groupby
+
+from django.conf import settings
+from django.utils import timezone
+
+import karrio.sdk as karrio
+import karrio.lib as lib
+from karrio.api.gateway import Gateway
+from karrio.api.interface import IRequestFrom
+from karrio.core.models import TrackingDetails, Message, TrackingEvent
+
+import karrio.server.core.utils as utils
+import karrio.server.manager.models as models
+import karrio.server.tracing.utils as tracing
+import karrio.server.core.datatypes as datatypes
+import karrio.server.manager.serializers as serializers
+
+logger = logging.getLogger(__name__)
+Delay = int
+RequestBatches = typing.Tuple[
+    Gateway, IRequestFrom, Delay, typing.List[models.Tracking]
+]
+BatchResponse = typing.List[typing.Tuple[TrackingDetails, typing.List[Message]]]
+
+DEFAULT_TRACKERS_UPDATE_INTERVAL = getattr(
+    settings, "DEFAULT_TRACKERS_UPDATE_INTERVAL", 7200
+)
+
+
+def update_trackers(
+    delta: datetime.timedelta = datetime.timedelta(
+        seconds=DEFAULT_TRACKERS_UPDATE_INTERVAL
+    ),
+    tracker_ids: typing.List[str] = [],
+):
+    logger.info("> starting scheduled trackers update")
+
+    active_trackers = lib.identity(
+        models.Tracking.objects.filter(id__in=tracker_ids)
+        if any(tracker_ids)
+        else models.Tracking.objects.filter(
+            delivered=False,
+            updated_at__lt=timezone.now() - delta,
+        )
+    )
+
+    if any(active_trackers):
+        trackers_grouped_by_carrier = [
+            list(g) for _, g in groupby(active_trackers, key=lambda t: t.carrier_id)
+        ]
+        request_batches: typing.List[RequestBatches] = sum(
+            [create_request_batches(group) for group in trackers_grouped_by_carrier], []
+        )
+
+        responses = lib.run_concurently(fetch_tracking_info, request_batches, 2)
+        save_tracing_records(request_batches)
+        save_updated_trackers(responses, active_trackers)
+    else:
+        logger.info("no active trackers found needing update")
+
+    logger.info("> ending scheduled trackers update")
+
+
+def create_request_batches(
+    trackers: typing.List[models.Tracking],
+) -> typing.List[RequestBatches]:
+    start = 0
+    end = 10
+    batches = []
+
+    while any(trackers[start:end]):
+        try:
+            # Add a request delay to avoid sending two request batches to a carrier at the same time
+            delay = int(((end / 10) * 10) - 10)
+            # Get the common tracking carrier
+            carrier = trackers[0].tracking_carrier
+            # Collect the 5 trackers between the start and end indexes
+            batch_trackers = trackers[start:end]
+            tracking_numbers = [t.tracking_number for t in batch_trackers]
+            options: dict = functools.reduce(
+                lambda acc, t: {**acc, **(t.options or {})}, batch_trackers, {}
+            )
+
+            logger.debug(f"prepare tracking request for {tracking_numbers}")
+
+            # Prepare and send tracking request(s) using the karrio interface.
+            request: IRequestFrom = karrio.Tracking.fetch(
+                datatypes.TrackingRequest(
+                    tracking_numbers=tracking_numbers, options=options
+                )
+            )
+            gateway: Gateway = carrier.gateway
+
+            batches.append((gateway, request, delay, batch_trackers))
+
+        except Exception as request_error:
+            logger.warning(f"failed to prepare tracking batch ({start}, {end}) request")
+            logger.error(request_error, exc_info=True)
+
+        end += 10
+        start += 10
+
+    return batches
+
+
+def fetch_tracking_info(request_batch: RequestBatches) -> BatchResponse:
+    gateway, request, delay, trackers = request_batch
+    logger.debug(f"fetching batch {[t.tracking_number for t in trackers]}")
+    time.sleep(delay)  # apply delay before request
+
+    try:
+        return utils.identity(lambda: request.from_(gateway).parse())
+    except Exception as request_error:
+        logger.warning("batch request failed")
+        logger.error(request_error, exc_info=True)
+
+    return []
+
+
+@utils.error_wrapper
+def save_tracing_records(request_batches: typing.List[RequestBatches]):
+    logger.info("> saving tracing records...")
+
+    try:
+        for request_batch in request_batches:
+            gateway, _, __, trackers = request_batch
+
+            if not any(trackers):
+                continue
+
+            context = serializers.get_object_context(trackers[0])
+            tracing.bulk_save_tracing_records(gateway.tracer, context=context)
+    except Exception as error:
+        print(error)
+        logger.warning("Failed failed saving tracing record...")
+        logger.error(error, exc_info=True)
+
+
+def save_updated_trackers(
+    responses: typing.List[BatchResponse], trackers: typing.List[models.Tracking]
+):
+    logger.info("> saving updated trackers")
+
+    for tracking_details, _ in responses:
+        for details in tracking_details or []:
+            try:
+                logger.debug(f"update tracking info for {details.tracking_number}")
+                related_trackers = [
+                    t for t in trackers if t.tracking_number == details.tracking_number
+                ]
+                for tracker in related_trackers:
+                    # update values only if changed; This is important for webhooks notification
+                    changes = []
+                    meta = details.meta or {}
+                    status = utils.compute_tracking_status(details).value
+                    events = utils.process_events(
+                        response_events=details.events,
+                        current_events=tracker.events,
+                    )
+                    options = {
+                        **(tracker.options or {}),
+                        tracker.tracking_number: details.meta,
+                    }
+                    info = lib.to_dict(details.info or {})
+
+                    if events != tracker.events:
+                        tracker.events = events
+                        changes.append("events")
+
+                    if options != tracker.options:
+                        tracker.options = options
+                        changes.append("options")
+
+                    if details.meta != tracker.meta:
+                        tracker.meta = meta
+                        changes.append("meta")
+
+                    if details.delivered != tracker.delivered:
+                        tracker.delivered = details.delivered
+                        changes.append("delivered")
+
+                    if status != tracker.status:
+                        tracker.status = status
+                        changes.append("status")
+
+                    if details.estimated_delivery != tracker.estimated_delivery:
+                        tracker.estimated_delivery = details.estimated_delivery
+                        changes.append("estimated_delivery")
+
+                    if details.images is not None and (
+                        details.images.delivery_image != tracker.delivery_image
+                        or details.images.signature_image != tracker.signature_image
+                    ):
+                        changes.append("delivery_image")
+                        changes.append("signature_image")
+                        tracker.delivery_image = (
+                            details.images.delivery_image or tracker.delivery_image
+                        )
+                        tracker.signature_image = (
+                            details.images.signature_image or tracker.signature_image
+                        )
+
+                    if any(info.keys()) and info != tracker.info:
+                        tracker.info = serializers.process_dictionaries_mutations(
+                            ["info"], dict(info=info), tracker
+                        )["info"]
+                        changes.append("info")
+
+                    if any(changes):
+                        tracker.save(update_fields=changes)
+                        serializers.update_shipment_tracker(tracker)
+                        logger.debug(
+                            f"tracking info {details.tracking_number} updated successfully"
+                        )
+                    else:
+                        logger.debug(f"no changes detect")
+
+            except Exception as update_error:
+                logger.warning(
+                    f"failed to update tracker with tracking number: {details.tracking_number}"
+                )
+                logger.error(update_error, exc_info=True)

karrio/server/events/task_definitions/base/webhook.py

@@ -0,0 +1,116 @@
+import typing
+import requests
+import logging
+from datetime import datetime
+from django.conf import settings
+from django.db.models import Q
+from django.contrib.auth import get_user_model
+
+from karrio.core import utils
+from karrio.server.core.utils import identity
+from karrio.server.serializers import Context
+from karrio.server.events import models
+import karrio.server.events.serializers.event as serializers
+
+logger = logging.getLogger(__name__)
+NotificationResponse = typing.Tuple[str, requests.Response]
+User = get_user_model()
+
+
+def notify_webhook_subscribers(
+    event: str,
+    data: dict,
+    event_at: datetime,
+    ctx: dict,
+    **kwargs,
+):
+    logger.info(f"> starting {event} subscribers notification")
+    context = retrieve_context(ctx)
+    query = (
+        (Q(enabled_events__icontains=event) | Q(enabled_events__icontains="all")),
+        (Q(disabled__isnull=True) | Q(disabled=False)),
+    )
+
+    webhooks = models.Webhook.access_by(context).filter(*query)
+    serializers.EventSerializer.map(
+        data=dict(
+            type=event,
+            data=data,
+            test_mode=context.test_mode,
+            pending_webhooks=webhooks.count(),
+        ),
+        context=context,
+    ).save()
+
+    if any(webhooks):
+        payload = dict(event=event, data=data)
+        responses: typing.List[NotificationResponse] = notify_subscribers(
+            webhooks, payload
+        )
+        update_notified_webhooks(webhooks, responses, event_at)
+    else:
+        logger.info("no subscribers found")
+
+    logger.info(f"> ending {event} subscribers notification")
+
+
+def notify_subscribers(webhooks: typing.List[models.Webhook], payload: dict):
+    def notify_subscriber(webhook: models.Webhook):
+        response = identity(
+            lambda: requests.post(
+                webhook.url,
+                json=payload,
+                headers={
+                    "Content-type": "application/json",
+                    "X-Event-Id": webhook.secret,
+                },
+            )
+        )
+
+        return webhook.id, response
+
+    return utils.exec_async(notify_subscriber, webhooks)
+
+
+def update_notified_webhooks(
+    webhooks: typing.List[models.Webhook],
+    responses: typing.List[NotificationResponse],
+    event_at: datetime,
+):
+    logger.info("> saving updated webhooks")
+
+    for webhook_id, response in responses:
+        try:
+            logger.debug(f"update webhook {webhook_id}")
+
+            webhook = next((w for w in webhooks if w.id == webhook_id))
+            if response.ok:
+                webhook.last_event_at = event_at
+                webhook.failure_streak_count = 0
+            else:
+                webhook.failure_streak_count += 1
+                # Disable the webhook if notification failed more than 5 times
+                webhook.disabled = webhook.failure_streak_count > 5
+
+            webhook.save()
+
+            logger.debug(f"webhook {webhook_id} updated successfully")
+
+        except Exception as update_error:
+            logger.warning(f"failed to update webhook {webhook_id}")
+            logger.error(update_error, exc_info=True)
+
+
+def retrieve_context(info: dict) -> Context:
+    org = None
+
+    if settings.MULTI_ORGANIZATIONS and "org_id" in info:
+        import karrio.server.orgs.models as orgs_models
+
+        org = orgs_models.Organization.objects.filter(id=info["org_id"]).first()
+
+    return Context(
+        org=org,
+        user=User.objects.filter(id=info["user_id"]).first(),
+        test_mode=info.get("test_mode"),
+    )

karrio/server/events/tasks.py

@@ -0,0 +1,20 @@
+import typing
+import logging
+import pkgutil
+import karrio.server.events.task_definitions as definitions
+
+logger = logging.getLogger(__name__)
+DEFINITIONS: typing.List[typing.Any] = []
+
+# Register karrio background tasks
+for _, name, _ in pkgutil.iter_modules(definitions.__path__):  # type: ignore
+    try:
+        definition = __import__(f"{definitions.__name__}.{name}", fromlist=[name])  # type: ignore
+        if hasattr(definition, "TASK_DEFINITIONS"):
+            DEFINITIONS += definition.TASK_DEFINITIONS
+    except Exception as e:
+        logger.warning(f'Failed to register "{name}" schema')
+        logger.exception(e)
+
+for wrapper in DEFINITIONS:
+    globals()[wrapper.task_class.__name__] = wrapper

karrio/server/events/tests/__init__.py

@@ -0,0 +1,7 @@
+import logging
+
+logging.disable(logging.CRITICAL)
+
+from karrio.server.events.tests.test_tracking_tasks import *
+from karrio.server.events.tests.test_webhooks import *
+from karrio.server.events.tests.test_events import *