karrio-server-data 2025.5rc1__py3-none-any.whl

karrio/server/data/serializers/__init__.py

@@ -0,0 +1,26 @@
+from karrio.server.serializers import *
+from karrio.server.core.serializers import *
+from karrio.server.events.serializers.base import *
+from karrio.server.orders.serializers import OrderData
+from karrio.server.orders.serializers.order import OrderSerializer
+from karrio.server.events.serializers.event import EventSerializer
+from karrio.server.manager.serializers import (
+    ShipmentSerializer,
+    ShipmentData,
+    TrackingSerializer,
+    TrackingData,
+)
+from karrio.server.data.serializers.base import (
+    ImportData,
+    BatchObject,
+    ResourceType,
+    ResourceStatus,
+    BatchOperation,
+    BatchOperationData,
+    BatchOperationStatus,
+    OPERATION_STATUS,
+    RESOURCE_TYPE,
+)
+from karrio.server.data.serializers.batch_orders import BatchOrderData
+from karrio.server.data.serializers.batch_trackers import BatchTrackerData
+from karrio.server.data.serializers.batch_shipments import BatchShipmentData

karrio/server/data/serializers/base.py

@@ -0,0 +1,107 @@
+import karrio.lib as lib
+import rest_framework.fields as fields
+import karrio.server.serializers as serializers
+
+
+class ResourceType(lib.StrEnum):
+    order = "orders"
+    shipment = "shipments"
+    trackers = "trackers"
+    billing = "billing"
+
+    @classmethod
+    def get_default_mapping(cls, resource_type: str) -> dict:
+        from karrio.server.data import resources
+
+        if resource_type == "orders":
+            return resources.orders.DEFAULT_HEADERS
+        if resource_type == "shipments":
+            return resources.shipments.DEFAULT_HEADERS
+        if resource_type == "trackers":
+            return resources.trackers.DEFAULT_HEADERS
+
+        return {}
+
+    @classmethod
+    def get_model(cls, resource_type: str) -> dict:
+        if resource_type == "orders":
+            from karrio.server.orders.models import Order
+
+            return Order
+        if resource_type == "shipments":
+            from karrio.server.manager.models import Shipment
+
+            return Shipment
+        if resource_type == "trackers":
+            from karrio.server.manager.models import Tracking
+
+            return Tracking
+
+        return None
+
+    @classmethod
+    def get_serialiazer(cls, resource_type: str) -> dict:
+        if resource_type == "orders":
+            from karrio.server.data.serializers import BatchOrderData
+
+            return BatchOrderData
+        if resource_type == "shipments":
+            from karrio.server.data.serializers import BatchShipmentData
+
+            return BatchShipmentData
+        if resource_type == "trackers":
+            from karrio.server.data.serializers import BatchTrackerData
+
+            return BatchTrackerData
+
+        return None
+
+
+class ResourceStatus(lib.StrEnum):
+    queued = "queued"
+    created = "created"
+    has_errors = "has_errors"
+    incomplete = "incomplete"
+    processed = "processed"
+
+
+class BatchOperationStatus(lib.StrEnum):
+    queued = "queued"
+    running = "running"
+    failed = "failed"
+    completed = "completed"
+    completed_with_errors = "completed_with_errors"
+
+
+RESOURCE_TYPE = [(c.value, c.value) for c in list(ResourceType)]
+OPERATION_STATUS = [(c.value, c.value) for c in list(BatchOperationStatus)]
+
+
+class ImportData(serializers.Serializer):
+    resource_type = fields.ChoiceField(required=True, choices=RESOURCE_TYPE)
+    data_template = fields.CharField(required=False)
+    data_file = fields.FileField(required=True)
+
+
+class BatchObject(serializers.EntitySerializer):
+    status = fields.ChoiceField(
+        choices=OPERATION_STATUS,
+        help_text="The batch operation resource status",
+    )
+    errors = serializers.PlainDictField(
+        required=False,
+        allow_null=True,
+        help_text="Resource processing errors",
+    )
+
+
+class BatchOperationData(serializers.Serializer):
+    status = fields.ChoiceField(choices=OPERATION_STATUS)
+    resource_type = fields.ChoiceField(required=True, choices=RESOURCE_TYPE)
+    resources = BatchObject(many=True)
+
+
+class BatchOperation(serializers.EntitySerializer, BatchOperationData):
+    created_at = fields.DateTimeField()
+    updated_at = fields.DateTimeField()
+    test_mode = fields.BooleanField(required=True)

karrio/server/data/serializers/batch.py

@@ -0,0 +1,9 @@
+import karrio.server.serializers as serializers
+import karrio.server.data.models as models
+
+
+@serializers.owned_model_serializer
+class BatchOperationModelSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.BatchOperation
+        exclude = ["created_at", "updated_at", "created_by"]

karrio/server/data/serializers/batch_orders.py

@@ -0,0 +1,99 @@
+from django.db import transaction
+
+import karrio.lib as lib
+import karrio.server.conf as conf
+import karrio.server.orders.models as orders
+import karrio.server.serializers as serializers
+import karrio.server.data.serializers.base as base
+import karrio.server.core.exceptions as exceptions
+import karrio.server.orders.serializers as order_serializers
+
+
+@serializers.owned_model_serializer
+class BatchOrderData(serializers.Serializer):
+    orders = order_serializers.OrderData(
+        many=True,
+        allow_empty=False,
+        help_text="The list of orders to process.",
+    )
+
+    @transaction.atomic
+    def create(self, validated_data: dict, context: serializers.Context, **kwargs):
+        import karrio.server.events.tasks as tasks
+        import karrio.server.data.serializers.batch as batch
+
+        operation_data = dict(resource_type="orders", test_mode=context.test_mode)
+        operation = (
+            batch.BatchOperationModelSerializer.map(
+                data=operation_data, context=context
+            )
+            .save()
+            .instance
+        )
+
+        sid = transaction.savepoint()
+        resources = BatchOrderData.save_resources(
+            context=context,
+            data=validated_data,
+            batch_id=operation.id,
+            format_errors=False,
+        )
+        errors = [r["errors"] for r in resources if r.get("errors") is not None]
+        transaction.savepoint_rollback(sid)
+
+        if any(errors):
+            raise exceptions.APIExceptions(errors, code="invalid_data")
+
+        tasks.save_batch_resources(
+            operation.id,
+            data=validated_data,
+            schema=conf.settings.schema,
+            ctx=dict(
+                test_mode=context.test_mode,
+                org_id=getattr(context.org, "id", None),
+                user_id=getattr(context.user, "id", None),
+            ),
+        )
+
+        return operation
+
+    @staticmethod
+    def save_resources(
+        data: dict,
+        batch_id: str,
+        context: serializers.Context,
+        format_errors: bool = True,
+    ):
+        orders_data = data["orders"]
+        resources = []
+
+        for index, order_data in enumerate(orders_data):
+            try:
+                queryset = orders.Order.access_by(context).filter(
+                    order_id=order_data["order_id"], source=order_data["source"]
+                )
+                order = (
+                    queryset.first()
+                    if queryset.exists()
+                    else (
+                        order_serializers.OrderSerializer.map(
+                            data=order_data, context=context
+                        )
+                        .save()
+                        .instance
+                    )
+                )
+                resources.append(
+                    dict(id=order.id, status=base.ResourceStatus.queued.value)
+                )
+            except Exception as e:
+                setattr(e, "index", index)
+                resources.append(
+                    dict(
+                        id=index,
+                        status=base.ResourceStatus.has_errors.value,
+                        errors=(lib.to_dict(e) if format_errors else e),
+                    )
+                )
+
+        return resources

karrio/server/data/serializers/batch_shipments.py

@@ -0,0 +1,102 @@
+from django.db import transaction
+
+import karrio.lib as lib
+import karrio.server.conf as conf
+import karrio.server.manager.models as manager
+import karrio.server.serializers as serializers
+import karrio.server.core.exceptions as exceptions
+import karrio.server.data.serializers.base as base
+import karrio.server.manager.serializers as manager_serializers
+
+
+class ShipmentDataReference(manager_serializers.ShipmentData):
+    id = serializers.CharField(
+        help_text="The shipment id.",
+        required=False,
+    )
+
+
+@serializers.owned_model_serializer
+class BatchShipmentData(serializers.Serializer):
+    shipments = ShipmentDataReference(
+        many=True,
+        allow_empty=False,
+        help_text="The list of shipments to process.",
+    )
+
+    @transaction.atomic
+    def create(self, validated_data: dict, context: serializers.Context, **kwargs):
+        import karrio.server.events.tasks as tasks
+        import karrio.server.data.serializers.batch as batch
+
+        operation_data = dict(resource_type="shipments", test_mode=context.test_mode)
+        operation = (
+            batch.BatchOperationModelSerializer.map(
+                data=operation_data, context=context
+            )
+            .save()
+            .instance
+        )
+
+        sid = transaction.savepoint()
+        resources = BatchShipmentData.save_resources(
+            context=context,
+            data=validated_data,
+            batch_id=operation.id,
+            format_errors=False,
+        )
+        errors = [r["errors"] for r in resources if r.get("errors") is not None]
+        transaction.savepoint_rollback(sid)
+
+        if any(errors):
+            raise exceptions.APIExceptions(errors, code="invalid_data")
+
+        tasks.save_batch_resources(
+            operation.id,
+            data=validated_data,
+            schema=conf.settings.schema,
+            ctx=dict(
+                test_mode=context.test_mode,
+                org_id=getattr(context.org, "id", None),
+                user_id=getattr(context.user, "id", None),
+            ),
+        )
+
+        return operation
+
+    @staticmethod
+    def save_resources(
+        data: dict,
+        batch_id: str,
+        context: serializers.Context,
+        format_errors: bool = True,
+    ):
+        shipments_data = data["shipments"]
+        resources = []
+
+        for index, shipment_data in enumerate(shipments_data):
+            try:
+                shipment = (
+                    manager.Shipment.access_by(context).get(id=shipment_data["id"])
+                    if shipment_data.get("id") is not None
+                    else (
+                        manager_serializers.ShipmentSerializer.map(
+                            data=shipment_data, context=context
+                        )
+                        .save(fetch_rates=False)
+                        .instance
+                    )
+                )
+                resources.append(
+                    dict(id=shipment.id, status=base.ResourceStatus.queued.value)
+                )
+            except Exception as e:
+                resources.append(
+                    dict(
+                        id=index,
+                        status=base.ResourceStatus.has_errors.value,
+                        errors=(lib.to_dict(e) if format_errors else e),
+                    )
+                )
+
+        return resources

karrio/server/data/serializers/batch_trackers.py

@@ -0,0 +1,131 @@
+from django.db import transaction
+
+import karrio.lib as lib
+import karrio.server.conf as conf
+import karrio.server.core.utils as utils
+import karrio.server.core.gateway as gateway
+import karrio.server.core.exceptions as exceptions
+import karrio.server.serializers as serializers
+import karrio.server.manager.models as models
+import karrio.server.manager.serializers as manager
+import karrio.server.data.serializers.base as base
+
+
+@serializers.owned_model_serializer
+class BatchTrackerData(serializers.Serializer):
+    trackers = manager.TrackingData(
+        many=True,
+        allow_empty=False,
+        help_text="The list of tracking info to process.",
+    )
+
+    @transaction.atomic
+    def create(self, validated_data: dict, context: serializers.Context, **kwargs):
+        import karrio.server.events.tasks as tasks
+        import karrio.server.data.serializers.batch as batch
+
+        operation_data = dict(resource_type="trackers", test_mode=context.test_mode)
+        operation = (
+            batch.BatchOperationModelSerializer
+            .map(data=operation_data, context=context)
+            .save()
+            .instance
+        )
+
+        sid = transaction.savepoint()
+        resources = BatchTrackerData.save_resources(
+            context=context,
+            data=validated_data,
+            batch_id=operation.id,
+            format_errors=False,
+        )
+        errors = [r['errors'] for r in resources if r.get('errors') is not None]
+        transaction.savepoint_rollback(sid)
+
+        if any(errors):
+            raise exceptions.APIExceptions(errors, code="invalid_data")
+
+        tasks.save_batch_resources(
+            operation.id,
+            data=validated_data,
+            schema=conf.settings.schema,
+            ctx=dict(
+                test_mode=context.test_mode,
+                org_id=getattr(context.org, "id", None),
+                user_id=getattr(context.user, "id", None),
+            ),
+        )
+
+        return operation
+
+    @staticmethod
+    def save_resources(data: dict, batch_id: str, context: serializers.Context, format_errors: bool = True):
+        meta = dict(batch_id=batch_id)
+        trackers_data = data['trackers']
+        carrier_names = set([t['carrier_name'] for t in trackers_data])
+        carriers = {
+            carrier_name: utils.failsafe(lambda: gateway.Carriers.first(
+                context=context,
+                capability='tracking',
+                carrier_name=carrier_name,
+                raise_not_found=False,
+            ))
+            for carrier_name in carrier_names
+        }
+        resources = []
+        trackers = []
+
+        for index, tracker_data in enumerate(trackers_data):
+            try:
+                carrier_name = tracker_data['carrier_name']
+                carrier = carriers[carrier_name]
+
+                if carrier is None:
+                    raise exceptions.APIException(
+                        f"No carrier connection found for '{tracker_data['carrier_name']}'",
+                        code="invalid_carrier",
+                    )
+
+                _tracker = (
+                    models.Tracking.access_by(context)
+                    .filter(tracking_number=tracker_data["tracking_number"])
+                    .first()
+                )
+                _exists = getattr(_tracker, "carrier_name", None) == carrier_name
+                tracker = (
+                    _tracker if _exists
+                    else models.Tracking(
+                        meta=meta,
+                        status="unknown",
+                        test_mode=context.test_mode,
+                        created_by_id=context.user.id,
+                        tracking_carrier_id=carrier.id,
+                        tracking_number=tracker_data["tracking_number"],
+                        events = utils.default_tracking_event(
+                            description="Awaiting update from carrier...",
+                            code="UNKNOWN",
+                        ),
+                    )
+                )
+
+                if _exists is False:
+                    trackers.append(tracker)
+
+                resources.append(dict(
+                    id=tracker.id,
+                    status=(base.ResourceStatus.processed.value
+                        if _exists else base.ResourceStatus.queued.value)
+                ))
+            except Exception as e:
+                setattr(e, "index", index)
+                resources.append(dict(
+                    id=index,
+                    status=base.ResourceStatus.has_errors.value,
+                    errors=(lib.to_dict(e) if format_errors else e),
+                ))
+
+        if any(trackers):
+            models.Tracking.objects.bulk_create(trackers)
+            serializers.bulk_link_org(trackers, context)
+
+        return resources

karrio/server/data/serializers/data.py

@@ -0,0 +1,109 @@
+import tablib
+import logging
+from django.db import transaction
+
+from karrio.server.conf import settings
+import karrio.server.core.exceptions as exceptions
+import karrio.server.data.models as models
+import karrio.server.data.resources as resources
+import karrio.server.data.serializers as serializers
+import karrio.server.data.serializers.batch as batch
+
+logger = logging.getLogger(__name__)
+
+
+@serializers.owned_model_serializer
+class DataTemplateModelSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.DataTemplate
+        exclude = ["created_at", "updated_at", "created_by", "org"]
+
+
+@serializers.owned_model_serializer
+class ImportDataSerializer(serializers.ImportData):
+    @transaction.atomic
+    def create(
+        self, validated_data: dict, context: serializers.Context, **kwargs
+    ) -> models.BatchOperation:
+        import karrio.server.events.tasks as tasks
+
+        resource_type = validated_data["resource_type"]
+        data_field = validated_data["data_file"]
+        template = (
+            models.DataTemplate.access_by(context).first(
+                slug=validated_data.get("data_template")
+            )
+            if "data_template" in validated_data
+            else None
+        )
+        data_fields: dict = getattr(
+            template,
+            "data_fields",
+            serializers.ResourceType.get_default_mapping(resource_type),
+        )
+        resource = resources.get_import_resource(
+            resource_type=resource_type,
+            data_fields=data_fields,
+            params=validated_data,
+            context=context,
+        )
+        dataset = tablib.Dataset().load(
+            data_field.read().decode(),
+            headers=data_fields.values(),
+        )
+
+        # dry run data import to validate file content.
+        validation = resource.import_data(dataset, dry_run=True)
+        check_dataset_validation_errors(validation)
+
+        operation = (
+            batch.BatchOperationModelSerializer.map(
+                data=dict(resource_type=resource_type, test_mode=context.test_mode),
+                context=context,
+            )
+            .save()
+            .instance
+        )
+
+        tasks.queue_batch_import(
+            operation.id,
+            data=dict(
+                dataset=dataset,
+                import_data=validated_data,
+            ),
+            ctx=dict(
+                org_id=getattr(context.org, "id", None),
+                user_id=getattr(context.user, "id", None),
+                test_mode=context.test_mode,
+            ),
+            schema=settings.schema,
+        )
+
+        return operation
+
+
+def check_dataset_validation_errors(validation):
+    if any(validation.base_errors):
+        raise exceptions.APIExceptions(
+            validation.base_errors,
+            code="invalid_data",
+        )
+
+    errors = []
+    flattened_row_errors = sum(
+        [
+            [(i, e.error) for e in row.errors]
+            for i, row in enumerate(validation.rows)
+        ],
+        []
+    )
+
+    for index, error in flattened_row_errors:
+        setattr(error, "index", index)
+        errors.append(error)
+
+    if any(errors):
+        raise exceptions.APIExceptions(
+            errors,
+            code="invalid_data",
+        )

karrio/server/data/signals.py

@@ -0,0 +1,52 @@
+import logging
+from django.db.models import signals
+
+from karrio.server.conf import settings
+import karrio.server.core.utils as utils
+import karrio.server.data.models as models
+import karrio.server.events.tasks as tasks
+import karrio.server.data.serializers as serializers
+
+logger = logging.getLogger(__name__)
+
+
+def register_all():
+    signals.post_save.connect(batch_operation_updated, sender=models.BatchOperation)
+
+    logger.info("karrio.data signals registered...")
+
+
+@utils.disable_for_loaddata
+def batch_operation_updated(sender, instance, *args, **kwargs):
+    """Process batch related events
+    - Create event on batch creation and completion
+    - Notifiy webhook subscribers of batch operation updates
+    """
+    changes = kwargs.get("update_fields") or []
+    post_create = "created_at" in changes
+
+    if post_create:
+        event = serializers.EventTypes.batch_queued.value
+    elif instance.status == serializers.BatchOperationStatus.running.value:
+        event = serializers.EventTypes.batch_running.value
+        # Run post creation processing for batch_operation resources
+        tasks.process_batch_resources(instance.id, schema=settings.schema)
+    elif instance.status == serializers.BatchOperationStatus.completed.value:
+        event = serializers.EventTypes.batch_completed.value
+    else:
+        return
+
+    data = serializers.BatchOperation(instance).data
+    event_at = instance.updated_at
+    context = dict(
+        user_id=utils.failsafe(lambda: instance.created_by.id),
+        test_mode=instance.test_mode,
+        org_id=utils.failsafe(
+            lambda: instance.org.first().id if hasattr(instance, "org") else None
+        ),
+    )
+
+    if settings.MULTI_ORGANIZATIONS and context["org_id"] is None:
+        return
+
+    tasks.notify_webhooks(event, data, event_at, context, schema=settings.schema)

karrio/server/data/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

karrio/server/data/urls.py

@@ -0,0 +1,13 @@
+"""
+karrio server data module urls
+"""
+from django.urls import include, path
+
+app_name = "karrio.server.data"
+urlpatterns = [
+    path("v1/", include("karrio.server.data.views.data")),
+    path("v1/", include("karrio.server.data.views.batch")),
+    path("v1/", include("karrio.server.data.views.batch_order")),
+    path("v1/", include("karrio.server.data.views.batch_shipment")),
+    path("v1/", include("karrio.server.data.views.batch_tracking")),
+]