karrio-server-data 2025.5rc1__py3-none-any.whl

karrio/server/data/views/batch.py

@@ -0,0 +1,72 @@
+from django.urls import path
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.pagination import LimitOffsetPagination
+from django_filters.rest_framework import DjangoFilterBackend
+
+from karrio.server.data.filters import BatchOperationFilter
+import karrio.server.data.serializers as serializers
+import karrio.server.data.models as models
+import karrio.server.core.views.api as api
+import karrio.server.openapi as openapi
+
+ENDPOINT_ID = "&&&&$"  # This endpoint id is used to make operation ids unique make sure not to duplicate
+BatchOperations = serializers.PaginatedResult(
+    "BatchOperations", serializers.BatchOperation
+)
+
+
+class BatchList(api.GenericAPIView):
+    pagination_class = type(
+        "CustomPagination", (LimitOffsetPagination,), dict(default_limit=20)
+    )
+    filter_backends = (DjangoFilterBackend,)
+    filterset_class = BatchOperationFilter
+    serializer_class = BatchOperations
+    model = models.BatchOperation
+
+    @openapi.extend_schema(
+        tags=["Batches"],
+        operation_id=f"{ENDPOINT_ID}list",
+        extensions={"x-operationId": "listBatchOperations"},
+        summary="List all batch operations",
+        responses={
+            200: BatchOperations(),
+            404: serializers.ErrorResponse(),
+            500: serializers.ErrorResponse(),
+        },
+    )
+    def get(self, _: Request):
+        """Retrieve all batch operations. `Beta`"""
+
+        batches = self.filter_queryset(self.get_queryset())
+        response = self.paginate_queryset(
+            serializers.BatchOperation(batches, many=True).data
+        )
+
+        return self.get_paginated_response(response)
+
+
+class BatchDetails(api.APIView):
+    @openapi.extend_schema(
+        tags=["Batches"],
+        operation_id=f"{ENDPOINT_ID}retrieve",
+        extensions={"x-operationId": "retrieveBatchOperation"},
+        summary="Retrieve a batch operation",
+        responses={
+            200: serializers.BatchOperation(),
+            404: serializers.ErrorResponse(),
+            500: serializers.ErrorResponse(),
+        },
+    )
+    def get(self, request: Request, pk: str):
+        """Retrieve a batch operation. `Beta`"""
+        batch = models.BatchOperation.access_by(request).get(pk=pk)
+
+        return Response(serializers.BatchOperation(batch).data)
+
+
+urlpatterns = [
+    path("batches/operations", BatchList.as_view(), name="batch-list"),
+    path("batches/operations/<str:pk>", BatchDetails.as_view(), name="batch-details"),
+]

karrio/server/data/views/batch_order.py

@@ -0,0 +1,40 @@
+from django.urls import path
+from rest_framework import status
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+import karrio.server.data.serializers as serializers
+import karrio.server.core.views.api as api
+import karrio.server.openapi as openapi
+
+ENDPOINT_ID = "&&&&$"  # This endpoint id is used to make operation ids unique make sure not to duplicate
+
+
+class BatchList(api.APIView):
+    @openapi.extend_schema(
+        tags=["Batches"],
+        operation_id=f"{ENDPOINT_ID}create_orders",
+        summary="Create order batch",
+        responses={
+            200: serializers.BatchOperation(),
+            404: serializers.ErrorResponse(),
+            500: serializers.ErrorResponse(),
+        },
+        request=serializers.BatchOrderData(),
+    )
+    def post(self, request: Request):
+        """Create order batch. `Beta`"""
+        operation = (
+            serializers.BatchOrderData.map(data=request.data, context=request)
+            .save()
+            .instance
+        )
+
+        return Response(
+            serializers.BatchOperation(operation).data, status=status.HTTP_201_CREATED
+        )
+
+
+urlpatterns = [
+    path("batches/orders", BatchList.as_view(), name="batch-orders"),
+]

karrio/server/data/views/batch_shipment.py

@@ -0,0 +1,40 @@
+from django.urls import path
+from rest_framework import status
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+import karrio.server.data.serializers as serializers
+import karrio.server.core.views.api as api
+import karrio.server.openapi as openapi
+
+ENDPOINT_ID = "&&&&$"  # This endpoint id is used to make operation ids unique make sure not to duplicate
+
+
+class BatchList(api.APIView):
+    @openapi.extend_schema(
+        tags=["Batches"],
+        operation_id=f"{ENDPOINT_ID}create_shipments",
+        summary="Create shipment batch",
+        responses={
+            200: serializers.BatchOperation(),
+            404: serializers.ErrorResponse(),
+            500: serializers.ErrorResponse(),
+        },
+        request=serializers.BatchShipmentData(),
+    )
+    def post(self, request: Request):
+        """Create shipment batch. `Beta`"""
+        operation = (
+            serializers.BatchShipmentData.map(data=request.data, context=request)
+            .save()
+            .instance
+        )
+
+        return Response(
+            serializers.BatchOperation(operation).data, status=status.HTTP_201_CREATED
+        )
+
+
+urlpatterns = [
+    path("batches/shipments", BatchList.as_view(), name="batch-shipments"),
+]

karrio/server/data/views/batch_tracking.py

@@ -0,0 +1,40 @@
+from django.urls import path
+from rest_framework import status
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+import karrio.server.data.serializers as serializers
+import karrio.server.core.views.api as api
+import karrio.server.openapi as openapi
+
+ENDPOINT_ID = "&&&&$"  # This endpoint id is used to make operation ids unique make sure not to duplicate
+
+
+class BatchList(api.APIView):
+    @openapi.extend_schema(
+        tags=["Batches"],
+        operation_id=f"{ENDPOINT_ID}create_trackers",
+        summary="Create tracker batch",
+        responses={
+            200: serializers.BatchOperation(),
+            404: serializers.ErrorResponse(),
+            500: serializers.ErrorResponse(),
+        },
+        request=serializers.BatchTrackerData(),
+    )
+    def post(self, request: Request):
+        """Create tracker batch. `Beta`"""
+        operation = (
+            serializers.BatchTrackerData.map(data=request.data, context=request)
+            .save()
+            .instance
+        )
+
+        return Response(
+            serializers.BatchOperation(operation).data, status=status.HTTP_201_CREATED
+        )
+
+
+urlpatterns = [
+    path("batches/trackers", BatchList.as_view(), name="batch-trackers"),
+]

karrio/server/data/views/data.py

@@ -0,0 +1,171 @@
+import io
+from django.http import JsonResponse
+from django.urls import re_path, path
+from django.core.files.base import ContentFile
+from django_downloadview import VirtualDownloadView
+from django.views.decorators.csrf import csrf_exempt
+from rest_framework.parsers import MultiPartParser, FormParser
+from rest_framework.response import Response
+from rest_framework.request import Request
+from rest_framework import status
+
+from karrio.server.data.serializers.data import ImportDataSerializer
+import karrio.server.data.serializers as serializers
+import karrio.server.data.resources as resources
+import karrio.server.core.views.api as api
+import karrio.server.openapi as openapi
+
+ENDPOINT_ID = "&&&&$"  # This endpoint id is used to make operation ids unique make sure not to duplicate
+DataImportParameters: list = [
+    openapi.OpenApiParameter(
+        name="resource_type",
+        type=openapi.OpenApiTypes.STR,
+        enum=[e.name for e in list(serializers.ResourceType)],
+        description="The type of the resource to import",
+    ),
+    openapi.OpenApiParameter(
+        "data_template",
+        type=openapi.OpenApiTypes.STR,
+        required=False,
+        description="""A data template slug to use for the import.<br/>
+        **When nothing is specified, the system default headers are expected.**
+        """,
+    ),
+    openapi.OpenApiParameter(
+        name="data_file",
+        type=openapi.OpenApiTypes.BINARY,
+    ),
+]
+
+
+class DataImport(api.BaseAPIView):
+    parser_classes = [MultiPartParser, FormParser]
+
+    @openapi.extend_schema(
+        tags=["Batches"],
+        operation_id=f"{ENDPOINT_ID}import_file",
+        summary="Import data files",
+        responses={
+            202: serializers.BatchOperation(),
+            400: serializers.ErrorResponse(),
+            500: serializers.ErrorResponse(),
+        },
+        request={
+            'multipart/form-data': {
+                'type': 'object',
+                'properties': {
+                    'resource_type': {
+                        'type': 'string',
+                    },
+                    'data_template': {
+                        'type': 'string',
+                    },
+                    'data_file': {
+                        'type': 'string',
+                        'format': 'binary'
+                    }
+                }
+            }
+        },
+        parameters=DataImportParameters,
+    )
+    def post(self, request: Request):
+        """Import csv, xls and xlsx data files for: `Beta`<br/>
+        - trackers data
+        - orders data
+        - shipments data
+        - billing data (soon)<br/><br/>
+        **This operation will return a batch operation that you can poll to follow
+        the import progression.**
+        """
+        operation = (
+            ImportDataSerializer.map(data=request.data, context=request)
+            .save()
+            .instance
+        )
+
+        return Response(
+            serializers.BatchOperation(operation).data, status=status.HTTP_202_ACCEPTED
+        )
+
+
+DataExportParameters: list = [
+    openapi.OpenApiParameter(
+        name="resource_type",
+        location=openapi.OpenApiParameter.PATH,
+        type=openapi.OpenApiTypes.STR,
+        enum=[e.name for e in list(serializers.ResourceType)],
+    ),
+    openapi.OpenApiParameter(
+        name="export_format",
+        location=openapi.OpenApiParameter.PATH,
+        type=openapi.OpenApiTypes.STR,
+        enum=[e.name for e in list(serializers.ResourceType)],
+    ),
+    openapi.OpenApiParameter(
+        "data_template",
+        location=openapi.OpenApiParameter.QUERY,
+        type=openapi.OpenApiTypes.STR,
+        required=False,
+        description="""A data template slug to use for the import.<br/>
+        **When nothing is specified, the system default headers are expected.**
+        """,
+    ),
+]
+
+
+class DataExport(api.LoginRequiredView, VirtualDownloadView):
+    @openapi.extend_schema(
+        tags=["Batches"],
+        operation_id=f"{ENDPOINT_ID}export_file",
+        summary="Export data files",
+        responses={
+            (200, "application/octet-stream"): openapi.OpenApiTypes.BINARY,
+            409: serializers.ErrorResponse(),
+            500: serializers.ErrorResponse(),
+        },
+        parameters=DataExportParameters,
+    )
+    def get(
+        self,
+        request: Request,
+        resource_type: str = "orders",
+        export_format: str = "csv",
+        **kwargs,
+    ):
+        try:
+            """Generate a file to export."""
+            query_params = request.GET
+            self.attachment = "download" in query_params
+            self.resource = resource_type
+            self.format = export_format
+
+            self.dataset = resources.export(
+                resource_type, query_params, context=request
+            )
+
+            response = super(DataExport, self).get(request, **kwargs)
+            response["X-Frame-Options"] = "ALLOWALL"
+            return response
+        except Exception as e:
+            return JsonResponse(
+                dict(errors=[{"message": str(e)}]),
+                status=status.HTTP_409_CONFLICT,
+            )
+
+    def get_file(self):
+        content = getattr(self.dataset, self.format, "")
+        buffer = io.StringIO() if type(content) == str else io.BytesIO()
+        buffer.write(content)
+
+        return ContentFile(buffer.getvalue(), name=f"{self.resource}.{self.format}")
+
+
+urlpatterns = [
+    path("batches/data/import", DataImport.as_view(), name="data-import"),
+    re_path(
+        r"^batches/data/export/(?P<resource_type>\w+).(?P<export_format>\w+)",
+        csrf_exempt(DataExport.as_view()),
+        name="data-export",
+    ),
+]

karrio/server/events/task_definitions/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore

karrio/server/events/task_definitions/data/__init__.py

@@ -0,0 +1,136 @@
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore
+
+import typing
+import logging
+from huey.contrib.djhuey import db_task
+
+import karrio.server.core.utils as utils
+import karrio.server.data.models as models
+import karrio.server.data.serializers as serializers
+
+logger = logging.getLogger(__name__)
+
+
+@db_task()
+@utils.error_wrapper
+@utils.tenant_aware
+def queue_batch_import(*args, **kwargs):
+    from karrio.server.events.task_definitions.data import batch
+
+    batch.trigger_batch_import(*args, **kwargs)
+
+
+@db_task()
+@utils.error_wrapper
+@utils.tenant_aware
+def save_batch_resources(*args, **kwargs):
+    from karrio.server.events.task_definitions.data import batch
+
+    batch.trigger_batch_saving(*args, **kwargs)
+
+
+@db_task()
+@utils.tenant_aware
+def process_batch_resources(batch_id, **kwargs):
+    logger.info(f"> start batch ({batch_id}) resources processing...")
+    try:
+        batch_operation = models.BatchOperation.objects.get(pk=batch_id)
+
+        if batch_operation.resource_type == serializers.ResourceType.trackers.value:
+            batch_operation.resources = _process_trackers(batch_operation.resources)
+
+        elif batch_operation.resource_type == serializers.ResourceType.order.value:
+            batch_operation.resources = _process_orders(batch_operation.resources)
+
+        elif batch_operation.resource_type == serializers.ResourceType.shipment.value:
+            batch_operation.resources = _process_shipments(batch_operation.resources)
+
+        elif batch_operation.resource_type == serializers.ResourceType.billing.value:
+            pass
+
+        batch_operation.status = serializers.BatchOperationStatus.completed.value
+        batch_operation.save(update_fields=["resources", "status"])
+    except Exception as e:
+        logger.exception(e)
+        batch_operation.status = serializers.BatchOperationStatus.failed.value
+        batch_operation.save()
+
+    logger.info(f"> ending batch ({batch_id}) resources processing...")
+
+
+def _process_shipments(resources: typing.List[dict]):
+    from karrio.server.manager import models
+    from karrio.server.events.task_definitions.data import shipments
+
+    resource_ids = [res["id"] for res in resources]
+    shipment_ids = [
+        res["id"]
+        for res in resources
+        if res["status"] != serializers.ResourceStatus.processed.value
+    ]
+    shipments.process_shipments(shipment_ids=shipment_ids)
+    # check results and update resource statuses
+    results = models.Shipment.objects.filter(id__in=resource_ids)
+
+    def _compute_state(shipment=None):
+        # shipment with service not purchased
+        if (
+            any(shipment.options.get("perferred_service") or "")
+            and shipment.status == "draft"
+        ):
+            return serializers.ResourceStatus.incomplete
+        # shipment has errors and no rates
+        if len(shipment.rates) == 0 and any(shipment.messages):
+            return serializers.ResourceStatus.has_errors
+        # shipment is at the right state
+        return serializers.ResourceStatus.processed.value
+
+    return [
+        dict(
+            id=res["id"],
+            status=_compute_state(results.filter(id=res["id"]).first()),
+        )
+        for res in resources
+    ]
+
+
+def _process_orders(resources: typing.List[dict]):
+    resource_ids = [res["id"] for res in resources]
+
+    return [dict(id=id, status="processed") for id in resource_ids]
+
+
+def _process_trackers(resources: typing.List[dict]):
+    from karrio.server.manager import models
+    from karrio.server.events.task_definitions.base import tracking
+
+    resource_ids = [res["id"] for res in resources]
+    tracker_ids = [
+        res["id"]
+        for res in resources
+        if res["status"] != serializers.ResourceStatus.processed.value
+    ]
+    tracking.update_trackers(tracker_ids=tracker_ids)
+    # check results and update resource statuses
+    results = models.Tracking.objects.filter(id__in=resource_ids)
+
+    def _compute_tracker_state(tracker=None):
+        if tracker is None:
+            return serializers.ResourceStatus.incomplete.value
+
+        return serializers.ResourceStatus.processed.value
+
+    return [
+        dict(
+            id=res["id"],
+            status=_compute_tracker_state(results.filter(id=res["id"]).first()),
+        )
+        for res in resources
+    ]
+
+
+TASK_DEFINITIONS = [
+    queue_batch_import,
+    save_batch_resources,
+    process_batch_resources,
+]

karrio/server/events/task_definitions/data/batch.py

@@ -0,0 +1,130 @@
+import typing
+import tablib
+import logging
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from import_export.resources import ModelResource
+
+import karrio.server.core.utils as utils
+import karrio.server.data.serializers as serializers
+import karrio.server.data.resources as resources
+import karrio.server.data.models as models
+
+logger = logging.getLogger(__name__)
+User = get_user_model()
+
+
+@utils.tenant_aware
+def trigger_batch_import(
+    batch_id: str,
+    data: dict,
+    ctx: dict,
+    **kwargs,
+):
+    logger.info(f"> starting batch import operation ({batch_id})")
+    try:
+        context = retrieve_context(ctx)
+        batch_operation = (
+            models.BatchOperation.access_by(context).filter(pk=batch_id).first()
+        )
+
+        if batch_operation is not None:
+            dataset = data["dataset"]
+            import_data = data["import_data"]
+            resource = resources.get_import_resource(
+                resource_type=batch_operation.resource_type,
+                params=import_data,
+                context=context,
+                batch_id=batch_id,
+            )
+
+            batch_resources = process_resources(resource, dataset)
+            update_batch_operation_resources(batch_operation, batch_resources)
+        else:
+            logger.info("batch operation not found")
+
+    except Exception as e:
+        logger.exception(e)
+
+    logger.info(f"> ending batch import operation ({batch_id})")
+
+
+@utils.tenant_aware
+def trigger_batch_saving(
+    batch_id: str,
+    data: dict,
+    ctx: dict,
+    **kwargs,
+):
+    logger.info(f"> beging batch resources saving ({batch_id})")
+    try:
+        context = retrieve_context(ctx)
+        batch_operation = (
+            models.BatchOperation.access_by(context).filter(pk=batch_id).first()
+        )
+
+        if batch_operation is not None:
+            batch_seriazlizer = serializers.ResourceType.get_serialiazer(
+                batch_operation.resource_type
+            )
+            batch_resources = batch_seriazlizer.save_resources(data, batch_id, context)
+            update_batch_operation_resources(batch_operation, batch_resources)
+        else:
+            logger.info("batch operation not found")
+
+    except Exception as e:
+        logger.exception(e)
+
+    logger.info(f"> ending batch resources saving ({batch_id})")
+
+
+def process_resources(
+    resource: ModelResource,
+    dataset: tablib.Dataset,
+):
+    result = resource.import_data(dataset, dry_run=False)
+    _object_ids = [(row.object_id, row.errors) for row in result.rows]
+
+    return [
+        dict(
+            id=id,
+            status=(
+                serializers.ResourceStatus.failed.value
+                if any(errors)
+                else serializers.ResourceStatus.queued.value
+            ),
+        )
+        for id, errors in _object_ids
+    ]
+
+
+def update_batch_operation_resources(
+    batch_operation: models.BatchOperation,
+    batch_resources: typing.List[dict],
+):
+    try:
+        logger.debug(f"update batch operation {batch_operation.id}")
+
+        batch_operation.resources = batch_resources
+        batch_operation.status = serializers.BatchOperationStatus.running.value
+        batch_operation.save(update_fields=["resources", "status"])
+
+        logger.debug(f"batch operation {batch_operation.id} updated successfully")
+    except Exception as update_error:
+        logger.warning(f"failed to update batch operation {batch_operation.id}")
+        logger.error(update_error, exc_info=True)
+
+
+def retrieve_context(info: dict) -> serializers.Context:
+    org = None
+
+    if settings.MULTI_ORGANIZATIONS and "org_id" in info:
+        import karrio.server.orgs.models as orgs_models
+
+        org = orgs_models.Organization.objects.filter(id=info["org_id"]).first()
+
+    return serializers.Context(
+        org=org,
+        user=User.objects.filter(id=info["user_id"]).first(),
+        test_mode=(info.get("test_mode") or False),
+    )

karrio/server/events/task_definitions/data/shipments.py

@@ -0,0 +1,51 @@
+import logging
+
+import karrio.server.core.utils as utils
+import karrio.server.manager.models as models
+import karrio.server.serializers as serializers
+from karrio.server.manager.serializers import (
+    fetch_shipment_rates,
+    can_mutate_shipment,
+    buy_shipment_label,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@utils.error_wrapper
+def process_shipments(shipment_ids=[]):
+    logger.info("> starting batch shipments processing")
+
+    shipments = models.Shipment.objects.filter(id__in=shipment_ids, status="draft")
+
+    if any(shipments):
+        for shipment in shipments:
+            process_shipment(shipment)
+    else:
+        logger.info("no shipment found")
+
+    logger.info("> ending batch shipments processing")
+
+
+@utils.error_wrapper
+def process_shipment(shipment):
+    preferred_service = shipment.options.get("preferred_service")
+    should_purchase = any(preferred_service or "")
+    should_update = should_purchase or len(shipment.rates) == 0
+    context = serializers.get_object_context(shipment)
+
+    can_mutate_shipment(
+        shipment,
+        update=should_update,
+        purchase=should_purchase,
+    )
+
+    if len(shipment.rates) == 0:
+        shipment = fetch_shipment_rates(shipment, context=context)
+
+    if should_purchase:
+        shipment = buy_shipment_label(
+            shipment,
+            context=context,
+            service=preferred_service,
+        )

karrio/server/graph/schemas/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore