mpt-extension-sdk 4.5.0__py3-none-any.whl

mpt_extension_sdk/runtime/djapp/apps.py

@@ -0,0 +1,46 @@
+from django.apps import AppConfig
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+
+from mpt_extension_sdk.core.extension import Extension
+
+ext = Extension()
+
+
+class DjAppConfig(AppConfig):
+    name = "mpt_extension_sdk.runtime.djapp"
+
+    def ready(self):
+        if not hasattr(settings, "MPT_PRODUCTS_IDS") or not settings.MPT_PRODUCTS_IDS:
+            raise ImproperlyConfigured(
+                f"Extension {self.verbose_name} is not properly configured."
+                "MPT_PRODUCTS_IDS is missing or empty."
+            )
+
+        self.extension_ready()
+
+    def extension_ready(self):
+        pass
+
+
+class ExtensionConfig(DjAppConfig):
+    name = "mpt_extension_sdk"
+    verbose_name = "SWO Extension SDK"
+    extension = ext
+
+    def extension_ready(self):
+        error_msgs = []
+
+        for product_id in settings.MPT_PRODUCTS_IDS:
+            if (
+                "WEBHOOKS_SECRETS" not in settings.EXTENSION_CONFIG
+                or product_id not in settings.EXTENSION_CONFIG["WEBHOOKS_SECRETS"]
+            ):
+                msg = (
+                    f"The webhook secret for {product_id} is not found. "
+                    f"Please, specify it in EXT_WEBHOOKS_SECRETS environment variable."
+                )
+                error_msgs.append(msg)
+
+        if error_msgs:
+            raise ImproperlyConfigured("\n".join(error_msgs))

mpt_extension_sdk/runtime/djapp/conf/__init__.py

@@ -0,0 +1,12 @@
+from typing import Any
+
+
+def extract_product_ids(product_ids: str) -> list[str]:
+    return product_ids.split(",")
+
+
+def get_for_product(settings, variable_name: str, product_id: str) -> Any:
+    """
+    A shortcut to return product scoped variable from the extension settings.
+    """
+    return settings.EXTENSION_CONFIG[variable_name][product_id]

mpt_extension_sdk/runtime/djapp/conf/default.py

@@ -0,0 +1,225 @@
+"""
+Django settings for pippo project.
+
+Generated by 'django-admin startproject' using Django 4.2.8.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/4.2/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/4.2/ref/settings/
+"""
+
+import os
+from pathlib import Path
+
+from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = os.getenv(
+    "MPT_DJANGO_SECRET_KEY",
+    "",
+)
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ["*"]
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
+    "mpt_extension_sdk.runtime.djapp.apps.DjAppConfig",
+]
+
+MIDDLEWARE = [
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+    "mpt_extension_sdk.runtime.djapp.middleware.MPTClientMiddleware",
+]
+
+ROOT_URLCONF = "mpt_extension_sdk.runtime.djapp.conf.urls"
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+            ],
+        },
+    },
+]
+
+
+# Database
+# https://docs.djangoproject.com/en/4.2/ref/settings/#databases
+
+DATABASES = {}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/4.2/topics/i18n/
+
+LANGUAGE_CODE = "en-us"
+
+TIME_ZONE = "UTC"
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/4.2/howto/static-files/
+
+STATIC_URL = "static/"
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
+# OpenTelemetry configuration
+APPLICATIONINSIGHTS_CONNECTION_STRING = os.getenv(
+    "APPLICATIONINSIGHTS_CONNECTION_STRING", ""
+)
+USE_APPLICATIONINSIGHTS = APPLICATIONINSIGHTS_CONNECTION_STRING != ""
+
+
+if USE_APPLICATIONINSIGHTS:  # pragma: no cover
+    logger_provider = LoggerProvider()
+    set_logger_provider(logger_provider)
+    exporter = AzureMonitorLogExporter(
+        connection_string=APPLICATIONINSIGHTS_CONNECTION_STRING
+    )
+    logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "verbose": {
+            "format": "{asctime} {name} {levelname} (pid: {process}, thread: {thread}) {message}",
+            "style": "{",
+        },
+        "rich": {
+            "format": "{message}",
+            "style": "{",
+        },
+        "opentelemetry": {
+            "format": "(pid: {process}, thread: {thread}) {message}",
+            "style": "{",
+        },
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "verbose",
+        },
+        "rich": {
+            "class": "mpt_extension_sdk.runtime.logging.RichHandler",
+            "formatter": "rich",
+            "log_time_format": lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3],
+            "rich_tracebacks": True,
+        },
+        "opentelemetry": {
+            "class": "opentelemetry.sdk._logs.LoggingHandler",
+            "formatter": "opentelemetry",
+        },
+    },
+    "root": {
+        "handlers": ["rich"],
+        "level": "WARNING",
+    },
+    "loggers": {
+        "django": {
+            "handlers": ["rich"],
+            "level": "INFO",
+            "propagate": False,
+        },
+        "swo.mpt": {
+            "handlers": ["rich"],
+            "level": "DEBUG",
+            "propagate": False,
+        },
+        "azure": {
+            "handlers": ["rich"],
+            "level": "WARNING",
+            "propagate": False,
+        },
+    },
+}
+
+# Proxy settings
+USE_X_FORWARDED_HOST = True
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+# MPT settings
+
+MPT_API_BASE_URL = os.getenv("MPT_API_BASE_URL", "http://localhost:8000")
+MPT_API_TOKEN = os.getenv("MPT_API_TOKEN", "change-me!")
+MPT_API_TOKEN_OPERATIONS = os.getenv("MPT_API_TOKEN_OPERATIONS", "change-me!")
+MPT_PRODUCTS_IDS = os.getenv("MPT_PRODUCTS_IDS", "PRD-1111-1111")
+MPT_PORTAL_BASE_URL = os.getenv("MPT_PORTAL_BASE_URL", "https://portal.s1.show")
+MPT_KEY_VAULT_NAME = os.getenv("MPT_KEY_VAULT_NAME", "mpt-key-vault")
+
+MPT_ORDERS_API_POLLING_INTERVAL_SECS = int(
+    os.getenv("MPT_ORDERS_API_POLLING_INTERVAL_SECS", "120")
+)
+
+EXTENSION_CONFIG = {
+    "DUE_DATE_DAYS": "30",
+    "ORDER_CREATION_WINDOW_HOURS": os.getenv("EXT_ORDER_CREATION_WINDOW_HOURS", "24"),
+}
+
+MPT_SETUP_CONTEXTS_FUNC = os.getenv(
+    "MPT_SETUP_CONTEXTS_FUNC",
+    "mpt_extension_sdk.runtime.events.utils.setup_contexts",
+)

mpt_extension_sdk/runtime/djapp/conf/urls.py

@@ -0,0 +1,9 @@
+from mpt_extension_sdk.constants import (
+    DEFAULT_APP_CONFIG_GROUP,
+    DEFAULT_APP_CONFIG_NAME,
+)
+from mpt_extension_sdk.runtime.utils import get_extension, get_urlpatterns
+
+extension = get_extension(name=DEFAULT_APP_CONFIG_NAME, group=DEFAULT_APP_CONFIG_GROUP)
+
+urlpatterns = get_urlpatterns(extension)

mpt_extension_sdk/runtime/djapp/management/commands/consume_events.py

@@ -0,0 +1,38 @@
+import signal  # pragma: no cover
+from threading import Event  # pragma: no cover
+
+from django.core.management.base import BaseCommand  # pragma: no cover
+
+from mpt_extension_sdk.constants import CONSUME_EVENTS_HELP_TEXT  # pragma: no cover
+from mpt_extension_sdk.runtime.events.dispatcher import Dispatcher  # pragma: no cover
+from mpt_extension_sdk.runtime.events.producers import (
+    OrderEventProducer,  # pragma: no cover
+)
+
+
+class Command(BaseCommand):  # pragma: no cover
+    help = CONSUME_EVENTS_HELP_TEXT
+    producer_classes = [
+        OrderEventProducer,
+    ]
+    producers = []
+
+    def handle(self, *args, **options):
+        self.shutdown_event = Event()
+        self.dispatcher = Dispatcher()
+        self.dispatcher.start()
+
+        def shutdown(signum, frame):
+            self.shutdown_event.set()
+
+        signal.signal(signal.SIGTERM, shutdown)
+        signal.signal(signal.SIGINT, shutdown)
+        for producer_cls in self.producer_classes:
+            producer = producer_cls(self.dispatcher)
+            self.producers.append(producer)
+            producer.start()
+
+        self.shutdown_event.wait()
+        for producer in self.producers:
+            producer.stop()
+        self.dispatcher.stop()

mpt_extension_sdk/runtime/djapp/middleware.py

@@ -0,0 +1,21 @@
+from django.conf import settings
+
+from mpt_extension_sdk.mpt_http.base import MPTClient
+
+_CLIENT = None
+
+
+class MPTClientMiddleware:  # pragma: no cover
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        global _CLIENT
+        if not _CLIENT:
+            _CLIENT = MPTClient(
+                f"{settings.MPT_API_BASE_URL}/v1/",
+                settings.MPT_API_TOKEN,
+            )
+        request.client = _CLIENT
+        response = self.get_response(request)
+        return response

mpt_extension_sdk/runtime/events/dispatcher.py

@@ -0,0 +1,83 @@
+import functools
+import logging
+import threading
+import time
+from collections import deque
+from concurrent.futures import ThreadPoolExecutor
+
+from mpt_extension_sdk.constants import (
+    DEFAULT_APP_CONFIG_GROUP,
+    DEFAULT_APP_CONFIG_NAME,
+)
+from mpt_extension_sdk.core.events.dataclasses import Event
+from mpt_extension_sdk.core.events.registry import EventsRegistry
+from mpt_extension_sdk.core.utils import setup_client
+from mpt_extension_sdk.runtime.events.utils import wrap_for_trace
+from mpt_extension_sdk.runtime.utils import get_events_registry
+
+logger = logging.getLogger(__name__)
+
+
+def done_callback(futures, key, future):  # pragma: no cover
+    del futures[key]
+    exc = future.exception()
+    if not exc:
+        logger.debug(f"Future for {key} has been completed successfully")
+        return
+    logger.error(f"Future for {key} has failed: {exc}")
+
+
+class Dispatcher:
+    def __init__(self, group=DEFAULT_APP_CONFIG_GROUP, name=DEFAULT_APP_CONFIG_NAME):
+        self.registry: EventsRegistry = get_events_registry(group=group, name=name)
+        self.queue = deque()
+        self.futures = {}
+        self.executor = ThreadPoolExecutor()
+        self.running_event = threading.Event()
+        self.processor = threading.Thread(target=self.process_events)
+        self.client = setup_client()
+
+    def start(self):
+        self.running_event.set()
+        self.processor.start()
+
+    def stop(self):
+        self.running_event.clear()
+        self.processor.join()
+
+    @property
+    def running(self):
+        return self.running_event.is_set()
+
+    def dispatch_event(self, event: Event):  # pragma: no cover
+        if self.registry.is_event_supported(event.type):
+            logger.info(f"event of type {event.type} with id {event.id} accepted")
+            self.queue.appendleft((event.type, event))
+
+    def process_events(self):  # pragma: no cover
+        while self.running:
+            skipped = []
+            while len(self.queue) > 0:
+                event_type, event = self.queue.pop()
+                logger.debug(
+                    f"got event of type {event_type} ({event.id}) from queue..."
+                )
+                listener = wrap_for_trace(
+                    self.registry.get_listener(event_type), event_type
+                )
+                if (event.type, event.id) not in self.futures:
+                    future = self.executor.submit(listener, self.client, event)
+                    self.futures[(event.type, event.id)] = future
+                    future.add_done_callback(
+                        functools.partial(
+                            done_callback, self.futures, (event.type, event.id)
+                        )
+                    )
+                else:
+                    logger.info(
+                        f"An event for {(event.type, event.id)} is already processing, skip it"
+                    )
+                    skipped.append((event.type, event))
+
+            self.queue.extendleft(skipped)
+            time.sleep(0.5)

mpt_extension_sdk/runtime/events/producers.py

@@ -0,0 +1,108 @@
+import logging
+import threading
+import time
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+
+import requests
+from django.conf import settings
+
+from mpt_extension_sdk.core.events.dataclasses import Event
+from mpt_extension_sdk.core.utils import setup_client
+
+logger = logging.getLogger(__name__)
+
+
+class EventProducer(ABC):
+    def __init__(self, dispatcher):
+        self.dispatcher = dispatcher
+        self.running_event = threading.Event()
+        self.producer = threading.Thread(target=self.produce_events)
+
+    @property
+    def running(self):
+        return self.running_event.is_set()
+
+    def start(self):
+        self.running_event.set()
+        self.producer.start()
+
+    def stop(self):
+        self.running_event.clear()
+        self.producer.join()
+
+    @contextmanager
+    def sleep(self, secs, interval=0.5):  # pragma: no cover
+        yield
+        sleeped = 0
+        while sleeped < secs and self.running_event.is_set():
+            time.sleep(interval)
+            sleeped += interval
+
+    @abstractmethod
+    def produce_events(self):
+        pass
+
+    @abstractmethod
+    def produce_events_with_context(self):
+        pass
+
+
+class OrderEventProducer(EventProducer):
+    def __init__(self, dispatcher):
+        super().__init__(dispatcher)
+        self.client = setup_client()
+
+    def produce_events(self):
+        while self.running:
+            with self.sleep(settings.MPT_ORDERS_API_POLLING_INTERVAL_SECS):
+                orders = self.get_processing_orders()
+                logger.info(f"{len(orders)} orders found for processing...")
+                for order in orders:
+                    self.dispatcher.dispatch_event(Event(order["id"], "orders", order))
+
+    def produce_events_with_context(self):  # pragma: no cover
+        while self.running:
+            with self.sleep(settings.MPT_ORDERS_API_POLLING_INTERVAL_SECS):
+                orders = self.get_processing_orders()
+                orders, contexts = self.filter_and_enrich(self.client, orders)
+                logger.info(f"{len(orders)} orders found for processing...")
+                for order, context in zip(orders, contexts):
+                    self.dispatcher.dispatch_event(
+                        Event(order["id"], "orders", order, context)
+                    )
+
+    def get_processing_orders(self):
+        products = ",".join(settings.MPT_PRODUCTS_IDS)
+        orders = []
+        rql_query = f"and(in(agreement.product.id,({products})),eq(status,processing))"
+        url = (
+            f"/commerce/orders?{rql_query}&select=audit,parameters,lines,subscriptions,"
+            f"subscriptions.lines,agreement,buyer,seller&order=audit.created.at"
+        )
+        page = None
+        limit = 10
+        offset = 0
+        while self.has_more_pages(page):
+            try:
+                response = self.client.get(f"{url}&limit={limit}&offset={offset}")
+            except requests.RequestException:
+                logger.exception("Cannot retrieve orders")
+                return []
+            if response.status_code == 200:
+                page = response.json()
+                orders.extend(page["data"])
+            else:
+                logger.warning(
+                    f"Order API error: {response.status_code} {response.content}"
+                )
+                return []
+            offset += limit
+
+        return orders
+
+    def has_more_pages(self, orders):
+        if not orders:
+            return True
+        pagination = orders["$meta"]["pagination"]
+        return pagination["total"] > pagination["limit"] + pagination["offset"]

mpt_extension_sdk/runtime/events/utils.py

@@ -0,0 +1,85 @@
+import logging
+from functools import wraps
+
+from azure.monitor.opentelemetry.exporter import (
+    AzureMonitorTraceExporter,
+)
+from django.conf import settings
+from opentelemetry import trace
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+from mpt_extension_sdk.flows.context import Context
+
+logger = logging.getLogger(__name__)
+
+
+def _response_hook(span, request, response):  # pragma: no cover
+    span.set_attribute(
+        "request.header.x-correlation-id",
+        request.headers.get("x-correlation-id", ""),
+    )
+    span.set_attribute(
+        "request.header.x-request-id",
+        request.headers.get("x-request-id", ""),
+    )
+    if not response.ok:
+        span.set_attribute("request.body", request.body or "")
+        span.set_attribute("response.body", response.content or "")
+
+
+def instrument_logging():  # pragma: no cover
+    exporter = AzureMonitorTraceExporter(
+        connection_string=settings.APPLICATIONINSIGHTS_CONNECTION_STRING
+    )
+
+    trace_provider = TracerProvider()
+    trace_provider.add_span_processor(BatchSpanProcessor(exporter))
+    trace.set_tracer_provider(trace_provider)
+
+    DjangoInstrumentor().instrument()
+    RequestsInstrumentor().instrument(response_hook=_response_hook)
+    LoggingInstrumentor().instrument(set_logging_format=True)
+
+
+def wrap_for_trace(func, event_type):  # pragma: no cover
+    @wraps(func)
+    def opentelemetry_wrapper(client, event):
+        tracer = trace.get_tracer(event_type)
+        object_id = event.id
+
+        with tracer.start_as_current_span(
+            f"Event {event_type} for {object_id}"
+        ) as span:
+            try:
+                func(client, event)
+            except Exception:
+                logger.exception("Unhandled exception!")
+            finally:
+                if span.is_recording():
+                    span.set_attribute("order.id", object_id)
+
+    @wraps(func)
+    def wrapper(client, event):
+        try:
+            func(client, event)
+        except Exception:
+            logger.exception("Unhandled exception!")
+
+    return opentelemetry_wrapper if settings.USE_APPLICATIONINSIGHTS else wrapper
+
+
+def setup_contexts(mpt_client, orders):
+    """
+    List of contexts from orders
+    Args:
+        mpt_client (MPTClient): MPT client
+        orders (list): List of orders
+
+    Returns: List of contexts
+
+    """
+    return [Context(order=order) for order in orders]