stapel-core 0.3.1__py3-none-any.whl

stapel_core/__init__.py

@@ -0,0 +1,95 @@
+"""Stapel Core — shared Django utilities for the Stapel framework.
+
+The building blocks every Stapel package sits on:
+
+- ``comm`` — Action/Function inter-module communication (``emit``,
+  ``on_action``, ``call``, ``function``) plus long-running tasks
+  (``start``, ``status``, ``task_handler``). Transports are deployment
+  configuration, not code.
+- ``bus`` — transport-agnostic message bus (``publish``, ``get_bus``,
+  ``Event``) with Kafka/NATS/in-memory backends.
+- ``conf.AppSettings`` — per-app settings namespaces (the DRF
+  ``api_settings`` pattern, generalized).
+- ``signals`` — in-process Django signals for business milestones.
+- ``django.api`` — API conventions: ``StapelResponse``,
+  ``StapelErrorResponse`` and ``StapelDataclassSerializer``.
+- ``gdpr`` — GDPR provider protocol and in-process registry.
+- ``django.users.AbstractStapelUser`` — base user model.
+- ``core`` — framework-agnostic JWT primitives (``JWTHandler``,
+  ``TokenManager``, ``TokenBlacklist``, ``JWTConfig``).
+
+All attributes are exported lazily (PEP 562), so importing
+``stapel_core`` stays cheap and never touches Django until a
+Django-dependent attribute is actually used.
+"""
+
+from importlib import import_module
+
+try:  # single source of truth: pyproject's version via package metadata
+    from importlib.metadata import version as _pkg_version
+
+    __version__ = _pkg_version("stapel-core")
+except Exception:  # editable/vendored checkout without dist-info
+    __version__ = "0.3.0"
+
+# Attribute name -> (relative module, attribute in that module).
+# An attribute of None means the module itself is the export.
+_LAZY_EXPORTS = {
+    # comm — Actions, Functions, long-running tasks
+    "emit": (".comm", "emit"),
+    "on_action": (".comm", "on_action"),
+    "call": (".comm", "call"),
+    "function": (".comm", "function"),
+    "start": (".comm", "start"),
+    "status": (".comm", "status"),
+    "task_handler": (".comm", "task_handler"),
+    # flows — self-documenting business scenarios
+    "Flow": (".flows", "Flow"),
+    "flow_step": (".flows", "flow_step"),
+    "flow_registry": (".flows", "flow_registry"),
+    # verification — step-up (OTP/TOTP/passkey) on any endpoint
+    "requires_verification": (".verification", "requires_verification"),
+    "register_factor": (".verification", "register_factor"),
+    "VerificationFactor": (".verification", "VerificationFactor"),
+    "get_user_policy": (".verification", "get_user_policy"),
+    "invalidate_policy_cache": (".verification", "invalidate_policy_cache"),
+    # bus — transport-agnostic message bus
+    "publish": (".bus", "publish"),
+    "get_bus": (".bus", "get_bus"),
+    "Event": (".bus", "Event"),
+    # conf — per-app settings namespaces
+    "AppSettings": (".conf", "AppSettings"),
+    # signals — in-process business milestones (module export)
+    "signals": (".signals", None),
+    # API conventions — responses, errors, serializers
+    "StapelResponse": (".django.api.errors", "StapelResponse"),
+    "StapelErrorResponse": (".django.api.errors", "StapelErrorResponse"),
+    "StapelDataclassSerializer": (".django.api.serializers", "StapelDataclassSerializer"),
+    # GDPR — provider protocol + in-process registry
+    "GDPRProvider": (".gdpr", "GDPRProvider"),
+    "gdpr_registry": (".gdpr", "gdpr_registry"),
+    # Users — base user model
+    "AbstractStapelUser": (".django.users.models", "AbstractStapelUser"),
+    # Framework-agnostic JWT primitives (0.1.x root exports, kept stable)
+    "JWTHandler": (".core.jwt_handler", "JWTHandler"),
+    "TokenManager": (".core.token_manager", "TokenManager"),
+    "TokenBlacklist": (".core.token_blacklist", "TokenBlacklist"),
+    "JWTConfig": (".core.config", "JWTConfig"),
+}
+
+__all__ = sorted([*_LAZY_EXPORTS, "__version__"])
+
+
+def __getattr__(name):
+    try:
+        module_path, attr = _LAZY_EXPORTS[name]
+    except KeyError:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
+    module = import_module(module_path, __name__)
+    value = module if attr is None else getattr(module, attr)
+    globals()[name] = value  # cache so __getattr__ runs once per name
+    return value
+
+
+def __dir__():
+    return sorted(set(globals()) | set(_LAZY_EXPORTS))

stapel_core/bus/__init__.py

@@ -0,0 +1,35 @@
+"""
+stapel_core.bus — transport-agnostic message bus.
+
+Public API:
+    publish(topic, event)         — send an event
+    get_bus()                     — get the configured backend instance
+    reset_bus()                   — force re-init (tests)
+    Event                         — message envelope dataclass
+    BusBackend                    — ABC for custom backends
+    BaseBusConsumerCommand        — base Django management command for consumers
+
+Backend is set via Django setting:
+    STAPEL_BUS_BACKEND = "stapel_core.bus.backends.kafka.KafkaBus"   # default/prod
+    STAPEL_BUS_BACKEND = "stapel_core.bus.backends.memory.MemoryBus"  # tests/dev
+"""
+
+from .base import BusBackend
+from .consumer import BaseBusConsumerCommand
+from .event import Event
+from .router import get_bus, reset_bus
+
+
+def publish(topic: str, event: Event) -> None:
+    """Publish *event* to *topic* via the configured backend."""
+    get_bus().publish(topic, event)
+
+
+__all__ = [
+    "publish",
+    "get_bus",
+    "reset_bus",
+    "Event",
+    "BusBackend",
+    "BaseBusConsumerCommand",
+]

stapel_core/bus/_config.py

@@ -0,0 +1,81 @@
+"""
+Connection settings for bus backends.
+
+Resolution order for every value: environment variable first, then the
+Django setting of the same name, then the default — a deployment switches
+transports and endpoints purely through the environment (12-factor), while
+tests keep configuring Django settings.
+"""
+from __future__ import annotations
+
+import os
+
+
+def _get(name: str, default: str = "") -> str:
+    value = os.environ.get(name)
+    if value:
+        return value
+    try:
+        from django.conf import settings
+
+        return getattr(settings, name, default) or default
+    except Exception:  # settings not configured (plain scripts)
+        return default
+
+
+class KafkaBusConfig:
+    @staticmethod
+    def _base() -> dict:
+        cfg: dict = {
+            "bootstrap.servers": _get("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092"),
+        }
+        protocol = _get("KAFKA_SECURITY_PROTOCOL", "PLAINTEXT")
+        if protocol != "PLAINTEXT":
+            cfg["security.protocol"] = protocol
+        mechanism = _get("KAFKA_SASL_MECHANISM", "")
+        if mechanism:
+            cfg["sasl.mechanism"] = mechanism
+            cfg["sasl.username"] = _get("KAFKA_SASL_USERNAME", "")
+            cfg["sasl.password"] = _get("KAFKA_SASL_PASSWORD", "")
+        return cfg
+
+    @classmethod
+    def producer_config(cls) -> dict:
+        return {**cls._base(), "acks": "all"}
+
+    @classmethod
+    def consumer_config(cls, group: str) -> dict:
+        return {
+            **cls._base(),
+            "group.id": group,
+            "auto.offset.reset": "earliest",
+            "enable.auto.commit": False,
+        }
+
+
+class NatsBusConfig:
+    """JetStream backend configuration.
+
+    NATS_URL                 broker address        (nats://nats:4222)
+    STAPEL_NATS_STREAM       JetStream stream name (stapel-events)
+    STAPEL_NATS_EVENT_PREFIX subject prefix        (stapel.evt)
+
+    Every bus topic maps to the subject ``<prefix>.<topic>``; the stream
+    captures ``<prefix>.>`` so new topics need no broker-side changes.
+    """
+
+    @staticmethod
+    def url() -> str:
+        return _get("NATS_URL", "nats://nats:4222")
+
+    @staticmethod
+    def stream() -> str:
+        return _get("STAPEL_NATS_STREAM", "stapel-events")
+
+    @staticmethod
+    def subject_prefix() -> str:
+        return _get("STAPEL_NATS_EVENT_PREFIX", "stapel.evt")
+
+    @classmethod
+    def subject_for(cls, topic: str) -> str:
+        return f"{cls.subject_prefix()}.{topic}"

stapel_core/bus/backends/kafka.py

@@ -0,0 +1,194 @@
+"""
+Kafka bus backend — production transport via confluent-kafka.
+
+Set in Django settings:
+    STAPEL_BUS_BACKEND = "stapel_core.bus.backends.kafka.KafkaBus"
+"""
+from __future__ import annotations
+
+import logging
+import os
+import signal
+import threading
+import time
+from typing import Callable
+
+from ..base import BusBackend
+from ..event import Event
+
+logger = logging.getLogger(__name__)
+
+HEARTBEAT_PATH = os.environ.get("KAFKA_CONSUMER_HEARTBEAT", "/tmp/kafka_consumer_alive")
+HEARTBEAT_STALENESS_S = int(os.environ.get("KAFKA_CONSUMER_HEARTBEAT_STALENESS_S", "120"))
+WATCHDOG_INTERVAL_S = int(os.environ.get("KAFKA_CONSUMER_WATCHDOG_INTERVAL_S", "30"))
+
+DLQ_SUFFIX = ".dlq"
+
+
+def _dlq_topic(topic: str) -> str:
+    return topic + DLQ_SUFFIX
+
+
+class KafkaBus(BusBackend):
+    """Thin wrapper around confluent-kafka Producer/Consumer."""
+
+    def __init__(self) -> None:
+        self._producer = None
+        self._producer_lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+    # Publish
+    # ------------------------------------------------------------------
+
+    def _get_producer(self):
+        if self._producer is not None:
+            return self._producer
+        with self._producer_lock:
+            if self._producer is not None:
+                return self._producer
+            from confluent_kafka import Producer
+            from stapel_core.bus._config import KafkaBusConfig
+            self._producer = Producer(KafkaBusConfig.producer_config())
+        return self._producer
+
+    def publish(self, topic: str, event: Event) -> None:
+        producer = self._get_producer()
+        key_bytes = (event.key or event.event_id).encode("utf-8")
+        producer.produce(
+            topic,
+            key=key_bytes,
+            value=event.to_bytes(),
+            callback=self._delivery_callback,
+        )
+        producer.poll(0)
+
+    @staticmethod
+    def _delivery_callback(err, msg):
+        if err:
+            logger.error("KafkaBus delivery failed: %s topic=%s", err, msg.topic())
+        else:
+            logger.debug("KafkaBus delivered topic=%s offset=%s", msg.topic(), msg.offset())
+
+    # ------------------------------------------------------------------
+    # Consume
+    # ------------------------------------------------------------------
+
+    def consume(
+        self,
+        topics: list[str],
+        group: str,
+        handler: Callable[[Event], None],
+        *,
+        poll_timeout: float = 0.1,
+    ) -> None:
+        from confluent_kafka import Consumer, KafkaError
+        from stapel_core.bus._config import KafkaBusConfig
+
+        config = KafkaBusConfig.consumer_config(group)
+        consumer = Consumer(config)
+        consumer.subscribe(topics)
+
+        running = threading.Event()
+        running.set()
+
+        def _shutdown(signum, frame):
+            logger.info("KafkaBus shutdown signal received")
+            running.clear()
+
+        signal.signal(signal.SIGINT, _shutdown)
+        signal.signal(signal.SIGTERM, _shutdown)
+
+        self._start_watchdog(running)
+
+        try:
+            while running.is_set():
+                msg = consumer.poll(timeout=poll_timeout)
+                self._touch_heartbeat()
+                if msg is None:
+                    continue
+                if msg.error():
+                    if msg.error().code() == KafkaError._PARTITION_EOF:
+                        continue
+                    logger.error("KafkaBus consumer error: %s", msg.error())
+                    continue
+
+                try:
+                    event = Event.from_bytes(msg.value())
+                except Exception:
+                    # Poison message: deserialization failure outside the
+                    # retry loop would crash consume() and, with the offset
+                    # uncommitted, wedge the partition on restart.
+                    logger.exception(
+                        "KafkaBus undecodable message on %s, sending raw to DLQ",
+                        msg.topic(),
+                    )
+                    if self._send_raw_to_dlq(msg.topic(), msg.value()):
+                        consumer.commit(msg)
+                    continue
+
+                retries = 0
+                dlq_ok = True
+                while retries <= 3:
+                    try:
+                        handler(event)
+                        break
+                    except Exception:
+                        retries += 1
+                        if retries > 3:
+                            logger.exception("KafkaBus DLQ event_id=%s", event.event_id)
+                            dlq_ok = self._send_to_dlq(msg.topic(), event)
+                        else:
+                            time.sleep(2 ** retries)
+                # Commit only when the message was handled or confirmed in
+                # the DLQ — otherwise the offset would advance past a
+                # message that exists nowhere else (silent loss).
+                if dlq_ok:
+                    consumer.commit(msg)
+        finally:
+            consumer.close()
+
+    def _send_to_dlq(self, original_topic: str, event: Event) -> bool:
+        try:
+            self.publish(_dlq_topic(original_topic), event)
+            return True
+        except Exception:
+            logger.exception("KafkaBus failed to send to DLQ")
+            return False
+
+    def _send_raw_to_dlq(self, original_topic: str, raw: bytes) -> bool:
+        """DLQ a message that could not even be deserialized."""
+        try:
+            event = Event(
+                event_type="__undecodable__",
+                service="bus",
+                payload={"raw": raw.decode("utf-8", errors="replace"), "topic": original_topic},
+            )
+            self.publish(_dlq_topic(original_topic), event)
+            return True
+        except Exception:
+            logger.exception("KafkaBus failed to DLQ undecodable message")
+            return False
+
+    @staticmethod
+    def _touch_heartbeat() -> None:
+        try:
+            open(HEARTBEAT_PATH, "w").close()
+        except OSError:
+            pass
+
+    def _start_watchdog(self, running: threading.Event) -> None:
+        def _watch():
+            while running.is_set():
+                time.sleep(WATCHDOG_INTERVAL_S)
+                try:
+                    mtime = os.path.getmtime(HEARTBEAT_PATH)
+                    age = time.time() - mtime
+                    if age > HEARTBEAT_STALENESS_S:
+                        logger.critical("KafkaBus heartbeat stale (%.0fs), exiting", age)
+                        running.clear()
+                        os.kill(os.getpid(), signal.SIGTERM)
+                except FileNotFoundError:
+                    pass
+
+        t = threading.Thread(target=_watch, daemon=True)
+        t.start()

stapel_core/bus/backends/memory.py

@@ -0,0 +1,84 @@
+"""
+In-memory bus backend — for tests and local dev without a broker.
+
+Published events are stored in ``MemoryBus.events`` so tests can assert on them:
+
+    from stapel_core.bus import get_bus
+    bus = get_bus()
+    assert bus.events[-1].event_type == "profile.changed"
+"""
+from __future__ import annotations
+
+import logging
+import queue
+import threading
+from collections import defaultdict
+from typing import Callable
+
+from ..base import BusBackend
+from ..event import Event
+
+logger = logging.getLogger(__name__)
+
+
+class MemoryBus(BusBackend):
+    """
+    Thread-safe in-memory bus. Subscribers are registered per topic.
+    ``publish()`` delivers synchronously to all registered handlers,
+    then appends to ``self.events`` for test introspection.
+    """
+
+    def __init__(self) -> None:
+        self.events: list[Event] = []
+        self._subscribers: dict[str, list[Callable[[Event], None]]] = defaultdict(list)
+        self._queue: queue.Queue[Event] = queue.Queue()
+        self._lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+    # BusBackend interface
+    # ------------------------------------------------------------------
+
+    def publish(self, topic: str, event: Event) -> None:
+        logger.debug("MemoryBus.publish topic=%s event_id=%s", topic, event.event_id)
+        with self._lock:
+            self.events.append(event)
+        for handler in self._subscribers.get(topic, []):
+            try:
+                handler(event)
+            except Exception:
+                logger.exception("MemoryBus handler error topic=%s", topic)
+        self._queue.put(event)
+
+    def consume(
+        self,
+        topics: list[str],
+        group: str,
+        handler: Callable[[Event], None],
+        *,
+        poll_timeout: float = 0.1,
+    ) -> None:
+        """Drain the queue, calling *handler* for each event whose type matches *topics*."""
+        logger.debug("MemoryBus.consume topics=%s group=%s", topics, group)
+        try:
+            while True:
+                event = self._queue.get(timeout=poll_timeout)
+                if event.event_type in topics:
+                    handler(event)
+        except queue.Empty:
+            pass
+
+    # ------------------------------------------------------------------
+    # Test helpers
+    # ------------------------------------------------------------------
+
+    def subscribe(self, topic: str, handler: Callable[[Event], None]) -> None:
+        """Register a synchronous handler called on every publish to *topic*."""
+        self._subscribers[topic].append(handler)
+
+    def clear(self) -> None:
+        """Reset state between tests."""
+        with self._lock:
+            self.events.clear()
+            self._subscribers.clear()
+        while not self._queue.empty():
+            self._queue.get_nowait()