youeduc-sdk-messaging 0.4.0__py3-none-any.whl

sdk_messaging/__init__.py

@@ -0,0 +1,104 @@
+"""SDK de mensageria assíncrona sobre Apache Kafka."""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _version
+
+from sdk_messaging import naming
+from sdk_messaging.admin import ensure_topics
+from sdk_messaging.config import KafkaConfig
+from sdk_messaging.consumer import (
+    Consumer,
+    HandlerCancelledError,
+    InvalidHandlerError,
+    InvalidPayloadError,
+)
+from sdk_messaging.dedup import InMemoryDuplicateChecker
+from sdk_messaging.dlq import ReplayReport, replay_dlq
+from sdk_messaging.exceptions import (
+    MessagingError,
+    NotStartedError,
+    PermanentProcessingError,
+    PublishError,
+    SchemaNotFoundError,
+    SchemaValidationError,
+)
+from sdk_messaging.messaging import Messaging
+from sdk_messaging.publisher import Publisher
+from sdk_messaging.retry import RetryPolicy
+from sdk_messaging.schema import (
+    ChainSchemaLoader,
+    FileSystemSchemaLoader,
+    InMemorySchemaLoader,
+    JsonSchemaValidator,
+    SchemaRegistry,
+)
+from sdk_messaging.telemetry import NoopTelemetry, OtelTelemetry
+from sdk_messaging.transport import KafkaProducerSender
+from sdk_messaging.types import (
+    ConsumedMessage,
+    ConsumeOutcome,
+    ConsumeSpan,
+    DlqReason,
+    DuplicateChecker,
+    MessageHandler,
+    MessageInfo,
+    MessageSender,
+    PayloadValidator,
+    PublishResult,
+    PublishStatus,
+    SchemaLoader,
+    SchemaValidator,
+    SentRecord,
+    Telemetry,
+)
+
+try:
+    __version__ = _version("youeduc-sdk-messaging")
+except PackageNotFoundError:  # rodando do código-fonte, sem instalar
+    __version__ = "0.0.0"
+
+__all__ = [
+    "ChainSchemaLoader",
+    "ConsumeOutcome",
+    "ConsumeSpan",
+    "ConsumedMessage",
+    "Consumer",
+    "DlqReason",
+    "DuplicateChecker",
+    "FileSystemSchemaLoader",
+    "HandlerCancelledError",
+    "InMemoryDuplicateChecker",
+    "InMemorySchemaLoader",
+    "InvalidHandlerError",
+    "InvalidPayloadError",
+    "JsonSchemaValidator",
+    "KafkaConfig",
+    "KafkaProducerSender",
+    "MessageHandler",
+    "MessageInfo",
+    "MessageSender",
+    "Messaging",
+    "MessagingError",
+    "NoopTelemetry",
+    "NotStartedError",
+    "OtelTelemetry",
+    "PayloadValidator",
+    "PermanentProcessingError",
+    "PublishError",
+    "PublishResult",
+    "PublishStatus",
+    "Publisher",
+    "ReplayReport",
+    "RetryPolicy",
+    "SchemaLoader",
+    "SchemaNotFoundError",
+    "SchemaRegistry",
+    "SchemaValidationError",
+    "SchemaValidator",
+    "SentRecord",
+    "Telemetry",
+    "__version__",
+    "ensure_topics",
+    "naming",
+    "replay_dlq",
+]

sdk_messaging/_schema_semantics.py

@@ -0,0 +1,254 @@
+"""Semântica de JSON Schema idêntica nas SDKs Python e Node (spec §10).
+
+- ``pattern`` / ``patternProperties``: regex ECMA-262 (flag ``u``) traduzida para o ``re``
+  do Python. ``\\d``/``\\w``/``\\b`` são ASCII, ``\\s`` é o conjunto de espaços do ECMA,
+  ``.`` não casa terminadores de linha e ``$`` só casa no fim absoluto (no Python casaria
+  antes de um ``\\n`` final). Grupos nomeados ``(?<n>...)`` / ``\\k<n>`` viram a sintaxe
+  do Python. Outras construções são copiadas como estão: o que o ``re`` não entende é
+  recusado na construção do validator.
+- ``format``: só os formatos da spec são verificados, com regras exatas (sem depender de
+  pacotes opcionais do ``jsonschema`` instalados no ambiente).
+"""
+
+from __future__ import annotations
+
+import calendar
+import functools
+import re
+from typing import Any
+
+from jsonschema import Draft202012Validator, FormatChecker
+
+# --------------------------------------------------------------------------- regex ECMA
+
+_WORD = "A-Za-z0-9_"
+# WhiteSpace + LineTerminator do ECMA-262 (\s).
+_SPACE = "\\t\\n\\x0b\\x0c\\r \\xa0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000\\ufeff"
+_LINE_TERMINATORS = "\\n\\r\\u2028\\u2029"
+_ANY_CHAR = "[\\x00-\\U0010ffff]"
+_NOTHING = "(?!)"
+
+# Escapes de classe (conteúdo de [...]); as versões maiúsculas são complementos.
+_CLASS_SETS = {"d": "0-9", "w": _WORD, "s": _SPACE}
+_OUTSIDE_ESCAPES = {
+    "d": "[0-9]",
+    "D": "[^0-9]",
+    "w": f"[{_WORD}]",
+    "W": f"[^{_WORD}]",
+    "s": f"[{_SPACE}]",
+    "S": f"[^{_SPACE}]",
+    "b": f"(?:(?<=[{_WORD}])(?![{_WORD}])|(?<![{_WORD}])(?=[{_WORD}]))",
+    "B": f"(?:(?<=[{_WORD}])(?=[{_WORD}])|(?<![{_WORD}])(?![{_WORD}]))",
+}
+# Dentro de [...] estes caracteres formam operações de conjunto futuras no ``re``.
+_CLASS_LITERALS_TO_ESCAPE = frozenset("[&~|")
+
+
+@functools.lru_cache(maxsize=1024)
+def ecma_pattern(pattern: str) -> str:
+    """Traduz uma regex ECMA-262 para uma regex ``re`` com o mesmo comportamento."""
+    out: list[str] = []
+    i, size = 0, len(pattern)
+    while i < size:
+        char = pattern[i]
+        if char == "\\" and i + 1 < size:
+            escaped = pattern[i + 1]
+            if escaped in _OUTSIDE_ESCAPES:
+                out.append(_OUTSIDE_ESCAPES[escaped])
+            elif escaped == "k" and pattern.startswith("<", i + 2):
+                end = pattern.find(">", i + 3)
+                if end != -1:
+                    out.append(f"(?P={pattern[i + 3 : end]})")
+                    i = end + 1
+                    continue
+                out.append("\\k")
+            else:
+                out.append(char + escaped)
+            i += 2
+        elif char == "[":
+            i = _translate_class(pattern, i, out)
+        elif (
+            pattern.startswith("(?<", i) and i + 3 < size and pattern[i + 3] not in "=!"
+        ):
+            out.append("(?P<")
+            i += 3
+        elif char == ".":
+            out.append(f"[^{_LINE_TERMINATORS}]")
+            i += 1
+        elif char == "$":
+            out.append("\\Z")
+            i += 1
+        else:
+            out.append(char)
+            i += 1
+    return "".join(out)
+
+
+def _translate_class(pattern: str, start: int, out: list[str]) -> int:
+    """Traduz ``[...]`` a partir de ``start``; retorna o índice após o ``]``."""
+    i = start + 1
+    negated = i < len(pattern) and pattern[i] == "^"
+    if negated:
+        i += 1
+    items: list[str] = []
+    complements: list[str] = []
+    while i < len(pattern):
+        char = pattern[i]
+        if char == "]":
+            break
+        if char == "\\" and i + 1 < len(pattern):
+            escaped = pattern[i + 1]
+            if escaped in _CLASS_SETS:
+                items.append(_CLASS_SETS[escaped])
+            elif escaped in "DWS":
+                complements.append(_CLASS_SETS[escaped.lower()])
+            else:
+                items.append(char + escaped)
+            i += 2
+            continue
+        items.append("\\" + char if char in _CLASS_LITERALS_TO_ESCAPE else char)
+        i += 1
+    else:
+        # Sem ']' de fechamento: o re do Python recusa o padrão na construção.
+        out.append(pattern[start:])
+        return len(pattern)
+    out.append(_class_expression("".join(items), complements, negated=negated))
+    return i + 1
+
+
+def _class_expression(body: str, complements: list[str], *, negated: bool) -> str:
+    if not complements:
+        if not body:  # ECMA: [] não casa nada, [^] casa qualquer caractere
+            return _ANY_CHAR if negated else _NOTHING
+        return f"[{'^' if negated else ''}{body}]"
+    if not negated:  # união: body ou complemento de cada conjunto
+        parts = ([f"[{body}]"] if body else []) + [f"[^{c}]" for c in complements]
+        return "(?:" + "|".join(parts) + ")"
+    # [^...\D...]: fora de body e dentro de cada conjunto complementado.
+    prefix = f"(?![{body}])" if body else ""
+    lookaheads = "".join(f"(?=[{c}])" for c in complements[:-1])
+    return f"(?:{prefix}{lookaheads}[{complements[-1]}])"
+
+
+# Palavras-chave cujo valor é subschema, mapa de subschemas ou lista de subschemas.
+_SUBSCHEMA_MAPS = (
+    "properties",
+    "patternProperties",
+    "$defs",
+    "definitions",
+    "dependentSchemas",
+)
+_SUBSCHEMA_VALUES = (
+    "additionalProperties",
+    "unevaluatedProperties",
+    "items",
+    "additionalItems",
+    "unevaluatedItems",
+    "contains",
+    "propertyNames",
+    "not",
+    "if",
+    "then",
+    "else",
+)
+_SUBSCHEMA_LISTS = ("allOf", "anyOf", "oneOf", "prefixItems")
+
+
+def ecma_schema(schema: Any) -> Any:  # noqa: ANN401 - schema JSON arbitrário
+    """Cópia do schema com ``pattern`` e chaves de ``patternProperties`` traduzidos.
+
+    Só percorre posições de subschema (nunca ``const``/``enum``/``default``/``examples``
+    nem nomes de propriedades), então dados e nomes ficam intactos.
+    """
+    if not isinstance(schema, dict):
+        return schema
+    result = dict(schema)
+    pattern = result.get("pattern")
+    if isinstance(pattern, str):
+        result["pattern"] = ecma_pattern(pattern)
+    for keyword in _SUBSCHEMA_MAPS:
+        value = result.get(keyword)
+        if isinstance(value, dict):
+            translate_key = keyword == "patternProperties"
+            result[keyword] = {
+                (ecma_pattern(key) if translate_key else key): ecma_schema(sub)
+                for key, sub in value.items()
+            }
+    for keyword in _SUBSCHEMA_VALUES:
+        if keyword in result:
+            value = result[keyword]
+            result[keyword] = (
+                [ecma_schema(sub) for sub in value]
+                if isinstance(value, list)
+                else ecma_schema(value)
+            )
+    for keyword in _SUBSCHEMA_LISTS:
+        value = result.get(keyword)
+        if isinstance(value, list):
+            result[keyword] = [ecma_schema(sub) for sub in value]
+    return result
+
+
+# --------------------------------------------------------------------------- formatos
+
+_DATE_TIME_RE = re.compile(
+    r"([0-9]{4})-(0[1-9]|1[0-2])-([0-9]{2})[Tt]"
+    r"(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](?:\.[0-9]+)?"
+    r"(?:[Zz]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])",
+    re.ASCII,
+)
+_UUID_RE = re.compile(
+    r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}",
+    re.ASCII,
+)
+
+
+def is_date_time(instance: object) -> bool:
+    """RFC 3339 ``date-time`` exato (fullmatch: nem ``\\n`` final), sem segundo bissexto."""
+    if not isinstance(instance, str):
+        return True
+    match = _DATE_TIME_RE.fullmatch(instance)
+    if match is None:
+        return False
+    year, month, day = (int(group) for group in match.groups())
+    return year >= 1 and 1 <= day <= calendar.monthrange(year, month)[1]
+
+
+def is_time(instance: object) -> bool:
+    if not isinstance(instance, str):
+        return True
+    return is_date_time("1970-01-01T" + instance)
+
+
+def is_uuid(instance: object) -> bool:
+    """``8-4-4-4-12`` hexadecimal exato (sem ``urn:uuid:``, chaves ou hífens extras)."""
+    if not isinstance(instance, str):
+        return True
+    return _UUID_RE.fullmatch(instance) is not None
+
+
+def is_ecma_regex(instance: object) -> bool:
+    if not isinstance(instance, str):
+        return True
+    re.compile(ecma_pattern(instance))
+    return True
+
+
+def _payload_format_checker() -> FormatChecker:
+    checker = FormatChecker(formats=())
+    reference = Draft202012Validator.FORMAT_CHECKER.checkers
+    # Regras do jsonschema que já coincidem com a spec.
+    for name in ("date", "email", "idn-email", "ipv4", "ipv6"):
+        checker.checkers[name] = reference[name]
+    checker.checks("date-time")(is_date_time)
+    checker.checks("time")(is_time)
+    checker.checks("uuid")(is_uuid)
+    return checker
+
+
+# Payloads: só os formatos da spec; qualquer outro é anotação.
+FORMAT_CHECKER = _payload_format_checker()
+
+# Metaschema (check_schema): ``pattern``/``patternProperties`` são regex ECMA.
+META_FORMAT_CHECKER = FormatChecker(formats=())
+META_FORMAT_CHECKER.checks("regex", raises=re.error)(is_ecma_regex)

sdk_messaging/admin.py

@@ -0,0 +1,161 @@
+"""Provisionamento de tópicos (uso em bootstrap/dev; em produção prefira IaC)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Protocol
+
+import structlog
+from aiokafka.admin import AIOKafkaAdminClient, NewTopic
+from aiokafka.errors import TopicAlreadyExistsError, for_code
+
+from sdk_messaging.exceptions import MessagingError
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Mapping, Sequence
+
+    from sdk_messaging.config import KafkaConfig
+
+logger = structlog.get_logger(__name__)
+
+
+class _CreateTopicsResponse(Protocol):
+    @property
+    def topic_errors(self) -> Sequence[Sequence[Any]]: ...
+
+
+class _Closable(Protocol):
+    async def close(self) -> None: ...
+
+
+@dataclass(frozen=True)
+class TopicSpec:
+    """Um tópico a criar: nome, partições e configs (usado por ``Messaging.ensure_topics``
+    para variar partições/configs por tipo de tópico, spec §2.6)."""
+
+    name: str
+    num_partitions: int
+    topic_configs: Mapping[str, str] = field(default_factory=dict)
+
+
+async def ensure_topics(
+    config: KafkaConfig,
+    topics: Iterable[str],
+    *,
+    num_partitions: int = 3,
+    replication_factor: int = 1,
+    topic_configs: Mapping[str, str] | None = None,
+    admin_factory: Callable[..., Any] | None = None,
+) -> list[str]:
+    """Cria os tópicos que ainda não existem e devolve os nomes criados.
+
+    Tópico já existente não é erro. Qualquer outro erro por tópico levanta
+    ``MessagingError`` listando cada tópico e o erro; erros da requisição como um
+    todo (conexão, timeout) propagam como vierem do aiokafka.
+    """
+    if isinstance(topics, str):
+        raise TypeError("'topics' must be an iterable of topic names, not a str")
+    names = list(dict.fromkeys(topics))
+    if not names:
+        return []
+    for name in names:
+        if not isinstance(name, str) or not name:
+            raise ValueError(f"topic names must be non-empty strings; got {name!r}")
+    _require_count("num_partitions", num_partitions)
+    configs = dict(topic_configs or {})
+    return await create_topics(
+        config,
+        [TopicSpec(name, num_partitions, configs) for name in names],
+        replication_factor=replication_factor,
+        admin_factory=admin_factory,
+    )
+
+
+def _require_count(name: str, value: object) -> None:
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        raise ValueError(f"{name!r} must be >= 1; got {value!r}")
+
+
+async def create_topics(
+    config: KafkaConfig,
+    specs: Sequence[TopicSpec],
+    *,
+    replication_factor: int = 1,
+    admin_factory: Callable[..., Any] | None = None,
+) -> list[str]:
+    """Como :func:`ensure_topics`, com partições e configs por tópico. Nomes repetidos:
+    vale a primeira definição."""
+    unique: dict[str, TopicSpec] = {}
+    for spec in specs:
+        if not isinstance(spec.name, str) or not spec.name:
+            raise ValueError(
+                f"topic names must be non-empty strings; got {spec.name!r}"
+            )
+        _require_count("num_partitions", spec.num_partitions)
+        unique.setdefault(spec.name, spec)
+    if not unique:
+        return []
+    _require_count("replication_factor", replication_factor)
+
+    names = list(unique)
+    new_topics = [
+        NewTopic(
+            name=spec.name,
+            num_partitions=spec.num_partitions,
+            replication_factor=replication_factor,
+            topic_configs=dict(spec.topic_configs),
+        )
+        for spec in unique.values()
+    ]
+    factory: Callable[..., Any] = (
+        admin_factory if admin_factory is not None else AIOKafkaAdminClient
+    )
+    admin = factory(**config.client_kwargs())
+    try:
+        await admin.start()
+        response = await admin.create_topics(new_topics)
+    finally:
+        await _close_quietly(admin)
+
+    created, failures = _check_response(names, response)
+    if created:
+        logger.info("kafka_topics_created", messaging={"topics": created})
+    if failures:
+        raise MessagingError("failed to create topics: " + "; ".join(failures))
+    return created
+
+
+def _check_response(
+    names: list[str], response: _CreateTopicsResponse
+) -> tuple[list[str], list[str]]:
+    """Interpreta ``topic_errors``: tuplas ``(topic, error_code[, error_message])``."""
+    succeeded: set[str] = set()
+    failures: list[str] = []
+    seen: set[str] = set()
+    for entry in response.topic_errors:
+        topic, code = entry[0], entry[1]
+        message = entry[2] if len(entry) > 2 else None
+        seen.add(topic)
+        if code == 0:
+            succeeded.add(topic)
+        elif code == TopicAlreadyExistsError.errno:
+            continue
+        else:
+            detail = f"{for_code(code).__name__} (error code {code})"
+            if message:
+                detail += f": {message}"
+            failures.append(f"{topic!r}: {detail}")
+    failures.extend(
+        f"{name!r}: missing from broker response" for name in names if name not in seen
+    )
+    return [name for name in names if name in succeeded], failures
+
+
+async def _close_quietly(admin: _Closable) -> None:
+    try:
+        await admin.close()
+    except Exception as exc:
+        logger.warning(
+            "kafka_admin_close_failed",
+            error={"type": type(exc).__name__, "message": str(exc)},
+        )

sdk_messaging/config.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import ssl
+    from collections.abc import Mapping
+
+_SECURITY_PROTOCOLS = ("PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL")
+_TLS_PROTOCOLS = ("SSL", "SASL_SSL")
+
+
+@dataclass(frozen=True)
+class KafkaConfig:
+    """Parâmetros de conexão compartilhados por producer, consumer e admin."""
+
+    bootstrap_servers: str
+    client_id: str | None = None
+    security_protocol: str = "PLAINTEXT"
+    sasl_mechanism: str | None = None
+    sasl_username: str | None = None
+    sasl_password: str | None = field(default=None, repr=False)
+    ssl_context: ssl.SSLContext | None = field(default=None, repr=False)
+
+    def __post_init__(self) -> None:
+        if not self.bootstrap_servers.strip():
+            raise ValueError("'bootstrap_servers' must not be empty")
+        if self.security_protocol not in _SECURITY_PROTOCOLS:
+            raise ValueError(
+                f"'security_protocol' must be one of {_SECURITY_PROTOCOLS}; "
+                f"got {self.security_protocol!r}"
+            )
+        if self.security_protocol.startswith("SASL") and self.sasl_mechanism is None:
+            raise ValueError("'sasl_mechanism' is required for SASL security protocols")
+
+    def client_kwargs(self) -> dict[str, Any]:
+        """kwargs comuns aceitos por AIOKafkaProducer, AIOKafkaConsumer e AIOKafkaAdminClient.
+
+        Com TLS (``SSL``/``SASL_SSL``) e sem ``ssl_context`` explícito, usa o
+        contexto padrão do aiokafka (CAs do sistema, verificação de hostname).
+        """
+        kwargs: dict[str, Any] = {
+            "bootstrap_servers": self.bootstrap_servers,
+            "security_protocol": self.security_protocol,
+        }
+        if self.client_id is not None:
+            kwargs["client_id"] = self.client_id
+        if self.sasl_mechanism is not None:
+            kwargs["sasl_mechanism"] = self.sasl_mechanism
+            kwargs["sasl_plain_username"] = self.sasl_username
+            kwargs["sasl_plain_password"] = self.sasl_password
+        if self.ssl_context is not None:
+            kwargs["ssl_context"] = self.ssl_context
+        elif self.security_protocol in _TLS_PROTOCOLS:
+            from aiokafka.helpers import create_ssl_context
+
+            kwargs["ssl_context"] = create_ssl_context()
+        return kwargs
+
+    @classmethod
+    def from_env(
+        cls, prefix: str = "KAFKA_", environ: Mapping[str, str] | None = None
+    ) -> KafkaConfig:
+        """Lê ``{prefix}BOOTSTRAP_SERVERS``, ``CLIENT_ID``, ``SECURITY_PROTOCOL``,
+        ``SASL_MECHANISM``, ``SASL_USERNAME`` e ``SASL_PASSWORD``.
+
+        TLS com CA própria exige montar o ``ssl_context`` no código; sem ele,
+        :meth:`client_kwargs` usa o contexto padrão.
+        """
+        env = os.environ if environ is None else environ
+        try:
+            servers = env[f"{prefix}BOOTSTRAP_SERVERS"]
+        except KeyError:
+            raise ValueError(
+                f"environment variable {prefix}BOOTSTRAP_SERVERS is required"
+            ) from None
+        return cls(
+            bootstrap_servers=servers,
+            client_id=env.get(f"{prefix}CLIENT_ID"),
+            security_protocol=env.get(f"{prefix}SECURITY_PROTOCOL", "PLAINTEXT"),
+            sasl_mechanism=env.get(f"{prefix}SASL_MECHANISM"),
+            sasl_username=env.get(f"{prefix}SASL_USERNAME"),
+            sasl_password=env.get(f"{prefix}SASL_PASSWORD"),
+        )