mq-bridge-basic 0.4.17__cp38-abi3-win_amd64.whl

mq_bridge/__init__.py

@@ -0,0 +1,92 @@
+import json
+import platform
+import sys
+from pathlib import Path
+from typing import Union
+
+from ._mq_bridge import (
+    Consumer,
+    MemoryDrainer,
+    Message,
+    NonRetryableError,
+    Publisher,
+    RetryableError,
+    Route,
+    __version__,
+    config_schema,
+    init_logging,
+    is_shutdown_requested,
+    load_endpoint_plugin,
+    register_endpoint,
+    register_middleware,
+    request_shutdown,
+    unregister_endpoint,
+    unregister_middleware,
+)
+
+
+def _plugin_platform_tag() -> str:
+    machine = platform.machine().lower()
+    arch = {"aarch64": "arm64", "amd64": "x64", "x86_64": "x64"}.get(machine, machine)
+    suffix = "-gnu" if sys.platform == "linux" else "-msvc" if sys.platform == "win32" else ""
+    return f"{sys.platform}-{arch}{suffix}"
+
+
+def plugin_library_path(package: Union[str, Path]) -> str:
+    """Resolve the native library in an installed mq-bridge plugin package."""
+    root = Path(package).resolve()
+    if root.is_file():
+        root = root.parent
+    manifest_path = root / "mq-bridge-plugin.json"
+    if not manifest_path.is_file():
+        raise FileNotFoundError(f"mq-bridge plugin manifest not found: {manifest_path}")
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    try:
+        library = manifest["library"]
+        name = manifest["name"]
+    except KeyError as error:
+        raise ValueError(f"{manifest_path} must contain 'name' and 'library'") from error
+    if not isinstance(name, str) or not isinstance(library, str):
+        raise ValueError(f"{manifest_path} must contain string fields 'name' and 'library'")
+    file_name = (
+        f"{library}.dll"
+        if sys.platform == "win32"
+        else f"lib{library}.dylib"
+        if sys.platform == "darwin"
+        else f"lib{library}.so"
+    )
+    candidates = (root / "prebuilds" / _plugin_platform_tag() / file_name, root / file_name)
+    for candidate in candidates:
+        if candidate.is_file():
+            return str(candidate)
+    raise FileNotFoundError(
+        f"no native library for {_plugin_platform_tag()} in mq-bridge plugin package {root}; "
+        f"looked in {', '.join(map(str, candidates))}"
+    )
+
+
+def load_plugin_package(package: Union[str, Path]) -> str:
+    """Resolve and load the native library in an mq-bridge plugin package."""
+    return load_endpoint_plugin(plugin_library_path(package))
+
+__all__ = [
+    "Consumer",
+    "MemoryDrainer",
+    "Message",
+    "NonRetryableError",
+    "Publisher",
+    "RetryableError",
+    "Route",
+    "__version__",
+    "config_schema",
+    "init_logging",
+    "is_shutdown_requested",
+    "load_endpoint_plugin",
+    "load_plugin_package",
+    "plugin_library_path",
+    "register_endpoint",
+    "register_middleware",
+    "request_shutdown",
+    "unregister_endpoint",
+    "unregister_middleware",
+]

mq_bridge/__init__.pyi

@@ -0,0 +1,351 @@
+from types import TracebackType
+from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Tuple, Type, Union
+
+__version__: str
+
+def config_schema() -> Dict[str, Any]:
+    """Return the route-config JSON Schema, generated from the Rust models."""
+    ...
+
+def init_logging(level: Optional[str] = ...) -> None:
+    """Route the library's internal ``tracing`` events into the standard
+    ``logging`` module. Call once at startup, then configure output with
+    ``logging`` as usual (``logging.basicConfig``, handlers, formatters).
+
+    ``level`` seeds the Rust-side filter (default ``"warn"``); the
+    ``MQ_BRIDGE_LOG`` / ``RUST_LOG`` environment variables override it.
+    Filtering happens in Rust, so suppressed events never cross into Python.
+    Raises if logging was already initialized."""
+    ...
+
+def request_shutdown() -> bool:
+    """Request a graceful shutdown of every route: running ones stop, and ones
+    started afterwards stop immediately. Wire it to your own signal handling,
+    e.g. ``signal.signal(signal.SIGTERM, lambda *_: request_shutdown())``.
+    Returns ``True`` only for the first request. It cannot be undone."""
+    ...
+
+def is_shutdown_requested() -> bool:
+    """``True`` once :func:`request_shutdown` has been called."""
+    ...
+
+def register_endpoint(
+    name: str, factory: Callable[[str, Dict[str, Any]], Any]
+) -> None:
+    """Register a custom endpoint implemented in Python under ``name``, making it
+    usable as an endpoint type in route configs — either as ``{"pulsar": {...}}``
+    or explicitly as ``{"custom": {"name": "pulsar", "config": {...}}}``.
+
+    ``factory`` is called as ``factory(route_name, config)`` once per route leg.
+    It must return an object implementing ``receive_batch(max_messages)`` to be
+    usable as an input and/or ``send_batch(messages)`` to be usable as an output,
+    plus optional ``commit(dispositions)`` and ``close()``.
+
+    ``receive_batch`` returns an iterable of ``Message``/bytes/str/JSON values,
+    or ``None``/``[]`` when nothing is available right now; raise
+    ``StopIteration`` to signal end of stream — that meaning applies only to
+    ``receive_batch``; from any other method it is an ordinary error. ``commit``
+    receives a list of ``"ack"``/``"nack"`` strings, one per message in the batch.
+
+    Register before starting a route that names it; registering the same name
+    twice raises and keeps the first factory. All calls into one endpoint object
+    are serialized on its own thread, so it need not be thread-safe."""
+    ...
+
+def unregister_endpoint(name: str) -> bool:
+    """Drop the endpoint factory registered under ``name``, releasing the
+    reference it holds on the Python factory object.
+
+    Returns ``True`` when a factory was removed, ``False`` when ``name`` was not
+    registered. Call only after every route using the endpoint has stopped;
+    routes already holding an instance keep running."""
+    ...
+
+def register_middleware(
+    name: str, factory: Callable[[str, Dict[str, Any]], Any]
+) -> None:
+    """Register a custom middleware implemented in Python under ``name``, usable
+    in any endpoint's ``middlewares`` list as
+    ``{"custom": {"name": name, "config": {...}}}``.
+
+    ``factory`` is called as ``factory(route_name, config)`` once per endpoint
+    the middleware is attached to. It must return an object implementing
+    ``on_receive(messages)`` (applies on an input endpoint) and/or
+    ``on_send(messages)`` (applies on an output endpoint); a side the object does
+    not implement passes through untouched.
+
+    Both hooks receive the batch and must return one item per input message: a
+    ``Message`` (kept, possibly rewritten) or ``None`` to drop it. Keeping the
+    length fixed is what lets acknowledgements stay aligned with the source
+    batch."""
+    ...
+
+def unregister_middleware(name: str) -> bool:
+    """Drop the middleware factory registered under ``name``, releasing the
+    reference it holds on the Python factory object.
+
+    Returns ``True`` when a factory was removed, ``False`` when ``name`` was not
+    registered. Call only after every route using the middleware has stopped."""
+    ...
+
+JsonValue = Any
+HandlerResult = Optional[Union["Message", bytes, str, Dict[str, JsonValue], List[JsonValue], int, float, bool]]
+
+
+class RetryableError(Exception): ...
+
+
+class NonRetryableError(Exception): ...
+
+
+class Message:
+    """A message payload with optional metadata and id.
+
+    ``id`` accepts a UUID string, a ``0x``-prefixed hex literal, or a decimal
+    integer. Any other string is hashed to a stable id rather than rejected, so
+    an arbitrary string no longer raises ``ValueError``; equal strings always
+    yield the same id.
+    """
+
+    def __init__(
+        self,
+        payload: bytes,
+        metadata: Optional[Mapping[str, str]] = ...,
+        id: Optional[str] = ...,
+    ) -> None: ...
+
+    @classmethod
+    def from_json(
+        cls,
+        data: JsonValue,
+        metadata: Optional[Mapping[str, str]] = ...,
+        id: Optional[str] = ...,
+    ) -> "Message": ...
+
+    @property
+    def payload(self) -> bytes: ...
+
+    @property
+    def metadata(self) -> Dict[str, str]: ...
+
+    @property
+    def id(self) -> Optional[str]: ...
+
+    def json(self) -> JsonValue: ...
+
+    def text(self) -> str: ...
+
+    def with_json(self, data: JsonValue) -> "Message": ...
+
+    def with_payload(self, payload: bytes) -> "Message": ...
+
+
+class Route:
+    @classmethod
+    def from_file(cls, path: str, name: Optional[str] = ...) -> "Route": ...
+
+    @classmethod
+    def from_str(cls, text: str, name: Optional[str] = ...) -> "Route": ...
+
+    @classmethod
+    def from_config(
+        cls, config: Mapping[str, Any], name: Optional[str] = ...
+    ) -> "Route": ...
+
+    # Deprecated: use from_file (emits DeprecationWarning at runtime).
+    @classmethod
+    def from_yaml(cls, path: str, name: Optional[str] = ...) -> "Route": ...
+
+    # Deprecated: use from_str (emits DeprecationWarning at runtime).
+    @classmethod
+    def from_yaml_str(cls, text: str, name: Optional[str] = ...) -> "Route": ...
+
+    def with_handler(self, handler: Callable[[Message], HandlerResult]) -> "Route": ...
+
+    def add_handler(
+        self,
+        kind: str,
+        handler: Callable[[JsonValue], HandlerResult],
+    ) -> "Route": ...
+
+    def run(self) -> None:
+        """Deploy and block the calling thread until ``stop()`` is called. A
+        ``KeyboardInterrupt`` stops the route gracefully and is re-raised."""
+        ...
+
+    def start(self) -> None:
+        """Deploy on a background thread and return immediately."""
+        ...
+
+    def join(self) -> None:
+        """Block until a route started with ``start()`` has stopped. A
+        ``KeyboardInterrupt`` stops the route gracefully and is re-raised."""
+        ...
+
+    def stop(self) -> None: ...
+
+    def __enter__(self) -> "Route": ...
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> bool: ...
+
+
+class Consumer:
+    @classmethod
+    def from_file(cls, path: str, name: Optional[str] = ...) -> "Consumer": ...
+
+    @classmethod
+    def from_str(cls, text: str, name: Optional[str] = ...) -> "Consumer": ...
+
+    @classmethod
+    def from_config(
+        cls, config: Mapping[str, Any], name: Optional[str] = ...
+    ) -> "Consumer": ...
+
+    def poll(
+        self,
+        max: int = ...,
+        timeout_ms: Optional[int] = ...,
+    ) -> List["Message"]:
+        """Receive up to ``max`` messages without acking. Empty list once
+        ``timeout_ms`` milliseconds elapse with nothing received, or when the
+        source is exhausted; omit ``timeout_ms`` to block until a message
+        arrives. The returned messages are acked by the next ``commit()`` call —
+        you must call it (see ``commit``)."""
+        ...
+
+    def poll_batch(
+        self,
+        max: int = ...,
+        timeout_ms: Optional[int] = ...,
+    ) -> Tuple[List["Message"], Optional[int]]:
+        """Like ``poll()``, but also return the batch's token so it can be acked
+        or nacked individually with ``ack(token)`` / ``nack(token)`` — the shape a
+        ``dlt`` resource wants (``poll → yield → commit load package →
+        ack(token)``). Returns ``(messages, token)``, or ``([], None)`` on timeout
+        or end-of-stream. Tokens stay outstanding until acked/nacked; ``commit()``
+        still acks every outstanding batch at once, so don't mix the two styles on
+        one consumer."""
+        ...
+
+    def ack(self, token: int) -> None:
+        """Ack a single batch by the token from ``poll_batch()``, advancing the
+        consumer offset for just that batch. Raises if the token is unknown
+        (already acked/nacked, or never polled)."""
+        ...
+
+    def nack(self, token: Optional[int] = ...) -> None:
+        """Negatively acknowledge so the broker can redeliver. With a ``token``,
+        nacks just that batch; without one, nacks every outstanding batch (oldest
+        first). On Kafka there is no per-message nack — this leaves the offset
+        unadvanced, so redelivery happens on the next run/rebalance, not at
+        once."""
+        ...
+
+    def commit(self) -> None:
+        """Ack every batch returned by ``poll()`` since the last ``commit()``,
+        advancing the consumer offset.
+
+        Calling this is required, not optional. Without it the offset never
+        advances (messages are re-delivered on the next run), most brokers stall
+        once their unacknowledged/prefetch window fills, and uncommitted batches
+        are held in memory so the process grows unbounded. To retry a failed
+        batch, simply don't commit it — it will be redelivered."""
+        ...
+
+    def status(self) -> Dict[str, Any]:
+        """Status snapshot for the underlying endpoint: ``healthy``, ``target``,
+        optional ``pending`` (broker backlog/lag where reported — Kafka offset
+        lag, AMQP queue depth, NATS JetStream ``num_pending``), optional
+        ``capacity``/``error``, and ``details``. ``pending == 0`` is a precise
+        "caught up" signal; ``None`` where the broker exposes no backlog."""
+        ...
+
+    def close(self) -> None:
+        """Release the broker connection. Idempotent; ``poll()``/``status()``
+        raise afterwards. The context-manager form calls this on exit."""
+        ...
+
+    @property
+    def exhausted(self) -> bool: ...
+
+    def __enter__(self) -> "Consumer": ...
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> bool: ...
+
+
+class MemoryDrainer:
+    @classmethod
+    def from_topic(
+        cls,
+        topic: str,
+        capacity: int = ...,
+    ) -> "MemoryDrainer": ...
+
+    def drain(
+        self,
+        count: int,
+        timeout: Optional[float] = ...,
+        batch_size: int = ...,
+    ) -> int: ...
+
+
+class Publisher:
+    @classmethod
+    def from_file(cls, path: str, name: Optional[str] = ...) -> "Publisher": ...
+
+    @classmethod
+    def from_str(cls, text: str, name: Optional[str] = ...) -> "Publisher": ...
+
+    @classmethod
+    def from_config(
+        cls, config: Mapping[str, Any], name: Optional[str] = ...
+    ) -> "Publisher": ...
+
+    # Deprecated: use from_file (emits DeprecationWarning at runtime).
+    @classmethod
+    def from_yaml(cls, path: str, name: Optional[str] = ...) -> "Publisher": ...
+
+    # Deprecated: use from_str (emits DeprecationWarning at runtime).
+    @classmethod
+    def from_yaml_str(cls, text: str, name: Optional[str] = ...) -> "Publisher": ...
+
+    def send(
+        self,
+        message: Union[Message, bytes],
+        metadata: Optional[Mapping[str, str]] = ...,
+    ) -> None: ...
+
+    def send_batch(
+        self,
+        messages: Iterable[Union[Message, bytes]],
+    ) -> None: ...
+
+    def request(
+        self,
+        message: Union[Message, bytes],
+        metadata: Optional[Mapping[str, str]] = ...,
+    ) -> Message: ...
+
+    def send_json(
+        self,
+        data: JsonValue,
+        metadata: Optional[Mapping[str, str]] = ...,
+        id: Optional[str] = ...,
+    ) -> None: ...
+
+    def request_json(
+        self,
+        data: JsonValue,
+        metadata: Optional[Mapping[str, str]] = ...,
+        id: Optional[str] = ...,
+    ) -> Message: ...

mq_bridge/_mq_bridge.pyd

@@ -0,0 +1,4 @@
+[diffend] Oversized file quarantined before diffing.
+name: mq_bridge/_mq_bridge.pyd
+size: 66819072 bytes
+sha256: 98c6f8121c5e68ad88ea29f51e705b52f66b5ccc436088bb49cb6cce5aa8db44

mq_bridge/config.py

@@ -0,0 +1,180 @@
+# Generated by scripts/gen_config_types.py from mq-bridge.schema.json.
+# Do not edit by hand; re-run the generator after changing the schema.
+
+from __future__ import annotations
+
+AggregateEntry = dict
+AggregateMiddleware = dict
+AmqpConfig = dict
+AwsConfig = dict
+BufferMiddleware = dict
+ClickHouseConfig = dict
+CompressionMiddleware = dict
+CookieJarMiddleware = dict
+DeadLetterQueueMiddleware = dict
+DeduplicationMiddleware = dict
+DelayMiddleware = dict
+DetailedMappingRule = dict
+DirSpoolConfig = dict
+EncryptionConfig = dict
+Endpoint = dict
+FileConfig = dict
+GrpcConfig = dict
+HttpConfig = dict
+IbmMqConfig = dict
+IbmTlsConfig = dict
+KafkaConfig = dict
+LimiterMiddleware = dict
+LookupEntry = dict
+LookupMiddleware = dict
+MemoryConfig = dict
+MetricsMiddleware = dict
+Middleware = dict
+MongoDbConfig = dict
+MqttConfig = dict
+NatsConfig = dict
+ObjectStoreConfig = dict
+OtelMiddleware = dict
+PackMiddleware = dict
+PostgresCdcConfig = dict
+RandomPanicMiddleware = dict
+RedisStreamsConfig = dict
+RequestForwardConfig = dict
+ResponseConfig = dict
+RetryMiddleware = dict
+Route = dict
+SequenceConfig = dict
+SledConfig = dict
+SqlxConfig = dict
+StreamBufferConfig = dict
+SwitchCase = dict
+SwitchConfig = dict
+TimeoutMiddleware = dict
+TlsConfig = dict
+TransformMiddleware = dict
+UnpackMiddleware = dict
+WeakJoinMiddleware = dict
+WebSocketConfig = dict
+ZeroMqConfig = dict
+AggregateConsistency = str
+AggregateEmit = str
+CipherKind = str
+Compression = str
+DatePartitionStyle = str
+DeliveryGuarantee = str
+FaultMode = str
+FileFormat = str
+HttpServerProtocol = str
+IdHash = str
+MappingRule = str
+MongoConsume = str
+MongoDbFormat = str
+MqttProtocol = str
+NameBy = str
+NatsDeliverPolicy = str
+PackFormat = str
+PostgresConsume = str
+SpoolClaim = str
+SpoolDone = str
+SpoolFsync = str
+StaticConfig = str
+TransformErrorPolicy = str
+WeakJoinAck = str
+WeakJoinTimeout = str
+WebSocketExecutionMode = str
+ZeroMqBackend = str
+ZeroMqFormat = str
+ZeroMqSocketType = str
+RouteConfig = Route
+EndpointConfig = Endpoint
+RoutesConfig = dict
+ConfigDocument = dict
+
+__all__ = [
+    "AggregateConsistency",
+    "AggregateEmit",
+    "AggregateEntry",
+    "AggregateMiddleware",
+    "AmqpConfig",
+    "AwsConfig",
+    "BufferMiddleware",
+    "CipherKind",
+    "ClickHouseConfig",
+    "Compression",
+    "CompressionMiddleware",
+    "ConfigDocument",
+    "CookieJarMiddleware",
+    "DatePartitionStyle",
+    "DeadLetterQueueMiddleware",
+    "DeduplicationMiddleware",
+    "DelayMiddleware",
+    "DeliveryGuarantee",
+    "DetailedMappingRule",
+    "DirSpoolConfig",
+    "EncryptionConfig",
+    "Endpoint",
+    "EndpointConfig",
+    "FaultMode",
+    "FileConfig",
+    "FileFormat",
+    "GrpcConfig",
+    "HttpConfig",
+    "HttpServerProtocol",
+    "IbmMqConfig",
+    "IbmTlsConfig",
+    "IdHash",
+    "KafkaConfig",
+    "LimiterMiddleware",
+    "LookupEntry",
+    "LookupMiddleware",
+    "MappingRule",
+    "MemoryConfig",
+    "MetricsMiddleware",
+    "Middleware",
+    "MongoConsume",
+    "MongoDbConfig",
+    "MongoDbFormat",
+    "MqttConfig",
+    "MqttProtocol",
+    "NameBy",
+    "NatsConfig",
+    "NatsDeliverPolicy",
+    "ObjectStoreConfig",
+    "OtelMiddleware",
+    "PackFormat",
+    "PackMiddleware",
+    "PostgresCdcConfig",
+    "PostgresConsume",
+    "RandomPanicMiddleware",
+    "RedisStreamsConfig",
+    "RequestForwardConfig",
+    "ResponseConfig",
+    "RetryMiddleware",
+    "Route",
+    "RouteConfig",
+    "RoutesConfig",
+    "SequenceConfig",
+    "SledConfig",
+    "SpoolClaim",
+    "SpoolDone",
+    "SpoolFsync",
+    "SqlxConfig",
+    "StaticConfig",
+    "StreamBufferConfig",
+    "SwitchCase",
+    "SwitchConfig",
+    "TimeoutMiddleware",
+    "TlsConfig",
+    "TransformErrorPolicy",
+    "TransformMiddleware",
+    "UnpackMiddleware",
+    "WeakJoinAck",
+    "WeakJoinMiddleware",
+    "WeakJoinTimeout",
+    "WebSocketConfig",
+    "WebSocketExecutionMode",
+    "ZeroMqBackend",
+    "ZeroMqConfig",
+    "ZeroMqFormat",
+    "ZeroMqSocketType",
+]