natsio-pcgroups 0.1.0__py3-none-any.whl

natsio/pcgroups/__init__.py

@@ -0,0 +1,147 @@
+"""Partitioned consumer groups over JetStream — static and elastic.
+
+A consumer group splits a stream into ``max_members`` partitions keyed by
+subject, and hands each *member* a fixed subset of them. Members consume in
+parallel while messages that share a partitioning key are still processed one
+at a time and in order — the throughput a single ``max_ack_pending=1`` consumer
+cannot give you.
+
+Two flavours, mirroring
+[`orbit.go/pcgroups`](https://github.com/synadia-io/orbit.go/tree/main/pcgroups):
+
+**Static** — the stream already carries the partition number as the first
+subject token (put there by a stream subject transform on ingest). One durable
+JetStream consumer per member, created straight on that stream. The membership
+is fixed for the life of the group.
+
+    await create_static(js, "ORDERS", "cg", 10, ["orders.*"], ["m1", "m2"])
+    ctx = await static_consume(js, "ORDERS", "cg", "m1", handle)
+
+**Elastic** — the stream needs no partition token. Creating the group creates a
+sourced *work-queue* stream that inserts the partition token on the way in, and
+the membership can change at any time; running members start and stop consuming
+as they are added and dropped.
+
+    await create_elastic(js, "ORDERS", "cg", 10, [PartitioningFilter(filter="orders.*", partitioning_wildcards=[1])])
+    ctx = await elastic_consume(js, "ORDERS", "cg", "m1", handle)
+    await add_members(js, "ORDERS", "cg", ["m1", "m2"])
+
+In both cases the handler is invoked with the partition token already stripped
+from the subject, and running several instances of the same member gives you
+hot-standby HA: the server's ADR-42 pinned-client priority group serves exactly
+one instance at a time.
+
+    async def handle(msg: PartitionedMsg) -> None:
+        print(msg.subject, msg.data)   # the subject as it was before partitioning
+        await msg.ack()
+
+Requires nats-server 2.11+ (priority groups, subject transforms).
+"""
+
+from natsio.pcgroups.elastic import (
+    ElasticConsumeContext,
+    add_members,
+    create_elastic,
+    delete_elastic,
+    delete_member_mappings,
+    delete_members,
+    elastic_consume,
+    elastic_is_in_membership_and_active,
+    elastic_member_step_down,
+    get_elastic_config,
+    list_elastic_active_members,
+    list_elastic_groups,
+    set_member_mappings,
+)
+from natsio.pcgroups.entities import (
+    DEFAULT_ACK_WAIT,
+    ELASTIC_BUCKET,
+    PRIORITY_GROUP,
+    STATIC_BUCKET,
+    ElasticConsumerGroupConfig,
+    MemberMapping,
+    PartitioningFilter,
+    StaticConsumerGroupConfig,
+    validate_elastic_config,
+    validate_static_config,
+)
+from natsio.pcgroups.errors import (
+    ConsumerGroupConfigError,
+    ConsumerGroupError,
+    ConsumerGroupExistsError,
+    ConsumerGroupNotFoundError,
+    GroupConfigChangedError,
+    MemberNotInGroupError,
+)
+from natsio.pcgroups.partitions import (
+    compose_group_stream_name,
+    compose_key,
+    compose_static_consumer_name,
+    elastic_get_partition_filters,
+    generate_partition_filters,
+    partitioning_transform_destination,
+    static_get_partition_filters,
+    strip_partition,
+)
+from natsio.pcgroups.session import ConsumerGroupConsumeContext, MessageHandler, PartitionedMsg
+from natsio.pcgroups.static import (
+    StaticConsumeContext,
+    create_static,
+    delete_static,
+    get_static_config,
+    list_static_active_members,
+    list_static_groups,
+    static_consume,
+    static_member_step_down,
+)
+
+__all__ = [
+    "DEFAULT_ACK_WAIT",
+    "ELASTIC_BUCKET",
+    "PRIORITY_GROUP",
+    "STATIC_BUCKET",
+    "ConsumerGroupConfigError",
+    "ConsumerGroupConsumeContext",
+    "ConsumerGroupError",
+    "ConsumerGroupExistsError",
+    "ConsumerGroupNotFoundError",
+    "ElasticConsumeContext",
+    "ElasticConsumerGroupConfig",
+    "GroupConfigChangedError",
+    "MemberMapping",
+    "MemberNotInGroupError",
+    "MessageHandler",
+    "PartitionedMsg",
+    "PartitioningFilter",
+    "StaticConsumeContext",
+    "StaticConsumerGroupConfig",
+    "add_members",
+    "compose_group_stream_name",
+    "compose_key",
+    "compose_static_consumer_name",
+    "create_elastic",
+    "create_static",
+    "delete_elastic",
+    "delete_member_mappings",
+    "delete_members",
+    "delete_static",
+    "elastic_consume",
+    "elastic_get_partition_filters",
+    "elastic_is_in_membership_and_active",
+    "elastic_member_step_down",
+    "generate_partition_filters",
+    "get_elastic_config",
+    "get_static_config",
+    "list_elastic_active_members",
+    "list_elastic_groups",
+    "list_static_active_members",
+    "list_static_groups",
+    "partitioning_transform_destination",
+    "set_member_mappings",
+    "static_consume",
+    "static_get_partition_filters",
+    "static_member_step_down",
+    "strip_partition",
+    "validate_elastic_config",
+    "validate_static_config",
+]

natsio/pcgroups/buckets.py

@@ -0,0 +1,45 @@
+"""Access to the two KV buckets the group configs live in.
+
+Both flavours store their configs in a KV bucket keyed ``"<stream>.<group>"``
+(`STATIC_BUCKET` / `ELASTIC_BUCKET`), created on demand with the same shape the
+oracle uses: file storage, replicated like the stream the groups sit on.
+"""
+
+from natsio.jetstream import JetStreamContext, StorageType
+from natsio.kv import KeyValue, KeyValueConfig
+from natsio.kv.errors import BucketExistsError, BucketNotFoundError
+
+from .errors import ConsumerGroupError, ConsumerGroupNotFoundError
+
+__all__ = ["open_bucket", "open_or_create_bucket", "require_names"]
+
+
+async def open_bucket(js: JetStreamContext, bucket: str) -> KeyValue:
+    """Open a consumer-group KV bucket, mapping "no bucket" to "no group".
+
+    A missing bucket means no group of that flavour has ever been created in
+    this account, which is indistinguishable from "this group doesn't exist"
+    for every caller here.
+    """
+    try:
+        return await js.key_value(bucket)
+    except BucketNotFoundError as exc:
+        raise ConsumerGroupNotFoundError(f"the {bucket!r} KV bucket does not exist") from exc
+
+
+async def open_or_create_bucket(js: JetStreamContext, bucket: str, replicas: int) -> KeyValue:
+    """Open a consumer-group KV bucket, creating it if it isn't there yet."""
+    try:
+        return await js.key_value(bucket)
+    except BucketNotFoundError:
+        pass
+    try:
+        return await js.create_key_value(KeyValueConfig(bucket=bucket, replicas=replicas, storage=StorageType.FILE))
+    except BucketExistsError:
+        return await js.key_value(bucket)  # lost a creation race: theirs is as good as ours
+
+
+def require_names(stream: str, group: str) -> None:
+    """Reject blank stream/group names before they become a malformed KV key."""
+    if not stream or not group:
+        raise ConsumerGroupError(f"invalid stream name {stream!r} or consumer group name {group!r}")