apache-iggy 0.9.0.dev3__tar.gz → 0.9.0.dev5__tar.gz

{apache_iggy-0.9.0.dev3 → apache_iggy-0.9.0.dev5}/Cargo.toml

@@ -44,6 +44,7 @@ members = [
     "core/connectors/sinks/mongodb_sink",
     "core/connectors/sinks/postgres_sink",
     "core/connectors/sinks/quickwit_sink",
+    "core/connectors/sinks/redshift_sink",
     "core/connectors/sinks/s3_sink",
     "core/connectors/sinks/stdout_sink",
     "core/connectors/sinks/surrealdb_sink",
@@ -68,6 +69,8 @@ members = [
     "core/system_stats",
     "core/tools",
     "examples/rust",
+    "gateways/kafka",
+    "gateways/kafka/tools/kafka-tool",
 ]
 exclude = ["foreign/cpp", "foreign/php", "foreign/python"]
 resolver = "3"
@@ -75,6 +78,9 @@ resolver = "3"
 [workspace.package]
 rust-version = "1.95"
 
+[workspace.lints.rust]
+warnings = "deny"
+
 [workspace.dependencies]
 actix-cors = "0.7.1"
 actix-files = "0.6.10"
@@ -189,6 +195,7 @@ gloo = "0.12"
 governor = "0.10.4"
 harness_derive = { path = "core/harness_derive" }
 hash32 = "1.0.0"
+hex = "0.4.3"
 hostname = "0.4.2"
 http = "1.4.2"
 human-repr = "1.1.0"
@@ -199,11 +206,11 @@ hyper-util = { version = "0.1.20", features = ["server-auto", "service"] }
 iceberg = "0.9.1"
 iceberg-catalog-rest = "0.9.1"
 iceberg-storage-opendal = "0.9.1"
-iggy = { path = "core/sdk", version = "0.11.0-edge.3" }
-iggy-cli = { path = "core/cli", version = "0.14.0-edge.3" }
-iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.0-edge.3" }
-iggy_common = { path = "core/common", version = "0.11.0-edge.3" }
-iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.4.0-edge.2" }
+iggy = { path = "core/sdk", version = "0.11.0-edge.5" }
+iggy-cli = { path = "core/cli", version = "0.14.0-edge.5" }
+iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.0-edge.5" }
+iggy_common = { path = "core/common", version = "0.11.0-edge.5" }
+iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.4.0-edge.3" }
 indexmap = "2.14.0"
 integration = { path = "core/integration" }
 ipnet = "2.12.0"
@@ -213,7 +220,6 @@ jsonwebtoken = { version = "10.4.0", features = ["rust_crypto"] }
 keyring-core = "1.0.0"
 lazy_static = "1.5.0"
 left-right = "0.11"
-lending-iterator = "0.1.7"
 libc = "0.2.188"
 log = "0.4.33"
 lz4_flex = "0.14.0"
@@ -255,6 +261,7 @@ parquet = "57.3.1"
 partitions = { path = "core/partitions" }
 passterm = "2.0.6"
 paste = "1.0"
+pgwire = "0.40.4"
 postcard = { version = "1.1.3", features = ["alloc"] }
 predicates = "3.1.4"
 proc-macro2 = "1"
@@ -298,9 +305,9 @@ server_common = { path = "core/server_common" }
 shard = { path = "core/shard" }
 shard_allocator = { path = "core/shard_allocator" }
 simd-json = { version = "0.17.3", features = ["serde_impl"] }
-slab = "0.4.12"
 smallvec = "1.15"
 socket2 = "0.6.5"
+sqlparser = { version = "0.62.0", features = ["visitor"] }
 sqlx = { version = "0.9.0", features = [
     "runtime-tokio",
     "tls-rustls",
@@ -316,6 +323,7 @@ sqlx = { version = "0.9.0", features = [
 static-toml = "1.3.0"
 strum = { version = "0.28.0", features = ["derive"] }
 strum_macros = "0.28.0"
+subtle = "2.6.1"
 # Pinned to 2 because darling 0.23 still emits syn 2 types.
 syn = { version = "2", features = ["full", "extra-traits"] }
 sysinfo = "0.39.6"
@@ -327,6 +335,7 @@ testcontainers = { version = "0.27.3", features = ["reusable-containers"] }
 testcontainers-modules = { version = "0.15.0", features = ["postgres", "http_wait"] }
 thiserror = "2.0.19"
 tokio = { version = "1.53.1", features = ["full"] }
+tokio-postgres = "0.7.18"
 tokio-rustls = "0.26.4"
 tokio-tungstenite = { version = "0.30", features = ["rustls-tls-webpki-roots"] }
 tokio-util = { version = "0.7.18", features = ["compat"] }

{apache_iggy-0.9.0.dev3 → apache_iggy-0.9.0.dev5}/NOTICE

@@ -1,4 +1,4 @@
-Apache Iggy (Incubating)
+Apache Iggy
 Copyright 2026 The Apache Software Foundation
 
 This product includes software developed at

{apache_iggy-0.9.0.dev3 → apache_iggy-0.9.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-iggy
-Version: 0.9.0.dev3
+Version: 0.9.0.dev5
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
@@ -65,12 +65,6 @@ Project-URL: Repository, https://github.com/apache/iggy
 
 Apache Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second.
 
-> Apache Iggy (Incubating) is an effort undergoing incubation at the Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC.
->
-> Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects.
->
-> While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
-
 ## Installation
 
 ### Basic Installation

{apache_iggy-0.9.0.dev3 → apache_iggy-0.9.0.dev5}/README.md

@@ -12,12 +12,6 @@
 
 Apache Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second.
 
-> Apache Iggy (Incubating) is an effort undergoing incubation at the Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC.
->
-> Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects.
->
-> While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
-
 ## Installation
 
 ### Basic Installation

{apache_iggy-0.9.0.dev3 → apache_iggy-0.9.0.dev5}/apache_iggy.pyi

@@ -30,12 +30,13 @@ __all__ = [
     "AutoCommitAfter",
     "AutoCommitWhen",
     "AutoLogin",
+    "Consumer",
     "ConsumerGroup",
     "ConsumerGroupDetails",
     "ConsumerGroupMember",
+    "GlobalPermissions",
     "HeaderKey",
     "HeaderValue",
-    "GlobalPermissions",
     "IggyClient",
     "IggyConsumer",
     "IggyExpiry",
@@ -55,10 +56,10 @@ __all__ = [
     "Topic",
     "TopicDetails",
     "TopicPermissions",
+    "UserHeaders",
     "UserInfo",
     "UserInfoDetails",
     "UserStatus",
-    "UserHeaders",
 ]
 
 class AutoCommit:
@@ -289,6 +290,35 @@ class AutoLogin:
         """
     def __repr__(self) -> builtins.str: ...
 
+class Consumer:
+    r"""
+    The consumer polling the messages. It selects both the consumer kind and the
+    identifier the server keys the stored offset on.
+    """
+    @typing.final
+    class Single(Consumer):
+        r"""
+        A regular consumer, owning its offset on the polled partition.
+        """
+
+        __match_args__ = ("id",)
+        @property
+        def id(self) -> builtins.str | builtins.int: ...
+        def __new__(cls, id: builtins.str | builtins.int) -> Consumer.Single: ...
+
+    @typing.final
+    class Group(Consumer):
+        r"""
+        A member of the consumer group, sharing the group's offset.
+        """
+
+        __match_args__ = ("id",)
+        @property
+        def id(self) -> builtins.str | builtins.int: ...
+        def __new__(cls, id: builtins.str | builtins.int) -> Consumer.Group: ...
+
+    ...
+
 @typing.final
 class ConsumerGroup:
     @property
@@ -358,6 +388,105 @@ class ConsumerGroupMember:
         Gets the collection of partitions the consumer group member is consuming.
         """
 
+@typing.final
+class GlobalPermissions:
+    r"""
+    Global permissions, applied to all streams without specifying them one by one.
+    """
+    @property
+    def manage_servers(self) -> builtins.bool:
+        r"""
+        Whether managing servers is allowed; includes `read_servers`.
+        """
+    @property
+    def read_servers(self) -> builtins.bool:
+        r"""
+        Whether reading server info (stats, clients) is allowed.
+        """
+    @property
+    def manage_users(self) -> builtins.bool:
+        r"""
+        Whether managing users is allowed; includes `read_users`.
+        """
+    @property
+    def read_users(self) -> builtins.bool:
+        r"""
+        Whether reading user info is allowed.
+        """
+    @property
+    def manage_streams(self) -> builtins.bool:
+        r"""
+        Whether managing all streams is allowed; includes `read_streams` and
+        `manage_topics`.
+        """
+    @property
+    def read_streams(self) -> builtins.bool:
+        r"""
+        Whether reading all streams is allowed; includes `read_topics`.
+        """
+    @property
+    def manage_topics(self) -> builtins.bool:
+        r"""
+        Whether managing all topics is allowed; includes `read_topics` and
+        `send_messages`.
+        """
+    @property
+    def read_topics(self) -> builtins.bool:
+        r"""
+        Whether reading all topics and managing consumer groups is allowed;
+        includes `poll_messages`.
+        """
+    @property
+    def poll_messages(self) -> builtins.bool:
+        r"""
+        Whether polling messages from all streams and managing consumer
+        offsets is allowed.
+        """
+    @property
+    def send_messages(self) -> builtins.bool:
+        r"""
+        Whether sending messages to all streams is allowed.
+        """
+    def __eq__(self, other: builtins.object, /) -> builtins.bool: ...
+    def __new__(
+        cls,
+        *,
+        manage_servers: builtins.bool = False,
+        read_servers: builtins.bool = False,
+        manage_users: builtins.bool = False,
+        read_users: builtins.bool = False,
+        manage_streams: builtins.bool = False,
+        read_streams: builtins.bool = False,
+        manage_topics: builtins.bool = False,
+        read_topics: builtins.bool = False,
+        poll_messages: builtins.bool = False,
+        send_messages: builtins.bool = False,
+    ) -> GlobalPermissions:
+        r"""
+        Create global permissions. Every flag defaults to `False`.
+
+        The `includes` notes below are transitive: a flag also grants everything
+        its included flags grant. For example `manage_streams` includes
+        `manage_topics`, and through it `read_topics`, `poll_messages`, and
+        `send_messages`.
+
+        Args:
+            manage_servers: Allow managing servers; includes `read_servers`.
+            read_servers: Allow reading server info (stats, clients).
+            manage_users: Allow managing users; includes `read_users`.
+            read_users: Allow reading user info.
+            manage_streams: Allow managing all streams; includes `read_streams`
+                and `manage_topics`.
+            read_streams: Allow reading all streams; includes `read_topics`.
+            manage_topics: Allow managing all topics; includes `read_topics`
+                and `send_messages`.
+            read_topics: Allow reading all topics and managing consumer groups
+                (including create and delete); includes `poll_messages`.
+            poll_messages: Allow polling messages from all streams and managing
+                consumer offsets.
+            send_messages: Allow sending messages to all streams.
+        """
+
 class HeaderKey:
     r"""
     Typed key for an Iggy user header.
@@ -708,105 +837,6 @@ class HeaderValue:
         def value(self) -> builtins.float: ...
         def __new__(cls, value: builtins.float) -> HeaderValue.Float64: ...
 
-@typing.final
-class GlobalPermissions:
-    r"""
-    Global permissions, applied to all streams without specifying them one by one.
-    """
-    @property
-    def manage_servers(self) -> builtins.bool:
-        r"""
-        Whether managing servers is allowed; includes `read_servers`.
-        """
-    @property
-    def read_servers(self) -> builtins.bool:
-        r"""
-        Whether reading server info (stats, clients) is allowed.
-        """
-    @property
-    def manage_users(self) -> builtins.bool:
-        r"""
-        Whether managing users is allowed; includes `read_users`.
-        """
-    @property
-    def read_users(self) -> builtins.bool:
-        r"""
-        Whether reading user info is allowed.
-        """
-    @property
-    def manage_streams(self) -> builtins.bool:
-        r"""
-        Whether managing all streams is allowed; includes `read_streams` and
-        `manage_topics`.
-        """
-    @property
-    def read_streams(self) -> builtins.bool:
-        r"""
-        Whether reading all streams is allowed; includes `read_topics`.
-        """
-    @property
-    def manage_topics(self) -> builtins.bool:
-        r"""
-        Whether managing all topics is allowed; includes `read_topics` and
-        `send_messages`.
-        """
-    @property
-    def read_topics(self) -> builtins.bool:
-        r"""
-        Whether reading all topics and managing consumer groups is allowed;
-        includes `poll_messages`.
-        """
-    @property
-    def poll_messages(self) -> builtins.bool:
-        r"""
-        Whether polling messages from all streams and managing consumer
-        offsets is allowed.
-        """
-    @property
-    def send_messages(self) -> builtins.bool:
-        r"""
-        Whether sending messages to all streams is allowed.
-        """
-    def __eq__(self, other: builtins.object, /) -> builtins.bool: ...
-    def __new__(
-        cls,
-        *,
-        manage_servers: builtins.bool = False,
-        read_servers: builtins.bool = False,
-        manage_users: builtins.bool = False,
-        read_users: builtins.bool = False,
-        manage_streams: builtins.bool = False,
-        read_streams: builtins.bool = False,
-        manage_topics: builtins.bool = False,
-        read_topics: builtins.bool = False,
-        poll_messages: builtins.bool = False,
-        send_messages: builtins.bool = False,
-    ) -> GlobalPermissions:
-        r"""
-        Create global permissions. Every flag defaults to `False`.
-
-        The `includes` notes below are transitive: a flag also grants everything
-        its included flags grant. For example `manage_streams` includes
-        `manage_topics`, and through it `read_topics`, `poll_messages`, and
-        `send_messages`.
-
-        Args:
-            manage_servers: Allow managing servers; includes `read_servers`.
-            read_servers: Allow reading server info (stats, clients).
-            manage_users: Allow managing users; includes `read_users`.
-            read_users: Allow reading user info.
-            manage_streams: Allow managing all streams; includes `read_streams`
-                and `manage_topics`.
-            read_streams: Allow reading all streams; includes `read_topics`.
-            manage_topics: Allow managing all topics; includes `read_topics`
-                and `send_messages`.
-            read_topics: Allow reading all topics and managing consumer groups
-                (including create and delete); includes `poll_messages`.
-            poll_messages: Allow polling messages from all streams and managing
-                consumer offsets.
-            send_messages: Allow sending messages to all streams.
-        """
-
 @typing.final
 class IggyClient:
     r"""
@@ -1315,13 +1345,17 @@ class IggyClient:
         self,
         stream: builtins.str | builtins.int,
         topic: builtins.str | builtins.int,
-        partition_id: builtins.int,
+        *,
+        consumer: Consumer,
         polling_strategy: PollingStrategy,
         count: builtins.int,
         auto_commit: builtins.bool,
+        partition_id: builtins.int | None = None,
     ) -> collections.abc.Awaitable[list[ReceiveMessage]]:
         r"""
-        Polls for messages from the specified topic and partition.
+        Polls for messages from the specified topic on behalf of the given consumer.
+        Omitting `partition_id` reads partition 0 for a regular consumer, and
+        polls the member's assigned partitions for a consumer group.
         Returns a list of received messages or a RuntimeError on failure.
         """
     def consumer_group(
@@ -1378,11 +1412,15 @@ class IggyConsumer:
         self, partition_id: builtins.int
     ) -> builtins.int | None:
         r"""
-        Get the last consumed offset or `None` if no offset has been consumed yet.
+        Get the last consumed offset for the given partition, or `None` while that partition
+        is untracked. Polling starts tracking a partition at `0`, so `0` also means
+        "seen, nothing consumed yet".
         """
     def get_last_stored_offset(self, partition_id: builtins.int) -> builtins.int | None:
         r"""
-        Get the last stored offset or `None` if no offset has been stored yet.
+        Get the last stored offset for the given partition, or `None` while that partition is
+        untracked. Polling starts tracking a partition at `0`, so `0` also means
+        "seen, nothing stored yet", including under `AutoCommit.Disabled()`.
         """
     def name(self) -> builtins.str:
         r"""
@@ -1394,11 +1432,11 @@ class IggyConsumer:
         """
     def stream(self) -> builtins.str | builtins.int:
         r"""
-        Gets the name of the stream this consumer group is configured for.
+        Gets the identifier of the stream this consumer group is configured for.
         """
     def topic(self) -> builtins.str | builtins.int:
         r"""
-        Gets the name of the topic this consumer group is configured for.
+        Gets the identifier of the topic this consumer group is configured for.
         """
     def store_offset(
         self, offset: builtins.int, partition_id: builtins.int | None
@@ -1714,6 +1752,10 @@ class ReceiveMessage:
     def user_headers(self) -> UserHeaders | None:
         r"""
         Retrieves user headers attached to the received message.
+
+        Returns `None` when no headers are present or when the headers
+        on the wire are structurally malformed (those errors are logged
+        internally). Only known semantic decode errors raise `ValueError`.
         """
 
 @typing.final
@@ -1978,19 +2020,25 @@ class TcpReconnectionConfig:
 
         Args:
             enabled: Whether to reconnect at all. Defaults to enabled.
-            max_retries: Attempts before giving up, or `None` for unlimited.
-                Defaults to unlimited, which means a call awaited while the server
-                is down never returns: `connect()`, `send_messages()` and
+            max_retries: Passes over the known endpoints after the first, or
+                `None` for unlimited; `0` still makes that first pass. One pass
+                tries the endpoint the client is on, the address it was
+                configured with, and every node the roster named, so this counts
+                passes rather than dials. Defaults
+                to unlimited, which means a call awaited while the server is
+                down never returns: `connect()`, `send_messages()` and
                 `poll_messages()` all wait inside the retry loop. Set a finite
                 number for request/reply style usage, so a call fails instead.
-            interval: Delay between attempts. Defaults to 1 second.
-            reestablish_after: Cooldown before reconnecting after a previously
-                successful connection. Defaults to 5 seconds.
+            interval: Delay between passes. Defaults to 1 second. The first pass
+                runs at once when more than one endpoint is known.
+            reestablish_after: Cooldown before redialing the endpoint of the last
+                successful connection, measured from when it was established, so
+                a session that outlived the interval is redialed at once. Owed to
+                that endpoint alone. Defaults to 5 seconds.
 
         Raises:
             ValueError: If a duration is negative, if `max_retries` is outside the
-                range of an unsigned 32-bit integer, or if `interval` is zero while
-                reconnection is enabled and `max_retries` is unlimited.
+                range of an unsigned 32-bit integer, or if `interval` is zero.
         """
     def __repr__(self) -> builtins.str: ...
 
@@ -2188,6 +2236,36 @@ class TopicPermissions:
             send_messages: Allow sending messages to the topic.
         """
 
+@typing.final
+class UserHeaders(dict):
+    r"""
+    User headers dictionary returned by `ReceiveMessage.user_headers`.
+
+    This is a regular `dict[HeaderKey, HeaderValue]` (so all mapping
+    operations work) that additionally exposes `to_scalar_dict` for the convenient
+    scalar form.
+    """
+    def __new__(cls, mapping: dict | None = None) -> UserHeaders:
+        r"""
+        Wraps a mapping so its entries gain the `to_scalar_dict` helper.
+
+        Accepts a dict whose keys and values can each independently be
+        `HeaderKey`/`HeaderValue` or a plain scalar (`str | bytes | bool |
+        int | float`). The inherited `dict` initializer copies the provided
+        mapping.
+        """
+    def __setitem__(self, key: typing.Any, value: typing.Any) -> None: ...
+    def to_scalar_dict(
+        self,
+    ) -> dict[str | bytes | bool | int | float, str | bytes | bool | int | float]:
+        r"""
+        Converts these headers into the convenient plain dictionary form.
+
+        Returns an error if two distinct typed keys map to the same plain
+        Python scalar (e.g., `UnsignedInt8(1)` and `UnsignedInt16(1)` both
+        become `int(1)`), or if a stored field cannot be decoded.
+        """
+
 @typing.final
 class UserInfo:
     @property
@@ -2253,31 +2331,3 @@ class UserStatus(enum.Enum):
     r"""
     The user account is inactive and cannot be used.
     """
-
-class UserHeaders(dict):
-    r"""
-    User headers dictionary returned by `ReceiveMessage.user_headers`.
-
-    This is a regular `dict[HeaderKey, HeaderValue]` (so all mapping
-    operations work) that additionally exposes `to_scalar_dict` for the convenient
-    scalar form.
-    """
-    def __new__(cls, mapping: dict | None = None) -> UserHeaders:
-        r"""
-        Wraps a mapping so its entries gain the `to_scalar_dict` helper.
-
-        Accepts a dict whose keys and values can each independently be
-        `HeaderKey`/`HeaderValue` or a plain scalar (`str | bytes | bool |
-        int | float`). The inherited `dict` initializer copies the provided
-        mapping.
-        """
-    def to_scalar_dict(
-        self,
-    ) -> dict[str | bytes | bool | int | float, str | bytes | bool | int | float]:
-        r"""
-        Converts these headers into the convenient plain dictionary form.
-
-        Returns an error if two distinct typed keys map to the same plain
-        Python scalar (e.g., `UnsignedInt8(1)` and `UnsignedInt16(1)` both
-        become `int(1)`), or if a stored field cannot be decoded.
-        """

{apache_iggy-0.9.0.dev3 → apache_iggy-0.9.0.dev5}/core/binary_protocol/Cargo.toml

@@ -17,7 +17,7 @@
 
 [package]
 name = "iggy_binary_protocol"
-version = "0.11.0-edge.3"
+version = "0.11.0-edge.5"
 description = "Wire protocol types and codec for the Iggy binary protocol. Shared between server and SDK."
 edition = "2024"
 rust-version.workspace = true
@@ -44,3 +44,6 @@ aligned-vec = { workspace = true }
 enum_glob_use = "deny"
 nursery = "warn"
 pedantic = "deny"
+
+[lints.rust]
+warnings = "deny"

{apache_iggy-0.9.0.dev3 → apache_iggy-0.9.0.dev5}/core/binary_protocol/README.md

@@ -14,12 +14,6 @@ Wire protocol types and codec for the [Apache Iggy](https://iggy.apache.org) bin
 
 The language-neutral wire spec for the login protocol-version handshake (packed version layout, `ClientVersionInfo` prefix, gate semantics, rejection frame offsets) lives in the [`version` module docs](src/version.rs).
 
-> Apache Iggy (Incubating) is an effort undergoing incubation at the Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC.
->
-> Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects.
->
-> While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
-
 ## Links
 
 - [Project website](https://iggy.apache.org)