apache-iggy 0.9.0.dev8__tar.gz → 0.9.1.dev1__tar.gz

{apache_iggy-0.9.0.dev8 → apache_iggy-0.9.1.dev1}/Cargo.toml

@@ -50,6 +50,7 @@ members = [
     "core/connectors/sinks/stdout_sink",
     "core/connectors/sinks/surrealdb_sink",
     "core/connectors/sources/elasticsearch_source",
+    "core/connectors/sources/http_source",
     "core/connectors/sources/influxdb_source",
     "core/connectors/sources/postgres_source",
     "core/connectors/sources/random_source",
@@ -91,6 +92,7 @@ aligned-vec = "0.6.4"
 anyhow = "1.0.104"
 apache-avro = "0.22.0"
 apple-native-keyring-store = { version = "1.0.2", features = ["keychain"] }
+arc-swap = "1.9.2"
 argon2 = "0.6.0"
 # arrow 58: iceberg 0.10.1 and deltalake 0.32.4 still require it.
 arrow = "58.4.0"
@@ -112,8 +114,8 @@ bench-report = { path = "core/bench/report" }
 bench-runner = { path = "core/bench/runner" }
 bit-set = "0.11.1"
 blake3 = "1.8.7"
-bon = "3.10.0"
-byte-unit = { version = "5.2.5", default-features = false, features = ["serde", "byte", "std"] }
+bon = "3.10.1"
+byte-unit = { version = "5.2.6", default-features = false, features = ["serde", "byte", "std"] }
 bytemuck = { version = "1.25", features = ["derive", "min_const_generics"] }
 bytes = "1.12.1"
 cfg_aliases = "0.2.2"
@@ -146,8 +148,9 @@ configs_derive = { path = "core/configs_derive", version = "0.1.0" }
 consensus = { path = "core/consensus" }
 console-subscriber = "0.5.0"
 cpu_allocation = { path = "core/cpu_allocation" }
-crossbeam = "0.8.4"
-crossfire = "3.1.19"
+crc32c = "0.6.8"
+crossbeam = "0.8.5"
+crossfire = "3.1.20"
 csv = "1.4.0"
 ctor = "1.0.13"
 ctrlc = { version = "3.5", features = ["termination"] }
@@ -162,19 +165,20 @@ derive-new = "0.7.0"
 derive_builder = "0.20.2"
 derive_more = { version = "2.1.1", features = ["full"] }
 dircpy = "0.3.20"
-dirs = "6.0.0"
+dirs = "7.0.0"
 dlopen2 = "0.9.0"
 dotenvy = "0.15.7"
 dtor = "1.0.6"
 elasticsearch = { version = "9.1.0-alpha.1", features = ["rustls-tls"], default-features = false }
 enumset = "1.1"
 env_logger = "0.11.11"
-err_trail = { version = "0.11.1", features = ["tracing"] }
+err_trail = { version = "0.11.2", features = ["tracing"] }
 error_set = "0.9.2"
 figlet-rs = "1.0.0"
 figment = { version = "0.10.19", features = ["toml", "env"] }
 file-operation = "0.8.32"
 flatbuffers = "25.12.19"
+flate2 = "1.1.10"
 flume = "0.12.0"
 fs2 = "0.4.3"
 futures = "0.3.34"
@@ -199,16 +203,16 @@ hyper-util = { version = "0.1.20", features = ["server-auto", "service"] }
 iceberg = "0.10.1"
 iceberg-catalog-rest = "0.10.1"
 iceberg-storage-opendal = "0.10.1"
-iggy = { path = "core/sdk", version = "0.11.0-edge.8" }
-iggy-cli = { path = "core/cli", version = "0.14.0-edge.8" }
+iggy = { path = "core/sdk", version = "0.11.1-edge.1" }
+iggy-cli = { path = "core/cli", version = "0.14.1-edge.1" }
 iggy-gateway-kafka = { path = "gateways/kafka" }
-iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.0-edge.8" }
-iggy_common = { path = "core/common", version = "0.11.0-edge.8" }
+iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.1-edge.1" }
+iggy_common = { path = "core/common", version = "0.11.1-edge.1" }
 iggy_connector_doris_sink = { path = "core/connectors/sinks/doris_sink" }
-iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.4.0-edge.5" }
-indexmap = "2.14.1"
+iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.5.1-edge.1" }
+indexmap = "2.14.2"
 integration = { path = "core/integration" }
-ipnet = "2.12.1"
+ipnet = "2.12.2"
 journal = { path = "core/journal" }
 js-sys = "0.3"
 jsonwebtoken = { version = "11.0.0", features = ["rust_crypto"] }
@@ -223,6 +227,7 @@ lazy_static = "1.5.0"
 left-right = "=0.11.8"
 libc = "0.2.189"
 log = "0.4.34"
+lz4 = "1.28.1"
 lz4_flex = "0.14.0"
 meilisearch-sdk = { version = "0.33.0", default-features = false, features = [
     "reqwest",
@@ -234,11 +239,11 @@ metadata = { path = "core/metadata" }
 mimalloc = "0.1"
 mime_guess = "2.0"
 mockall = "0.15.0"
-mongodb = { version = "3.8.2", features = ["rustls-tls"] }
+mongodb = { version = "3.9.1", features = ["rustls-tls"] }
 nix = { version = "0.31.3", features = ["feature", "fs", "resource", "sched"] }
 nonzero_lit = "0.1.2"
 notify = "8.2.0"
-octocrab = "0.54.1"
+octocrab = "0.54.2"
 opentelemetry = { version = "0.32.0", features = ["trace", "logs"] }
 opentelemetry-appender-tracing = { version = "0.32.0", features = ["log"] }
 opentelemetry-otlp = { version = "0.32.0", features = [
@@ -278,19 +283,19 @@ rand_xoshiro = "0.8.1"
 rayon = "1.12.0"
 rcgen = "0.14.10"
 regex = "1.13.1"
-reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls"] }
+reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls"] }
 reqwest-middleware = { version = "0.5.2", features = ["json", "query"] }
 reqwest-retry = "0.9.1"
 reqwest-tracing = "0.7.1"
 ring = "0.17.14"
 ringbuffer = "0.16.0"
-rmcp = "3.2.0"
+rmcp = "3.3.0"
 rmp = "0.8.15"
 rmp-serde = "1.3.1"
 rolling-file = "0.2.0"
 rust-embed = "8.12.0"
 rust-s3 = { version = "0.37.2", default-features = false, features = ["tokio-rustls-tls", "tags"] }
-rustls = { version = "0.23.43", features = ["ring"] }
+rustls = { version = "0.23.45", features = ["ring"] }
 rustls-pemfile = "2.2.0"
 scopeguard = "1.2.0"
 sd-notify = "0.5"
@@ -298,7 +303,7 @@ secrecy = { version = "0.10", features = ["serde"] }
 send_wrapper = "0.6.0"
 serde = { version = "1.0.229", features = ["derive", "rc"] }
 serde_json = "1.0.151"
-serde_with = { version = "3.22.0", features = ["base64", "macros"] }
+serde_with = { version = "3.23.0", features = ["base64", "macros"] }
 serde_yaml_ng = "0.10.0"
 serial_test = "4.0.1"
 server = { path = "core/server", default-features = false }
@@ -306,13 +311,17 @@ server_common = { path = "core/server_common" }
 shard = { path = "core/shard" }
 simd-json = { version = "0.18.1", features = ["serde_impl"] }
 smallvec = "1.16"
+snap = "1.1.2"
 socket2 = "0.6.5"
-sqlparser = { version = "0.62.0", features = ["visitor"] }
+sqlparser = { version = "0.63.0", features = ["visitor"] }
 sqlx = { version = "0.9.0", features = [
     "runtime-tokio",
     "tls-rustls",
     "postgres",
-    # "mysql": the Doris integration fixture talks to Doris over its MySQL frontend.
+    "bigdecimal",
+    # "mysql": Doris exposes a MySQL-wire frontend; the Doris sink's
+    # integration-test fixture talks to it over this driver. Cargo unifies
+    # features across the workspace, so it is declared once here.
     "mysql",
     "chrono",
     "uuid",
@@ -334,10 +343,10 @@ testcontainers-modules = { version = "0.15.0", features = ["postgres", "http_wai
 thiserror = "2.0.20"
 tokio = { version = "1.53.1", features = ["full"] }
 tokio-postgres = "0.7.18"
-tokio-rustls = "0.26.4"
+tokio-rustls = "0.26.5"
 tokio-tungstenite = { version = "0.30", features = ["rustls-tls-webpki-roots"] }
 tokio-util = { version = "0.7.19", features = ["compat"] }
-toml = "1.1.4"
+toml = "1.1.6"
 tower-http = { version = "0.7.1", features = ["add-extension", "cors", "trace"] }
 tracing = "0.1.44"
 tracing-appender = "0.2.5"
@@ -353,7 +362,7 @@ twox-hash = { version = "2.1.4", features = ["xxhash32"] }
 ulid = "3.0.0"
 ureq = "3.4"
 url = "2.5.8"
-uuid = { version = "1.26.0", features = ["v4", "v7", "fast-rng", "serde", "zerocopy"] }
+uuid = { version = "1.26.1", features = ["v4", "v7", "fast-rng", "serde", "zerocopy"] }
 vergen-git2 = { version = "10.0.1", features = ["build", "cargo", "rustc", "si"] }
 walkdir = "2.5.0"
 wasm-bindgen = "0.2"
@@ -374,6 +383,7 @@ yew-router = "0.20"
 zbus-secret-service-keyring-store = { version = "1.0.1", features = ["rt-async-io-crypto-rust"] }
 zeroize = "1.9.0"
 zip = { version = "8.6.0", default-features = false, features = ["deflate"] }
+zstd = "0.13.3"
 
 [profile.release]
 lto = true

{apache_iggy-0.9.0.dev8 → apache_iggy-0.9.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-iggy
-Version: 0.9.0.dev8
+Version: 0.9.1.dev1
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
@@ -79,14 +79,19 @@ pip install apache-iggy
 
 ### Prerequisites
 
-Every installation below compiles the Rust extension, so you'll need:
-
 - Python 3.10+
+
+Published wheels include the Rust extension; installing a wheel does not require
+Rust. Building from source and running the development checks below also requires:
+
 - Rust toolchain: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
 - `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh`
 - All checks tooling from [CONTRIBUTING.md](https://github.com/apache/iggy/blob/master/CONTRIBUTING.md).
 - Docker
 
+Use an SDK release compatible with your server. For unreleased changes, build
+the SDK and server from the same source checkout.
+
 ### Local Development
 
 **IMPORTANT: All commands are supposed to be ran from `foreign/python` unless it's specified to run in repository's root folder.**
@@ -255,6 +260,155 @@ async def main():
 asyncio.run(main())
 ```
 
+## High-Level Producer
+
+The Python high-level producer API is a port of the Rust high-level producer
+API. For detailed producer semantics and configuration guidance, see the
+[Rust high-level SDK documentation](https://iggy.apache.org/docs/sdk/rust/high-level-sdk/).
+
+Use `IggyClient.producer()` when an application repeatedly publishes to one
+stream and topic. Producer creation is asynchronous because it initializes the
+destination before returning. By default, it creates a missing stream and topic,
+uses balanced partitioning, sends directly in batches of up to 1,000 messages,
+and retries failed sends up to three times with a one-second retry interval.
+
+The default mode is direct. Pass `BackgroundProducerConfig` to queue sends on
+background workers instead.
+
+```python
+import asyncio
+from datetime import timedelta
+
+from apache_iggy import DirectProducerConfig, IggyClient, Partitioning, SendMessage
+
+
+async def main():
+    client = IggyClient.from_connection_string("iggy+tcp://iggy:iggy@127.0.0.1:8090")
+    await client.connect()
+
+    producer = await client.producer(
+        "orders",
+        "created",
+        partitioning=Partitioning.balanced(),
+        mode=DirectProducerConfig(
+            batch_length=500,
+            linger_time=timedelta(milliseconds=5),
+        ),
+        create_stream_if_not_exists=True,
+        create_topic_if_not_exists=True,
+        topic_partitions_count=3,
+        topic_message_expiry=None,
+        topic_max_size=None,
+        send_retries=3,
+        send_retry_interval=timedelta(seconds=1),
+    )
+
+    async with producer:
+        await producer.send_one(SendMessage("order-1"))
+        response = await producer.send([SendMessage("order-2"), SendMessage("order-3")])
+        print(f"Received {len(response.confirmations)} partition confirmations")
+
+
+asyncio.run(main())
+```
+
+The producer is bound to the stream and topic passed to `producer()`. Use
+`send_with_partitioning(messages, partitioning)` to override its partitioning
+strategy for one call, or `send_to(stream, topic, messages, partitioning)` to
+send to another existing destination. `send_to()` does not create or initialize
+that destination.
+
+Direct sends use at-least-once delivery. A request can commit even when its
+response is lost, so any retry can write the same batch again. `send_retries`
+counts retries after the initial attempt. The first retry runs immediately, and
+`send_retry_interval` delays only later retries. Set `send_retries` to `None` or
+`0` to disable producer retries. Set `send_retry_interval` to `None` to run all
+enabled retries without a delay. A zero interval raises `ValueError`.
+
+Transport retries are separate from producer retries. For example, the default
+`HttpConfig(retries=3)` gives each producer attempt up to four HTTP attempts.
+
+A failed direct send raises `ProducerSendError`, which is a `RuntimeError`
+subclass. Its `cause` property contains the underlying error text. Its
+`committed` property contains confirmations from completed chunks, and `failed`
+contains the remaining unconfirmed messages. If encryption is enabled, the
+failed messages contain encrypted payloads. Restore the original payloads
+before submitting them to the same producer again.
+
+### Background Mode
+
+A successful background send means the dispatcher accepted the messages. The
+worker writes them later, so all four send methods return a
+`SendMessagesResponse` with an empty `confirmations` list. It does not mean the
+server has committed the messages.
+
+```python
+from datetime import timedelta
+
+from apache_iggy import (
+    BackgroundProducerConfig,
+    BackpressureMode,
+    ProducerSharding,
+    SendMessage,
+)
+
+producer = await client.producer(
+    "orders",
+    "created",
+    mode=BackgroundProducerConfig(
+        num_shards=4,
+        linger_time=timedelta(milliseconds=10),
+        batch_size=1024 * 1024,
+        batch_length=100,
+        max_buffer_size=32 * 1024 * 1024,
+        failure_mode=BackpressureMode.block_with_timeout(timedelta(seconds=1)),
+        max_in_flight=4,
+        sharding=ProducerSharding.ORDERED,
+    ),
+)
+
+async with producer:
+    accepted = await producer.send_one(SendMessage("order-1"))
+    assert accepted.confirmations == []
+```
+
+Each shard flushes when any configured condition is met: `batch_length` queued
+send calls, `batch_size` reported bytes, or `linger_time` since the first send
+entered an empty buffer. `batch_length` counts calls, not individual messages.
+Zero disables either batching threshold, while a zero linger flushes as soon as
+the worker receives a send.
+
+`ProducerSharding.ORDERED` hashes the stream/topic destination so its sends use
+one sequential worker and retain dispatch order. `ProducerSharding.BALANCED`
+assigns consecutive sends round-robin across the shards for throughput; ordering
+for one destination is not guaranteed.
+
+`max_buffer_size` bounds bytes queued or in flight across the whole producer.
+When it is full, `BackpressureMode.block()` waits indefinitely,
+`block_with_timeout(duration)` waits up to that duration, and
+`fail_immediately()` raises `RuntimeError` without accepting the batch. A single
+batch larger than the whole budget always fails. A zero byte budget is
+unlimited. `max_in_flight` separately bounds concurrent write requests across
+all shards; zero uses the runtime maximum.
+
+Producer retries and transport reconnection happen inside background workers.
+The Python API does not currently expose a background error callback, so a write
+that still fails after its retries is logged by the Rust SDK and its unconfirmed
+messages are dropped.
+
+See the complete runnable
+[`background_producer.py`](../../examples/python/high-level/background_producer.py)
+example for all background configuration fields and deterministic shutdown.
+
+Cleanup is asynchronous and must be explicit. Prefer `async with`, as above, so
+shutdown runs on both successful and exceptional exits. Otherwise, call
+`await producer.shutdown()` in a `finally` block. Shutdown waits for active
+sends, is safe to call more than once, and rejects later sends with
+`RuntimeError`. In background mode it also drains every queue and flushes all
+accepted messages before returning. Object destruction does not perform
+asynchronous cleanup; dropping a background producer without `shutdown()` can
+lose buffered messages.
+
 ## Examples
 
 Refer to the [examples/python/](https://github.com/apache/iggy/tree/master/examples/python) directory for usage examples.

{apache_iggy-0.9.0.dev8/foreign/python → apache_iggy-0.9.1.dev1}/README.md

@@ -28,14 +28,19 @@ pip install apache-iggy
 
 ### Prerequisites
 
-Every installation below compiles the Rust extension, so you'll need:
-
 - Python 3.10+
+
+Published wheels include the Rust extension; installing a wheel does not require
+Rust. Building from source and running the development checks below also requires:
+
 - Rust toolchain: `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
 - `uv`: `curl -LsSf https://astral.sh/uv/install.sh | sh`
 - All checks tooling from [CONTRIBUTING.md](https://github.com/apache/iggy/blob/master/CONTRIBUTING.md).
 - Docker
 
+Use an SDK release compatible with your server. For unreleased changes, build
+the SDK and server from the same source checkout.
+
 ### Local Development
 
 **IMPORTANT: All commands are supposed to be ran from `foreign/python` unless it's specified to run in repository's root folder.**
@@ -204,6 +209,155 @@ async def main():
 asyncio.run(main())
 ```
 
+## High-Level Producer
+
+The Python high-level producer API is a port of the Rust high-level producer
+API. For detailed producer semantics and configuration guidance, see the
+[Rust high-level SDK documentation](https://iggy.apache.org/docs/sdk/rust/high-level-sdk/).
+
+Use `IggyClient.producer()` when an application repeatedly publishes to one
+stream and topic. Producer creation is asynchronous because it initializes the
+destination before returning. By default, it creates a missing stream and topic,
+uses balanced partitioning, sends directly in batches of up to 1,000 messages,
+and retries failed sends up to three times with a one-second retry interval.
+
+The default mode is direct. Pass `BackgroundProducerConfig` to queue sends on
+background workers instead.
+
+```python
+import asyncio
+from datetime import timedelta
+
+from apache_iggy import DirectProducerConfig, IggyClient, Partitioning, SendMessage
+
+
+async def main():
+    client = IggyClient.from_connection_string("iggy+tcp://iggy:iggy@127.0.0.1:8090")
+    await client.connect()
+
+    producer = await client.producer(
+        "orders",
+        "created",
+        partitioning=Partitioning.balanced(),
+        mode=DirectProducerConfig(
+            batch_length=500,
+            linger_time=timedelta(milliseconds=5),
+        ),
+        create_stream_if_not_exists=True,
+        create_topic_if_not_exists=True,
+        topic_partitions_count=3,
+        topic_message_expiry=None,
+        topic_max_size=None,
+        send_retries=3,
+        send_retry_interval=timedelta(seconds=1),
+    )
+
+    async with producer:
+        await producer.send_one(SendMessage("order-1"))
+        response = await producer.send([SendMessage("order-2"), SendMessage("order-3")])
+        print(f"Received {len(response.confirmations)} partition confirmations")
+
+
+asyncio.run(main())
+```
+
+The producer is bound to the stream and topic passed to `producer()`. Use
+`send_with_partitioning(messages, partitioning)` to override its partitioning
+strategy for one call, or `send_to(stream, topic, messages, partitioning)` to
+send to another existing destination. `send_to()` does not create or initialize
+that destination.
+
+Direct sends use at-least-once delivery. A request can commit even when its
+response is lost, so any retry can write the same batch again. `send_retries`
+counts retries after the initial attempt. The first retry runs immediately, and
+`send_retry_interval` delays only later retries. Set `send_retries` to `None` or
+`0` to disable producer retries. Set `send_retry_interval` to `None` to run all
+enabled retries without a delay. A zero interval raises `ValueError`.
+
+Transport retries are separate from producer retries. For example, the default
+`HttpConfig(retries=3)` gives each producer attempt up to four HTTP attempts.
+
+A failed direct send raises `ProducerSendError`, which is a `RuntimeError`
+subclass. Its `cause` property contains the underlying error text. Its
+`committed` property contains confirmations from completed chunks, and `failed`
+contains the remaining unconfirmed messages. If encryption is enabled, the
+failed messages contain encrypted payloads. Restore the original payloads
+before submitting them to the same producer again.
+
+### Background Mode
+
+A successful background send means the dispatcher accepted the messages. The
+worker writes them later, so all four send methods return a
+`SendMessagesResponse` with an empty `confirmations` list. It does not mean the
+server has committed the messages.
+
+```python
+from datetime import timedelta
+
+from apache_iggy import (
+    BackgroundProducerConfig,
+    BackpressureMode,
+    ProducerSharding,
+    SendMessage,
+)
+
+producer = await client.producer(
+    "orders",
+    "created",
+    mode=BackgroundProducerConfig(
+        num_shards=4,
+        linger_time=timedelta(milliseconds=10),
+        batch_size=1024 * 1024,
+        batch_length=100,
+        max_buffer_size=32 * 1024 * 1024,
+        failure_mode=BackpressureMode.block_with_timeout(timedelta(seconds=1)),
+        max_in_flight=4,
+        sharding=ProducerSharding.ORDERED,
+    ),
+)
+
+async with producer:
+    accepted = await producer.send_one(SendMessage("order-1"))
+    assert accepted.confirmations == []
+```
+
+Each shard flushes when any configured condition is met: `batch_length` queued
+send calls, `batch_size` reported bytes, or `linger_time` since the first send
+entered an empty buffer. `batch_length` counts calls, not individual messages.
+Zero disables either batching threshold, while a zero linger flushes as soon as
+the worker receives a send.
+
+`ProducerSharding.ORDERED` hashes the stream/topic destination so its sends use
+one sequential worker and retain dispatch order. `ProducerSharding.BALANCED`
+assigns consecutive sends round-robin across the shards for throughput; ordering
+for one destination is not guaranteed.
+
+`max_buffer_size` bounds bytes queued or in flight across the whole producer.
+When it is full, `BackpressureMode.block()` waits indefinitely,
+`block_with_timeout(duration)` waits up to that duration, and
+`fail_immediately()` raises `RuntimeError` without accepting the batch. A single
+batch larger than the whole budget always fails. A zero byte budget is
+unlimited. `max_in_flight` separately bounds concurrent write requests across
+all shards; zero uses the runtime maximum.
+
+Producer retries and transport reconnection happen inside background workers.
+The Python API does not currently expose a background error callback, so a write
+that still fails after its retries is logged by the Rust SDK and its unconfirmed
+messages are dropped.
+
+See the complete runnable
+[`background_producer.py`](../../examples/python/high-level/background_producer.py)
+example for all background configuration fields and deterministic shutdown.
+
+Cleanup is asynchronous and must be explicit. Prefer `async with`, as above, so
+shutdown runs on both successful and exceptional exits. Otherwise, call
+`await producer.shutdown()` in a `finally` block. Shutdown waits for active
+sends, is safe to call more than once, and rejects later sends with
+`RuntimeError`. In background mode it also drains every queue and flushes all
+accepted messages before returning. Object destruction does not perform
+asynchronous cleanup; dropping a background producer without `shutdown()` can
+lose buffered messages.
+
 ## Examples
 
 Refer to the [examples/python/](https://github.com/apache/iggy/tree/master/examples/python) directory for usage examples.