mq-bridge-py-basic 0.4.1__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/Cargo.lock +4 -4
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/Cargo.toml +1 -1
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/PKG-INFO +1 -1
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/docs/ARCHITECTURE.md +13 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/mq-bridge.schema.json +1 -1
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/command_handler.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/file/mod.rs +6 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/file/tests.rs +94 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/object_store.rs +8 -2
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/sqlx/mod.rs +35 -2
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/sqlx/tests.rs +120 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/structural/fanout.rs +5 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/structural/request.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/structural/switch.rs +8 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/buffer.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/compression.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/cookie_jar.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/delay.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/dlq.rs +8 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/encryption.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/limiter.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/metrics.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/random_panic.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/retry.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/transform/mod.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/models.rs +3 -1
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/plugin/middleware.rs +4 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/route.rs +225 -8
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/traits.rs +38 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/dependabot.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/workflows/benchmark.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/workflows/ci.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/workflows/ibm-mq.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/workflows/node.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/workflows/publish-node.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/workflows/publish-python.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/workflows/python.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/workflows/release.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.github/workflows/test-matrix.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/.gitignore +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/CHANGELOG.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/CLAUDE.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/CONTRIBUTING.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/LICENSE +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/LICENSE.python +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/README.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/SECURITY.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/benches/ETL_BENCHMARKS.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/benches/performance_bench.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/benches/router_bench.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/benches/zeromq_backends.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/bindings-common/Cargo.toml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/bindings-common/src/lib.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/bindings-common/src/logging.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/build.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/deny.toml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/docs/CONFIGURATION.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/docs/DELIVERY.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/docs/EXTENDING.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/docs/PLUGINS.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/docs/REFERENCE.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/examples/ipc_worker_queue.yaml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/mq_bridge/__init__.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/mq_bridge/__init__.pyi +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/mq_bridge/config.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/mq_bridge/config.pyi +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/mq_bridge/plugin_packaging.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/mq_bridge/py.typed +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/package-lock.json +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/pyproject.toml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/Cargo.toml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/LICENSE.python +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/README.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/analysis/bench_http_native.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/example.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/examples/bench_memory.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/examples/bench_memory.yaml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/examples/consume_pull.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/examples/dlt_source.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/examples/json_route.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/examples/memory.yaml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/examples/raw_route.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/pyproject-basic.toml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/pyproject-reduced.toml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/scripts/gen_config_types.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/src/lib.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/tests/test_config_types.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/tests/test_custom_endpoint.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/tests/test_logging.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/tests/test_packaging.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/tests/test_performance_smoke.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/tests/test_plugin_package.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/tests/test_public_api.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/tests/test_routes.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/python/mq-bridge-py/uv.lock +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/analysis/benches/metadata_bench.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/analysis/debug_ibm_mq_tls.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/analysis/http/analyze_xctrace_time_profile.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/analysis/http/http_flamegraph.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/analysis/http/mq_bridge_http_profile.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/analysis/http/run_http_ladder.sh +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/ci/format_bench_output.sh +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/httparena/README.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/httparena/frameworks/mq-bridge-py/Dockerfile +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/httparena/frameworks/mq-bridge-py/meta.json +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/httparena/frameworks/mq-bridge-py/server.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/techempower/Python/mq-bridge-py/README.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/techempower/Python/mq-bridge-py/benchmark_config.json +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/techempower/Python/mq-bridge-py/mq-bridge-py.dockerfile +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/techempower/Python/mq-bridge-py/server.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/techempower/README.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/techempower/postgres.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/techempower/seed.sql +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/techempower/verify.sh +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/the-benchmarker/README.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/the-benchmarker/python/mq-bridge-py/config.yaml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/the-benchmarker/python/mq-bridge-py/pyproject.toml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/scripts/the-benchmarker/python/mq-bridge-py/server.py +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/canonical_message.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/checkpoint.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/amqp.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/aws.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/clickhouse.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/grpc.proto +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/grpc.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/http/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/http/publisher.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/http/stream.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/http/tests.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/ibm_mq.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/kafka.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/memory/endpoint.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/memory/framed.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/memory/ipc_unix.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/memory/ipc_windows.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/memory/memory_transport.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/memory/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/memory/transport.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/mongodb/consumer.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/mongodb/dedup.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/mongodb/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/mongodb/publisher.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/mongodb/readers.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/mongodb/tests.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/mqtt.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/nats.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/poll.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/postgres/VENDORED.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/postgres/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/postgres/pgoutput/decoder.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/postgres/pgoutput/messages.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/postgres/pgoutput/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/postgres/pgoutput/registry.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/postgres/pgoutput/values.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/postgres/replication.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/postgres/state.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/redis_streams.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/sled.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/sqlx/dedup.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/structural/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/structural/null.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/structural/reader.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/structural/response.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/structural/static_endpoint.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/structural/stream_buffer.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/websocket.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/zeromq/codec.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/zeromq/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/zeromq/omq.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/endpoints/zeromq/zmq.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/errors.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/event_handler.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/event_store.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/extensions.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/lib.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/deduplication.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/id.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/transform/coerce.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/transform/compiled.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/transform/error.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/transform/path.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/transform/schema.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/transform/tests.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/middleware/weak_join.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/models/builders.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/models/defaults.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/models/secrets.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/models/serde_support.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/models/tests.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/outcomes.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/plugin/conformance.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/plugin/endpoint.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/plugin/message.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/plugin/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/plugin/sdk.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/plugin/test_support.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/publisher.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/response.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/support/base64_engine.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/support/compression.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/support/connection_registry.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/support/crypto.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/support/crypto_envelope.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/support/interpolation.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/support/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/support/plugin_abi.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/support/source_ranges.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/test_utils.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/src/type_handler.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/README.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/armature_integration.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/amqp.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/aws.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/clickhouse.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/config.toml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/amqp.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/aws.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/clickhouse.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/ibm-mq-certs/client.crl +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/ibm-mq-certs/client.kdb +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/ibm-mq-certs/client.p12 +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/ibm-mq-certs/client.rdb +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/ibm-mq-certs/client.sth +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/ibm_mq.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/ibm_mq_tls.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/kafka-tls.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/kafka.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mariadb.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mongodb-replica.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mongodb-tls.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mongodb.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mosquitto.conf +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mosquitto_performance.conf +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mosquitto_persistence.conf +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mq_init.mqsc +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mq_init_tls.mqsc +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mqtt.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mqtt_performance.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/mysql.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/nats.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/object_store.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/postgres.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/postgres_cdc.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/docker-compose/redis.yml +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/file.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/grpc.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/grpc_tls.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/http.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/http_tls.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/ibm_mq.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/ibm_mq_tls.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/ipc.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/kafka.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/kafka_tls.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/logic_test.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/mariadb.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/memory.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/mod.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/mongodb.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/mongodb_raw.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/mongodb_tls.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/mqtt.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/mysql.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/nats.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/object_store.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/performance_static.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/postgres.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/postgres_cdc.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/redis_streams.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/route.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/scripts/gen_certs.sh +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/sqlite.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/tls_helpers.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/websocket.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration/zeromq.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/integration_test.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/memory_leak_test.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/ref_test.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/reference_docs_test.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/sqlite_test.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/tls_example.rs +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/unit_tests.md +0 -0
- {mq_bridge_py_basic-0.4.1 → mq_bridge_py_basic-0.4.2}/tests/websocket_test.rs +0 -0
|
@@ -3348,7 +3348,7 @@ dependencies = [
|
|
|
3348
3348
|
|
|
3349
3349
|
[[package]]
|
|
3350
3350
|
name = "mq-bridge"
|
|
3351
|
-
version = "0.4.
|
|
3351
|
+
version = "0.4.2"
|
|
3352
3352
|
dependencies = [
|
|
3353
3353
|
"aes-gcm",
|
|
3354
3354
|
"anyhow",
|
|
@@ -3431,7 +3431,7 @@ dependencies = [
|
|
|
3431
3431
|
|
|
3432
3432
|
[[package]]
|
|
3433
3433
|
name = "mq-bridge-bindings-common"
|
|
3434
|
-
version = "0.4.
|
|
3434
|
+
version = "0.4.2"
|
|
3435
3435
|
dependencies = [
|
|
3436
3436
|
"anyhow",
|
|
3437
3437
|
"fast-uuid-v7",
|
|
@@ -3445,7 +3445,7 @@ dependencies = [
|
|
|
3445
3445
|
|
|
3446
3446
|
[[package]]
|
|
3447
3447
|
name = "mq-bridge-node"
|
|
3448
|
-
version = "0.4.
|
|
3448
|
+
version = "0.4.2"
|
|
3449
3449
|
dependencies = [
|
|
3450
3450
|
"anyhow",
|
|
3451
3451
|
"async-trait",
|
|
@@ -3487,7 +3487,7 @@ dependencies = [
|
|
|
3487
3487
|
|
|
3488
3488
|
[[package]]
|
|
3489
3489
|
name = "mq-bridge-py"
|
|
3490
|
-
version = "0.4.
|
|
3490
|
+
version = "0.4.2"
|
|
3491
3491
|
dependencies = [
|
|
3492
3492
|
"anyhow",
|
|
3493
3493
|
"async-trait",
|
|
@@ -150,6 +150,19 @@ Batch processing is a core concept in mq-bridge and is required for all endpoint
|
|
|
150
150
|
- Each route can be configured with a `concurrency` parameter, which determines how many worker tasks will process batches in parallel.
|
|
151
151
|
- Batch size is also configurable per route.
|
|
152
152
|
|
|
153
|
+
#### Ordering
|
|
154
|
+
Two independent guarantees, each decided once per route by the endpoint itself:
|
|
155
|
+
|
|
156
|
+
- **Commits** follow `MessageConsumer::commit_requires_order()` (default `true`). Cumulative-ack sources such as Kafka funnel their commits through one sequencer; individually-acking sources commit concurrently, bounded by `commit_concurrency_limit`.
|
|
157
|
+
- **Publishing** follows `MessagePublisher::requires_ordered_publish()` (default `false`). Above `concurrency: 1` workers call `send_batch` in parallel, so whole batches can reach the sink out of source order — rows keep their order *within* a batch. The `file` sink declares `true` and the route sequences the `send_batch` calls; batch prep and commits stay parallel, so the ordered path measures within noise of an unordered one.
|
|
158
|
+
|
|
159
|
+
Only `file` declares it, for two reasons that any other candidate has to clear:
|
|
160
|
+
|
|
161
|
+
- **The cost must be low.** The file sink already holds a write lock across the whole batch, so sequencing changes who writes, not how many write at once. Sequencing a sink that ends in a network round trip instead costs the whole concurrency factor — one batch in flight instead of N.
|
|
162
|
+
- **The guarantee must be reachable.** `object_store` does *not* declare it: read order there is object-key order, and the uuidv7 key randomises everything below the millisecond, so ordering the writes would buy nothing. Ordered cloud export needs a source-sequenced key first.
|
|
163
|
+
|
|
164
|
+
Broker sinks keep it off. NATS (both Core and JetStream) publishes a batch through `send_batch_helper`, which keeps up to `SEND_BATCH_CONCURRENCY` publishes in flight, so wire order inside one batch is already unordered — only the results are re-sorted. Kafka and AMQP preserve order within a batch (they submit the whole batch, then await confirms), but for them the order-critical part is the cheap submit loop, not the round trip, so a useful fix would have to release the sequence after submit rather than after `send_batch`. For per-key Kafka ordering today, use `concurrency: 1`.
|
|
165
|
+
|
|
153
166
|
### Helper Utilities
|
|
154
167
|
mq-bridge provides several helper functions to make implementing batching easier, especially if your endpoint only supports single-message operations:
|
|
155
168
|
|
|
@@ -2972,7 +2972,7 @@
|
|
|
2972
2972
|
"minimum": 1
|
|
2973
2973
|
},
|
|
2974
2974
|
"concurrency": {
|
|
2975
|
-
"description": "(Optional) Number of concurrent processing tasks for this route. While it improves throughput for high-latency\nhandlers, it adds synchronization overhead for ordered commits and may lead to out-of-order processing\nin the handler.
|
|
2975
|
+
"description": "(Optional) Number of concurrent processing tasks for this route. While it improves throughput for high-latency\nhandlers, it adds synchronization overhead for ordered commits and may lead to out-of-order processing\nin the handler. Above 1, whole batches may also reach the sink out of source order (rows keep their order\nwithin a batch) unless the sink declares itself order-sensitive, as `file` does.\nDefaults to 1.",
|
|
2976
2976
|
"type": "integer",
|
|
2977
2977
|
"format": "uint",
|
|
2978
2978
|
"default": 1,
|
|
@@ -1017,6 +1017,12 @@ impl MessagePublisher for FilePublisher {
|
|
|
1017
1017
|
Ok(())
|
|
1018
1018
|
}
|
|
1019
1019
|
|
|
1020
|
+
/// A file is an ordered log by construction: appending batches in the order they
|
|
1021
|
+
/// were read is the whole point of exporting to JSONL/CSV.
|
|
1022
|
+
fn requires_ordered_publish(&self) -> bool {
|
|
1023
|
+
true
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1020
1026
|
fn as_any(&self) -> &dyn Any {
|
|
1021
1027
|
self
|
|
1022
1028
|
}
|
|
@@ -912,6 +912,100 @@ async fn test_file_consumer_subscribe_explicit_no_delete() {
|
|
|
912
912
|
|
|
913
913
|
use crate::models::{Endpoint, EndpointType, Route};
|
|
914
914
|
|
|
915
|
+
// Regression (issue #71): the reporter's repro — 20 numbered rows, file -> file,
|
|
916
|
+
// batch_size 5, concurrency 4 — used to emit whole batches out of source order
|
|
917
|
+
// (e.g. 15..19 first). A file is an ordered log, so `FilePublisher` declares
|
|
918
|
+
// `requires_ordered_publish()` and the route sequences the sends.
|
|
919
|
+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
920
|
+
async fn test_route_file_to_file_preserves_order_at_concurrency() {
|
|
921
|
+
let dir = tempdir().unwrap();
|
|
922
|
+
let src = dir.path().join("rows.jsonl");
|
|
923
|
+
let dst = dir.path().join("out.jsonl");
|
|
924
|
+
let rows: String = (0..20).map(|i| format!("{i}\n")).collect();
|
|
925
|
+
tokio::fs::write(&src, rows.as_bytes()).await.unwrap();
|
|
926
|
+
|
|
927
|
+
let input = Endpoint::new(EndpointType::File(FileConfig {
|
|
928
|
+
path: src.to_str().unwrap().to_string(),
|
|
929
|
+
mode: Some(FileConsumerMode::Consume { delete: false }),
|
|
930
|
+
format: FileFormat::Raw,
|
|
931
|
+
..Default::default()
|
|
932
|
+
}));
|
|
933
|
+
let output = Endpoint::new(EndpointType::File(FileConfig {
|
|
934
|
+
path: dst.to_str().unwrap().to_string(),
|
|
935
|
+
format: FileFormat::Raw,
|
|
936
|
+
..Default::default()
|
|
937
|
+
}));
|
|
938
|
+
let route = Route::new(input, output)
|
|
939
|
+
.with_concurrency(4)
|
|
940
|
+
.with_batch_size(5)
|
|
941
|
+
.with_exit_on_empty(true);
|
|
942
|
+
|
|
943
|
+
tokio::time::timeout(
|
|
944
|
+
std::time::Duration::from_secs(10),
|
|
945
|
+
route.run_until_err("file_order_regression", None, None),
|
|
946
|
+
)
|
|
947
|
+
.await
|
|
948
|
+
.expect("Route should drain and exit")
|
|
949
|
+
.expect("Route should complete without errors");
|
|
950
|
+
|
|
951
|
+
let content = tokio::fs::read_to_string(&dst).await.unwrap();
|
|
952
|
+
let written: Vec<&str> = content.lines().collect();
|
|
953
|
+
let expected: Vec<String> = (0..20).map(|i| i.to_string()).collect();
|
|
954
|
+
assert_eq!(written, expected, "File sink must preserve source order");
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
// A buffering publisher only returns from `send_batch` once its buffer flushed, so
|
|
958
|
+
// sequencing sends must not let it wait for a batch that is itself waiting to be sent.
|
|
959
|
+
// `max_delay_ms` guarantees the flush, but the interaction is worth pinning: this hangs
|
|
960
|
+
// if the ordered path ever gates on something the sink needs first.
|
|
961
|
+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
962
|
+
async fn test_route_ordered_file_sink_with_buffer_does_not_stall() {
|
|
963
|
+
use crate::models::{BufferMiddleware, Middleware};
|
|
964
|
+
|
|
965
|
+
let dir = tempdir().unwrap();
|
|
966
|
+
let src = dir.path().join("buf_rows.jsonl");
|
|
967
|
+
let dst = dir.path().join("buf_out.jsonl");
|
|
968
|
+
let rows: String = (0..20).map(|i| format!("{i}\n")).collect();
|
|
969
|
+
tokio::fs::write(&src, rows.as_bytes()).await.unwrap();
|
|
970
|
+
|
|
971
|
+
let input = Endpoint::new(EndpointType::File(FileConfig {
|
|
972
|
+
path: src.to_str().unwrap().to_string(),
|
|
973
|
+
mode: Some(FileConsumerMode::Consume { delete: false }),
|
|
974
|
+
format: FileFormat::Raw,
|
|
975
|
+
..Default::default()
|
|
976
|
+
}));
|
|
977
|
+
let mut output = Endpoint::new(EndpointType::File(FileConfig {
|
|
978
|
+
path: dst.to_str().unwrap().to_string(),
|
|
979
|
+
format: FileFormat::Raw,
|
|
980
|
+
..Default::default()
|
|
981
|
+
}));
|
|
982
|
+
// Buffer larger than one batch, so every flush is timer-driven.
|
|
983
|
+
output
|
|
984
|
+
.middlewares
|
|
985
|
+
.push(Middleware::Buffer(BufferMiddleware {
|
|
986
|
+
max_messages: 50,
|
|
987
|
+
max_delay_ms: 20,
|
|
988
|
+
}));
|
|
989
|
+
|
|
990
|
+
let route = Route::new(input, output)
|
|
991
|
+
.with_concurrency(4)
|
|
992
|
+
.with_batch_size(5)
|
|
993
|
+
.with_exit_on_empty(true);
|
|
994
|
+
|
|
995
|
+
tokio::time::timeout(
|
|
996
|
+
std::time::Duration::from_secs(10),
|
|
997
|
+
route.run_until_err("file_order_buffer", None, None),
|
|
998
|
+
)
|
|
999
|
+
.await
|
|
1000
|
+
.expect("Ordered sends through a buffer must not stall")
|
|
1001
|
+
.expect("Route should complete without errors");
|
|
1002
|
+
|
|
1003
|
+
let content = tokio::fs::read_to_string(&dst).await.unwrap();
|
|
1004
|
+
let written: Vec<&str> = content.lines().collect();
|
|
1005
|
+
let expected: Vec<String> = (0..20).map(|i| i.to_string()).collect();
|
|
1006
|
+
assert_eq!(written, expected);
|
|
1007
|
+
}
|
|
1008
|
+
|
|
915
1009
|
#[tokio::test]
|
|
916
1010
|
async fn test_route_file_consume_explicit_delete() {
|
|
917
1011
|
let dir = tempdir().unwrap();
|
|
@@ -9,8 +9,9 @@
|
|
|
9
9
|
//! - **Sink** ([`ObjectStorePublisher`]): each flushed batch is encoded (reusing the
|
|
10
10
|
//! file endpoint's [`FileFormat`] codecs) and written as one immutable object at
|
|
11
11
|
//! `<prefix>/[YYYY/MM/DD/]<uuidv7>.<ext>`. Objects are write-once; nothing is appended
|
|
12
|
-
//! or mutated. The uuidv7 name sorts by write time
|
|
13
|
-
//!
|
|
12
|
+
//! or mutated. The uuidv7 name sorts by write time to *millisecond* granularity only —
|
|
13
|
+
//! `rand_a`/`rand_b` are random, so objects written within the same millisecond sort in
|
|
14
|
+
//! arbitrary order. The date prefix is a readability / lifecycle-rule convenience only.
|
|
14
15
|
//! - **Source** ([`ObjectStoreConsumer`]): objects under `prefix` are listed in key order,
|
|
15
16
|
//! fetched whole, split by `delimiter`, and emitted. Progress is a durable cursor holding
|
|
16
17
|
//! the last fully-acked object key (via the external checkpoint store), so a restart
|
|
@@ -418,6 +419,11 @@ impl MessagePublisher for ObjectStorePublisher {
|
|
|
418
419
|
Ok(())
|
|
419
420
|
}
|
|
420
421
|
|
|
422
|
+
// Deliberately *not* declaring `requires_ordered_publish()`. Unlike the file sink, read
|
|
423
|
+
// order here is key order, and `next_key` randomises everything below the millisecond, so
|
|
424
|
+
// sequencing the writes would cost a latency-bound sink its concurrency without buying an
|
|
425
|
+
// ordering guarantee. Ordered cloud export needs a source-sequenced key first.
|
|
426
|
+
|
|
421
427
|
fn as_any(&self) -> &dyn Any {
|
|
422
428
|
self
|
|
423
429
|
}
|
|
@@ -1723,6 +1723,22 @@ fn mysql_data_type_is_any_safe(data_type: &str) -> bool {
|
|
|
1723
1723
|
)
|
|
1724
1724
|
}
|
|
1725
1725
|
|
|
1726
|
+
/// SQLite declared column types that sqlx maps to a `DataType` the `Any` driver rejects.
|
|
1727
|
+
/// sqlx-sqlite parses `sqlite3_column_decltype` with `DataType::from_str`, and only these exact
|
|
1728
|
+
/// spellings yield `Bool`/`Date`/`Time`/`Datetime` — everything else either maps to a supported
|
|
1729
|
+
/// type or fails to parse and falls back to the runtime value type, which is always safe.
|
|
1730
|
+
/// Notably this covers the `DATETIME` columns our own `auto_create_table` DDL emits.
|
|
1731
|
+
fn sqlite_decltype_is_any_safe(decl_type: &str) -> bool {
|
|
1732
|
+
!matches!(
|
|
1733
|
+
decl_type,
|
|
1734
|
+
"boolean" | "bool" | "date" | "time" | "datetime" | "timestamp"
|
|
1735
|
+
)
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
/// Binds are the table name and its schema (`main` unless the name was `schema.table`).
|
|
1739
|
+
const SQLITE_COLUMN_TYPES_SQL: &str =
|
|
1740
|
+
"SELECT name AS name, LOWER(type) AS typname FROM pragma_table_info(?, ?) ORDER BY cid";
|
|
1741
|
+
|
|
1726
1742
|
/// `$1::regclass` resolves the (optionally schema-qualified) table name against the current
|
|
1727
1743
|
/// search_path. `pg_attribute.attname`/`pg_type.typname` are Postgres `name`-typed columns,
|
|
1728
1744
|
/// which `Any` cannot decode directly (distinct from `text`), so cast them explicitly.
|
|
@@ -1744,8 +1760,10 @@ const MYSQL_COLUMN_TYPES_SQL: &str = "SELECT COLUMN_NAME AS name, LOWER(DATA_TYP
|
|
|
1744
1760
|
/// query on the first type it cannot map (`TIMESTAMPTZ` on Postgres, `DECIMAL`/`TIMESTAMP` on
|
|
1745
1761
|
/// MySQL). We introspect the table's columns and cast every `Any`-incompatible column to a
|
|
1746
1762
|
/// string type, so the copy succeeds (unmappable values arrive as strings) instead of failing
|
|
1747
|
-
/// every read forever.
|
|
1748
|
-
///
|
|
1763
|
+
/// every read forever. SQLite is affected too: its type info comes from the *declared* type, so
|
|
1764
|
+
/// a `DATETIME` column — including the ones `auto_create_table` writes — breaks the read.
|
|
1765
|
+
/// Any introspection failure (or an unsupported driver) falls back to `*`, preserving the
|
|
1766
|
+
/// previous behaviour.
|
|
1749
1767
|
async fn build_cursor_projection(pool: &AnyPool, driver_name: &str, table: &str) -> String {
|
|
1750
1768
|
type SafeFn = fn(&str) -> bool;
|
|
1751
1769
|
type CastFn = fn(&str) -> String;
|
|
@@ -1772,6 +1790,21 @@ async fn build_cursor_projection(pool: &AnyPool, driver_name: &str, table: &str)
|
|
|
1772
1790
|
|ident| format!("CAST({ident} AS CHAR) AS {ident}"),
|
|
1773
1791
|
)
|
|
1774
1792
|
}
|
|
1793
|
+
"SQLite" => {
|
|
1794
|
+
let (schema, name) = match table.split_once('.') {
|
|
1795
|
+
Some((s, t)) => (
|
|
1796
|
+
s.trim_matches('"').to_string(),
|
|
1797
|
+
t.trim_matches('"').to_string(),
|
|
1798
|
+
),
|
|
1799
|
+
None => ("main".to_string(), table.trim_matches('"').to_string()),
|
|
1800
|
+
};
|
|
1801
|
+
(
|
|
1802
|
+
SQLITE_COLUMN_TYPES_SQL,
|
|
1803
|
+
vec![name, schema],
|
|
1804
|
+
sqlite_decltype_is_any_safe,
|
|
1805
|
+
|ident| format!("CAST({ident} AS TEXT) AS {ident}"),
|
|
1806
|
+
)
|
|
1807
|
+
}
|
|
1775
1808
|
_ => return "*".to_string(),
|
|
1776
1809
|
};
|
|
1777
1810
|
|
|
@@ -1009,6 +1009,126 @@ async fn test_sqlx_multicolumn_batch() {
|
|
|
1009
1009
|
}
|
|
1010
1010
|
}
|
|
1011
1011
|
|
|
1012
|
+
// Regression (issue #71): a table written by the publisher with `auto_create_table` must be
|
|
1013
|
+
// readable by this library's own cursor reader. The generated DDL declares `locked_until`/
|
|
1014
|
+
// `created_at` as DATETIME, which the `Any` driver refuses to decode, so `SELECT *` failed the
|
|
1015
|
+
// whole read. The projection now casts those columns to TEXT.
|
|
1016
|
+
#[tokio::test]
|
|
1017
|
+
async fn test_sqlx_auto_created_table_is_cursor_readable() {
|
|
1018
|
+
sqlx::any::install_default_drivers();
|
|
1019
|
+
let dir = tempdir().unwrap();
|
|
1020
|
+
let path = dir.path().join("roundtrip.db");
|
|
1021
|
+
let url = sqlite_url(&path);
|
|
1022
|
+
drop(tokio::fs::File::create(&path).await.unwrap());
|
|
1023
|
+
|
|
1024
|
+
let config = SqlxConfig {
|
|
1025
|
+
url: url.clone(),
|
|
1026
|
+
table: "orders".to_string(),
|
|
1027
|
+
auto_create_table: true,
|
|
1028
|
+
..Default::default()
|
|
1029
|
+
};
|
|
1030
|
+
let publisher = SqlxPublisher::new(&config).await.unwrap();
|
|
1031
|
+
publisher
|
|
1032
|
+
.send_batch(
|
|
1033
|
+
(1..=3)
|
|
1034
|
+
.map(|i| CanonicalMessage::new(format!("row{i}").into_bytes(), None))
|
|
1035
|
+
.collect(),
|
|
1036
|
+
)
|
|
1037
|
+
.await
|
|
1038
|
+
.unwrap();
|
|
1039
|
+
|
|
1040
|
+
let read_config = SqlxConfig {
|
|
1041
|
+
url,
|
|
1042
|
+
table: "orders".to_string(),
|
|
1043
|
+
cursor_column: Some("id".to_string()),
|
|
1044
|
+
..Default::default()
|
|
1045
|
+
};
|
|
1046
|
+
let mut reader = SqlxCursorReader::new(&read_config).await.unwrap();
|
|
1047
|
+
let batch = reader.receive_batch(10).await.unwrap();
|
|
1048
|
+
assert_eq!(batch.messages.len(), 3);
|
|
1049
|
+
let v: serde_json::Value = serde_json::from_slice(&batch.messages[0].payload).unwrap();
|
|
1050
|
+
assert_eq!(v["id"], 1);
|
|
1051
|
+
// The DATETIME columns survive as strings rather than aborting the read. `is_some()`
|
|
1052
|
+
// alone would also pass on a JSON null, which is what a failed cast looks like.
|
|
1053
|
+
assert!(
|
|
1054
|
+
v["created_at"].as_str().is_some(),
|
|
1055
|
+
"got {}",
|
|
1056
|
+
v["created_at"]
|
|
1057
|
+
);
|
|
1058
|
+
assert!(v["locked_until"].is_null());
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
// A DATETIME cursor column is now readable on SQLite (the projection casts it to TEXT),
|
|
1062
|
+
// so the cursor value round-trips as text while `WHERE`/`ORDER BY` still compare the raw
|
|
1063
|
+
// column. Pin that resume neither skips nor re-emits rows.
|
|
1064
|
+
#[tokio::test]
|
|
1065
|
+
async fn test_sqlx_cursor_reader_datetime_cursor_column_resumes() {
|
|
1066
|
+
sqlx::any::install_default_drivers();
|
|
1067
|
+
let dir = tempdir().unwrap();
|
|
1068
|
+
let path = dir.path().join("dtcursor.db");
|
|
1069
|
+
let url = sqlite_url(&path);
|
|
1070
|
+
drop(tokio::fs::File::create(&path).await.unwrap());
|
|
1071
|
+
let pool = AnyPool::connect(&url).await.unwrap();
|
|
1072
|
+
sqlx::query("CREATE TABLE events (id INTEGER PRIMARY KEY, ts DATETIME, note TEXT)")
|
|
1073
|
+
.execute(&pool)
|
|
1074
|
+
.await
|
|
1075
|
+
.unwrap();
|
|
1076
|
+
for (i, ts) in [
|
|
1077
|
+
"2026-08-16 10:00:00",
|
|
1078
|
+
"2026-08-16 10:00:01",
|
|
1079
|
+
"2026-08-16 10:00:02",
|
|
1080
|
+
"2026-08-16 10:00:03",
|
|
1081
|
+
]
|
|
1082
|
+
.iter()
|
|
1083
|
+
.enumerate()
|
|
1084
|
+
{
|
|
1085
|
+
sqlx::query("INSERT INTO events (id, ts, note) VALUES (?, ?, ?)")
|
|
1086
|
+
.bind((i + 1) as i64)
|
|
1087
|
+
.bind(*ts)
|
|
1088
|
+
.bind(format!("n{i}"))
|
|
1089
|
+
.execute(&pool)
|
|
1090
|
+
.await
|
|
1091
|
+
.unwrap();
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
let config = SqlxConfig {
|
|
1095
|
+
url,
|
|
1096
|
+
table: "events".to_string(),
|
|
1097
|
+
cursor_column: Some("ts".to_string()),
|
|
1098
|
+
cursor_id: Some("dt-1".to_string()),
|
|
1099
|
+
..Default::default()
|
|
1100
|
+
};
|
|
1101
|
+
let mut reader = SqlxCursorReader::new(&config).await.unwrap();
|
|
1102
|
+
let b1 = reader.receive_batch(2).await.unwrap();
|
|
1103
|
+
assert_eq!(b1.messages.len(), 2);
|
|
1104
|
+
(b1.commit)(vec![MessageDisposition::Ack; 2]).await.unwrap();
|
|
1105
|
+
|
|
1106
|
+
// Resume from the *checkpoint*, not the in-memory cursor: a text-encoded timestamp has
|
|
1107
|
+
// to survive save/load/decode, which reading on through the same reader never exercises.
|
|
1108
|
+
drop(reader);
|
|
1109
|
+
let mut reader = SqlxCursorReader::new(&config).await.unwrap();
|
|
1110
|
+
let b2 = reader.receive_batch(2).await.unwrap();
|
|
1111
|
+
let notes: Vec<String> = b2
|
|
1112
|
+
.messages
|
|
1113
|
+
.iter()
|
|
1114
|
+
.map(|m| {
|
|
1115
|
+
serde_json::from_slice::<serde_json::Value>(&m.payload).unwrap()["note"]
|
|
1116
|
+
.as_str()
|
|
1117
|
+
.unwrap()
|
|
1118
|
+
.to_string()
|
|
1119
|
+
})
|
|
1120
|
+
.collect();
|
|
1121
|
+
assert_eq!(
|
|
1122
|
+
notes,
|
|
1123
|
+
vec!["n2", "n3"],
|
|
1124
|
+
"resume must continue, not repeat or skip"
|
|
1125
|
+
);
|
|
1126
|
+
(b2.commit)(vec![MessageDisposition::Ack; 2]).await.unwrap();
|
|
1127
|
+
|
|
1128
|
+
let b3 = reader.receive_batch(2).await.unwrap();
|
|
1129
|
+
assert!(b3.messages.is_empty(), "drained");
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1012
1132
|
#[tokio::test]
|
|
1013
1133
|
async fn test_sqlx_auto_create_rejects_tokens() {
|
|
1014
1134
|
let (_dir, url) = setup_db_file().await;
|
|
@@ -101,6 +101,11 @@ impl MessagePublisher for FanoutPublisher {
|
|
|
101
101
|
}
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
/// Ordered if any leg is: a fanout is only as order-tolerant as its strictest sink.
|
|
105
|
+
fn requires_ordered_publish(&self) -> bool {
|
|
106
|
+
self.publishers.iter().any(|p| p.requires_ordered_publish())
|
|
107
|
+
}
|
|
108
|
+
|
|
104
109
|
fn as_any(&self) -> &dyn Any {
|
|
105
110
|
self
|
|
106
111
|
}
|
|
@@ -99,6 +99,10 @@ impl MessagePublisher for RequestForwardPublisher {
|
|
|
99
99
|
self.request.status().await
|
|
100
100
|
}
|
|
101
101
|
|
|
102
|
+
fn requires_ordered_publish(&self) -> bool {
|
|
103
|
+
self.request.requires_ordered_publish() || self.forward.requires_ordered_publish()
|
|
104
|
+
}
|
|
105
|
+
|
|
102
106
|
fn as_any(&self) -> &dyn Any {
|
|
103
107
|
self
|
|
104
108
|
}
|
|
@@ -127,6 +127,14 @@ impl MessagePublisher for SwitchPublisher {
|
|
|
127
127
|
}
|
|
128
128
|
}
|
|
129
129
|
|
|
130
|
+
/// Ordered if any branch is: batches for a given branch must stay in source order.
|
|
131
|
+
fn requires_ordered_publish(&self) -> bool {
|
|
132
|
+
self.cases
|
|
133
|
+
.values()
|
|
134
|
+
.chain(self.default.iter())
|
|
135
|
+
.any(|p| p.requires_ordered_publish())
|
|
136
|
+
}
|
|
137
|
+
|
|
130
138
|
fn as_any(&self) -> &dyn Any {
|
|
131
139
|
self
|
|
132
140
|
}
|
|
@@ -311,6 +311,14 @@ impl MessagePublisher for DlqPublisher {
|
|
|
311
311
|
}
|
|
312
312
|
}
|
|
313
313
|
|
|
314
|
+
/// Deliberately follows `inner` only, not the DLQ. An order-sensitive DLQ behind an
|
|
315
|
+
/// unordered sink would sequence *every* batch — paying the primary sink's whole
|
|
316
|
+
/// concurrency factor — to order a sparse, failure-only side channel. When the gate
|
|
317
|
+
/// is on, the DLQ send is inside this `send_batch` and is sequenced with it anyway.
|
|
318
|
+
fn requires_ordered_publish(&self) -> bool {
|
|
319
|
+
self.inner.requires_ordered_publish()
|
|
320
|
+
}
|
|
321
|
+
|
|
314
322
|
fn as_any(&self) -> &dyn Any {
|
|
315
323
|
self
|
|
316
324
|
}
|
|
@@ -165,6 +165,10 @@ impl MessagePublisher for LimiterPublisher {
|
|
|
165
165
|
self.inner.send_batch(messages).await
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
+
fn requires_ordered_publish(&self) -> bool {
|
|
169
|
+
self.inner.requires_ordered_publish()
|
|
170
|
+
}
|
|
171
|
+
|
|
168
172
|
fn as_any(&self) -> &dyn Any {
|
|
169
173
|
self
|
|
170
174
|
}
|
|
@@ -143,7 +143,9 @@ pub struct RouteOptions {
|
|
|
143
143
|
pub description: String,
|
|
144
144
|
/// (Optional) Number of concurrent processing tasks for this route. While it improves throughput for high-latency
|
|
145
145
|
/// handlers, it adds synchronization overhead for ordered commits and may lead to out-of-order processing
|
|
146
|
-
/// in the handler.
|
|
146
|
+
/// in the handler. Above 1, whole batches may also reach the sink out of source order (rows keep their order
|
|
147
|
+
/// within a batch) unless the sink declares itself order-sensitive, as `file` does.
|
|
148
|
+
/// Defaults to 1.
|
|
147
149
|
#[serde(default = "default_concurrency")]
|
|
148
150
|
#[cfg_attr(feature = "schema", schemars(range(min = 1)))]
|
|
149
151
|
pub concurrency: usize,
|