ankka-flow 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. ankka_flow-0.1.0/.gitignore +34 -0
  2. ankka_flow-0.1.0/PKG-INFO +93 -0
  3. ankka_flow-0.1.0/README.md +79 -0
  4. ankka_flow-0.1.0/proto/DESCRIPTOR.md +109 -0
  5. ankka_flow-0.1.0/proto/README.md +80 -0
  6. ankka_flow-0.1.0/proto/fixtures/conversations/run.batch-order-within-partition.json +46 -0
  7. ankka_flow-0.1.0/proto/fixtures/conversations/run.config-applied.json +22 -0
  8. ankka_flow-0.1.0/proto/fixtures/conversations/run.echo-preserves-record.json +31 -0
  9. ankka_flow-0.1.0/proto/fixtures/conversations/run.emits-precede-ack.json +22 -0
  10. ankka_flow-0.1.0/proto/fixtures/conversations/run.fail-sends-fail.json +22 -0
  11. ankka_flow-0.1.0/proto/fixtures/conversations/run.fan-out.json +22 -0
  12. ankka_flow-0.1.0/proto/fixtures/conversations/run.header-order.json +39 -0
  13. ankka_flow-0.1.0/proto/fixtures/conversations/run.large-record.json +25 -0
  14. ankka_flow-0.1.0/proto/fixtures/conversations/run.rogue-outlet.json +22 -0
  15. ankka_flow-0.1.0/proto/fixtures/conversations/run.skip-acks-without-emit.json +22 -0
  16. ankka_flow-0.1.0/proto/fixtures/conversations/run.two-inlets.json +34 -0
  17. ankka_flow-0.1.0/proto/fixtures/conversations/run.two-partitions-interleave.json +34 -0
  18. ankka_flow-0.1.0/proto/fixtures/conversations/run.unkeyed-emit.json +22 -0
  19. ankka_flow-0.1.0/proto/fixtures/declarations/README.md +6 -0
  20. ankka_flow-0.1.0/proto/fixtures/declarations/cart-router.md +9 -0
  21. ankka_flow-0.1.0/proto/fixtures/declarations/conformance.md +12 -0
  22. ankka_flow-0.1.0/proto/fixtures/declarations/every-type.md +14 -0
  23. ankka_flow-0.1.0/proto/fixtures/declarations/many-ports.md +9 -0
  24. ankka_flow-0.1.0/proto/fixtures/declarations/minimal.md +7 -0
  25. ankka_flow-0.1.0/proto/fixtures/declarations/sink.md +7 -0
  26. ankka_flow-0.1.0/proto/fixtures/descriptors/cart-router.json +47 -0
  27. ankka_flow-0.1.0/proto/fixtures/descriptors/conformance.json +60 -0
  28. ankka_flow-0.1.0/proto/fixtures/descriptors/every-type.json +62 -0
  29. ankka_flow-0.1.0/proto/fixtures/descriptors/many-ports.json +94 -0
  30. ankka_flow-0.1.0/proto/fixtures/descriptors/minimal.json +30 -0
  31. ankka_flow-0.1.0/proto/fixtures/descriptors/sink.json +21 -0
  32. ankka_flow-0.1.0/proto/src/main/protobuf/ankka/flow/v1/discovery.proto +66 -0
  33. ankka_flow-0.1.0/proto/src/main/protobuf/ankka/flow/v1/payload.proto +31 -0
  34. ankka_flow-0.1.0/proto/src/main/protobuf/ankka/flow/v1/streamlet.proto +76 -0
  35. ankka_flow-0.1.0/pyproject.toml +60 -0
  36. ankka_flow-0.1.0/scripts/proto.py +68 -0
  37. ankka_flow-0.1.0/src/ankka_flow/__init__.py +40 -0
  38. ankka_flow-0.1.0/src/ankka_flow/_conformance.py +93 -0
  39. ankka_flow-0.1.0/src/ankka_flow/_descriptor.py +74 -0
  40. ankka_flow-0.1.0/src/ankka_flow/_proto/__init__.py +0 -0
  41. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/__init__.py +0 -0
  42. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/flow/__init__.py +0 -0
  43. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/flow/v1/__init__.py +0 -0
  44. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/flow/v1/discovery_pb2.py +53 -0
  45. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/flow/v1/discovery_pb2.pyi +94 -0
  46. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/flow/v1/discovery_pb2_grpc.py +146 -0
  47. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/flow/v1/payload_pb2.py +46 -0
  48. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/flow/v1/payload_pb2.pyi +47 -0
  49. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/flow/v1/payload_pb2_grpc.py +24 -0
  50. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/flow/v1/streamlet_pb2.py +57 -0
  51. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/flow/v1/streamlet_pb2.pyi +106 -0
  52. ankka_flow-0.1.0/src/ankka_flow/_proto/ankka/flow/v1/streamlet_pb2_grpc.py +101 -0
  53. ankka_flow-0.1.0/src/ankka_flow/descriptor.py +95 -0
  54. ankka_flow-0.1.0/src/ankka_flow/json.py +14 -0
  55. ankka_flow-0.1.0/src/ankka_flow/parameters.py +241 -0
  56. ankka_flow-0.1.0/src/ankka_flow/ports.py +76 -0
  57. ankka_flow-0.1.0/src/ankka_flow/py.typed +0 -0
  58. ankka_flow-0.1.0/src/ankka_flow/records.py +42 -0
  59. ankka_flow-0.1.0/src/ankka_flow/server.py +245 -0
  60. ankka_flow-0.1.0/src/ankka_flow/streamlet.py +90 -0
  61. ankka_flow-0.1.0/src/ankka_flow/testkit/__init__.py +112 -0
  62. ankka_flow-0.1.0/template/.dockerignore +4 -0
  63. ankka_flow-0.1.0/template/.gitignore +2 -0
  64. ankka_flow-0.1.0/template/Dockerfile +10 -0
  65. ankka_flow-0.1.0/template/README.md +19 -0
  66. ankka_flow-0.1.0/template/blueprint.conf +21 -0
  67. ankka_flow-0.1.0/template/docker-compose.yml +33 -0
  68. ankka_flow-0.1.0/template/flow/streamlet.conf +27 -0
  69. ankka_flow-0.1.0/template/pyproject.toml +19 -0
  70. ankka_flow-0.1.0/template/src/{{module}}/__init__.py +0 -0
  71. ankka_flow-0.1.0/template/src/{{module}}/main.py +6 -0
  72. ankka_flow-0.1.0/template/src/{{module}}/streamlet.py +14 -0
  73. ankka_flow-0.1.0/template/tests/test_streamlet.py +11 -0
  74. ankka_flow-0.1.0/tests/conftest.py +4 -0
  75. ankka_flow-0.1.0/tests/fixture_streamlets.py +103 -0
  76. ankka_flow-0.1.0/tests/sidecar_double.py +120 -0
  77. ankka_flow-0.1.0/tests/test_conformance_reference.py +43 -0
  78. ankka_flow-0.1.0/tests/test_descriptor_fixtures.py +29 -0
  79. ankka_flow-0.1.0/tests/test_harness.py +140 -0
  80. ankka_flow-0.1.0/tests/test_server.py +189 -0
  81. ankka_flow-0.1.0/uv.lock +481 -0
@@ -0,0 +1,34 @@
1
+ *.class
2
+ *.tasty
3
+ *.log
4
+
5
+ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
6
+ hs_err_pid*
7
+
8
+ # sbt, IDEs, OS
9
+ target/
10
+ .bsp/
11
+ .metals/
12
+ .bloop/
13
+ .idea/
14
+ .vscode/
15
+ .DS_Store
16
+ *.tmp
17
+ *.swp
18
+
19
+ # Python (sdks/python, samples)
20
+ __pycache__/
21
+ *.pyc
22
+ .venv/
23
+ venv/
24
+ dist/
25
+ *.egg-info/
26
+ .mypy_cache/
27
+ .pytest_cache/
28
+ sdks/python/src/ankka_flow/_proto/
29
+ samples/*/src/*/_proto/
30
+
31
+ # Kubernetes material that must never be committed
32
+ *.secret.yaml
33
+ kubeconfig*
34
+ .env*
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.5
2
+ Name: ankka-flow
3
+ Version: 0.1.0
4
+ Summary: Write ankka-flow streamlets in Python: declare ports, process batches, let the sidecar own Kafka.
5
+ Project-URL: Homepage, https://flow.ankka.cloud/
6
+ Project-URL: Documentation, https://flow.ankka.cloud/reference/python-sdk/
7
+ Project-URL: Repository, https://github.com/thinkmorestupidless/ankka-flow
8
+ License-Expression: Apache-2.0
9
+ Keywords: ankka,ankka-flow,kafka,sidecar,streaming
10
+ Requires-Python: >=3.12
11
+ Requires-Dist: grpcio<2,>=1.84
12
+ Requires-Dist: protobuf<8,>=6
13
+ Description-Content-Type: text/markdown
14
+
15
+ # ankka-flow for Python
16
+
17
+ Write [ankka-flow](https://flow.ankka.cloud/) streamlets in Python. You declare ports and parameters and
18
+ implement `process`; the sidecar the platform runs beside your container owns everything Kafka.
19
+
20
+ ```python
21
+ from collections.abc import Iterable
22
+ from ankka_flow import Batch, Emit, IntegerParameter, JsonInlet, JsonOutlet, Streamlet, json, serve
23
+
24
+ class CartRouter(Streamlet):
25
+ name = "cart-router"
26
+ inlet = JsonInlet("in", schema_name="cart-events.v1")
27
+ valid = JsonOutlet("valid", schema_name="cart-events.v1")
28
+ review = JsonOutlet("review", schema_name="cart-events.v1")
29
+ threshold = IntegerParameter("review-threshold", default=100)
30
+
31
+ def process(self, batch: Batch) -> Iterable[Emit]:
32
+ for record in batch:
33
+ event = json.loads(record.value)
34
+ yield (self.review if event["total"] > self.config[self.threshold] else self.valid).emit(record)
35
+
36
+ if __name__ == "__main__":
37
+ serve(CartRouter()) # 127.0.0.1:$FLOW_PROCESS_PORT (9010), loopback only
38
+ ```
39
+
40
+ - `process` runs once per batch on a worker thread. One batch per partition is in flight at a time,
41
+ so it never runs twice for one partition at once; different partitions run concurrently.
42
+ - Returning acknowledges the batch. Raising fails it, and the sidecar redelivers it from the last
43
+ commit. Yielding nothing for a record skips it.
44
+ - Records are bytes with a key and headers. The SDK decodes nothing; `ankka_flow.json` helps.
45
+
46
+ ## Descriptor
47
+
48
+ `uv run descriptor` writes `flow/descriptor.json` from the streamlet named by
49
+ `[tool.ankka-flow] streamlet = "module:Class"` in your `pyproject.toml` (or `$FLOW_STREAMLET`).
50
+ Commit it. `uv run descriptor --check` fails when it is stale. The format is
51
+ [`proto/DESCRIPTOR.md`](https://github.com/thinkmorestupidless/ankka-flow/blob/main/sdks/python/proto/DESCRIPTOR.md).
52
+
53
+ ## Testing without Kafka
54
+
55
+ ```python
56
+ from ankka_flow.testkit import Harness
57
+
58
+ h = Harness(CartRouter(), config={"review-threshold": 50})
59
+ h.inlet("in").put(key=b"cart-1", value=b'{"total": 10}')
60
+ h.run()
61
+ assert [r.key for r in h.outlet("valid").records] == [b"cart-1"]
62
+ ```
63
+
64
+ ## Developing the SDK
65
+
66
+ ```bash
67
+ uv sync
68
+ uv run python scripts/proto.py # copy ../../protocol into proto/ (committed) and generate src/ankka_flow/_proto/
69
+ uv run mypy # strict
70
+ uv run pytest -q # fixtures, the server against a scripted sidecar, the harness
71
+ uv run conformance # the reference streamlet against the platform's conformance suite
72
+ ```
73
+
74
+ `proto/` must equal `../../protocol` byte for byte; CI diffs it. `tests/test_descriptor_fixtures.py`
75
+ proves this SDK writes every fixture descriptor exactly.
76
+
77
+ A new project starts from [`template/`](https://github.com/thinkmorestupidless/ankka-flow/tree/main/sdks/python/template), which carries a compose file with Kafka and the
78
+ sidecar for the laptop loop.
79
+
80
+ ## Conformance
81
+
82
+ `uv run conformance` serves the reference streamlet (`ankka_flow._conformance`) on
83
+ `127.0.0.1:$FLOW_PROCESS_PORT` and runs the sidecar's conformance suite against it from the
84
+ repository root (sbt and a JDK are needed). Last run: every one of the 18 cases that apply to an SDK
85
+ passes; the five `violation.*` and `version.*` cases are skipped, as they only run against the
86
+ Scala double.
87
+
88
+ Two deliberate breaks prove the suite points at what broke:
89
+
90
+ | `ANKKA_FLOW_BREAK` | what it breaks | cases that fail |
91
+ |---|---|---|
92
+ | `keyless-empty-key` | a keyless emit carries an empty key | exactly `run.unkeyed-emit` (SC-005) |
93
+ | `ack-first` | the ack is sent before the batch's emits | the 12 cases that emit, `run.emits-precede-ack` among them |
@@ -0,0 +1,79 @@
1
+ # ankka-flow for Python
2
+
3
+ Write [ankka-flow](https://flow.ankka.cloud/) streamlets in Python. You declare ports and parameters and
4
+ implement `process`; the sidecar the platform runs beside your container owns everything Kafka.
5
+
6
+ ```python
7
+ from collections.abc import Iterable
8
+ from ankka_flow import Batch, Emit, IntegerParameter, JsonInlet, JsonOutlet, Streamlet, json, serve
9
+
10
+ class CartRouter(Streamlet):
11
+ name = "cart-router"
12
+ inlet = JsonInlet("in", schema_name="cart-events.v1")
13
+ valid = JsonOutlet("valid", schema_name="cart-events.v1")
14
+ review = JsonOutlet("review", schema_name="cart-events.v1")
15
+ threshold = IntegerParameter("review-threshold", default=100)
16
+
17
+ def process(self, batch: Batch) -> Iterable[Emit]:
18
+ for record in batch:
19
+ event = json.loads(record.value)
20
+ yield (self.review if event["total"] > self.config[self.threshold] else self.valid).emit(record)
21
+
22
+ if __name__ == "__main__":
23
+ serve(CartRouter()) # 127.0.0.1:$FLOW_PROCESS_PORT (9010), loopback only
24
+ ```
25
+
26
+ - `process` runs once per batch on a worker thread. One batch per partition is in flight at a time,
27
+ so it never runs twice for one partition at once; different partitions run concurrently.
28
+ - Returning acknowledges the batch. Raising fails it, and the sidecar redelivers it from the last
29
+ commit. Yielding nothing for a record skips it.
30
+ - Records are bytes with a key and headers. The SDK decodes nothing; `ankka_flow.json` helps.
31
+
32
+ ## Descriptor
33
+
34
+ `uv run descriptor` writes `flow/descriptor.json` from the streamlet named by
35
+ `[tool.ankka-flow] streamlet = "module:Class"` in your `pyproject.toml` (or `$FLOW_STREAMLET`).
36
+ Commit it. `uv run descriptor --check` fails when it is stale. The format is
37
+ [`proto/DESCRIPTOR.md`](https://github.com/thinkmorestupidless/ankka-flow/blob/main/sdks/python/proto/DESCRIPTOR.md).
38
+
39
+ ## Testing without Kafka
40
+
41
+ ```python
42
+ from ankka_flow.testkit import Harness
43
+
44
+ h = Harness(CartRouter(), config={"review-threshold": 50})
45
+ h.inlet("in").put(key=b"cart-1", value=b'{"total": 10}')
46
+ h.run()
47
+ assert [r.key for r in h.outlet("valid").records] == [b"cart-1"]
48
+ ```
49
+
50
+ ## Developing the SDK
51
+
52
+ ```bash
53
+ uv sync
54
+ uv run python scripts/proto.py # copy ../../protocol into proto/ (committed) and generate src/ankka_flow/_proto/
55
+ uv run mypy # strict
56
+ uv run pytest -q # fixtures, the server against a scripted sidecar, the harness
57
+ uv run conformance # the reference streamlet against the platform's conformance suite
58
+ ```
59
+
60
+ `proto/` must equal `../../protocol` byte for byte; CI diffs it. `tests/test_descriptor_fixtures.py`
61
+ proves this SDK writes every fixture descriptor exactly.
62
+
63
+ A new project starts from [`template/`](https://github.com/thinkmorestupidless/ankka-flow/tree/main/sdks/python/template), which carries a compose file with Kafka and the
64
+ sidecar for the laptop loop.
65
+
66
+ ## Conformance
67
+
68
+ `uv run conformance` serves the reference streamlet (`ankka_flow._conformance`) on
69
+ `127.0.0.1:$FLOW_PROCESS_PORT` and runs the sidecar's conformance suite against it from the
70
+ repository root (sbt and a JDK are needed). Last run: every one of the 18 cases that apply to an SDK
71
+ passes; the five `violation.*` and `version.*` cases are skipped, as they only run against the
72
+ Scala double.
73
+
74
+ Two deliberate breaks prove the suite points at what broke:
75
+
76
+ | `ANKKA_FLOW_BREAK` | what it breaks | cases that fail |
77
+ |---|---|---|
78
+ | `keyless-empty-key` | a keyless emit carries an empty key | exactly `run.unkeyed-emit` (SC-005) |
79
+ | `ack-first` | the ack is sent before the batch's emits | the 12 cases that emit, `run.emits-precede-ack` among them |
@@ -0,0 +1,109 @@
1
+ # The descriptor file
2
+
3
+ A streamlet's descriptor is the discovery `Spec` message written as JSON by the SDK at build time.
4
+ The same declaration produces the same bytes in every language, which the fixtures prove. It is
5
+ never written by hand: the CLI and the sidecar refuse one whose fingerprints do not match their
6
+ schema names.
7
+
8
+ It is read in three places: the CLI verifies a blueprint against it, the operator mounts it into the
9
+ sidecar, and the sidecar compares it field by field with the process's answer to `Discover`.
10
+
11
+ ## Canonical JSON
12
+
13
+ 1. Field names are the proto field names (snake_case): `protocol_version`, `schema_name`,
14
+ `config_parameters`, `default_value`.
15
+ 2. Object keys are sorted lexicographically (byte order) at every level.
16
+ 3. `inlets` and `outlets` are sorted by `name`; `config_parameters` by `key`. The SDK sorts; a
17
+ declaration order is never meaningful.
18
+ 4. Enums are their names (`"INTEGER"`), never numbers.
19
+ 5. A field at its proto3 default is omitted (`""`, `0`, `false`, an empty list, an unset
20
+ `optional`). `ConfigType.STRING` is therefore omitted, which is the rule, not a bug.
21
+ 6. Two-space indentation, `": "` and `,\n` separators, LF line endings, UTF-8, exactly one trailing
22
+ newline, no BOM.
23
+
24
+ Python: `json.dumps(MessageToDict(spec, preserving_proto_field_name=True), sort_keys=True,
25
+ indent=2, ensure_ascii=False) + "\n"`. Scala: `protocol`'s `DescriptorJson.write(spec)`, which is
26
+ tested against every fixture.
27
+
28
+ ## Example: the cart router
29
+
30
+ `protocol/fixtures/declarations/cart-router.md` describes the declaration; every SDK's sample
31
+ declares it and `protocol/fixtures/descriptors/cart-router.json` is the expected output:
32
+
33
+ ```json
34
+ {
35
+ "protocol_version": "1.0",
36
+ "sdk": {
37
+ "name": "fixture",
38
+ "version": "0.0.0"
39
+ },
40
+ "streamlet": {
41
+ "config_parameters": [
42
+ {
43
+ "default_value": "100",
44
+ "description": "Carts with a total above this go to the review outlet.",
45
+ "key": "review-threshold",
46
+ "type": "INTEGER"
47
+ }
48
+ ],
49
+ "description": "Routes cart events to the valid or review outlet.",
50
+ "inlets": [
51
+ {
52
+ "contract": {
53
+ "fingerprint": "nXhoFwNZSB7DKScFuZUtZ1gnAYkIxadumsXNHWwbLfM=",
54
+ "format": "json",
55
+ "schema_name": "cart-events.v1"
56
+ },
57
+ "name": "in"
58
+ }
59
+ ],
60
+ "name": "cart-router",
61
+ "outlets": [
62
+ {
63
+ "contract": {
64
+ "fingerprint": "nXhoFwNZSB7DKScFuZUtZ1gnAYkIxadumsXNHWwbLfM=",
65
+ "format": "json",
66
+ "schema_name": "cart-events.v1"
67
+ },
68
+ "name": "review"
69
+ },
70
+ {
71
+ "contract": {
72
+ "fingerprint": "nXhoFwNZSB7DKScFuZUtZ1gnAYkIxadumsXNHWwbLfM=",
73
+ "format": "json",
74
+ "schema_name": "cart-events.v1"
75
+ },
76
+ "name": "valid"
77
+ }
78
+ ]
79
+ }
80
+ }
81
+ ```
82
+
83
+ A fixture's `sdk` block is pinned to `{"name": "fixture", "version": "0.0.0"}` so that one file is
84
+ the expected output of every SDK. Outside the fixture test an SDK writes its real name and version.
85
+
86
+ ## Fixtures
87
+
88
+ | fixture | declares |
89
+ |---|---|
90
+ | `minimal` | one inlet, one outlet, no parameters, no description |
91
+ | `cart-router` | the sample above |
92
+ | `every-type` | one parameter of every `ConfigType`, one required (no default), unicode in a description |
93
+ | `many-ports` | five inlets and five outlets declared out of order, to prove sorting |
94
+ | `sink` | inlets only, no outlets |
95
+ | `conformance` | the reference streamlet of the conformance suite (`../README.md`) |
96
+
97
+ Each SDK's test suite declares each fixture's streamlet in its own language and asserts the
98
+ written bytes equal the fixture. The Scala `protocol` test does the same for `DescriptorJson` and
99
+ also parses each fixture back and re-writes it unchanged (FR-002, S5.2).
100
+
101
+ ## Validation (the CLI and the sidecar apply the same rules)
102
+
103
+ - `streamlet.name` matches `[a-z0-9-]{1,63}` and does not start or end with `-`.
104
+ - Port names match `[a-z][a-z0-9-]{0,62}` and are unique across inlets and outlets together.
105
+ - `contract.format` is `json` (version one); `contract.fingerprint` equals
106
+ `Base64(SHA-256(UTF-8(schema_name)))` with standard alphabet and padding.
107
+ - Parameter keys match `[a-z][a-z0-9-]*` and are unique; a `default_value` parses as its `type`
108
+ (`DURATION` as HOCON durations, `MEMORY_SIZE` as HOCON sizes).
109
+ - `protocol_version` is `MAJOR.MINOR` with both parts unsigned integers.
@@ -0,0 +1,80 @@
1
+ # The ankka-flow streamlet protocol, `ankka.flow.v1`
2
+
3
+ This directory is the platform's promise to every SDK. Copy it into an SDK **verbatim**; CI diffs
4
+ each copy against this one. It holds:
5
+
6
+ | path | what |
7
+ |---|---|
8
+ | `src/main/protobuf/ankka/flow/v1/payload.proto` | `Record`, `Header`, `Error`, `Problems` |
9
+ | `src/main/protobuf/ankka/flow/v1/discovery.proto` | `Discovery`: the process describes itself; the descriptor is its `Spec` |
10
+ | `src/main/protobuf/ankka/flow/v1/streamlet.proto` | `Streamlet.Run`: one bidirectional conversation per streamlet instance |
11
+ | `DESCRIPTOR.md` | the descriptor file's canonical JSON |
12
+ | `fixtures/declarations/` | streamlet declarations in prose |
13
+ | `fixtures/descriptors/` | the exact bytes each declaration must produce |
14
+ | `fixtures/conversations/` | the scripted inputs the conformance suite sends |
15
+
16
+ ## Directions
17
+
18
+ The developer's process serves `Discovery` and `Streamlet` on `127.0.0.1:$FLOW_PROCESS_PORT`
19
+ (default 9010), bound to loopback only. The sidecar dials it. The sidecar binds no gRPC port in
20
+ `1.x`; 9011 and `FLOW_SIDECAR_PORT` are reserved for a callback service a later minor may add.
21
+ The sidecar sends no HTTP/2 keepalive pings, so a process keeps its gRPC library's default ping
22
+ policy; one that ends connections for too many pings is never provoked.
23
+
24
+ ## Discovery
25
+
26
+ 1. The sidecar calls `Discover`, retrying with backoff (500 ms doubling to 10 s) until the process
27
+ answers. It is not ready until then.
28
+ 2. It checks the `Spec`'s protocol version, validates the descriptor, and compares it field by
29
+ field with the descriptor it was deployed with.
30
+ 3. On any problem it calls `ReportError` once with every problem, logs them, and exits.
31
+
32
+ ## Run
33
+
34
+ 1. The sidecar sends `Start`. The process sends nothing before it.
35
+ 2. The sidecar sends `Batch`es: at most one in flight per (inlet, partition), each partition's in
36
+ offset order. Batches for different partitions interleave freely.
37
+ 3. The process answers each batch with zero or more `Emit` and then exactly one `Ack` or `Fail`.
38
+ Messages for different batches may interleave.
39
+ 4. On `Ack` the sidecar produces the batch's emits, waits for the broker to confirm every one, and
40
+ only then commits the batch's offsets.
41
+ 5. On shutdown the sidecar sends `Stop` and half-closes.
42
+
43
+ ## Rules the messages do not state
44
+
45
+ - **One batch in flight per (inlet, partition).** A partition's batches arrive in offset order.
46
+ - **Emits precede the ack.** An emit after its batch's ack fails the stream.
47
+ - **Commit after the write.** Offsets are committed only after every emit for the batch is confirmed.
48
+ - **A `Fail` fails the stream, and nothing is skipped.** The sidecar voids every in-flight batch,
49
+ reconnects with backoff, repeats discovery, sends a new `Start`, and redelivers from the last
50
+ commit, indefinitely.
51
+ - **Skipping is acking without emitting.** The sidecar never knows a record was skipped.
52
+ - **A rebalance discards.** An emit or ack for a partition the sidecar no longer owns is dropped
53
+ silently and nothing is committed for it; the new owner reads it again. This is not a violation.
54
+ - **The sidecar never decodes a value.** A contract is a format and a fingerprint.
55
+ - **A keyless emit is partitioned by Kafka's default partitioner.** Per-key order is promised for
56
+ keyed records only.
57
+ - **A new `Start` voids everything.** State tied to an older `conversation_id` must be discarded.
58
+ - **Violations fail the stream:** an emit naming an outlet not in `Start.outlets`; an emit, ack or
59
+ fail for a batch that was never sent or is already acknowledged; an emit after its ack.
60
+ - **Message size.** Batches are bounded by count, bytes and time so they stay under 4 MiB. A single
61
+ input record over the limit fails the stream naming its topic, partition and offset. An emit must
62
+ stay under `Start.max_message_bytes`.
63
+
64
+ ## Versioning
65
+
66
+ `protocol_version` is `MAJOR.MINOR`, `1.0` here. The sidecar accepts a `Spec` of its own major
67
+ whose minor is not later than its own, and refuses anything else naming both. Adding an optional
68
+ field, a message, an rpc, a `ConfigType` value, a contract `format` or a fixture is a minor.
69
+ Renaming, removing or re-meaning anything is a major.
70
+
71
+ ## Proving an SDK
72
+
73
+ Two things define a compatible SDK:
74
+
75
+ - **The descriptor fixtures.** Declare each streamlet in `fixtures/declarations/` in your language,
76
+ write its descriptor with `sdk` pinned to `{"name": "fixture", "version": "0.0.0"}`, and assert
77
+ the bytes equal `fixtures/descriptors/<name>.json`.
78
+ - **The conformance suite.** Implement the `conformance` reference streamlet, serve it on a port,
79
+ and run `sbt 'sidecar/testOnly *ConformanceSuite' -Dflow.conformance.target=127.0.0.1:9010` from
80
+ the ankka-flow repository. Every case is named; a failure names the conversation that broke.
@@ -0,0 +1,46 @@
1
+ {
2
+ "batches": [
3
+ {
4
+ "inlet": "in",
5
+ "partition": 0,
6
+ "records": [
7
+ {
8
+ "headers": [],
9
+ "key": "echo",
10
+ "offset": 0,
11
+ "value_base64": "eyJuIjowfQ=="
12
+ }
13
+ ]
14
+ },
15
+ {
16
+ "inlet": "in",
17
+ "partition": 0,
18
+ "records": [
19
+ {
20
+ "headers": [],
21
+ "key": "echo",
22
+ "offset": 1,
23
+ "value_base64": "eyJuIjoxfQ=="
24
+ }
25
+ ]
26
+ },
27
+ {
28
+ "inlet": "in",
29
+ "partition": 0,
30
+ "records": [
31
+ {
32
+ "headers": [],
33
+ "key": "echo",
34
+ "offset": 2,
35
+ "value_base64": "eyJuIjoyfQ=="
36
+ }
37
+ ]
38
+ }
39
+ ],
40
+ "config": {
41
+ "factor": 1,
42
+ "mode": "echo"
43
+ },
44
+ "description": "Three sequential batches on one partition, acked in order.",
45
+ "name": "run.batch-order-within-partition"
46
+ }
@@ -0,0 +1,22 @@
1
+ {
2
+ "batches": [
3
+ {
4
+ "inlet": "in",
5
+ "partition": 0,
6
+ "records": [
7
+ {
8
+ "headers": [],
9
+ "key": "multiply",
10
+ "offset": 0,
11
+ "value_base64": "eyJuIjoxfQ=="
12
+ }
13
+ ]
14
+ }
15
+ ],
16
+ "config": {
17
+ "factor": 3,
18
+ "mode": "echo"
19
+ },
20
+ "description": "Three emits: the factor from Start.config_json.",
21
+ "name": "run.config-applied"
22
+ }
@@ -0,0 +1,31 @@
1
+ {
2
+ "batches": [
3
+ {
4
+ "inlet": "in",
5
+ "partition": 0,
6
+ "records": [
7
+ {
8
+ "headers": [
9
+ {
10
+ "key": "ce_type",
11
+ "value_base64": "SXRlbUFkZGVk"
12
+ },
13
+ {
14
+ "key": "raw",
15
+ "value_base64": "AP8qf4A="
16
+ }
17
+ ],
18
+ "key": "echo",
19
+ "offset": 7,
20
+ "value_base64": "eyJ0b3RhbCI6M30="
21
+ }
22
+ ]
23
+ }
24
+ ],
25
+ "config": {
26
+ "factor": 1,
27
+ "mode": "echo"
28
+ },
29
+ "description": "One Emit to out with the same key, headers in order and value bytes; then Ack.",
30
+ "name": "run.echo-preserves-record"
31
+ }
@@ -0,0 +1,22 @@
1
+ {
2
+ "batches": [
3
+ {
4
+ "inlet": "in",
5
+ "partition": 0,
6
+ "records": [
7
+ {
8
+ "headers": [],
9
+ "key": "multiply",
10
+ "offset": 0,
11
+ "value_base64": "eyJuIjoxfQ=="
12
+ }
13
+ ]
14
+ }
15
+ ],
16
+ "config": {
17
+ "factor": 5,
18
+ "mode": "echo"
19
+ },
20
+ "description": "Five emits, then Ack; nothing after.",
21
+ "name": "run.emits-precede-ack"
22
+ }
@@ -0,0 +1,22 @@
1
+ {
2
+ "batches": [
3
+ {
4
+ "inlet": "in",
5
+ "partition": 0,
6
+ "records": [
7
+ {
8
+ "headers": [],
9
+ "key": "fail",
10
+ "offset": 0,
11
+ "value_base64": "eyJuIjoxfQ=="
12
+ }
13
+ ]
14
+ }
15
+ ],
16
+ "config": {
17
+ "factor": 1,
18
+ "mode": "echo"
19
+ },
20
+ "description": "Fail with a message, no emits.",
21
+ "name": "run.fail-sends-fail"
22
+ }
@@ -0,0 +1,22 @@
1
+ {
2
+ "batches": [
3
+ {
4
+ "inlet": "in",
5
+ "partition": 0,
6
+ "records": [
7
+ {
8
+ "headers": [],
9
+ "key": "fan",
10
+ "offset": 0,
11
+ "value_base64": "eyJuIjoxfQ=="
12
+ }
13
+ ]
14
+ }
15
+ ],
16
+ "config": {
17
+ "factor": 1,
18
+ "mode": "echo"
19
+ },
20
+ "description": "Emits to out then other; then Ack.",
21
+ "name": "run.fan-out"
22
+ }
@@ -0,0 +1,39 @@
1
+ {
2
+ "batches": [
3
+ {
4
+ "inlet": "in",
5
+ "partition": 0,
6
+ "records": [
7
+ {
8
+ "headers": [
9
+ {
10
+ "key": "a",
11
+ "value_base64": "MQ=="
12
+ },
13
+ {
14
+ "key": "b",
15
+ "value_base64": "Mg=="
16
+ },
17
+ {
18
+ "key": "c",
19
+ "value_base64": "Mw=="
20
+ },
21
+ {
22
+ "key": "d",
23
+ "value_base64": "AP8qf4A="
24
+ }
25
+ ],
26
+ "key": "header-echo",
27
+ "offset": 0,
28
+ "value_base64": "eyJuIjoxfQ=="
29
+ }
30
+ ]
31
+ }
32
+ ],
33
+ "config": {
34
+ "factor": 1,
35
+ "mode": "echo"
36
+ },
37
+ "description": "Headers reversed, bytes intact.",
38
+ "name": "run.header-order"
39
+ }
@@ -0,0 +1,25 @@
1
+ {
2
+ "batches": [
3
+ {
4
+ "inlet": "in",
5
+ "partition": 0,
6
+ "records": [
7
+ {
8
+ "headers": [],
9
+ "key": "echo",
10
+ "offset": 0,
11
+ "value_repeat": {
12
+ "byte": "x",
13
+ "count": 3145728
14
+ }
15
+ }
16
+ ]
17
+ }
18
+ ],
19
+ "config": {
20
+ "factor": 1,
21
+ "mode": "echo"
22
+ },
23
+ "description": "A 3 MiB value echoed intact.",
24
+ "name": "run.large-record"
25
+ }
@@ -0,0 +1,22 @@
1
+ {
2
+ "batches": [
3
+ {
4
+ "inlet": "in",
5
+ "partition": 0,
6
+ "records": [
7
+ {
8
+ "headers": [],
9
+ "key": "rogue-outlet",
10
+ "offset": 0,
11
+ "value_base64": "eyJuIjoxfQ=="
12
+ }
13
+ ]
14
+ }
15
+ ],
16
+ "config": {
17
+ "factor": 1,
18
+ "mode": "echo"
19
+ },
20
+ "description": "The streamlet tries to emit to an undeclared outlet: the SDK fails the batch, or the sidecar fails the stream.",
21
+ "name": "run.rogue-outlet"
22
+ }