omnibase_infra 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. omnibase_infra/constants_topic_patterns.py +26 -0
  2. omnibase_infra/enums/__init__.py +3 -0
  3. omnibase_infra/enums/enum_consumer_group_purpose.py +92 -0
  4. omnibase_infra/enums/enum_handler_source_mode.py +16 -2
  5. omnibase_infra/errors/__init__.py +4 -0
  6. omnibase_infra/errors/error_binding_resolution.py +128 -0
  7. omnibase_infra/event_bus/configs/kafka_event_bus_config.yaml +0 -2
  8. omnibase_infra/event_bus/event_bus_inmemory.py +64 -10
  9. omnibase_infra/event_bus/event_bus_kafka.py +105 -47
  10. omnibase_infra/event_bus/mixin_kafka_broadcast.py +3 -7
  11. omnibase_infra/event_bus/mixin_kafka_dlq.py +12 -6
  12. omnibase_infra/event_bus/models/config/model_kafka_event_bus_config.py +0 -81
  13. omnibase_infra/event_bus/testing/__init__.py +26 -0
  14. omnibase_infra/event_bus/testing/adapter_protocol_event_publisher_inmemory.py +418 -0
  15. omnibase_infra/event_bus/testing/model_publisher_metrics.py +64 -0
  16. omnibase_infra/handlers/handler_consul.py +2 -0
  17. omnibase_infra/handlers/mixins/__init__.py +5 -0
  18. omnibase_infra/handlers/mixins/mixin_consul_service.py +274 -10
  19. omnibase_infra/handlers/mixins/mixin_consul_topic_index.py +585 -0
  20. omnibase_infra/handlers/models/model_filesystem_config.py +4 -4
  21. omnibase_infra/migrations/001_create_event_ledger.sql +166 -0
  22. omnibase_infra/migrations/001_drop_event_ledger.sql +18 -0
  23. omnibase_infra/mixins/mixin_node_introspection.py +189 -19
  24. omnibase_infra/models/__init__.py +8 -0
  25. omnibase_infra/models/bindings/__init__.py +59 -0
  26. omnibase_infra/models/bindings/constants.py +144 -0
  27. omnibase_infra/models/bindings/model_binding_resolution_result.py +103 -0
  28. omnibase_infra/models/bindings/model_operation_binding.py +44 -0
  29. omnibase_infra/models/bindings/model_operation_bindings_subcontract.py +152 -0
  30. omnibase_infra/models/bindings/model_parsed_binding.py +52 -0
  31. omnibase_infra/models/discovery/model_introspection_config.py +25 -17
  32. omnibase_infra/models/dispatch/__init__.py +8 -0
  33. omnibase_infra/models/dispatch/model_debug_trace_snapshot.py +114 -0
  34. omnibase_infra/models/dispatch/model_materialized_dispatch.py +141 -0
  35. omnibase_infra/models/handlers/model_handler_source_config.py +1 -1
  36. omnibase_infra/models/model_node_identity.py +126 -0
  37. omnibase_infra/models/projection/model_snapshot_topic_config.py +3 -2
  38. omnibase_infra/models/registration/__init__.py +9 -0
  39. omnibase_infra/models/registration/model_event_bus_topic_entry.py +59 -0
  40. omnibase_infra/models/registration/model_node_event_bus_config.py +99 -0
  41. omnibase_infra/models/registration/model_node_introspection_event.py +11 -0
  42. omnibase_infra/models/runtime/__init__.py +9 -0
  43. omnibase_infra/models/validation/model_coverage_metrics.py +2 -2
  44. omnibase_infra/nodes/__init__.py +9 -0
  45. omnibase_infra/nodes/contract_registry_reducer/__init__.py +29 -0
  46. omnibase_infra/nodes/contract_registry_reducer/contract.yaml +255 -0
  47. omnibase_infra/nodes/contract_registry_reducer/models/__init__.py +38 -0
  48. omnibase_infra/nodes/contract_registry_reducer/models/model_contract_registry_state.py +266 -0
  49. omnibase_infra/nodes/contract_registry_reducer/models/model_payload_cleanup_topic_references.py +55 -0
  50. omnibase_infra/nodes/contract_registry_reducer/models/model_payload_deactivate_contract.py +58 -0
  51. omnibase_infra/nodes/contract_registry_reducer/models/model_payload_mark_stale.py +49 -0
  52. omnibase_infra/nodes/contract_registry_reducer/models/model_payload_update_heartbeat.py +71 -0
  53. omnibase_infra/nodes/contract_registry_reducer/models/model_payload_update_topic.py +66 -0
  54. omnibase_infra/nodes/contract_registry_reducer/models/model_payload_upsert_contract.py +92 -0
  55. omnibase_infra/nodes/contract_registry_reducer/node.py +121 -0
  56. omnibase_infra/nodes/contract_registry_reducer/reducer.py +784 -0
  57. omnibase_infra/nodes/contract_registry_reducer/registry/__init__.py +9 -0
  58. omnibase_infra/nodes/contract_registry_reducer/registry/registry_infra_contract_registry_reducer.py +101 -0
  59. omnibase_infra/nodes/handlers/consul/contract.yaml +85 -0
  60. omnibase_infra/nodes/handlers/db/contract.yaml +72 -0
  61. omnibase_infra/nodes/handlers/graph/contract.yaml +127 -0
  62. omnibase_infra/nodes/handlers/http/contract.yaml +74 -0
  63. omnibase_infra/nodes/handlers/intent/contract.yaml +66 -0
  64. omnibase_infra/nodes/handlers/mcp/contract.yaml +69 -0
  65. omnibase_infra/nodes/handlers/vault/contract.yaml +91 -0
  66. omnibase_infra/nodes/node_ledger_projection_compute/__init__.py +50 -0
  67. omnibase_infra/nodes/node_ledger_projection_compute/contract.yaml +104 -0
  68. omnibase_infra/nodes/node_ledger_projection_compute/node.py +284 -0
  69. omnibase_infra/nodes/node_ledger_projection_compute/registry/__init__.py +29 -0
  70. omnibase_infra/nodes/node_ledger_projection_compute/registry/registry_infra_ledger_projection.py +118 -0
  71. omnibase_infra/nodes/node_ledger_write_effect/__init__.py +82 -0
  72. omnibase_infra/nodes/node_ledger_write_effect/contract.yaml +200 -0
  73. omnibase_infra/nodes/node_ledger_write_effect/handlers/__init__.py +22 -0
  74. omnibase_infra/nodes/node_ledger_write_effect/handlers/handler_ledger_append.py +372 -0
  75. omnibase_infra/nodes/node_ledger_write_effect/handlers/handler_ledger_query.py +597 -0
  76. omnibase_infra/nodes/node_ledger_write_effect/models/__init__.py +31 -0
  77. omnibase_infra/nodes/node_ledger_write_effect/models/model_ledger_append_result.py +54 -0
  78. omnibase_infra/nodes/node_ledger_write_effect/models/model_ledger_entry.py +92 -0
  79. omnibase_infra/nodes/node_ledger_write_effect/models/model_ledger_query.py +53 -0
  80. omnibase_infra/nodes/node_ledger_write_effect/models/model_ledger_query_result.py +41 -0
  81. omnibase_infra/nodes/node_ledger_write_effect/node.py +89 -0
  82. omnibase_infra/nodes/node_ledger_write_effect/protocols/__init__.py +13 -0
  83. omnibase_infra/nodes/node_ledger_write_effect/protocols/protocol_ledger_persistence.py +127 -0
  84. omnibase_infra/nodes/node_ledger_write_effect/registry/__init__.py +9 -0
  85. omnibase_infra/nodes/node_ledger_write_effect/registry/registry_infra_ledger_write.py +121 -0
  86. omnibase_infra/nodes/node_registration_orchestrator/registry/registry_infra_node_registration_orchestrator.py +7 -5
  87. omnibase_infra/nodes/reducers/models/__init__.py +7 -2
  88. omnibase_infra/nodes/reducers/models/model_payload_consul_register.py +11 -0
  89. omnibase_infra/nodes/reducers/models/model_payload_ledger_append.py +133 -0
  90. omnibase_infra/nodes/reducers/registration_reducer.py +1 -0
  91. omnibase_infra/protocols/__init__.py +3 -0
  92. omnibase_infra/protocols/protocol_dispatch_engine.py +152 -0
  93. omnibase_infra/runtime/__init__.py +60 -0
  94. omnibase_infra/runtime/binding_resolver.py +753 -0
  95. omnibase_infra/runtime/constants_security.py +70 -0
  96. omnibase_infra/runtime/contract_loaders/__init__.py +9 -0
  97. omnibase_infra/runtime/contract_loaders/operation_bindings_loader.py +789 -0
  98. omnibase_infra/runtime/emit_daemon/__init__.py +97 -0
  99. omnibase_infra/runtime/emit_daemon/cli.py +844 -0
  100. omnibase_infra/runtime/emit_daemon/client.py +811 -0
  101. omnibase_infra/runtime/emit_daemon/config.py +535 -0
  102. omnibase_infra/runtime/emit_daemon/daemon.py +812 -0
  103. omnibase_infra/runtime/emit_daemon/event_registry.py +477 -0
  104. omnibase_infra/runtime/emit_daemon/model_daemon_request.py +139 -0
  105. omnibase_infra/runtime/emit_daemon/model_daemon_response.py +191 -0
  106. omnibase_infra/runtime/emit_daemon/queue.py +618 -0
  107. omnibase_infra/runtime/event_bus_subcontract_wiring.py +466 -0
  108. omnibase_infra/runtime/handler_source_resolver.py +43 -2
  109. omnibase_infra/runtime/kafka_contract_source.py +984 -0
  110. omnibase_infra/runtime/models/__init__.py +13 -0
  111. omnibase_infra/runtime/models/model_contract_load_result.py +224 -0
  112. omnibase_infra/runtime/models/model_runtime_contract_config.py +268 -0
  113. omnibase_infra/runtime/models/model_runtime_scheduler_config.py +4 -3
  114. omnibase_infra/runtime/models/model_security_config.py +109 -0
  115. omnibase_infra/runtime/publisher_topic_scoped.py +294 -0
  116. omnibase_infra/runtime/runtime_contract_config_loader.py +406 -0
  117. omnibase_infra/runtime/service_kernel.py +76 -6
  118. omnibase_infra/runtime/service_message_dispatch_engine.py +558 -15
  119. omnibase_infra/runtime/service_runtime_host_process.py +770 -20
  120. omnibase_infra/runtime/transition_notification_publisher.py +3 -2
  121. omnibase_infra/runtime/util_wiring.py +206 -62
  122. omnibase_infra/services/mcp/service_mcp_tool_sync.py +27 -9
  123. omnibase_infra/services/session/config_consumer.py +25 -8
  124. omnibase_infra/services/session/config_store.py +2 -2
  125. omnibase_infra/services/session/consumer.py +1 -1
  126. omnibase_infra/topics/__init__.py +45 -0
  127. omnibase_infra/topics/platform_topic_suffixes.py +140 -0
  128. omnibase_infra/topics/util_topic_composition.py +95 -0
  129. omnibase_infra/types/typed_dict/__init__.py +9 -1
  130. omnibase_infra/types/typed_dict/typed_dict_envelope_build_params.py +115 -0
  131. omnibase_infra/utils/__init__.py +9 -0
  132. omnibase_infra/utils/util_consumer_group.py +232 -0
  133. omnibase_infra/validation/infra_validators.py +18 -1
  134. omnibase_infra/validation/validation_exemptions.yaml +192 -0
  135. {omnibase_infra-0.2.5.dist-info → omnibase_infra-0.2.7.dist-info}/METADATA +3 -3
  136. {omnibase_infra-0.2.5.dist-info → omnibase_infra-0.2.7.dist-info}/RECORD +139 -52
  137. {omnibase_infra-0.2.5.dist-info → omnibase_infra-0.2.7.dist-info}/entry_points.txt +1 -0
  138. {omnibase_infra-0.2.5.dist-info → omnibase_infra-0.2.7.dist-info}/WHEEL +0 -0
  139. {omnibase_infra-0.2.5.dist-info → omnibase_infra-0.2.7.dist-info}/licenses/LICENSE +0 -0
@@ -29,10 +29,6 @@ Environment Variables:
29
29
  Default: "local"
30
30
  Example: "dev", "staging", "prod"
31
31
 
32
- KAFKA_GROUP: Consumer group identifier
33
- Default: "default"
34
- Example: "my-service-group"
35
-
36
32
  Timeout and Retry Settings:
37
33
  KAFKA_TIMEOUT_SECONDS: Timeout for Kafka operations (integer seconds)
38
34
  Default: 30
@@ -134,6 +130,7 @@ Usage:
134
130
  ```python
135
131
  from omnibase_infra.event_bus.event_bus_kafka import EventBusKafka
136
132
  from omnibase_infra.event_bus.models.config import ModelKafkaEventBusConfig
133
+ from omnibase_infra.models import ModelNodeIdentity
137
134
 
138
135
  # Option 1: Use defaults with environment variable overrides
139
136
  bus = EventBusKafka.default()
@@ -147,10 +144,17 @@ Usage:
147
144
  bus = EventBusKafka(config=config)
148
145
  await bus.start()
149
146
 
150
- # Subscribe to a topic
147
+ # Subscribe to a topic with node identity
148
+ identity = ModelNodeIdentity(
149
+ env="dev",
150
+ service="my-service",
151
+ node_name="event-processor",
152
+ version="v1",
153
+ )
154
+
151
155
  async def handler(msg):
152
156
  print(f"Received: {msg.value}")
153
- unsubscribe = await bus.subscribe("events", "group1", handler)
157
+ unsubscribe = await bus.subscribe("events", identity, handler)
154
158
 
155
159
  # Publish a message
156
160
  await bus.publish("events", b"key", b"value")
@@ -163,6 +167,10 @@ Usage:
163
167
  Protocol Compatibility:
164
168
  This class implements ProtocolEventBus from omnibase_core using duck typing
165
169
  (no explicit inheritance required per ONEX patterns).
170
+
171
+ TODO: Consider formalizing the EventBusKafka interface as a Protocol
172
+ (ProtocolEventBusKafka) in the future to enable better static type checking
173
+ and IDE support for consumers that depend on Kafka-specific features.
166
174
  """
167
175
 
168
176
  from __future__ import annotations
@@ -180,7 +188,7 @@ from uuid import UUID, uuid4
180
188
  from aiokafka import AIOKafkaConsumer, AIOKafkaProducer
181
189
  from aiokafka.errors import KafkaError
182
190
 
183
- from omnibase_infra.enums import EnumInfraTransportType
191
+ from omnibase_infra.enums import EnumConsumerGroupPurpose, EnumInfraTransportType
184
192
  from omnibase_infra.errors import (
185
193
  InfraConnectionError,
186
194
  InfraTimeoutError,
@@ -197,6 +205,8 @@ from omnibase_infra.event_bus.models import (
197
205
  )
198
206
  from omnibase_infra.event_bus.models.config import ModelKafkaEventBusConfig
199
207
  from omnibase_infra.mixins import MixinAsyncCircuitBreaker
208
+ from omnibase_infra.models import ModelNodeIdentity
209
+ from omnibase_infra.utils import compute_consumer_group_id
200
210
 
201
211
  logger = logging.getLogger(__name__)
202
212
 
@@ -215,29 +225,30 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
215
225
  - Circuit breaker for connection failure protection
216
226
  - Retry with exponential backoff on publish failures
217
227
  - Dead letter queue (DLQ) for failed message processing
218
- - Environment and group-based message routing
228
+ - Environment-based message routing
219
229
  - Proper async producer/consumer lifecycle management
220
230
 
221
231
  Attributes:
222
232
  environment: Environment identifier (e.g., "local", "dev", "prod")
223
- group: Consumer group identifier
224
233
  adapter: Returns self (for protocol compatibility)
225
234
 
226
235
  Architecture:
227
236
  This class uses mixin composition to organize functionality:
228
- - MixinKafkaBroadcast: Environment/group broadcast messaging, envelope publishing
237
+ - MixinKafkaBroadcast: Environment broadcast messaging, envelope publishing
229
238
  - MixinKafkaDlq: Dead letter queue handling and metrics
230
239
  - MixinAsyncCircuitBreaker: Circuit breaker resilience pattern
231
240
 
232
241
  The core class provides:
233
242
  - Factory methods (3): from_config, from_yaml, default
234
- - Properties (4): config, adapter, environment, group
243
+ - Properties (3): config, adapter, environment
235
244
  - Lifecycle methods (4): start, initialize, shutdown, close
236
245
  - Pub/Sub methods (3): publish, subscribe, start_consuming
237
246
  - Health check (1): health_check
238
247
 
239
248
  Example:
240
249
  ```python
250
+ from omnibase_infra.models import ModelNodeIdentity
251
+
241
252
  config = ModelKafkaEventBusConfig(
242
253
  bootstrap_servers="kafka:9092",
243
254
  environment="dev",
@@ -245,10 +256,17 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
245
256
  bus = EventBusKafka(config=config)
246
257
  await bus.start()
247
258
 
248
- # Subscribe
259
+ # Subscribe with node identity
260
+ identity = ModelNodeIdentity(
261
+ env="dev",
262
+ service="my-service",
263
+ node_name="event-processor",
264
+ version="v1",
265
+ )
266
+
249
267
  async def handler(msg):
250
268
  print(f"Received: {msg.value}")
251
- unsubscribe = await bus.subscribe("events", "group1", handler)
269
+ unsubscribe = await bus.subscribe("events", identity, handler)
252
270
 
253
271
  # Publish
254
272
  await bus.publish("events", b"key", b"value")
@@ -296,7 +314,6 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
296
314
  # Apply config values
297
315
  self._bootstrap_servers = config.bootstrap_servers
298
316
  self._environment = config.environment
299
- self._group = config.group
300
317
  self._timeout_seconds = config.timeout_seconds
301
318
  self._max_retry_attempts = config.max_retry_attempts
302
319
  self._retry_backoff_base = config.retry_backoff_base
@@ -450,15 +467,6 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
450
467
  """
451
468
  return self._environment
452
469
 
453
- @property
454
- def group(self) -> str:
455
- """Get the consumer group identifier.
456
-
457
- Returns:
458
- Consumer group string
459
- """
460
- return self._group
461
-
462
470
  async def start(self) -> None:
463
471
  """Start the event bus and connect to Kafka.
464
472
 
@@ -511,7 +519,6 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
511
519
  "EventBusKafka started",
512
520
  extra={
513
521
  "environment": self._environment,
514
- "group": self._group,
515
522
  "bootstrap_servers": self._sanitize_bootstrap_servers(
516
523
  self._bootstrap_servers
517
524
  ),
@@ -612,7 +619,6 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
612
619
  Args:
613
620
  config: Configuration dictionary with optional keys:
614
621
  - environment: Override environment setting
615
- - group: Override group setting
616
622
  - bootstrap_servers: Override bootstrap servers
617
623
  - timeout_seconds: Override timeout setting
618
624
  """
@@ -620,8 +626,6 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
620
626
  async with self._lock:
621
627
  if "environment" in config:
622
628
  self._environment = str(config["environment"])
623
- if "group" in config:
624
- self._group = str(config["group"])
625
629
  if "bootstrap_servers" in config:
626
630
  self._bootstrap_servers = str(config["bootstrap_servers"])
627
631
  if "timeout_seconds" in config:
@@ -696,7 +700,7 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
696
700
 
697
701
  logger.info(
698
702
  "EventBusKafka closed",
699
- extra={"environment": self._environment, "group": self._group},
703
+ extra={"environment": self._environment},
700
704
  )
701
705
 
702
706
  async def publish(
@@ -739,7 +743,7 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
739
743
  # Create headers if not provided
740
744
  if headers is None:
741
745
  headers = ModelEventHeaders(
742
- source=f"{self._environment}.{self._group}",
746
+ source=self._environment,
743
747
  event_type=topic,
744
748
  timestamp=datetime.now(UTC),
745
749
  )
@@ -915,32 +919,58 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
915
919
  async def subscribe(
916
920
  self,
917
921
  topic: str,
918
- group_id: str,
922
+ node_identity: ModelNodeIdentity,
919
923
  on_message: Callable[[ModelEventMessage], Awaitable[None]],
924
+ *,
925
+ purpose: EnumConsumerGroupPurpose = EnumConsumerGroupPurpose.CONSUME,
920
926
  ) -> Callable[[], Awaitable[None]]:
921
927
  """Subscribe to topic with callback handler.
922
928
 
923
929
  Registers a callback to be invoked for each message received on the topic.
924
930
  Returns an unsubscribe function to remove the subscription.
925
931
 
932
+ The consumer group ID is derived from the node identity using the canonical
933
+ format: ``{env}.{service}.{node_name}.{purpose}.{version}``.
934
+
926
935
  Note: Unlike typical Kafka consumer groups, this implementation maintains
927
936
  a subscriber registry and fans out messages to all registered callbacks,
928
937
  matching the EventBusInmemory interface.
929
938
 
930
939
  Args:
931
940
  topic: Topic to subscribe to
932
- group_id: Consumer group identifier for this subscription
941
+ node_identity: Node identity used to derive the consumer group ID.
942
+ Contains env, service, node_name, and version components.
933
943
  on_message: Async callback invoked for each message
944
+ purpose: Consumer group purpose classification. Defaults to CONSUME.
945
+ Used in the consumer group ID derivation for disambiguation.
934
946
 
935
947
  Returns:
936
948
  Async unsubscribe function to remove this subscription
937
949
 
938
950
  Example:
939
951
  ```python
952
+ from omnibase_infra.models import ModelNodeIdentity
953
+ from omnibase_infra.enums import EnumConsumerGroupPurpose
954
+
955
+ identity = ModelNodeIdentity(
956
+ env="dev",
957
+ service="my-service",
958
+ node_name="event-processor",
959
+ version="v1",
960
+ )
961
+
940
962
  async def handler(msg):
941
963
  print(f"Received: {msg.value}")
942
964
 
943
- unsubscribe = await bus.subscribe("events", "group1", handler)
965
+ # Standard subscription (group_id: dev.my-service.event-processor.consume.v1)
966
+ unsubscribe = await bus.subscribe("events", identity, handler)
967
+
968
+ # With explicit purpose
969
+ unsubscribe = await bus.subscribe(
970
+ "events", identity, handler,
971
+ purpose=EnumConsumerGroupPurpose.INTROSPECTION,
972
+ )
973
+
944
974
  # ... later ...
945
975
  await unsubscribe()
946
976
  ```
@@ -948,22 +978,27 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
948
978
  subscription_id = str(uuid4())
949
979
  correlation_id = uuid4()
950
980
 
981
+ # Derive consumer group ID from node identity (no overrides allowed)
982
+ effective_group_id = compute_consumer_group_id(node_identity, purpose)
983
+
951
984
  # Validate topic name
952
985
  self._validate_topic_name(topic, correlation_id)
953
986
 
954
987
  async with self._lock:
955
988
  # Add to subscriber registry
956
- self._subscribers[topic].append((group_id, subscription_id, on_message))
989
+ self._subscribers[topic].append(
990
+ (effective_group_id, subscription_id, on_message)
991
+ )
957
992
 
958
993
  # Start consumer for this topic if not already running
959
994
  if topic not in self._consumers and self._started:
960
- await self._start_consumer_for_topic(topic, group_id)
995
+ await self._start_consumer_for_topic(topic, effective_group_id)
961
996
 
962
997
  logger.debug(
963
998
  "Subscriber added",
964
999
  extra={
965
1000
  "topic": topic,
966
- "group_id": group_id,
1001
+ "group_id": effective_group_id,
967
1002
  "subscription_id": subscription_id,
968
1003
  },
969
1004
  )
@@ -983,7 +1018,7 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
983
1018
  "Subscriber removed",
984
1019
  extra={
985
1020
  "topic": topic,
986
- "group_id": group_id,
1021
+ "group_id": effective_group_id,
987
1022
  "subscription_id": subscription_id,
988
1023
  },
989
1024
  )
@@ -1006,9 +1041,13 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
1006
1041
 
1007
1042
  Args:
1008
1043
  topic: Topic to consume from
1009
- group_id: Consumer group ID
1044
+ group_id: Fully qualified consumer group ID. This should be derived
1045
+ from ``compute_consumer_group_id()`` or an explicit override.
1046
+ The ID is used directly without any prefix modification.
1010
1047
 
1011
1048
  Raises:
1049
+ ProtocolConfigurationError: If group_id is empty (must be derived from
1050
+ compute_consumer_group_id or provided as explicit override)
1012
1051
  InfraTimeoutError: If consumer startup times out after timeout_seconds
1013
1052
  InfraConnectionError: If consumer fails to connect to Kafka brokers
1014
1053
  """
@@ -1018,15 +1057,29 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
1018
1057
  correlation_id = uuid4()
1019
1058
  sanitized_servers = self._sanitize_bootstrap_servers(self._bootstrap_servers)
1020
1059
 
1021
- # Normalize empty string to default group (treats "" same as None)
1022
- # This ensures consistent behavior when group_id is unset or empty
1023
- effective_group_id = group_id.strip() if group_id else self._group
1060
+ # Use group_id directly - it's already fully qualified from compute_consumer_group_id()
1061
+ # or an explicit override. Empty group_id indicates a bug in the caller.
1062
+ effective_group_id = group_id.strip()
1063
+ if not effective_group_id:
1064
+ context = ModelInfraErrorContext.with_correlation(
1065
+ correlation_id=correlation_id,
1066
+ transport_type=EnumInfraTransportType.KAFKA,
1067
+ operation="start_consumer",
1068
+ target_name=f"kafka.{topic}",
1069
+ )
1070
+ raise ProtocolConfigurationError(
1071
+ f"Consumer group ID is required for topic '{topic}'. "
1072
+ "Internal error: compute_consumer_group_id() should have been called.",
1073
+ context=context,
1074
+ parameter="group_id",
1075
+ value=group_id,
1076
+ )
1024
1077
 
1025
1078
  # Apply consumer configuration from config model
1026
1079
  consumer = AIOKafkaConsumer(
1027
1080
  topic,
1028
1081
  bootstrap_servers=self._bootstrap_servers,
1029
- group_id=f"{self._environment}.{effective_group_id}",
1082
+ group_id=effective_group_id,
1030
1083
  auto_offset_reset=self._config.auto_offset_reset,
1031
1084
  enable_auto_commit=self._config.enable_auto_commit,
1032
1085
  )
@@ -1194,6 +1247,15 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
1194
1247
  )
1195
1248
  break
1196
1249
 
1250
+ # Get subscribers snapshot early - needed for consumer group in DLQ
1251
+ async with self._lock:
1252
+ subscribers = list(self._subscribers.get(topic, []))
1253
+
1254
+ # Extract consumer group for DLQ traceability (all subscribers share the same consumer)
1255
+ effective_consumer_group = (
1256
+ subscribers[0][0] if subscribers else "unknown"
1257
+ )
1258
+
1197
1259
  # Convert Kafka message to ModelEventMessage - handle conversion errors
1198
1260
  try:
1199
1261
  event_message = self._kafka_msg_to_model(msg, topic)
@@ -1215,13 +1277,10 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
1215
1277
  error=e,
1216
1278
  correlation_id=correlation_id,
1217
1279
  failure_type="deserialization_error",
1280
+ consumer_group=effective_consumer_group,
1218
1281
  )
1219
1282
  continue # Skip this message but continue consuming
1220
1283
 
1221
- # Get subscribers snapshot
1222
- async with self._lock:
1223
- subscribers = list(self._subscribers.get(topic, []))
1224
-
1225
1284
  # Dispatch to all subscribers
1226
1285
  for group_id, subscription_id, callback in subscribers:
1227
1286
  try:
@@ -1255,6 +1314,7 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
1255
1314
  failed_message=event_message,
1256
1315
  error=e,
1257
1316
  correlation_id=correlation_id,
1317
+ consumer_group=group_id,
1258
1318
  )
1259
1319
  else:
1260
1320
  # Message still has retries available - log for potential republish
@@ -1340,7 +1400,6 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
1340
1400
  - healthy: Whether the bus is operational
1341
1401
  - started: Whether start() has been called
1342
1402
  - environment: Current environment
1343
- - group: Current consumer group
1344
1403
  - bootstrap_servers: Kafka bootstrap servers
1345
1404
  - circuit_state: Current circuit breaker state
1346
1405
  - subscriber_count: Total number of active subscriptions
@@ -1371,7 +1430,6 @@ class EventBusKafka(MixinKafkaBroadcast, MixinKafkaDlq, MixinAsyncCircuitBreaker
1371
1430
  "healthy": started and producer_healthy,
1372
1431
  "started": started,
1373
1432
  "environment": self._environment,
1374
- "group": self._group,
1375
1433
  "bootstrap_servers": self._sanitize_bootstrap_servers(
1376
1434
  self._bootstrap_servers
1377
1435
  ),
@@ -26,7 +26,6 @@ Usage:
26
26
  Design Note:
27
27
  This mixin assumes the parent class has:
28
28
  - self._environment: str for environment context
29
- - self._group: str for consumer group context
30
29
  - self.publish(): Async method to publish messages
31
30
  """
32
31
 
@@ -51,7 +50,6 @@ class ProtocolKafkaBroadcastHost(Protocol):
51
50
  """
52
51
 
53
52
  _environment: str
54
- _group: str
55
53
 
56
54
  async def publish(
57
55
  self,
@@ -77,7 +75,6 @@ class MixinKafkaBroadcast:
77
75
 
78
76
  Required attributes from parent class:
79
77
  _environment: str for environment context
80
- _group: str for consumer group context
81
78
 
82
79
  Required methods from parent class:
83
80
  publish: Async method to publish messages to a topic
@@ -85,7 +82,6 @@ class MixinKafkaBroadcast:
85
82
 
86
83
  # Type hints for attributes expected from parent class
87
84
  _environment: str
88
- _group: str
89
85
 
90
86
  async def broadcast_to_environment(
91
87
  self: ProtocolKafkaBroadcastHost,
@@ -108,7 +104,7 @@ class MixinKafkaBroadcast:
108
104
  value = json.dumps(value_dict).encode("utf-8")
109
105
 
110
106
  headers = ModelEventHeaders(
111
- source=f"{self._environment}.{self._group}",
107
+ source=self._environment,
112
108
  event_type="broadcast",
113
109
  content_type="application/json",
114
110
  timestamp=datetime.now(UTC),
@@ -136,7 +132,7 @@ class MixinKafkaBroadcast:
136
132
  value = json.dumps(value_dict).encode("utf-8")
137
133
 
138
134
  headers = ModelEventHeaders(
139
- source=f"{self._environment}.{self._group}",
135
+ source=self._environment,
140
136
  event_type="group_command",
141
137
  content_type="application/json",
142
138
  timestamp=datetime.now(UTC),
@@ -172,7 +168,7 @@ class MixinKafkaBroadcast:
172
168
  value = json.dumps(envelope_dict).encode("utf-8")
173
169
 
174
170
  headers = ModelEventHeaders(
175
- source=f"{self._environment}.{self._group}",
171
+ source=self._environment,
176
172
  event_type=topic,
177
173
  content_type="application/json",
178
174
  timestamp=datetime.now(UTC),
@@ -32,7 +32,6 @@ Design Note:
32
32
  This mixin assumes the parent class has:
33
33
  - self._config: ModelKafkaEventBusConfig with dead_letter_topic
34
34
  - self._environment: str for environment context
35
- - self._group: str for consumer group context
36
35
  - self._producer: AIOKafkaProducer | None
37
36
  - self._producer_lock: asyncio.Lock for producer access
38
37
  - self._timeout_seconds: int for publish timeout
@@ -75,7 +74,6 @@ class ProtocolKafkaDlqHost(Protocol):
75
74
  # Attributes from parent class (EventBusKafka)
76
75
  _config: ModelKafkaEventBusConfig
77
76
  _environment: str
78
- _group: str
79
77
  _producer: AIOKafkaProducer | None
80
78
  _producer_lock: asyncio.Lock
81
79
  _timeout_seconds: int
@@ -212,6 +210,8 @@ class MixinKafkaDlq:
212
210
  failed_message: ModelEventMessage,
213
211
  error: Exception,
214
212
  correlation_id: UUID,
213
+ *,
214
+ consumer_group: str,
215
215
  ) -> None:
216
216
  """Publish failed message to dead letter queue with metrics and alerting.
217
217
 
@@ -232,6 +232,8 @@ class MixinKafkaDlq:
232
232
  failed_message: The message that failed processing
233
233
  error: The exception that caused the failure
234
234
  correlation_id: Correlation ID for tracking
235
+ consumer_group: Consumer group ID that processed the message.
236
+ Required for DLQ traceability.
235
237
 
236
238
  Note:
237
239
  This method logs errors if DLQ publishing fails but does not raise
@@ -298,7 +300,7 @@ class MixinKafkaDlq:
298
300
 
299
301
  # Create DLQ headers with failure metadata
300
302
  dlq_headers = ModelEventHeaders(
301
- source=f"{self._environment}.{self._group}",
303
+ source=self._environment,
302
304
  event_type="dlq_message",
303
305
  content_type="application/json",
304
306
  correlation_id=correlation_id,
@@ -451,7 +453,7 @@ class MixinKafkaDlq:
451
453
  dlq_error_message=dlq_error_message,
452
454
  timestamp=end_time,
453
455
  environment=self._environment,
454
- consumer_group=self._group,
456
+ consumer_group=consumer_group,
455
457
  )
456
458
 
457
459
  # Update DLQ metrics (copy-on-write pattern)
@@ -504,6 +506,8 @@ class MixinKafkaDlq:
504
506
  error: Exception,
505
507
  correlation_id: UUID,
506
508
  failure_type: str,
509
+ *,
510
+ consumer_group: str,
507
511
  ) -> None:
508
512
  """Publish raw Kafka message to DLQ when deserialization fails.
509
513
 
@@ -518,6 +522,8 @@ class MixinKafkaDlq:
518
522
  error: The exception that caused the failure
519
523
  correlation_id: Correlation ID for tracking
520
524
  failure_type: Type of failure (e.g., "deserialization_error")
525
+ consumer_group: Consumer group ID that processed the message.
526
+ Required for DLQ traceability.
521
527
 
522
528
  Note:
523
529
  This method logs errors if DLQ publishing fails but does not raise
@@ -593,7 +599,7 @@ class MixinKafkaDlq:
593
599
 
594
600
  # Create DLQ headers
595
601
  dlq_headers = ModelEventHeaders(
596
- source=f"{self._environment}.{self._group}",
602
+ source=self._environment,
597
603
  event_type="dlq_raw_message",
598
604
  content_type="application/json",
599
605
  correlation_id=correlation_id,
@@ -746,7 +752,7 @@ class MixinKafkaDlq:
746
752
  dlq_error_message=dlq_error_message,
747
753
  timestamp=end_time,
748
754
  environment=self._environment,
749
- consumer_group=self._group,
755
+ consumer_group=consumer_group,
750
756
  )
751
757
 
752
758
  # Update DLQ metrics
@@ -27,10 +27,6 @@ Environment Variables:
27
27
  Default: "local"
28
28
  Example: "dev", "staging", "prod"
29
29
 
30
- KAFKA_GROUP: Consumer group identifier
31
- Default: "default"
32
- Example: "my-service-group"
33
-
34
30
  Timeout and Retry Settings (with validation):
35
31
  KAFKA_TIMEOUT_SECONDS: Timeout for operations (integer, 1-300)
36
32
  Default: 30
@@ -124,7 +120,6 @@ class ModelKafkaEventBusConfig(BaseModel):
124
120
  Attributes:
125
121
  bootstrap_servers: Kafka bootstrap servers (host:port format)
126
122
  environment: Environment identifier for message routing
127
- group: Consumer group identifier for message routing
128
123
  timeout_seconds: Timeout for Kafka operations in seconds
129
124
  max_retry_attempts: Maximum retry attempts for publish operations
130
125
  retry_backoff_base: Base delay in seconds for exponential backoff
@@ -167,11 +162,6 @@ class ModelKafkaEventBusConfig(BaseModel):
167
162
  description="Environment identifier for message routing (e.g., 'local', 'dev', 'prod')",
168
163
  min_length=1,
169
164
  )
170
- group: str = Field(
171
- default="default",
172
- description="Consumer group identifier for message routing",
173
- min_length=1,
174
- )
175
165
  timeout_seconds: int = Field(
176
166
  default=30,
177
167
  description="Timeout for Kafka operations in seconds",
@@ -408,73 +398,6 @@ class ModelKafkaEventBusConfig(BaseModel):
408
398
  )
409
399
  return v.strip()
410
400
 
411
- @field_validator("group", mode="before")
412
- @classmethod
413
- def validate_group(cls, v: object) -> str:
414
- """Validate consumer group identifier.
415
-
416
- Args:
417
- v: Group value (any type before Pydantic conversion)
418
-
419
- Returns:
420
- Validated group string
421
-
422
- Raises:
423
- ProtocolConfigurationError: If group is empty, invalid type, or contains invalid characters
424
- """
425
- context = ModelInfraErrorContext(
426
- transport_type=EnumInfraTransportType.KAFKA,
427
- operation="validate_config",
428
- target_name="kafka_config",
429
- correlation_id=uuid4(),
430
- )
431
-
432
- if v is None:
433
- raise ProtocolConfigurationError(
434
- "group cannot be None",
435
- context=context,
436
- parameter="group",
437
- value=None,
438
- )
439
- if not isinstance(v, str):
440
- raise ProtocolConfigurationError(
441
- f"group must be a string, got {type(v).__name__}",
442
- context=context,
443
- parameter="group",
444
- value=type(v).__name__,
445
- )
446
-
447
- group_name = v.strip()
448
- if not group_name:
449
- raise ProtocolConfigurationError(
450
- "group cannot be empty",
451
- context=context,
452
- parameter="group",
453
- value=v,
454
- )
455
-
456
- # Kafka group names have similar restrictions to topic names
457
- # but are generally more permissive
458
- if len(group_name) > 255:
459
- raise ProtocolConfigurationError(
460
- f"group name '{group_name}' exceeds maximum length of 255 characters",
461
- context=context,
462
- parameter="group",
463
- value=group_name,
464
- )
465
-
466
- # Check for invalid characters (control characters, null bytes)
467
- for char in group_name:
468
- if ord(char) < 32 or char == "\x7f":
469
- raise ProtocolConfigurationError(
470
- f"group name '{group_name}' contains invalid control character",
471
- context=context,
472
- parameter="group",
473
- value=group_name,
474
- )
475
-
476
- return group_name
477
-
478
401
  def apply_environment_overrides(self) -> ModelKafkaEventBusConfig:
479
402
  """Apply environment variable overrides to configuration.
480
403
 
@@ -482,7 +405,6 @@ class ModelKafkaEventBusConfig(BaseModel):
482
405
  - KAFKA_BOOTSTRAP_SERVERS -> bootstrap_servers
483
406
  - KAFKA_TIMEOUT_SECONDS -> timeout_seconds
484
407
  - KAFKA_ENVIRONMENT -> environment
485
- - KAFKA_GROUP -> group
486
408
  - KAFKA_MAX_RETRY_ATTEMPTS -> max_retry_attempts
487
409
  - KAFKA_CIRCUIT_BREAKER_THRESHOLD -> circuit_breaker_threshold
488
410
 
@@ -495,7 +417,6 @@ class ModelKafkaEventBusConfig(BaseModel):
495
417
  "KAFKA_BOOTSTRAP_SERVERS": "bootstrap_servers",
496
418
  "KAFKA_TIMEOUT_SECONDS": "timeout_seconds",
497
419
  "KAFKA_ENVIRONMENT": "environment",
498
- "KAFKA_GROUP": "group",
499
420
  "KAFKA_MAX_RETRY_ATTEMPTS": "max_retry_attempts",
500
421
  "KAFKA_CIRCUIT_BREAKER_THRESHOLD": "circuit_breaker_threshold",
501
422
  "KAFKA_CIRCUIT_BREAKER_RESET_TIMEOUT": "circuit_breaker_reset_timeout",
@@ -625,7 +546,6 @@ class ModelKafkaEventBusConfig(BaseModel):
625
546
  base_config = cls(
626
547
  bootstrap_servers="localhost:9092",
627
548
  environment="local",
628
- group="default",
629
549
  timeout_seconds=30,
630
550
  max_retry_attempts=3,
631
551
  retry_backoff_base=1.0,
@@ -662,7 +582,6 @@ class ModelKafkaEventBusConfig(BaseModel):
662
582
  ```yaml
663
583
  bootstrap_servers: "kafka:9092"
664
584
  environment: "prod"
665
- group: "my-service"
666
585
  timeout_seconds: 60
667
586
  max_retry_attempts: 5
668
587
  circuit_breaker_threshold: 10