trustgraph-base 2.2.19__tar.gz → 2.2.20__tar.gz

{trustgraph_base-2.2.19 → trustgraph_base-2.2.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-base
-Version: 2.2.19
+Version: 2.2.20
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_base-2.2.19 → trustgraph_base-2.2.20}/trustgraph/base/async_processor.py

@@ -1,7 +1,8 @@
 
 # Base class for processors.  Implements:
-# - Pulsar client, subscribe and consume basic
+# - Pub/sub client, subscribe and consume basic
 # - the async startup logic
+# - Config notify handling with subscribe-then-fetch pattern
 # - Initialising metrics
 
 import asyncio
@@ -12,12 +13,17 @@ import logging
 import os
 from prometheus_client import start_http_server, Info
 
-from .. schema import ConfigPush, config_push_queue
+from .. schema import ConfigPush, ConfigRequest, ConfigResponse
+from .. schema import config_push_queue, config_request_queue
+from .. schema import config_response_queue
 from .. log_level import LogLevel
 from . pubsub import get_pubsub, add_pubsub_args
 from . producer import Producer
 from . consumer import Consumer
-from . metrics import ProcessorMetrics, ConsumerMetrics
+from . subscriber import Subscriber
+from . request_response_spec import RequestResponse
+from . metrics import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
+from . metrics import SubscriberMetrics
 from . logging import add_logging_args, setup_logging
 
 default_config_queue = config_push_queue
@@ -57,9 +63,13 @@ class AsyncProcessor:
             "config_push_queue", default_config_queue
         )
 
-        # This records registered configuration handlers
+        # This records registered configuration handlers, each entry is:
+        # { "handler": async_fn, "types": set_or_none }
         self.config_handlers = []
 
+        # Track the current config version for dedup
+        self.config_version = 0
+
         # Create a random ID for this subscription to the configuration
         # service
         config_subscriber_id = str(uuid.uuid4())
@@ -68,8 +78,7 @@ class AsyncProcessor:
             processor = self.id, flow = None, name = "config",
         )
 
-        # Subscribe to config queue — exclusive so every processor
-        # gets its own copy of config pushes (broadcast pattern)
+        # Subscribe to config notify queue
         self.config_sub_task = Consumer(
 
             taskgroup = self.taskgroup,
@@ -80,21 +89,93 @@ class AsyncProcessor:
             topic = self.config_push_queue,
             schema = ConfigPush,
 
-            handler = self.on_config_change,
+            handler = self.on_config_notify,
 
             metrics = config_consumer_metrics,
 
-            start_of_messages = True,
+            start_of_messages = False,
             consumer_type = 'exclusive',
         )
 
         self.running = True
 
-    # This is called to start dynamic behaviour.  An over-ride point for
-    # extra functionality
+    def _create_config_client(self):
+        """Create a short-lived config request/response client."""
+        config_rr_id = str(uuid.uuid4())
+
+        config_req_metrics = ProducerMetrics(
+            processor = self.id, flow = None, name = "config-request",
+        )
+        config_resp_metrics = SubscriberMetrics(
+            processor = self.id, flow = None, name = "config-response",
+        )
+
+        return RequestResponse(
+            backend = self.pubsub_backend,
+            subscription = f"{self.id}--config--{config_rr_id}",
+            consumer_name = self.id,
+            request_topic = config_request_queue,
+            request_schema = ConfigRequest,
+            request_metrics = config_req_metrics,
+            response_topic = config_response_queue,
+            response_schema = ConfigResponse,
+            response_metrics = config_resp_metrics,
+        )
+
+    async def fetch_config(self):
+        """Fetch full config from config service using a short-lived
+        request/response client. Returns (config, version) or raises."""
+        client = self._create_config_client()
+        try:
+            await client.start()
+            resp = await client.request(
+                ConfigRequest(operation="config"),
+                timeout=10,
+            )
+            if resp.error:
+                raise RuntimeError(f"Config error: {resp.error.message}")
+            return resp.config, resp.version
+        finally:
+            await client.stop()
+
+    # This is called to start dynamic behaviour.
+    # Implements the subscribe-then-fetch pattern to avoid race conditions.
     async def start(self):
+
+        # 1. Start the notify consumer (begins buffering incoming notifys)
         await self.config_sub_task.start()
 
+        # 2. Fetch current config via request/response
+        await self.fetch_and_apply_config()
+
+        # 3. Any buffered notifys with version > fetched version will be
+        #    processed by on_config_notify, which does the version check
+
+    async def fetch_and_apply_config(self):
+        """Fetch full config from config service and apply to all handlers.
+        Retries until successful — config service may not be ready yet."""
+
+        while self.running:
+
+            try:
+                config, version = await self.fetch_config()
+
+                logger.info(f"Fetched config version {version}")
+
+                self.config_version = version
+
+                # Apply to all handlers (startup = invoke all)
+                for entry in self.config_handlers:
+                    await entry["handler"](config, version)
+
+                return
+
+            except Exception as e:
+                logger.warning(
+                    f"Config fetch failed: {e}, retrying in 2s..."
+                )
+                await asyncio.sleep(2)
+
     # This is called to stop all threads.  An over-ride point for extra
     # functionality
     def stop(self):
@@ -110,20 +191,66 @@ class AsyncProcessor:
     def pulsar_host(self): return self._pulsar_host
 
     # Register a new event handler for configuration change
-    def register_config_handler(self, handler):
-        self.config_handlers.append(handler)
+    def register_config_handler(self, handler, types=None):
+        self.config_handlers.append({
+            "handler": handler,
+            "types": set(types) if types else None,
+        })
 
-    # Called when a new configuration message push occurs
-    async def on_config_change(self, message, consumer, flow):
+    # Called when a config notify message arrives
+    async def on_config_notify(self, message, consumer, flow):
+
+        notify_version = message.value().version
+        notify_types = set(message.value().types)
+
+        # Skip if we already have this version or newer
+        if notify_version <= self.config_version:
+            logger.debug(
+                f"Ignoring config notify v{notify_version}, "
+                f"already at v{self.config_version}"
+            )
+            return
+
+        # Check if any handler cares about the affected types
+        if notify_types:
+            any_interested = False
+            for entry in self.config_handlers:
+                handler_types = entry["types"]
+                if handler_types is None or notify_types & handler_types:
+                    any_interested = True
+                    break
+
+            if not any_interested:
+                logger.debug(
+                    f"Ignoring config notify v{notify_version}, "
+                    f"no handlers for types {notify_types}"
+                )
+                self.config_version = notify_version
+                return
 
-        # Get configuration data and version number
-        config = message.value().config
-        version = message.value().version
+        logger.info(
+            f"Config notify v{notify_version} types={list(notify_types)}, "
+            f"fetching config..."
+        )
+
+        # Fetch full config using short-lived client
+        try:
+            config, version = await self.fetch_config()
+
+            self.config_version = version
 
-        # Invoke message handlers
-        logger.info(f"Config change event: version={version}")
-        for ch in self.config_handlers:
-            await ch(config, version)
+            # Invoke handlers that care about the affected types
+            for entry in self.config_handlers:
+                handler_types = entry["types"]
+                if handler_types is None:
+                    await entry["handler"](config, version)
+                elif not notify_types or notify_types & handler_types:
+                    await entry["handler"](config, version)
+
+        except Exception as e:
+            logger.error(
+                f"Failed to fetch config on notify: {e}", exc_info=True
+            )
 
     # This is the 'main' body of the handler.  It is a point to override
     # if needed.  By default does nothing.  Processors are implemented
@@ -181,7 +308,7 @@ class AsyncProcessor:
             prog=ident,
             description=doc
         )
-        
+
         parser.add_argument(
             '--id',
             default=ident,
@@ -271,4 +398,3 @@ class AsyncProcessor:
             default=8000,
             help=f'Pulsar host (default: 8000)',
         )
-

{trustgraph_base-2.2.19 → trustgraph_base-2.2.20}/trustgraph/base/flow_processor.py

@@ -26,7 +26,9 @@ class FlowProcessor(AsyncProcessor):
         super(FlowProcessor, self).__init__(**params)
 
         # Register configuration handler
-        self.register_config_handler(self.on_configure_flows)
+        self.register_config_handler(
+            self.on_configure_flows, types=["active-flow"]
+        )
 
         # Initialise flow information state
         self.flows = {}

trustgraph_base-2.2.20/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "2.2.20"

{trustgraph_base-2.2.19 → trustgraph_base-2.2.20}/trustgraph/schema/services/config.py

@@ -58,11 +58,11 @@ class ConfigResponse:
 @dataclass
 class ConfigPush:
     version: int = 0
-    config: dict[str, dict[str, str]] = field(default_factory=dict)
+    types: list[str] = field(default_factory=list)
 
 config_request_queue = queue('config', cls='request')
 config_response_queue = queue('config', cls='response')
-config_push_queue = queue('config', cls='state')
+config_push_queue = queue('config', cls='flow')
 
 ############################################################################
 

{trustgraph_base-2.2.19 → trustgraph_base-2.2.20}/trustgraph_base.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-base
-Version: 2.2.19
+Version: 2.2.20
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

trustgraph_base-2.2.19/trustgraph/base_version.py

@@ -1 +0,0 @@
-__version__ = "2.2.19"