trustgraph-flow 2.7.13__tar.gz → 2.8.0__tar.gz

{trustgraph_flow-2.7.13 → trustgraph_flow-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-flow
-Version: 2.7.13
+Version: 2.8.0
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph
@@ -8,7 +8,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: trustgraph-base<2.8,>=2.7
+Requires-Dist: trustgraph-base<2.9,>=2.8
 Requires-Dist: aiohttp
 Requires-Dist: anthropic
 Requires-Dist: scylla-driver

{trustgraph_flow-2.7.13 → trustgraph_flow-2.8.0}/pyproject.toml

@@ -10,7 +10,7 @@ description = "TrustGraph provides a means to run a pipeline of flexible AI proc
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
-    "trustgraph-base>=2.7,<2.8",
+    "trustgraph-base>=2.8,<2.9",
     "aiohttp",
     "anthropic",
     "scylla-driver",

{trustgraph_flow-2.7.13 → trustgraph_flow-2.8.0}/trustgraph/bootstrap/bootstrapper/service.py

@@ -17,9 +17,8 @@ from argparse import ArgumentParser
 from dataclasses import dataclass
 
 from trustgraph.base import AsyncProcessor
-from trustgraph.base import ProducerMetrics, SubscriberMetrics
-from trustgraph.base.config_client import ConfigClient
-from trustgraph.base.request_response_spec import RequestResponse
+from trustgraph.base.async_config_client import AsyncConfigClient
+from trustgraph.base.async_rr_wrapper import AsyncRequestResponseWrapper
 from trustgraph.schema import (
     ConfigRequest, ConfigResponse,
     config_request_queue, config_response_queue,
@@ -164,68 +163,40 @@ class Processor(AsyncProcessor):
     # Client construction (short-lived per wake cycle).
     # ------------------------------------------------------------------
 
-    def _make_config_client(self):
-        rr_id = str(uuid.uuid4())
-        return ConfigClient(
-            backend=self.pubsub_backend,
-            subscription=f"{self.id}--config--{rr_id}",
-            consumer_name=self.id,
+    async def _make_config_client(self):
+        return await AsyncConfigClient.create(
+            backend=self.async_backend,
             request_topic=config_request_queue,
-            request_schema=ConfigRequest,
-            request_metrics=ProducerMetrics(
-                processor=self.id, producer="config-request",
-            ),
             response_topic=config_response_queue,
-            response_schema=ConfigResponse,
-            response_metrics=SubscriberMetrics(
-                processor=self.id, subscriber="config-response",
-            ),
+            subscription=f"{self.id}--config--{uuid.uuid4()}",
         )
 
     def _make_flow_client(self, workspace):
-        rr_id = str(uuid.uuid4())
-        return RequestResponse(
-            backend=self.pubsub_backend,
-            subscription=f"{self.id}--flow--{rr_id}",
-            consumer_name=self.id,
+        return AsyncRequestResponseWrapper(
+            backend=self.async_backend,
             request_topic=f"{flow_request_queue}:{workspace}",
-            request_schema=FlowRequest,
-            request_metrics=ProducerMetrics(
-                processor=self.id, producer="flow-request",
-            ),
             response_topic=f"{flow_response_queue}:{workspace}",
+            request_schema=FlowRequest,
             response_schema=FlowResponse,
-            response_metrics=SubscriberMetrics(
-                processor=self.id, subscriber="flow-response",
-            ),
+            subscription=f"{self.id}--flow--{uuid.uuid4()}",
         )
 
     def _make_iam_client(self):
-        rr_id = str(uuid.uuid4())
-        return RequestResponse(
-            backend=self.pubsub_backend,
-            subscription=f"{self.id}--iam--{rr_id}",
-            consumer_name=self.id,
+        return AsyncRequestResponseWrapper(
+            backend=self.async_backend,
             request_topic=iam_request_queue,
-            request_schema=IamRequest,
-            request_metrics=ProducerMetrics(
-                processor=self.id, producer="iam-request",
-            ),
             response_topic=iam_response_queue,
+            request_schema=IamRequest,
             response_schema=IamResponse,
-            response_metrics=SubscriberMetrics(
-                processor=self.id, subscriber="iam-response",
-            ),
+            subscription=f"{self.id}--iam--{uuid.uuid4()}",
         )
 
     async def _open_clients(self):
-        config = self._make_config_client()
-        await config.start()
-        return config
+        return await self._make_config_client()
 
     async def _safe_stop(self, client):
         try:
-            await client.stop()
+            await client.close()
         except Exception:
             pass
 

{trustgraph_flow-2.7.13 → trustgraph_flow-2.8.0}/trustgraph/config/service/service.py

@@ -21,13 +21,12 @@ from trustgraph.schema import WorkspaceChanges
 from trustgraph.schema import config_request_queue, config_response_queue
 from trustgraph.schema import config_push_queue
 
-from trustgraph.base import AsyncProcessor, Consumer, Producer
+from trustgraph.base import AsyncProcessor
 from trustgraph.base.cassandra_config import add_cassandra_args, resolve_cassandra_config
 
 from . config import Configuration, WORKSPACES_NAMESPACE, WORKSPACE_TYPE
 
 from ... base import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
-from ... base import Consumer, Producer
 
 # Module logger
 logger = logging.getLogger(__name__)
@@ -70,7 +69,7 @@ class Processor(AsyncProcessor):
         self.config_response_queue_base = params.get(
             "config_response_queue", default_config_response_queue
         )
-        config_push_queue = params.get(
+        self.config_push_queue_name = params.get(
             "config_push_queue", default_config_push_queue
         )
 
@@ -105,44 +104,10 @@ class Processor(AsyncProcessor):
             }
         )
 
-        config_request_metrics = ConsumerMetrics(
-            processor=self.id, consumer="config-request",
-        )
-        config_response_metrics = ProducerMetrics(
-            processor=self.id, producer="config-response",
-        )
-        config_push_metrics = ProducerMetrics(
-            processor=self.id, producer="config-push",
-        )
-
+        # Store queue names and schemas for pool registration in start()
         self.config_request_queue_base = config_request_queue
         self.config_request_subscriber = id
 
-        self.system_consumer = Consumer(
-            taskgroup = self.taskgroup,
-            backend = self.pubsub,
-            flow = None,
-            topic = config_request_queue,
-            subscriber = id,
-            schema = ConfigRequest,
-            handler = self.on_system_config_request,
-            metrics = config_request_metrics,
-        )
-
-        self.config_response_producer = Producer(
-            backend = self.pubsub,
-            topic = self.config_response_queue_base,
-            schema = ConfigResponse,
-            metrics = config_response_metrics,
-        )
-
-        self.config_push_producer = Producer(
-            backend = self.pubsub,
-            topic = config_push_queue,
-            schema = ConfigPush,
-            metrics = config_push_metrics,
-        )
-
         self.config = Configuration(
             host = self.cassandra_host,
             username = self.cassandra_username,
@@ -217,42 +182,32 @@ class Processor(AsyncProcessor):
             self.config_response_queue_base, workspace_id,
         )
 
-        await self.pubsub.ensure_topic(req_queue)
-        await self.pubsub.ensure_topic(resp_queue)
+        await self.async_backend.ensure_topic(req_queue)
+        await self.async_backend.ensure_topic(resp_queue)
 
-        response_producer = Producer(
-            backend=self.pubsub,
+        response_handle = await self.sender_pool.add_producer(
             topic=resp_queue,
             schema=ConfigResponse,
-            metrics=ProducerMetrics(
-                processor=self.id, producer="config-response",
-                workspace=workspace_id,
-            ),
         )
 
-        consumer = Consumer(
-            taskgroup=self.taskgroup,
-            backend=self.pubsub,
-            flow=None,
+        handler = partial(
+            self.on_workspace_config_request,
+            workspace=workspace_id,
+        )
+
+        async def wrapper(message):
+            await handler(message, None, None)
+
+        consumer_reg = await self.receiver_pool.add_consumer(
             topic=req_queue,
-            subscriber=self.id,
+            subscription=self.id,
             schema=ConfigRequest,
-            handler=partial(
-                self.on_workspace_config_request,
-                workspace=workspace_id,
-            ),
-            metrics=ConsumerMetrics(
-                processor=self.id, consumer="config-request",
-                workspace=workspace_id,
-            ),
+            handler=wrapper,
         )
 
-        await response_producer.start()
-        await consumer.start()
-
         self.workspace_consumers[workspace_id] = {
-            "consumer": consumer,
-            "response": response_producer,
+            "consumer": consumer_reg,
+            "response": response_handle,
         }
 
         logger.info(
@@ -262,22 +217,55 @@ class Processor(AsyncProcessor):
     async def _remove_workspace_consumer(self, workspace_id):
         clients = self.workspace_consumers.pop(workspace_id, None)
         if clients:
-            for client in clients.values():
-                await client.stop()
+            await clients["consumer"].unregister()
+            await clients["response"].unregister()
             logger.info(
                 f"Unsubscribed from workspace config queue: {workspace_id}"
             )
 
     async def start(self):
 
-        await self.pubsub.ensure_topic(self.config_request_queue_base)
-        await self.config_response_producer.start()
+        # Start the pools since we don't call super().start()
+        await self.receiver_pool.start()
+        await self.sender_pool.start()
+
+        await self.async_backend.ensure_topic(self.config_request_queue_base)
+
+        # Create system-level producers via sender_pool
+        self.config_response_handle = await self.sender_pool.add_producer(
+            topic=self.config_response_queue_base,
+            schema=ConfigResponse,
+        )
+
+        self.config_push_handle = await self.sender_pool.add_producer(
+            topic=self.config_push_queue_name,
+            schema=ConfigPush,
+        )
+
         await self.push()  # Startup poke: empty types = everything
-        await self.system_consumer.start()
+
+        # Create system consumer via receiver_pool
+        async def system_handler_wrapper(message):
+            await self.on_system_config_request(message, None, None)
+
+        self._system_consumer_reg = await self.receiver_pool.add_consumer(
+            topic=self.config_request_queue_base,
+            subscription=self.id,
+            schema=ConfigRequest,
+            handler=system_handler_wrapper,
+        )
 
         # Start the config push subscriber so we receive our own
         # workspace change notifications.
-        await self.config_sub_task.start()
+        async def config_notify_wrapper(message):
+            await self.on_config_notify(message, None, None)
+
+        self._config_sub_reg = await self.receiver_pool.add_consumer(
+            topic=self.config_push_queue,
+            subscription=self.id,
+            schema=ConfigPush,
+            handler=config_notify_wrapper,
+        )
 
         await self._discover_workspaces()
 
@@ -306,7 +294,7 @@ class Processor(AsyncProcessor):
             workspace_changes = workspace_changes,
         )
 
-        await self.config_push_producer.send(resp)
+        await self.config_push_handle.send(resp)
 
         logger.info(
             f"Pushed config poke version {version}, "
@@ -330,11 +318,11 @@ class Processor(AsyncProcessor):
                 f"workspace={workspace}..."
             )
 
-            producer = self.workspace_consumers[workspace]["response"]
+            handle = self.workspace_consumers[workspace]["response"]
 
             resp = await self.config.handle_workspace(v, workspace)
 
-            await producer.send(
+            await handle.send(
                 resp, properties={"id": id}
             )
 
@@ -347,7 +335,7 @@ class Processor(AsyncProcessor):
                 ),
             )
 
-            await producer.send(
+            await handle.send(
                 resp, properties={"id": id}
             )
 
@@ -364,7 +352,7 @@ class Processor(AsyncProcessor):
 
             resp = await self.config.handle_system(v)
 
-            await self.config_response_producer.send(
+            await self.config_response_handle.send(
                 resp, properties={"id": id}
             )
 
@@ -377,7 +365,7 @@ class Processor(AsyncProcessor):
                 ),
             )
 
-            await self.config_response_producer.send(
+            await self.config_response_handle.send(
                 resp, properties={"id": id}
             )
 

{trustgraph_flow-2.7.13 → trustgraph_flow-2.8.0}/trustgraph/cores/service.py

@@ -9,10 +9,9 @@ import base64
 import json
 import logging
 
-from .. base import WorkspaceProcessor, Consumer, Producer, Publisher, Subscriber
+from .. base import WorkspaceProcessor, AsyncLibrarianClient
 from .. base import ConsumerMetrics, ProducerMetrics
 from .. base.cassandra_config import add_cassandra_args, resolve_cassandra_config
-from .. base import LibrarianClient
 
 from .. schema import KnowledgeRequest, KnowledgeResponse, Error
 from .. schema import knowledge_request_queue, knowledge_response_queue
@@ -112,59 +111,42 @@ class Processor(WorkspaceProcessor):
             self.knowledge_response_queue_base, workspace,
         )
 
-        await self.pubsub.ensure_topic(req_queue)
-        await self.pubsub.ensure_topic(resp_queue)
+        await self.async_backend.ensure_topic(req_queue)
+        await self.async_backend.ensure_topic(resp_queue)
 
-        response_producer = Producer(
-            backend=self.pubsub,
+        response_handle = await self.sender_pool.add_producer(
             topic=resp_queue,
             schema=KnowledgeResponse,
-            metrics=ProducerMetrics(
-                processor=self.id, producer="knowledge-response",
-                workspace=workspace,
-            ),
         )
 
-        consumer = Consumer(
-            taskgroup=self.taskgroup,
-            backend=self.pubsub,
-            flow=None,
+        async def handler(message):
+            await self.on_knowledge_request(
+                message, None, None, workspace=workspace,
+            )
+
+        consumer_reg = await self.receiver_pool.add_consumer(
             topic=req_queue,
-            subscriber=self.id,
+            subscription=self.id,
             schema=KnowledgeRequest,
-            handler=partial(
-                self.on_knowledge_request, workspace=workspace,
-            ),
-            metrics=ConsumerMetrics(
-                processor=self.id, consumer="knowledge-request",
-                workspace=workspace,
-            ),
+            handler=handler,
         )
 
-        librarian_client = LibrarianClient(
-            id=self.id,
-            backend=self.pubsub,
-            taskgroup=self.taskgroup,
-            librarian_request_queue=workspace_queue(
+        librarian_client = await AsyncLibrarianClient.create(
+            backend=self.async_backend,
+            request_topic=workspace_queue(
                 librarian_request_queue, workspace,
             ),
-            librarian_response_queue=workspace_queue(
+            response_topic=workspace_queue(
                 librarian_response_queue, workspace,
             ),
-            librarian_subscriber=(
-                f"{self.id}--{workspace}--librarian"
-            ),
+            subscription=f"{self.id}--{workspace}--librarian",
         )
 
-        await response_producer.start()
-        await consumer.start()
-        await librarian_client.start()
-
         self.librarian_clients[workspace] = librarian_client
 
         self.workspace_consumers[workspace] = {
-            "consumer": consumer,
-            "response": response_producer,
+            "consumer": consumer_reg,
+            "response": response_handle,
             "librarian": librarian_client,
         }
 
@@ -176,8 +158,9 @@ class Processor(WorkspaceProcessor):
 
         clients = self.workspace_consumers.pop(workspace, None)
         if clients:
-            for client in clients.values():
-                await client.stop()
+            await clients["consumer"].unregister()
+            await clients["response"].unregister()
+            await clients["librarian"].stop()
             logger.info(f"Unsubscribed from workspace queue: {workspace}")
 
     async def start(self):
@@ -301,4 +284,3 @@ class Processor(WorkspaceProcessor):
 def run():
 
     Processor.launch(default_ident, __doc__)
-

{trustgraph_flow-2.7.13 → trustgraph_flow-2.8.0}/trustgraph/extract/kg/agent/extract.py

@@ -345,13 +345,6 @@ class Processor(FlowProcessor):
     @staticmethod
     def add_args(parser):
 
-        parser.add_argument(
-            '-c', '--concurrency',
-            type=int,
-            default=default_concurrency,
-            help=f'Concurrent processing threads (default: {default_concurrency})'
-        )
-
         parser.add_argument(
             "--template-id",
             type=str,

{trustgraph_flow-2.7.13 → trustgraph_flow-2.8.0}/trustgraph/extract/kg/definitions/extract.py

@@ -239,13 +239,6 @@ class Processor(FlowProcessor):
     @staticmethod
     def add_args(parser):
 
-        parser.add_argument(
-            '-c', '--concurrency',
-            type=int,
-            default=default_concurrency,
-            help=f'Concurrent processing threads (default: {default_concurrency})'
-        )
-
         parser.add_argument(
             '--triples-batch-size',
             type=int,

{trustgraph_flow-2.7.13 → trustgraph_flow-2.8.0}/trustgraph/extract/kg/ontology/extract.py

@@ -965,12 +965,6 @@ class Processor(FlowProcessor):
     @staticmethod
     def add_args(parser):
         """Add command-line arguments."""
-        parser.add_argument(
-            '-c', '--concurrency',
-            type=int,
-            default=default_concurrency,
-            help=f'Concurrent processing threads (default: {default_concurrency})'
-        )
         parser.add_argument(
             '--top-k',
             type=int,

{trustgraph_flow-2.7.13 → trustgraph_flow-2.8.0}/trustgraph/extract/kg/relationships/extract.py

@@ -220,13 +220,6 @@ class Processor(FlowProcessor):
     @staticmethod
     def add_args(parser):
 
-        parser.add_argument(
-            '-c', '--concurrency',
-            type=int,
-            default=default_concurrency,
-            help=f'Concurrent processing threads (default: {default_concurrency})'
-        )
-
         parser.add_argument(
             '--triples-batch-size',
             type=int,

{trustgraph_flow-2.7.13 → trustgraph_flow-2.8.0}/trustgraph/extract/kg/rows/processor.py

@@ -315,13 +315,6 @@ class Processor(FlowProcessor):
     def add_args(parser):
         """Add command-line arguments"""
 
-        parser.add_argument(
-            '-c', '--concurrency',
-            type=int,
-            default=default_concurrency,
-            help=f'Concurrent processing threads (default: {default_concurrency})'
-        )
-
         parser.add_argument(
             '--config-type',
             default='schema',

{trustgraph_flow-2.7.13 → trustgraph_flow-2.8.0}/trustgraph/flow/service/flow.py

@@ -13,10 +13,10 @@ DELETE_RETRY_DELAY = 2  # seconds
 
 
 class FlowConfig:
-    def __init__(self, config, pubsub):
+    def __init__(self, config, async_backend):
 
         self.config = config
-        self.pubsub = pubsub
+        self.async_backend = async_backend
         # Per-workspace cache for parameter type definitions
         # Keyed by (workspace, type-name)
         self.param_type_cache = {}
@@ -241,7 +241,7 @@ class FlowConfig:
         # before processors receive their config and start connecting.
         topics = self._collect_flow_topics(cls, repl_template_with_params)
         for topic in topics:
-            await self.pubsub.create_topic(topic)
+            await self.async_backend.create_topic(topic)
 
         # Build all processor config updates, then write in a single batch.
         updates = []
@@ -364,7 +364,7 @@ class FlowConfig:
 
                     topics = self._collect_flow_topics(cls, repl_template)
                     for topic in topics:
-                        await self.pubsub.ensure_topic(topic)
+                        await self.async_backend.ensure_topic(topic)
 
                     logger.info(
                         f"Ensured topics for existing flow "
@@ -524,7 +524,7 @@ class FlowConfig:
 
             for topic in topics:
                 try:
-                    await self.pubsub.delete_topic(topic)
+                    await self.async_backend.delete_topic(topic)
                 except Exception as e:
                     logger.warning(
                         f"Topic delete failed (attempt {attempt + 1}/"