trustgraph-base 2.3.12__tar.gz → 2.3.14__tar.gz

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-base
-Version: 2.3.12
+Version: 2.3.14
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/__init__.py

@@ -5,7 +5,7 @@ from . consumer import Consumer
 from . producer import Producer
 from . publisher import Publisher
 from . subscriber import Subscriber
-from . metrics import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
+from . metrics import ProcessorMetrics, ConsumerMetrics, ProducerMetrics, SubscriberMetrics
 from . logging import add_logging_args, setup_logging
 from . flow_processor import FlowProcessor
 from . consumer_spec import ConsumerSpec
@@ -22,6 +22,7 @@ from . text_completion_client import (
     TextCompletionClientSpec, TextCompletionClient, TextCompletionResult,
 )
 from . prompt_client import PromptClientSpec, PromptClient, PromptResult
+from . config_client import ConfigClientSpec, ConfigClient
 from . triples_store_service import TriplesStoreService
 from . graph_embeddings_store_service import GraphEmbeddingsStoreService
 from . document_embeddings_store_service import DocumentEmbeddingsStoreService

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/agent_service.py

@@ -1,8 +1,11 @@
-
 """
 Agent manager service completion base class
 """
 
+from __future__ import annotations
+
+from argparse import ArgumentParser
+
 import time
 import logging
 from prometheus_client import Histogram
@@ -97,7 +100,7 @@ class AgentService(FlowProcessor):
             )
 
     @staticmethod
-    def add_args(parser):
+    def add_args(parser: ArgumentParser) -> None:
 
         FlowProcessor.add_args(parser)
 

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/async_processor.py

@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from argparse import ArgumentParser
+from typing import Any, Callable
 
 # Base class for processors.  Implements:
 # - Pub/sub client, subscribe and consume basic
@@ -178,20 +182,20 @@ class AsyncProcessor:
 
     # This is called to stop all threads.  An over-ride point for extra
     # functionality
-    def stop(self):
+    def stop(self) -> None:
         self.pubsub_backend.close()
         self.running = False
 
     # Returns the pub/sub backend (new interface)
     @property
-    def pubsub(self): return self.pubsub_backend
+    def pubsub(self) -> Any: return self.pubsub_backend
 
     # Returns the pulsar host (backward compatibility)
     @property
-    def pulsar_host(self): return self._pulsar_host
+    def pulsar_host(self) -> str: return self._pulsar_host
 
     # Register a new event handler for configuration change
-    def register_config_handler(self, handler, types=None):
+    def register_config_handler(self, handler: Callable[..., Any], types: list[type] | None = None) -> None:
         self.config_handlers.append({
             "handler": handler,
             "types": set(types) if types else None,
@@ -295,13 +299,13 @@ class AsyncProcessor:
             raise e
 
     @classmethod
-    def setup_logging(cls, args):
+    def setup_logging(cls, args: dict[str, Any]) -> None:
         """Configure logging for the entire application"""
         setup_logging(args)
 
     # Startup fabric.  launch calls launch_async in async mode.
     @classmethod
-    def launch(cls, ident, doc):
+    def launch(cls, ident: str, doc: str) -> None:
 
         # Start assembling CLI arguments
         parser = argparse.ArgumentParser(
@@ -374,7 +378,7 @@ class AsyncProcessor:
     # The command-line arguments are built using a stack of add_args
     # invocations
     @staticmethod
-    def add_args(parser):
+    def add_args(parser: ArgumentParser) -> None:
 
         add_pubsub_args(parser)
         add_logging_args(parser)

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/backend.py

@@ -121,7 +121,7 @@ class PubSubBackend(Protocol):
         Create a producer for a topic.
 
         Args:
-            topic: Generic topic format (qos/tenant/namespace/queue)
+            topic: Queue identifier in class:topicspace:topic format
             schema: Dataclass type for messages
             **options: Backend-specific options (e.g., chunking_enabled)
 
@@ -159,6 +159,58 @@ class PubSubBackend(Protocol):
         """
         ...
 
+    async def create_topic(self, topic: str) -> None:
+        """
+        Create the broker-side resources for a logical topic.
+
+        For RabbitMQ this creates a fanout exchange.  For Pulsar this is
+        a no-op (topics auto-create on first use).
+
+        Idempotent — creating an already-existing topic succeeds silently.
+
+        Args:
+            topic: Topic identifier in class:topicspace:topic format
+        """
+        ...
+
+    async def delete_topic(self, topic: str) -> None:
+        """
+        Delete a topic and discard any in-flight messages.
+
+        For RabbitMQ this deletes the fanout exchange; consumer queues
+        lose their binding and drain naturally.
+
+        Idempotent — deleting a non-existent topic succeeds silently.
+
+        Args:
+            topic: Topic identifier in class:topicspace:topic format
+        """
+        ...
+
+    async def topic_exists(self, topic: str) -> bool:
+        """
+        Check whether a topic exists.
+
+        Args:
+            topic: Topic identifier in class:topicspace:topic format
+
+        Returns:
+            True if the topic exists, False otherwise.
+        """
+        ...
+
+    async def ensure_topic(self, topic: str) -> None:
+        """
+        Ensure a topic exists, creating it if necessary.
+
+        Convenience wrapper — checks existence, creates if missing.
+        Used by the flow service and system services on startup.
+
+        Args:
+            topic: Topic identifier in class:topicspace:topic format
+        """
+        ...
+
     def close(self) -> None:
         """Close the backend connection."""
         ...

trustgraph_base-2.3.14/trustgraph/base/config_client.py

@@ -0,0 +1,92 @@
+
+from . request_response_spec import RequestResponse, RequestResponseSpec
+from .. schema import ConfigRequest, ConfigResponse, ConfigKey, ConfigValue
+
+CONFIG_TIMEOUT = 10
+
+
+class ConfigClient(RequestResponse):
+
+    async def _request(self, timeout=CONFIG_TIMEOUT, **kwargs):
+        resp = await self.request(
+            ConfigRequest(**kwargs),
+            timeout=timeout,
+        )
+        if resp.error:
+            raise RuntimeError(
+                f"{resp.error.type}: {resp.error.message}"
+            )
+        return resp
+
+    async def get(self, type, key, timeout=CONFIG_TIMEOUT):
+        """Get a single config value. Returns the value string or None."""
+        resp = await self._request(
+            operation="get",
+            keys=[ConfigKey(type=type, key=key)],
+            timeout=timeout,
+        )
+        if resp.values and len(resp.values) > 0:
+            return resp.values[0].value
+        return None
+
+    async def put(self, type, key, value, timeout=CONFIG_TIMEOUT):
+        """Put a single config value."""
+        await self._request(
+            operation="put",
+            values=[ConfigValue(type=type, key=key, value=value)],
+            timeout=timeout,
+        )
+
+    async def put_many(self, values, timeout=CONFIG_TIMEOUT):
+        """Put multiple config values in a single request.
+        values is a list of (type, key, value) tuples."""
+        await self._request(
+            operation="put",
+            values=[
+                ConfigValue(type=t, key=k, value=v)
+                for t, k, v in values
+            ],
+            timeout=timeout,
+        )
+
+    async def delete(self, type, key, timeout=CONFIG_TIMEOUT):
+        """Delete a single config key."""
+        await self._request(
+            operation="delete",
+            keys=[ConfigKey(type=type, key=key)],
+            timeout=timeout,
+        )
+
+    async def delete_many(self, keys, timeout=CONFIG_TIMEOUT):
+        """Delete multiple config keys in a single request.
+        keys is a list of (type, key) tuples."""
+        await self._request(
+            operation="delete",
+            keys=[
+                ConfigKey(type=t, key=k)
+                for t, k in keys
+            ],
+            timeout=timeout,
+        )
+
+    async def keys(self, type, timeout=CONFIG_TIMEOUT):
+        """List all keys for a config type."""
+        resp = await self._request(
+            operation="list",
+            type=type,
+            timeout=timeout,
+        )
+        return resp.directory
+
+
+class ConfigClientSpec(RequestResponseSpec):
+    def __init__(
+            self, request_name, response_name,
+    ):
+        super(ConfigClientSpec, self).__init__(
+            request_name=request_name,
+            request_schema=ConfigRequest,
+            response_name=response_name,
+            response_schema=ConfigResponse,
+            impl=ConfigClient,
+        )

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/consumer_spec.py

@@ -1,16 +1,19 @@
+from __future__ import annotations
+
+from typing import Any
 
 from . metrics import ConsumerMetrics
 from . consumer import Consumer
 from . spec import Spec
 
 class ConsumerSpec(Spec):
-    def __init__(self, name, schema, handler, concurrency = 1):
+    def __init__(self, name: str, schema: Any, handler: Any, concurrency: int = 1) -> None:
         self.name = name
         self.schema = schema
         self.handler = handler
         self.concurrency = concurrency
 
-    def add(self, flow, processor, definition):
+    def add(self, flow: Any, processor: Any, definition: dict[str, Any]) -> None:
 
         consumer_metrics = ConsumerMetrics(
             processor = flow.id, flow = flow.name, name = self.name,
@@ -20,7 +23,7 @@ class ConsumerSpec(Spec):
             taskgroup = processor.taskgroup,
             flow = flow,
             backend = processor.pubsub,
-            topic = definition[self.name],
+            topic = definition["topics"][self.name],
             subscriber = processor.id + "--" + flow.name + "--" + self.name,
             schema = self.schema,
             handler = self.handler,

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/document_embeddings_query_service.py

@@ -1,9 +1,12 @@
-
 """
 Document embeddings query service.  Input is vectors.  Output is list of
 embeddings.
 """
 
+from __future__ import annotations
+
+from argparse import ArgumentParser
+
 import logging
 
 from .. schema import DocumentEmbeddingsRequest, DocumentEmbeddingsResponse
@@ -82,7 +85,7 @@ class DocumentEmbeddingsQueryService(FlowProcessor):
             await flow("response").send(r, properties={"id": id})
 
     @staticmethod
-    def add_args(parser):
+    def add_args(parser: ArgumentParser) -> None:
 
         FlowProcessor.add_args(parser)
 
@@ -93,7 +96,7 @@ class DocumentEmbeddingsQueryService(FlowProcessor):
             help=f'Number of concurrent requests (default: {default_concurrency})'
         )
 
-def run():
+def run() -> None:
 
     Processor.launch(default_ident, __doc__)
 

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/document_embeddings_store_service.py

@@ -1,8 +1,11 @@
-
 """
 Document embeddings store base class
 """
 
+from __future__ import annotations
+
+from argparse import ArgumentParser
+
 import logging
 
 from .. schema import DocumentEmbeddings
@@ -49,7 +52,7 @@ class DocumentEmbeddingsStoreService(FlowProcessor):
             raise e
 
     @staticmethod
-    def add_args(parser):
+    def add_args(parser: ArgumentParser) -> None:
 
         FlowProcessor.add_args(parser)
 

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/dynamic_tool_service.py

@@ -1,4 +1,3 @@
-
 """
 Base class for dynamically pluggable tool services.
 
@@ -11,6 +10,10 @@ Uses direct Pulsar topics (no flow configuration required):
 - Response: non-persistent://tg/response/{topic}
 """
 
+from __future__ import annotations
+
+from argparse import ArgumentParser
+
 import json
 import logging
 import asyncio
@@ -173,7 +176,7 @@ class DynamicToolService(AsyncProcessor):
         raise NotImplementedError("Subclasses must implement invoke()")
 
     @staticmethod
-    def add_args(parser):
+    def add_args(parser: ArgumentParser) -> None:
 
         AsyncProcessor.add_args(parser)
 

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/embeddings_service.py

@@ -1,8 +1,11 @@
-
 """
 Embeddings resolution base class
 """
 
+from __future__ import annotations
+
+from argparse import ArgumentParser
+
 import time
 import logging
 from prometheus_client import Histogram
@@ -100,7 +103,7 @@ class EmbeddingsService(FlowProcessor):
             )
 
     @staticmethod
-    def add_args(parser):
+    def add_args(parser: ArgumentParser) -> None:
 
         parser.add_argument(
             '-c', '--concurrency',
@@ -112,4 +115,3 @@ class EmbeddingsService(FlowProcessor):
         FlowProcessor.add_args(parser)
 
 
-

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/flow.py

@@ -2,6 +2,13 @@
 import asyncio
 
 class Flow:
+    """
+    Runtime representation of a deployed flow process.
+    
+    This class maintains internal processor states and orchestrates
+    lifecycles (start, stop) for inputs (consumers) and parameters 
+    that drive data flowing across linked nodes.
+    """
     def __init__(self, id, flow, processor, defn):
 
         self.id = id

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/flow_processor.py

@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from typing import Any
+from argparse import ArgumentParser
 
 # Base class for processor with management of flows in & out which are managed
 # by configuration.  This is probably all processor types, except for the
@@ -25,9 +29,9 @@ class FlowProcessor(AsyncProcessor):
         # Initialise base class
         super(FlowProcessor, self).__init__(**params)
 
-        # Register configuration handler
+        # Register configuration handler for this processor's config type
         self.register_config_handler(
-            self.on_configure_flows, types=["active-flow"]
+            self.on_configure_flows, types=[f"processor:{self.id}"]
         )
 
         # Initialise flow information state
@@ -41,7 +45,7 @@ class FlowProcessor(AsyncProcessor):
         logger.info("Service initialised.")
 
     # Register a configuration variable
-    def register_specification(self, spec):
+    def register_specification(self, spec: Any) -> None:
         self.specifications.append(spec)
 
     # Start processing for a new flow
@@ -62,17 +66,16 @@ class FlowProcessor(AsyncProcessor):
 
         logger.info(f"Got config version {version}")
 
-        # Skip over invalid data
-        if "active-flow" not in config: return
-
-        # Check there's configuration information for me
-        if self.id in config["active-flow"]:
-
-            # Get my flow config
-            flow_config = json.loads(config["active-flow"][self.id])
+        config_type = f"processor:{self.id}"
 
+        # Get my flow config — each key is a variant, each value is
+        # the JSON config for that flow variant
+        if config_type in config:
+            flow_config = {
+                k: json.loads(v)
+                for k, v in config[config_type].items()
+            }
         else:
-
             logger.debug("No configuration settings for me.")
             flow_config = {}
 
@@ -99,7 +102,7 @@ class FlowProcessor(AsyncProcessor):
         await super(FlowProcessor, self).start()
 
     @staticmethod
-    def add_args(parser):
+    def add_args(parser: ArgumentParser) -> None:
 
         AsyncProcessor.add_args(parser)
 

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/graph_embeddings_client.py

@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+from typing import Any
 
 import logging
 
@@ -9,7 +12,7 @@ from .. knowledge import Uri, Literal
 logger = logging.getLogger(__name__)
 
 
-def to_value(x):
+def to_value(x: Any) -> Any:
     """Convert schema Term to Uri or Literal."""
     if x.type == IRI:
         return Uri(x.iri)

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/graph_embeddings_query_service.py

@@ -1,9 +1,12 @@
-
 """
 Graph embeddings query service.  Input is vectors.  Output is list of
 embeddings.
 """
 
+from __future__ import annotations
+
+from argparse import ArgumentParser
+
 import logging
 
 from .. schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse
@@ -82,7 +85,7 @@ class GraphEmbeddingsQueryService(FlowProcessor):
             await flow("response").send(r, properties={"id": id})
 
     @staticmethod
-    def add_args(parser):
+    def add_args(parser: ArgumentParser) -> None:
 
         FlowProcessor.add_args(parser)
 
@@ -93,7 +96,7 @@ class GraphEmbeddingsQueryService(FlowProcessor):
             help=f'Number of concurrent requests (default: {default_concurrency})'
         )
 
-def run():
+def run() -> None:
 
     Processor.launch(default_ident, __doc__)
 

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/graph_embeddings_store_service.py

@@ -1,8 +1,11 @@
-
 """
 Graph embeddings store base class
 """
 
+from __future__ import annotations
+
+from argparse import ArgumentParser
+
 import logging
 
 from .. schema import GraphEmbeddings
@@ -49,7 +52,7 @@ class GraphEmbeddingsStoreService(FlowProcessor):
             raise e
 
     @staticmethod
-    def add_args(parser):
+    def add_args(parser: ArgumentParser) -> None:
 
         FlowProcessor.add_args(parser)
 

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/llm_service.py

@@ -1,8 +1,11 @@
-
 """
 LLM text completion base class
 """
 
+from __future__ import annotations
+
+from argparse import ArgumentParser
+
 import time
 import logging
 from prometheus_client import Histogram, Info
@@ -42,6 +45,12 @@ class LlmChunk:
     __slots__ = ["text", "in_token", "out_token", "model", "is_final"]
 
 class LlmService(FlowProcessor):
+    """
+    Extensible service processing requests to Large Language Models (LLMs).
+    
+    This class handles the core logic of dispatching text completion or chat requests
+    to integrated underlying LLM providers (e.g. OpenAI, vertex ai).
+    """
 
     def __init__(self, **params):
 
@@ -199,7 +208,7 @@ class LlmService(FlowProcessor):
                 properties={"id": id}
             )
 
-    def supports_streaming(self):
+    def supports_streaming(self) -> bool:
         """
         Override in subclass to indicate streaming support.
         Returns False by default.
@@ -215,7 +224,7 @@ class LlmService(FlowProcessor):
         raise NotImplementedError("Streaming not implemented for this provider")
 
     @staticmethod
-    def add_args(parser):
+    def add_args(parser: ArgumentParser) -> None:
 
         parser.add_argument(
             '-c', '--concurrency',

{trustgraph_base-2.3.12 → trustgraph_base-2.3.14}/trustgraph/base/logging.py

@@ -11,7 +11,9 @@ Supports dual output to console and Loki for centralized log aggregation.
 import contextvars
 import logging
 import logging.handlers
+from argparse import ArgumentParser
 from queue import Queue
+from typing import Any
 import os
 
 
@@ -53,7 +55,7 @@ class _ProcessorIdFilter(logging.Filter):
         return True
 
 
-def add_logging_args(parser):
+def add_logging_args(parser: ArgumentParser) -> None:
     """
     Add standard logging arguments to an argument parser.
 
@@ -100,7 +102,7 @@ def add_logging_args(parser):
     )
 
 
-def setup_logging(args):
+def setup_logging(args: dict[str, Any]) -> None:
     """
     Configure logging from parsed command-line arguments.