trustgraph-base 2.2.14__tar.gz → 2.2.15__tar.gz

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-base
-Version: 2.2.14
+Version: 2.2.15
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/base/__init__.py

@@ -1,5 +1,5 @@
 
-from . pubsub import PulsarClient, get_pubsub
+from . pubsub import get_pubsub, add_pubsub_args
 from . async_processor import AsyncProcessor
 from . consumer import Consumer
 from . producer import Producer

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/base/async_processor.py

@@ -6,7 +6,6 @@
 
 import asyncio
 import argparse
-import _pulsar
 import time
 import uuid
 import logging
@@ -15,7 +14,7 @@ from prometheus_client import start_http_server, Info
 
 from .. schema import ConfigPush, config_push_queue
 from .. log_level import LogLevel
-from . pubsub import PulsarClient, get_pubsub
+from . pubsub import get_pubsub, add_pubsub_args
 from . producer import Producer
 from . consumer import Consumer
 from . metrics import ProcessorMetrics, ConsumerMetrics
@@ -223,8 +222,8 @@ class AsyncProcessor:
                 logger.info("Keyboard interrupt.")
                 return
 
-            except _pulsar.Interrupted:
-                logger.info("Pulsar Interrupted.")
+            except KeyboardInterrupt:
+                logger.info("Interrupted.")
                 return
 
             # Exceptions from a taskgroup come in as an exception group
@@ -250,15 +249,7 @@ class AsyncProcessor:
     @staticmethod
     def add_args(parser):
 
-        # Pub/sub backend selection
-        parser.add_argument(
-            '--pubsub-backend',
-            default=os.getenv('PUBSUB_BACKEND', 'pulsar'),
-            choices=['pulsar', 'mqtt'],
-            help='Pub/sub backend (default: pulsar, env: PUBSUB_BACKEND)',
-        )
-
-        PulsarClient.add_args(parser)
+        add_pubsub_args(parser)
         add_logging_args(parser)
 
         parser.add_argument(

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/base/flow_processor.py

@@ -6,8 +6,6 @@
 import json
 import logging
 
-from pulsar.schema import JsonSchema
-
 from .. schema import Error
 from .. schema import config_request_queue, config_response_queue
 from .. schema import config_push_queue

trustgraph_base-2.2.15/trustgraph/base/pubsub.py

@@ -0,0 +1,72 @@
+
+import os
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Default connection settings from environment
+DEFAULT_PULSAR_HOST = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
+DEFAULT_PULSAR_API_KEY = os.getenv("PULSAR_API_KEY", None)
+
+
+def get_pubsub(**config):
+    """
+    Factory function to create a pub/sub backend based on configuration.
+
+    Args:
+        config: Configuration dictionary from command-line args.
+                Key 'pubsub_backend' selects the backend (default: 'pulsar').
+
+    Returns:
+        Backend instance implementing the PubSubBackend protocol.
+    """
+    backend_type = config.get('pubsub_backend', 'pulsar')
+
+    if backend_type == 'pulsar':
+        from .pulsar_backend import PulsarBackend
+        return PulsarBackend(
+            host=config.get('pulsar_host', DEFAULT_PULSAR_HOST),
+            api_key=config.get('pulsar_api_key', DEFAULT_PULSAR_API_KEY),
+            listener=config.get('pulsar_listener'),
+        )
+    else:
+        raise ValueError(f"Unknown pub/sub backend: {backend_type}")
+
+
+STANDALONE_PULSAR_HOST = 'pulsar://localhost:6650'
+
+
+def add_pubsub_args(parser, standalone=False):
+    """Add pub/sub CLI arguments to an argument parser.
+
+    Args:
+        parser: argparse.ArgumentParser
+        standalone: If True, default host is localhost (for CLI tools
+                    that run outside containers)
+    """
+    host = STANDALONE_PULSAR_HOST if standalone else DEFAULT_PULSAR_HOST
+    listener_default = 'localhost' if standalone else None
+
+    parser.add_argument(
+        '--pubsub-backend',
+        default=os.getenv('PUBSUB_BACKEND', 'pulsar'),
+        help='Pub/sub backend (default: pulsar, env: PUBSUB_BACKEND)',
+    )
+
+    parser.add_argument(
+        '-p', '--pulsar-host',
+        default=host,
+        help=f'Pulsar host (default: {host})',
+    )
+
+    parser.add_argument(
+        '--pulsar-api-key',
+        default=DEFAULT_PULSAR_API_KEY,
+        help='Pulsar API key',
+    )
+
+    parser.add_argument(
+        '--pulsar-listener',
+        default=listener_default,
+        help=f'Pulsar listener (default: {listener_default or "none"})',
+    )

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/base/pulsar_backend.py

@@ -181,8 +181,11 @@ class PulsarBackendConsumer:
         self._schema_cls = schema_cls
 
     def receive(self, timeout_millis: int = 2000) -> Message:
-        """Receive a message."""
-        pulsar_msg = self._consumer.receive(timeout_millis=timeout_millis)
+        """Receive a message. Raises TimeoutError if no message available."""
+        try:
+            pulsar_msg = self._consumer.receive(timeout_millis=timeout_millis)
+        except _pulsar.Timeout:
+            raise TimeoutError("No message received within timeout")
         return PulsarMessage(pulsar_msg, self._schema_cls)
 
     def acknowledge(self, message: Message) -> None:
@@ -237,38 +240,44 @@ class PulsarBackend:
         self.client = pulsar.Client(**client_args)
         logger.info(f"Pulsar client connected to {host}")
 
-    def map_topic(self, generic_topic: str) -> str:
+    def map_topic(self, queue_id: str) -> str:
         """
-        Map generic topic format to Pulsar URI.
+        Map queue identifier to Pulsar URI.
 
-        Format: qos/tenant/namespace/queue
-        Example: q1/tg/flow/my-queue -> persistent://tg/flow/my-queue
+        Format: class:topicspace:topic
+        Example: flow:tg:text-completion-request -> persistent://tg/flow/text-completion-request
 
         Args:
-            generic_topic: Generic topic string or already-formatted Pulsar URI
+            queue_id: Queue identifier string or already-formatted Pulsar URI
 
         Returns:
             Pulsar topic URI
         """
         # If already a Pulsar URI, return as-is
-        if '://' in generic_topic:
-            return generic_topic
+        if '://' in queue_id:
+            return queue_id
 
-        parts = generic_topic.split('/', 3)
-        if len(parts) != 4:
-            raise ValueError(f"Invalid topic format: {generic_topic}, expected qos/tenant/namespace/queue")
+        parts = queue_id.split(':', 2)
+        if len(parts) != 3:
+            raise ValueError(
+                f"Invalid queue format: {queue_id}, "
+                f"expected class:topicspace:topic"
+            )
 
-        qos, tenant, namespace, queue = parts
+        cls, topicspace, topic = parts
 
-        # Map QoS to persistence
-        if qos == 'q0':
-            persistence = 'non-persistent'
-        elif qos in ['q1', 'q2']:
+        # Map class to Pulsar persistence and namespace
+        if cls in ('flow', 'state'):
             persistence = 'persistent'
+        elif cls in ('request', 'response'):
+            persistence = 'non-persistent'
         else:
-            raise ValueError(f"Invalid QoS level: {qos}, expected q0, q1, or q2")
+            raise ValueError(
+                f"Invalid queue class: {cls}, "
+                f"expected flow, request, response, or state"
+            )
 
-        return f"{persistence}://{tenant}/{namespace}/{queue}"
+        return f"{persistence}://{topicspace}/{cls}/{topic}"
 
     def create_producer(self, topic: str, schema: type, **options) -> BackendProducer:
         """

trustgraph_base-2.2.15/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "2.2.15"

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/agent_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 
 from .. schema import AgentRequest, AgentResponse
 from .. schema import agent_request_queue
@@ -7,15 +6,11 @@ from .. schema import agent_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class AgentClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -27,7 +22,6 @@ class AgentClient(BaseClient):
         if output_queue is None: output_queue = agent_response_queue
 
         super(AgentClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/base.py

@@ -1,10 +1,6 @@
 
-import pulsar
-import _pulsar
-import hashlib
 import uuid
 import time
-from pulsar.schema import JsonSchema
 
 from .. exceptions import *
 from ..base.pubsub import get_pubsub
@@ -12,16 +8,11 @@ from ..base.pubsub import get_pubsub
 # Default timeout for a request/response.  In seconds.
 DEFAULT_TIMEOUT=300
 
-# Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class BaseClient:
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -87,7 +78,7 @@ class BaseClient:
 
             try:
                 msg = self.consumer.receive(timeout_millis=2500)
-            except pulsar.exceptions.Timeout:
+            except TimeoutError:
                 continue
 
             mid = msg.properties()["id"]
@@ -139,4 +130,3 @@ class BaseClient:
 
         if hasattr(self, "backend"):
             self.backend.close()
-

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/config_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 import json
 import dataclasses
 
@@ -9,10 +8,6 @@ from .. schema import config_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 @dataclasses.dataclass
 class Definition:
@@ -34,7 +29,7 @@ class Topic:
 class ConfigClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -50,7 +45,6 @@ class ConfigClient(BaseClient):
             output_queue = config_response_queue
 
         super(ConfigClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/document_embeddings_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 
 from .. schema import DocumentEmbeddingsRequest, DocumentEmbeddingsResponse
 from .. schema import document_embeddings_request_queue
@@ -7,15 +6,11 @@ from .. schema import document_embeddings_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class DocumentEmbeddingsClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -30,7 +25,6 @@ class DocumentEmbeddingsClient(BaseClient):
             output_queue = document_embeddings_response_queue
             
         super(DocumentEmbeddingsClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/document_rag_client.py

@@ -1,21 +1,15 @@
 
-import _pulsar
 
 from .. schema import DocumentRagQuery, DocumentRagResponse
 from .. schema import document_rag_request_queue, document_rag_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class DocumentRagClient(BaseClient):
 
     def __init__(
             self,
-            log_level=ERROR,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -30,7 +24,6 @@ class DocumentRagClient(BaseClient):
             output_queue = document_rag_response_queue
   
         super(DocumentRagClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/embeddings_client.py

@@ -1,20 +1,14 @@
 
-from pulsar.schema import JsonSchema
 from .. schema import EmbeddingsRequest, EmbeddingsResponse
 from . base import BaseClient
 
-import _pulsar
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class EmbeddingsClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             input_queue=None,
             output_queue=None,
             subscriber=None,
@@ -23,7 +17,6 @@ class EmbeddingsClient(BaseClient):
     ):
 
         super(EmbeddingsClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/graph_embeddings_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 
 from .. schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse
 from .. schema import graph_embeddings_request_queue
@@ -7,15 +6,11 @@ from .. schema import graph_embeddings_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class GraphEmbeddingsClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -30,7 +25,6 @@ class GraphEmbeddingsClient(BaseClient):
             output_queue = graph_embeddings_response_queue
             
         super(GraphEmbeddingsClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/graph_rag_client.py

@@ -1,21 +1,15 @@
 
-import _pulsar
 
 from .. schema import GraphRagQuery, GraphRagResponse
 from .. schema import graph_rag_request_queue, graph_rag_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class GraphRagClient(BaseClient):
 
     def __init__(
             self,
-            log_level=ERROR,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -30,7 +24,6 @@ class GraphRagClient(BaseClient):
             output_queue = graph_rag_response_queue
   
         super(GraphRagClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/llm_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 
 from .. schema import TextCompletionRequest, TextCompletionResponse
 from .. schema import text_completion_request_queue
@@ -8,15 +7,11 @@ from . base import BaseClient
 from .. exceptions import LlmError
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class LlmClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -28,7 +23,6 @@ class LlmClient(BaseClient):
         if output_queue is None: output_queue = text_completion_response_queue
 
         super(LlmClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/prompt_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 import json
 import dataclasses
 
@@ -9,10 +8,6 @@ from .. schema import prompt_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 @dataclasses.dataclass
 class Definition:
@@ -34,7 +29,7 @@ class Topic:
 class PromptClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -49,7 +44,6 @@ class PromptClient(BaseClient):
             output_queue = prompt_response_queue
 
         super(PromptClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/row_embeddings_client.py

@@ -1,5 +1,4 @@
 
-import _pulsar
 
 from .. schema import RowEmbeddingsRequest, RowEmbeddingsResponse
 from .. schema import row_embeddings_request_queue
@@ -7,15 +6,11 @@ from .. schema import row_embeddings_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class RowEmbeddingsClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -30,7 +25,6 @@ class RowEmbeddingsClient(BaseClient):
             output_queue = row_embeddings_response_queue
 
         super(RowEmbeddingsClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/clients/triples_query_client.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import _pulsar
 
 from .. schema import TriplesQueryRequest, TriplesQueryResponse, Term, IRI, LITERAL
 from .. schema import triples_request_queue
@@ -8,15 +7,11 @@ from .. schema import triples_response_queue
 from . base import BaseClient
 
 # Ugly
-ERROR=_pulsar.LoggerLevel.Error
-WARN=_pulsar.LoggerLevel.Warn
-INFO=_pulsar.LoggerLevel.Info
-DEBUG=_pulsar.LoggerLevel.Debug
 
 class TriplesQueryClient(BaseClient):
 
     def __init__(
-            self, log_level=ERROR,
+            self,
             subscriber=None,
             input_queue=None,
             output_queue=None,
@@ -31,7 +26,6 @@ class TriplesQueryClient(BaseClient):
             output_queue = triples_response_queue
 
         super(TriplesQueryClient, self).__init__(
-            log_level=log_level,
             subscriber=subscriber,
             input_queue=input_queue,
             output_queue=output_queue,

trustgraph_base-2.2.15/trustgraph/log_level.py

@@ -0,0 +1,12 @@
+
+from enum import Enum
+
+
+class LogLevel(Enum):
+    DEBUG = 'debug'
+    INFO = 'info'
+    WARN = 'warn'
+    ERROR = 'error'
+
+    def __str__(self):
+        return self.value

{trustgraph_base-2.2.14 → trustgraph_base-2.2.15}/trustgraph/messaging/translators/agent.py

@@ -6,7 +6,7 @@ from .base import MessageTranslator
 class AgentRequestTranslator(MessageTranslator):
     """Translator for AgentRequest schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> AgentRequest:
+    def decode(self, data: Dict[str, Any]) -> AgentRequest:
         return AgentRequest(
             question=data["question"],
             state=data.get("state", None),
@@ -26,7 +26,7 @@ class AgentRequestTranslator(MessageTranslator):
             expected_siblings=data.get("expected_siblings", 0),
         )
 
-    def from_pulsar(self, obj: AgentRequest) -> Dict[str, Any]:
+    def encode(self, obj: AgentRequest) -> Dict[str, Any]:
         return {
             "question": obj.question,
             "state": obj.state,
@@ -50,10 +50,10 @@ class AgentRequestTranslator(MessageTranslator):
 class AgentResponseTranslator(MessageTranslator):
     """Translator for AgentResponse schema objects"""
     
-    def to_pulsar(self, data: Dict[str, Any]) -> AgentResponse:
+    def decode(self, data: Dict[str, Any]) -> AgentResponse:
         raise NotImplementedError("Response translation to Pulsar not typically needed")
     
-    def from_pulsar(self, obj: AgentResponse) -> Dict[str, Any]:
+    def encode(self, obj: AgentResponse) -> Dict[str, Any]:
         result = {}
 
         if obj.chunk_type:
@@ -81,7 +81,7 @@ class AgentResponseTranslator(MessageTranslator):
 
         return result
 
-    def from_response_with_completion(self, obj: AgentResponse) -> Tuple[Dict[str, Any], bool]:
+    def encode_with_completion(self, obj: AgentResponse) -> Tuple[Dict[str, Any], bool]:
         """Returns (response_dict, is_final)"""
         is_final = getattr(obj, 'end_of_dialog', False)
-        return self.from_pulsar(obj), is_final
+        return self.encode(obj), is_final