trustgraph-base 1.0.16__tar.gz → 1.0.17__tar.gz

{trustgraph-base-1.0.16 → trustgraph-base-1.0.17}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 1.0.16
+Version: 1.0.17
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v1.0.16.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v1.0.17.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-base-1.0.17/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "1.0.17"

trustgraph-base-1.0.17/trustgraph/messaging/__init__.py

@@ -0,0 +1,105 @@
+from .registry import TranslatorRegistry
+from .translators import *
+
+# Auto-register all translators
+from .translators.agent import AgentRequestTranslator, AgentResponseTranslator
+from .translators.embeddings import EmbeddingsRequestTranslator, EmbeddingsResponseTranslator
+from .translators.text_completion import TextCompletionRequestTranslator, TextCompletionResponseTranslator
+from .translators.retrieval import (
+    DocumentRagRequestTranslator, DocumentRagResponseTranslator,
+    GraphRagRequestTranslator, GraphRagResponseTranslator
+)
+from .translators.triples import TriplesQueryRequestTranslator, TriplesQueryResponseTranslator
+from .translators.knowledge import KnowledgeRequestTranslator, KnowledgeResponseTranslator
+from .translators.library import LibraryDocumentTranslator, LibraryProcessingTranslator
+from .translators.document_loading import DocumentTranslator, TextDocumentTranslator
+from .translators.config import ConfigRequestTranslator, ConfigResponseTranslator
+from .translators.flow import FlowRequestTranslator, FlowResponseTranslator
+from .translators.prompt import PromptRequestTranslator, PromptResponseTranslator
+from .translators.embeddings_query import (
+    DocumentEmbeddingsRequestTranslator, DocumentEmbeddingsResponseTranslator,
+    GraphEmbeddingsRequestTranslator, GraphEmbeddingsResponseTranslator
+)
+
+# Register all service translators
+TranslatorRegistry.register_service(
+    "agent", 
+    AgentRequestTranslator(), 
+    AgentResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "embeddings", 
+    EmbeddingsRequestTranslator(), 
+    EmbeddingsResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "text-completion", 
+    TextCompletionRequestTranslator(), 
+    TextCompletionResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "document-rag", 
+    DocumentRagRequestTranslator(), 
+    DocumentRagResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "graph-rag", 
+    GraphRagRequestTranslator(), 
+    GraphRagResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "triples-query", 
+    TriplesQueryRequestTranslator(), 
+    TriplesQueryResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "knowledge", 
+    KnowledgeRequestTranslator(), 
+    KnowledgeResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "librarian", 
+    LibraryDocumentTranslator(), 
+    LibraryProcessingTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "config", 
+    ConfigRequestTranslator(), 
+    ConfigResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "flow", 
+    FlowRequestTranslator(), 
+    FlowResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "prompt", 
+    PromptRequestTranslator(), 
+    PromptResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "document-embeddings-query", 
+    DocumentEmbeddingsRequestTranslator(), 
+    DocumentEmbeddingsResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "graph-embeddings-query", 
+    GraphEmbeddingsRequestTranslator(), 
+    GraphEmbeddingsResponseTranslator()
+)
+
+# Register single-direction translators for document loading
+TranslatorRegistry.register_request("document", DocumentTranslator())
+TranslatorRegistry.register_request("text-document", TextDocumentTranslator())

trustgraph-base-1.0.17/trustgraph/messaging/registry.py

@@ -0,0 +1,51 @@
+from typing import Dict, List, Union
+from .translators.base import MessageTranslator
+
+
+class TranslatorRegistry:
+    """Registry for service translators"""
+    
+    _request_translators: Dict[str, MessageTranslator] = {}
+    _response_translators: Dict[str, MessageTranslator] = {}
+    
+    @classmethod
+    def register_request(cls, service_name: str, translator: MessageTranslator):
+        """Register a request translator for a service"""
+        cls._request_translators[service_name] = translator
+    
+    @classmethod
+    def register_response(cls, service_name: str, translator: MessageTranslator):
+        """Register a response translator for a service"""
+        cls._response_translators[service_name] = translator
+    
+    @classmethod
+    def register_service(cls, service_name: str, request_translator: MessageTranslator, 
+                        response_translator: MessageTranslator):
+        """Register both request and response translators for a service"""
+        cls.register_request(service_name, request_translator)
+        cls.register_response(service_name, response_translator)
+    
+    @classmethod
+    def get_request_translator(cls, service_name: str) -> MessageTranslator:
+        """Get request translator for a service"""
+        if service_name not in cls._request_translators:
+            raise KeyError(f"No request translator registered for service: {service_name}")
+        return cls._request_translators[service_name]
+    
+    @classmethod
+    def get_response_translator(cls, service_name: str) -> MessageTranslator:
+        """Get response translator for a service"""
+        if service_name not in cls._response_translators:
+            raise KeyError(f"No response translator registered for service: {service_name}")
+        return cls._response_translators[service_name]
+    
+    @classmethod
+    def list_services(cls) -> List[str]:
+        """List all registered services"""
+        return sorted(set(cls._request_translators.keys()) | set(cls._response_translators.keys()))
+    
+    @classmethod
+    def has_service(cls, service_name: str) -> bool:
+        """Check if a service is registered"""
+        return (service_name in cls._request_translators or 
+                service_name in cls._response_translators)

trustgraph-base-1.0.17/trustgraph/messaging/translators/__init__.py

@@ -0,0 +1,19 @@
+from .base import Translator, MessageTranslator
+from .primitives import ValueTranslator, TripleTranslator, SubgraphTranslator
+from .metadata import DocumentMetadataTranslator, ProcessingMetadataTranslator
+from .agent import AgentRequestTranslator, AgentResponseTranslator
+from .embeddings import EmbeddingsRequestTranslator, EmbeddingsResponseTranslator
+from .text_completion import TextCompletionRequestTranslator, TextCompletionResponseTranslator
+from .retrieval import DocumentRagRequestTranslator, DocumentRagResponseTranslator
+from .retrieval import GraphRagRequestTranslator, GraphRagResponseTranslator
+from .triples import TriplesQueryRequestTranslator, TriplesQueryResponseTranslator
+from .knowledge import KnowledgeRequestTranslator, KnowledgeResponseTranslator
+from .library import LibraryDocumentTranslator, LibraryProcessingTranslator
+from .document_loading import DocumentTranslator, TextDocumentTranslator, ChunkTranslator, DocumentEmbeddingsTranslator
+from .config import ConfigRequestTranslator, ConfigResponseTranslator
+from .flow import FlowRequestTranslator, FlowResponseTranslator
+from .prompt import PromptRequestTranslator, PromptResponseTranslator
+from .embeddings_query import (
+    DocumentEmbeddingsRequestTranslator, DocumentEmbeddingsResponseTranslator,
+    GraphEmbeddingsRequestTranslator, GraphEmbeddingsResponseTranslator
+)

trustgraph-base-1.0.17/trustgraph/messaging/translators/agent.py

@@ -0,0 +1,44 @@
+from typing import Dict, Any, Tuple
+from ...schema import AgentRequest, AgentResponse
+from .base import MessageTranslator
+
+
+class AgentRequestTranslator(MessageTranslator):
+    """Translator for AgentRequest schema objects"""
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> AgentRequest:
+        return AgentRequest(
+            question=data["question"],
+            plan=data.get("plan", ""),
+            state=data.get("state", ""),
+            history=data.get("history", [])
+        )
+    
+    def from_pulsar(self, obj: AgentRequest) -> Dict[str, Any]:
+        return {
+            "question": obj.question,
+            "plan": obj.plan,
+            "state": obj.state,
+            "history": obj.history
+        }
+
+
+class AgentResponseTranslator(MessageTranslator):
+    """Translator for AgentResponse schema objects"""
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> AgentResponse:
+        raise NotImplementedError("Response translation to Pulsar not typically needed")
+    
+    def from_pulsar(self, obj: AgentResponse) -> Dict[str, Any]:
+        result = {}
+        if obj.answer:
+            result["answer"] = obj.answer
+        if obj.thought:
+            result["thought"] = obj.thought  
+        if obj.observation:
+            result["observation"] = obj.observation
+        return result
+    
+    def from_response_with_completion(self, obj: AgentResponse) -> Tuple[Dict[str, Any], bool]:
+        """Returns (response_dict, is_final)"""
+        return self.from_pulsar(obj), (obj.answer is not None)

trustgraph-base-1.0.17/trustgraph/messaging/translators/base.py

@@ -0,0 +1,43 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Any, Tuple
+from pulsar.schema import Record
+
+
+class Translator(ABC):
+    """Base class for bidirectional Pulsar ↔ dict translation"""
+    
+    @abstractmethod
+    def to_pulsar(self, data: Dict[str, Any]) -> Record:
+        """Convert dict to Pulsar schema object"""
+        pass
+    
+    @abstractmethod  
+    def from_pulsar(self, obj: Record) -> Dict[str, Any]:
+        """Convert Pulsar schema object to dict"""
+        pass
+
+
+class MessageTranslator(Translator):
+    """For complete request/response message translation"""
+    
+    def from_response_with_completion(self, obj: Record) -> Tuple[Dict[str, Any], bool]:
+        """Returns (response_dict, is_final) - for streaming responses"""
+        return self.from_pulsar(obj), True
+
+
+class SendTranslator(Translator):
+    """For fire-and-forget send operations (like ServiceSender)"""
+    
+    def from_pulsar(self, obj: Record) -> Dict[str, Any]:
+        """Usually not needed for send-only operations"""
+        raise NotImplementedError("Send translators typically don't need from_pulsar")
+
+
+def handle_optional_fields(obj: Record, fields: list) -> Dict[str, Any]:
+    """Helper to extract optional fields from Pulsar object"""
+    result = {}
+    for field in fields:
+        value = getattr(obj, field, None)
+        if value is not None:
+            result[field] = value
+    return result

trustgraph-base-1.0.17/trustgraph/messaging/translators/config.py

@@ -0,0 +1,100 @@
+from typing import Dict, Any, Tuple
+from ...schema import ConfigRequest, ConfigResponse, ConfigKey, ConfigValue
+from .base import MessageTranslator
+
+
+class ConfigRequestTranslator(MessageTranslator):
+    """Translator for ConfigRequest schema objects"""
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> ConfigRequest:
+        keys = None
+        if "keys" in data:
+            keys = [
+                ConfigKey(
+                    type=k["type"],
+                    key=k["key"]
+                )
+                for k in data["keys"]
+            ]
+        
+        values = None
+        if "values" in data:
+            values = [
+                ConfigValue(
+                    type=v["type"],
+                    key=v["key"],
+                    value=v["value"]
+                )
+                for v in data["values"]
+            ]
+        
+        return ConfigRequest(
+            operation=data.get("operation"),
+            keys=keys,
+            type=data.get("type"),
+            values=values
+        )
+    
+    def from_pulsar(self, obj: ConfigRequest) -> Dict[str, Any]:
+        result = {}
+        
+        if obj.operation:
+            result["operation"] = obj.operation
+        if obj.type:
+            result["type"] = obj.type
+        
+        if obj.keys:
+            result["keys"] = [
+                {
+                    "type": k.type,
+                    "key": k.key
+                }
+                for k in obj.keys
+            ]
+        
+        if obj.values:
+            result["values"] = [
+                {
+                    "type": v.type,
+                    "key": v.key,
+                    "value": v.value
+                }
+                for v in obj.values
+            ]
+        
+        return result
+
+
+class ConfigResponseTranslator(MessageTranslator):
+    """Translator for ConfigResponse schema objects"""
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> ConfigResponse:
+        raise NotImplementedError("Response translation to Pulsar not typically needed")
+    
+    def from_pulsar(self, obj: ConfigResponse) -> Dict[str, Any]:
+        result = {}
+        
+        if obj.version is not None:
+            result["version"] = obj.version
+        
+        if obj.values:
+            result["values"] = [
+                {
+                    "type": v.type,
+                    "key": v.key,
+                    "value": v.value
+                }
+                for v in obj.values
+            ]
+        
+        if obj.directory:
+            result["directory"] = obj.directory
+        
+        if obj.config:
+            result["config"] = obj.config
+        
+        return result
+    
+    def from_response_with_completion(self, obj: ConfigResponse) -> Tuple[Dict[str, Any], bool]:
+        """Returns (response_dict, is_final)"""
+        return self.from_pulsar(obj), True

trustgraph-base-1.0.17/trustgraph/messaging/translators/document_loading.py

@@ -0,0 +1,191 @@
+import base64
+from typing import Dict, Any
+from ...schema import Document, TextDocument, Chunk, DocumentEmbeddings, ChunkEmbeddings
+from .base import SendTranslator
+from .metadata import DocumentMetadataTranslator
+from .primitives import SubgraphTranslator
+
+
+class DocumentTranslator(SendTranslator):
+    """Translator for Document schema objects (PDF docs etc.)"""
+    
+    def __init__(self):
+        self.subgraph_translator = SubgraphTranslator()
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> Document:
+        metadata = data.get("metadata", [])
+        
+        # Handle base64 content validation
+        doc = base64.b64decode(data["data"])
+        
+        from ...schema import Metadata
+        return Document(
+            metadata=Metadata(
+                id=data.get("id"),
+                metadata=self.subgraph_translator.to_pulsar(metadata) if metadata else [],
+                user=data.get("user", "trustgraph"),
+                collection=data.get("collection", "default"),
+            ),
+            data=base64.b64encode(doc).decode("utf-8")
+        )
+    
+    def from_pulsar(self, obj: Document) -> Dict[str, Any]:
+        result = {
+            "data": obj.data
+        }
+        
+        if obj.metadata:
+            metadata_dict = {}
+            if obj.metadata.id:
+                metadata_dict["id"] = obj.metadata.id
+            if obj.metadata.user:
+                metadata_dict["user"] = obj.metadata.user
+            if obj.metadata.collection:
+                metadata_dict["collection"] = obj.metadata.collection
+            if obj.metadata.metadata:
+                metadata_dict["metadata"] = self.subgraph_translator.from_pulsar(obj.metadata.metadata)
+                
+            result["metadata"] = metadata_dict
+            
+        return result
+
+
+class TextDocumentTranslator(SendTranslator):
+    """Translator for TextDocument schema objects"""
+    
+    def __init__(self):
+        self.subgraph_translator = SubgraphTranslator()
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> TextDocument:
+        metadata = data.get("metadata", [])
+        charset = data.get("charset", "utf-8")
+        
+        # Text is base64 encoded in input
+        text = base64.b64decode(data["text"]).decode(charset)
+        
+        from ...schema import Metadata
+        return TextDocument(
+            metadata=Metadata(
+                id=data.get("id"),
+                metadata=self.subgraph_translator.to_pulsar(metadata) if metadata else [],
+                user=data.get("user", "trustgraph"),
+                collection=data.get("collection", "default"),
+            ),
+            text=text.encode("utf-8")
+        )
+    
+    def from_pulsar(self, obj: TextDocument) -> Dict[str, Any]:
+        result = {
+            "text": obj.text.decode("utf-8") if isinstance(obj.text, bytes) else obj.text
+        }
+        
+        if obj.metadata:
+            metadata_dict = {}
+            if obj.metadata.id:
+                metadata_dict["id"] = obj.metadata.id
+            if obj.metadata.user:
+                metadata_dict["user"] = obj.metadata.user
+            if obj.metadata.collection:
+                metadata_dict["collection"] = obj.metadata.collection
+            if obj.metadata.metadata:
+                metadata_dict["metadata"] = self.subgraph_translator.from_pulsar(obj.metadata.metadata)
+                
+            result["metadata"] = metadata_dict
+            
+        return result
+
+
+class ChunkTranslator(SendTranslator):
+    """Translator for Chunk schema objects"""
+    
+    def __init__(self):
+        self.subgraph_translator = SubgraphTranslator()
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> Chunk:
+        metadata = data.get("metadata", [])
+        
+        from ...schema import Metadata
+        return Chunk(
+            metadata=Metadata(
+                id=data.get("id"),
+                metadata=self.subgraph_translator.to_pulsar(metadata) if metadata else [],
+                user=data.get("user", "trustgraph"),
+                collection=data.get("collection", "default"),
+            ),
+            chunk=data["chunk"].encode("utf-8") if isinstance(data["chunk"], str) else data["chunk"]
+        )
+    
+    def from_pulsar(self, obj: Chunk) -> Dict[str, Any]:
+        result = {
+            "chunk": obj.chunk.decode("utf-8") if isinstance(obj.chunk, bytes) else obj.chunk
+        }
+        
+        if obj.metadata:
+            metadata_dict = {}
+            if obj.metadata.id:
+                metadata_dict["id"] = obj.metadata.id
+            if obj.metadata.user:
+                metadata_dict["user"] = obj.metadata.user
+            if obj.metadata.collection:
+                metadata_dict["collection"] = obj.metadata.collection
+            if obj.metadata.metadata:
+                metadata_dict["metadata"] = self.subgraph_translator.from_pulsar(obj.metadata.metadata)
+                
+            result["metadata"] = metadata_dict
+            
+        return result
+
+
+class DocumentEmbeddingsTranslator(SendTranslator):
+    """Translator for DocumentEmbeddings schema objects"""
+    
+    def __init__(self):
+        self.subgraph_translator = SubgraphTranslator()
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> DocumentEmbeddings:
+        metadata = data.get("metadata", {})
+        
+        chunks = [
+            ChunkEmbeddings(
+                chunk=chunk["chunk"].encode("utf-8") if isinstance(chunk["chunk"], str) else chunk["chunk"],
+                vectors=chunk["vectors"]
+            )
+            for chunk in data.get("chunks", [])
+        ]
+        
+        from ...schema import Metadata
+        return DocumentEmbeddings(
+            metadata=Metadata(
+                id=metadata.get("id"),
+                metadata=self.subgraph_translator.to_pulsar(metadata.get("metadata", [])),
+                user=metadata.get("user", "trustgraph"),
+                collection=metadata.get("collection", "default"),
+            ),
+            chunks=chunks
+        )
+    
+    def from_pulsar(self, obj: DocumentEmbeddings) -> Dict[str, Any]:
+        result = {
+            "chunks": [
+                {
+                    "chunk": chunk.chunk.decode("utf-8") if isinstance(chunk.chunk, bytes) else chunk.chunk,
+                    "vectors": chunk.vectors
+                }
+                for chunk in obj.chunks
+            ]
+        }
+        
+        if obj.metadata:
+            metadata_dict = {}
+            if obj.metadata.id:
+                metadata_dict["id"] = obj.metadata.id
+            if obj.metadata.user:
+                metadata_dict["user"] = obj.metadata.user
+            if obj.metadata.collection:
+                metadata_dict["collection"] = obj.metadata.collection
+            if obj.metadata.metadata:
+                metadata_dict["metadata"] = self.subgraph_translator.from_pulsar(obj.metadata.metadata)
+                
+            result["metadata"] = metadata_dict
+            
+        return result

trustgraph-base-1.0.17/trustgraph/messaging/translators/embeddings.py

@@ -0,0 +1,33 @@
+from typing import Dict, Any, Tuple
+from ...schema import EmbeddingsRequest, EmbeddingsResponse
+from .base import MessageTranslator
+
+
+class EmbeddingsRequestTranslator(MessageTranslator):
+    """Translator for EmbeddingsRequest schema objects"""
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> EmbeddingsRequest:
+        return EmbeddingsRequest(
+            text=data["text"]
+        )
+    
+    def from_pulsar(self, obj: EmbeddingsRequest) -> Dict[str, Any]:
+        return {
+            "text": obj.text
+        }
+
+
+class EmbeddingsResponseTranslator(MessageTranslator):
+    """Translator for EmbeddingsResponse schema objects"""
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> EmbeddingsResponse:
+        raise NotImplementedError("Response translation to Pulsar not typically needed")
+    
+    def from_pulsar(self, obj: EmbeddingsResponse) -> Dict[str, Any]:
+        return {
+            "vectors": obj.vectors
+        }
+    
+    def from_response_with_completion(self, obj: EmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
+        """Returns (response_dict, is_final)"""
+        return self.from_pulsar(obj), True