trustgraph-base 0.23.17__tar.gz → 1.7.1__tar.gz

{trustgraph-base-0.23.17 → trustgraph_base-1.7.1}/PKG-INFO

@@ -1,15 +1,15 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: trustgraph-base
-Version: 0.23.17
+Version: 1.7.1
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
-Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.17.tar.gz
-Author: trustgraph.ai
-Author-email: security@trustgraph.ai
+Author-email: "trustgraph.ai" <security@trustgraph.ai>
+Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph
 Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
+Requires-Dist: pulsar-client
+Requires-Dist: prometheus-client
+Requires-Dist: requests
 
 See https://trustgraph.ai/

trustgraph_base-1.7.1/pyproject.toml

@@ -0,0 +1,29 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "trustgraph-base"
+dynamic = ["version"]
+authors = [{name = "trustgraph.ai", email = "security@trustgraph.ai"}]
+description = "TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline."
+readme = "README.md"
+requires-python = ">=3.8"
+dependencies = [
+    "pulsar-client",
+    "prometheus-client",
+    "requests",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+Homepage = "https://github.com/trustgraph-ai/trustgraph"
+
+[tool.setuptools.packages.find]
+include = ["trustgraph*"]
+
+[tool.setuptools.dynamic]
+version = {attr = "trustgraph.base_version.__version__"}

trustgraph_base-1.7.1/trustgraph/api/__init__.py

@@ -0,0 +1,80 @@
+
+# Core API
+from .api import Api
+
+# Flow clients
+from .flow import Flow, FlowInstance
+from .async_flow import AsyncFlow, AsyncFlowInstance
+
+# WebSocket clients
+from .socket_client import SocketClient, SocketFlowInstance
+from .async_socket_client import AsyncSocketClient, AsyncSocketFlowInstance
+
+# Bulk operation clients
+from .bulk_client import BulkClient
+from .async_bulk_client import AsyncBulkClient
+
+# Metrics clients
+from .metrics import Metrics
+from .async_metrics import AsyncMetrics
+
+# Types
+from .types import (
+    Triple,
+    ConfigKey,
+    ConfigValue,
+    DocumentMetadata,
+    ProcessingMetadata,
+    CollectionMetadata,
+    StreamingChunk,
+    AgentThought,
+    AgentObservation,
+    AgentAnswer,
+    RAGChunk,
+)
+
+# Exceptions
+from .exceptions import ProtocolException, ApplicationException
+
+__all__ = [
+    # Core API
+    "Api",
+
+    # Flow clients
+    "Flow",
+    "FlowInstance",
+    "AsyncFlow",
+    "AsyncFlowInstance",
+
+    # WebSocket clients
+    "SocketClient",
+    "SocketFlowInstance",
+    "AsyncSocketClient",
+    "AsyncSocketFlowInstance",
+
+    # Bulk operation clients
+    "BulkClient",
+    "AsyncBulkClient",
+
+    # Metrics clients
+    "Metrics",
+    "AsyncMetrics",
+
+    # Types
+    "Triple",
+    "ConfigKey",
+    "ConfigValue",
+    "DocumentMetadata",
+    "ProcessingMetadata",
+    "CollectionMetadata",
+    "StreamingChunk",
+    "AgentThought",
+    "AgentObservation",
+    "AgentAnswer",
+    "RAGChunk",
+
+    # Exceptions
+    "ProtocolException",
+    "ApplicationException",
+]
+

trustgraph_base-1.7.1/trustgraph/api/api.py

@@ -0,0 +1,182 @@
+
+import requests
+import json
+import base64
+import time
+from typing import Optional
+
+from . library import Library
+from . flow import Flow
+from . config import Config
+from . knowledge import Knowledge
+from . collection import Collection
+from . exceptions import *
+from . types import *
+
+def check_error(response):
+
+    if "error" in response:
+
+        try:
+            msg = response["error"]["message"]
+            tp = response["error"]["type"]
+        except:
+            raise ApplicationException(response["error"])
+
+        raise ApplicationException(f"{tp}: {msg}")
+
+class Api:
+
+    def __init__(self, url="http://localhost:8088/", timeout=60, token: Optional[str] = None):
+
+        self.url = url
+
+        if not url.endswith("/"):
+            self.url += "/"
+
+        self.url += "api/v1/"
+
+        self.timeout = timeout
+        self.token = token
+
+        # Lazy initialization for new clients
+        self._socket_client = None
+        self._bulk_client = None
+        self._async_flow = None
+        self._async_socket_client = None
+        self._async_bulk_client = None
+        self._metrics = None
+        self._async_metrics = None
+
+    def flow(self):
+        return Flow(api=self)
+
+    def config(self):
+        return Config(api=self)
+
+    def knowledge(self):
+        return Knowledge(api=self)
+
+    def request(self, path, request):
+
+        url = f"{self.url}{path}"
+
+        headers = {}
+        if self.token:
+            headers["Authorization"] = f"Bearer {self.token}"
+
+        # Invoke the API, input is passed as JSON
+        resp = requests.post(url, json=request, timeout=self.timeout, headers=headers)
+
+        # Should be a 200 status code
+        if resp.status_code != 200:
+            raise ProtocolException(f"Status code {resp.status_code}")
+
+        try:
+            # Parse the response as JSON
+            object = resp.json()
+        except:
+            raise ProtocolException(f"Expected JSON response")
+
+        check_error(object)
+
+        return object
+
+    def library(self):
+        return Library(self)
+
+    def collection(self):
+        return Collection(self)
+
+    # New synchronous methods
+    def socket(self):
+        """Synchronous WebSocket-based interface for streaming operations"""
+        if self._socket_client is None:
+            from . socket_client import SocketClient
+            # Extract base URL (remove api/v1/ suffix)
+            base_url = self.url.rsplit("api/v1/", 1)[0].rstrip("/")
+            self._socket_client = SocketClient(base_url, self.timeout, self.token)
+        return self._socket_client
+
+    def bulk(self):
+        """Synchronous bulk operations interface for import/export"""
+        if self._bulk_client is None:
+            from . bulk_client import BulkClient
+            # Extract base URL (remove api/v1/ suffix)
+            base_url = self.url.rsplit("api/v1/", 1)[0].rstrip("/")
+            self._bulk_client = BulkClient(base_url, self.timeout, self.token)
+        return self._bulk_client
+
+    def metrics(self):
+        """Synchronous metrics interface"""
+        if self._metrics is None:
+            from . metrics import Metrics
+            # Extract base URL (remove api/v1/ suffix)
+            base_url = self.url.rsplit("api/v1/", 1)[0].rstrip("/")
+            self._metrics = Metrics(base_url, self.timeout, self.token)
+        return self._metrics
+
+    # New asynchronous methods
+    def async_flow(self):
+        """Asynchronous REST-based flow interface"""
+        if self._async_flow is None:
+            from . async_flow import AsyncFlow
+            self._async_flow = AsyncFlow(self.url, self.timeout, self.token)
+        return self._async_flow
+
+    def async_socket(self):
+        """Asynchronous WebSocket-based interface for streaming operations"""
+        if self._async_socket_client is None:
+            from . async_socket_client import AsyncSocketClient
+            # Extract base URL (remove api/v1/ suffix)
+            base_url = self.url.rsplit("api/v1/", 1)[0].rstrip("/")
+            self._async_socket_client = AsyncSocketClient(base_url, self.timeout, self.token)
+        return self._async_socket_client
+
+    def async_bulk(self):
+        """Asynchronous bulk operations interface for import/export"""
+        if self._async_bulk_client is None:
+            from . async_bulk_client import AsyncBulkClient
+            # Extract base URL (remove api/v1/ suffix)
+            base_url = self.url.rsplit("api/v1/", 1)[0].rstrip("/")
+            self._async_bulk_client = AsyncBulkClient(base_url, self.timeout, self.token)
+        return self._async_bulk_client
+
+    def async_metrics(self):
+        """Asynchronous metrics interface"""
+        if self._async_metrics is None:
+            from . async_metrics import AsyncMetrics
+            # Extract base URL (remove api/v1/ suffix)
+            base_url = self.url.rsplit("api/v1/", 1)[0].rstrip("/")
+            self._async_metrics = AsyncMetrics(base_url, self.timeout, self.token)
+        return self._async_metrics
+
+    # Resource management
+    def close(self):
+        """Close all synchronous connections"""
+        if self._socket_client:
+            self._socket_client.close()
+        if self._bulk_client:
+            self._bulk_client.close()
+
+    async def aclose(self):
+        """Close all asynchronous connections"""
+        if self._async_socket_client:
+            await self._async_socket_client.aclose()
+        if self._async_bulk_client:
+            await self._async_bulk_client.aclose()
+        if self._async_flow:
+            await self._async_flow.aclose()
+
+    # Context manager support
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *args):
+        await self.aclose()

trustgraph_base-1.7.1/trustgraph/api/async_bulk_client.py

@@ -0,0 +1,131 @@
+
+import json
+import websockets
+from typing import Optional, AsyncIterator, Dict, Any, Iterator
+
+from . types import Triple
+
+
+class AsyncBulkClient:
+    """Asynchronous bulk operations client"""
+
+    def __init__(self, url: str, timeout: int, token: Optional[str]) -> None:
+        self.url: str = self._convert_to_ws_url(url)
+        self.timeout: int = timeout
+        self.token: Optional[str] = token
+
+    def _convert_to_ws_url(self, url: str) -> str:
+        """Convert HTTP URL to WebSocket URL"""
+        if url.startswith("http://"):
+            return url.replace("http://", "ws://", 1)
+        elif url.startswith("https://"):
+            return url.replace("https://", "wss://", 1)
+        elif url.startswith("ws://") or url.startswith("wss://"):
+            return url
+        else:
+            return f"ws://{url}"
+
+    async def import_triples(self, flow: str, triples: AsyncIterator[Triple], **kwargs: Any) -> None:
+        """Bulk import triples via WebSocket"""
+        ws_url = f"{self.url}/api/v1/flow/{flow}/import/triples"
+        if self.token:
+            ws_url = f"{ws_url}?token={self.token}"
+
+        async with websockets.connect(ws_url, ping_interval=20, ping_timeout=self.timeout) as websocket:
+            async for triple in triples:
+                message = {
+                    "s": triple.s,
+                    "p": triple.p,
+                    "o": triple.o
+                }
+                await websocket.send(json.dumps(message))
+
+    async def export_triples(self, flow: str, **kwargs: Any) -> AsyncIterator[Triple]:
+        """Bulk export triples via WebSocket"""
+        ws_url = f"{self.url}/api/v1/flow/{flow}/export/triples"
+        if self.token:
+            ws_url = f"{ws_url}?token={self.token}"
+
+        async with websockets.connect(ws_url, ping_interval=20, ping_timeout=self.timeout) as websocket:
+            async for raw_message in websocket:
+                data = json.loads(raw_message)
+                yield Triple(
+                    s=data.get("s", ""),
+                    p=data.get("p", ""),
+                    o=data.get("o", "")
+                )
+
+    async def import_graph_embeddings(self, flow: str, embeddings: AsyncIterator[Dict[str, Any]], **kwargs: Any) -> None:
+        """Bulk import graph embeddings via WebSocket"""
+        ws_url = f"{self.url}/api/v1/flow/{flow}/import/graph-embeddings"
+        if self.token:
+            ws_url = f"{ws_url}?token={self.token}"
+
+        async with websockets.connect(ws_url, ping_interval=20, ping_timeout=self.timeout) as websocket:
+            async for embedding in embeddings:
+                await websocket.send(json.dumps(embedding))
+
+    async def export_graph_embeddings(self, flow: str, **kwargs: Any) -> AsyncIterator[Dict[str, Any]]:
+        """Bulk export graph embeddings via WebSocket"""
+        ws_url = f"{self.url}/api/v1/flow/{flow}/export/graph-embeddings"
+        if self.token:
+            ws_url = f"{ws_url}?token={self.token}"
+
+        async with websockets.connect(ws_url, ping_interval=20, ping_timeout=self.timeout) as websocket:
+            async for raw_message in websocket:
+                yield json.loads(raw_message)
+
+    async def import_document_embeddings(self, flow: str, embeddings: AsyncIterator[Dict[str, Any]], **kwargs: Any) -> None:
+        """Bulk import document embeddings via WebSocket"""
+        ws_url = f"{self.url}/api/v1/flow/{flow}/import/document-embeddings"
+        if self.token:
+            ws_url = f"{ws_url}?token={self.token}"
+
+        async with websockets.connect(ws_url, ping_interval=20, ping_timeout=self.timeout) as websocket:
+            async for embedding in embeddings:
+                await websocket.send(json.dumps(embedding))
+
+    async def export_document_embeddings(self, flow: str, **kwargs: Any) -> AsyncIterator[Dict[str, Any]]:
+        """Bulk export document embeddings via WebSocket"""
+        ws_url = f"{self.url}/api/v1/flow/{flow}/export/document-embeddings"
+        if self.token:
+            ws_url = f"{ws_url}?token={self.token}"
+
+        async with websockets.connect(ws_url, ping_interval=20, ping_timeout=self.timeout) as websocket:
+            async for raw_message in websocket:
+                yield json.loads(raw_message)
+
+    async def import_entity_contexts(self, flow: str, contexts: AsyncIterator[Dict[str, Any]], **kwargs: Any) -> None:
+        """Bulk import entity contexts via WebSocket"""
+        ws_url = f"{self.url}/api/v1/flow/{flow}/import/entity-contexts"
+        if self.token:
+            ws_url = f"{ws_url}?token={self.token}"
+
+        async with websockets.connect(ws_url, ping_interval=20, ping_timeout=self.timeout) as websocket:
+            async for context in contexts:
+                await websocket.send(json.dumps(context))
+
+    async def export_entity_contexts(self, flow: str, **kwargs: Any) -> AsyncIterator[Dict[str, Any]]:
+        """Bulk export entity contexts via WebSocket"""
+        ws_url = f"{self.url}/api/v1/flow/{flow}/export/entity-contexts"
+        if self.token:
+            ws_url = f"{ws_url}?token={self.token}"
+
+        async with websockets.connect(ws_url, ping_interval=20, ping_timeout=self.timeout) as websocket:
+            async for raw_message in websocket:
+                yield json.loads(raw_message)
+
+    async def import_objects(self, flow: str, objects: AsyncIterator[Dict[str, Any]], **kwargs: Any) -> None:
+        """Bulk import objects via WebSocket"""
+        ws_url = f"{self.url}/api/v1/flow/{flow}/import/objects"
+        if self.token:
+            ws_url = f"{ws_url}?token={self.token}"
+
+        async with websockets.connect(ws_url, ping_interval=20, ping_timeout=self.timeout) as websocket:
+            async for obj in objects:
+                await websocket.send(json.dumps(obj))
+
+    async def aclose(self) -> None:
+        """Close connections"""
+        # Cleanup handled by context managers
+        pass

trustgraph_base-1.7.1/trustgraph/api/async_flow.py

@@ -0,0 +1,245 @@
+
+import aiohttp
+import json
+from typing import Optional, Dict, Any, List
+
+from . exceptions import ProtocolException, ApplicationException
+
+
+def check_error(response):
+    if "error" in response:
+        try:
+            msg = response["error"]["message"]
+            tp = response["error"]["type"]
+        except:
+            raise ApplicationException(response["error"])
+
+        raise ApplicationException(f"{tp}: {msg}")
+
+
+class AsyncFlow:
+    """Asynchronous REST-based flow interface"""
+
+    def __init__(self, url: str, timeout: int, token: Optional[str]) -> None:
+        self.url: str = url
+        self.timeout: int = timeout
+        self.token: Optional[str] = token
+
+    async def request(self, path: str, request_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Make async HTTP request to Gateway API"""
+        url = f"{self.url}{path}"
+
+        headers = {"Content-Type": "application/json"}
+        if self.token:
+            headers["Authorization"] = f"Bearer {self.token}"
+
+        timeout = aiohttp.ClientTimeout(total=self.timeout)
+
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.post(url, json=request_data, headers=headers) as resp:
+                if resp.status != 200:
+                    raise ProtocolException(f"Status code {resp.status}")
+
+                try:
+                    obj = await resp.json()
+                except:
+                    raise ProtocolException(f"Expected JSON response")
+
+                check_error(obj)
+                return obj
+
+    async def list(self) -> List[str]:
+        """List all flows"""
+        result = await self.request("flow", {"operation": "list-flows"})
+        return result.get("flow-ids", [])
+
+    async def get(self, id: str) -> Dict[str, Any]:
+        """Get flow definition"""
+        result = await self.request("flow", {
+            "operation": "get-flow",
+            "flow-id": id
+        })
+        return json.loads(result.get("flow", "{}"))
+
+    async def start(self, class_name: str, id: str, description: str, parameters: Optional[Dict] = None):
+        """Start a flow"""
+        request_data = {
+            "operation": "start-flow",
+            "flow-id": id,
+            "class-name": class_name,
+            "description": description
+        }
+        if parameters:
+            request_data["parameters"] = json.dumps(parameters)
+
+        await self.request("flow", request_data)
+
+    async def stop(self, id: str):
+        """Stop a flow"""
+        await self.request("flow", {
+            "operation": "stop-flow",
+            "flow-id": id
+        })
+
+    async def list_classes(self) -> List[str]:
+        """List flow classes"""
+        result = await self.request("flow", {"operation": "list-classes"})
+        return result.get("class-names", [])
+
+    async def get_class(self, class_name: str) -> Dict[str, Any]:
+        """Get flow class definition"""
+        result = await self.request("flow", {
+            "operation": "get-class",
+            "class-name": class_name
+        })
+        return json.loads(result.get("class-definition", "{}"))
+
+    async def put_class(self, class_name: str, definition: Dict[str, Any]):
+        """Create/update flow class"""
+        await self.request("flow", {
+            "operation": "put-class",
+            "class-name": class_name,
+            "class-definition": json.dumps(definition)
+        })
+
+    async def delete_class(self, class_name: str):
+        """Delete flow class"""
+        await self.request("flow", {
+            "operation": "delete-class",
+            "class-name": class_name
+        })
+
+    def id(self, flow_id: str):
+        """Get async flow instance"""
+        return AsyncFlowInstance(self, flow_id)
+
+    async def aclose(self) -> None:
+        """Close connection (cleanup handled by aiohttp session)"""
+        pass
+
+
+class AsyncFlowInstance:
+    """Asynchronous REST flow instance"""
+
+    def __init__(self, flow: AsyncFlow, flow_id: str):
+        self.flow = flow
+        self.flow_id = flow_id
+
+    async def request(self, service: str, request_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Make request to flow-scoped service"""
+        return await self.flow.request(f"flow/{self.flow_id}/service/{service}", request_data)
+
+    async def agent(self, question: str, user: str, state: Optional[Dict] = None,
+                    group: Optional[str] = None, history: Optional[List] = None, **kwargs: Any) -> Dict[str, Any]:
+        """Execute agent (non-streaming, use async_socket for streaming)"""
+        request_data = {
+            "question": question,
+            "user": user,
+            "streaming": False  # REST doesn't support streaming
+        }
+        if state is not None:
+            request_data["state"] = state
+        if group is not None:
+            request_data["group"] = group
+        if history is not None:
+            request_data["history"] = history
+        request_data.update(kwargs)
+
+        return await self.request("agent", request_data)
+
+    async def text_completion(self, system: str, prompt: str, **kwargs: Any) -> str:
+        """Text completion (non-streaming, use async_socket for streaming)"""
+        request_data = {
+            "system": system,
+            "prompt": prompt,
+            "streaming": False
+        }
+        request_data.update(kwargs)
+
+        result = await self.request("text-completion", request_data)
+        return result.get("response", "")
+
+    async def graph_rag(self, question: str, user: str, collection: str,
+                        max_subgraph_size: int = 1000, max_subgraph_count: int = 5,
+                        max_entity_distance: int = 3, **kwargs: Any) -> str:
+        """Graph RAG (non-streaming, use async_socket for streaming)"""
+        request_data = {
+            "question": question,
+            "user": user,
+            "collection": collection,
+            "max-subgraph-size": max_subgraph_size,
+            "max-subgraph-count": max_subgraph_count,
+            "max-entity-distance": max_entity_distance,
+            "streaming": False
+        }
+        request_data.update(kwargs)
+
+        result = await self.request("graph-rag", request_data)
+        return result.get("response", "")
+
+    async def document_rag(self, question: str, user: str, collection: str,
+                           doc_limit: int = 10, **kwargs: Any) -> str:
+        """Document RAG (non-streaming, use async_socket for streaming)"""
+        request_data = {
+            "question": question,
+            "user": user,
+            "collection": collection,
+            "doc-limit": doc_limit,
+            "streaming": False
+        }
+        request_data.update(kwargs)
+
+        result = await self.request("document-rag", request_data)
+        return result.get("response", "")
+
+    async def graph_embeddings_query(self, text: str, user: str, collection: str, limit: int = 10, **kwargs: Any):
+        """Query graph embeddings for semantic search"""
+        request_data = {
+            "text": text,
+            "user": user,
+            "collection": collection,
+            "limit": limit
+        }
+        request_data.update(kwargs)
+
+        return await self.request("graph-embeddings", request_data)
+
+    async def embeddings(self, text: str, **kwargs: Any):
+        """Generate text embeddings"""
+        request_data = {"text": text}
+        request_data.update(kwargs)
+
+        return await self.request("embeddings", request_data)
+
+    async def triples_query(self, s=None, p=None, o=None, user=None, collection=None, limit=100, **kwargs: Any):
+        """Triple pattern query"""
+        request_data = {"limit": limit}
+        if s is not None:
+            request_data["s"] = str(s)
+        if p is not None:
+            request_data["p"] = str(p)
+        if o is not None:
+            request_data["o"] = str(o)
+        if user is not None:
+            request_data["user"] = user
+        if collection is not None:
+            request_data["collection"] = collection
+        request_data.update(kwargs)
+
+        return await self.request("triples", request_data)
+
+    async def objects_query(self, query: str, user: str, collection: str, variables: Optional[Dict] = None,
+                            operation_name: Optional[str] = None, **kwargs: Any):
+        """GraphQL query"""
+        request_data = {
+            "query": query,
+            "user": user,
+            "collection": collection
+        }
+        if variables:
+            request_data["variables"] = variables
+        if operation_name:
+            request_data["operationName"] = operation_name
+        request_data.update(kwargs)
+
+        return await self.request("objects", request_data)