trustgraph-flow 1.2.0__tar.gz → 1.2.4__tar.gz

trustgraph_flow-1.2.4/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: trustgraph-flow
+Version: 1.2.4
+Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
+Author-email: "trustgraph.ai" <security@trustgraph.ai>
+Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: trustgraph-base<1.3,>=1.2
+Requires-Dist: aiohttp
+Requires-Dist: anthropic
+Requires-Dist: cassandra-driver
+Requires-Dist: cohere
+Requires-Dist: cryptography
+Requires-Dist: falkordb
+Requires-Dist: fastembed
+Requires-Dist: google-genai
+Requires-Dist: ibis
+Requires-Dist: jsonschema
+Requires-Dist: langchain
+Requires-Dist: langchain-community
+Requires-Dist: langchain-core
+Requires-Dist: langchain-text-splitters
+Requires-Dist: mcp
+Requires-Dist: minio
+Requires-Dist: mistralai
+Requires-Dist: neo4j
+Requires-Dist: ollama
+Requires-Dist: openai
+Requires-Dist: pinecone[grpc]
+Requires-Dist: prometheus-client
+Requires-Dist: pulsar-client
+Requires-Dist: pymilvus
+Requires-Dist: pypdf
+Requires-Dist: mistralai
+Requires-Dist: pyyaml
+Requires-Dist: qdrant-client
+Requires-Dist: rdflib
+Requires-Dist: requests
+Requires-Dist: tabulate
+Requires-Dist: tiktoken
+Requires-Dist: urllib3
+
+See https://trustgraph.ai/

trustgraph_flow-1.2.4/pyproject.toml

@@ -0,0 +1,123 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "trustgraph-flow"
+dynamic = ["version"]
+authors = [{name = "trustgraph.ai", email = "security@trustgraph.ai"}]
+description = "TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline."
+readme = "README.md"
+requires-python = ">=3.8"
+dependencies = [
+    "trustgraph-base>=1.2,<1.3",
+    "aiohttp",
+    "anthropic",
+    "cassandra-driver",
+    "cohere",
+    "cryptography",
+    "falkordb",
+    "fastembed",
+    "google-genai",
+    "ibis",
+    "jsonschema",
+    "langchain",
+    "langchain-community",
+    "langchain-core",
+    "langchain-text-splitters",
+    "mcp",
+    "minio",
+    "mistralai",
+    "neo4j",
+    "ollama",
+    "openai",
+    "pinecone[grpc]",
+    "prometheus-client",
+    "pulsar-client",
+    "pymilvus",
+    "pypdf",
+    "mistralai",
+    "pyyaml",
+    "qdrant-client",
+    "rdflib",
+    "requests",
+    "tabulate",
+    "tiktoken",
+    "urllib3",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+Homepage = "https://github.com/trustgraph-ai/trustgraph"
+
+[project.scripts]
+agent-manager-react = "trustgraph.agent.react:run"
+api-gateway = "trustgraph.gateway:run"
+chunker-recursive = "trustgraph.chunking.recursive:run"
+chunker-token = "trustgraph.chunking.token:run"
+config-svc = "trustgraph.config.service:run"
+de-query-milvus = "trustgraph.query.doc_embeddings.milvus:run"
+de-query-pinecone = "trustgraph.query.doc_embeddings.pinecone:run"
+de-query-qdrant = "trustgraph.query.doc_embeddings.qdrant:run"
+de-write-milvus = "trustgraph.storage.doc_embeddings.milvus:run"
+de-write-pinecone = "trustgraph.storage.doc_embeddings.pinecone:run"
+de-write-qdrant = "trustgraph.storage.doc_embeddings.qdrant:run"
+document-embeddings = "trustgraph.embeddings.document_embeddings:run"
+document-rag = "trustgraph.retrieval.document_rag:run"
+embeddings-fastembed = "trustgraph.embeddings.fastembed:run"
+embeddings-ollama = "trustgraph.embeddings.ollama:run"
+ge-query-milvus = "trustgraph.query.graph_embeddings.milvus:run"
+ge-query-pinecone = "trustgraph.query.graph_embeddings.pinecone:run"
+ge-query-qdrant = "trustgraph.query.graph_embeddings.qdrant:run"
+ge-write-milvus = "trustgraph.storage.graph_embeddings.milvus:run"
+ge-write-pinecone = "trustgraph.storage.graph_embeddings.pinecone:run"
+ge-write-qdrant = "trustgraph.storage.graph_embeddings.qdrant:run"
+graph-embeddings = "trustgraph.embeddings.graph_embeddings:run"
+graph-rag = "trustgraph.retrieval.graph_rag:run"
+kg-extract-agent = "trustgraph.extract.kg.agent:run"
+kg-extract-definitions = "trustgraph.extract.kg.definitions:run"
+kg-extract-relationships = "trustgraph.extract.kg.relationships:run"
+kg-extract-topics = "trustgraph.extract.kg.topics:run"
+kg-manager = "trustgraph.cores:run"
+kg-store = "trustgraph.storage.knowledge:run"
+librarian = "trustgraph.librarian:run"
+mcp-tool = "trustgraph.agent.mcp_tool:run"
+metering = "trustgraph.metering:run"
+object-extract-row = "trustgraph.extract.object.row:run"
+oe-write-milvus = "trustgraph.storage.object_embeddings.milvus:run"
+pdf-decoder = "trustgraph.decoding.pdf:run"
+pdf-ocr-mistral = "trustgraph.decoding.mistral_ocr:run"
+prompt-template = "trustgraph.prompt.template:run"
+rev-gateway = "trustgraph.rev_gateway:run"
+rows-write-cassandra = "trustgraph.storage.rows.cassandra:run"
+run-processing = "trustgraph.processing:run"
+text-completion-azure = "trustgraph.model.text_completion.azure:run"
+text-completion-azure-openai = "trustgraph.model.text_completion.azure_openai:run"
+text-completion-claude = "trustgraph.model.text_completion.claude:run"
+text-completion-cohere = "trustgraph.model.text_completion.cohere:run"
+text-completion-googleaistudio = "trustgraph.model.text_completion.googleaistudio:run"
+text-completion-llamafile = "trustgraph.model.text_completion.llamafile:run"
+text-completion-lmstudio = "trustgraph.model.text_completion.lmstudio:run"
+text-completion-mistral = "trustgraph.model.text_completion.mistral:run"
+text-completion-ollama = "trustgraph.model.text_completion.ollama:run"
+text-completion-openai = "trustgraph.model.text_completion.openai:run"
+text-completion-tgi = "trustgraph.model.text_completion.tgi:run"
+text-completion-vllm = "trustgraph.model.text_completion.vllm:run"
+triples-query-cassandra = "trustgraph.query.triples.cassandra:run"
+triples-query-falkordb = "trustgraph.query.triples.falkordb:run"
+triples-query-memgraph = "trustgraph.query.triples.memgraph:run"
+triples-query-neo4j = "trustgraph.query.triples.neo4j:run"
+triples-write-cassandra = "trustgraph.storage.triples.cassandra:run"
+triples-write-falkordb = "trustgraph.storage.triples.falkordb:run"
+triples-write-memgraph = "trustgraph.storage.triples.memgraph:run"
+triples-write-neo4j = "trustgraph.storage.triples.neo4j:run"
+wikipedia-lookup = "trustgraph.external.wikipedia:run"
+
+[tool.setuptools.packages.find]
+include = ["trustgraph*"]
+
+[tool.setuptools.dynamic]
+version = {attr = "trustgraph.flow_version.__version__"}

{trustgraph-flow-1.2.0 → trustgraph_flow-1.2.4}/trustgraph/agent/mcp_tool/service.py

@@ -5,11 +5,15 @@ name + parameters, output is the response, either a string or an object.
 """
 
 import json
+import logging
 from mcp.client.streamable_http import streamablehttp_client
 from mcp import ClientSession
 
 from ... base import ToolService
 
+# Module logger
+logger = logging.getLogger(__name__)
+
 default_ident = "mcp-tool"
 
 class Service(ToolService):
@@ -26,7 +30,7 @@ class Service(ToolService):
 
     async def on_mcp_config(self, config, version):
 
-        print("Got config version", version)
+        logger.info(f"Got config version {version}")
 
         if "mcp" not in config: return
 
@@ -52,7 +56,7 @@ class Service(ToolService):
             else:
                 remote_name = name
 
-            print("Invoking", remote_name, "at", url, flush=True)
+            logger.info(f"Invoking {remote_name} at {url}")
 
             # Connect to a streamable HTTP server
             async with streamablehttp_client(url) as (
@@ -86,13 +90,13 @@ class Service(ToolService):
         except BaseExceptionGroup as e:
 
             for child in e.exceptions:
-                print(child)
+                logger.debug(f"Child: {child}")
 
             raise e.exceptions[0]
 
         except Exception as e:
 
-            print(e)
+            logger.error(f"Error invoking MCP tool: {e}", exc_info=True)
             raise e
             
     @staticmethod

{trustgraph-flow-1.2.0 → trustgraph_flow-1.2.4}/trustgraph/agent/react/agent_manager.py

@@ -164,18 +164,18 @@ class AgentManager:
 
     async def reason(self, question, history, context):
 
-        print(f"calling reason: {question}", flush=True)
+        logger.debug(f"calling reason: {question}")
 
         tools = self.tools
 
-        print(f"in reason", flush=True)
-        print(tools, flush=True)
+        logger.debug("in reason")
+        logger.debug(f"tools: {tools}")
 
         tool_names = ",".join([
             t for t in self.tools.keys()
         ])
 
-        print("Tool names:", tool_names, flush=True)
+        logger.debug(f"Tool names: {tool_names}")
 
         variables = {
             "question": question,
@@ -208,14 +208,14 @@ class AgentManager:
             ]
         }
 
-        print(json.dumps(variables, indent=4), flush=True)
+        logger.debug(f"Variables: {json.dumps(variables, indent=4)}")
 
         logger.info(f"prompt: {variables}")
 
         # Get text response from prompt service
         response_text = await context("prompt-request").agent_react(variables)
 
-        print(f"Response text:\n{response_text}", flush=True)
+        logger.debug(f"Response text:\n{response_text}")
 
         logger.info(f"response: {response_text}")
 
@@ -233,7 +233,6 @@ class AgentManager:
     async def react(self, question, history, think, observe, context):
 
         logger.info(f"question: {question}")
-        print(f"question: {question}", flush=True)
 
         act = await self.reason(
             question = question,
@@ -256,7 +255,7 @@ class AgentManager:
             else:
                 raise RuntimeError(f"No action for {act.name}!")
 
-            print("TOOL>>>", act, flush=True)
+            logger.debug(f"TOOL>>> {act}")
 
             resp = await action.implementation(context).invoke(
                 **act.arguments

{trustgraph-flow-1.2.0 → trustgraph_flow-1.2.4}/trustgraph/agent/react/service.py

@@ -8,7 +8,7 @@ import sys
 import functools
 import logging
 
-logging.basicConfig(level=logging.DEBUG)
+# Module logger
 logger = logging.getLogger(__name__)
 
 from ... base import AgentService, TextCompletionClientSpec, PromptClientSpec
@@ -81,7 +81,7 @@ class Processor(AgentService):
 
     async def on_tools_config(self, config, version):
 
-        print("Loading configuration version", version)
+        logger.info(f"Loading configuration version {version}")
 
         try:
 
@@ -151,13 +151,13 @@ class Processor(AgentService):
                 additional_context=additional
             )
 
-            print(f"Loaded {len(tools)} tools", flush=True)
-            print("Tool configuration reloaded.", flush=True)
+            logger.info(f"Loaded {len(tools)} tools")
+            logger.info("Tool configuration reloaded.")
 
         except Exception as e:
 
-            print("on_tools_config Exception:", e, flush=True)
-            print("Configuration reload failed", flush=True)
+            logger.error(f"on_tools_config Exception: {e}", exc_info=True)
+            logger.error("Configuration reload failed")
 
     async def agent_request(self, request, respond, next, flow):
 
@@ -176,16 +176,16 @@ class Processor(AgentService):
             else:
                 history = []
 
-            print(f"Question: {request.question}", flush=True)
+            logger.info(f"Question: {request.question}")
 
             if len(history) >= self.max_iterations:
                 raise RuntimeError("Too many agent iterations")
 
-            print(f"History: {history}", flush=True)
+            logger.debug(f"History: {history}")
 
             async def think(x):
 
-                print(f"Think: {x}", flush=True)
+                logger.debug(f"Think: {x}")
 
                 r = AgentResponse(
                     answer=None,
@@ -198,7 +198,7 @@ class Processor(AgentService):
 
             async def observe(x):
 
-                print(f"Observe: {x}", flush=True)
+                logger.debug(f"Observe: {x}")
 
                 r = AgentResponse(
                     answer=None,
@@ -209,7 +209,7 @@ class Processor(AgentService):
 
                 await respond(r)
 
-            print("Call React", flush=True)
+            logger.debug("Call React")
 
             act = await self.agent.react(
                 question = request.question,
@@ -219,11 +219,11 @@ class Processor(AgentService):
                 context = flow,
             )
 
-            print(f"Action: {act}", flush=True)
+            logger.debug(f"Action: {act}")
 
             if isinstance(act, Final):
 
-                print("Send final response...", flush=True)
+                logger.debug("Send final response...")
 
                 if isinstance(act.final, str):
                     f = act.final
@@ -238,11 +238,11 @@ class Processor(AgentService):
 
                 await respond(r)
 
-                print("Done.", flush=True)
+                logger.debug("Done.")
 
                 return
 
-            print("Send next...", flush=True)
+            logger.debug("Send next...")
 
             history.append(act)
 
@@ -263,15 +263,15 @@ class Processor(AgentService):
 
             await next(r)
 
-            print("Done.", flush=True)
+            logger.debug("React agent processing complete")
 
             return
 
         except Exception as e:
 
-            print(f"agent_request Exception: {e}")
+            logger.error(f"agent_request Exception: {e}", exc_info=True)
 
-            print("Send error response...", flush=True)
+            logger.debug("Send error response...")
 
             r = AgentResponse(
                 error=Error(

{trustgraph-flow-1.2.0 → trustgraph_flow-1.2.4}/trustgraph/agent/react/tools.py

@@ -1,7 +1,11 @@
 
 import json
+import logging
 from .types import Argument
 
+# Module logger
+logger = logging.getLogger(__name__)
+
 # This tool implementation knows how to put a question to the graph RAG
 # service
 class KnowledgeQueryImpl:
@@ -21,7 +25,7 @@ class KnowledgeQueryImpl:
     
     async def invoke(self, **arguments):
         client = self.context("graph-rag-request")
-        print("Graph RAG question...", flush=True)
+        logger.debug("Graph RAG question...")
         return await client.rag(
             arguments.get("question")
         )
@@ -44,7 +48,7 @@ class TextCompletionImpl:
     
     async def invoke(self, **arguments):
         client = self.context("prompt-request")
-        print("Prompt question...", flush=True)
+        logger.debug("Prompt question...")
         return await client.question(
             arguments.get("question")
         )
@@ -67,13 +71,13 @@ class McpToolImpl:
 
         client = self.context("mcp-tool-request")
 
-        print(f"MCP tool invocation: {self.mcp_tool_id}...", flush=True)
+        logger.debug(f"MCP tool invocation: {self.mcp_tool_id}...")
         output = await client.invoke(
             name = self.mcp_tool_id,
             parameters = arguments,  # Pass the actual arguments
         )
 
-        print(output)
+        logger.debug(f"MCP tool output: {output}")
 
         if isinstance(output, str):
             return output
@@ -94,7 +98,7 @@ class PromptImpl:
     
     async def invoke(self, **arguments):
         client = self.context("prompt-request")
-        print(f"Prompt template invocation: {self.template_id}...", flush=True)
+        logger.debug(f"Prompt template invocation: {self.template_id}...")
         return await client.prompt(
             id=self.template_id,
             variables=arguments

{trustgraph-flow-1.2.0 → trustgraph_flow-1.2.4}/trustgraph/chunking/recursive/chunker.py

@@ -4,12 +4,16 @@ Simple decoder, accepts text documents on input, outputs chunks from the
 as text as separate output objects.
 """
 
+import logging
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from prometheus_client import Histogram
 
 from ... schema import TextDocument, Chunk
 from ... base import FlowProcessor, ConsumerSpec, ProducerSpec
 
+# Module logger
+logger = logging.getLogger(__name__)
+
 default_ident = "chunker"
 
 class Processor(FlowProcessor):
@@ -54,12 +58,12 @@ class Processor(FlowProcessor):
             )
         )
 
-        print("Chunker initialised", flush=True)
+        logger.info("Recursive chunker initialized")
 
     async def on_message(self, msg, consumer, flow):
 
         v = msg.value()
-        print(f"Chunking {v.metadata.id}...", flush=True)
+        logger.info(f"Chunking document {v.metadata.id}...")
 
         texts = self.text_splitter.create_documents(
             [v.text.decode("utf-8")]
@@ -67,7 +71,7 @@ class Processor(FlowProcessor):
 
         for ix, chunk in enumerate(texts):
 
-            print("Chunk", len(chunk.page_content), flush=True)
+            logger.debug(f"Created chunk of size {len(chunk.page_content)}")
 
             r = Chunk(
                 metadata=v.metadata,
@@ -80,7 +84,7 @@ class Processor(FlowProcessor):
 
             await flow("output").send(r)
 
-        print("Done.", flush=True)
+        logger.debug("Document chunking complete")
 
     @staticmethod
     def add_args(parser):

{trustgraph-flow-1.2.0 → trustgraph_flow-1.2.4}/trustgraph/chunking/token/chunker.py

@@ -4,12 +4,16 @@ Simple decoder, accepts text documents on input, outputs chunks from the
 as text as separate output objects.
 """
 
+import logging
 from langchain_text_splitters import TokenTextSplitter
 from prometheus_client import Histogram
 
 from ... schema import TextDocument, Chunk
 from ... base import FlowProcessor
 
+# Module logger
+logger = logging.getLogger(__name__)
+
 default_ident = "chunker"
 
 class Processor(FlowProcessor):
@@ -53,12 +57,12 @@ class Processor(FlowProcessor):
             )
         )
 
-        print("Chunker initialised", flush=True)
+        logger.info("Token chunker initialized")
 
     async def on_message(self, msg, consumer, flow):
 
         v = msg.value()
-        print(f"Chunking {v.metadata.id}...", flush=True)
+        logger.info(f"Chunking document {v.metadata.id}...")
 
         texts = self.text_splitter.create_documents(
             [v.text.decode("utf-8")]
@@ -66,7 +70,7 @@ class Processor(FlowProcessor):
 
         for ix, chunk in enumerate(texts):
 
-            print("Chunk", len(chunk.page_content), flush=True)
+            logger.debug(f"Created chunk of size {len(chunk.page_content)}")
 
             r = Chunk(
                 metadata=v.metadata,
@@ -79,7 +83,7 @@ class Processor(FlowProcessor):
 
             await flow("output").send(r)
 
-        print("Done.", flush=True)
+        logger.debug("Document chunking complete")
 
     @staticmethod
     def add_args(parser):

{trustgraph-flow-1.2.0 → trustgraph_flow-1.2.4}/trustgraph/config/service/config.py

@@ -1,9 +1,14 @@
 
+import logging
+
 from trustgraph.schema import ConfigResponse
 from trustgraph.schema import ConfigValue, Error
 
 from ... tables.config import ConfigTableStore
 
+# Module logger
+logger = logging.getLogger(__name__)
+
 class ConfigurationClass:
     
     async def keys(self):
@@ -228,7 +233,7 @@ class Configuration:
 
     async def handle(self, msg):
 
-        print("Handle message ", msg.operation)
+        logger.debug(f"Handling config message: {msg.operation}")
 
         if msg.operation == "get":
 

{trustgraph-flow-1.2.0 → trustgraph_flow-1.2.4}/trustgraph/config/service/flow.py

@@ -1,6 +1,10 @@
 
 from trustgraph.schema import FlowResponse, Error
 import json
+import logging
+
+# Module logger
+logger = logging.getLogger(__name__)
 
 class FlowConfig:
     def __init__(self, config):
@@ -41,7 +45,7 @@ class FlowConfig:
     
     async def handle_delete_class(self, msg):
 
-        print(msg)
+        logger.debug(f"Flow config message: {msg}")
 
         await self.config.get("flow-classes").delete(msg.class_name)
 
@@ -218,7 +222,7 @@ class FlowConfig:
     
     async def handle(self, msg):
 
-        print("Handle message ", msg.operation)
+        logger.debug(f"Handling flow message: {msg.operation}")
 
         if msg.operation == "list-classes":
             resp = await self.handle_list_classes(msg)

{trustgraph-flow-1.2.0 → trustgraph_flow-1.2.4}/trustgraph/config/service/service.py

@@ -3,6 +3,8 @@
 Config service.  Manages system global configuration state
 """
 
+import logging
+
 from trustgraph.schema import Error
 
 from trustgraph.schema import ConfigRequest, ConfigResponse, ConfigPush
@@ -20,6 +22,9 @@ from . flow import FlowConfig
 from ... base import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
 from ... base import Consumer, Producer
 
+# Module logger
+logger = logging.getLogger(__name__)
+
 # FIXME: How to ensure this doesn't conflict with other usage?
 keyspace = "config"
 
@@ -146,7 +151,7 @@ class Processor(AsyncProcessor):
 
         self.flow = FlowConfig(self.config)
 
-        print("Service initialised.")
+        logger.info("Config service initialized")
 
     async def start(self):
 
@@ -172,7 +177,7 @@ class Processor(AsyncProcessor):
 
         # Race condition, should make sure version & config sync
 
-        print("Pushed version ", await self.config.get_version())
+        logger.info(f"Pushed configuration version {await self.config.get_version()}")
         
     async def on_config_request(self, msg, consumer, flow):
 
@@ -183,7 +188,7 @@ class Processor(AsyncProcessor):
             # Sender-produced ID
             id = msg.properties()["id"]
 
-            print(f"Handling {id}...", flush=True)
+            logger.info(f"Handling config request {id}...")
 
             resp = await self.config.handle(v)
 
@@ -214,7 +219,7 @@ class Processor(AsyncProcessor):
             # Sender-produced ID
             id = msg.properties()["id"]
 
-            print(f"Handling {id}...", flush=True)
+            logger.info(f"Handling flow request {id}...")
 
             resp = await self.flow.handle(v)