trustgraph-flow 0.23.13__tar.gz → 0.23.14__tar.gz

{trustgraph-flow-0.23.13 → trustgraph-flow-0.23.14}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.23.13
+Version: 0.23.14
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.13.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.14.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-flow-0.23.14/scripts/kg-manager

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.cores import run
+
+run()
+

trustgraph-flow-0.23.14/scripts/kg-store

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.knowledge import run
+
+run()
+

{trustgraph-flow-0.23.13 → trustgraph-flow-0.23.14}/setup.py

@@ -95,6 +95,8 @@ setuptools.setup(
         "scripts/kg-extract-definitions",
         "scripts/kg-extract-relationships",
         "scripts/kg-extract-topics",
+        "scripts/kg-store",
+        "scripts/kg-manager",
         "scripts/librarian",
         "scripts/metering",
         "scripts/object-extract-row",

trustgraph-flow-0.23.14/trustgraph/cores/__init__.py

@@ -0,0 +1,3 @@
+
+from . service import run
+

trustgraph-flow-0.23.14/trustgraph/cores/__main__.py

@@ -0,0 +1,5 @@
+
+from . service import run
+
+if __name__ == '__main__':
+    run()

trustgraph-flow-0.23.14/trustgraph/cores/knowledge.py

@@ -0,0 +1,104 @@
+
+from .. schema import KnowledgeResponse, Error
+from .. knowledge import hash
+from .. exceptions import RequestError
+from .. tables.knowledge import KnowledgeTableStore
+import base64
+
+import uuid
+
+class KnowledgeManager:
+
+    def __init__(
+            self, cassandra_host, cassandra_user, cassandra_password,
+            keyspace,
+    ):
+
+        self.table_store = KnowledgeTableStore(
+            cassandra_host, cassandra_user, cassandra_password, keyspace
+        )
+
+    async def delete_kg_core(self, request, respond):
+
+        print("Deleting core...", flush=True)
+
+        await self.table_store.delete_kg_core(
+            request.user, request.id
+        )
+
+        await respond(
+            KnowledgeResponse(
+                error = None,
+                ids = None,
+                eos = False,
+                triples = None,
+                graph_embeddings = None,
+            )
+        )
+
+    async def fetch_kg_core(self, request, respond):
+
+        print("Fetch core...", flush=True)
+
+        async def publish_triples(t):
+            await respond(
+                KnowledgeResponse(
+                    error = None,
+                    ids = None,
+                    eos = False,
+                    triples = t,
+                    graph_embeddings = None,
+                )
+            )
+
+        # Remove doc table row
+        await self.table_store.get_triples(
+            request.user,
+            request.id,
+            publish_triples,
+        )
+
+        async def publish_ge(g):
+            await respond(
+                KnowledgeResponse(
+                    error = None,
+                    ids = None,
+                    eos = False,
+                    triples = None,
+                    graph_embeddings = g,
+                )
+            )
+
+        # Remove doc table row
+        await self.table_store.get_graph_embeddings(
+            request.user,
+            request.id,
+            publish_ge,
+        )
+
+        print("Fetch complete", flush=True)
+
+        await respond(
+            KnowledgeResponse(
+                error = None,
+                ids = None,
+                eos = True,
+                triples = None,
+                graph_embeddings = None,
+            )
+        )
+
+    async def list_kg_cores(self, request, respond):
+
+        ids = await self.table_store.list_kg_cores(request.user)
+
+        await respond(
+            KnowledgeResponse(
+                error = None,
+                ids = ids,
+                eos = False,
+                triples = None,
+                graph_embeddings = None
+            )
+        )
+

trustgraph-flow-0.23.14/trustgraph/cores/service.py

@@ -0,0 +1,228 @@
+
+"""
+Knowledge core service, manages cores and exports them
+"""
+
+from functools import partial
+import asyncio
+import base64
+import json
+
+from .. base import AsyncProcessor, Consumer, Producer, Publisher, Subscriber
+from .. base import ConsumerMetrics, ProducerMetrics
+
+from .. schema import KnowledgeRequest, KnowledgeResponse, Error
+from .. schema import knowledge_request_queue, knowledge_response_queue
+
+from .. schema import Document, Metadata
+from .. schema import TextDocument, Metadata
+
+from .. exceptions import RequestError
+
+from . knowledge import KnowledgeManager
+
+default_ident = "knowledge"
+
+default_knowledge_request_queue = knowledge_request_queue
+default_knowledge_response_queue = knowledge_response_queue
+
+default_cassandra_host = "cassandra"
+
+# FIXME: How to ensure this doesn't conflict with other usage?
+keyspace = "knowledge"
+
+class Processor(AsyncProcessor):
+
+    def __init__(self, **params):
+
+        id = params.get("id")
+
+        knowledge_request_queue = params.get(
+            "knowledge_request_queue", default_knowledge_request_queue
+        )
+
+        knowledge_response_queue = params.get(
+            "knowledge_response_queue", default_knowledge_response_queue
+        )
+
+        cassandra_host = params.get("cassandra_host", default_cassandra_host)
+        cassandra_user = params.get("cassandra_user")
+        cassandra_password = params.get("cassandra_password")
+
+        super(Processor, self).__init__(
+            **params | {
+                "knowledge_request_queue": knowledge_request_queue,
+                "knowledge_response_queue": knowledge_response_queue,
+                "cassandra_host": cassandra_host,
+                "cassandra_user": cassandra_user,
+            }
+        )
+
+        knowledge_request_metrics = ConsumerMetrics(
+            processor = self.id, flow = None, name = "knowledge-request"
+        )
+
+        knowledge_response_metrics = ProducerMetrics(
+            processor = self.id, flow = None, name = "knowledge-response"
+        )
+
+        self.knowledge_request_consumer = Consumer(
+            taskgroup = self.taskgroup,
+            client = self.pulsar_client,
+            flow = None,
+            topic = knowledge_request_queue,
+            subscriber = id,
+            schema = KnowledgeRequest,
+            handler = self.on_knowledge_request,
+            metrics = knowledge_request_metrics,
+        )
+
+        self.knowledge_response_producer = Producer(
+            client = self.pulsar_client,
+            topic = knowledge_response_queue,
+            schema = KnowledgeResponse,
+            metrics = knowledge_response_metrics,
+        )
+
+        self.knowledge = KnowledgeManager(
+            cassandra_host = cassandra_host.split(","),
+            cassandra_user = cassandra_user,
+            cassandra_password = cassandra_password,
+            keyspace = keyspace,
+        )
+
+        self.register_config_handler(self.on_knowledge_config)
+
+        self.flows = {}
+
+        print("Initialised.", flush=True)
+
+    async def start(self):
+
+        await super(Processor, self).start()
+        await self.knowledge_request_consumer.start()
+        await self.knowledge_response_producer.start()
+
+    async def on_knowledge_config(self, config, version):
+
+        print("config version", version)
+
+        if "flows" in config:
+
+            self.flows = {
+                k: json.loads(v)
+                for k, v in config["flows"].items()
+            }
+
+        print(self.flows)
+
+    async def process_request(self, v, id):
+
+        if v.operation is None:
+            raise RequestError("Null operation")
+
+        print("request", v.operation)
+
+        impls = {
+            "list-kg-cores": self.knowledge.list_kg_cores,
+            "fetch-kg-core": self.knowledge.fetch_kg_core,
+            "delete-kg-core": self.knowledge.delete_kg_core,
+        }
+
+        if v.operation not in impls:
+            raise RequestError(f"Invalid operation: {v.operation}")
+
+        async def respond(x):
+            await self.knowledge_response_producer.send(
+                x, { "id": id }
+            )
+        return await impls[v.operation](v, respond)
+
+    async def on_knowledge_request(self, msg, consumer, flow):
+
+        v = msg.value()
+
+        # Sender-produced ID
+
+        id = msg.properties()["id"]
+
+        print(f"Handling input {id}...", flush=True)
+
+        try:
+
+            await self.process_request(v, id)
+
+#            await self.knowledge_response_producer.send(
+#                resp, properties={"id": id}
+#            )
+
+            return
+
+        except RequestError as e:
+            resp = KnowledgeResponse(
+                error = Error(
+                    type = "request-error",
+                    message = str(e),
+                )
+            )
+
+            await self.knowledge_response_producer.send(
+                resp, properties={"id": id}
+            )
+
+            return
+        except Exception as e:
+            resp = KnowledgeResponse(
+                error = Error(
+                    type = "unexpected-error",
+                    message = str(e),
+                )
+            )
+
+            await self.knowledge_response_producer.send(
+                resp, properties={"id": id}
+            )
+
+            return
+
+        print("Done.", flush=True)
+
+    @staticmethod
+    def add_args(parser):
+
+        AsyncProcessor.add_args(parser)
+
+        parser.add_argument(
+            '--knowledge-request-queue',
+            default=default_knowledge_request_queue,
+            help=f'Config request queue (default: {default_knowledge_request_queue})'
+        )
+
+        parser.add_argument(
+            '--knowledge-response-queue',
+            default=default_knowledge_response_queue,
+            help=f'Config response queue {default_knowledge_response_queue}',
+        )
+
+        parser.add_argument(
+            '--cassandra-host',
+            default="cassandra",
+            help=f'Graph host (default: cassandra)'
+        )
+
+        parser.add_argument(
+            '--cassandra-user',
+            default=None,
+            help=f'Cassandra user'
+        )
+
+        parser.add_argument(
+            '--cassandra-password',
+            default=None,
+            help=f'Cassandra password'
+        )
+
+def run():
+
+    Processor.launch(default_ident, __doc__)
+

trustgraph-flow-0.23.14/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "0.23.14"

trustgraph-flow-0.23.14/trustgraph/gateway/dispatch/knowledge.py

@@ -0,0 +1,68 @@
+
+import base64
+
+from ... schema import KnowledgeRequest, KnowledgeResponse
+from ... schema import knowledge_request_queue
+from ... schema import knowledge_response_queue
+
+from . requestor import ServiceRequestor
+from . serialize import serialize_graph_embeddings
+from . serialize import serialize_triples
+from . serialize import to_document_metadata, to_processing_metadata
+
+class KnowledgeRequestor(ServiceRequestor):
+    def __init__(self, pulsar_client, consumer, subscriber, timeout=120):
+
+        super(KnowledgeRequestor, self).__init__(
+            pulsar_client=pulsar_client,
+            consumer_name = consumer,
+            subscription = subscriber,
+            request_queue=knowledge_request_queue,
+            response_queue=knowledge_response_queue,
+            request_schema=KnowledgeRequest,
+            response_schema=KnowledgeResponse,
+            timeout=timeout,
+        )
+
+    def to_request(self, body):
+
+        return KnowledgeRequest(
+            operation = body.get("operation", None),
+            user = body.get("user", None),
+            id = body.get("id", None),
+        )
+
+    def from_response(self, message):
+
+        print("Processing message")
+
+        # Response to list, 
+        if message.ids is not None:
+            print("-> IDS")
+            return {
+                "ids": message.ids
+            }, True
+
+        if message.triples:
+            print("-> triples")
+            return {
+                "triples": serialize_triples(message.triples)
+            }, False
+
+        if message.graph_embeddings:
+            print("-> ge")
+            return {
+                "graph-embeddings": serialize_graph_embeddings(
+                    message.graph_embeddings
+                )
+            }, False
+
+        if message.eos is True:
+            print("-> eos")
+            return {
+                "eos": True
+            }, True
+
+        # Empty case, return from successful delete.
+        return {}, True
+

{trustgraph-flow-0.23.13 → trustgraph-flow-0.23.14}/trustgraph/gateway/dispatch/manager.py

@@ -5,6 +5,7 @@ import uuid
 from . config import ConfigRequestor
 from . flow import FlowRequestor
 from . librarian import LibrarianRequestor
+from . knowledge import KnowledgeRequestor
 
 from . embeddings import EmbeddingsRequestor
 from . agent import AgentRequestor
@@ -44,6 +45,7 @@ global_dispatchers = {
     "config": ConfigRequestor,
     "flow": FlowRequestor,
     "librarian": LibrarianRequestor,
+    "knowledge": KnowledgeRequestor,
 }
 
 sender_dispatchers = {

{trustgraph-flow-0.23.13 → trustgraph-flow-0.23.14}/trustgraph/gateway/dispatch/requestor.py

@@ -82,7 +82,7 @@ class ServiceRequestor:
 
                 resp, fin = self.from_response(resp)
 
-                print(resp, fin)
+                print(resp, fin, flush=True)
 
                 if responder:
                     await responder(resp, fin)

{trustgraph-flow-0.23.13 → trustgraph-flow-0.23.14}/trustgraph/librarian/librarian.py

@@ -2,7 +2,7 @@
 from .. schema import LibrarianRequest, LibrarianResponse, Error, Triple
 from .. knowledge import hash
 from .. exceptions import RequestError
-from . table_store import TableStore
+from .. tables.library import LibraryTableStore
 from . blob_store import BlobStore
 import base64
 
@@ -21,7 +21,7 @@ class Librarian:
             minio_host, minio_access_key, minio_secret_key, bucket_name
         )
 
-        self.table_store = TableStore(
+        self.table_store = LibraryTableStore(
             cassandra_host, cassandra_user, cassandra_password, keyspace
         )
 

{trustgraph-flow-0.23.13 → trustgraph-flow-0.23.14}/trustgraph/librarian/service.py

@@ -210,7 +210,7 @@ class Processor(AsyncProcessor):
         if v.operation is None:
             raise RequestError("Null operation")
 
-        print("requets", v.operation)
+        print("request", v.operation)
 
         impls = {
             "add-document": self.librarian.add_document,

{trustgraph-flow-0.23.13 → trustgraph-flow-0.23.14}/trustgraph/model/text_completion/cohere/llm.py

@@ -70,6 +70,8 @@ class Processor(LlmService):
                 model = self.model
             )
 
+            return resp
+
         # FIXME: Wrong exception, don't know what this LLM throws
         # for a rate limit
         except cohere.TooManyRequestsError:

{trustgraph-flow-0.23.13 → trustgraph-flow-0.23.14}/trustgraph/model/text_completion/lmstudio/llm.py

@@ -12,7 +12,6 @@ from .... base import LlmService, LlmResult
 
 default_ident = "text-completion"
 
-default_subscriber = module
 default_model = 'gemma3:9b'
 default_url = os.getenv("LMSTUDIO_URL", "http://localhost:1234/")
 default_temperature = 0.0

{trustgraph-flow-0.23.13 → trustgraph-flow-0.23.14}/trustgraph/model/text_completion/mistral/llm.py

@@ -86,6 +86,8 @@ class Processor(LlmService):
                 model = self.model
             )
 
+            return resp
+
         # FIXME: Wrong exception.  The MistralAI library has retry logic
         # so retry-able errors are retried transparently.  It means we
         # don't get rate limit events.

trustgraph-flow-0.23.14/trustgraph/storage/knowledge/__init__.py

@@ -0,0 +1,3 @@
+
+from . store import run
+

trustgraph-flow-0.23.14/trustgraph/storage/knowledge/__main__.py

@@ -0,0 +1,5 @@
+
+from . store import run
+
+if __name__ == '__main__':
+    run()

trustgraph-flow-0.23.14/trustgraph/storage/knowledge/store.py

@@ -0,0 +1,78 @@
+
+"""
+Stores knowledge-cores in Cassandra
+"""
+
+import json
+import urllib.parse
+
+from ... schema import Triples, GraphEmbeddings
+from ... base import FlowProcessor, ConsumerSpec
+
+from ... tables.knowledge import KnowledgeTableStore
+
+default_ident = "kg-store"
+
+default_cassandra_host = "cassandra"
+keyspace = "knowledge"
+
+class Processor(FlowProcessor):
+
+    def __init__(self, **params):
+
+        id = params.get("id")
+
+        cassandra_host = params.get("cassandra_host", default_cassandra_host)
+        cassandra_user = params.get("cassandra_user")
+        cassandra_password = params.get("cassandra_password")
+
+        super(Processor, self).__init__(
+            **params | {
+                "id": id,
+                "cassandra_host": cassandra_host,
+                "cassandra_user": cassandra_user,
+            }
+        )
+
+        self.register_specification(
+            ConsumerSpec(
+                name = "triples-input",
+                schema = Triples,
+                handler = self.on_triples
+            )
+        )
+
+        self.register_specification(
+            ConsumerSpec(
+                name = "graph-embeddings-input",
+                schema = GraphEmbeddings,
+                handler = self.on_graph_embeddings
+            )
+        )
+
+        self.table_store = KnowledgeTableStore(
+            cassandra_host = cassandra_host.split(","),
+            cassandra_user = cassandra_user,
+            cassandra_password = cassandra_password,
+            keyspace = keyspace,
+        )
+
+    async def on_triples(self, msg, consumer, flow):
+
+        v = msg.value()
+        await self.table_store.add_triples(v)
+
+    async def on_graph_embeddings(self, msg, consumer, flow):
+
+        v = msg.value()
+        await self.table_store.add_graph_embeddings(v)
+
+    @staticmethod
+    def add_args(parser):
+
+        FlowProcessor.add_args(parser)
+
+def run():
+
+    Processor.launch(default_ident, __doc__)
+