trustgraph-flow 0.23.16__tar.gz → 0.23.18__tar.gz

{trustgraph-flow-0.23.16 → trustgraph-flow-0.23.18}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.23.16
+Version: 0.23.18
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.16.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.18.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

{trustgraph-flow-0.23.16 → trustgraph-flow-0.23.18}/trustgraph/config/service/config.py

@@ -120,31 +120,31 @@ class Configuration:
 
     async def handle_getvalues(self, v):
 
-        if v.type not in self:
-
-            return ConfigResponse(
-                version = None,
-                values = None,
-                directory = None,
-                config = None,
-                error = Error(
-                    type = "key-error",
-                    message = f"Key error"
-                )
-            )
+        # if v.type not in self:
+
+        #     return ConfigResponse(
+        #         version = None,
+        #         values = None,
+        #         directory = None,
+        #         config = None,
+        #         error = Error(
+        #             type = "key-error",
+        #             message = f"Key error"
+        #         )
+        #     )
 
-        v = await self.table_store.get_values(v.type)
+        vals = await self.table_store.get_values(v.type)
 
         values = map(
             lambda x: ConfigValue(
                 type = v.type, key = x[0], value = x[1]
             ),
-            v
+            vals
         )
 
         return ConfigResponse(
             version = await self.get_version(),
-            values = values,
+            values = list(values),
             directory = None,
             config = None,
             error = None,

trustgraph-flow-0.23.18/trustgraph/cores/knowledge.py

@@ -0,0 +1,285 @@
+
+from .. schema import KnowledgeResponse, Error, Triples, GraphEmbeddings
+from .. knowledge import hash
+from .. exceptions import RequestError
+from .. tables.knowledge import KnowledgeTableStore
+from .. base import Publisher
+
+import base64
+import asyncio
+import uuid
+
+class KnowledgeManager:
+
+    def __init__(
+            self, cassandra_host, cassandra_user, cassandra_password,
+            keyspace, flow_config,
+    ):
+
+        self.table_store = KnowledgeTableStore(
+            cassandra_host, cassandra_user, cassandra_password, keyspace
+        )
+
+        self.loader_queue = asyncio.Queue(maxsize=20)
+        self.background_task = None
+        self.flow_config = flow_config
+
+    async def delete_kg_core(self, request, respond):
+
+        print("Deleting core...", flush=True)
+
+        await self.table_store.delete_kg_core(
+            request.user, request.id
+        )
+
+        await respond(
+            KnowledgeResponse(
+                error = None,
+                ids = None,
+                eos = False,
+                triples = None,
+                graph_embeddings = None,
+            )
+        )
+
+    async def get_kg_core(self, request, respond):
+
+        print("Get core...", flush=True)
+
+        async def publish_triples(t):
+            await respond(
+                KnowledgeResponse(
+                    error = None,
+                    ids = None,
+                    eos = False,
+                    triples = t,
+                    graph_embeddings = None,
+                )
+            )
+
+        # Remove doc table row
+        await self.table_store.get_triples(
+            request.user,
+            request.id,
+            publish_triples,
+        )
+
+        async def publish_ge(g):
+            await respond(
+                KnowledgeResponse(
+                    error = None,
+                    ids = None,
+                    eos = False,
+                    triples = None,
+                    graph_embeddings = g,
+                )
+            )
+
+        # Remove doc table row
+        await self.table_store.get_graph_embeddings(
+            request.user,
+            request.id,
+            publish_ge,
+        )
+
+        print("Get complete", flush=True)
+
+        await respond(
+            KnowledgeResponse(
+                error = None,
+                ids = None,
+                eos = True,
+                triples = None,
+                graph_embeddings = None,
+            )
+        )
+
+    async def list_kg_cores(self, request, respond):
+
+        ids = await self.table_store.list_kg_cores(request.user)
+
+        await respond(
+            KnowledgeResponse(
+                error = None,
+                ids = ids,
+                eos = False,
+                triples = None,
+                graph_embeddings = None
+            )
+        )
+
+    async def put_kg_core(self, request, respond):
+
+        if request.triples:
+            await self.table_store.add_triples(request.triples)
+
+        if request.graph_embeddings:
+            await self.table_store.add_graph_embeddings(
+                request.graph_embeddings
+            )
+
+        await respond(
+            KnowledgeResponse(
+                error = None,
+                ids = None,
+                eos = False,
+                triples = None,
+                graph_embeddings = None
+            )
+        )
+
+    async def load_kg_core(self, request, respond):
+
+        if self.background_task is None:
+            self.background_task = asyncio.create_task(
+                self.core_loader()
+            )
+            # Wait for it to start (yuck)
+#            await asyncio.sleep(0.5)
+
+        await self.loader_queue.put((request, respond))
+
+        # Not sending a response, the loader thread can do that
+
+    async def unload_kg_core(self, request, respond):
+
+        await respond(
+            KnowledgeResponse(
+                error = Error(
+                    type = "not-implemented",
+                    message = "Not implemented"
+                ),
+                ids = None,
+                eos = False,
+                triples = None,
+                graph_embeddings = None
+            )
+        )
+
+    async def core_loader(self):
+
+        print("Running...", flush=True)
+        while True:
+
+            print("Wait for next load...", flush=True)
+            request, respond = await self.loader_queue.get()
+
+            print("Loading...", request.id, flush=True)
+
+            try:
+
+                if request.id is None:
+                    raise RuntimeError("Core ID must be specified")
+
+                if request.flow is None:
+                    raise RuntimeError("Flow ID must be specified")
+
+                if request.flow not in self.flow_config.flows:
+                    raise RuntimeError("Invalid flow")
+
+                flow = self.flow_config.flows[request.flow]
+
+                if "interfaces" not in flow:
+                    raise RuntimeError("No defined interfaces")
+
+                if "triples-store" not in flow["interfaces"]:
+                    raise RuntimeError("Flow has no triples-store")
+
+                if "graph-embeddings-store" not in flow["interfaces"]:
+                    raise RuntimeError("Flow has no graph-embeddings-store")
+
+                t_q = flow["interfaces"]["triples-store"]
+                ge_q = flow["interfaces"]["graph-embeddings-store"]
+
+                # Got this far, it should all work
+                await respond(
+                    KnowledgeResponse(
+                        error = None,
+                        ids = None,
+                        eos = False,
+                        triples = None,
+                        graph_embeddings = None
+                    )
+                )
+
+            except Exception as e:
+
+                print("Exception:", e, flush=True)
+                await respond(
+                    KnowledgeResponse(
+                        error = Error(
+                            type = "load-error",
+                            message = str(e),
+                        ),
+                        ids = None,
+                        eos = False,
+                        triples = None,
+                        graph_embeddings = None
+                    )
+                )
+
+
+            print("Going to start loading...", flush=True)
+
+            try:
+
+                t_pub = None
+                ge_pub = None
+
+                print(t_q, flush=True)
+                print(ge_q, flush=True)
+
+                t_pub = Publisher(
+                    self.flow_config.pulsar_client, t_q,
+                    schema=Triples,
+                )
+                ge_pub = Publisher(
+                    self.flow_config.pulsar_client, ge_q,
+                    schema=GraphEmbeddings
+                )
+
+                print("Start publishers...", flush=True)
+
+                await t_pub.start()
+                await ge_pub.start()
+
+                async def publish_triples(t):
+                    await t_pub.send(None, t)
+
+                print("Publish triples...", flush=True)
+
+                # Remove doc table row
+                await self.table_store.get_triples(
+                    request.user,
+                    request.id,
+                    publish_triples,
+                )
+
+                async def publish_ge(g):
+                    await ge_pub.send(None, g)
+
+                print("Publish GEs...", flush=True)
+
+                # Remove doc table row
+                await self.table_store.get_graph_embeddings(
+                    request.user,
+                    request.id,
+                    publish_ge,
+                )
+
+                print("Completed that.", flush=True)
+
+            except Exception as e:
+
+                print("Exception:", e, flush=True)
+
+            finally:
+
+                print("Stopping publishers...", flush=True)
+
+                if t_pub: await t_pub.stop()
+                if ge_pub: await ge_pub.stop()
+
+            print("Done", flush=True)
+
+            continue

{trustgraph-flow-0.23.16 → trustgraph-flow-0.23.18}/trustgraph/cores/service.py

@@ -89,6 +89,7 @@ class Processor(AsyncProcessor):
             cassandra_user = cassandra_user,
             cassandra_password = cassandra_password,
             keyspace = keyspace,
+            flow_config = self,
         )
 
         self.register_config_handler(self.on_knowledge_config)
@@ -128,6 +129,8 @@ class Processor(AsyncProcessor):
             "get-kg-core": self.knowledge.get_kg_core,
             "delete-kg-core": self.knowledge.delete_kg_core,
             "put-kg-core": self.knowledge.put_kg_core,
+            "load-kg-core": self.knowledge.load_kg_core,
+            "unload-kg-core": self.knowledge.unload_kg_core,
         }
 
         if v.operation not in impls:

trustgraph-flow-0.23.18/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "0.23.18"

{trustgraph-flow-0.23.16 → trustgraph-flow-0.23.18}/trustgraph/gateway/dispatch/knowledge.py

@@ -61,29 +61,26 @@ class KnowledgeRequestor(ServiceRequestor):
             operation = body.get("operation", None),
             user = body.get("user", None),
             id = body.get("id", None),
+            flow = body.get("flow", None),
+            collection = body.get("collection", None),
             triples = triples,
             graph_embeddings = ge,
         )
 
     def from_response(self, message):
 
-        print("Processing message")
-
         # Response to list, 
         if message.ids is not None:
-            print("-> IDS")
             return {
                 "ids": message.ids
             }, True
 
         if message.triples:
-            print("-> triples")
             return {
                 "triples": serialize_triples(message.triples)
             }, False
 
         if message.graph_embeddings:
-            print("-> ge")
             return {
                 "graph-embeddings": serialize_graph_embeddings(
                     message.graph_embeddings
@@ -91,7 +88,6 @@ class KnowledgeRequestor(ServiceRequestor):
             }, False
 
         if message.eos is True:
-            print("-> eos")
             return {
                 "eos": True
             }, True

{trustgraph-flow-0.23.16 → trustgraph-flow-0.23.18}/trustgraph/tables/config.py

@@ -161,6 +161,7 @@ class ConfigTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -180,6 +181,7 @@ class ConfigTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -204,6 +206,7 @@ class ConfigTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -228,6 +231,7 @@ class ConfigTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -251,6 +255,7 @@ class ConfigTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -275,6 +280,7 @@ class ConfigTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -297,6 +303,7 @@ class ConfigTableStore:
 
             except Exception as e:
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 

{trustgraph-flow-0.23.16 → trustgraph-flow-0.23.18}/trustgraph/tables/knowledge.py

@@ -232,6 +232,7 @@ class KnowledgeTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -276,6 +277,7 @@ class KnowledgeTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -320,6 +322,7 @@ class KnowledgeTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -340,6 +343,7 @@ class KnowledgeTableStore:
 
             except Exception as e:
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -370,6 +374,7 @@ class KnowledgeTableStore:
 
             except Exception as e:
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -386,6 +391,7 @@ class KnowledgeTableStore:
 
             except Exception as e:
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -406,6 +412,7 @@ class KnowledgeTableStore:
 
             except Exception as e:
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -463,6 +470,7 @@ class KnowledgeTableStore:
 
             except Exception as e:
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 

{trustgraph-flow-0.23.16 → trustgraph-flow-0.23.18}/trustgraph/tables/library.py

@@ -232,6 +232,7 @@ class LibraryTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -267,6 +268,7 @@ class LibraryTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -292,6 +294,7 @@ class LibraryTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -314,6 +317,7 @@ class LibraryTableStore:
 
             except Exception as e:
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -361,6 +365,7 @@ class LibraryTableStore:
 
             except Exception as e:
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -407,6 +412,7 @@ class LibraryTableStore:
 
             except Exception as e:
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -455,6 +461,7 @@ class LibraryTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -480,6 +487,7 @@ class LibraryTableStore:
             except Exception as e:
 
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 
@@ -502,6 +510,7 @@ class LibraryTableStore:
 
             except Exception as e:
                 print("Exception:", type(e))
+                raise e
                 print(f"{e}, retry...", flush=True)
                 await asyncio.sleep(1)
 

{trustgraph-flow-0.23.16 → trustgraph-flow-0.23.18}/trustgraph_flow.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.23.16
+Version: 0.23.18
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.16.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.18.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3