trustgraph-flow 0.23.15__tar.gz → 0.23.17__tar.gz

{trustgraph-flow-0.23.15 → trustgraph-flow-0.23.17}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.23.15
+Version: 0.23.17
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.15.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.17.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-flow-0.23.17/trustgraph/config/service/config.py

@@ -0,0 +1,269 @@
+
+from trustgraph.schema import ConfigResponse
+from trustgraph.schema import ConfigValue, Error
+
+from ... tables.config import ConfigTableStore
+
+class ConfigurationClass:
+    
+    async def keys(self):
+        return await self.table_store.get_keys(self.type)
+    
+    async def values(self):
+        vals = await self.table_store.get_values(self.type)
+        return {
+            v[0]: v[1]
+            for v in vals
+        }
+
+    async def get(self, key):
+        return await self.table_store.get_value(self.type, key)        
+
+    async def put(self, key, value):
+        return await self.table_store.put_config(self.type, key, value)
+
+    async def delete(self, key):
+        return await self.table_store.delete_key(self.type, key)
+
+    async def has(self, key):
+        val = await self.table_store.get_value(self.type, key)
+        return val is not None
+
+class Configuration:
+
+    # FIXME: The state is held internally. This only works if there's
+    # one config service.  Should be more than one, and use a
+    # back-end state store.
+
+    # FIXME: This has state now, but does it address all of the above?
+    # REVIEW: Above
+
+    # FIXME: Some version vs config race conditions
+
+    def __init__(self, push, host, user, password, keyspace):
+
+        # External function to respond to update
+        self.push = push
+
+        self.table_store = ConfigTableStore(
+            host, user, password, keyspace
+        )
+
+    async def inc_version(self):
+        await self.table_store.inc_version()
+
+    async def get_version(self):
+        return await self.table_store.get_version()
+
+    def get(self, type):
+
+        c = ConfigurationClass()
+        c.table_store = self.table_store
+        c.type = type
+
+        return c
+
+    async def handle_get(self, v):
+
+        # for k in v.keys:
+        #     if k.type not in self or k.key not in self[k.type]:
+        #         return ConfigResponse(
+        #             version = None,
+        #             values = None,
+        #             directory = None,
+        #             config = None,
+        #             error = Error(
+        #                 type = "key-error",
+        #                 message = f"Key error"
+        #             )
+        #         )
+
+        values = [
+            ConfigValue(
+                type = k.type,
+                key = k.key,
+                value = await self.table_store.get_value(k.type, k.key)
+            )
+            for k in v.keys
+        ]
+
+        return ConfigResponse(
+            version = await self.get_version(),
+            values = values,
+            directory = None,
+            config = None,
+            error = None,
+        )
+    
+    async def handle_list(self, v):
+
+        # if v.type not in self:
+
+        #     return ConfigResponse(
+        #         version = None,
+        #         values = None,
+        #         directory = None,
+        #         config = None,
+        #         error = Error(
+        #             type = "key-error",
+        #             message = "No such type",
+        #         ),
+        #     )
+
+        return ConfigResponse(
+            version = await self.get_version(),
+            values = None,
+            directory = await self.table_store.get_keys(v.type),
+            config = None,
+            error = None,
+        )
+
+    async def handle_getvalues(self, v):
+
+        if v.type not in self:
+
+            return ConfigResponse(
+                version = None,
+                values = None,
+                directory = None,
+                config = None,
+                error = Error(
+                    type = "key-error",
+                    message = f"Key error"
+                )
+            )
+
+        v = await self.table_store.get_values(v.type)
+
+        values = map(
+            lambda x: ConfigValue(
+                type = v.type, key = x[0], value = x[1]
+            ),
+            v
+        )
+
+        return ConfigResponse(
+            version = await self.get_version(),
+            values = values,
+            directory = None,
+            config = None,
+            error = None,
+        )
+
+    async def handle_delete(self, v):
+
+        # for k in v.keys:
+        #     if k.type not in self or k.key not in self[k.type]:
+        #         return ConfigResponse(
+        #             version = None,
+        #             values = None,
+        #             directory = None,
+        #             config = None,
+        #             error = Error(
+        #                 type = "key-error",
+        #                 message = f"Key error"
+        #             )
+        #         )
+
+        for k in v.keys:
+
+            await self.table_store.delete_key(k.type, k.key)
+
+        await self.inc_version()
+
+        await self.push()
+
+        return ConfigResponse(
+            version = None,
+            value = None,
+            directory = None,
+            values = None,
+            config = None,
+            error = None,
+        )
+
+    async def handle_put(self, v):
+
+        for k in v.values:
+
+            await self.table_store.put_config(k.type, k.key, k.value)
+
+        await self.inc_version()
+
+        await self.push()
+
+        return ConfigResponse(
+            version = None,
+            value = None,
+            directory = None,
+            values = None,
+            error = None,
+        )
+
+    async def get_config(self):
+
+        table = await self.table_store.get_all()
+
+        config = {}
+
+        for row in table:
+            if row[0] not in config:
+                config[row[0]] = {}
+            config[row[0]][row[1]] = row[2]
+
+        return config
+
+    async def handle_config(self, v):
+
+        config = await self.get_config()
+
+        return ConfigResponse(
+            version = await self.get_version(),
+            value = None,
+            directory = None,
+            values = None,
+            config = config,
+            error = None,
+        )
+
+    async def handle(self, msg):
+
+        print("Handle message ", msg.operation)
+
+        if msg.operation == "get":
+
+            resp = await self.handle_get(msg)
+
+        elif msg.operation == "list":
+
+            resp = await self.handle_list(msg)
+
+        elif msg.operation == "getvalues":
+
+            resp = await self.handle_getvalues(msg)
+
+        elif msg.operation == "delete":
+
+            resp = await self.handle_delete(msg)
+
+        elif msg.operation == "put":
+
+            resp = await self.handle_put(msg)
+
+        elif msg.operation == "config":
+
+            resp = await self.handle_config(msg)
+
+        else:
+
+            resp = ConfigResponse(
+                value=None,
+                directory=None,
+                values=None,
+                error=Error(
+                    type = "bad-operation",
+                    message = "Bad operation"
+                )
+            )
+
+        return resp

{trustgraph-flow-0.23.15 → trustgraph-flow-0.23.17}/trustgraph/config/service/flow.py

@@ -9,7 +9,7 @@ class FlowConfig:
 
     async def handle_list_classes(self, msg):
 
-        names = list(self.config["flow-classes"].keys())
+        names = list(await self.config.get("flow-classes").keys())
 
         return FlowResponse(
             error = None,
@@ -20,14 +20,18 @@ class FlowConfig:
 
         return FlowResponse(
             error = None,
-            class_definition = self.config["flow-classes"][msg.class_name],
+            class_definition = await self.config.get(
+                "flow-classes"
+            ).get(msg.class_name),
         )
     
     async def handle_put_class(self, msg):
 
-        self.config["flow-classes"][msg.class_name] = msg.class_definition
+        await self.config.get("flow-classes").put(
+            msg.class_name, msg.class_definition
+        )
 
-        self.config.version += 1
+        await self.config.inc_version()
 
         await self.config.push()
 
@@ -39,9 +43,9 @@ class FlowConfig:
 
         print(msg)
 
-        del self.config["flow-classes"][msg.class_name]
+        await self.config.get("flow-classes").delete(msg.class_name)
 
-        self.config.version += 1
+        await self.config.inc_version()
 
         await self.config.push()
 
@@ -51,7 +55,7 @@ class FlowConfig:
     
     async def handle_list_flows(self, msg):
 
-        names = list(self.config["flows"].keys())
+        names = list(await self.config.get("flows").keys())
 
         return FlowResponse(
             error = None,
@@ -60,7 +64,7 @@ class FlowConfig:
     
     async def handle_get_flow(self, msg):
 
-        flow = self.config["flows"][msg.flow_id]
+        flow = await self.config.get("flows").get(msg.flow_id)
 
         return FlowResponse(
             error = None,
@@ -75,13 +79,13 @@ class FlowConfig:
         if msg.flow_id is None:
             raise RuntimeError("No flow ID")
 
-        if msg.flow_id in self.config["flows"]:
+        if msg.flow_id in await self.config.get("flows").values():
             raise RuntimeError("Flow already exists")
 
         if msg.description is None:
             raise RuntimeError("No description")
 
-        if msg.class_name not in self.config["flow-classes"]:
+        if msg.class_name not in await self.config.get("flow-classes").values():
             raise RuntimeError("Class does not exist")
 
         def repl_template(tmp):
@@ -91,7 +95,9 @@ class FlowConfig:
                 "{id}", msg.flow_id
             )
 
-        cls = json.loads(self.config["flow-classes"][msg.class_name])
+        cls = json.loads(
+            await self.config.get("flow-classes").get(msg.class_name)
+        )
 
         for kind in ("class", "flow"):
 
@@ -106,15 +112,18 @@ class FlowConfig:
                     for k2, v2 in v.items()
                 }
 
-                if processor in self.config["flows-active"]:
-                    target = json.loads(self.config["flows-active"][processor])
+                flac = await self.config.get("flows-active").values()
+                if processor in flac:
+                    target = json.loads(flac[processor])
                 else:
                     target = {}
 
                 if variant not in target:
                     target[variant] = v
 
-                self.config["flows-active"][processor] = json.dumps(target)
+                await self.config.get("flows-active").put(
+                    processor, json.dumps(target)
+                )
 
         def repl_interface(i):
             if isinstance(i, str):
@@ -133,13 +142,16 @@ class FlowConfig:
         else:
             interfaces = {}
 
-        self.config["flows"][msg.flow_id] = json.dumps({
-            "description": msg.description,
-            "class-name": msg.class_name,
-            "interfaces": interfaces,
-        })
+        await self.config.get("flows").put(
+            msg.flow_id,
+            json.dumps({
+                "description": msg.description,
+                "class-name": msg.class_name,
+                "interfaces": interfaces,
+            })
+        )
 
-        self.config.version += 1
+        await self.config.inc_version()
 
         await self.config.push()
 
@@ -152,17 +164,17 @@ class FlowConfig:
         if msg.flow_id is None:
             raise RuntimeError("No flow ID")
 
-        if msg.flow_id not in self.config["flows"]:
+        if msg.flow_id not in await self.config.get("flows").keys():
             raise RuntimeError("Flow ID invalid")
 
-        flow = json.loads(self.config["flows"][msg.flow_id])
+        flow = json.loads(await self.config.get("flows").get(msg.flow_id))
 
         if "class-name" not in flow:
             raise RuntimeError("Internal error: flow has no flow class")
 
         class_name = flow["class-name"]
 
-        cls = json.loads(self.config["flow-classes"][class_name])
+        cls = json.loads(await self.config.get("flow-classes").get(class_name))
 
         def repl_template(tmp):
             return tmp.replace(
@@ -179,20 +191,24 @@ class FlowConfig:
 
                 variant = repl_template(variant)
 
-                if processor in self.config["flows-active"]:
-                    target = json.loads(self.config["flows-active"][processor])
+                flac = await self.config.get("flows-active").values()
+
+                if processor in flac:
+                    target = json.loads(flac[processor])
                 else:
                     target = {}
 
                 if variant in target:
                     del target[variant]
 
-                self.config["flows-active"][processor] = json.dumps(target)
+                await self.config.get("flows-active").put(
+                    processor, json.dumps(target)
+                )
 
-        if msg.flow_id in self.config["flows"]:
-            del self.config["flows"][msg.flow_id]
+        if msg.flow_id in await self.config.get("flows").values():
+            await self.config.get("flows").delete(msg.flow_id)
 
-        self.config.version += 1
+        await self.config.inc_version()
 
         await self.config.push()
 

{trustgraph-flow-0.23.15 → trustgraph-flow-0.23.17}/trustgraph/config/service/service.py

@@ -20,6 +20,9 @@ from . flow import FlowConfig
 from ... base import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
 from ... base import Consumer, Producer
 
+# FIXME: How to ensure this doesn't conflict with other usage?
+keyspace = "config"
+
 default_ident = "config-svc"
 
 default_config_request_queue = config_request_queue
@@ -29,6 +32,8 @@ default_config_push_queue = config_push_queue
 default_flow_request_queue = flow_request_queue
 default_flow_response_queue = flow_response_queue
 
+default_cassandra_host = "cassandra"
+
 class Processor(AsyncProcessor):
 
     def __init__(self, **params):
@@ -50,6 +55,10 @@ class Processor(AsyncProcessor):
             "flow_response_queue", default_flow_response_queue
         )
 
+        cassandra_host = params.get("cassandra_host", default_cassandra_host)
+        cassandra_user = params.get("cassandra_user")
+        cassandra_password = params.get("cassandra_password")
+
         id = params.get("id")
 
         flow_request_schema = FlowRequest
@@ -62,6 +71,8 @@ class Processor(AsyncProcessor):
                 "config_push_schema": ConfigPush.__name__,
                 "flow_request_schema": FlowRequest.__name__,
                 "flow_response_schema": FlowResponse.__name__,
+                "cassandra_host": cassandra_host,
+                "cassandra_user": cassandra_user,
             }
         )
 
@@ -125,7 +136,14 @@ class Processor(AsyncProcessor):
             metrics = flow_response_metrics,
         )
 
-        self.config = Configuration(self.push)
+        self.config = Configuration(
+            host = cassandra_host.split(","),
+            user = cassandra_user,
+            password = cassandra_password,
+            keyspace = keyspace,
+            push = self.push
+        )
+
         self.flow = FlowConfig(self.config)
 
         print("Service initialised.")
@@ -138,18 +156,23 @@ class Processor(AsyncProcessor):
         
     async def push(self):
 
+        config = await self.config.get_config()
+        version = await self.config.get_version()
+
         resp = ConfigPush(
-            version = self.config.version,
+            version = version,
             value = None,
             directory = None,
             values = None,
-            config = self.config,
+            config = config,
             error = None,
         )
 
         await self.config_push_producer.send(resp)
 
-        print("Pushed version ", self.config.version)
+        # Race condition, should make sure version & config sync
+
+        print("Pushed version ", await self.config.get_version())
         
     async def on_config_request(self, msg, consumer, flow):
 
@@ -248,6 +271,24 @@ class Processor(AsyncProcessor):
             help=f'Flow response queue {default_flow_response_queue}',
         )
 
+        parser.add_argument(
+            '--cassandra-host',
+            default="cassandra",
+            help=f'Graph host (default: cassandra)'
+        )
+
+        parser.add_argument(
+            '--cassandra-user',
+            default=None,
+            help=f'Cassandra user'
+        )
+
+        parser.add_argument(
+            '--cassandra-password',
+            default=None,
+            help=f'Cassandra password'
+        )
+
 def run():
 
     Processor.launch(default_ident, __doc__)

{trustgraph-flow-0.23.15 → trustgraph-flow-0.23.17}/trustgraph/cores/knowledge.py

@@ -36,9 +36,9 @@ class KnowledgeManager:
             )
         )
 
-    async def fetch_kg_core(self, request, respond):
+    async def get_kg_core(self, request, respond):
 
-        print("Fetch core...", flush=True)
+        print("Get core...", flush=True)
 
         async def publish_triples(t):
             await respond(
@@ -76,7 +76,7 @@ class KnowledgeManager:
             publish_ge,
         )
 
-        print("Fetch complete", flush=True)
+        print("Get complete", flush=True)
 
         await respond(
             KnowledgeResponse(
@@ -102,3 +102,23 @@ class KnowledgeManager:
             )
         )
 
+    async def put_kg_core(self, request, respond):
+
+        if request.triples:
+            await self.table_store.add_triples(request.triples)
+
+        if request.graph_embeddings:
+            await self.table_store.add_graph_embeddings(
+                request.graph_embeddings
+            )
+
+        await respond(
+            KnowledgeResponse(
+                error = None,
+                ids = None,
+                eos = False,
+                triples = None,
+                graph_embeddings = None
+            )
+        )
+

{trustgraph-flow-0.23.15 → trustgraph-flow-0.23.17}/trustgraph/cores/service.py

@@ -125,8 +125,9 @@ class Processor(AsyncProcessor):
 
         impls = {
             "list-kg-cores": self.knowledge.list_kg_cores,
-            "fetch-kg-core": self.knowledge.fetch_kg_core,
+            "get-kg-core": self.knowledge.get_kg_core,
             "delete-kg-core": self.knowledge.delete_kg_core,
+            "put-kg-core": self.knowledge.put_kg_core,
         }
 
         if v.operation not in impls:
@@ -150,12 +151,10 @@ class Processor(AsyncProcessor):
 
         try:
 
+            # We don't send a response back here, the processing
+            # implementation sends whatever it needs to send.
             await self.process_request(v, id)
 
-#            await self.knowledge_response_producer.send(
-#                resp, properties={"id": id}
-#            )
-
             return
 
         except RequestError as e:

trustgraph-flow-0.23.17/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "0.23.17"