trustgraph-base 0.21.19__tar.gz → 0.22.1__tar.gz

{trustgraph-base-0.21.19 → trustgraph-base-0.22.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.21.19
+Version: 0.22.1
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.21.19.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.22.1.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

{trustgraph-base-0.21.19 → trustgraph-base-0.22.1}/trustgraph/api/api.py

@@ -382,3 +382,32 @@ class Api:
         if resp.status_code != 200:
             raise ProtocolException(f"Status code {resp.status_code}")
 
+    def get_config(self):
+
+        # The input consists of system and prompt strings
+        input = {
+            "operation": "config"
+        }
+
+        url = f"{self.url}config"
+
+        # Invoke the API, input is passed as JSON
+        resp = requests.post(url, json=input)
+
+        # Should be a 200 status code
+        if resp.status_code != 200:
+            raise ProtocolException(f"Status code {resp.status_code}")
+
+        try:
+            # Parse the response as JSON
+            object = resp.json()
+        except:
+            raise ProtocolException(f"Expected JSON response")
+
+        self.check_error(resp)
+
+        try:
+            return object["config"], object["version"]
+        except:
+            raise ProtocolException(f"Response not formatted correctly")
+

{trustgraph-base-0.21.19 → trustgraph-base-0.22.1}/trustgraph/base/base_processor.py

@@ -3,12 +3,18 @@ import asyncio
 import os
 import argparse
 import pulsar
+from pulsar.schema import JsonSchema
 import _pulsar
 import time
+import uuid
 from prometheus_client import start_http_server, Info
 
+from .. schema import ConfigPush, config_push_queue
 from .. log_level import LogLevel
 
+default_config_queue = config_push_queue
+config_subscriber_id = str(uuid.uuid4())
+
 class BaseProcessor:
 
     default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
@@ -34,6 +40,11 @@ class BaseProcessor:
         pulsar_api_key = params.get("pulsar_api_key", None)
         log_level = params.get("log_level", LogLevel.INFO)
 
+        self.config_push_queue = params.get(
+            "config_push_queue",
+            default_config_queue
+        )
+
         self.pulsar_host = pulsar_host
         self.pulsar_api_key = pulsar_api_key
 
@@ -53,6 +64,12 @@ class BaseProcessor:
 
         self.pulsar_listener = pulsar_listener
 
+        self.config_subscriber = self.client.subscribe(
+            self.config_push_queue, config_subscriber_id,
+            consumer_type=pulsar.ConsumerType.Shared,
+            schema=JsonSchema(ConfigPush),         
+        )
+
     def __del__(self):
 
         if hasattr(self, "client"):
@@ -74,6 +91,12 @@ class BaseProcessor:
             help=f'Pulsar API key',
         )
 
+        parser.add_argument(
+            '--config-push-queue',
+            default=default_config_queue,
+            help=f'Config push queue {default_config_queue}',
+        )
+
         parser.add_argument(
             '--pulsar-listener',
             help=f'Pulsar listener (default: none)',
@@ -104,14 +127,44 @@ class BaseProcessor:
     async def start(self):
         pass
 
+    async def run_config_queue(self):
+
+        if self.module == "config.service":
+            print("I am config-svc, not looking at config queue", flush=True)
+            return
+
+        print("Config thread running", flush=True)
+        
+        while True:
+
+            try:
+                msg = await asyncio.to_thread(
+                    self.config_subscriber.receive, timeout_millis=2000
+                )
+            except pulsar.Timeout:
+                continue
+
+            v = msg.value()
+            print("Got config version", v.version, flush=True)
+
+            await self.on_config(v.version, v.config)
+
+    async def on_config(self, version, config):
+        pass
+
     async def run(self):
         raise RuntimeError("Something should have implemented the run method")
 
     @classmethod
-    async def launch_async(cls, args):
+    async def launch_async(cls, args, prog):
         p = cls(**args)
+        p.module = prog
         await p.start()
-        await p.run()
+
+        task1 = asyncio.create_task(p.run_config_queue())
+        task2 = asyncio.create_task(p.run())
+
+        await asyncio.gather(task1, task2)
 
     @classmethod
     def launch(cls, prog, doc):
@@ -135,7 +188,7 @@ class BaseProcessor:
 
             try:
 
-                asyncio.run(cls.launch_async(args))
+                asyncio.run(cls.launch_async(args, prog))
 
             except KeyboardInterrupt:
                 print("Keyboard interrupt.")

{trustgraph-base-0.21.19 → trustgraph-base-0.22.1}/trustgraph/base/consumer.py

@@ -82,7 +82,7 @@ class Consumer(BaseProcessor):
 
         while True:
 
-            msg = self.consumer.receive()
+            msg = await asyncio.to_thread(self.consumer.receive)
 
             expiry = time.time() + self.rate_limit_timeout
 

trustgraph-base-0.22.1/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "0.22.1"

{trustgraph-base-0.21.19 → trustgraph-base-0.22.1}/trustgraph/clients/base.py

@@ -28,6 +28,7 @@ class BaseClient:
             output_schema=None,
             pulsar_host="pulsar://pulsar:6650",
             pulsar_api_key=None,
+            listener=None,
     ):
 
         if input_queue == None: raise RuntimeError("Need input_queue")
@@ -41,14 +42,16 @@ class BaseClient:
         if pulsar_api_key:
             auth = pulsar.AuthenticationToken(pulsar_api_key)
             self.client = pulsar.Client(
-            pulsar_host,
-            logger=pulsar.ConsoleLogger(log_level),
-            authentication=auth,
+                pulsar_host,
+                logger=pulsar.ConsoleLogger(log_level),
+                authentication=auth,
+                listener=listener,
             )
         else:
             self.client = pulsar.Client(
-            pulsar_host,
-            logger=pulsar.ConsoleLogger(log_level)
+                pulsar_host,
+                logger=pulsar.ConsoleLogger(log_level),
+                listener_name=listener,
             )
 
         self.producer = self.client.create_producer(

trustgraph-base-0.22.1/trustgraph/clients/config_client.py

@@ -0,0 +1,161 @@
+
+import _pulsar
+import json
+import dataclasses
+
+from .. schema import ConfigRequest, ConfigResponse, ConfigKey, ConfigValue
+from .. schema import config_request_queue
+from .. schema import config_response_queue
+from . base import BaseClient
+
+# Ugly
+ERROR=_pulsar.LoggerLevel.Error
+WARN=_pulsar.LoggerLevel.Warn
+INFO=_pulsar.LoggerLevel.Info
+DEBUG=_pulsar.LoggerLevel.Debug
+
+@dataclasses.dataclass
+class Definition:
+    name: str
+    definition: str
+
+@dataclasses.dataclass
+class Relationship:
+    s: str
+    p: str
+    o: str
+    o_entity: str
+
+@dataclasses.dataclass
+class Topic:
+    name: str
+    definition: str
+
+class ConfigClient(BaseClient):
+
+    def __init__(
+            self, log_level=ERROR,
+            subscriber=None,
+            input_queue=None,
+            output_queue=None,
+            pulsar_host="pulsar://pulsar:6650",
+            listener=None,
+            pulsar_api_key=None,
+    ):
+
+        if input_queue == None:
+            input_queue = config_request_queue
+
+        if output_queue == None:
+            output_queue = config_response_queue
+
+        super(ConfigClient, self).__init__(
+            log_level=log_level,
+            subscriber=subscriber,
+            input_queue=input_queue,
+            output_queue=output_queue,
+            pulsar_host=pulsar_host,
+            pulsar_api_key=pulsar_api_key,
+            input_schema=ConfigRequest,
+            output_schema=ConfigResponse,
+            listener=listener,
+        )
+
+    def request_get(self, keys, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="get",
+            keys=[
+                ConfigKey(
+                    type = k["type"],
+                    key = k["key"]
+                )
+                for k in keys
+            ],
+            timeout=timeout
+        )
+
+        return [
+            {
+                "type": v.type,
+                "key": v.key,
+                "value": v.value
+            }
+            for v in resp.values
+        ]
+
+    def request_list(self, type, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="list",
+            type=type,
+            timeout=timeout
+        )
+
+        return resp.directory
+
+    def request_getvalues(self, type, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="getvalues",
+            type=type,
+            timeout=timeout
+        )
+
+        return [
+            {
+                "type": v.type,
+                "key": v.key,
+                "value": v.value
+            }
+            for v in resp.values
+        ]
+
+    def request_delete(self, keys, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="delete",
+            keys=[
+                ConfigKey(
+                    type = k["type"],
+                    key = k["key"]
+                )
+                for k in keys
+            ],
+            timeout=timeout
+        )
+
+        return None
+
+    def request_put(self, value, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="put",
+            values=[
+                ConfigValue(
+                    type = k["type"],
+                    key = k["key"],
+                    value = k["value"]
+                )
+                for k in keys
+            ],
+            timeout=timeout
+        )
+
+        return None
+
+    def request_config(self, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="config",
+            timeout=timeout
+        )
+
+        return resp.config
+

{trustgraph-base-0.21.19 → trustgraph-base-0.22.1}/trustgraph/schema/__init__.py

@@ -11,5 +11,6 @@ from . metadata import *
 from . agent import *
 from . lookup import *
 from . library import *
+from . config import *
 
 

trustgraph-base-0.22.1/trustgraph/schema/config.py

@@ -0,0 +1,71 @@
+
+from pulsar.schema import Record, Bytes, String, Boolean, Array, Map, Integer
+
+from . topic import topic
+from . types import Error, RowSchema
+
+############################################################################
+
+# Config service:
+#   get(keys) -> (version, values)
+#   list(type) -> (version, values)
+#   getvalues(type) -> (version, values)
+#   put(values) -> ()
+#   delete(keys) -> ()
+#   config() -> (version, config)
+class ConfigKey(Record):
+    type = String()
+    key = String()
+
+class ConfigValue(Record):
+    type = String()
+    key = String()
+    value = String()
+
+# Prompt services, abstract the prompt generation
+class ConfigRequest(Record):
+
+    operation = String() # get, list, getvalues, delete, put, config
+
+    # get, delete
+    keys = Array(ConfigKey())
+
+    # list, getvalues
+    type = String()
+
+    # put
+    values = Array(ConfigValue())
+
+class ConfigResponse(Record):
+
+    # get, list, getvalues, config
+    version = Integer()
+
+    # get, getvalues
+    values = Array(ConfigValue())
+
+    # list
+    directory = Array(String())
+
+    # config
+    config = Map(Map(String()))
+
+    # Everything
+    error = Error()
+
+class ConfigPush(Record):
+    version = Integer()
+    config = Map(Map(String()))
+
+config_request_queue = topic(
+    'config', kind='non-persistent', namespace='request'
+)
+config_response_queue = topic(
+    'config', kind='non-persistent', namespace='response'
+)
+config_push_queue = topic(
+    'config', kind='persistent', namespace='config'
+)
+
+############################################################################
+

{trustgraph-base-0.21.19 → trustgraph-base-0.22.1}/trustgraph_base.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.21.19
+Version: 0.22.1
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.21.19.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.22.1.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

{trustgraph-base-0.21.19 → trustgraph-base-0.22.1}/trustgraph_base.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ trustgraph/base/subscriber.py
 trustgraph/clients/__init__.py
 trustgraph/clients/agent_client.py
 trustgraph/clients/base.py
+trustgraph/clients/config_client.py
 trustgraph/clients/document_embeddings_client.py
 trustgraph/clients/document_rag_client.py
 trustgraph/clients/embeddings_client.py
@@ -35,6 +36,7 @@ trustgraph/objects/field.py
 trustgraph/objects/object.py
 trustgraph/schema/__init__.py
 trustgraph/schema/agent.py
+trustgraph/schema/config.py
 trustgraph/schema/documents.py
 trustgraph/schema/graph.py
 trustgraph/schema/library.py

trustgraph-base-0.21.19/trustgraph/base_version.py

@@ -1 +0,0 @@
-__version__ = "0.21.19"