trustgraph-base 0.22.0__tar.gz → 0.22.1__tar.gz

{trustgraph-base-0.22.0 → trustgraph-base-0.22.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.22.0
+Version: 0.22.1
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.22.0.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.22.1.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

{trustgraph-base-0.22.0 → trustgraph-base-0.22.1}/trustgraph/api/api.py

@@ -411,4 +411,3 @@ class Api:
         except:
             raise ProtocolException(f"Response not formatted correctly")
 
-        

{trustgraph-base-0.22.0 → trustgraph-base-0.22.1}/trustgraph/base/base_processor.py

@@ -3,12 +3,18 @@ import asyncio
 import os
 import argparse
 import pulsar
+from pulsar.schema import JsonSchema
 import _pulsar
 import time
+import uuid
 from prometheus_client import start_http_server, Info
 
+from .. schema import ConfigPush, config_push_queue
 from .. log_level import LogLevel
 
+default_config_queue = config_push_queue
+config_subscriber_id = str(uuid.uuid4())
+
 class BaseProcessor:
 
     default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
@@ -34,6 +40,11 @@ class BaseProcessor:
         pulsar_api_key = params.get("pulsar_api_key", None)
         log_level = params.get("log_level", LogLevel.INFO)
 
+        self.config_push_queue = params.get(
+            "config_push_queue",
+            default_config_queue
+        )
+
         self.pulsar_host = pulsar_host
         self.pulsar_api_key = pulsar_api_key
 
@@ -53,6 +64,12 @@ class BaseProcessor:
 
         self.pulsar_listener = pulsar_listener
 
+        self.config_subscriber = self.client.subscribe(
+            self.config_push_queue, config_subscriber_id,
+            consumer_type=pulsar.ConsumerType.Shared,
+            schema=JsonSchema(ConfigPush),         
+        )
+
     def __del__(self):
 
         if hasattr(self, "client"):
@@ -74,6 +91,12 @@ class BaseProcessor:
             help=f'Pulsar API key',
         )
 
+        parser.add_argument(
+            '--config-push-queue',
+            default=default_config_queue,
+            help=f'Config push queue {default_config_queue}',
+        )
+
         parser.add_argument(
             '--pulsar-listener',
             help=f'Pulsar listener (default: none)',
@@ -104,14 +127,44 @@ class BaseProcessor:
     async def start(self):
         pass
 
+    async def run_config_queue(self):
+
+        if self.module == "config.service":
+            print("I am config-svc, not looking at config queue", flush=True)
+            return
+
+        print("Config thread running", flush=True)
+        
+        while True:
+
+            try:
+                msg = await asyncio.to_thread(
+                    self.config_subscriber.receive, timeout_millis=2000
+                )
+            except pulsar.Timeout:
+                continue
+
+            v = msg.value()
+            print("Got config version", v.version, flush=True)
+
+            await self.on_config(v.version, v.config)
+
+    async def on_config(self, version, config):
+        pass
+
     async def run(self):
         raise RuntimeError("Something should have implemented the run method")
 
     @classmethod
-    async def launch_async(cls, args):
+    async def launch_async(cls, args, prog):
         p = cls(**args)
+        p.module = prog
         await p.start()
-        await p.run()
+
+        task1 = asyncio.create_task(p.run_config_queue())
+        task2 = asyncio.create_task(p.run())
+
+        await asyncio.gather(task1, task2)
 
     @classmethod
     def launch(cls, prog, doc):
@@ -135,7 +188,7 @@ class BaseProcessor:
 
             try:
 
-                asyncio.run(cls.launch_async(args))
+                asyncio.run(cls.launch_async(args, prog))
 
             except KeyboardInterrupt:
                 print("Keyboard interrupt.")

{trustgraph-base-0.22.0 → trustgraph-base-0.22.1}/trustgraph/base/consumer.py

@@ -82,7 +82,7 @@ class Consumer(BaseProcessor):
 
         while True:
 
-            msg = self.consumer.receive()
+            msg = await asyncio.to_thread(self.consumer.receive)
 
             expiry = time.time() + self.rate_limit_timeout
 

trustgraph-base-0.22.1/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "0.22.1"

{trustgraph-base-0.22.0 → trustgraph-base-0.22.1}/trustgraph_base.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.22.0
+Version: 0.22.1
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.22.0.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.22.1.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-base-0.22.0/trustgraph/base_version.py

@@ -1 +0,0 @@
-__version__ = "0.22.0"