trustgraph-base 0.23.2__tar.gz → 0.23.4__tar.gz

{trustgraph-base-0.23.2 → trustgraph-base-0.23.4}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.23.2
+Version: 0.23.4
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.2.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.4.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

{trustgraph-base-0.23.2 → trustgraph-base-0.23.4}/trustgraph/base/async_processor.py

@@ -114,7 +114,7 @@ class AsyncProcessor:
         version = message.value().version
 
         # Invoke message handlers
-        print("Config change event", config, version, flush=True)
+        print("Config change event", version, flush=True)
         for ch in self.config_handlers:
             await ch(config, version)
 

{trustgraph-base-0.23.2 → trustgraph-base-0.23.4}/trustgraph/base/consumer_spec.py

@@ -20,7 +20,7 @@ class ConsumerSpec(Spec):
             flow = flow,
             client = processor.pulsar_client,
             topic = definition[self.name],
-            subscriber = processor.id + "--" + self.name,
+            subscriber = processor.id + "--" + flow.name + "--" + self.name,
             schema = self.schema,
             handler = self.handler,
             metrics = consumer_metrics,

{trustgraph-base-0.23.2 → trustgraph-base-0.23.4}/trustgraph/base/flow_processor.py

@@ -75,7 +75,8 @@ class FlowProcessor(AsyncProcessor):
         # Get list of flows which should be running and are currently
         # running
         wanted_flows = flow_config.keys()
-        current_flows = self.flows.keys()
+        # This takes a copy, needed because dict gets modified by stop_flow
+        current_flows = list(self.flows.keys())
 
         # Start all the flows which arent currently running
         for flow in wanted_flows:

{trustgraph-base-0.23.2 → trustgraph-base-0.23.4}/trustgraph/base/publisher.py

@@ -21,6 +21,7 @@ class Publisher:
 
     async def stop(self):
         self.running = False
+        await self.task
 
     async def join(self):
         await self.stop()
@@ -42,7 +43,7 @@ class Publisher:
                     try:
                         id, item = await asyncio.wait_for(
                             self.q.get(),
-                            timeout=0.5
+                            timeout=0.25
                         )
                     except asyncio.TimeoutError:
                         continue
@@ -57,8 +58,11 @@ class Publisher:
             except Exception as e:
                 print("Exception:", e, flush=True)
 
+            if not self.running:
+                return
+
             # If handler drops out, sleep a retry
-            time.sleep(2)
+            await asyncio.sleep(1)
 
     async def send(self, id, item):
         await self.q.put((id, item))

{trustgraph-base-0.23.2 → trustgraph-base-0.23.4}/trustgraph/base/request_response_spec.py

@@ -127,7 +127,9 @@ class RequestResponseSpec(Spec):
 
         rr = self.impl(
             client = processor.pulsar_client,
-            subscription = flow.id,
+            subscription = (
+                processor.id + "--" + flow.name + "--" + self.request_name
+            ),
             consumer_name = flow.id,
             request_topic = definition[self.request_name],
             request_schema = self.request_schema,

{trustgraph-base-0.23.2 → trustgraph-base-0.23.4}/trustgraph/base/subscriber.py

@@ -20,7 +20,7 @@ class Subscriber:
         self.running = True
         self.metrics = metrics
 
-    async def __del__(self):
+    def __del__(self):
         self.running = False
 
     async def start(self):
@@ -28,6 +28,7 @@ class Subscriber:
 
     async def stop(self):
         self.running = False
+        await self.task
 
     async def join(self):
         await self.stop()
@@ -35,6 +36,8 @@ class Subscriber:
 
     async def run(self):
 
+        consumer = None
+
         while self.running:
 
             if self.metrics:
@@ -59,7 +62,7 @@ class Subscriber:
                     try:
                         msg = await asyncio.to_thread(
                             consumer.receive,
-                            timeout_millis=2000
+                            timeout_millis=250
                         )
                     except _pulsar.Timeout:
                         continue
@@ -91,7 +94,7 @@ class Subscriber:
                                 # FIXME: Timeout means data goes missing
                                 await asyncio.wait_for(
                                     self.q[id].put(value),
-                                    timeout=2
+                                    timeout=1
                                 )
 
                             except Exception as e:
@@ -103,7 +106,7 @@ class Subscriber:
                                 # FIXME: Timeout means data goes missing
                                 await asyncio.wait_for(
                                     q.put(value),
-                                    timeout=2
+                                    timeout=1
                                 )
                             except Exception as e:
                                 self.metrics.dropped()
@@ -112,13 +115,21 @@ class Subscriber:
             except Exception as e:
                 print("Subscriber exception:", e, flush=True)
 
-            consumer.close()
+            finally:
+
+                if consumer:
+                    consumer.close()
+                    consumer = None
+                
          
             if self.metrics:
                 self.metrics.state("stopped")
 
+            if not self.running:
+                return
+            
             # If handler drops out, sleep a retry
-            time.sleep(2)
+            await asyncio.sleep(1)
 
     async def subscribe(self, id):
 

trustgraph-base-0.23.4/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "0.23.4"

{trustgraph-base-0.23.2 → trustgraph-base-0.23.4}/trustgraph_base.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.23.2
+Version: 0.23.4
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.2.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.4.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-base-0.23.2/trustgraph/base_version.py

@@ -1 +0,0 @@
-__version__ = "0.23.2"