trustgraph-base 0.21.0__tar.gz → 0.21.2__tar.gz

{trustgraph-base-0.21.0 → trustgraph-base-0.21.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.21.0
+Version: 0.21.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.21.0.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.21.2.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

{trustgraph-base-0.21.0 → trustgraph-base-0.21.2}/trustgraph/base/__init__.py

@@ -3,4 +3,6 @@ from . base_processor import BaseProcessor
 from . consumer import Consumer
 from . producer import Producer
 from . consumer_producer import ConsumerProducer
+from . publisher import Publisher
+from . subscriber import Subscriber
 

{trustgraph-base-0.21.0 → trustgraph-base-0.21.2}/trustgraph/base/base_processor.py

@@ -1,4 +1,5 @@
 
+import asyncio
 import os
 import argparse
 import pulsar
@@ -28,15 +29,19 @@ class BaseProcessor:
         })
 
         pulsar_host = params.get("pulsar_host", self.default_pulsar_host)
+        pulsar_listener = params.get("pulsar_listener", None)
         log_level = params.get("log_level", LogLevel.INFO)
 
         self.pulsar_host = pulsar_host
 
         self.client = pulsar.Client(
             pulsar_host,
+            listener_name=pulsar_listener,
             logger=pulsar.ConsoleLogger(log_level.to_pulsar())
         )
 
+        self.pulsar_listener = pulsar_listener
+
     def __del__(self):
 
         if hasattr(self, "client"):
@@ -52,6 +57,11 @@ class BaseProcessor:
             help=f'Pulsar host (default: {__class__.default_pulsar_host})',
         )
 
+        parser.add_argument(
+            '--pulsar-listener',
+            help=f'Pulsar listener (default: none)',
+        )
+
         parser.add_argument(
             '-l', '--log-level',
             type=LogLevel,
@@ -74,11 +84,20 @@ class BaseProcessor:
             help=f'Pulsar host (default: 8000)',
         )
 
-    def run(self):
+    async def start(self):
+        pass
+
+    async def run(self):
         raise RuntimeError("Something should have implemented the run method")
 
     @classmethod
-    def start(cls, prog, doc):
+    async def launch_async(cls, args):
+        p = cls(**args)
+        await p.start()
+        await p.run()
+
+    @classmethod
+    def launch(cls, prog, doc):
 
         parser = argparse.ArgumentParser(
             prog=prog,
@@ -99,8 +118,7 @@ class BaseProcessor:
 
             try:
 
-                p = cls(**args)
-                p.run()
+                asyncio.run(cls.launch_async(args))
 
             except KeyboardInterrupt:
                 print("Keyboard interrupt.")
@@ -118,3 +136,4 @@ class BaseProcessor:
                 print("Will retry...", flush=True)
 
                 time.sleep(4)
+

{trustgraph-base-0.21.0 → trustgraph-base-0.21.2}/trustgraph/base/consumer.py

@@ -1,4 +1,5 @@
 
+import asyncio
 from pulsar.schema import JsonSchema
 import pulsar
 from prometheus_client import Histogram, Info, Counter, Enum
@@ -75,7 +76,7 @@ class Consumer(BaseProcessor):
 
         print("Initialised consumer.", flush=True)
 
-    def run(self):
+    async def run(self):
 
         __class__.state_metric.state('running')
 
@@ -104,7 +105,7 @@ class Consumer(BaseProcessor):
                 try:
 
                     with __class__.request_metric.time():
-                        self.handle(msg)
+                        await self.handle(msg)
 
                     # Acknowledge successful processing of the message
                     self.consumer.acknowledge(msg)

{trustgraph-base-0.21.0 → trustgraph-base-0.21.2}/trustgraph/base/consumer_producer.py

@@ -42,7 +42,7 @@ class ConsumerProducer(Consumer):
 
         print("Initialised consumer/producer.")
 
-    def send(self, msg, properties={}):
+    async def send(self, msg, properties={}):
         self.producer.send(msg, properties)
         __class__.output_metric.inc()
 

{trustgraph-base-0.21.0 → trustgraph-base-0.21.2}/trustgraph/base/producer.py

@@ -37,7 +37,7 @@ class Producer(BaseProcessor):
             chunking_enabled=True,
         )
 
-    def send(self, msg, properties={}):
+    async def send(self, msg, properties={}):
         self.producer.send(msg, properties)
         __class__.output_metric.inc()
 

trustgraph-base-0.21.2/trustgraph/base/publisher.py

@@ -0,0 +1,63 @@
+
+import queue
+import time
+import pulsar
+import threading
+
+class Publisher:
+
+    def __init__(self, pulsar_client, topic, schema=None, max_size=10,
+                 chunking_enabled=True):
+        self.client = pulsar_client
+        self.topic = topic
+        self.schema = schema
+        self.q = queue.Queue(maxsize=max_size)
+        self.chunking_enabled = chunking_enabled
+        self.running = True
+
+    def start(self):
+        self.task = threading.Thread(target=self.run)
+        self.task.start()
+
+    def stop(self):
+        self.running = False
+
+    def join(self):
+        self.stop()
+        self.task.join()
+
+    def run(self):
+
+        while self.running:
+
+            try:
+
+                print(self.chunking_enabled)
+                producer = self.client.create_producer(
+                    topic=self.topic,
+                    schema=self.schema,
+                    chunking_enabled=self.chunking_enabled,
+                )
+
+                while self.running:
+
+                    try:
+                        id, item = self.q.get(timeout=0.5)
+                    except queue.Empty:
+                        continue
+
+                    if id:
+                        producer.send(item, { "id": id })
+                    else:
+                        producer.send(item)
+
+            except Exception as e:
+                print("Exception:", e, flush=True)
+
+            # If handler drops out, sleep a retry
+            time.sleep(2)
+
+    def send(self, id, msg):
+        self.q.put((id, msg))
+
+        

trustgraph-base-0.21.2/trustgraph/base/subscriber.py

@@ -0,0 +1,114 @@
+
+import queue
+import pulsar
+import threading
+import time
+
+class Subscriber:
+
+    def __init__(self, pulsar_client, topic, subscription, consumer_name,
+                 schema=None, max_size=100):
+        self.client = pulsar_client
+        self.topic = topic
+        self.subscription = subscription
+        self.consumer_name = consumer_name
+        self.schema = schema
+        self.q = {}
+        self.full = {}
+        self.max_size = max_size
+        self.lock = threading.Lock()
+        self.running = True
+
+    def start(self):
+        self.task = threading.Thread(target=self.run)
+        self.task.start()
+
+    def stop(self):
+        self.running = False
+
+    def join(self):
+        self.task.join()
+
+    def run(self):
+
+        while self.running:
+
+            try:
+
+                consumer = self.client.subscribe(
+                    topic=self.topic,
+                    subscription_name=self.subscription,
+                    consumer_name=self.consumer_name,
+                    schema=self.schema,
+                )
+
+                while self.running:
+
+                    msg = consumer.receive()
+
+                    # Acknowledge successful reception of the message
+                    consumer.acknowledge(msg)
+
+                    try:
+                        id = msg.properties()["id"]
+                    except:
+                        id = None
+
+                    value = msg.value()
+
+                    with self.lock:
+
+                        if id in self.q:
+                            try:
+                                # FIXME: Timeout means data goes missing
+                                self.q[id].put(value, timeout=0.5)
+                            except:
+                                pass
+
+                        for q in self.full.values():
+                            try:
+                                # FIXME: Timeout means data goes missing
+                                q.put(value, timeout=0.5)
+                            except:
+                                pass
+
+            except Exception as e:
+                print("Exception:", e, flush=True)
+         
+            # If handler drops out, sleep a retry
+            time.sleep(2)
+
+    def subscribe(self, id):
+
+        with self.lock:
+
+            q = queue.Queue(maxsize=self.max_size)
+            self.q[id] = q
+
+        return q
+
+    def unsubscribe(self, id):
+        
+        with self.lock:
+
+            if id in self.q:
+#                self.q[id].shutdown(immediate=True)
+                del self.q[id]
+    
+    def subscribe_all(self, id):
+
+        with self.lock:
+
+            q = queue.Queue(maxsize=self.max_size)
+            self.full[id] = q
+
+        return q
+
+    def unsubscribe_all(self, id):
+
+        with self.lock:
+
+            if id in self.full:
+#                self.full[id].shutdown(immediate=True)
+                del self.full[id]
+

trustgraph-base-0.21.2/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "0.21.2"

{trustgraph-base-0.21.0 → trustgraph-base-0.21.2}/trustgraph/exceptions.py

@@ -8,3 +8,6 @@ class LlmError(Exception):
 class ParseError(Exception):
     pass
 
+class RequestError(Exception):
+    pass
+

{trustgraph-base-0.21.0 → trustgraph-base-0.21.2}/trustgraph/schema/__init__.py

@@ -10,5 +10,6 @@ from . retrieval import *
 from . metadata import *
 from . agent import *
 from . lookup import *
+from . library import *
 
 

trustgraph-base-0.21.2/trustgraph/schema/library.py

@@ -0,0 +1,58 @@
+
+from pulsar.schema import Record, Bytes, String, Array
+from . types import Triple
+from . topic import topic
+from . types import Error
+from . metadata import Metadata
+from . documents import Document, TextDocument
+
+# add(Metadata, Bytes) : error?
+# copy(id, user, collection)
+# move(id, user, collection)
+# delete(id)
+# get(id) : Bytes
+# reindex(id)
+# list(user, collection) : id[]
+# info(id[]) : DocumentInfo[]
+# search(<key,op,value>[]) : id[]
+
+class DocumentPackage(Record):
+    metadata = Array(Triple())
+    document = Bytes()
+    kind = String()
+    user = String()
+    collection = String()
+    title = String()
+    comments = String()
+
+class DocumentInfo(Record):
+    metadata = Array(Triple())
+    kind = String()
+    user = String()
+    collection = String()
+
+class Criteria(Record):
+    key = String()
+    value = String()
+    operator = String()
+
+class LibrarianRequest(Record):
+    operation = String()
+    id = String()
+    document = DocumentPackage()
+    user = String()
+    collection = String()
+    criteria = Array(Criteria())
+
+class LibrarianResponse(Record):
+    error = Error()
+    document = DocumentPackage()
+    info = Array(DocumentInfo())
+
+librarian_request_queue = topic(
+    'librarian', kind='non-persistent', namespace='request'
+)
+librarian_response_queue = topic(
+    'librarian', kind='non-persistent', namespace='response',
+)
+

{trustgraph-base-0.21.0 → trustgraph-base-0.21.2}/trustgraph_base.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.21.0
+Version: 0.21.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.21.0.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.21.2.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

{trustgraph-base-0.21.0 → trustgraph-base-0.21.2}/trustgraph_base.egg-info/SOURCES.txt

@@ -11,6 +11,8 @@ trustgraph/base/base_processor.py
 trustgraph/base/consumer.py
 trustgraph/base/consumer_producer.py
 trustgraph/base/producer.py
+trustgraph/base/publisher.py
+trustgraph/base/subscriber.py
 trustgraph/clients/__init__.py
 trustgraph/clients/agent_client.py
 trustgraph/clients/base.py
@@ -35,6 +37,7 @@ trustgraph/schema/__init__.py
 trustgraph/schema/agent.py
 trustgraph/schema/documents.py
 trustgraph/schema/graph.py
+trustgraph/schema/library.py
 trustgraph/schema/lookup.py
 trustgraph/schema/metadata.py
 trustgraph/schema/models.py

trustgraph-base-0.21.0/trustgraph/base_version.py

@@ -1 +0,0 @@
-__version__ = "0.21.0"