trustgraph-flow 0.23.8__tar.gz → 0.23.9__tar.gz

{trustgraph-flow-0.23.8 → trustgraph-flow-0.23.9}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.23.8
+Version: 0.23.9
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.8.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.9.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

{trustgraph-flow-0.23.8 → trustgraph-flow-0.23.9}/trustgraph/config/service/service.py

@@ -3,8 +3,6 @@
 Config service.  Manages system global configuration state
 """
 
-from pulsar.schema import JsonSchema
-
 from trustgraph.schema import Error
 
 from trustgraph.schema import ConfigRequest, ConfigResponse, ConfigPush
@@ -14,7 +12,6 @@ from trustgraph.schema import config_push_queue
 from trustgraph.schema import FlowRequest, FlowResponse
 from trustgraph.schema import flow_request_queue, flow_response_queue
 
-from trustgraph.log_level import LogLevel
 from trustgraph.base import AsyncProcessor, Consumer, Producer
 
 from . config import Configuration

trustgraph-flow-0.23.9/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "0.23.9"

trustgraph-flow-0.23.9/trustgraph/gateway/dispatch/librarian.py

@@ -0,0 +1,93 @@
+
+import base64
+
+from ... schema import LibrarianRequest, LibrarianResponse
+from ... schema import librarian_request_queue
+from ... schema import librarian_response_queue
+
+from . requestor import ServiceRequestor
+from . serialize import serialize_document_metadata
+from . serialize import serialize_processing_metadata
+from . serialize import to_document_metadata, to_processing_metadata
+from . serialize import to_criteria
+
+class LibrarianRequestor(ServiceRequestor):
+    def __init__(self, pulsar_client, consumer, subscriber, timeout=120):
+
+        super(LibrarianRequestor, self).__init__(
+            pulsar_client=pulsar_client,
+            consumer_name = consumer,
+            subscription = subscriber,
+            request_queue=librarian_request_queue,
+            response_queue=librarian_response_queue,
+            request_schema=LibrarianRequest,
+            response_schema=LibrarianResponse,
+            timeout=timeout,
+        )
+
+    def to_request(self, body):
+
+        # Content gets base64 decoded & encoded again.  It at least makes
+        # sure payload is valid base64.
+
+        if "document-metadata" in body:
+            dm = to_document_metadata(body["document-metadata"])
+        else:
+            dm = None
+
+        if "processing-metadata" in body:
+            pm = to_processing_metadata(body["processing-metadata"])
+        else:
+            pm = None
+
+        if "criteria" in body:
+            criteria = to_criteria(body["criteria"])
+        else:
+            criteria = None
+
+        if "content" in body:
+            content = base64.b64decode(body["content"].encode("utf-8"))
+            content = base64.b64encode(content).decode("utf-8")
+        else:
+            content = None
+
+        return LibrarianRequest(
+            operation = body.get("operation", None),
+            document_id = body.get("document-id", None),
+            processing_id = body.get("processing-id", None),
+            document_metadata = dm,
+            processing_metadata = pm,
+            content = content,
+            user = body.get("user", None),
+            collection = body.get("collection", None),
+            criteria = criteria,
+        )
+
+    def from_response(self, message):
+
+        print(message)
+
+        response = {}
+
+        if message.document_metadata:
+            response["document-metadata"] = serialize_document_metadata(
+                message.document_metadata
+            )
+
+        if message.content:
+            response["content"] = message.content.decode("utf-8")
+
+        if message.document_metadatas != None:
+            response["document-metadatas"] = [
+                serialize_document_metadata(v)
+                for v in message.document_metadatas
+            ]
+
+        if message.processing_metadatas != None:
+            response["processing-metadatas"] = [
+                serialize_processing_metadata(v)
+                for v in message.processing_metadatas
+            ]
+        
+        return response, True
+

{trustgraph-flow-0.23.8 → trustgraph-flow-0.23.9}/trustgraph/gateway/dispatch/serialize.py

@@ -1,7 +1,7 @@
 
 import base64
 
-from ... schema import Value, Triple, DocumentPackage, DocumentInfo
+from ... schema import Value, Triple, DocumentMetadata, ProcessingMetadata
 
 def to_value(x):
     return Value(value=x["v"], is_uri=x["e"])
@@ -80,42 +80,51 @@ def serialize_document_embeddings(message):
         ],
     }
 
-def serialize_document_package(message):
+def serialize_document_metadata(message):
 
     ret = {}
 
     if message.id:
         ret["id"] = message.id
 
-    if message.metadata:
-        ret["metadata"] = serialize_subgraph(message.metdata)
-
-    if message.document:
-        blob = base64.b64encode(
-            message.document.encode("utf-8")
-        ).decode("utf-8")
-        ret["document"] = blob
+    if message.time:
+        ret["time"] = message.time
 
     if message.kind:
         ret["kind"] = message.kind
 
+    if message.title:
+        ret["title"] = message.title
+
+    if message.comments:
+        ret["comments"] = message.comments
+
+    if message.metadata:
+        ret["metadata"] = serialize_subgraph(message.metadata)
+
     if message.user:
         ret["user"] = message.user
 
-    if message.collection:
-        ret["collection"] = message.collection
+    if message.tags:
+        ret["tags"] = message.tags
 
     return ret
 
-def serialize_document_info(message):
+def serialize_processing_metadata(message):
 
     ret = {}
 
     if message.id:
         ret["id"] = message.id
 
-    if message.kind:
-        ret["kind"] = message.kind
+    if message.id:
+        ret["document-id"] = message.document_id
+
+    if message.time:
+        ret["time"] = message.time
+
+    if message.flow:
+        ret["flow"] = message.flow
 
     if message.user:
         ret["user"] = message.user
@@ -123,45 +132,34 @@ def serialize_document_info(message):
     if message.collection:
         ret["collection"] = message.collection
 
-    if message.title:
-        ret["title"] = message.title
-
-    if message.comments:
-        ret["comments"] = message.comments
-
-    if message.time:
-        ret["time"] = message.time
-
-    if message.metadata:
-        ret["metadata"] = serialize_subgraph(message.metadata)
+    if message.tags:
+        ret["tags"] = message.tags
 
     return ret
 
-def to_document_package(x):
+def to_document_metadata(x):
 
-    return DocumentPackage(
+    return DocumentMetadata(
         id = x.get("id", None),
+        time = x.get("time", None),
         kind = x.get("kind", None),
-        user = x.get("user", None),
-        collection = x.get("collection", None),
         title = x.get("title", None),
         comments = x.get("comments", None),
-        time = x.get("time", None),
-        document = x.get("document", None),
         metadata = to_subgraph(x["metadata"]),
+        user = x.get("user", None),
+        tags = x.get("tags", None),
     )
 
-def to_document_info(x):
+def to_processing_metadata(x):
 
-    return DocumentInfo(
+    return ProcessingMetadata(
         id = x.get("id", None),
-        kind = x.get("kind", None),
+        document_id = x.get("document-id", None),
+        time = x.get("time", None),
+        flow = x.get("flow", None),
         user = x.get("user", None),
         collection = x.get("collection", None),
-        title = x.get("title", None),
-        comments = x.get("comments", None),
-        time = x.get("time", None),
-        metadata = to_subgraph(x["metadata"]),
+        tags = x.get("tags", None),
     )
 
 def to_criteria(x):
@@ -169,3 +167,4 @@ def to_criteria(x):
         Critera(v["key"], v["value"], v["operator"])
         for v in x
     ]
+

{trustgraph-flow-0.23.8 → trustgraph-flow-0.23.9}/trustgraph/gateway/service.py

@@ -95,7 +95,6 @@ class Api:
 
         await self.config_receiver.start()
 
-
         for ep in self.endpoints:
             ep.add_routes(self.app)
 

{trustgraph-flow-0.23.8 → trustgraph-flow-0.23.9}/trustgraph/librarian/blob_store.py

@@ -37,7 +37,7 @@ class BlobStore:
         else:
             print("Bucket", self.bucket_name, "already exists", flush=True)
 
-    def add(self, object_id, blob, kind):
+    async def add(self, object_id, blob, kind):
 
         # FIXME: Loop retry
         self.minio.put_object(
@@ -49,3 +49,25 @@ class BlobStore:
         )
 
         print("Add blob complete", flush=True)
+
+    async def remove(self, object_id):
+
+        # FIXME: Loop retry
+        self.minio.remove_object(
+            bucket_name = self.bucket_name,
+            object_name = "doc/" + str(object_id),
+        )
+
+        print("Remove blob complete", flush=True)
+
+
+    async def get(self, object_id):
+
+        # FIXME: Loop retry
+        resp = self.minio.get_object(
+            bucket_name = self.bucket_name,
+            object_name = "doc/" + str(object_id),
+        )
+
+        return resp.read()
+

trustgraph-flow-0.23.9/trustgraph/librarian/librarian.py

@@ -0,0 +1,267 @@
+
+from .. schema import LibrarianRequest, LibrarianResponse, Error, Triple
+from .. knowledge import hash
+from .. exceptions import RequestError
+from . table_store import TableStore
+from . blob_store import BlobStore
+import base64
+
+import uuid
+
+class Librarian:
+
+    def __init__(
+            self,
+            cassandra_host, cassandra_user, cassandra_password,
+            minio_host, minio_access_key, minio_secret_key,
+            bucket_name, keyspace, load_document, load_text,
+    ):
+
+        self.blob_store = BlobStore(
+            minio_host, minio_access_key, minio_secret_key, bucket_name
+        )
+
+        self.table_store = TableStore(
+            cassandra_host, cassandra_user, cassandra_password, keyspace
+        )
+
+        self.load_document = load_document
+        self.load_text = load_text
+
+    async def add_document(self, request):
+
+        if request.document_metadata.kind not in (
+                "text/plain", "application/pdf"
+        ):
+            raise RequestError(
+                "Invalid document kind: " + request.document_metadata.kind
+            )
+
+        if await self.table_store.document_exists(
+                request.document_metadata.user,
+                request.document_metadata.id
+        ):
+            raise RuntimeError("Document already exists")
+
+        # Create object ID for blob
+        object_id = uuid.uuid4()
+
+        print("Add blob...")
+
+        await self.blob_store.add(
+            object_id, base64.b64decode(request.content),
+            request.document_metadata.kind
+        )
+
+        print("Add table...")
+
+        await self.table_store.add_document(
+            request.document_metadata, object_id
+        )
+
+        print("Add complete", flush=True)
+
+        return LibrarianResponse(
+            error = None,
+            document_metadata = None,
+            content = None,
+            document_metadatas = None,
+            processing_metadatas = None,
+        )
+
+    async def remove_document(self, request):
+
+        print("Removing doc...")
+
+        if not await self.table_store.document_exists(
+                request.user,
+                request.document_id,
+        ):
+            raise RuntimeError("Document does not exist")
+
+        object_id = await self.table_store.get_document_object_id(
+            request.user,
+            request.document_id
+        )
+
+        # Remove blob...
+        await self.blob_store.remove(object_id)
+
+        # Remove doc table row
+        await self.table_store.remove_document(
+            request.user,
+            request.document_id
+        )
+
+        print("Remove complete", flush=True)
+
+        return LibrarianResponse(
+            error = None,
+            document_metadata = None,
+            content = None,
+            document_metadatas = None,
+            processing_metadatas = None,
+        )
+
+    async def update_document(self, request):
+
+        print("Updating doc...")
+
+        # You can't update the document ID, user or kind.
+
+        if not await self.table_store.document_exists(
+                request.document_metadata.user,
+                request.document_metadata.id
+        ):
+            raise RuntimeError("Document does not exist")
+
+        await self.table_store.update_document(request.document_metadata)
+
+        print("Update complete", flush=True)
+
+        return LibrarianResponse(
+            error = None,
+            document_metadata = None,
+            content = None,
+            document_metadatas = None,
+            processing_metadatas = None,
+        )
+
+    async def get_document_metadata(self, request):
+
+        print("Get doc...")
+
+        doc = await self.table_store.get_document(
+            request.user,
+            request.document_id
+        )
+
+        print("Get complete", flush=True)
+
+        return LibrarianResponse(
+            error = None,
+            document_metadata = doc,
+            content = None,
+            document_metadatas = None,
+            processing_metadatas = None,
+        )
+
+    async def get_document_content(self, request):
+
+        print("Get doc content...")
+
+        object_id = await self.table_store.get_document_object_id(
+            request.user,
+            request.document_id
+        )
+
+        content = await self.blob_store.get(
+            object_id
+        )
+
+        print("Get complete", flush=True)
+
+        return LibrarianResponse(
+            error = None,
+            document_metadata = None,
+            content = base64.b64encode(content),
+            document_metadatas = None,
+            processing_metadatas = None,
+        )
+
+    async def add_processing(self, request):
+
+        print("Add processing")
+
+        if await self.table_store.processing_exists(
+                request.processing_metadata.user,
+                request.processing_metadata.id
+        ):
+            raise RuntimeError("Processing already exists")
+
+        doc = await self.table_store.get_document(
+            request.processing_metadata.user,
+            request.processing_metadata.document_id
+        )
+
+        object_id = await self.table_store.get_document_object_id(
+            request.processing_metadata.user,
+            request.processing_metadata.document_id
+        )
+
+        content = await self.blob_store.get(
+            object_id
+        )
+
+        print("Got content")
+
+        print("Add processing...")
+
+        await self.table_store.add_processing(request.processing_metadata)
+
+        print("Add complete", flush=True)
+
+        return LibrarianResponse(
+            error = None,
+            document_metadata = None,
+            content = None,
+            document_metadatas = None,
+            processing_metadatas = None,
+        )
+        
+
+        # if document.kind == "application/pdf":
+        #     await self.load_document(document)
+        # elif document.kind == "text/plain":
+        #     await self.load_text(document)
+
+    async def remove_processing(self, request):
+
+        print("Removing processing...")
+
+        if not await self.table_store.processing_exists(
+                request.user,
+                request.processing_id,
+        ):
+            raise RuntimeError("Processing object does not exist")
+
+        # Remove doc table row
+        await self.table_store.remove_processing(
+            request.user,
+            request.processing_id
+        )
+
+        print("Remove complete", flush=True)
+
+        return LibrarianResponse(
+            error = None,
+            document_metadata = None,
+            content = None,
+            document_metadatas = None,
+            processing_metadatas = None,
+        )
+
+    async def list_documents(self, request):
+
+        docs = await self.table_store.list_documents(request.user)
+
+        return LibrarianResponse(
+            error = None,
+            document_metadata = None,
+            content = None,
+            document_metadatas = docs,
+            processing_metadatas = None,
+        )
+
+    async def list_processing(self, request):
+
+        procs = await self.table_store.list_processing(request.user)
+
+        return LibrarianResponse(
+            error = None,
+            document_metadata = None,
+            content = None,
+            document_metadatas = None,
+            processing_metadatas = procs,
+        )
+