trustgraph-base 0.23.10__tar.gz → 0.23.12__tar.gz

{trustgraph-base-0.23.10 → trustgraph-base-0.23.12}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.23.10
+Version: 0.23.12
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.10.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.12.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

trustgraph-base-0.23.12/trustgraph/api/api.py

@@ -0,0 +1,69 @@
+
+import requests
+import json
+import base64
+import time
+
+from . library import Library
+from . flow import Flow
+from . config import Config
+from . exceptions import *
+from . types import *
+
+def check_error(response):
+
+    if "error" in response:
+
+        try:
+            msg = response["error"]["message"]
+            tp = response["error"]["type"]
+        except:
+            raise ApplicationException(response["error"])
+
+        raise ApplicationException(f"{tp}: {msg}")
+
+class Api:
+
+    def __init__(self, url="http://localhost:8088/"):
+
+        self.url = url
+
+        if not url.endswith("/"):
+            self.url += "/"
+
+        self.url += "api/v1/"
+
+    def flow(self):
+        return Flow(api=self)
+
+    def config(self):
+        return Config(api=self)
+
+    def request(self, path, request):
+
+        url = f"{self.url}{path}"
+
+#        print("uri:", url)
+#        print(json.dumps(request, indent=4))
+
+        # Invoke the API, input is passed as JSON
+        resp = requests.post(url, json=request)
+
+        # Should be a 200 status code
+        if resp.status_code != 200:
+            raise ProtocolException(f"Status code {resp.status_code}")
+
+#        print(resp.text)
+
+        try:
+            # Parse the response as JSON
+            object = resp.json()
+        except:
+            raise ProtocolException(f"Expected JSON response")
+
+        check_error(object)
+
+        return object
+
+    def library(self):
+        return Library(self)

trustgraph-base-0.23.12/trustgraph/api/config.py

@@ -0,0 +1,97 @@
+
+from . exceptions import *
+from . types import ConfigValue
+
+class Config:
+
+    def __init__(self, api):
+        self.api = api
+
+    def request(self, request):
+        return self.api.request("config", request)
+
+    def get(self, keys):
+
+        # The input consists of system and prompt strings
+        input = {
+            "operation": "get",
+            "keys": [
+                { "type": k.type, "key": k.key }
+                for k in keys
+            ]
+        }
+
+        object = self.request(input)
+
+        try:
+            return [
+                ConfigValue(
+                    type = v["type"],
+                    key = v["key"],
+                    value = v["value"]
+                )
+                for v in object["values"]
+            ]
+        except Exception as e:
+            print(e)
+            raise ProtocolException("Response not formatted correctly")
+
+    def put(self, values):
+
+        # The input consists of system and prompt strings
+        input = {
+            "operation": "put",
+            "values": [
+                { "type": v.type, "key": v.key, "value": v.value }
+                for v in values
+            ]
+        }
+
+        self.request(input)
+
+    def list(self, type):
+
+        # The input consists of system and prompt strings
+        input = {
+            "operation": "list",
+            "type": type,
+        }
+
+        return self.request(input)["directory"]
+
+    def get_values(self, type):
+
+        # The input consists of system and prompt strings
+        input = {
+            "operation": "getvalues",
+            "type": type,
+        }
+
+        object = self.request(input)["directory"]
+
+        try:
+            return [
+                ConfigValue(
+                    type = v["type"],
+                    key = v["key"],
+                    value = v["value"]
+                )
+                for v in object["values"]
+            ]
+        except:
+            raise ProtocolException(f"Response not formatted correctly")
+
+    def all(self):
+
+        # The input consists of system and prompt strings
+        input = {
+            "operation": "config"
+        }
+
+        object = self.request(input)
+
+        try:
+            return object["config"], object["version"]
+        except:
+            raise ProtocolException(f"Response not formatted correctly")
+

trustgraph-base-0.23.12/trustgraph/api/exceptions.py

@@ -0,0 +1,6 @@
+
+class ProtocolException(Exception):
+    pass
+
+class ApplicationException(Exception):
+    pass

trustgraph-base-0.23.10/trustgraph/api/api.py → trustgraph-base-0.23.12/trustgraph/api/flow.py

@@ -1,175 +1,41 @@
 
-import requests
 import json
-import dataclasses
 import base64
 
-from trustgraph.knowledge import hash, Uri, Literal
+from .. knowledge import hash, Uri, Literal
 
-class ProtocolException(Exception):
-    pass
+def to_value(x):
+    if x["e"]: return Uri(x["v"])
+    return Literal(x["v"])
 
-class ApplicationException(Exception):
-    pass
-
-@dataclasses.dataclass
-class Triple:
-    s : str
-    p : str
-    o : str
-
-@dataclasses.dataclass
-class ConfigKey:
-    type : str
-    key : str
-
-@dataclasses.dataclass
-class ConfigValue:
-    type : str
-    key : str
-    value : str
-
-def check_error(response):
-
-    if "error" in response:
-
-        try:
-            msg = response["error"]["message"]
-            tp = response["error"]["type"]
-        except:
-            raise ApplicationException(response["error"])
-
-        raise ApplicationException(f"{tp}: {msg}")
-
-class Api:
-
-    def __init__(self, url="http://localhost:8088/"):
-
-        self.url = url
-
-        if not url.endswith("/"):
-            self.url += "/"
-
-        self.url += "api/v1/"
-
-    def flow(self, flow="0000"):
-        return Flow(api=self, flow=flow)
-
-    def request(self, path, request):
-
-        url = f"{self.url}{path}"
-
-        # Invoke the API, input is passed as JSON
-        resp = requests.post(url, json=request)
-
-        # Should be a 200 status code
-        if resp.status_code != 200:
-            raise ProtocolException(f"Status code {resp.status_code}")
-
-        try:
-            # Parse the response as JSON
-            object = resp.json()
-        except:
-            raise ProtocolException(f"Expected JSON response")
-
-        check_error(object)
-
-        return object
-
-    def config_all(self):
-
-        # The input consists of system and prompt strings
-        input = {
-            "operation": "config"
-        }
-
-        object = self.request("config", input)
-
-        try:
-            return object["config"], object["version"]
-        except:
-            raise ProtocolException(f"Response not formatted correctly")
-
-    def config_get(self, keys):
-
-        # The input consists of system and prompt strings
-        input = {
-            "operation": "get",
-            "keys": [
-                { "type": k.type, "key": k.key }
-                for k in keys
-            ]
-        }
-
-        object = self.request("config", input)
-
-        try:
-            return [
-                ConfigValue(
-                    type = v["type"],
-                    key = v["key"],
-                    value = v["value"]
-                )
-                for v in object["values"]
-            ]
-        except:
-            raise ProtocolException(f"Response not formatted correctly")
-
-    def config_put(self, values):
-
-        # The input consists of system and prompt strings
-        input = {
-            "operation": "put",
-            "values": [
-                { "type": v.type, "key": v.key, "value": v.value }
-                for v in values
-            ]
-        }
-
-        self.request("config", input)
-
-    def config_list(self, type):
-
-        # The input consists of system and prompt strings
-        input = {
-            "operation": "list",
-            "type": type,
-        }
+class Flow:
 
-        return self.request("config", input)["directory"]
+    def __init__(self, api):
+        self.api = api
 
-    def config_getvalues(self, type):
+    def request(self, path=None, request=None):
 
-        # The input consists of system and prompt strings
-        input = {
-            "operation": "getvalues",
-            "type": type,
-        }
+        if request is None:
+            raise RuntimeError("request must be specified")
 
-        object = self.request("config", input)["directory"]
+        if path:
+            return self.api.request(f"flow/{path}", request)
+        else:
+            return self.api.request(f"flow", request)
 
-        try:
-            return [
-                ConfigValue(
-                    type = v["type"],
-                    key = v["key"],
-                    value = v["value"]
-                )
-                for v in object["values"]
-            ]
-        except:
-            raise ProtocolException(f"Response not formatted correctly")
+    def id(self, id="0000"):
+        return FlowInstance(api=self, id=id)
 
-    def flow_list_classes(self):
+    def list_classes(self):
 
         # The input consists of system and prompt strings
         input = {
             "operation": "list-classes",
         }
 
-        return self.request("flow", input)["class-names"]
+        return self.request(request = input)["class-names"]
 
-    def flow_get_class(self, class_name):
+    def get_class(self, class_name):
 
         # The input consists of system and prompt strings
         input = {
@@ -177,9 +43,9 @@ class Api:
             "class-name": class_name,
         }
 
-        return json.loads(self.request("flow", input)["class-definition"])
+        return json.loads(self.request(request = input)["class-definition"])
 
-    def flow_put_class(self, class_name, definition):
+    def put_class(self, class_name, definition):
 
         # The input consists of system and prompt strings
         input = {
@@ -188,9 +54,9 @@ class Api:
             "class-definition": json.dumps(definition),
         }
 
-        self.request("flow", input)
+        self.request(request = input)
 
-    def flow_delete_class(self, class_name):
+    def delete_class(self, class_name):
 
         # The input consists of system and prompt strings
         input = {
@@ -198,18 +64,18 @@ class Api:
             "class-name": class_name,
         }
 
-        self.request("flow", input)
+        self.request(request = input)
 
-    def flow_list(self):
+    def list(self):
 
         # The input consists of system and prompt strings
         input = {
             "operation": "list-flows",
         }
 
-        return self.request("flow", input)["flow-ids"]
+        return self.request(request = input)["flow-ids"]
 
-    def flow_get(self, id):
+    def get(self, id):
 
         # The input consists of system and prompt strings
         input = {
@@ -217,9 +83,9 @@ class Api:
             "flow-id": id,
         }
 
-        return json.loads(self.request("flow", input)["flow"])
+        return json.loads(self.request(request = input)["flow"])
 
-    def flow_start(self, class_name, id, description):
+    def start(self, class_name, id, description):
 
         # The input consists of system and prompt strings
         input = {
@@ -229,9 +95,9 @@ class Api:
             "description": description,
         }
 
-        self.request("flow", input)
+        self.request(request = input)
 
-    def flow_stop(self, id):
+    def stop(self, id):
 
         # The input consists of system and prompt strings
         input = {
@@ -239,13 +105,17 @@ class Api:
             "flow-id": id,
         }
 
-        self.request("flow", input)
-
-class Flow:
+        self.request(request = input)
+        
+class FlowInstance:
 
-    def __init__(self, api, flow):
+    def __init__(self, api, id):
         self.api = api
-        self.flow = flow
+        self.id = id
+
+    def request(self, path, request):
+
+        return self.api.request(path = f"{self.id}/{path}", request = request)
 
     def text_completion(self, system, prompt):
 
@@ -255,8 +125,8 @@ class Flow:
             "prompt": prompt
         }
 
-        return self.api.request(
-            f"flow/{self.flow}/service/text-completion",
+        return self.request(
+            "service/text-completion",
             input
         )["response"]
 
@@ -267,8 +137,8 @@ class Flow:
             "question": question
         }
 
-        return self.api.request(
-            f"flow/{self.flow}/service/agent",
+        return self.request(
+            "service/agent",
             input
         )["answer"]
 
@@ -289,8 +159,8 @@ class Flow:
             "max-path-length": max_path_length,
         }
 
-        return self.api.request(
-            f"flow/{self.flow}/service/graph-rag",
+        return self.request(
+            "service/graph-rag",
             input
         )["response"]
 
@@ -307,8 +177,8 @@ class Flow:
             "doc-limit": doc_limit,
         }
 
-        return self.api.request(
-            f"flow/{self.flow}/service/document-rag",
+        return self.request(
+            "service/document-rag",
             input
         )["response"]
 
@@ -319,8 +189,8 @@ class Flow:
             "text": text
         }
 
-        return self.api.request(
-            f"flow/{self.flow}/service/embeddings",
+        return self.request(
+            "service/embeddings",
             input
         )["vectors"]
 
@@ -332,8 +202,8 @@ class Flow:
             "variables": variables
         }
 
-        object = self.api.request(
-            f"flow/{self.flow}/service/prompt",
+        object = self.request(
+            "service/prompt",
             input
         )
 
@@ -381,15 +251,11 @@ class Flow:
                 raise RuntimeError("o must be Uri or Literal")
             input["o"] = { "v": str(o), "e": isinstance(o, Uri), }
 
-        object = self.api.request(
-            f"flow/{self.flow}/service/triples",
+        object = self.request(
+            "service/triples",
             input
         )
 
-        def to_value(x):
-            if x["e"]: return Uri(x["v"])
-            return Literal(x["v"])
-            
         return [
             Triple(
                 s=to_value(t["s"]),
@@ -441,8 +307,8 @@ class Flow:
         if collection:
             input["collection"] = collection
 
-        return self.api.request(
-            f"flow/{self.flow}/service/document-load",
+        return self.request(
+            "service/document-load",
             input
         )
 
@@ -486,8 +352,8 @@ class Flow:
         if collection:
             input["collection"] = collection
 
-        return self.api.request(
-            f"flow/{self.flow}/service/text-load",
+        return self.request(
+            "service/text-load",
             input
         )
 

trustgraph-base-0.23.12/trustgraph/api/library.py

@@ -0,0 +1,259 @@
+
+import datetime
+import time
+import base64
+
+from . types import DocumentMetadata, ProcessingMetadata, Triple
+from .. knowledge import hash, Uri, Literal
+from . exceptions import *
+
+def to_value(x):
+    if x["e"]: return Uri(x["v"])
+    return Literal(x["v"])
+
+class Library:
+
+    def __init__(self, api):
+        self.api = api
+
+    def request(self, request):
+        return self.api.request(f"librarian", request)
+
+    def add_document(
+            self, document, id, metadata, user, title, comments,
+            kind="text/plain", tags=[], 
+    ):
+
+        if id is None:
+
+            if metadata is not None:
+
+                # Situation makes no sense.  What can the metadata possibly
+                # mean if the caller doesn't know the document ID.
+                # Metadata should relate to the document by ID
+                raise RuntimeError("Can't specify metadata without id")
+
+            id = hash(document)
+
+        if not title: title = ""
+        if not comments: comments = ""
+
+        triples = []
+
+        def emit(t):
+            triples.append(t)
+
+        if metadata:
+            metadata.emit(
+                lambda t: triples.append({
+                    "s": { "v": t["s"], "e": isinstance(t["s"], Uri) },
+                    "p": { "v": t["p"], "e": isinstance(t["p"], Uri) },
+                    "o": { "v": t["o"], "e": isinstance(t["o"], Uri) }
+                })
+            )
+
+        input = {
+            "operation": "add-document",
+            "document-metadata": {
+                "id": id,
+                "time": int(time.time()),
+                "kind": kind,
+                "title": title,
+                "comments": comments,
+                "metadata": triples,
+                "user": user,
+                "tags": tags
+            },
+            "content": base64.b64encode(document).decode("utf-8"),
+        }
+
+        return self.request(input)
+
+    def get_documents(self, user):
+
+        input = {
+            "operation": "list-documents",
+            "user": user,
+        }
+
+        object = self.request(input)
+
+        try:
+            return [
+                DocumentMetadata(
+                    id = v["id"],
+                    time = datetime.datetime.fromtimestamp(v["time"]),
+                    kind = v["kind"],
+                    title = v["title"],
+                    comments = v.get("comments", ""),
+                    metadata = [
+                        Triple(
+                            s = to_value(w["s"]),
+                            p = to_value(w["p"]),
+                            o = to_value(w["o"])
+                        )
+                        for w in v["metadata"]
+                    ],
+                    user = v["user"],
+                    tags = v["tags"]
+                )
+                for v in object["document-metadatas"]
+            ]
+        except Exception as e:
+            print(e)
+            raise ProtocolException(f"Response not formatted correctly")
+
+    def get_document(self, user, id):
+
+        input = {
+            "operation": "get-document",
+            "user": user,
+            "document-id": id,
+        }
+
+        object = self.request(input)
+        doc = object["document-metadata"]
+
+        try:
+            DocumentMetadata(
+                id = doc["id"],
+                time = datetime.datetime.fromtimestamp(doc["time"]),
+                kind = doc["kind"],
+                title = doc["title"],
+                comments = doc.get("comments", ""),
+                metadata = [
+                    Triple(
+                        s = to_value(w["s"]),
+                        p = to_value(w["p"]),
+                        o = to_value(w["o"])
+                    )
+                    for w in doc["metadata"]
+                ],
+                user = doc["user"],
+                tags = doc["tags"]
+            )
+        except Exception as e:
+            print(e)
+            raise ProtocolException(f"Response not formatted correctly")
+
+    def update_document(self, user, id, metadata):
+
+        input = {
+            "operation": "update-document",
+            "document-metadata": {
+                "user": user,
+                "document-id": id,
+                "time": metadata.time,
+                "title": metadata.title,
+                "comments": metadata.comments,
+                "metadata": [
+                    {
+                        "s": { "v": t["s"], "e": isinstance(t["s"], Uri) },
+                        "p": { "v": t["p"], "e": isinstance(t["p"], Uri) },
+                        "o": { "v": t["o"], "e": isinstance(t["o"], Uri) }
+                    }
+                    for t in metadata.metadata
+                ],
+                "tags": metadata.tags,
+            }
+        }
+
+        object = self.request(input)
+        doc = object["document-metadata"]
+
+        try:
+            DocumentMetadata(
+                id = doc["id"],
+                time = datetime.datetime.fromtimestamp(doc["time"]),
+                kind = doc["kind"],
+                title = doc["title"],
+                comments = doc.get("comments", ""),
+                metadata = [
+                    Triple(
+                        s = to_value(w["s"]),
+                        p = to_value(w["p"]),
+                        o = to_value(w["o"])
+                    )
+                    for w in doc["metadata"]
+                ],
+                user = doc["user"],
+                tags = doc["tags"]
+            )
+        except Exception as e:
+            print(e)
+            raise ProtocolException(f"Response not formatted correctly")
+
+    def remove_document(self, user, id):
+
+        input = {
+            "operation": "remove-document",
+            "user": user,
+            "document-id": id,
+        }
+
+        object = self.request(input)
+
+        return {}
+
+    def start_processing(
+            self, id, document_id, flow="0000",
+            user="trustgraph", collection="default", tags=[],
+    ):
+
+        input = {
+            "operation": "add-processing",
+            "processing-metadata": {
+                "id": id,
+                "document-id": document_id,
+                "time": int(time.time()),
+                "flow": flow,
+                "user": user,
+                "collection": collection,
+                "tags": tags,
+            }
+        }
+
+        object = self.request(input)
+
+        return {}
+
+    def stop_processing(
+            self, id, user="trustgraph", 
+    ):
+
+        input = {
+            "operation": "remove-processing",
+            "processing-id": id,
+            "user": user,
+        }
+
+        object = self.request(input)
+
+        return {}
+
+    def get_processings(self, user="trustgraph"):
+
+        input = {
+            "operation": "list-processing",
+            "user": user,
+        }
+
+        object = self.request(input)
+
+        try:
+            return [
+                ProcessingMetadata(
+                    id = v["id"],
+                    document_id = v["document-id"],
+                    time = datetime.datetime.fromtimestamp(v["time"]),
+                    flow = v["flow"],
+                    user = v["user"],
+                    collection = v["collection"],
+                    tags = v["tags"],
+                )
+                for v in object["processing-metadatas"]
+            ]
+        except Exception as e:
+            print(e)
+            raise ProtocolException(f"Response not formatted correctly")
+

trustgraph-base-0.23.12/trustgraph/api/types.py

@@ -0,0 +1,42 @@
+
+import dataclasses
+import datetime
+from typing import List
+
+@dataclasses.dataclass
+class Triple:
+    s : str
+    p : str
+    o : str
+
+@dataclasses.dataclass
+class ConfigKey:
+    type : str
+    key : str
+
+@dataclasses.dataclass
+class ConfigValue:
+    type : str
+    key : str
+    value : str
+
+@dataclasses.dataclass
+class DocumentMetadata:
+    id : str
+    time : datetime.datetime
+    kind : str
+    title : str
+    comments : str
+    metadata : List[Triple]
+    user : str
+    tags : List[str]
+
+@dataclasses.dataclass
+class ProcessingMetadata:
+    id : str
+    document_id : str
+    time : datetime.datetime
+    flow : str
+    user : str
+    collection : str
+    tags : List[str]

{trustgraph-base-0.23.10 → trustgraph-base-0.23.12}/trustgraph/base/producer.py

@@ -4,7 +4,9 @@ import asyncio
 
 class Producer:
 
-    def __init__(self, client, topic, schema, metrics=None):
+    def __init__(self, client, topic, schema, metrics=None,
+                 chunking_enabled=True):
+
         self.client = client
         self.topic = topic
         self.schema = schema
@@ -14,6 +16,8 @@ class Producer:
         self.running = True
         self.producer = None
 
+        self.chunking_enabled = chunking_enabled
+
     def __del__(self):
 
         self.running = False
@@ -38,7 +42,8 @@ class Producer:
                 print("Connect publisher to", self.topic, "...", flush=True)
                 self.producer = self.client.create_producer(
                     topic = self.topic,
-                    schema = JsonSchema(self.schema)
+                    schema = JsonSchema(self.schema),
+                    chunking_enabled = self.chunking_enabled,
                 )
                 print("Connected to", self.topic, flush=True)
             except Exception as e:

{trustgraph-base-0.23.10 → trustgraph-base-0.23.12}/trustgraph/base/publisher.py

@@ -37,6 +37,7 @@ class Publisher:
         while self.running:
 
             try:
+
                 producer = self.client.create_producer(
                     topic=self.topic,
                     schema=JsonSchema(self.schema),

trustgraph-base-0.23.12/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "0.23.12"

{trustgraph-base-0.23.10 → trustgraph-base-0.23.12}/trustgraph/schema/library.py

@@ -125,10 +125,13 @@ class LibrarianResponse(Record):
     document_metadatas = Array(DocumentMetadata())
     processing_metadatas = Array(ProcessingMetadata())
 
+# FIXME: Is this right?  Using persistence on librarian so that
+# message chunking works
+
 librarian_request_queue = topic(
-    'librarian', kind='non-persistent', namespace='request'
+    'librarian', kind='persistent', namespace='request'
 )
 librarian_response_queue = topic(
-    'librarian', kind='non-persistent', namespace='response',
+    'librarian', kind='persistent', namespace='response',
 )
 

{trustgraph-base-0.23.10 → trustgraph-base-0.23.12}/trustgraph_base.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph-base
-Version: 0.23.10
+Version: 0.23.12
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.10.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.23.12.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3

{trustgraph-base-0.23.10 → trustgraph-base-0.23.12}/trustgraph_base.egg-info/SOURCES.txt

@@ -6,6 +6,11 @@ trustgraph/log_level.py
 trustgraph/rdf.py
 trustgraph/api/__init__.py
 trustgraph/api/api.py
+trustgraph/api/config.py
+trustgraph/api/exceptions.py
+trustgraph/api/flow.py
+trustgraph/api/library.py
+trustgraph/api/types.py
 trustgraph/base/__init__.py
 trustgraph/base/agent_client.py
 trustgraph/base/agent_service.py

trustgraph-base-0.23.10/trustgraph/base_version.py

@@ -1 +0,0 @@
-__version__ = "0.23.10"