trustgraph 0.5.1__tar.gz → 0.5.2__tar.gz

{trustgraph-0.5.1 → trustgraph-0.5.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph
-Version: 0.5.1
+Version: 0.5.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.5.1.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.5.2.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3
@@ -32,6 +32,7 @@ Requires-Dist: anthropic
 Requires-Dist: google-cloud-aiplatform
 Requires-Dist: pyyaml
 Requires-Dist: prometheus-client
+Requires-Dist: pyarrow
 
 
 # TrustGraph
@@ -95,11 +96,13 @@ package installed can also run the entire architecture.
   chunking algorithm to produce smaller text chunks.
 - `embeddings-hf` - A service which analyses text and returns a vector
   embedding using one of the HuggingFace embeddings models.
+- `embeddings-ollama` - A service which analyses text and returns a vector
+  embedding using an Ollama embeddings model.
 - `embeddings-vectorize` - Uses an embeddings service to get a vector
   embedding which is added to the processor payload.
 - `graph-rag` - A query service which applies a Graph RAG algorithm to
   provide a response to a text prompt.
-- `graph-write-cassandra` - Takes knowledge graph edges and writes them to
+- `triples-write-cassandra` - Takes knowledge graph edges and writes them to
   a Cassandra store.
 - `kg-extract-definitions` - knowledge extractor - examines text and
   produces graph edges.
@@ -115,15 +118,15 @@ package installed can also run the entire architecture.
   format.  For instance, the wrapping of text between lines in a PDF document
   is not semantically encoded, so the decoder will see wrapped lines as
   space-separated.
-- `vector-write-milvus` - Takes vector-entity mappings and records them
+- `ge-write-milvus` - Takes graph embeddings mappings and records them
   in the vector embeddings store.
 
 ## LM Specific Modules
 
-- `llm-azure-text` - Sends request to AzureAI serverless endpoint
-- `llm-claude-text` - Sends request to Anthropic's API
-- `llm-ollama-text` -  Sends request to LM running using Ollama
-- `llm-vertexai-text` -  Sends request to model available through VertexAI API
+- `text-completion-azure` - Sends request to AzureAI serverless endpoint
+- `text-completion-claude` - Sends request to Anthropic's API
+- `text-completion-ollama` -  Sends request to LM running using Ollama
+- `text-completion-vertexai` -  Sends request to model available through VertexAI API
 
 ## Quickstart Guide
 

{trustgraph-0.5.1 → trustgraph-0.5.2}/README.md

@@ -60,11 +60,13 @@ package installed can also run the entire architecture.
   chunking algorithm to produce smaller text chunks.
 - `embeddings-hf` - A service which analyses text and returns a vector
   embedding using one of the HuggingFace embeddings models.
+- `embeddings-ollama` - A service which analyses text and returns a vector
+  embedding using an Ollama embeddings model.
 - `embeddings-vectorize` - Uses an embeddings service to get a vector
   embedding which is added to the processor payload.
 - `graph-rag` - A query service which applies a Graph RAG algorithm to
   provide a response to a text prompt.
-- `graph-write-cassandra` - Takes knowledge graph edges and writes them to
+- `triples-write-cassandra` - Takes knowledge graph edges and writes them to
   a Cassandra store.
 - `kg-extract-definitions` - knowledge extractor - examines text and
   produces graph edges.
@@ -80,15 +82,15 @@ package installed can also run the entire architecture.
   format.  For instance, the wrapping of text between lines in a PDF document
   is not semantically encoded, so the decoder will see wrapped lines as
   space-separated.
-- `vector-write-milvus` - Takes vector-entity mappings and records them
+- `ge-write-milvus` - Takes graph embeddings mappings and records them
   in the vector embeddings store.
 
 ## LM Specific Modules
 
-- `llm-azure-text` - Sends request to AzureAI serverless endpoint
-- `llm-claude-text` - Sends request to Anthropic's API
-- `llm-ollama-text` -  Sends request to LM running using Ollama
-- `llm-vertexai-text` -  Sends request to model available through VertexAI API
+- `text-completion-azure` - Sends request to AzureAI serverless endpoint
+- `text-completion-claude` - Sends request to Anthropic's API
+- `text-completion-ollama` -  Sends request to LM running using Ollama
+- `text-completion-vertexai` -  Sends request to model available through VertexAI API
 
 ## Quickstart Guide
 

trustgraph-0.5.2/scripts/dump-parquet

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+import pyarrow.parquet as pq
+import sys
+
+for file in sys.argv[1:]:
+
+    table = pq.read_table(file).to_pandas()
+    print(table)
+
+
+

trustgraph-0.5.2/scripts/triples-dump-parquet

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.dump.triples.parquet import run
+
+run()
+

{trustgraph-0.5.1 → trustgraph-0.5.2}/setup.py

@@ -4,7 +4,7 @@ import os
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
-version = "0.5.1"
+version = "0.5.2"
 
 setuptools.setup(
     name="trustgraph",
@@ -44,6 +44,7 @@ setuptools.setup(
         "google-cloud-aiplatform",
         "pyyaml",
         "prometheus-client",
+        "pyarrow",
     ],
     scripts=[
         "scripts/chunker-recursive",
@@ -66,5 +67,7 @@ setuptools.setup(
         "scripts/text-completion-ollama",
         "scripts/text-completion-vertexai",
         "scripts/triples-write-cassandra",
+        "scripts/dump-parquet",
+        "scripts/triples-dump-parquet",
     ]
 )

trustgraph-0.5.2/trustgraph/dump/triples/parquet/processor.py

@@ -0,0 +1,87 @@
+
+"""
+Write graphs triples to parquet files in a directory.
+"""
+
+import pulsar
+import base64
+import os
+import argparse
+import time
+
+from .... trustgraph import TrustGraph
+from .... schema import Triple
+from .... schema import triples_store_queue
+from .... log_level import LogLevel
+from .... base import Consumer
+
+from . writer import ParquetWriter
+
+module = ".".join(__name__.split(".")[1:-1])
+
+default_input_queue = triples_store_queue
+default_subscriber = module
+default_graph_host='localhost'
+default_directory = "."
+default_file_template = "triples-{id}.parquet"
+default_rotation_time = 60
+        
+class Processor(Consumer):
+
+    def __init__(self, **params):
+        
+        input_queue = params.get("input_queue", default_input_queue)
+        subscriber = params.get("subscriber", default_subscriber)
+        directory = params.get("directory", default_directory)
+        file_template = params.get("file_template", default_file_template)
+        rotation_time = params.get("rotation_time", default_rotation_time)
+
+        super(Processor, self).__init__(
+            **params | {
+                "input_queue": input_queue,
+                "subscriber": subscriber,
+                "input_schema": Triple,
+            }
+        )
+
+        self.writer = ParquetWriter(directory, file_template, rotation_time)
+
+    def __del__(self):
+        if hasattr(self, "writer"):
+            del self.writer
+
+    def handle(self, msg):
+
+        v = msg.value()
+        self.writer.write(v.s.value, v.p.value, v.o.value)
+
+    @staticmethod
+    def add_args(parser):
+
+        Consumer.add_args(
+            parser, default_input_queue, default_subscriber,
+        )
+
+        parser.add_argument(
+            '-d', '--directory',
+            default=default_directory,
+            help=f'Directory to write to (default: {default_directory})'
+        )
+
+        parser.add_argument(
+            '-f', '--file-template',
+            default=default_file_template,
+            help=f'Directory to write to (default: {default_file_template})'
+        )
+
+        parser.add_argument(
+            '-t', '--rotation-time',
+            type=int,
+            default=default_rotation_time,
+            help=f'Rotation time / seconds (default: {default_rotation_time})'
+        )
+
+def run():
+
+    Processor.start(module, __doc__)
+

trustgraph-0.5.2/trustgraph/dump/triples/parquet/writer.py

@@ -0,0 +1,96 @@
+
+import threading
+import queue
+import time
+import uuid
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+class ParquetWriter:
+
+    def __init__(self, directory, file_template, rotation_time):
+        self.directory = directory
+        self.file_template = file_template
+        self.rotation_time = rotation_time
+
+        self.q = queue.Queue()
+
+        self.running = True
+
+        self.thread = threading.Thread(target=(self.writer_thread))
+        self.thread.start()
+
+    def writer_thread(self):
+
+        triples = []
+
+        timeout = None
+
+        while self.running:
+
+            try:
+
+                item = self.q.get(timeout=1)
+
+                if timeout == None:
+                    timeout = time.time() + self.rotation_time
+
+                triples.append(item)
+
+            except queue.Empty:
+                pass
+
+            if timeout:
+                if time.time() > timeout:
+
+                    self.write_file(triples)
+                    timeout = None
+                    triples = []
+
+    def write_file(self, triples):
+
+        try:
+
+            schema = pa.schema([
+                pa.field('s', pa.string()),
+                pa.field('p', pa.string()),
+                pa.field('o', pa.string()),
+            ])
+
+            fname = self.file_template.format(id=str(uuid.uuid4()))
+            path = f"{self.directory}/{fname}"
+
+            writer = pq.ParquetWriter(path, schema)
+
+            batch = pa.record_batch(
+                [
+                    [tpl[0] for tpl in triples],
+                    [tpl[1] for tpl in triples],
+                    [tpl[2] for tpl in triples],
+                ],
+                names=['s', 'p', 'o']
+            )
+
+            writer.write_batch(batch)
+
+            writer.close()
+
+            print(f"Wrote {path}.")
+
+        except Exception as e:
+
+            print("Parquet write:", e)
+
+    def write(self, s, p, o):
+        self.q.put((s, p, o))
+
+    def __del__(self):
+
+        self.running = False
+
+        if hasattr(self, "q"):
+            self.thread.join()
+
+            
+
+        

trustgraph-0.5.2/trustgraph/embeddings/ollama/__init__.py

@@ -0,0 +1,3 @@
+
+from . processor import *
+

{trustgraph-0.5.1 → trustgraph-0.5.2}/trustgraph/embeddings/vectorize/vectorize.py

@@ -34,7 +34,10 @@ class Processor(ConsumerProducer):
             }
         )
 
-        self.embeddings = EmbeddingsClient(pulsar_host=self.pulsar_host)
+        self.embeddings = EmbeddingsClient(
+            pulsar_host=self.pulsar_host,
+            subscriber=module + "emb",
+        )
 
     def emit(self, source, chunk, vectors):
 

{trustgraph-0.5.1 → trustgraph-0.5.2}/trustgraph/embeddings_client.py

@@ -17,14 +17,14 @@ DEBUG=_pulsar.LoggerLevel.Debug
 class EmbeddingsClient:
 
     def __init__(
-            self, log_level=ERROR, client_id=None,
+            self, log_level=ERROR, subscriber=None,
             pulsar_host="pulsar://pulsar:6650",
     ):
 
         self.client = None
 
-        if client_id == None:
-            client_id = str(uuid.uuid4())
+        if subscriber == None:
+            subscriber = str(uuid.uuid4())
 
         self.client = pulsar.Client(
             pulsar_host,
@@ -38,7 +38,7 @@ class EmbeddingsClient:
         )
 
         self.consumer = self.client.subscribe(
-            embeddings_response_queue, client_id,
+            embeddings_response_queue, subscriber,
             schema=JsonSchema(EmbeddingsResponse),
         )
 
@@ -67,6 +67,13 @@ class EmbeddingsClient:
 
     def __del__(self):
 
-        if self.client:
-            self.client.close()
+        if hasattr(self, "consumer"):
+#             self.consumer.unsubscribe()
+            self.consumer.close()
+            
+        if hasattr(self, "producer"):
+            self.producer.flush()
+            self.producer.close()
+            
+        self.client.close()
 

{trustgraph-0.5.1 → trustgraph-0.5.2}/trustgraph/graph_rag.py

@@ -19,6 +19,7 @@ class GraphRag:
             entity_limit=50,
             triple_limit=30,
             max_subgraph_size=3000,
+            module="test",
     ):
 
         self.verbose=verbose
@@ -31,7 +32,10 @@ class GraphRag:
 
         self.graph = TrustGraph(graph_hosts)
 
-        self.embeddings = EmbeddingsClient(pulsar_host=pulsar_host)
+        self.embeddings = EmbeddingsClient(
+            pulsar_host=pulsar_host,
+            subscriber=module + "-emb",
+        )
 
         self.vecstore = TripleVectors(vector_store)
 
@@ -41,7 +45,10 @@ class GraphRag:
 
         self.label_cache = {}
 
-        self.llm = LlmClient(pulsar_host=pulsar_host)
+        self.llm = LlmClient(
+            pulsar_host=pulsar_host,
+            subscriber=module + "-llm",
+        )
 
         if self.verbose:
             print("Initialised", flush=True)

{trustgraph-0.5.1 → trustgraph-0.5.2}/trustgraph/graph_rag_client.py

@@ -18,12 +18,12 @@ DEBUG=_pulsar.LoggerLevel.Debug
 class GraphRagClient:
 
     def __init__(
-            self, log_level=ERROR, client_id=None,
+            self, log_level=ERROR, subscriber=None,
             pulsar_host="pulsar://pulsar:6650",
     ):
 
-        if client_id == None:
-            client_id = str(uuid.uuid4())
+        if subscriber == None:
+            subscriber = str(uuid.uuid4())
 
         self.client = pulsar.Client(
             pulsar_host,
@@ -37,7 +37,7 @@ class GraphRagClient:
         )
 
         self.consumer = self.client.subscribe(
-            graph_rag_response_queue, client_id,
+            graph_rag_response_queue, subscriber,
             schema=JsonSchema(GraphRagResponse),
         )
 
@@ -66,5 +66,13 @@ class GraphRagClient:
 
     def __del__(self):
 
+        if hasattr(self, "consumer"):
+#             self.consumer.unsubscribe()
+            self.consumer.close()
+            
+        if hasattr(self, "producer"):
+            self.producer.flush()
+            self.producer.close()
+            
         self.client.close()
 

{trustgraph-0.5.1 → trustgraph-0.5.2}/trustgraph/kg/extract_definitions/extract.py

@@ -41,7 +41,10 @@ class Processor(ConsumerProducer):
             }
         )
 
-        self.llm = LlmClient(pulsar_host=self.pulsar_host)
+        self.llm = LlmClient(
+            pulsar_host=self.pulsar_host,
+            subscriber = module + "-llm",
+        )
 
     def to_uri(self, text):
 

{trustgraph-0.5.1 → trustgraph-0.5.2}/trustgraph/kg/extract_relationships/extract.py

@@ -61,7 +61,10 @@ class Processor(ConsumerProducer):
             "vector_schema": GraphEmbeddings.__name__,
         })
 
-        self.llm = LlmClient(pulsar_host=self.pulsar_host)
+        self.llm = LlmClient(
+            pulsar_host = self.pulsar_host,
+            subscriber = module + "-llm",
+        )
 
     def to_uri(self, text):
 

{trustgraph-0.5.1 → trustgraph-0.5.2}/trustgraph/llm_client.py

@@ -19,12 +19,12 @@ DEBUG=_pulsar.LoggerLevel.Debug
 class LlmClient:
 
     def __init__(
-            self, log_level=ERROR, client_id=None,
+            self, log_level=ERROR, subscriber=None,
             pulsar_host="pulsar://pulsar:6650",
     ):
 
-        if client_id == None:
-            client_id = str(uuid.uuid4())
+        if subscriber == None:
+            subscriber = str(uuid.uuid4())
 
         self.client = pulsar.Client(
             pulsar_host,
@@ -38,7 +38,7 @@ class LlmClient:
         )
 
         self.consumer = self.client.subscribe(
-            text_completion_response_queue, client_id,
+            text_completion_response_queue, subscriber,
             schema=JsonSchema(TextCompletionResponse),
         )
 
@@ -68,7 +68,13 @@ class LlmClient:
 
     def __del__(self):
 
-        self.producer.close()
-        self.consumer.close()
+        if hasattr(self, "consumer"):
+#             self.consumer.unsubscribe()
+            self.consumer.close()
+            
+        if hasattr(self, "producer"):
+            self.producer.flush()
+            self.producer.close()
+            
         self.client.close()
 

trustgraph-0.5.2/trustgraph/storage/triples/cassandra/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+from . write import run
+
+if __name__ == '__main__':
+    run()
+

{trustgraph-0.5.1 → trustgraph-0.5.2}/trustgraph.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph
-Version: 0.5.1
+Version: 0.5.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.5.1.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.5.2.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3
@@ -32,6 +32,7 @@ Requires-Dist: anthropic
 Requires-Dist: google-cloud-aiplatform
 Requires-Dist: pyyaml
 Requires-Dist: prometheus-client
+Requires-Dist: pyarrow
 
 
 # TrustGraph
@@ -95,11 +96,13 @@ package installed can also run the entire architecture.
   chunking algorithm to produce smaller text chunks.
 - `embeddings-hf` - A service which analyses text and returns a vector
   embedding using one of the HuggingFace embeddings models.
+- `embeddings-ollama` - A service which analyses text and returns a vector
+  embedding using an Ollama embeddings model.
 - `embeddings-vectorize` - Uses an embeddings service to get a vector
   embedding which is added to the processor payload.
 - `graph-rag` - A query service which applies a Graph RAG algorithm to
   provide a response to a text prompt.
-- `graph-write-cassandra` - Takes knowledge graph edges and writes them to
+- `triples-write-cassandra` - Takes knowledge graph edges and writes them to
   a Cassandra store.
 - `kg-extract-definitions` - knowledge extractor - examines text and
   produces graph edges.
@@ -115,15 +118,15 @@ package installed can also run the entire architecture.
   format.  For instance, the wrapping of text between lines in a PDF document
   is not semantically encoded, so the decoder will see wrapped lines as
   space-separated.
-- `vector-write-milvus` - Takes vector-entity mappings and records them
+- `ge-write-milvus` - Takes graph embeddings mappings and records them
   in the vector embeddings store.
 
 ## LM Specific Modules
 
-- `llm-azure-text` - Sends request to AzureAI serverless endpoint
-- `llm-claude-text` - Sends request to Anthropic's API
-- `llm-ollama-text` -  Sends request to LM running using Ollama
-- `llm-vertexai-text` -  Sends request to model available through VertexAI API
+- `text-completion-azure` - Sends request to AzureAI serverless endpoint
+- `text-completion-claude` - Sends request to Anthropic's API
+- `text-completion-ollama` -  Sends request to LM running using Ollama
+- `text-completion-vertexai` -  Sends request to model available through VertexAI API
 
 ## Quickstart Guide
 

{trustgraph-0.5.1 → trustgraph-0.5.2}/trustgraph.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ LICENSE
 README.md
 setup.py
 scripts/chunker-recursive
+scripts/dump-parquet
 scripts/embeddings-hf
 scripts/embeddings-ollama
 scripts/embeddings-vectorize
@@ -20,6 +21,7 @@ scripts/text-completion-azure
 scripts/text-completion-claude
 scripts/text-completion-ollama
 scripts/text-completion-vertexai
+scripts/triples-dump-parquet
 scripts/triples-write-cassandra
 trustgraph/__init__.py
 trustgraph/embeddings_client.py
@@ -50,6 +52,12 @@ trustgraph/decoding/__init__.py
 trustgraph/decoding/pdf/__init__.py
 trustgraph/decoding/pdf/__main__.py
 trustgraph/decoding/pdf/pdf_decoder.py
+trustgraph/dump/__init__.py
+trustgraph/dump/triples/__init__.py
+trustgraph/dump/triples/parquet/__init__.py
+trustgraph/dump/triples/parquet/__main__.py
+trustgraph/dump/triples/parquet/processor.py
+trustgraph/dump/triples/parquet/writer.py
 trustgraph/embeddings/__init__.py
 trustgraph/embeddings/hf/__init__.py
 trustgraph/embeddings/hf/__main__.py

{trustgraph-0.5.1 → trustgraph-0.5.2}/trustgraph.egg-info/requires.txt

@@ -18,3 +18,4 @@ anthropic
 google-cloud-aiplatform
 pyyaml
 prometheus-client
+pyarrow