trustgraph 0.4.2__tar.gz → 0.5.2__tar.gz

{trustgraph-0.4.2 → trustgraph-0.5.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: trustgraph
-Version: 0.4.2
+Version: 0.5.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.4.2.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.5.2.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 Classifier: Programming Language :: Python :: 3
@@ -32,6 +32,7 @@ Requires-Dist: anthropic
 Requires-Dist: google-cloud-aiplatform
 Requires-Dist: pyyaml
 Requires-Dist: prometheus-client
+Requires-Dist: pyarrow
 
 
 # TrustGraph
@@ -95,11 +96,13 @@ package installed can also run the entire architecture.
   chunking algorithm to produce smaller text chunks.
 - `embeddings-hf` - A service which analyses text and returns a vector
   embedding using one of the HuggingFace embeddings models.
+- `embeddings-ollama` - A service which analyses text and returns a vector
+  embedding using an Ollama embeddings model.
 - `embeddings-vectorize` - Uses an embeddings service to get a vector
   embedding which is added to the processor payload.
 - `graph-rag` - A query service which applies a Graph RAG algorithm to
   provide a response to a text prompt.
-- `graph-write-cassandra` - Takes knowledge graph edges and writes them to
+- `triples-write-cassandra` - Takes knowledge graph edges and writes them to
   a Cassandra store.
 - `kg-extract-definitions` - knowledge extractor - examines text and
   produces graph edges.
@@ -115,15 +118,15 @@ package installed can also run the entire architecture.
   format.  For instance, the wrapping of text between lines in a PDF document
   is not semantically encoded, so the decoder will see wrapped lines as
   space-separated.
-- `vector-write-milvus` - Takes vector-entity mappings and records them
+- `ge-write-milvus` - Takes graph embeddings mappings and records them
   in the vector embeddings store.
 
 ## LM Specific Modules
 
-- `llm-azure-text` - Sends request to AzureAI serverless endpoint
-- `llm-claude-text` - Sends request to Anthropic's API
-- `llm-ollama-text` -  Sends request to LM running using Ollama
-- `llm-vertexai-text` -  Sends request to model available through VertexAI API
+- `text-completion-azure` - Sends request to AzureAI serverless endpoint
+- `text-completion-claude` - Sends request to Anthropic's API
+- `text-completion-ollama` -  Sends request to LM running using Ollama
+- `text-completion-vertexai` -  Sends request to model available through VertexAI API
 
 ## Quickstart Guide
 

{trustgraph-0.4.2 → trustgraph-0.5.2}/README.md

@@ -60,11 +60,13 @@ package installed can also run the entire architecture.
   chunking algorithm to produce smaller text chunks.
 - `embeddings-hf` - A service which analyses text and returns a vector
   embedding using one of the HuggingFace embeddings models.
+- `embeddings-ollama` - A service which analyses text and returns a vector
+  embedding using an Ollama embeddings model.
 - `embeddings-vectorize` - Uses an embeddings service to get a vector
   embedding which is added to the processor payload.
 - `graph-rag` - A query service which applies a Graph RAG algorithm to
   provide a response to a text prompt.
-- `graph-write-cassandra` - Takes knowledge graph edges and writes them to
+- `triples-write-cassandra` - Takes knowledge graph edges and writes them to
   a Cassandra store.
 - `kg-extract-definitions` - knowledge extractor - examines text and
   produces graph edges.
@@ -80,15 +82,15 @@ package installed can also run the entire architecture.
   format.  For instance, the wrapping of text between lines in a PDF document
   is not semantically encoded, so the decoder will see wrapped lines as
   space-separated.
-- `vector-write-milvus` - Takes vector-entity mappings and records them
+- `ge-write-milvus` - Takes graph embeddings mappings and records them
   in the vector embeddings store.
 
 ## LM Specific Modules
 
-- `llm-azure-text` - Sends request to AzureAI serverless endpoint
-- `llm-claude-text` - Sends request to Anthropic's API
-- `llm-ollama-text` -  Sends request to LM running using Ollama
-- `llm-vertexai-text` -  Sends request to model available through VertexAI API
+- `text-completion-azure` - Sends request to AzureAI serverless endpoint
+- `text-completion-claude` - Sends request to Anthropic's API
+- `text-completion-ollama` -  Sends request to LM running using Ollama
+- `text-completion-vertexai` -  Sends request to model available through VertexAI API
 
 ## Quickstart Guide
 

trustgraph-0.5.2/scripts/chunker-recursive

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.chunking.recursive import run
+
+run()
+

trustgraph-0.5.2/scripts/dump-parquet

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+import pyarrow.parquet as pq
+import sys
+
+for file in sys.argv[1:]:
+
+    table = pq.read_table(file).to_pandas()
+    print(table)
+
+
+

trustgraph-0.5.2/scripts/ge-write-milvus

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.graph_embeddings.milvus import run
+
+run()
+

trustgraph-0.5.2/scripts/graph-rag

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.retrieval.graph_rag import run
+
+run()
+

trustgraph-0.5.2/scripts/pdf-decoder

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.decoding.pdf import run
+
+run()
+

trustgraph-0.5.2/scripts/text-completion-azure

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.azure import run
+
+run()
+

trustgraph-0.5.2/scripts/text-completion-claude

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.claude import run
+
+run()
+

trustgraph-0.5.2/scripts/text-completion-ollama

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.ollama import run
+
+run()
+

trustgraph-0.5.2/scripts/text-completion-vertexai

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.vertexai import run
+
+run()
+

trustgraph-0.5.2/scripts/triples-dump-parquet

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.dump.triples.parquet import run
+
+run()
+

trustgraph-0.5.2/scripts/triples-write-cassandra

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.triples.cassandra import run
+
+run()
+

{trustgraph-0.4.2 → trustgraph-0.5.2}/setup.py

@@ -4,7 +4,7 @@ import os
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
-version = "0.4.2"
+version = "0.5.2"
 
 setuptools.setup(
     name="trustgraph",
@@ -44,27 +44,30 @@ setuptools.setup(
         "google-cloud-aiplatform",
         "pyyaml",
         "prometheus-client",
+        "pyarrow",
     ],
     scripts=[
         "scripts/chunker-recursive",
         "scripts/embeddings-hf",
         "scripts/embeddings-ollama",
         "scripts/embeddings-vectorize",
+        "scripts/ge-write-milvus",
         "scripts/graph-rag",
         "scripts/graph-show",
         "scripts/graph-to-turtle",
-        "scripts/graph-write-cassandra",
         "scripts/init-pulsar-manager",
         "scripts/kg-extract-definitions",
         "scripts/kg-extract-relationships",
-        "scripts/llm-azure-text",
-        "scripts/llm-claude-text",
-        "scripts/llm-ollama-text",
-        "scripts/llm-vertexai-text",
         "scripts/loader",
         "scripts/pdf-decoder",
         "scripts/query",
         "scripts/run-processing",
-        "scripts/vector-write-milvus",
+        "scripts/text-completion-azure",
+        "scripts/text-completion-claude",
+        "scripts/text-completion-ollama",
+        "scripts/text-completion-vertexai",
+        "scripts/triples-write-cassandra",
+        "scripts/dump-parquet",
+        "scripts/triples-dump-parquet",
     ]
 )

trustgraph-0.5.2/trustgraph/base/__init__.py

@@ -0,0 +1,6 @@
+
+from . base_processor import BaseProcessor
+from . consumer import Consumer
+from . producer import Producer
+from . consumer_producer import ConsumerProducer
+

trustgraph-0.5.2/trustgraph/base/base_processor.py

@@ -0,0 +1,117 @@
+
+import os
+import argparse
+import pulsar
+import _pulsar
+import time
+from prometheus_client import start_http_server, Info
+
+from .. log_level import LogLevel
+
+class BaseProcessor:
+
+    default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
+
+    def __init__(self, **params):
+
+        self.client = None
+
+        if not hasattr(__class__, "params_metric"):
+            __class__.params_metric = Info(
+                'params', 'Parameters configuration'
+            )
+
+        # FIXME: Maybe outputs information it should not
+        __class__.params_metric.info({
+            k: str(params[k])
+            for k in params
+        })
+
+        pulsar_host = params.get("pulsar_host", self.default_pulsar_host)
+        log_level = params.get("log_level", LogLevel.INFO)
+
+        self.pulsar_host = pulsar_host
+
+        self.client = pulsar.Client(
+            pulsar_host,
+            logger=pulsar.ConsoleLogger(log_level.to_pulsar())
+        )
+
+    def __del__(self):
+
+        if self.client:
+            self.client.close()
+
+    @staticmethod
+    def add_args(parser):
+
+        parser.add_argument(
+            '-p', '--pulsar-host',
+            default=__class__.default_pulsar_host,
+            help=f'Pulsar host (default: {__class__.default_pulsar_host})',
+        )
+
+        parser.add_argument(
+            '-l', '--log-level',
+            type=LogLevel,
+            default=LogLevel.INFO,
+            choices=list(LogLevel),
+            help=f'Output queue (default: info)'
+        )
+
+        parser.add_argument(
+            '-M', '--metrics-enabled',
+            type=bool,
+            default=True,
+            help=f'Pulsar host (default: true)',
+        )
+
+        parser.add_argument(
+            '-P', '--metrics-port',
+            type=int,
+            default=8000,
+            help=f'Pulsar host (default: 8000)',
+        )
+
+    def run(self):
+        raise RuntimeError("Something should have implemented the run method")
+
+    @classmethod
+    def start(cls, prog, doc):
+
+        while True:
+
+            parser = argparse.ArgumentParser(
+                prog=prog,
+                description=doc
+            )
+
+            cls.add_args(parser)
+
+            args = parser.parse_args()
+            args = vars(args)
+
+            if args["metrics_enabled"]:
+                start_http_server(args["metrics_port"])
+
+            try:
+
+                p = cls(**args)
+                p.run()
+
+            except KeyboardInterrupt:
+                print("Keyboard interrupt.")
+                return
+
+            except _pulsar.Interrupted:
+                print("Pulsar Interrupted.")
+                return
+
+            except Exception as e:
+
+                print(type(e))
+
+                print("Exception:", e, flush=True)
+                print("Will retry...", flush=True)
+
+                time.sleep(10)

trustgraph-0.5.2/trustgraph/base/consumer.py

@@ -0,0 +1,87 @@
+
+from pulsar.schema import JsonSchema
+from prometheus_client import start_http_server, Histogram, Info, Counter
+
+from . base_processor import BaseProcessor
+
+class Consumer(BaseProcessor):
+
+    def __init__(self, **params):
+
+        super(Consumer, self).__init__(**params)
+
+        input_queue = params.get("input_queue")
+        subscriber = params.get("subscriber")
+        input_schema = params.get("input_schema")
+
+        if input_schema == None:
+            raise RuntimeError("input_schema must be specified")
+
+        if not hasattr(__class__, "request_metric"):
+            __class__.request_metric = Histogram(
+                'request_latency', 'Request latency (seconds)'
+            )
+
+        if not hasattr(__class__, "pubsub_metric"):
+            __class__.pubsub_metric = Info(
+                'pubsub', 'Pub/sub configuration'
+            )
+
+        if not hasattr(__class__, "processing_metric"):
+            __class__.processing_metric = Counter(
+                'processing_count', 'Processing count', ["status"]
+            )
+
+        __class__.pubsub_metric.info({
+            "input_queue": input_queue,
+            "subscriber": subscriber,
+            "input_schema": input_schema.__name__,
+        })
+
+        self.consumer = self.client.subscribe(
+            input_queue, subscriber,
+            schema=JsonSchema(input_schema),
+        )
+
+    def run(self):
+
+        while True:
+
+            msg = self.consumer.receive()
+
+            try:
+
+                with __class__.request_metric.time():
+                    self.handle(msg)
+
+                # Acknowledge successful processing of the message
+                self.consumer.acknowledge(msg)
+
+                __class__.processing_metric.labels(status="success").inc()
+
+            except Exception as e:
+
+                print("Exception:", e, flush=True)
+
+                # Message failed to be processed
+                self.consumer.negative_acknowledge(msg)
+
+                __class__.processing_metric.labels(status="error").inc()
+
+    @staticmethod
+    def add_args(parser, default_input_queue, default_subscriber):
+
+        BaseProcessor.add_args(parser)
+
+        parser.add_argument(
+            '-i', '--input-queue',
+            default=default_input_queue,
+            help=f'Input queue (default: {default_input_queue})'
+        )
+
+        parser.add_argument(
+            '-s', '--subscriber',
+            default=default_subscriber,
+            help=f'Queue subscriber name (default: {default_subscriber})'
+        )
+

trustgraph-0.5.2/trustgraph/base/consumer_producer.py

@@ -0,0 +1,168 @@
+
+from pulsar.schema import JsonSchema
+from prometheus_client import Histogram, Info, Counter
+
+from . base_processor import BaseProcessor
+
+# FIXME: Derive from consumer?  And producer?
+
+class ConsumerProducer(BaseProcessor):
+
+    def __init__(self, **params):
+
+        input_queue = params.get("input_queue")
+        output_queue = params.get("output_queue")
+        subscriber = params.get("subscriber")
+        input_schema = params.get("input_schema")
+        output_schema = params.get("output_schema")
+
+        if not hasattr(__class__, "request_metric"):
+            __class__.request_metric = Histogram(
+                'request_latency', 'Request latency (seconds)'
+            )
+
+        if not hasattr(__class__, "output_metric"):
+            __class__.output_metric = Counter(
+                'output_count', 'Output items created'
+            )
+
+        if not hasattr(__class__, "pubsub_metric"):
+            __class__.pubsub_metric = Info(
+                'pubsub', 'Pub/sub configuration'
+            )
+
+        if not hasattr(__class__, "processing_metric"):
+            __class__.processing_metric = Counter(
+                'processing_count', 'Processing count', ["status"]
+            )
+
+        __class__.pubsub_metric.info({
+            "input_queue": input_queue,
+            "output_queue": output_queue,
+            "subscriber": subscriber,
+            "input_schema": input_schema.__name__,
+            "output_schema": output_schema.__name__,
+        })
+
+        super(ConsumerProducer, self).__init__(**params)
+
+        if input_schema == None:
+            raise RuntimeError("input_schema must be specified")
+
+        if output_schema == None:
+            raise RuntimeError("output_schema must be specified")
+
+        self.consumer = self.client.subscribe(
+            input_queue, subscriber,
+            schema=JsonSchema(input_schema),
+        )
+
+        self.producer = self.client.create_producer(
+            topic=output_queue,
+            schema=JsonSchema(output_schema),
+        )
+
+    def run(self):
+
+        while True:
+
+            msg = self.consumer.receive()
+
+            try:
+
+                with __class__.request_metric.time():
+                    resp = self.handle(msg)
+
+                # Acknowledge successful processing of the message
+                self.consumer.acknowledge(msg)
+
+                __class__.processing_metric.labels(status="success").inc()
+
+            except Exception as e:
+
+                print("Exception:", e, flush=True)
+
+                # Message failed to be processed
+                self.consumer.negative_acknowledge(msg)
+
+                __class__.processing_metric.labels(status="error").inc()
+
+    def send(self, msg, properties={}):
+        self.producer.send(msg, properties)
+        __class__.output_metric.inc()
+
+    @staticmethod
+    def add_args(
+            parser, default_input_queue, default_subscriber,
+            default_output_queue,
+    ):
+
+        BaseProcessor.add_args(parser)
+
+        parser.add_argument(
+            '-i', '--input-queue',
+            default=default_input_queue,
+            help=f'Input queue (default: {default_input_queue})'
+        )
+
+        parser.add_argument(
+            '-s', '--subscriber',
+            default=default_subscriber,
+            help=f'Queue subscriber name (default: {default_subscriber})'
+        )
+
+        parser.add_argument(
+            '-o', '--output-queue',
+            default=default_output_queue,
+            help=f'Output queue (default: {default_output_queue})'
+        )
+
+class Producer(BaseProcessor):
+
+    def __init__(self, **params):
+
+        output_queue = params.get("output_queue")
+        output_schema = params.get("output_schema")
+
+        if not hasattr(__class__, "output_metric"):
+            __class__.output_metric = Counter(
+                'output_count', 'Output items created'
+            )
+
+        if not hasattr(__class__, "pubsub_metric"):
+            __class__.pubsub_metric = Info(
+                'pubsub', 'Pub/sub configuration'
+            )
+
+        __class__.pubsub_metric.info({
+            "output_queue": output_queue,
+            "output_schema": output_schema.__name__,
+        })
+
+        super(Producer, self).__init__(**params)
+
+        if output_schema == None:
+            raise RuntimeError("output_schema must be specified")
+
+        self.producer = self.client.create_producer(
+            topic=output_queue,
+            schema=JsonSchema(output_schema),
+        )
+
+    def send(self, msg, properties={}):
+        self.producer.send(msg, properties)
+        __class__.output_metric.inc()
+
+    @staticmethod
+    def add_args(
+            parser, default_input_queue, default_subscriber,
+            default_output_queue,
+    ):
+
+        BaseProcessor.add_args(parser)
+
+        parser.add_argument(
+            '-o', '--output-queue',
+            default=default_output_queue,
+            help=f'Output queue (default: {default_output_queue})'
+        )

trustgraph-0.5.2/trustgraph/base/producer.py

@@ -0,0 +1,55 @@
+
+from pulsar.schema import JsonSchema
+from prometheus_client import Info, Counter
+
+from . base_processor import BaseProcessor
+
+class Producer(BaseProcessor):
+
+    def __init__(self, **params):
+
+        output_queue = params.get("output_queue")
+        output_schema = params.get("output_schema")
+
+        if not hasattr(__class__, "output_metric"):
+            __class__.output_metric = Counter(
+                'output_count', 'Output items created'
+            )
+
+        if not hasattr(__class__, "pubsub_metric"):
+            __class__.pubsub_metric = Info(
+                'pubsub', 'Pub/sub configuration'
+            )
+
+        __class__.pubsub_metric.info({
+            "output_queue": output_queue,
+            "output_schema": output_schema.__name__,
+        })
+
+        super(Producer, self).__init__(**params)
+
+        if output_schema == None:
+            raise RuntimeError("output_schema must be specified")
+
+        self.producer = self.client.create_producer(
+            topic=output_queue,
+            schema=JsonSchema(output_schema),
+        )
+
+    def send(self, msg, properties={}):
+        self.producer.send(msg, properties)
+        __class__.output_metric.inc()
+
+    @staticmethod
+    def add_args(
+            parser, default_input_queue, default_subscriber,
+            default_output_queue,
+    ):
+
+        BaseProcessor.add_args(parser)
+
+        parser.add_argument(
+            '-o', '--output-queue',
+            default=default_output_queue,
+            help=f'Output queue (default: {default_output_queue})'
+        )