trustgraph-flow 0.11.11__tar.gz

trustgraph-flow-0.11.11/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.1
+Name: trustgraph-flow
+Version: 0.11.11
+Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
+Home-page: https://github.com/trustgraph-ai/trustgraph
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.11.11.tar.gz
+Author: trustgraph.ai
+Author-email: security@trustgraph.ai
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: trustgraph-base
+Requires-Dist: urllib3
+Requires-Dist: rdflib
+Requires-Dist: pymilvus
+Requires-Dist: langchain
+Requires-Dist: langchain-core
+Requires-Dist: langchain-text-splitters
+Requires-Dist: langchain-community
+Requires-Dist: requests
+Requires-Dist: cassandra-driver
+Requires-Dist: pulsar-client
+Requires-Dist: pypdf
+Requires-Dist: qdrant-client
+Requires-Dist: tabulate
+Requires-Dist: anthropic
+Requires-Dist: pyyaml
+Requires-Dist: prometheus-client
+Requires-Dist: cohere
+Requires-Dist: openai
+Requires-Dist: neo4j
+Requires-Dist: tiktoken
+
+See https://trustgraph.ai/

trustgraph-flow-0.11.11/README.md

@@ -0,0 +1 @@
+See https://trustgraph.ai/

trustgraph-flow-0.11.11/scripts/chunker-recursive

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.chunking.recursive import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/chunker-token

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.chunking.token import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/de-query-milvus

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.query.doc_embeddings.milvus import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/de-query-qdrant

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.query.doc_embeddings.qdrant import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/de-write-milvus

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.doc_embeddings.milvus import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/de-write-qdrant

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.doc_embeddings.qdrant import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/document-rag

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.retrieval.document_rag import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/embeddings-ollama

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.embeddings.ollama import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/embeddings-vectorize

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.embeddings.vectorize import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/ge-query-milvus

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.query.graph_embeddings.milvus import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/ge-query-qdrant

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.query.graph_embeddings.qdrant import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/ge-write-milvus

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.graph_embeddings.milvus import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/ge-write-qdrant

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.graph_embeddings.qdrant import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/graph-rag

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.retrieval.graph_rag import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/kg-extract-definitions

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.extract.kg.definitions import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/kg-extract-relationships

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.extract.kg.relationships import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/kg-extract-topics

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.extract.kg.topics import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/metering

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+from trustgraph.metering import run
+
+run()

trustgraph-flow-0.11.11/scripts/object-extract-row

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.extract.object.row import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/oe-write-milvus

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.object_embeddings.milvus import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/pdf-decoder

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.decoding.pdf import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/prompt-generic

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.prompt.generic import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/prompt-template

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.prompt.template import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/rows-write-cassandra

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.rows.cassandra import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/run-processing

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.processing import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/text-completion-azure

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.azure import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/text-completion-claude

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.claude import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/text-completion-cohere

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.cohere import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/text-completion-llamafile

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.llamafile import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/text-completion-ollama

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.ollama import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/text-completion-openai

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.openai import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/triples-query-cassandra

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.query.triples.cassandra import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/triples-query-neo4j

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.query.triples.neo4j import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/triples-write-cassandra

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.triples.cassandra import run
+
+run()
+

trustgraph-flow-0.11.11/scripts/triples-write-neo4j

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.storage.triples.neo4j import run
+
+run()
+

trustgraph-flow-0.11.11/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

trustgraph-flow-0.11.11/setup.py

@@ -0,0 +1,96 @@
+import setuptools
+import os
+import importlib
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+# Load a version number module
+spec = importlib.util.spec_from_file_location(
+    'version', 'trustgraph/flow_version.py'
+)
+version_module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(version_module)
+
+version = version_module.__version__
+
+setuptools.setup(
+    name="trustgraph-flow",
+    version=version,
+    author="trustgraph.ai",
+    author_email="security@trustgraph.ai",
+    description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/trustgraph-ai/trustgraph",
+    packages=setuptools.find_namespace_packages(
+        where='./',
+    ),
+    classifiers=[ 
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+        "Operating System :: OS Independent",
+    ],
+    python_requires='>=3.8',
+    download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
+    install_requires=[
+        "trustgraph-base",
+        "urllib3",
+        "rdflib",
+        "pymilvus",
+        "langchain",
+        "langchain-core",
+        "langchain-text-splitters",
+        "langchain-community",
+        "requests",
+        "cassandra-driver",
+        "pulsar-client",
+        "pypdf",
+        "qdrant-client",
+        "tabulate",
+        "anthropic",
+        "pyyaml",
+        "prometheus-client",
+        "cohere",
+        "openai",
+        "neo4j",
+        "tiktoken",
+    ],
+    scripts=[
+        "scripts/chunker-recursive",
+        "scripts/chunker-token",
+        "scripts/de-query-milvus",
+        "scripts/de-query-qdrant",
+        "scripts/de-write-milvus",
+        "scripts/de-write-qdrant",
+        "scripts/document-rag",
+        "scripts/embeddings-ollama",
+        "scripts/embeddings-vectorize",
+        "scripts/ge-query-milvus",
+        "scripts/ge-query-qdrant",
+        "scripts/ge-write-milvus",
+        "scripts/ge-write-qdrant",
+        "scripts/graph-rag",
+        "scripts/kg-extract-definitions",
+        "scripts/kg-extract-topics",
+        "scripts/kg-extract-relationships",
+        "scripts/metering",
+        "scripts/object-extract-row",
+        "scripts/oe-write-milvus",
+        "scripts/pdf-decoder",
+        "scripts/prompt-generic",
+        "scripts/prompt-template",
+        "scripts/rows-write-cassandra",
+        "scripts/run-processing",
+        "scripts/text-completion-azure",
+        "scripts/text-completion-claude",
+        "scripts/text-completion-cohere",
+        "scripts/text-completion-llamafile",
+        "scripts/text-completion-ollama",
+        "scripts/text-completion-openai",
+        "scripts/triples-query-cassandra",
+        "scripts/triples-query-neo4j",
+        "scripts/triples-write-cassandra",
+        "scripts/triples-write-neo4j",
+    ]
+)

trustgraph-flow-0.11.11/trustgraph/chunking/recursive/__init__.py

@@ -0,0 +1,3 @@
+
+from . chunker import *
+

trustgraph-flow-0.11.11/trustgraph/chunking/recursive/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+from . chunker import run
+
+if __name__ == '__main__':
+    run()
+

trustgraph-flow-0.11.11/trustgraph/chunking/recursive/chunker.py

@@ -0,0 +1,108 @@
+
+"""
+Simple decoder, accepts text documents on input, outputs chunks from the
+as text as separate output objects.
+"""
+
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from prometheus_client import Histogram
+
+from ... schema import TextDocument, Chunk, Source
+from ... schema import text_ingest_queue, chunk_ingest_queue
+from ... log_level import LogLevel
+from ... base import ConsumerProducer
+
+module = ".".join(__name__.split(".")[1:-1])
+
+default_input_queue = text_ingest_queue
+default_output_queue = chunk_ingest_queue
+default_subscriber = module
+
+class Processor(ConsumerProducer):
+
+    def __init__(self, **params):
+
+        input_queue = params.get("input_queue", default_input_queue)
+        output_queue = params.get("output_queue", default_output_queue)
+        subscriber = params.get("subscriber", default_subscriber)
+        chunk_size = params.get("chunk_size", 2000)
+        chunk_overlap = params.get("chunk_overlap", 100)
+        
+        super(Processor, self).__init__(
+            **params | {
+                "input_queue": input_queue,
+                "output_queue": output_queue,
+                "subscriber": subscriber,
+                "input_schema": TextDocument,
+                "output_schema": Chunk,
+            }
+        )
+
+        if not hasattr(__class__, "chunk_metric"):
+            __class__.chunk_metric = Histogram(
+                'chunk_size', 'Chunk size',
+                buckets=[100, 160, 250, 400, 650, 1000, 1600,
+                         2500, 4000, 6400, 10000, 16000]
+            )
+
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            length_function=len,
+            is_separator_regex=False,
+        )
+
+    def handle(self, msg):
+
+        v = msg.value()
+        print(f"Chunking {v.source.id}...", flush=True)
+
+        texts = self.text_splitter.create_documents(
+            [v.text.decode("utf-8")]
+        )
+
+        for ix, chunk in enumerate(texts):
+
+            id = v.source.id + "-c" + str(ix)
+
+            r = Chunk(
+                source=Source(
+                    source=v.source.source,
+                    id=id,
+                    title=v.source.title
+                ),
+                chunk=chunk.page_content.encode("utf-8"),
+            )
+
+            __class__.chunk_metric.observe(len(chunk.page_content))
+
+            self.send(r)
+
+        print("Done.", flush=True)
+
+    @staticmethod
+    def add_args(parser):
+
+        ConsumerProducer.add_args(
+            parser, default_input_queue, default_subscriber,
+            default_output_queue,
+        )
+
+        parser.add_argument(
+            '-z', '--chunk-size',
+            type=int,
+            default=2000,
+            help=f'Chunk size (default: 2000)'
+        )
+
+        parser.add_argument(
+            '-v', '--chunk-overlap',
+            type=int,
+            default=100,
+            help=f'Chunk overlap (default: 100)'
+        )
+
+def run():
+
+    Processor.start(module, __doc__)
+

trustgraph-flow-0.11.11/trustgraph/chunking/token/__init__.py

@@ -0,0 +1,3 @@
+
+from . chunker import *
+

trustgraph-flow-0.11.11/trustgraph/chunking/token/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+from . chunker import run
+
+if __name__ == '__main__':
+    run()
+

trustgraph-flow-0.11.11/trustgraph/chunking/token/chunker.py

@@ -0,0 +1,107 @@
+
+"""
+Simple decoder, accepts text documents on input, outputs chunks from the
+as text as separate output objects.
+"""
+
+from langchain_text_splitters import TokenTextSplitter
+from prometheus_client import Histogram
+
+from ... schema import TextDocument, Chunk, Source
+from ... schema import text_ingest_queue, chunk_ingest_queue
+from ... log_level import LogLevel
+from ... base import ConsumerProducer
+
+module = ".".join(__name__.split(".")[1:-1])
+
+default_input_queue = text_ingest_queue
+default_output_queue = chunk_ingest_queue
+default_subscriber = module
+
+class Processor(ConsumerProducer):
+
+    def __init__(self, **params):
+
+        input_queue = params.get("input_queue", default_input_queue)
+        output_queue = params.get("output_queue", default_output_queue)
+        subscriber = params.get("subscriber", default_subscriber)
+        chunk_size = params.get("chunk_size", 250)
+        chunk_overlap = params.get("chunk_overlap", 15)
+        
+        super(Processor, self).__init__(
+            **params | {
+                "input_queue": input_queue,
+                "output_queue": output_queue,
+                "subscriber": subscriber,
+                "input_schema": TextDocument,
+                "output_schema": Chunk,
+            }
+        )
+
+        if not hasattr(__class__, "chunk_metric"):
+            __class__.chunk_metric = Histogram(
+                'chunk_size', 'Chunk size',
+                buckets=[100, 160, 250, 400, 650, 1000, 1600,
+                         2500, 4000, 6400, 10000, 16000]
+            )
+
+        self.text_splitter = TokenTextSplitter(
+            encoding_name="cl100k_base",
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+        )
+
+    def handle(self, msg):
+
+        v = msg.value()
+        print(f"Chunking {v.source.id}...", flush=True)
+
+        texts = self.text_splitter.create_documents(
+            [v.text.decode("utf-8")]
+        )
+
+        for ix, chunk in enumerate(texts):
+
+            id = v.source.id + "-c" + str(ix)
+
+            r = Chunk(
+                source=Source(
+                    source=v.source.source,
+                    id=id,
+                    title=v.source.title
+                ),
+                chunk=chunk.page_content.encode("utf-8"),
+            )
+
+            __class__.chunk_metric.observe(len(chunk.page_content))
+
+            self.send(r)
+
+        print("Done.", flush=True)
+
+    @staticmethod
+    def add_args(parser):
+
+        ConsumerProducer.add_args(
+            parser, default_input_queue, default_subscriber,
+            default_output_queue,
+        )
+
+        parser.add_argument(
+            '-z', '--chunk-size',
+            type=int,
+            default=250,
+            help=f'Chunk size (default: 250)'
+        )
+
+        parser.add_argument(
+            '-v', '--chunk-overlap',
+            type=int,
+            default=15,
+            help=f'Chunk overlap (default: 15)'
+        )
+
+def run():
+
+    Processor.start(module, __doc__)
+