trustgraph-flow 0.11.19__tar.gz → 0.12.1__tar.gz

{trustgraph-flow-0.11.19 → trustgraph-flow-0.12.1}/PKG-INFO

@@ -1,36 +1,19 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.11.19
+Version: 0.12.1
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.11.19.tar.gz
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
+License: UNKNOWN
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.12.1.tar.gz
+Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: trustgraph-base
-Requires-Dist: urllib3
-Requires-Dist: rdflib
-Requires-Dist: pymilvus
-Requires-Dist: langchain
-Requires-Dist: langchain-core
-Requires-Dist: langchain-text-splitters
-Requires-Dist: langchain-community
-Requires-Dist: requests
-Requires-Dist: cassandra-driver
-Requires-Dist: pulsar-client
-Requires-Dist: pypdf
-Requires-Dist: qdrant-client
-Requires-Dist: tabulate
-Requires-Dist: anthropic
-Requires-Dist: pyyaml
-Requires-Dist: prometheus-client
-Requires-Dist: cohere
-Requires-Dist: openai
-Requires-Dist: neo4j
-Requires-Dist: tiktoken
 
 See https://trustgraph.ai/
+
+

trustgraph-flow-0.12.1/scripts/text-completion-azure-openai

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.model.text_completion.azure_openai import run
+
+run()
+

{trustgraph-flow-0.11.19 → trustgraph-flow-0.12.1}/setup.py

@@ -83,6 +83,7 @@ setuptools.setup(
         "scripts/rows-write-cassandra",
         "scripts/run-processing",
         "scripts/text-completion-azure",
+        "scripts/text-completion-azure-openai",
         "scripts/text-completion-claude",
         "scripts/text-completion-cohere",
         "scripts/text-completion-llamafile",

{trustgraph-flow-0.11.19 → trustgraph-flow-0.12.1}/trustgraph/chunking/recursive/chunker.py

@@ -7,7 +7,7 @@ as text as separate output objects.
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from prometheus_client import Histogram
 
-from ... schema import TextDocument, Chunk, Source
+from ... schema import TextDocument, Chunk, Metadata
 from ... schema import text_ingest_queue, chunk_ingest_queue
 from ... log_level import LogLevel
 from ... base import ConsumerProducer
@@ -55,7 +55,7 @@ class Processor(ConsumerProducer):
     def handle(self, msg):
 
         v = msg.value()
-        print(f"Chunking {v.source.id}...", flush=True)
+        print(f"Chunking {v.metadata.id}...", flush=True)
 
         texts = self.text_splitter.create_documents(
             [v.text.decode("utf-8")]
@@ -63,13 +63,15 @@ class Processor(ConsumerProducer):
 
         for ix, chunk in enumerate(texts):
 
-            id = v.source.id + "-c" + str(ix)
+            id = v.metadata.id + "-c" + str(ix)
 
             r = Chunk(
-                source=Source(
-                    source=v.source.source,
+                metadata=Metadata(
+                    source=v.metadata.source,
                     id=id,
-                    title=v.source.title
+                    title=v.metadata.title,
+                    user=v.metadata.user,
+                    collection=v.metadata.collection,
                 ),
                 chunk=chunk.page_content.encode("utf-8"),
             )

{trustgraph-flow-0.11.19 → trustgraph-flow-0.12.1}/trustgraph/chunking/token/chunker.py

@@ -7,7 +7,7 @@ as text as separate output objects.
 from langchain_text_splitters import TokenTextSplitter
 from prometheus_client import Histogram
 
-from ... schema import TextDocument, Chunk, Source
+from ... schema import TextDocument, Chunk, Metadata
 from ... schema import text_ingest_queue, chunk_ingest_queue
 from ... log_level import LogLevel
 from ... base import ConsumerProducer
@@ -54,7 +54,7 @@ class Processor(ConsumerProducer):
     def handle(self, msg):
 
         v = msg.value()
-        print(f"Chunking {v.source.id}...", flush=True)
+        print(f"Chunking {v.metadata.id}...", flush=True)
 
         texts = self.text_splitter.create_documents(
             [v.text.decode("utf-8")]
@@ -62,13 +62,15 @@ class Processor(ConsumerProducer):
 
         for ix, chunk in enumerate(texts):
 
-            id = v.source.id + "-c" + str(ix)
+            id = v.metadata.id + "-c" + str(ix)
 
             r = Chunk(
-                source=Source(
-                    source=v.source.source,
+                metadata=Metadata(
+                    source=v.metadata.source,
                     id=id,
-                    title=v.source.title
+                    title=v.metadata.title,
+                    user=v.metadata.user,
+                    collection=v.metadata.collection,
                 ),
                 chunk=chunk.page_content.encode("utf-8"),
             )

{trustgraph-flow-0.11.19 → trustgraph-flow-0.12.1}/trustgraph/decoding/pdf/pdf_decoder.py

@@ -8,7 +8,7 @@ import tempfile
 import base64
 from langchain_community.document_loaders import PyPDFLoader
 
-from ... schema import Document, TextDocument, Source
+from ... schema import Document, TextDocument, Metadata
 from ... schema import document_ingest_queue, text_ingest_queue
 from ... log_level import LogLevel
 from ... base import ConsumerProducer
@@ -45,7 +45,7 @@ class Processor(ConsumerProducer):
 
         v = msg.value()
 
-        print(f"Decoding {v.source.id}...", flush=True)
+        print(f"Decoding {v.metadata.id}...", flush=True)
 
         with tempfile.NamedTemporaryFile(delete_on_close=False) as fp:
 
@@ -59,12 +59,14 @@ class Processor(ConsumerProducer):
 
                 for ix, page in enumerate(pages):
 
-                    id = v.source.id + "-p" + str(ix)
+                    id = v.metadata.id + "-p" + str(ix)
                     r = TextDocument(
-                        source=Source(
-                            source=v.source.source,
-                            title=v.source.title,
+                        metadata=Metadata(
+                            source=v.metadata.source,
+                            title=v.metadata.title,
                             id=id,
+                            user=v.metadata.user,
+                            collection=v.metadata.collection,
                         ),
                         text=page.page_content.encode("utf-8"),
                     )

{trustgraph-flow-0.11.19 → trustgraph-flow-0.12.1}/trustgraph/direct/cassandra.py

@@ -4,10 +4,16 @@ from cassandra.auth import PlainTextAuthProvider
 
 class TrustGraph:
 
-    def __init__(self, hosts=None):
+    def __init__(
+            self, hosts=None,
+            keyspace="trustgraph", table="default",
+    ):
 
         if hosts is None:
             hosts = ["localhost"]
+
+        self.keyspace = keyspace
+        self.table = table
             
         self.cluster = Cluster(hosts)
         self.session = self.cluster.connect()
@@ -16,26 +22,26 @@ class TrustGraph:
 
     def clear(self):
 
-        self.session.execute("""
-            drop keyspace if exists trustgraph;
+        self.session.execute(f"""
+            drop keyspace if exists {self.keyspace};
         """);
 
         self.init()
 
     def init(self):
 
-        self.session.execute("""
-            create keyspace if not exists trustgraph
-                with replication = { 
+        self.session.execute(f"""
+            create keyspace if not exists {self.keyspace}
+                with replication = {{ 
                    'class' : 'SimpleStrategy', 
                    'replication_factor' : 1 
-                };
+                }};
         """);
 
-        self.session.set_keyspace('trustgraph')
+        self.session.set_keyspace(self.keyspace)
 
-        self.session.execute("""
-            create table if not exists triples (
+        self.session.execute(f"""
+            create table if not exists {self.table} (
                 s text,
                 p text,
                 o text,
@@ -43,66 +49,66 @@ class TrustGraph:
             );
         """);
 
-        self.session.execute("""
-            create index if not exists triples_p
-                ON triples (p);
+        self.session.execute(f"""
+            create index if not exists {self.table}_p
+                ON {self.table} (p);
         """);
 
-        self.session.execute("""
-            create index if not exists triples_o
-                ON triples (o);
+        self.session.execute(f"""
+            create index if not exists {self.table}_o
+                ON {self.table} (o);
         """);
 
     def insert(self, s, p, o):
     
         self.session.execute(
-            "insert into triples (s, p, o) values (%s, %s, %s)",
+            f"insert into {self.table} (s, p, o) values (%s, %s, %s)",
             (s, p, o)
         )
 
     def get_all(self, limit=50):
         return self.session.execute(
-            f"select s, p, o from triples limit {limit}"
+            f"select s, p, o from {self.table} limit {limit}"
         )
 
     def get_s(self, s, limit=10):
         return self.session.execute(
-            f"select p, o from triples where s = %s limit {limit}",
+            f"select p, o from {self.table} where s = %s limit {limit}",
             (s,)
         )
 
     def get_p(self, p, limit=10):
         return self.session.execute(
-            f"select s, o from triples where p = %s limit {limit}",
+            f"select s, o from {self.table} where p = %s limit {limit}",
             (p,)
         )
 
     def get_o(self, o, limit=10):
         return self.session.execute(
-            f"select s, p from triples where o = %s limit {limit}",
+            f"select s, p from {self.table} where o = %s limit {limit}",
             (o,)
         )
 
     def get_sp(self, s, p, limit=10):
         return self.session.execute(
-            f"select o from triples where s = %s and p = %s limit {limit}",
+            f"select o from {self.table} where s = %s and p = %s limit {limit}",
             (s, p)
         )
 
     def get_po(self, p, o, limit=10):
         return self.session.execute(
-            f"select s from triples where p = %s and o = %s allow filtering limit {limit}",
+            f"select s from {self.table} where p = %s and o = %s allow filtering limit {limit}",
             (p, o)
         )
 
     def get_os(self, o, s, limit=10):
         return self.session.execute(
-            f"select p from triples where o = %s and s = %s limit {limit}",
+            f"select p from {self.table} where o = %s and s = %s limit {limit}",
             (o, s)
         )
 
     def get_spo(self, s, p, o, limit=10):
         return self.session.execute(
-            f"""select s as x from triples where s = %s and p = %s and o = %s limit {limit}""",
+            f"""select s as x from {self.table} where s = %s and p = %s and o = %s limit {limit}""",
             (s, p, o)
         )

{trustgraph-flow-0.11.19 → trustgraph-flow-0.12.1}/trustgraph/embeddings/vectorize/vectorize.py

@@ -50,15 +50,15 @@ class Processor(ConsumerProducer):
             subscriber=module + "-emb",
         )
 
-    def emit(self, source, chunk, vectors):
+    def emit(self, metadata, chunk, vectors):
 
-        r = ChunkEmbeddings(source=source, chunk=chunk, vectors=vectors)
+        r = ChunkEmbeddings(metadata=metadata, chunk=chunk, vectors=vectors)
         self.producer.send(r)
 
     def handle(self, msg):
 
         v = msg.value()
-        print(f"Indexing {v.source.id}...", flush=True)
+        print(f"Indexing {v.metadata.id}...", flush=True)
 
         chunk = v.chunk.decode("utf-8")
 
@@ -67,7 +67,7 @@ class Processor(ConsumerProducer):
             vectors = self.embeddings.request(chunk)
 
             self.emit(
-                source=v.source,
+                metadata=v.metadata,
                 chunk=chunk.encode("utf-8"),
                 vectors=vectors
             )

{trustgraph-flow-0.11.19 → trustgraph-flow-0.12.1}/trustgraph/extract/kg/definitions/extract.py

@@ -7,7 +7,7 @@ get entity definitions which are output as graph edges.
 import urllib.parse
 import json
 
-from .... schema import ChunkEmbeddings, Triple, Source, Value
+from .... schema import ChunkEmbeddings, Triple, Metadata, Value
 from .... schema import chunk_embeddings_ingest_queue, triples_store_queue
 from .... schema import prompt_request_queue
 from .... schema import prompt_response_queue
@@ -69,15 +69,15 @@ class Processor(ConsumerProducer):
 
         return self.prompt.request_definitions(chunk)
 
-    def emit_edge(self, s, p, o):
+    def emit_edge(self, metadata, s, p, o):
 
-        t = Triple(s=s, p=p, o=o)
+        t = Triple(metadata=metadata, s=s, p=p, o=o)
         self.producer.send(t)
 
     def handle(self, msg):
 
         v = msg.value()
-        print(f"Indexing {v.source.id}...", flush=True)
+        print(f"Indexing {v.metadata.id}...", flush=True)
 
         chunk = v.chunk.decode("utf-8")
 
@@ -101,7 +101,7 @@ class Processor(ConsumerProducer):
                 s_value = Value(value=str(s_uri), is_uri=True)
                 o_value = Value(value=str(o), is_uri=False)
 
-                self.emit_edge(s_value, DEFINITION_VALUE, o_value)
+                self.emit_edge(v.metadata, s_value, DEFINITION_VALUE, o_value)
 
         except Exception as e:
             print("Exception: ", e, flush=True)

{trustgraph-flow-0.11.19 → trustgraph-flow-0.12.1}/trustgraph/extract/kg/relationships/extract.py

@@ -9,7 +9,8 @@ import urllib.parse
 import os
 from pulsar.schema import JsonSchema
 
-from .... schema import ChunkEmbeddings, Triple, GraphEmbeddings, Source, Value
+from .... schema import ChunkEmbeddings, Triple, GraphEmbeddings
+from .... schema import Metadata, Value
 from .... schema import chunk_embeddings_ingest_queue, triples_store_queue
 from .... schema import graph_embeddings_store_queue
 from .... schema import prompt_request_queue
@@ -91,20 +92,20 @@ class Processor(ConsumerProducer):
 
         return self.prompt.request_relationships(chunk)
 
-    def emit_edge(self, s, p, o):
+    def emit_edge(self, metadata, s, p, o):
 
-        t = Triple(s=s, p=p, o=o)
+        t = Triple(metadata=metadata, s=s, p=p, o=o)
         self.producer.send(t)
 
-    def emit_vec(self, ent, vec):
+    def emit_vec(self, metadata, ent, vec):
 
-        r = GraphEmbeddings(entity=ent, vectors=vec)
+        r = GraphEmbeddings(metadata=metadata, entity=ent, vectors=vec)
         self.vec_prod.send(r)
 
     def handle(self, msg):
 
         v = msg.value()
-        print(f"Indexing {v.source.id}...", flush=True)
+        print(f"Indexing {v.metadata.id}...", flush=True)
 
         chunk = v.chunk.decode("utf-8")
 
@@ -139,6 +140,7 @@ class Processor(ConsumerProducer):
                     o_value = Value(value=str(o), is_uri=False)
 
                 self.emit_edge(
+                    v.metadata,
                     s_value,
                     p_value,
                     o_value
@@ -146,6 +148,7 @@ class Processor(ConsumerProducer):
 
                 # Label for s
                 self.emit_edge(
+                    v.metadata,
                     s_value,
                     RDF_LABEL_VALUE,
                     Value(value=str(s), is_uri=False)
@@ -153,6 +156,7 @@ class Processor(ConsumerProducer):
 
                 # Label for p
                 self.emit_edge(
+                    v.metadata,
                     p_value,
                     RDF_LABEL_VALUE,
                     Value(value=str(p), is_uri=False)
@@ -161,15 +165,16 @@ class Processor(ConsumerProducer):
                 if rel.o_entity:
                     # Label for o
                     self.emit_edge(
+                        v.metadata,
                         o_value,
                         RDF_LABEL_VALUE,
                         Value(value=str(o), is_uri=False)
                     )
 
-                self.emit_vec(s_value, v.vectors)
-                self.emit_vec(p_value, v.vectors)
+                self.emit_vec(v.metadata, s_value, v.vectors)
+                self.emit_vec(v.metadata, p_value, v.vectors)
                 if rel.o_entity:
-                    self.emit_vec(o_value, v.vectors)
+                    self.emit_vec(v.metadata, o_value, v.vectors)
 
         except Exception as e:
             print("Exception: ", e, flush=True)

{trustgraph-flow-0.11.19 → trustgraph-flow-0.12.1}/trustgraph/extract/kg/topics/extract.py

@@ -7,7 +7,7 @@ get entity definitions which are output as graph edges.
 import urllib.parse
 import json
 
-from .... schema import ChunkEmbeddings, Triple, Source, Value
+from .... schema import ChunkEmbeddings, Triple, Metadata, Value
 from .... schema import chunk_embeddings_ingest_queue, triples_store_queue
 from .... schema import prompt_request_queue
 from .... schema import prompt_response_queue
@@ -69,15 +69,15 @@ class Processor(ConsumerProducer):
 
         return self.prompt.request_topics(chunk)
 
-    def emit_edge(self, s, p, o):
+    def emit_edge(self, metadata, s, p, o):
 
-        t = Triple(s=s, p=p, o=o)
+        t = Triple(metadata=metadata, s=s, p=p, o=o)
         self.producer.send(t)
 
     def handle(self, msg):
 
         v = msg.value()
-        print(f"Indexing {v.source.id}...", flush=True)
+        print(f"Indexing {v.metadata.id}...", flush=True)
 
         chunk = v.chunk.decode("utf-8")
 
@@ -101,7 +101,7 @@ class Processor(ConsumerProducer):
                 s_value = Value(value=str(s_uri), is_uri=True)
                 o_value = Value(value=str(o), is_uri=False)
 
-                self.emit_edge(s_value, DEFINITION_VALUE, o_value)
+                self.emit_edge(v. metadata, s_value, DEFINITION_VALUE, o_value)
 
         except Exception as e:
             print("Exception: ", e, flush=True)

{trustgraph-flow-0.11.19 → trustgraph-flow-0.12.1}/trustgraph/extract/object/row/extract.py

@@ -8,7 +8,7 @@ import urllib.parse
 import os
 from pulsar.schema import JsonSchema
 
-from .... schema import ChunkEmbeddings, Rows, ObjectEmbeddings, Source
+from .... schema import ChunkEmbeddings, Rows, ObjectEmbeddings, Metadata
 from .... schema import RowSchema, Field
 from .... schema import chunk_embeddings_ingest_queue, rows_store_queue
 from .... schema import object_embeddings_store_queue
@@ -124,24 +124,24 @@ class Processor(ConsumerProducer):
     def get_rows(self, chunk):
         return self.prompt.request_rows(self.schema, chunk)
 
-    def emit_rows(self, source, rows):
+    def emit_rows(self, metadata, rows):
 
         t = Rows(
-            source=source, row_schema=self.row_schema, rows=rows
+            metadata=metadata, row_schema=self.row_schema, rows=rows
         )
         self.producer.send(t)
 
-    def emit_vec(self, source, name, vec, key_name, key):
+    def emit_vec(self, metadata, name, vec, key_name, key):
 
         r = ObjectEmbeddings(
-            source=source, vectors=vec, name=name, key_name=key_name, id=key
+            metadata=metadata, vectors=vec, name=name, key_name=key_name, id=key
         )
         self.vec_prod.send(r)
 
     def handle(self, msg):
 
         v = msg.value()
-        print(f"Indexing {v.source.id}...", flush=True)
+        print(f"Indexing {v.metadata.id}...", flush=True)
 
         chunk = v.chunk.decode("utf-8")
 
@@ -150,13 +150,13 @@ class Processor(ConsumerProducer):
             rows = self.get_rows(chunk)
 
             self.emit_rows(
-                source=v.source,
+                metadata=v.metadata,
                 rows=rows
             )
 
             for row in rows:
                 self.emit_vec(
-                    source=v.source, vec=v.vectors,
+                    metadata=v.metadata, vec=v.vectors,
                     name=self.schema.name, key_name=self.primary.name,
                     key=row[self.primary.name]
                 )

trustgraph-flow-0.12.1/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "0.12.1"