trustgraph-flow 1.3.18__tar.gz → 1.4.1__tar.gz

{trustgraph_flow-1.3.18 → trustgraph_flow-1.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-flow
-Version: 1.3.18
+Version: 1.4.1
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph
@@ -8,7 +8,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: trustgraph-base<1.4,>=1.3
+Requires-Dist: trustgraph-base<1.5,>=1.4
 Requires-Dist: aiohttp
 Requires-Dist: anthropic
 Requires-Dist: cassandra-driver

{trustgraph_flow-1.3.18 → trustgraph_flow-1.4.1}/pyproject.toml

@@ -10,7 +10,7 @@ description = "TrustGraph provides a means to run a pipeline of flexible AI proc
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
-    "trustgraph-base>=1.3,<1.4",
+    "trustgraph-base>=1.4,<1.5",
     "aiohttp",
     "anthropic",
     "cassandra-driver",

trustgraph_flow-1.4.1/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "1.4.1"

{trustgraph_flow-1.3.18 → trustgraph_flow-1.4.1}/trustgraph/query/doc_embeddings/pinecone/service.py

@@ -47,6 +47,39 @@ class Processor(DocumentEmbeddingsQueryService):
             }
         )
 
+        self.last_index_name = None
+
+    def ensure_index_exists(self, index_name, dim):
+        """Ensure index exists, create if it doesn't"""
+        if index_name != self.last_index_name:
+            if not self.pinecone.has_index(index_name):
+                try:
+                    self.pinecone.create_index(
+                        name=index_name,
+                        dimension=dim,
+                        metric="cosine",
+                        spec=ServerlessSpec(
+                            cloud="aws",
+                            region="us-east-1",
+                        )
+                    )
+                    logger.info(f"Created index: {index_name}")
+                    
+                    # Wait for index to be ready
+                    import time
+                    for i in range(0, 1000):
+                        if self.pinecone.describe_index(index_name).status["ready"]:
+                            break
+                        time.sleep(1)
+                        
+                    if not self.pinecone.describe_index(index_name).status["ready"]:
+                        raise RuntimeError("Gave up waiting for index creation")
+                        
+                except Exception as e:
+                    logger.error(f"Pinecone index creation failed: {e}")
+                    raise e
+            self.last_index_name = index_name
+
     async def query_document_embeddings(self, msg):
 
         try:
@@ -65,6 +98,8 @@ class Processor(DocumentEmbeddingsQueryService):
                     "d-" + msg.user + "-" + msg.collection + "-" + str(dim)
                 )
 
+                self.ensure_index_exists(index_name, dim)
+
                 index = self.pinecone.Index(index_name)
 
                 results = index.query(

{trustgraph_flow-1.3.18 → trustgraph_flow-1.4.1}/trustgraph/query/doc_embeddings/qdrant/service.py

@@ -38,6 +38,24 @@ class Processor(DocumentEmbeddingsQueryService):
         )
 
         self.qdrant = QdrantClient(url=store_uri, api_key=api_key)
+        self.last_collection = None
+
+    def ensure_collection_exists(self, collection, dim):
+        """Ensure collection exists, create if it doesn't"""
+        if collection != self.last_collection:
+            if not self.qdrant.collection_exists(collection):
+                try:
+                    self.qdrant.create_collection(
+                        collection_name=collection,
+                        vectors_config=VectorParams(
+                            size=dim, distance=Distance.COSINE
+                        ),
+                    )
+                    logger.info(f"Created collection: {collection}")
+                except Exception as e:
+                    logger.error(f"Qdrant collection creation failed: {e}")
+                    raise e
+            self.last_collection = collection
 
     async def query_document_embeddings(self, msg):
 
@@ -53,6 +71,8 @@ class Processor(DocumentEmbeddingsQueryService):
                     str(dim)
                 )
 
+                self.ensure_collection_exists(collection, dim)
+
                 search_result = self.qdrant.query_points(
                     collection_name=collection,
                     query=vec,

{trustgraph_flow-1.3.18 → trustgraph_flow-1.4.1}/trustgraph/query/graph_embeddings/pinecone/service.py

@@ -49,6 +49,39 @@ class Processor(GraphEmbeddingsQueryService):
             }
         )
 
+        self.last_index_name = None
+
+    def ensure_index_exists(self, index_name, dim):
+        """Ensure index exists, create if it doesn't"""
+        if index_name != self.last_index_name:
+            if not self.pinecone.has_index(index_name):
+                try:
+                    self.pinecone.create_index(
+                        name=index_name,
+                        dimension=dim,
+                        metric="cosine",
+                        spec=ServerlessSpec(
+                            cloud="aws",
+                            region="us-east-1",
+                        )
+                    )
+                    logger.info(f"Created index: {index_name}")
+                    
+                    # Wait for index to be ready
+                    import time
+                    for i in range(0, 1000):
+                        if self.pinecone.describe_index(index_name).status["ready"]:
+                            break
+                        time.sleep(1)
+                        
+                    if not self.pinecone.describe_index(index_name).status["ready"]:
+                        raise RuntimeError("Gave up waiting for index creation")
+                        
+                except Exception as e:
+                    logger.error(f"Pinecone index creation failed: {e}")
+                    raise e
+            self.last_index_name = index_name
+
     def create_value(self, ent):
         if ent.startswith("http://") or ent.startswith("https://"):
             return Value(value=ent, is_uri=True)
@@ -74,6 +107,8 @@ class Processor(GraphEmbeddingsQueryService):
                     "t-" + msg.user + "-" + msg.collection + "-" + str(dim)
                 )
 
+                self.ensure_index_exists(index_name, dim)
+
                 index = self.pinecone.Index(index_name)
 
                 # Heuristic hack, get (2*limit), so that we have more chance

{trustgraph_flow-1.3.18 → trustgraph_flow-1.4.1}/trustgraph/query/graph_embeddings/qdrant/service.py

@@ -38,6 +38,24 @@ class Processor(GraphEmbeddingsQueryService):
         )
 
         self.qdrant = QdrantClient(url=store_uri, api_key=api_key)
+        self.last_collection = None
+
+    def ensure_collection_exists(self, collection, dim):
+        """Ensure collection exists, create if it doesn't"""
+        if collection != self.last_collection:
+            if not self.qdrant.collection_exists(collection):
+                try:
+                    self.qdrant.create_collection(
+                        collection_name=collection,
+                        vectors_config=VectorParams(
+                            size=dim, distance=Distance.COSINE
+                        ),
+                    )
+                    logger.info(f"Created collection: {collection}")
+                except Exception as e:
+                    logger.error(f"Qdrant collection creation failed: {e}")
+                    raise e
+            self.last_collection = collection
 
     def create_value(self, ent):
         if ent.startswith("http://") or ent.startswith("https://"):
@@ -60,6 +78,8 @@ class Processor(GraphEmbeddingsQueryService):
                     str(dim)
                 )
 
+                self.ensure_collection_exists(collection, dim)
+
                 # Heuristic hack, get (2*limit), so that we have more chance
                 # of getting (limit) entities
                 search_result = self.qdrant.query_points(

{trustgraph_flow-1.3.18 → trustgraph_flow-1.4.1}/trustgraph/query/triples/memgraph/service.py

@@ -55,6 +55,10 @@ class Processor(TriplesQueryService):
 
         try:
 
+            # Extract user and collection, use defaults if not provided
+            user = query.user if query.user else "default"
+            collection = query.collection if query.collection else "default"
+            
             triples = []
 
             if query.s is not None:
@@ -64,10 +68,13 @@ class Processor(TriplesQueryService):
                         # SPO
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal {value: $value}) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $rel, user: $user, collection: $collection}]->"
+                            "(dest:Literal {value: $value, user: $user, collection: $collection}) "
                             "RETURN $src as src "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, rel=query.p.value, value=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -75,10 +82,13 @@ class Processor(TriplesQueryService):
                             triples.append((query.s.value, query.p.value, query.o.value))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node {uri: $uri}) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $rel, user: $user, collection: $collection}]->"
+                            "(dest:Node {uri: $uri, user: $user, collection: $collection}) "
                             "RETURN $src as src "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, rel=query.p.value, uri=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -90,10 +100,13 @@ class Processor(TriplesQueryService):
                         # SP
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $rel, user: $user, collection: $collection}]->"
+                            "(dest:Literal {user: $user, collection: $collection}) "
                             "RETURN dest.value as dest "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, rel=query.p.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -102,10 +115,13 @@ class Processor(TriplesQueryService):
                             triples.append((query.s.value, query.p.value, data["dest"]))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $rel, user: $user, collection: $collection}]->"
+                            "(dest:Node {user: $user, collection: $collection}) "
                             "RETURN dest.uri as dest "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, rel=query.p.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -120,10 +136,13 @@ class Processor(TriplesQueryService):
                         # SO
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal {value: $value}) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Literal {value: $value, user: $user, collection: $collection}) "
                             "RETURN rel.uri as rel "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, value=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -132,10 +151,13 @@ class Processor(TriplesQueryService):
                             triples.append((query.s.value, data["rel"], query.o.value))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node {uri: $uri}) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Node {uri: $uri, user: $user, collection: $collection}) "
                             "RETURN rel.uri as rel "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, uri=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -148,10 +170,13 @@ class Processor(TriplesQueryService):
                         # S
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Literal {user: $user, collection: $collection}) "
                             "RETURN rel.uri as rel, dest.value as dest "
                             "LIMIT " + str(query.limit),
                             src=query.s.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -160,10 +185,13 @@ class Processor(TriplesQueryService):
                             triples.append((query.s.value, data["rel"], data["dest"]))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Node {user: $user, collection: $collection}) "
                             "RETURN rel.uri as rel, dest.uri as dest "
                             "LIMIT " + str(query.limit),
                             src=query.s.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -181,10 +209,13 @@ class Processor(TriplesQueryService):
                         # PO
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Literal {value: $value}) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $uri, user: $user, collection: $collection}]->"
+                            "(dest:Literal {value: $value, user: $user, collection: $collection}) "
                             "RETURN src.uri as src "
                             "LIMIT " + str(query.limit),
                             uri=query.p.value, value=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -193,10 +224,13 @@ class Processor(TriplesQueryService):
                             triples.append((data["src"], query.p.value, query.o.value))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Node {uri: $dest}) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $uri, user: $user, collection: $collection}]->"
+                            "(dest:Node {uri: $dest, user: $user, collection: $collection}) "
                             "RETURN src.uri as src "
                             "LIMIT " + str(query.limit),
                             uri=query.p.value, dest=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -209,10 +243,13 @@ class Processor(TriplesQueryService):
                         # P
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Literal) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $uri, user: $user, collection: $collection}]->"
+                            "(dest:Literal {user: $user, collection: $collection}) "
                             "RETURN src.uri as src, dest.value as dest "
                             "LIMIT " + str(query.limit),
                             uri=query.p.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -221,10 +258,13 @@ class Processor(TriplesQueryService):
                             triples.append((data["src"], query.p.value, data["dest"]))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Node) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $uri, user: $user, collection: $collection}]->"
+                            "(dest:Node {user: $user, collection: $collection}) "
                             "RETURN src.uri as src, dest.uri as dest "
                             "LIMIT " + str(query.limit),
                             uri=query.p.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -239,10 +279,13 @@ class Processor(TriplesQueryService):
                         # O
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel]->(dest:Literal {value: $value}) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Literal {value: $value, user: $user, collection: $collection}) "
                             "RETURN src.uri as src, rel.uri as rel "
                             "LIMIT " + str(query.limit),
                             value=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -251,10 +294,13 @@ class Processor(TriplesQueryService):
                             triples.append((data["src"], data["rel"], query.o.value))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel]->(dest:Node {uri: $uri}) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Node {uri: $uri, user: $user, collection: $collection}) "
                             "RETURN src.uri as src, rel.uri as rel "
                             "LIMIT " + str(query.limit),
                             uri=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -267,9 +313,12 @@ class Processor(TriplesQueryService):
                         # *
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel]->(dest:Literal) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Literal {user: $user, collection: $collection}) "
                             "RETURN src.uri as src, rel.uri as rel, dest.value as dest "
                             "LIMIT " + str(query.limit),
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -278,9 +327,12 @@ class Processor(TriplesQueryService):
                             triples.append((data["src"], data["rel"], data["dest"]))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel]->(dest:Node) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Node {user: $user, collection: $collection}) "
                             "RETURN src.uri as src, rel.uri as rel, dest.uri as dest "
                             "LIMIT " + str(query.limit),
+                            user=user, collection=collection,
                             database_=self.db,
                         )