trustgraph-flow 1.3.17__tar.gz → 1.4.0__tar.gz

{trustgraph_flow-1.3.17 → trustgraph_flow-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-flow
-Version: 1.3.17
+Version: 1.4.0
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph
@@ -8,7 +8,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: trustgraph-base<1.4,>=1.3
+Requires-Dist: trustgraph-base<1.5,>=1.4
 Requires-Dist: aiohttp
 Requires-Dist: anthropic
 Requires-Dist: cassandra-driver

{trustgraph_flow-1.3.17 → trustgraph_flow-1.4.0}/pyproject.toml

@@ -10,7 +10,7 @@ description = "TrustGraph provides a means to run a pipeline of flexible AI proc
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
-    "trustgraph-base>=1.3,<1.4",
+    "trustgraph-base>=1.4,<1.5",
     "aiohttp",
     "anthropic",
     "cassandra-driver",
@@ -90,7 +90,6 @@ metering = "trustgraph.metering:run"
 nlp-query = "trustgraph.retrieval.nlp_query:run"
 objects-write-cassandra = "trustgraph.storage.objects.cassandra:run"
 objects-query-cassandra = "trustgraph.query.objects.cassandra:run"
-oe-write-milvus = "trustgraph.storage.object_embeddings.milvus:run"
 pdf-decoder = "trustgraph.decoding.pdf:run"
 pdf-ocr-mistral = "trustgraph.decoding.mistral_ocr:run"
 prompt-template = "trustgraph.prompt.template:run"

{trustgraph_flow-1.3.17 → trustgraph_flow-1.4.0}/trustgraph/direct/milvus_doc_embeddings.py

@@ -2,9 +2,32 @@
 from pymilvus import MilvusClient, CollectionSchema, FieldSchema, DataType
 import time
 import logging
+import re
 
 logger = logging.getLogger(__name__)
 
+def make_safe_collection_name(user, collection, dimension, prefix):
+    """
+    Create a safe Milvus collection name from user/collection parameters.
+    Milvus only allows letters, numbers, and underscores.
+    """
+    def sanitize(s):
+        # Replace non-alphanumeric characters (except underscore) with underscore
+        # Then collapse multiple underscores into single underscore
+        safe = re.sub(r'[^a-zA-Z0-9_]', '_', s)
+        safe = re.sub(r'_+', '_', safe)
+        # Remove leading/trailing underscores
+        safe = safe.strip('_')
+        # Ensure it's not empty
+        if not safe:
+            safe = 'default'
+        return safe
+    
+    safe_user = sanitize(user)
+    safe_collection = sanitize(collection)
+    
+    return f"{prefix}_{safe_user}_{safe_collection}_{dimension}"
+
 class DocVectors:
 
     def __init__(self, uri="http://localhost:19530", prefix='doc'):
@@ -26,9 +49,9 @@ class DocVectors:
         self.next_reload = time.time() + self.reload_time
         logger.debug(f"Reload at {self.next_reload}")
 
-    def init_collection(self, dimension):
+    def init_collection(self, dimension, user, collection):
 
-        collection_name = self.prefix + "_" + str(dimension)
+        collection_name = make_safe_collection_name(user, collection, dimension, self.prefix)
 
         pkey_field = FieldSchema(
             name="id",
@@ -75,14 +98,14 @@ class DocVectors:
             index_params=index_params
         )
 
-        self.collections[dimension] = collection_name
+        self.collections[(dimension, user, collection)] = collection_name
 
-    def insert(self, embeds, doc):
+    def insert(self, embeds, doc, user, collection):
 
         dim = len(embeds)
 
-        if dim not in self.collections:
-            self.init_collection(dim)
+        if (dim, user, collection) not in self.collections:
+            self.init_collection(dim, user, collection)
     
         data = [
             {
@@ -92,18 +115,18 @@ class DocVectors:
         ]
 
         self.client.insert(
-            collection_name=self.collections[dim],
+            collection_name=self.collections[(dim, user, collection)],
             data=data
         )
 
-    def search(self, embeds, fields=["doc"], limit=10):
+    def search(self, embeds, user, collection, fields=["doc"], limit=10):
 
         dim = len(embeds)
 
-        if dim not in self.collections:
-            self.init_collection(dim)
+        if (dim, user, collection) not in self.collections:
+            self.init_collection(dim, user, collection)
 
-        coll = self.collections[dim]
+        coll = self.collections[(dim, user, collection)]
 
         search_params = {
             "metric_type": "COSINE",

{trustgraph_flow-1.3.17 → trustgraph_flow-1.4.0}/trustgraph/direct/milvus_graph_embeddings.py

@@ -2,9 +2,32 @@
 from pymilvus import MilvusClient, CollectionSchema, FieldSchema, DataType
 import time
 import logging
+import re
 
 logger = logging.getLogger(__name__)
 
+def make_safe_collection_name(user, collection, dimension, prefix):
+    """
+    Create a safe Milvus collection name from user/collection parameters.
+    Milvus only allows letters, numbers, and underscores.
+    """
+    def sanitize(s):
+        # Replace non-alphanumeric characters (except underscore) with underscore
+        # Then collapse multiple underscores into single underscore
+        safe = re.sub(r'[^a-zA-Z0-9_]', '_', s)
+        safe = re.sub(r'_+', '_', safe)
+        # Remove leading/trailing underscores
+        safe = safe.strip('_')
+        # Ensure it's not empty
+        if not safe:
+            safe = 'default'
+        return safe
+    
+    safe_user = sanitize(user)
+    safe_collection = sanitize(collection)
+    
+    return f"{prefix}_{safe_user}_{safe_collection}_{dimension}"
+
 class EntityVectors:
 
     def __init__(self, uri="http://localhost:19530", prefix='entity'):
@@ -26,9 +49,9 @@ class EntityVectors:
         self.next_reload = time.time() + self.reload_time
         logger.debug(f"Reload at {self.next_reload}")
 
-    def init_collection(self, dimension):
+    def init_collection(self, dimension, user, collection):
 
-        collection_name = self.prefix + "_" + str(dimension)
+        collection_name = make_safe_collection_name(user, collection, dimension, self.prefix)
 
         pkey_field = FieldSchema(
             name="id",
@@ -75,14 +98,14 @@ class EntityVectors:
             index_params=index_params
         )
 
-        self.collections[dimension] = collection_name
+        self.collections[(dimension, user, collection)] = collection_name
 
-    def insert(self, embeds, entity):
+    def insert(self, embeds, entity, user, collection):
 
         dim = len(embeds)
 
-        if dim not in self.collections:
-            self.init_collection(dim)
+        if (dim, user, collection) not in self.collections:
+            self.init_collection(dim, user, collection)
     
         data = [
             {
@@ -92,18 +115,18 @@ class EntityVectors:
         ]
 
         self.client.insert(
-            collection_name=self.collections[dim],
+            collection_name=self.collections[(dim, user, collection)],
             data=data
         )
 
-    def search(self, embeds, fields=["entity"], limit=10):
+    def search(self, embeds, user, collection, fields=["entity"], limit=10):
 
         dim = len(embeds)
 
-        if dim not in self.collections:
-            self.init_collection(dim)
+        if (dim, user, collection) not in self.collections:
+            self.init_collection(dim, user, collection)
 
-        coll = self.collections[dim]
+        coll = self.collections[(dim, user, collection)]
 
         search_params = {
             "metric_type": "COSINE",

trustgraph_flow-1.4.0/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "1.4.0"

{trustgraph_flow-1.3.17 → trustgraph_flow-1.4.0}/trustgraph/query/doc_embeddings/milvus/service.py

@@ -43,7 +43,12 @@ class Processor(DocumentEmbeddingsQueryService):
 
             for vec in msg.vectors:
 
-                resp = self.vecstore.search(vec, limit=msg.limit)
+                resp = self.vecstore.search(
+                    vec, 
+                    msg.user, 
+                    msg.collection, 
+                    limit=msg.limit
+                )
 
                 for r in resp:
                     chunk = r["entity"]["doc"]

{trustgraph_flow-1.3.17 → trustgraph_flow-1.4.0}/trustgraph/query/graph_embeddings/milvus/service.py

@@ -50,7 +50,12 @@ class Processor(GraphEmbeddingsQueryService):
 
             for vec in msg.vectors:
 
-                resp = self.vecstore.search(vec, limit=msg.limit * 2)
+                resp = self.vecstore.search(
+                    vec, 
+                    msg.user, 
+                    msg.collection, 
+                    limit=msg.limit * 2
+                )
 
                 for r in resp:
                     ent = r["entity"]["entity"]

{trustgraph_flow-1.3.17 → trustgraph_flow-1.4.0}/trustgraph/query/triples/memgraph/service.py

@@ -55,6 +55,10 @@ class Processor(TriplesQueryService):
 
         try:
 
+            # Extract user and collection, use defaults if not provided
+            user = query.user if query.user else "default"
+            collection = query.collection if query.collection else "default"
+            
             triples = []
 
             if query.s is not None:
@@ -64,10 +68,13 @@ class Processor(TriplesQueryService):
                         # SPO
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal {value: $value}) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $rel, user: $user, collection: $collection}]->"
+                            "(dest:Literal {value: $value, user: $user, collection: $collection}) "
                             "RETURN $src as src "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, rel=query.p.value, value=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -75,10 +82,13 @@ class Processor(TriplesQueryService):
                             triples.append((query.s.value, query.p.value, query.o.value))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node {uri: $uri}) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $rel, user: $user, collection: $collection}]->"
+                            "(dest:Node {uri: $uri, user: $user, collection: $collection}) "
                             "RETURN $src as src "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, rel=query.p.value, uri=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -90,10 +100,13 @@ class Processor(TriplesQueryService):
                         # SP
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Literal) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $rel, user: $user, collection: $collection}]->"
+                            "(dest:Literal {user: $user, collection: $collection}) "
                             "RETURN dest.value as dest "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, rel=query.p.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -102,10 +115,13 @@ class Processor(TriplesQueryService):
                             triples.append((query.s.value, query.p.value, data["dest"]))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel {uri: $rel}]->(dest:Node) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $rel, user: $user, collection: $collection}]->"
+                            "(dest:Node {user: $user, collection: $collection}) "
                             "RETURN dest.uri as dest "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, rel=query.p.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -120,10 +136,13 @@ class Processor(TriplesQueryService):
                         # SO
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal {value: $value}) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Literal {value: $value, user: $user, collection: $collection}) "
                             "RETURN rel.uri as rel "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, value=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -132,10 +151,13 @@ class Processor(TriplesQueryService):
                             triples.append((query.s.value, data["rel"], query.o.value))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node {uri: $uri}) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Node {uri: $uri, user: $user, collection: $collection}) "
                             "RETURN rel.uri as rel "
                             "LIMIT " + str(query.limit),
                             src=query.s.value, uri=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -148,10 +170,13 @@ class Processor(TriplesQueryService):
                         # S
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Literal) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Literal {user: $user, collection: $collection}) "
                             "RETURN rel.uri as rel, dest.value as dest "
                             "LIMIT " + str(query.limit),
                             src=query.s.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -160,10 +185,13 @@ class Processor(TriplesQueryService):
                             triples.append((query.s.value, data["rel"], data["dest"]))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node {uri: $src})-[rel:Rel]->(dest:Node) "
+                            "MATCH (src:Node {uri: $src, user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Node {user: $user, collection: $collection}) "
                             "RETURN rel.uri as rel, dest.uri as dest "
                             "LIMIT " + str(query.limit),
                             src=query.s.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -181,10 +209,13 @@ class Processor(TriplesQueryService):
                         # PO
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Literal {value: $value}) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $uri, user: $user, collection: $collection}]->"
+                            "(dest:Literal {value: $value, user: $user, collection: $collection}) "
                             "RETURN src.uri as src "
                             "LIMIT " + str(query.limit),
                             uri=query.p.value, value=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -193,10 +224,13 @@ class Processor(TriplesQueryService):
                             triples.append((data["src"], query.p.value, query.o.value))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Node {uri: $dest}) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $uri, user: $user, collection: $collection}]->"
+                            "(dest:Node {uri: $dest, user: $user, collection: $collection}) "
                             "RETURN src.uri as src "
                             "LIMIT " + str(query.limit),
                             uri=query.p.value, dest=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -209,10 +243,13 @@ class Processor(TriplesQueryService):
                         # P
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Literal) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $uri, user: $user, collection: $collection}]->"
+                            "(dest:Literal {user: $user, collection: $collection}) "
                             "RETURN src.uri as src, dest.value as dest "
                             "LIMIT " + str(query.limit),
                             uri=query.p.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -221,10 +258,13 @@ class Processor(TriplesQueryService):
                             triples.append((data["src"], query.p.value, data["dest"]))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel {uri: $uri}]->(dest:Node) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {uri: $uri, user: $user, collection: $collection}]->"
+                            "(dest:Node {user: $user, collection: $collection}) "
                             "RETURN src.uri as src, dest.uri as dest "
                             "LIMIT " + str(query.limit),
                             uri=query.p.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -239,10 +279,13 @@ class Processor(TriplesQueryService):
                         # O
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel]->(dest:Literal {value: $value}) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Literal {value: $value, user: $user, collection: $collection}) "
                             "RETURN src.uri as src, rel.uri as rel "
                             "LIMIT " + str(query.limit),
                             value=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -251,10 +294,13 @@ class Processor(TriplesQueryService):
                             triples.append((data["src"], data["rel"], query.o.value))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel]->(dest:Node {uri: $uri}) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Node {uri: $uri, user: $user, collection: $collection}) "
                             "RETURN src.uri as src, rel.uri as rel "
                             "LIMIT " + str(query.limit),
                             uri=query.o.value,
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -267,9 +313,12 @@ class Processor(TriplesQueryService):
                         # *
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel]->(dest:Literal) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Literal {user: $user, collection: $collection}) "
                             "RETURN src.uri as src, rel.uri as rel, dest.value as dest "
                             "LIMIT " + str(query.limit),
+                            user=user, collection=collection,
                             database_=self.db,
                         )
 
@@ -278,9 +327,12 @@ class Processor(TriplesQueryService):
                             triples.append((data["src"], data["rel"], data["dest"]))
 
                         records, summary, keys = self.io.execute_query(
-                            "MATCH (src:Node)-[rel:Rel]->(dest:Node) "
+                            "MATCH (src:Node {user: $user, collection: $collection})-"
+                            "[rel:Rel {user: $user, collection: $collection}]->"
+                            "(dest:Node {user: $user, collection: $collection}) "
                             "RETURN src.uri as src, rel.uri as rel, dest.uri as dest "
                             "LIMIT " + str(query.limit),
+                            user=user, collection=collection,
                             database_=self.db,
                         )