trustgraph-flow 0.17.16__tar.gz → 0.18.1__tar.gz

{trustgraph-flow-0.17.16 → trustgraph-flow-0.18.1}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.17.16
+Version: 0.18.1
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 License: UNKNOWN
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.17.16.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.18.1.tar.gz
 Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)

{trustgraph-flow-0.17.16 → trustgraph-flow-0.18.1}/setup.py

@@ -34,7 +34,7 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
-        "trustgraph-base>=0.17,<0.18",
+        "trustgraph-base>=0.18,<0.19",
         "urllib3",
         "rdflib",
         "pymilvus",

trustgraph-flow-0.18.1/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "0.18.1"

trustgraph-flow-0.18.1/trustgraph/gateway/graph_embeddings_query.py

@@ -0,0 +1,40 @@
+
+from .. schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse
+from .. schema import graph_embeddings_request_queue
+from .. schema import graph_embeddings_response_queue
+
+from . endpoint import ServiceEndpoint
+from . requestor import ServiceRequestor
+from . serialize import serialize_value
+
+class GraphEmbeddingsQueryRequestor(ServiceRequestor):
+    def __init__(self, pulsar_host, timeout, auth):
+
+        super(GraphEmbeddingsQueryRequestor, self).__init__(
+            pulsar_host=pulsar_host,
+            request_queue=graph_embeddings_request_queue,
+            response_queue=graph_embeddings_response_queue,
+            request_schema=GraphEmbeddingsRequest,
+            response_schema=GraphEmbeddingsResponse,
+            timeout=timeout,
+        )
+
+    def to_request(self, body):
+
+        limit = int(body.get("limit", 20))
+
+        return GraphEmbeddingsRequest(
+            vectors = body["vectors"],
+            limit = limit,
+            user = body.get("user", "trustgraph"),
+            collection = body.get("collection", "default"),
+        )
+
+    def from_response(self, message):
+
+        return {
+            "entities": [
+                serialize_value(ent) for ent in message.entities
+            ]
+        }, True
+

{trustgraph-flow-0.17.16 → trustgraph-flow-0.18.1}/trustgraph/gateway/service.py

@@ -35,6 +35,7 @@ from . text_completion import TextCompletionRequestor
 from . prompt import PromptRequestor
 from . graph_rag import GraphRagRequestor
 from . triples_query import TriplesQueryRequestor
+from . graph_embeddings_query import GraphEmbeddingsQueryRequestor
 from . embeddings import EmbeddingsRequestor
 from . encyclopedia import EncyclopediaRequestor
 from . agent import AgentRequestor
@@ -95,6 +96,10 @@ class Api:
                 pulsar_host=self.pulsar_host, timeout=self.timeout,
                 auth = self.auth,
             ),
+            "graph-embeddings-query": GraphEmbeddingsQueryRequestor(
+                pulsar_host=self.pulsar_host, timeout=self.timeout,
+                auth = self.auth,
+            ),
             "embeddings": EmbeddingsRequestor(
                 pulsar_host=self.pulsar_host, timeout=self.timeout,
                 auth = self.auth,
@@ -134,6 +139,11 @@ class Api:
                 endpoint_path = "/api/v1/triples-query", auth=self.auth,
                 requestor = self.services["triples-query"],
             ),
+            ServiceEndpoint(
+                endpoint_path = "/api/v1/graph-embeddings-query",
+                auth=self.auth,
+                requestor = self.services["graph-embeddings-query"],
+            ),
             ServiceEndpoint(
                 endpoint_path = "/api/v1/embeddings", auth=self.auth,
                 requestor = self.services["embeddings"],

{trustgraph-flow-0.17.16 → trustgraph-flow-0.18.1}/trustgraph/query/graph_embeddings/pinecone/service.py

@@ -73,7 +73,8 @@ class Processor(ConsumerProducer):
 
             print(f"Handling input {id}...", flush=True)
 
-            entities = set()
+            entity_set = set()
+            entities = []
 
             for vec in v.vectors:
 
@@ -85,20 +86,30 @@ class Processor(ConsumerProducer):
 
                 index = self.pinecone.Index(index_name)
 
+                # Heuristic hack, get (2*limit), so that we have more chance
+                # of getting (limit) entities
                 results = index.query(
                     namespace=v.collection,
                     vector=vec,
-                    top_k=v.limit,
+                    top_k=v.limit * 2,
                     include_values=False,
                     include_metadata=True
                 )
 
                 for r in results.matches:
+
                     ent = r.metadata["entity"]
-                    entities.add(ent)
 
-            # Convert set to list
-            entities = list(entities)
+                    # De-dupe entities
+                    if ent not in entity_set:
+                        entity_set.add(ent)
+                        entities.append(ent)
+
+                    # Keep adding entities until limit
+                    if len(entity_set) >= v.limit: break
+
+                # Keep adding entities until limit
+                if len(entity_set) >= v.limit: break
 
             ents2 = []
 

{trustgraph-flow-0.17.16 → trustgraph-flow-0.18.1}/trustgraph/query/graph_embeddings/qdrant/service.py

@@ -61,7 +61,8 @@ class Processor(ConsumerProducer):
 
             print(f"Handling input {id}...", flush=True)
 
-            entities = set()
+            entity_set = set()
+            entities = []
 
             for vec in v.vectors:
 
@@ -71,19 +72,28 @@ class Processor(ConsumerProducer):
                     str(dim)
                 )
 
+                # Heuristic hack, get (2*limit), so that we have more chance
+                # of getting (limit) entities
                 search_result = self.client.query_points(
                     collection_name=collection,
                     query=vec,
-                    limit=v.limit,
+                    limit=v.limit * 2,
                     with_payload=True,
                 ).points
 
                 for r in search_result:
                     ent = r.payload["entity"]
-                    entities.add(ent)
 
-            # Convert set to list
-            entities = list(entities)
+                    # De-dupe entities
+                    if ent not in entity_set:
+                        entity_set.add(ent)
+                        entities.append(ent)
+
+                    # Keep adding entities until limit
+                    if len(entity_set) >= v.limit: break
+
+                # Keep adding entities until limit
+                if len(entity_set) >= v.limit: break
 
             ents2 = []
 

{trustgraph-flow-0.17.16 → trustgraph-flow-0.18.1}/trustgraph/storage/graph_embeddings/qdrant/write.py

@@ -62,7 +62,7 @@ class Processor(Consumer):
                         self.client.create_collection(
                             collection_name=collection,
                             vectors_config=VectorParams(
-                                size=dim, distance=Distance.DOT
+                                size=dim, distance=Distance.COSINE
                             ),
                         )
                     except Exception as e:

{trustgraph-flow-0.17.16 → trustgraph-flow-0.18.1}/trustgraph_flow.egg-info/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: trustgraph-flow
-Version: 0.17.16
+Version: 0.18.1
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Home-page: https://github.com/trustgraph-ai/trustgraph
 Author: trustgraph.ai
 Author-email: security@trustgraph.ai
 License: UNKNOWN
-Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.17.16.tar.gz
+Download-URL: https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v0.18.1.tar.gz
 Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)

{trustgraph-flow-0.17.16 → trustgraph-flow-0.18.1}/trustgraph_flow.egg-info/SOURCES.txt

@@ -108,6 +108,7 @@ trustgraph/gateway/embeddings.py
 trustgraph/gateway/encyclopedia.py
 trustgraph/gateway/endpoint.py
 trustgraph/gateway/graph_embeddings_load.py
+trustgraph/gateway/graph_embeddings_query.py
 trustgraph/gateway/graph_embeddings_stream.py
 trustgraph/gateway/graph_rag.py
 trustgraph/gateway/internet_search.py

{trustgraph-flow-0.17.16 → trustgraph-flow-0.18.1}/trustgraph_flow.egg-info/requires.txt

@@ -22,5 +22,5 @@ rdflib
 requests
 tabulate
 tiktoken
-trustgraph-base<0.18,>=0.17
+trustgraph-base<0.19,>=0.18
 urllib3

trustgraph-flow-0.17.16/trustgraph/flow_version.py

@@ -1 +0,0 @@
-__version__ = "0.17.16"