trustgraph-flow 2.8.1__tar.gz → 2.8.3__tar.gz

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-flow
-Version: 2.8.1
+Version: 2.8.3
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/pyproject.toml

@@ -91,7 +91,6 @@ kg-extract-agent = "trustgraph.extract.kg.agent:run"
 kg-extract-definitions = "trustgraph.extract.kg.definitions:run"
 kg-extract-rows = "trustgraph.extract.kg.rows:run"
 kg-extract-relationships = "trustgraph.extract.kg.relationships:run"
-kg-extract-topics = "trustgraph.extract.kg.topics:run"
 kg-extract-ontology = "trustgraph.extract.kg.ontology:run"
 kg-manager = "trustgraph.cores:run"
 kg-store = "trustgraph.storage.knowledge:run"

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/cores/knowledge.py

@@ -5,7 +5,6 @@ from .. schema import LibrarianRequest, DocumentMetadata
 from .. knowledge import hash
 from .. exceptions import RequestError
 from .. tables.knowledge import KnowledgeTableStore
-from .. base import Publisher
 
 import base64
 import asyncio
@@ -491,32 +490,25 @@ class KnowledgeManager:
             )
         )
 
-        t_pub = None
-        ge_pub = None
+        t_prod = None
+        ge_prod = None
 
         try:
 
             logger.debug(f"Triples queue: {t_q}")
             logger.debug(f"Graph embeddings queue: {ge_q}")
 
-            t_pub = Publisher(
-                self.flow_config.pubsub, t_q,
-                schema=Triples,
+            t_prod = await self.flow_config.pubsub.create_producer(
+                topic=t_q, schema=Triples,
             )
-            ge_pub = Publisher(
-                self.flow_config.pubsub, ge_q,
-                schema=GraphEmbeddings
+            ge_prod = await self.flow_config.pubsub.create_producer(
+                topic=ge_q, schema=GraphEmbeddings,
             )
 
-            logger.debug("Starting publishers...")
-
-            await t_pub.start()
-            await ge_pub.start()
-
             async def publish_triples(t):
                 if hasattr(t, 'metadata') and hasattr(t.metadata, 'collection'):
                     t.metadata.collection = request.collection or "default"
-                await t_pub.send(None, t)
+                await t_prod.send(t)
 
             logger.debug("Publishing triples...")
 
@@ -529,7 +521,7 @@ class KnowledgeManager:
             async def publish_ge(g):
                 if hasattr(g, 'metadata') and hasattr(g.metadata, 'collection'):
                     g.metadata.collection = request.collection or "default"
-                await ge_pub.send(None, g)
+                await ge_prod.send(g)
 
             logger.debug("Publishing graph embeddings...")
 
@@ -547,10 +539,8 @@ class KnowledgeManager:
 
         finally:
 
-            logger.debug("Stopping publishers...")
-
-            if t_pub: await t_pub.stop()
-            if ge_pub: await ge_pub.stop()
+            if t_prod: await t_prod.close()
+            if ge_prod: await ge_prod.close()
 
     async def _load_de_core(self, request, respond, workspace, flow):
 
@@ -569,25 +559,20 @@ class KnowledgeManager:
             )
         )
 
-        de_pub = None
+        de_prod = None
 
         try:
 
             logger.debug(f"Document embeddings queue: {de_q}")
 
-            de_pub = Publisher(
-                self.flow_config.pubsub, de_q,
-                schema=DocumentEmbeddings,
+            de_prod = await self.flow_config.pubsub.create_producer(
+                topic=de_q, schema=DocumentEmbeddings,
             )
 
-            logger.debug("Starting publisher...")
-
-            await de_pub.start()
-
             async def publish_de(de):
                 if hasattr(de, 'metadata') and hasattr(de.metadata, 'collection'):
                     de.metadata.collection = request.collection or "default"
-                await de_pub.send(None, de)
+                await de_prod.send(de)
 
             logger.debug("Publishing document embeddings...")
 
@@ -605,6 +590,4 @@ class KnowledgeManager:
 
         finally:
 
-            logger.debug("Stopping publisher...")
-
-            if de_pub: await de_pub.stop()
+            if de_prod: await de_prod.close()

trustgraph_flow-2.8.3/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "2.8.3"

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/gateway/service.py

@@ -127,7 +127,6 @@ class Api:
         self.audit_publisher = AuditPublisher(
             async_backend=self.pubsub_backend,
             component_name="api-gateway",
-            processor_id=config.get("id", "api-gateway"),
         )
 
         self.endpoint_manager = EndpointManager(

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/iam/service/service.py

@@ -120,7 +120,6 @@ class Processor(AsyncProcessor):
 
         self.audit = AuditPublisher(
             component_name="iam-service",
-            processor_id=self.id,
         )
 
         self.iam = IamService(

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/storage/doc_embeddings/milvus/write.py

@@ -7,7 +7,7 @@ import logging
 
 from .... direct.milvus_doc_embeddings import DocVectors
 from .... base import DocumentEmbeddingsStoreService, CollectionConfigHandler
-from .... base import AsyncProcessor, Consumer, Producer
+from .... base import AsyncProcessor
 from .... base import ConsumerMetrics, ProducerMetrics
 
 # Module logger

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/storage/doc_embeddings/pinecone/write.py

@@ -12,7 +12,7 @@ import os
 import logging
 
 from .... base import DocumentEmbeddingsStoreService, CollectionConfigHandler
-from .... base import AsyncProcessor, Consumer, Producer
+from .... base import AsyncProcessor
 from .... base import ConsumerMetrics, ProducerMetrics
 
 # Module logger

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/storage/doc_embeddings/qdrant/write.py

@@ -12,7 +12,7 @@ from qdrant_client.models import PointStruct
 from qdrant_client.models import Distance, VectorParams
 
 from .... base import DocumentEmbeddingsStoreService, CollectionConfigHandler
-from .... base import AsyncProcessor, Consumer, Producer
+from .... base import AsyncProcessor
 from .... base import ConsumerMetrics, ProducerMetrics
 from .... base.qdrant_config import add_qdrant_args, resolve_qdrant_config
 

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/storage/graph_embeddings/milvus/write.py

@@ -7,7 +7,7 @@ import logging
 
 from .... direct.milvus_graph_embeddings import EntityVectors
 from .... base import GraphEmbeddingsStoreService, CollectionConfigHandler
-from .... base import AsyncProcessor, Consumer, Producer
+from .... base import AsyncProcessor
 from .... base import ConsumerMetrics, ProducerMetrics
 from .... schema import IRI, LITERAL
 

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/storage/graph_embeddings/pinecone/write.py

@@ -12,7 +12,7 @@ import os
 import logging
 
 from .... base import GraphEmbeddingsStoreService, CollectionConfigHandler
-from .... base import AsyncProcessor, Consumer, Producer
+from .... base import AsyncProcessor
 from .... base import ConsumerMetrics, ProducerMetrics
 from .... schema import IRI, LITERAL
 

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/storage/graph_embeddings/qdrant/write.py

@@ -12,7 +12,7 @@ from qdrant_client.models import PointStruct
 from qdrant_client.models import Distance, VectorParams
 
 from .... base import GraphEmbeddingsStoreService, CollectionConfigHandler
-from .... base import AsyncProcessor, Consumer, Producer
+from .... base import AsyncProcessor
 from .... base import ConsumerMetrics, ProducerMetrics
 from .... base.qdrant_config import add_qdrant_args, resolve_qdrant_config
 from .... schema import IRI, LITERAL

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/storage/triples/cassandra/write.py

@@ -10,7 +10,7 @@ from .... direct.cassandra_kg import (
     EntityCentricKnowledgeGraph, DEFAULT_GRAPH
 )
 from .... base import TriplesStoreService, CollectionConfigHandler
-from .... base import AsyncProcessor, Consumer, Producer
+from .... base import AsyncProcessor
 from .... base import ConsumerMetrics, ProducerMetrics
 from .... base.cassandra_config import add_cassandra_args, resolve_cassandra_config
 from .... schema import IRI, LITERAL, BLANK, TRIPLE

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/storage/triples/falkordb/write.py

@@ -12,7 +12,7 @@ import logging
 from falkordb import FalkorDB
 
 from .... base import TriplesStoreService, CollectionConfigHandler
-from .... base import AsyncProcessor, Consumer, Producer
+from .... base import AsyncProcessor
 from .... base import ConsumerMetrics, ProducerMetrics
 from .... schema import IRI, LITERAL
 

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/storage/triples/memgraph/write.py

@@ -12,7 +12,7 @@ import logging
 from neo4j import GraphDatabase
 
 from .... base import TriplesStoreService, CollectionConfigHandler
-from .... base import AsyncProcessor, Consumer, Producer
+from .... base import AsyncProcessor
 from .... base import ConsumerMetrics, ProducerMetrics
 from .... schema import IRI, LITERAL
 

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph/storage/triples/neo4j/write.py

@@ -11,7 +11,7 @@ import logging
 
 from neo4j import GraphDatabase
 from .... base import TriplesStoreService, CollectionConfigHandler
-from .... base import AsyncProcessor, Consumer, Producer
+from .... base import AsyncProcessor
 from .... base import ConsumerMetrics, ProducerMetrics
 from .... schema import IRI, LITERAL
 

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph_flow.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-flow
-Version: 2.8.1
+Version: 2.8.3
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph_flow.egg-info/SOURCES.txt

@@ -157,7 +157,6 @@ trustgraph/gateway/dispatch/rows_query.py
 trustgraph/gateway/dispatch/sender.py
 trustgraph/gateway/dispatch/serialize.py
 trustgraph/gateway/dispatch/sparql_query.py
-trustgraph/gateway/dispatch/streamer.py
 trustgraph/gateway/dispatch/structured_diag.py
 trustgraph/gateway/dispatch/structured_query.py
 trustgraph/gateway/dispatch/text_completion.py

{trustgraph_flow-2.8.1 → trustgraph_flow-2.8.3}/trustgraph_flow.egg-info/entry_points.txt

@@ -33,7 +33,6 @@ kg-extract-definitions = trustgraph.extract.kg.definitions:run
 kg-extract-ontology = trustgraph.extract.kg.ontology:run
 kg-extract-relationships = trustgraph.extract.kg.relationships:run
 kg-extract-rows = trustgraph.extract.kg.rows:run
-kg-extract-topics = trustgraph.extract.kg.topics:run
 kg-manager = trustgraph.cores:run
 kg-store = trustgraph.storage.knowledge:run
 kw-index-fts5 = trustgraph.storage.kw_index.fts5:run

trustgraph_flow-2.8.1/trustgraph/flow_version.py

@@ -1 +0,0 @@
-__version__ = "2.8.1"

trustgraph_flow-2.8.1/trustgraph/gateway/dispatch/streamer.py

@@ -1,97 +0,0 @@
-
-import asyncio
-import uuid
-import logging
-
-from ... base import Publisher
-from ... base import Subscriber
-
-logger = logging.getLogger("requestor")
-logger.setLevel(logging.INFO)
-
-class ServiceRequestor:
-
-    def __init__(
-            self,
-            backend,
-            queue, schema,
-            handler,
-            subscription="api-gateway", consumer_name="api-gateway",
-            timeout=600,
-    ):
-
-        self.sub = Subscriber(
-            backend, queue,
-            subscription, consumer_name,
-            schema
-        )
-
-        self.timeout = timeout
-
-        self.running = True
-
-        self.receiver = handler
-
-    async def start(self):
-        await self.sub.start()
-        self.streamer = asyncio.create_task(self.stream())
-        sub.start()
-        self.running = True
-
-    async def stop(self):
-        await self.sub.stop()
-        self.running = False
-
-    def from_inbound(self, response):
-        raise RuntimeError("Not defined")
-
-    async def stream(self):
-
-        id = str(uuid.uuid4())
-
-        try:
-
-            q = await self.sub.subscribe(id)
-
-            while self.running:
-
-                try:
-                    resp = await asyncio.wait_for(
-                        q.get(), timeout=self.timeout
-                    )
-                except Exception as e:
-                    raise RuntimeError("Timeout")
-
-                if resp.error:
-                    err = { "error": {
-                        "type": resp.error.type,
-                        "message": resp.error.message,
-                    } }
-
-                    fin = False
-
-                    await self.receiver(err, fin)
-
-                else:
-
-                    resp, fin = self.from_inbound(resp)
-
-                    await self.receiver(resp, fin)
-
-                if fin: break
-
-        except Exception as e:
-
-            logging.error(f"Exception: {e}")
-
-            err = { "error": {
-                "type": "gateway-error",
-                "message": str(e),
-            } }
-            if responder:
-                await responder(err, True)
-            return err
-
-        finally:
-            await self.sub.unsubscribe(id)
-