trustgraph-flow 1.3.5__tar.gz → 1.3.7__tar.gz

{trustgraph_flow-1.3.5 → trustgraph_flow-1.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-flow
-Version: 1.3.5
+Version: 1.3.7
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_flow-1.3.5 → trustgraph_flow-1.3.7}/pyproject.toml

@@ -87,6 +87,7 @@ kg-store = "trustgraph.storage.knowledge:run"
 librarian = "trustgraph.librarian:run"
 mcp-tool = "trustgraph.agent.mcp_tool:run"
 metering = "trustgraph.metering:run"
+nlp-query = "trustgraph.retrieval.nlp_query:run"
 objects-write-cassandra = "trustgraph.storage.objects.cassandra:run"
 objects-query-cassandra = "trustgraph.query.objects.cassandra:run"
 oe-write-milvus = "trustgraph.storage.object_embeddings.milvus:run"
@@ -95,6 +96,7 @@ pdf-ocr-mistral = "trustgraph.decoding.mistral_ocr:run"
 prompt-template = "trustgraph.prompt.template:run"
 rev-gateway = "trustgraph.rev_gateway:run"
 run-processing = "trustgraph.processing:run"
+structured-query = "trustgraph.retrieval.structured_query:run"
 text-completion-azure = "trustgraph.model.text_completion.azure:run"
 text-completion-azure-openai = "trustgraph.model.text_completion.azure_openai:run"
 text-completion-claude = "trustgraph.model.text_completion.claude:run"

{trustgraph_flow-1.3.5 → trustgraph_flow-1.3.7}/trustgraph/config/service/service.py

@@ -15,6 +15,7 @@ from trustgraph.schema import FlowRequest, FlowResponse
 from trustgraph.schema import flow_request_queue, flow_response_queue
 
 from trustgraph.base import AsyncProcessor, Consumer, Producer
+from trustgraph.base.cassandra_config import add_cassandra_args, resolve_cassandra_config
 
 from . config import Configuration
 from . flow import FlowConfig
@@ -60,9 +61,21 @@ class Processor(AsyncProcessor):
             "flow_response_queue", default_flow_response_queue
         )
 
-        cassandra_host = params.get("cassandra_host", default_cassandra_host)
+        cassandra_host = params.get("cassandra_host")
         cassandra_username = params.get("cassandra_username")
         cassandra_password = params.get("cassandra_password")
+        
+        # Resolve configuration with environment variable fallback
+        hosts, username, password = resolve_cassandra_config(
+            host=cassandra_host,
+            username=cassandra_username,
+            password=cassandra_password
+        )
+        
+        # Store resolved configuration
+        self.cassandra_host = hosts
+        self.cassandra_username = username
+        self.cassandra_password = password
 
         id = params.get("id")
 
@@ -76,8 +89,9 @@ class Processor(AsyncProcessor):
                 "config_push_schema": ConfigPush.__name__,
                 "flow_request_schema": FlowRequest.__name__,
                 "flow_response_schema": FlowResponse.__name__,
-                "cassandra_host": cassandra_host,
-                "cassandra_username": cassandra_username,
+                "cassandra_host": self.cassandra_host,
+                "cassandra_username": self.cassandra_username,
+                "cassandra_password": self.cassandra_password,
             }
         )
 
@@ -142,9 +156,9 @@ class Processor(AsyncProcessor):
         )
 
         self.config = Configuration(
-            host = cassandra_host.split(","),
-            username = cassandra_username,
-            password = cassandra_password,
+            host = self.cassandra_host,
+            username = self.cassandra_username,
+            password = self.cassandra_password,
             keyspace = keyspace,
             push = self.push
         )
@@ -276,23 +290,7 @@ class Processor(AsyncProcessor):
             help=f'Flow response queue {default_flow_response_queue}',
         )
 
-        parser.add_argument(
-            '--cassandra-host',
-            default="cassandra",
-            help=f'Graph host (default: cassandra)'
-        )
-
-        parser.add_argument(
-            '--cassandra-user',
-            default=None,
-            help=f'Cassandra user'
-        )
-
-        parser.add_argument(
-            '--cassandra-password',
-            default=None,
-            help=f'Cassandra password'
-        )
+        add_cassandra_args(parser)
 
 def run():
 

{trustgraph_flow-1.3.5 → trustgraph_flow-1.3.7}/trustgraph/cores/service.py

@@ -11,6 +11,7 @@ import logging
 
 from .. base import AsyncProcessor, Consumer, Producer, Publisher, Subscriber
 from .. base import ConsumerMetrics, ProducerMetrics
+from .. base.cassandra_config import add_cassandra_args, resolve_cassandra_config
 
 from .. schema import KnowledgeRequest, KnowledgeResponse, Error
 from .. schema import knowledge_request_queue, knowledge_response_queue
@@ -49,16 +50,29 @@ class Processor(AsyncProcessor):
             "knowledge_response_queue", default_knowledge_response_queue
         )
 
-        cassandra_host = params.get("cassandra_host", default_cassandra_host)
+        cassandra_host = params.get("cassandra_host")
         cassandra_username = params.get("cassandra_username")
         cassandra_password = params.get("cassandra_password")
+        
+        # Resolve configuration with environment variable fallback
+        hosts, username, password = resolve_cassandra_config(
+            host=cassandra_host,
+            username=cassandra_username,
+            password=cassandra_password
+        )
+        
+        # Store resolved configuration
+        self.cassandra_host = hosts
+        self.cassandra_username = username
+        self.cassandra_password = password
 
         super(Processor, self).__init__(
             **params | {
                 "knowledge_request_queue": knowledge_request_queue,
                 "knowledge_response_queue": knowledge_response_queue,
-                "cassandra_host": cassandra_host,
-                "cassandra_user": cassandra_user,
+                "cassandra_host": self.cassandra_host,
+                "cassandra_username": self.cassandra_username,
+                "cassandra_password": self.cassandra_password,
             }
         )
 
@@ -89,9 +103,9 @@ class Processor(AsyncProcessor):
         )
 
         self.knowledge = KnowledgeManager(
-            cassandra_host = cassandra_host.split(","),
-            cassandra_username = cassandra_username,
-            cassandra_password = cassandra_password,
+            cassandra_host = self.cassandra_host,
+            cassandra_username = self.cassandra_username,
+            cassandra_password = self.cassandra_password,
             keyspace = keyspace,
             flow_config = self,
         )
@@ -210,23 +224,7 @@ class Processor(AsyncProcessor):
             help=f'Config response queue {default_knowledge_response_queue}',
         )
 
-        parser.add_argument(
-            '--cassandra-host',
-            default="cassandra",
-            help=f'Graph host (default: cassandra)'
-        )
-
-        parser.add_argument(
-            '--cassandra-user',
-            default=None,
-            help=f'Cassandra user'
-        )
-
-        parser.add_argument(
-            '--cassandra-password',
-            default=None,
-            help=f'Cassandra password'
-        )
+        add_cassandra_args(parser)
 
 def run():
 

trustgraph_flow-1.3.7/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "1.3.7"

{trustgraph_flow-1.3.5 → trustgraph_flow-1.3.7}/trustgraph/gateway/dispatch/manager.py

@@ -20,6 +20,8 @@ from . graph_rag import GraphRagRequestor
 from . document_rag import DocumentRagRequestor
 from . triples_query import TriplesQueryRequestor
 from . objects_query import ObjectsQueryRequestor
+from . nlp_query import NLPQueryRequestor
+from . structured_query import StructuredQueryRequestor
 from . embeddings import EmbeddingsRequestor
 from . graph_embeddings_query import GraphEmbeddingsQueryRequestor
 from . mcp_tool import McpToolRequestor
@@ -52,6 +54,8 @@ request_response_dispatchers = {
     "graph-embeddings": GraphEmbeddingsQueryRequestor,
     "triples": TriplesQueryRequestor,
     "objects": ObjectsQueryRequestor,
+    "nlp-query": NLPQueryRequestor,
+    "structured-query": StructuredQueryRequestor,
 }
 
 global_dispatchers = {

trustgraph_flow-1.3.7/trustgraph/gateway/dispatch/nlp_query.py

@@ -0,0 +1,30 @@
+from ... schema import QuestionToStructuredQueryRequest, QuestionToStructuredQueryResponse
+from ... messaging import TranslatorRegistry
+
+from . requestor import ServiceRequestor
+
+class NLPQueryRequestor(ServiceRequestor):
+    def __init__(
+            self, pulsar_client, request_queue, response_queue, timeout,
+            consumer, subscriber,
+    ):
+
+        super(NLPQueryRequestor, self).__init__(
+            pulsar_client=pulsar_client,
+            request_queue=request_queue,
+            response_queue=response_queue,
+            request_schema=QuestionToStructuredQueryRequest,
+            response_schema=QuestionToStructuredQueryResponse,
+            subscription = subscriber,
+            consumer_name = consumer,
+            timeout=timeout,
+        )
+
+        self.request_translator = TranslatorRegistry.get_request_translator("nlp-query")
+        self.response_translator = TranslatorRegistry.get_response_translator("nlp-query")
+
+    def to_request(self, body):
+        return self.request_translator.to_pulsar(body)
+
+    def from_response(self, message):
+        return self.response_translator.from_response_with_completion(message)

trustgraph_flow-1.3.7/trustgraph/gateway/dispatch/structured_query.py

@@ -0,0 +1,30 @@
+from ... schema import StructuredQueryRequest, StructuredQueryResponse
+from ... messaging import TranslatorRegistry
+
+from . requestor import ServiceRequestor
+
+class StructuredQueryRequestor(ServiceRequestor):
+    def __init__(
+            self, pulsar_client, request_queue, response_queue, timeout,
+            consumer, subscriber,
+    ):
+
+        super(StructuredQueryRequestor, self).__init__(
+            pulsar_client=pulsar_client,
+            request_queue=request_queue,
+            response_queue=response_queue,
+            request_schema=StructuredQueryRequest,
+            response_schema=StructuredQueryResponse,
+            subscription = subscriber,
+            consumer_name = consumer,
+            timeout=timeout,
+        )
+
+        self.request_translator = TranslatorRegistry.get_request_translator("structured-query")
+        self.response_translator = TranslatorRegistry.get_response_translator("structured-query")
+
+    def to_request(self, body):
+        return self.request_translator.to_pulsar(body)
+
+    def from_response(self, message):
+        return self.response_translator.from_response_with_completion(message)

{trustgraph_flow-1.3.5 → trustgraph_flow-1.3.7}/trustgraph/librarian/service.py

@@ -11,6 +11,7 @@ import logging
 
 from .. base import AsyncProcessor, Consumer, Producer, Publisher, Subscriber
 from .. base import ConsumerMetrics, ProducerMetrics
+from .. base.cassandra_config import add_cassandra_args, resolve_cassandra_config
 
 from .. schema import LibrarianRequest, LibrarianResponse, Error
 from .. schema import librarian_request_queue, librarian_response_queue
@@ -66,9 +67,21 @@ class Processor(AsyncProcessor):
             default_minio_secret_key
         )
 
-        cassandra_host = params.get("cassandra_host", default_cassandra_host)
+        cassandra_host = params.get("cassandra_host")
         cassandra_username = params.get("cassandra_username")
         cassandra_password = params.get("cassandra_password")
+        
+        # Resolve configuration with environment variable fallback
+        hosts, username, password = resolve_cassandra_config(
+            host=cassandra_host,
+            username=cassandra_username,
+            password=cassandra_password
+        )
+        
+        # Store resolved configuration
+        self.cassandra_host = hosts
+        self.cassandra_username = username
+        self.cassandra_password = password
 
         super(Processor, self).__init__(
             **params | {
@@ -76,8 +89,9 @@ class Processor(AsyncProcessor):
                 "librarian_response_queue": librarian_response_queue,
                 "minio_host": minio_host,
                 "minio_access_key": minio_access_key,
-                "cassandra_host": cassandra_host,
-                "cassandra_username": cassandra_username,
+                "cassandra_host": self.cassandra_host,
+                "cassandra_username": self.cassandra_username,
+                "cassandra_password": self.cassandra_password,
             }
         )
 
@@ -108,9 +122,9 @@ class Processor(AsyncProcessor):
         )
 
         self.librarian = Librarian(
-            cassandra_host = cassandra_host.split(","),
-            cassandra_username = cassandra_username,
-            cassandra_password = cassandra_password,
+            cassandra_host = self.cassandra_host,
+            cassandra_username = self.cassandra_username,
+            cassandra_password = self.cassandra_password,
             minio_host = minio_host,
             minio_access_key = minio_access_key,
             minio_secret_key = minio_secret_key,
@@ -319,23 +333,7 @@ class Processor(AsyncProcessor):
             f'(default: {default_minio_access_key})',
         )
 
-        parser.add_argument(
-            '--cassandra-host',
-            default="cassandra",
-            help=f'Graph host (default: cassandra)'
-        )
-
-        parser.add_argument(
-            '--cassandra-user',
-            default=None,
-            help=f'Cassandra user'
-        )
-
-        parser.add_argument(
-            '--cassandra-password',
-            default=None,
-            help=f'Cassandra password'
-        )
+        add_cassandra_args(parser)
 
 def run():
 

trustgraph_flow-1.3.7/trustgraph/retrieval/nlp_query/__init__.py

@@ -0,0 +1 @@
+from . service import *

trustgraph_flow-1.3.7/trustgraph/retrieval/nlp_query/__main__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+from . service import run
+
+run()

trustgraph_flow-1.3.7/trustgraph/retrieval/nlp_query/service.py

@@ -0,0 +1,301 @@
+"""
+NLP to Structured Query Service - converts natural language questions to GraphQL queries.
+Two-phase approach: 1) Select relevant schemas, 2) Generate GraphQL query.
+"""
+
+import json
+import logging
+from typing import Dict, Any, Optional, List
+
+from ...schema import QuestionToStructuredQueryRequest, QuestionToStructuredQueryResponse
+from ...schema import PromptRequest
+from ...schema import Error, RowSchema, Field as SchemaField
+
+from ...base import FlowProcessor, ConsumerSpec, ProducerSpec, PromptClientSpec
+
+# Module logger
+logger = logging.getLogger(__name__)
+
+default_ident = "nlp-query"
+default_schema_selection_template = "schema-selection"
+default_graphql_generation_template = "graphql-generation"
+
+class Processor(FlowProcessor):
+    
+    def __init__(self, **params):
+        
+        id = params.get("id", default_ident)
+        
+        # Config key for schemas
+        self.config_key = params.get("config_type", "schema")
+        
+        # Configurable prompt template names
+        self.schema_selection_template = params.get("schema_selection_template", default_schema_selection_template)
+        self.graphql_generation_template = params.get("graphql_generation_template", default_graphql_generation_template)
+        
+        super(Processor, self).__init__(
+            **params | {
+                "id": id,
+                "config-type": self.config_key,
+            }
+        )
+        
+        self.register_specification(
+            ConsumerSpec(
+                name = "request",
+                schema = QuestionToStructuredQueryRequest,
+                handler = self.on_message
+            )
+        )
+        
+        self.register_specification(
+            ProducerSpec(
+                name = "response",
+                schema = QuestionToStructuredQueryResponse,
+            )
+        )
+        
+        # Client spec for calling prompt service
+        self.register_specification(
+            PromptClientSpec(
+                request_name = "prompt-request",
+                response_name = "prompt-response",
+            )
+        )
+        
+        # Register config handler for schema updates
+        self.register_config_handler(self.on_schema_config)
+        
+        # Schema storage: name -> RowSchema
+        self.schemas: Dict[str, RowSchema] = {}
+        
+        logger.info("NLP Query service initialized")
+
+    async def on_schema_config(self, config, version):
+        """Handle schema configuration updates"""
+        logger.info(f"Loading schema configuration version {version}")
+        
+        # Clear existing schemas
+        self.schemas = {}
+        
+        # Check if our config type exists
+        if self.config_key not in config:
+            logger.warning(f"No '{self.config_key}' type in configuration")
+            return
+        
+        # Get the schemas dictionary for our type
+        schemas_config = config[self.config_key]
+        
+        # Process each schema in the schemas config
+        for schema_name, schema_json in schemas_config.items():
+            try:
+                # Parse the JSON schema definition
+                schema_def = json.loads(schema_json)
+                
+                # Create Field objects
+                fields = []
+                for field_def in schema_def.get("fields", []):
+                    field = SchemaField(
+                        name=field_def["name"],
+                        type=field_def["type"],
+                        size=field_def.get("size", 0),
+                        primary=field_def.get("primary_key", False),
+                        description=field_def.get("description", ""),
+                        required=field_def.get("required", False),
+                        enum_values=field_def.get("enum", []),
+                        indexed=field_def.get("indexed", False)
+                    )
+                    fields.append(field)
+                
+                # Create RowSchema
+                row_schema = RowSchema(
+                    name=schema_def.get("name", schema_name),
+                    description=schema_def.get("description", ""),
+                    fields=fields
+                )
+                
+                self.schemas[schema_name] = row_schema
+                logger.info(f"Loaded schema: {schema_name} with {len(fields)} fields")
+                
+            except Exception as e:
+                logger.error(f"Failed to parse schema {schema_name}: {e}", exc_info=True)
+        
+        logger.info(f"Schema configuration loaded: {len(self.schemas)} schemas")
+
+    async def phase1_select_schemas(self, question: str) -> List[str]:
+        """Phase 1: Use prompt service to select relevant schemas for the question"""
+        logger.info("Starting Phase 1: Schema selection")
+        
+        # Prepare schema information for the prompt
+        schema_info = []
+        for name, schema in self.schemas.items():
+            schema_desc = {
+                "name": name,
+                "description": schema.description,
+                "fields": [{"name": f.name, "type": f.type, "description": f.description} 
+                          for f in schema.fields]
+            }
+            schema_info.append(schema_desc)
+        
+        # Create prompt variables
+        variables = {
+            "question": question,
+            "schemas": schema_info  # Pass structured data directly
+        }
+        
+        # Call prompt service for schema selection
+        prompt_request = PromptRequest(
+            template=self.schema_selection_template,
+            variables=variables
+        )
+        
+        try:
+            response = await self.client("prompt-request").request(prompt_request)
+            
+            if response.error is not None:
+                raise Exception(f"Prompt service error: {response.error}")
+            
+            # Parse the response to get selected schema names
+            # Expecting response.text to contain JSON array of schema names
+            selected_schemas = json.loads(response.text)
+            
+            logger.info(f"Phase 1 selected schemas: {selected_schemas}")
+            return selected_schemas
+            
+        except Exception as e:
+            logger.error(f"Phase 1 schema selection failed: {e}")
+            raise
+
+    async def phase2_generate_graphql(self, question: str, selected_schemas: List[str]) -> Dict[str, Any]:
+        """Phase 2: Generate GraphQL query using selected schemas"""
+        logger.info(f"Starting Phase 2: GraphQL generation for schemas: {selected_schemas}")
+        
+        # Get detailed schema information for selected schemas only
+        selected_schema_info = []
+        for schema_name in selected_schemas:
+            if schema_name in self.schemas:
+                schema = self.schemas[schema_name]
+                schema_desc = {
+                    "name": schema_name,
+                    "description": schema.description,
+                    "fields": [
+                        {
+                            "name": f.name, 
+                            "type": f.type, 
+                            "description": f.description,
+                            "required": f.required,
+                            "primary_key": f.primary,
+                            "indexed": f.indexed,
+                            "enum_values": f.enum_values if f.enum_values else []
+                        } 
+                        for f in schema.fields
+                    ]
+                }
+                selected_schema_info.append(schema_desc)
+        
+        # Create prompt variables for GraphQL generation
+        variables = {
+            "question": question,
+            "schemas": selected_schema_info  # Pass structured data directly
+        }
+        
+        # Call prompt service for GraphQL generation
+        prompt_request = PromptRequest(
+            template=self.graphql_generation_template,
+            variables=variables
+        )
+        
+        try:
+            response = await self.client("prompt-request").request(prompt_request)
+            
+            if response.error is not None:
+                raise Exception(f"Prompt service error: {response.error}")
+            
+            # Parse the response to get GraphQL query and variables
+            # Expecting response.text to contain JSON with "query" and "variables" fields
+            result = json.loads(response.text)
+            
+            logger.info(f"Phase 2 generated GraphQL: {result.get('query', '')[:100]}...")
+            return result
+            
+        except Exception as e:
+            logger.error(f"Phase 2 GraphQL generation failed: {e}")
+            raise
+
+    async def on_message(self, msg, consumer, flow):
+        """Handle incoming question to structured query request"""
+        
+        try:
+            request = msg.value()
+            
+            # Sender-produced ID
+            id = msg.properties()["id"]
+            
+            logger.info(f"Handling NLP query request {id}: {request.question[:100]}...")
+            
+            # Phase 1: Select relevant schemas
+            selected_schemas = await self.phase1_select_schemas(request.question)
+            
+            # Phase 2: Generate GraphQL query
+            graphql_result = await self.phase2_generate_graphql(request.question, selected_schemas)
+            
+            # Create response
+            response = QuestionToStructuredQueryResponse(
+                error=None,
+                graphql_query=graphql_result.get("query", ""),
+                variables=graphql_result.get("variables", {}),
+                detected_schemas=selected_schemas,
+                confidence=graphql_result.get("confidence", 0.8)  # Default confidence
+            )
+            
+            logger.info("Sending NLP query response...")
+            await flow("response").send(response, properties={"id": id})
+            
+            logger.info("NLP query request completed")
+            
+        except Exception as e:
+            
+            logger.error(f"Exception in NLP query service: {e}", exc_info=True)
+            
+            logger.info("Sending error response...")
+            
+            response = QuestionToStructuredQueryResponse(
+                error = Error(
+                    type = "nlp-query-error",
+                    message = str(e),
+                ),
+                graphql_query = "",
+                variables = {},
+                detected_schemas = [],
+                confidence = 0.0
+            )
+            
+            await flow("response").send(response, properties={"id": id})
+
+    @staticmethod
+    def add_args(parser):
+        """Add command-line arguments"""
+        
+        FlowProcessor.add_args(parser)
+        
+        parser.add_argument(
+            '--config-type',
+            default='schema',
+            help='Configuration type prefix for schemas (default: schema)'
+        )
+        
+        parser.add_argument(
+            '--schema-selection-template',
+            default=default_schema_selection_template,
+            help=f'Prompt template name for schema selection (default: {default_schema_selection_template})'
+        )
+        
+        parser.add_argument(
+            '--graphql-generation-template', 
+            default=default_graphql_generation_template,
+            help=f'Prompt template name for GraphQL generation (default: {default_graphql_generation_template})'
+        )
+
+def run():
+    """Entry point for nlp-query command"""
+    Processor.launch(default_ident, __doc__)

trustgraph_flow-1.3.7/trustgraph/retrieval/structured_query/__init__.py

@@ -0,0 +1 @@
+from . service import *

trustgraph_flow-1.3.7/trustgraph/retrieval/structured_query/__main__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+from . service import run
+
+run()