trustgraph-base 1.3.15__tar.gz → 1.5.1__tar.gz

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-base
-Version: 1.3.15
+Version: 1.5.1
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph
@@ -10,5 +10,6 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 Requires-Dist: pulsar-client
 Requires-Dist: prometheus-client
+Requires-Dist: requests
 
 See https://trustgraph.ai/

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/pyproject.toml

@@ -12,6 +12,7 @@ requires-python = ">=3.8"
 dependencies = [
     "pulsar-client",
     "prometheus-client",
+    "requests",
 ]
 classifiers = [
     "Programming Language :: Python :: 3",

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/trustgraph/api/api.py

@@ -8,6 +8,7 @@ from . library import Library
 from . flow import Flow
 from . config import Config
 from . knowledge import Knowledge
+from . collection import Collection
 from . exceptions import *
 from . types import *
 
@@ -68,3 +69,6 @@ class Api:
 
     def library(self):
         return Library(self)
+
+    def collection(self):
+        return Collection(self)

trustgraph_base-1.5.1/trustgraph/api/collection.py

@@ -0,0 +1,98 @@
+import datetime
+import logging
+
+from . types import CollectionMetadata
+from . exceptions import *
+
+logger = logging.getLogger(__name__)
+
+class Collection:
+
+    def __init__(self, api):
+        self.api = api
+
+    def request(self, request):
+        return self.api.request(f"collection-management", request)
+
+    def list_collections(self, user, tag_filter=None):
+
+        input = {
+            "operation": "list-collections",
+            "user": user,
+        }
+
+        if tag_filter:
+            input["tag_filter"] = tag_filter
+
+        object = self.request(input)
+
+        try:
+            # Handle case where collections might be None or missing
+            if object is None or "collections" not in object:
+                return []
+
+            collections = object.get("collections", [])
+            if collections is None:
+                return []
+
+            return [
+                CollectionMetadata(
+                    user = v["user"],
+                    collection = v["collection"],
+                    name = v["name"],
+                    description = v["description"],
+                    tags = v["tags"],
+                    created_at = v["created_at"],
+                    updated_at = v["updated_at"]
+                )
+                for v in collections
+            ]
+        except Exception as e:
+            logger.error("Failed to parse collection list response", exc_info=True)
+            raise ProtocolException(f"Response not formatted correctly")
+
+    def update_collection(self, user, collection, name=None, description=None, tags=None):
+
+        input = {
+            "operation": "update-collection",
+            "user": user,
+            "collection": collection,
+        }
+
+        if name is not None:
+            input["name"] = name
+        if description is not None:
+            input["description"] = description
+        if tags is not None:
+            input["tags"] = tags
+
+        object = self.request(input)
+
+        try:
+            if "collections" in object and object["collections"]:
+                v = object["collections"][0]
+                return CollectionMetadata(
+                    user = v["user"],
+                    collection = v["collection"],
+                    name = v["name"],
+                    description = v["description"],
+                    tags = v["tags"],
+                    created_at = v["created_at"],
+                    updated_at = v["updated_at"]
+                )
+            return None
+        except Exception as e:
+            logger.error("Failed to parse collection update response", exc_info=True)
+            raise ProtocolException(f"Response not formatted correctly")
+
+    def delete_collection(self, user, collection):
+
+        input = {
+            "operation": "delete-collection",
+            "user": user,
+            "collection": collection,
+        }
+
+        object = self.request(input)
+
+        return {}

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/trustgraph/api/flow.py

@@ -87,7 +87,7 @@ class Flow:
 
         return json.loads(self.request(request = input)["flow"])
 
-    def start(self, class_name, id, description):
+    def start(self, class_name, id, description, parameters=None):
 
         # The input consists of system and prompt strings
         input = {
@@ -97,6 +97,9 @@ class Flow:
             "description": description,
         }
 
+        if parameters:
+            input["parameters"] = parameters
+
         self.request(request = input)
 
     def stop(self, id):
@@ -132,12 +135,24 @@ class FlowInstance:
             input
         )["response"]
 
-    def agent(self, question):
+    def agent(self, question, user="trustgraph", state=None, group=None, history=None):
 
-        # The input consists of a question
+        # The input consists of a question and optional context
         input = {
-            "question": question
+            "question": question,
+            "user": user,
         }
+        
+        # Only include state if it has a value
+        if state is not None:
+            input["state"] = state
+            
+        # Only include group if it has a value
+        if group is not None:
+            input["group"] = group
+            
+        # Always include history (empty list if None)
+        input["history"] = history or []
 
         return self.request(
             "service/agent",
@@ -456,32 +471,172 @@ class FlowInstance:
         
         return response
 
-    def structured_query(self, question):
+    def structured_query(self, question, user="trustgraph", collection="default"):
         """
         Execute a natural language question against structured data.
         Combines NLP query conversion and GraphQL execution.
         
         Args:
             question: Natural language question
+            user: Cassandra keyspace identifier (default: "trustgraph")
+            collection: Data collection identifier (default: "default")
             
         Returns:
             dict with data and optional errors
         """
         
         input = {
-            "question": question
+            "question": question,
+            "user": user,
+            "collection": collection
         }
         
         response = self.request(
             "service/structured-query",
             input
         )
-        
+
         # Check for system-level error
         if "error" in response and response["error"]:
             error_type = response["error"].get("type", "unknown")
             error_message = response["error"].get("message", "Unknown error")
             raise ProtocolException(f"{error_type}: {error_message}")
-        
+
+        return response
+
+    def detect_type(self, sample):
+        """
+        Detect the data type of a structured data sample.
+
+        Args:
+            sample: Data sample to analyze (string content)
+
+        Returns:
+            dict with detected_type, confidence, and optional metadata
+        """
+
+        input = {
+            "operation": "detect-type",
+            "sample": sample
+        }
+
+        response = self.request(
+            "service/structured-diag",
+            input
+        )
+
+        # Check for system-level error
+        if "error" in response and response["error"]:
+            error_type = response["error"].get("type", "unknown")
+            error_message = response["error"].get("message", "Unknown error")
+            raise ProtocolException(f"{error_type}: {error_message}")
+
+        return response["detected-type"]
+
+    def generate_descriptor(self, sample, data_type, schema_name, options=None):
+        """
+        Generate a descriptor for structured data mapping to a specific schema.
+
+        Args:
+            sample: Data sample to analyze (string content)
+            data_type: Data type (csv, json, xml)
+            schema_name: Target schema name for descriptor generation
+            options: Optional parameters (e.g., delimiter for CSV)
+
+        Returns:
+            dict with descriptor and metadata
+        """
+
+        input = {
+            "operation": "generate-descriptor",
+            "sample": sample,
+            "type": data_type,
+            "schema-name": schema_name
+        }
+
+        if options:
+            input["options"] = options
+
+        response = self.request(
+            "service/structured-diag",
+            input
+        )
+
+        # Check for system-level error
+        if "error" in response and response["error"]:
+            error_type = response["error"].get("type", "unknown")
+            error_message = response["error"].get("message", "Unknown error")
+            raise ProtocolException(f"{error_type}: {error_message}")
+
+        return response["descriptor"]
+
+    def diagnose_data(self, sample, schema_name=None, options=None):
+        """
+        Perform combined data diagnosis: detect type and generate descriptor.
+
+        Args:
+            sample: Data sample to analyze (string content)
+            schema_name: Optional target schema name for descriptor generation
+            options: Optional parameters (e.g., delimiter for CSV)
+
+        Returns:
+            dict with detected_type, confidence, descriptor, and metadata
+        """
+
+        input = {
+            "operation": "diagnose",
+            "sample": sample
+        }
+
+        if schema_name:
+            input["schema-name"] = schema_name
+
+        if options:
+            input["options"] = options
+
+        response = self.request(
+            "service/structured-diag",
+            input
+        )
+
+        # Check for system-level error
+        if "error" in response and response["error"]:
+            error_type = response["error"].get("type", "unknown")
+            error_message = response["error"].get("message", "Unknown error")
+            raise ProtocolException(f"{error_type}: {error_message}")
+
         return response
 
+    def schema_selection(self, sample, options=None):
+        """
+        Select matching schemas for a data sample using prompt analysis.
+
+        Args:
+            sample: Data sample to analyze (string content)
+            options: Optional parameters
+
+        Returns:
+            dict with schema_matches array and metadata
+        """
+
+        input = {
+            "operation": "schema-selection",
+            "sample": sample
+        }
+
+        if options:
+            input["options"] = options
+
+        response = self.request(
+            "service/structured-diag",
+            input
+        )
+
+        # Check for system-level error
+        if "error" in response and response["error"]:
+            error_type = response["error"].get("type", "unknown")
+            error_message = response["error"].get("message", "Unknown error")
+            raise ProtocolException(f"{error_type}: {error_message}")
+
+        return response["schema-matches"]
+

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/trustgraph/api/types.py

@@ -41,3 +41,13 @@ class ProcessingMetadata:
     user : str
     collection : str
     tags : List[str]
+
+@dataclasses.dataclass
+class CollectionMetadata:
+    user : str
+    collection : str
+    name : str
+    description : str
+    tags : List[str]
+    created_at : str
+    updated_at : str

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/trustgraph/base/__init__.py

@@ -8,11 +8,12 @@ from . subscriber import Subscriber
 from . metrics import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
 from . flow_processor import FlowProcessor
 from . consumer_spec import ConsumerSpec
-from . setting_spec import SettingSpec
+from . parameter_spec import ParameterSpec
 from . producer_spec import ProducerSpec
 from . subscriber_spec import SubscriberSpec
 from . request_response_spec import RequestResponseSpec
 from . llm_service import LlmService, LlmResult
+from . chunking_service import ChunkingService
 from . embeddings_service import EmbeddingsService
 from . embeddings_client import EmbeddingsClientSpec
 from . text_completion_client import TextCompletionClientSpec

trustgraph_base-1.5.1/trustgraph/base/chunking_service.py

@@ -0,0 +1,62 @@
+"""
+Base chunking service that provides parameter specification functionality
+for chunk-size and chunk-overlap parameters
+"""
+
+import logging
+from .flow_processor import FlowProcessor
+from .parameter_spec import ParameterSpec
+
+# Module logger
+logger = logging.getLogger(__name__)
+
+class ChunkingService(FlowProcessor):
+    """Base service for chunking processors with parameter specification support"""
+
+    def __init__(self, **params):
+
+        # Call parent constructor
+        super(ChunkingService, self).__init__(**params)
+
+        # Register parameter specifications for chunk-size and chunk-overlap
+        self.register_specification(
+            ParameterSpec(name="chunk-size")
+        )
+
+        self.register_specification(
+            ParameterSpec(name="chunk-overlap")
+        )
+
+        logger.debug("ChunkingService initialized with parameter specifications")
+
+    async def chunk_document(self, msg, consumer, flow, default_chunk_size, default_chunk_overlap):
+        """
+        Extract chunk parameters from flow and return effective values
+
+        Args:
+            msg: The message containing the document to chunk
+            consumer: The consumer spec
+            flow: The flow context
+            default_chunk_size: Default chunk size from processor config
+            default_chunk_overlap: Default chunk overlap from processor config
+
+        Returns:
+            tuple: (chunk_size, chunk_overlap) - effective values to use
+        """
+        # Extract parameters from flow (flow-configurable parameters)
+        chunk_size = flow("chunk-size")
+        chunk_overlap = flow("chunk-overlap")
+
+        # Use provided values or fall back to defaults
+        effective_chunk_size = chunk_size if chunk_size is not None else default_chunk_size
+        effective_chunk_overlap = chunk_overlap if chunk_overlap is not None else default_chunk_overlap
+
+        logger.debug(f"Using chunk-size: {effective_chunk_size}")
+        logger.debug(f"Using chunk-overlap: {effective_chunk_overlap}")
+
+        return effective_chunk_size, effective_chunk_overlap
+
+    @staticmethod
+    def add_args(parser):
+        """Add chunking service arguments to parser"""
+        FlowProcessor.add_args(parser)

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/trustgraph/base/flow.py

@@ -12,7 +12,7 @@ class Flow:
         # Consumers and publishers.  Is this a bit untidy?
         self.consumer = {}
 
-        self.setting = {}
+        self.parameter = {}
 
         for spec in processor.specifications:
             spec.add(self, processor, defn)
@@ -28,5 +28,5 @@ class Flow:
     def __call__(self, key):
         if key in self.producer: return self.producer[key]
         if key in self.consumer: return self.consumer[key]
-        if key in self.setting: return self.setting[key].value
+        if key in self.parameter: return self.parameter[key].value
         return None

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/trustgraph/base/flow_processor.py

@@ -35,7 +35,7 @@ class FlowProcessor(AsyncProcessor):
 
         # These can be overriden by a derived class:
 
-        # Array of specifications: ConsumerSpec, ProducerSpec, SettingSpec
+        # Array of specifications: ConsumerSpec, ProducerSpec, ParameterSpec
         self.specifications = []
 
         logger.info("Service initialised.")

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/trustgraph/base/llm_service.py

@@ -5,11 +5,11 @@ LLM text completion base class
 
 import time
 import logging
-from prometheus_client import Histogram
+from prometheus_client import Histogram, Info
 
 from .. schema import TextCompletionRequest, TextCompletionResponse, Error
 from .. exceptions import TooManyRequests
-from .. base import FlowProcessor, ConsumerSpec, ProducerSpec
+from .. base import FlowProcessor, ConsumerSpec, ProducerSpec, ParameterSpec
 
 # Module logger
 logger = logging.getLogger(__name__)
@@ -32,7 +32,7 @@ class LlmService(FlowProcessor):
 
     def __init__(self, **params):
 
-        id = params.get("id")
+        id = params.get("id", default_ident)
         concurrency = params.get("concurrency", 1)
 
         super(LlmService, self).__init__(**params | {
@@ -56,6 +56,18 @@ class LlmService(FlowProcessor):
             )
         )
 
+        self.register_specification(
+            ParameterSpec(
+                name = "model",
+            )
+        )
+
+        self.register_specification(
+            ParameterSpec(
+                name = "temperature",
+            )
+        )
+
         if not hasattr(__class__, "text_completion_metric"):
             __class__.text_completion_metric = Histogram(
                 'text_completion_duration',
@@ -70,6 +82,13 @@ class LlmService(FlowProcessor):
                 ]
             )
 
+        if not hasattr(__class__, "text_completion_model_metric"):
+            __class__.text_completion_model_metric = Info(
+                'text_completion_model',
+                'Text completion model',
+                ["processor", "flow"]
+            )
+
     async def on_request(self, msg, consumer, flow):
 
         try:
@@ -85,10 +104,21 @@ class LlmService(FlowProcessor):
                     flow=f"{flow.name}-{consumer.name}",
             ).time():
 
+                model = flow("model")
+                temperature = flow("temperature")
+
                 response = await self.generate_content(
-                    request.system, request.prompt
+                    request.system, request.prompt, model, temperature
                 )
 
+            __class__.text_completion_model_metric.labels(
+                processor = self.id,
+                flow = flow.name
+            ).info({
+                "model": str(model) if model is not None else "",
+                "temperature": str(temperature) if temperature is not None else "",
+            })
+
             await flow("response").send(
                 TextCompletionResponse(
                     error=None,

trustgraph_base-1.3.15/trustgraph/base/setting_spec.py → trustgraph_base-1.5.1/trustgraph/base/parameter_spec.py

@@ -1,7 +1,7 @@
 
 from . spec import Spec
 
-class Setting:
+class Parameter:
     def __init__(self, value):
         self.value = value
     async def start():
@@ -9,11 +9,13 @@ class Setting:
     async def stop():
         pass
         
-class SettingSpec(Spec):
+class ParameterSpec(Spec):
     def __init__(self, name):
         self.name = name
 
     def add(self, flow, processor, definition):
 
-        flow.config[self.name] = Setting(definition[self.name])
+        value = definition.get(self.name, None)
+
+        flow.parameter[self.name] = Parameter(value)
 

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/trustgraph/base/request_response_spec.py

@@ -49,8 +49,6 @@ class RequestResponse(Subscriber):
 
         id = str(uuid.uuid4())
 
-        logger.debug(f"Sending request {id}...")
-
         q = await self.subscribe(id)
 
         try:
@@ -75,8 +73,6 @@ class RequestResponse(Subscriber):
                     timeout=timeout
                 )
 
-                logger.debug("Received response")
-
                 if recipient is None:
 
                     # If no recipient handler, just return the first

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/trustgraph/base/structured_query_client.py

@@ -2,10 +2,12 @@ from . request_response_spec import RequestResponse, RequestResponseSpec
 from .. schema import StructuredQueryRequest, StructuredQueryResponse
 
 class StructuredQueryClient(RequestResponse):
-    async def structured_query(self, question, timeout=600):
+    async def structured_query(self, question, user="trustgraph", collection="default", timeout=600):
         resp = await self.request(
             StructuredQueryRequest(
-                question = question
+                question = question,
+                user = user,
+                collection = collection
             ),
             timeout=timeout
         )

trustgraph_base-1.5.1/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "1.5.1"

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/trustgraph/messaging/__init__.py

@@ -24,6 +24,8 @@ from .translators.embeddings_query import (
 from .translators.objects_query import ObjectsQueryRequestTranslator, ObjectsQueryResponseTranslator
 from .translators.nlp_query import QuestionToStructuredQueryRequestTranslator, QuestionToStructuredQueryResponseTranslator
 from .translators.structured_query import StructuredQueryRequestTranslator, StructuredQueryResponseTranslator
+from .translators.diagnosis import StructuredDataDiagnosisRequestTranslator, StructuredDataDiagnosisResponseTranslator
+from .translators.collection import CollectionManagementRequestTranslator, CollectionManagementResponseTranslator
 
 # Register all service translators
 TranslatorRegistry.register_service(
@@ -123,11 +125,23 @@ TranslatorRegistry.register_service(
 )
 
 TranslatorRegistry.register_service(
-    "structured-query", 
-    StructuredQueryRequestTranslator(), 
+    "structured-query",
+    StructuredQueryRequestTranslator(),
     StructuredQueryResponseTranslator()
 )
 
+TranslatorRegistry.register_service(
+    "structured-diag",
+    StructuredDataDiagnosisRequestTranslator(),
+    StructuredDataDiagnosisResponseTranslator()
+)
+
+TranslatorRegistry.register_service(
+    "collection-management",
+    CollectionManagementRequestTranslator(),
+    CollectionManagementResponseTranslator()
+)
+
 # Register single-direction translators for document loading
 TranslatorRegistry.register_request("document", DocumentTranslator())
 TranslatorRegistry.register_request("text-document", TextDocumentTranslator())

{trustgraph_base-1.3.15 → trustgraph_base-1.5.1}/trustgraph/messaging/translators/__init__.py

@@ -18,3 +18,4 @@ from .embeddings_query import (
     GraphEmbeddingsRequestTranslator, GraphEmbeddingsResponseTranslator
 )
 from .objects_query import ObjectsQueryRequestTranslator, ObjectsQueryResponseTranslator
+from .diagnosis import StructuredDataDiagnosisRequestTranslator, StructuredDataDiagnosisResponseTranslator