trustgraph-base 1.2.8__tar.gz → 1.2.9__tar.gz

{trustgraph_base-1.2.8 → trustgraph_base-1.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-base
-Version: 1.2.8
+Version: 1.2.9
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_base-1.2.8 → trustgraph_base-1.2.9}/trustgraph/base/prompt_client.py

@@ -40,6 +40,13 @@ class PromptClient(RequestResponse):
             timeout = timeout,
         )
 
+    async def extract_objects(self, text, schema, timeout=600):
+        return await self.prompt(
+            id = "extract-rows",
+            variables = { "text": text, "schema": schema, },
+            timeout = timeout,
+        )
+
     async def kg_prompt(self, query, kg, timeout=600):
         return await self.prompt(
             id = "kg-prompt",

trustgraph_base-1.2.9/trustgraph/base_version.py

@@ -0,0 +1 @@
+__version__ = "1.2.9"

{trustgraph_base-1.2.8 → trustgraph_base-1.2.9}/trustgraph/messaging/translators/__init__.py

@@ -1,5 +1,5 @@
 from .base import Translator, MessageTranslator
-from .primitives import ValueTranslator, TripleTranslator, SubgraphTranslator
+from .primitives import ValueTranslator, TripleTranslator, SubgraphTranslator, RowSchemaTranslator, FieldTranslator, row_schema_translator, field_translator
 from .metadata import DocumentMetadataTranslator, ProcessingMetadataTranslator
 from .agent import AgentRequestTranslator, AgentResponseTranslator
 from .embeddings import EmbeddingsRequestTranslator, EmbeddingsResponseTranslator

trustgraph_base-1.2.9/trustgraph/messaging/translators/primitives.py

@@ -0,0 +1,140 @@
+from typing import Dict, Any, List
+from ...schema import Value, Triple, RowSchema, Field
+from .base import Translator
+
+
+class ValueTranslator(Translator):
+    """Translator for Value schema objects"""
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> Value:
+        return Value(value=data["v"], is_uri=data["e"])
+    
+    def from_pulsar(self, obj: Value) -> Dict[str, Any]:
+        return {"v": obj.value, "e": obj.is_uri}
+
+
+class TripleTranslator(Translator):
+    """Translator for Triple schema objects"""
+    
+    def __init__(self):
+        self.value_translator = ValueTranslator()
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> Triple:
+        return Triple(
+            s=self.value_translator.to_pulsar(data["s"]),
+            p=self.value_translator.to_pulsar(data["p"]),
+            o=self.value_translator.to_pulsar(data["o"])
+        )
+    
+    def from_pulsar(self, obj: Triple) -> Dict[str, Any]:
+        return {
+            "s": self.value_translator.from_pulsar(obj.s),
+            "p": self.value_translator.from_pulsar(obj.p),
+            "o": self.value_translator.from_pulsar(obj.o)
+        }
+
+
+class SubgraphTranslator(Translator):
+    """Translator for lists of Triple objects (subgraphs)"""
+    
+    def __init__(self):
+        self.triple_translator = TripleTranslator()
+    
+    def to_pulsar(self, data: List[Dict[str, Any]]) -> List[Triple]:
+        return [self.triple_translator.to_pulsar(t) for t in data]
+    
+    def from_pulsar(self, obj: List[Triple]) -> List[Dict[str, Any]]:
+        return [self.triple_translator.from_pulsar(t) for t in obj]
+
+
+class RowSchemaTranslator(Translator):
+    """Translator for RowSchema objects"""
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> RowSchema:
+        """Convert dict to RowSchema Pulsar object"""
+        fields = []
+        for field_data in data.get("fields", []):
+            field = Field(
+                name=field_data.get("name", ""),
+                type=field_data.get("type", "string"),
+                size=field_data.get("size", 0),
+                primary=field_data.get("primary", False),
+                description=field_data.get("description", ""),
+                required=field_data.get("required", False),
+                indexed=field_data.get("indexed", False),
+                enum_values=field_data.get("enum_values", [])
+            )
+            fields.append(field)
+        
+        return RowSchema(
+            name=data.get("name", ""),
+            description=data.get("description", ""),
+            fields=fields
+        )
+    
+    def from_pulsar(self, obj: RowSchema) -> Dict[str, Any]:
+        """Convert RowSchema Pulsar object to JSON-serializable dictionary"""
+        result = {
+            "name": obj.name,
+            "description": obj.description,
+            "fields": []
+        }
+        
+        for field in obj.fields:
+            field_dict = {
+                "name": field.name,
+                "type": field.type,
+                "size": field.size,
+                "primary": field.primary,
+                "description": field.description,
+                "required": field.required,
+                "indexed": field.indexed
+            }
+            
+            # Handle enum_values array
+            if field.enum_values:
+                field_dict["enum_values"] = list(field.enum_values)
+            
+            result["fields"].append(field_dict)
+        
+        return result
+
+
+class FieldTranslator(Translator):
+    """Translator for Field objects"""
+    
+    def to_pulsar(self, data: Dict[str, Any]) -> Field:
+        """Convert dict to Field Pulsar object"""
+        return Field(
+            name=data.get("name", ""),
+            type=data.get("type", "string"),
+            size=data.get("size", 0),
+            primary=data.get("primary", False),
+            description=data.get("description", ""),
+            required=data.get("required", False),
+            indexed=data.get("indexed", False),
+            enum_values=data.get("enum_values", [])
+        )
+    
+    def from_pulsar(self, obj: Field) -> Dict[str, Any]:
+        """Convert Field Pulsar object to JSON-serializable dictionary"""
+        result = {
+            "name": obj.name,
+            "type": obj.type,
+            "size": obj.size,
+            "primary": obj.primary,
+            "description": obj.description,
+            "required": obj.required,
+            "indexed": obj.indexed
+        }
+        
+        # Handle enum_values array
+        if obj.enum_values:
+            result["enum_values"] = list(obj.enum_values)
+        
+        return result
+
+
+# Create singleton instances for easy access
+row_schema_translator = RowSchemaTranslator()
+field_translator = FieldTranslator()

trustgraph_base-1.2.9/trustgraph/schema/__init__.py

@@ -0,0 +1,10 @@
+
+# Import core types and primitives
+from .core import *
+
+# Import knowledge schemas
+from .knowledge import *
+
+# Import service schemas
+from .services import *
+

trustgraph_base-1.2.9/trustgraph/schema/core/__init__.py

@@ -0,0 +1,3 @@
+from .primitives import *
+from .metadata import *
+from .topic import *

{trustgraph_base-1.2.8/trustgraph/schema → trustgraph_base-1.2.9/trustgraph/schema/core}/metadata.py

@@ -1,6 +1,6 @@
 
 from pulsar.schema import Record, String, Array
-from . types import Triple
+from .primitives import Triple
 
 class Metadata(Record):
 

trustgraph_base-1.2.8/trustgraph/schema/types.py → trustgraph_base-1.2.9/trustgraph/schema/core/primitives.py

@@ -17,11 +17,15 @@ class Triple(Record):
 
 class Field(Record):
     name = String()
-    # int, string, long, bool, float, double
+    # int, string, long, bool, float, double, timestamp
     type = String()
     size = Integer()
     primary = Boolean()
     description = String()
+    # NEW FIELDS for structured data:
+    required = Boolean()  # Whether field is required
+    enum_values = Array(String())  # For enum type fields
+    indexed = Boolean()  # Whether field should be indexed
 
 class RowSchema(Record):
     name = String()

trustgraph_base-1.2.9/trustgraph/schema/knowledge/__init__.py

@@ -0,0 +1,8 @@
+from .graph import *
+from .document import *
+from .embeddings import *
+from .knowledge import *
+from .nlp import *
+from .rows import *
+from .structured import *
+from .object import *

trustgraph_base-1.2.9/trustgraph/schema/knowledge/document.py

@@ -0,0 +1,29 @@
+from pulsar.schema import Record, Bytes
+
+from ..core.metadata import Metadata
+from ..core.topic import topic
+
+############################################################################
+
+# PDF docs etc.
+class Document(Record):
+    metadata = Metadata()
+    data = Bytes()
+
+############################################################################
+
+# Text documents / text from PDF
+
+class TextDocument(Record):
+    metadata = Metadata()
+    text = Bytes()
+
+############################################################################
+
+# Chunks of text
+
+class Chunk(Record):
+    metadata = Metadata()
+    chunk = Bytes()
+
+############################################################################

trustgraph_base-1.2.8/trustgraph/schema/graph.py → trustgraph_base-1.2.9/trustgraph/schema/knowledge/embeddings.py

@@ -1,22 +1,8 @@
+from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double, Map
 
-from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double
-
-from . types import Error, Value, Triple
-from . topic import topic
-from . metadata import Metadata
-
-############################################################################
-
-# Entity context are an entity associated with textual context
-
-class EntityContext(Record):
-    entity = Value()
-    context = String()
-
-# This is a 'batching' mechanism for the above data
-class EntityContexts(Record):
-    metadata = Metadata()
-    entities = Array(EntityContext())
+from ..core.metadata import Metadata
+from ..core.primitives import Value, RowSchema
+from ..core.topic import topic
 
 ############################################################################
 
@@ -33,39 +19,38 @@ class GraphEmbeddings(Record):
 
 ############################################################################
 
-# Graph embeddings query
+# Document embeddings are embeddings associated with a chunk
 
-class GraphEmbeddingsRequest(Record):
+class ChunkEmbeddings(Record):
+    chunk = Bytes()
     vectors = Array(Array(Double()))
-    limit = Integer()
-    user = String()
-    collection = String()
 
-class GraphEmbeddingsResponse(Record):
-    error = Error()
-    entities = Array(Value())
+# This is a 'batching' mechanism for the above data
+class DocumentEmbeddings(Record):
+    metadata = Metadata()
+    chunks = Array(ChunkEmbeddings())
 
 ############################################################################
 
-# Graph triples
+# Object embeddings are embeddings associated with the primary key of an
+# object
 
-class Triples(Record):
+class ObjectEmbeddings(Record):
     metadata = Metadata()
-    triples = Array(Triple())
+    vectors = Array(Array(Double()))
+    name = String()
+    key_name = String()
+    id = String()
 
 ############################################################################
 
-# Triples query
+# Structured object embeddings with enhanced capabilities
 
-class TriplesQueryRequest(Record):
-    s = Value()
-    p = Value()
-    o = Value()
-    limit = Integer()
-    user = String()
-    collection = String()
-
-class TriplesQueryResponse(Record):
-    error = Error()
-    triples = Array(Triple())
+class StructuredObjectEmbedding(Record):
+    metadata = Metadata()
+    vectors = Array(Array(Double()))
+    schema_name = String()
+    object_id = String()  # Primary key value
+    field_embeddings = Map(Array(Double()))  # Per-field embeddings
 
+############################################################################

trustgraph_base-1.2.9/trustgraph/schema/knowledge/graph.py

@@ -0,0 +1,28 @@
+from pulsar.schema import Record, String, Array
+
+from ..core.primitives import Value, Triple
+from ..core.metadata import Metadata
+from ..core.topic import topic
+
+############################################################################
+
+# Entity context are an entity associated with textual context
+
+class EntityContext(Record):
+    entity = Value()
+    context = String()
+
+# This is a 'batching' mechanism for the above data
+class EntityContexts(Record):
+    metadata = Metadata()
+    entities = Array(EntityContext())
+
+############################################################################
+
+# Graph triples
+
+class Triples(Record):
+    metadata = Metadata()
+    triples = Array(Triple())
+
+############################################################################

{trustgraph_base-1.2.8/trustgraph/schema → trustgraph_base-1.2.9/trustgraph/schema/knowledge}/knowledge.py

@@ -1,11 +1,11 @@
 
 from pulsar.schema import Record, Bytes, String, Array, Long, Boolean
-from . types import Triple
-from . topic import topic
-from . types import Error
-from . metadata import Metadata
-from . documents import Document, TextDocument
-from . graph import Triples, GraphEmbeddings
+from ..core.primitives import Triple, Error
+from ..core.topic import topic
+from ..core.metadata import Metadata
+from .document import Document, TextDocument
+from .graph import Triples
+from .embeddings import GraphEmbeddings
 
 # get-kg-core
 #   -> (???)

trustgraph_base-1.2.9/trustgraph/schema/knowledge/nlp.py

@@ -0,0 +1,26 @@
+from pulsar.schema import Record, String, Boolean
+
+from ..core.topic import topic
+
+############################################################################
+
+# NLP extraction data types
+
+class Definition(Record):
+    name = String()
+    definition = String()
+
+class Topic(Record):
+    name = String()
+    definition = String()
+
+class Relationship(Record):
+    s = String()
+    p = String()
+    o = String()
+    o_entity = Boolean()
+
+class Fact(Record):
+    s = String()
+    p = String()
+    o = String()

trustgraph_base-1.2.9/trustgraph/schema/knowledge/object.py

@@ -0,0 +1,17 @@
+from pulsar.schema import Record, String, Map, Double
+
+from ..core.metadata import Metadata
+from ..core.topic import topic
+
+############################################################################
+
+# Extracted object from text processing
+
+class ExtractedObject(Record):
+    metadata = Metadata()
+    schema_name = String()  # Which schema this object belongs to
+    values = Map(String())  # Field name -> value
+    confidence = Double()
+    source_span = String()  # Text span where object was found
+
+############################################################################

trustgraph_base-1.2.9/trustgraph/schema/knowledge/rows.py

@@ -0,0 +1,16 @@
+from pulsar.schema import Record, Array, Map, String
+
+from ..core.metadata import Metadata
+from ..core.primitives import RowSchema
+from ..core.topic import topic
+
+############################################################################
+
+# Stores rows of information
+
+class Rows(Record):
+    metadata = Metadata()
+    row_schema = RowSchema()
+    rows = Array(Map(String()))
+
+############################################################################

trustgraph_base-1.2.9/trustgraph/schema/knowledge/structured.py

@@ -0,0 +1,17 @@
+from pulsar.schema import Record, String, Bytes, Map
+
+from ..core.metadata import Metadata
+from ..core.topic import topic
+
+############################################################################
+
+# Structured data submission for fire-and-forget processing
+
+class StructuredDataSubmission(Record):
+    metadata = Metadata()
+    format = String()  # "json", "csv", "xml"
+    schema_name = String()  # Reference to schema in config
+    data = Bytes()  # Raw data to ingest
+    options = Map(String())  # Format-specific options
+
+############################################################################

trustgraph_base-1.2.9/trustgraph/schema/services/__init__.py

@@ -0,0 +1,11 @@
+from .llm import *
+from .retrieval import *
+from .query import *
+from .agent import *
+from .flow import *
+from .prompt import *
+from .config import *
+from .library import *
+from .lookup import *
+from .nlp_query import *
+from .structured_query import *

{trustgraph_base-1.2.8/trustgraph/schema → trustgraph_base-1.2.9/trustgraph/schema/services}/agent.py

@@ -1,8 +1,8 @@
 
 from pulsar.schema import Record, String, Array, Map
 
-from . topic import topic
-from . types import Error
+from ..core.topic import topic
+from ..core.primitives import Error
 
 ############################################################################
 

{trustgraph_base-1.2.8/trustgraph/schema → trustgraph_base-1.2.9/trustgraph/schema/services}/config.py

@@ -1,8 +1,8 @@
 
 from pulsar.schema import Record, Bytes, String, Boolean, Array, Map, Integer
 
-from . topic import topic
-from . types import Error
+from ..core.topic import topic
+from ..core.primitives import Error
 
 ############################################################################
 

trustgraph_base-1.2.8/trustgraph/schema/flows.py → trustgraph_base-1.2.9/trustgraph/schema/services/flow.py

@@ -1,8 +1,8 @@
 
 from pulsar.schema import Record, Bytes, String, Boolean, Array, Map, Integer
 
-from . topic import topic
-from . types import Error
+from ..core.topic import topic
+from ..core.primitives import Error
 
 ############################################################################
 

{trustgraph_base-1.2.8/trustgraph/schema → trustgraph_base-1.2.9/trustgraph/schema/services}/library.py

@@ -1,10 +1,9 @@
 
 from pulsar.schema import Record, Bytes, String, Array, Long
-from . types import Triple
-from . topic import topic
-from . types import Error
-from . metadata import Metadata
-from . documents import Document, TextDocument
+from ..core.primitives import Triple, Error
+from ..core.topic import topic
+from ..core.metadata import Metadata
+from ..knowledge.document import Document, TextDocument
 
 # add-document
 #   -> (document_id, document_metadata, content)

trustgraph_base-1.2.8/trustgraph/schema/models.py → trustgraph_base-1.2.9/trustgraph/schema/services/llm.py

@@ -1,8 +1,8 @@
 
 from pulsar.schema import Record, String, Array, Double, Integer
 
-from . topic import topic
-from . types import Error
+from ..core.topic import topic
+from ..core.primitives import Error
 
 ############################################################################
 

{trustgraph_base-1.2.8/trustgraph/schema → trustgraph_base-1.2.9/trustgraph/schema/services}/lookup.py

@@ -1,9 +1,9 @@
 
 from pulsar.schema import Record, String
 
-from . types import Error, Value, Triple
-from . topic import topic
-from . metadata import Metadata
+from ..core.primitives import Error, Value, Triple
+from ..core.topic import topic
+from ..core.metadata import Metadata
 
 ############################################################################
 

trustgraph_base-1.2.9/trustgraph/schema/services/nlp_query.py

@@ -0,0 +1,22 @@
+from pulsar.schema import Record, String, Array, Map, Integer, Double
+
+from ..core.primitives import Error
+from ..core.topic import topic
+
+############################################################################
+
+# NLP to Structured Query Service - converts natural language to GraphQL
+
+class NLPToStructuredQueryRequest(Record):
+    natural_language_query = String()
+    max_results = Integer()
+    context_hints = Map(String())  # Optional context for query generation
+
+class NLPToStructuredQueryResponse(Record):
+    error = Error()
+    graphql_query = String()  # Generated GraphQL query
+    variables = Map(String())  # GraphQL variables if any
+    detected_schemas = Array(String())  # Which schemas the query targets
+    confidence = Double()
+
+############################################################################

{trustgraph_base-1.2.8/trustgraph/schema → trustgraph_base-1.2.9/trustgraph/schema/services}/prompt.py

@@ -1,32 +1,12 @@
+from pulsar.schema import Record, String, Map
 
-from pulsar.schema import Record, Bytes, String, Boolean, Array, Map, Integer
-
-from . topic import topic
-from . types import Error, RowSchema
+from ..core.primitives import Error
+from ..core.topic import topic
 
 ############################################################################
 
 # Prompt services, abstract the prompt generation
 
-class Definition(Record):
-    name = String()
-    definition = String()
-
-class Topic(Record):
-    name = String()
-    definition = String()
-
-class Relationship(Record):
-    s = String()
-    p = String()
-    o = String()
-    o_entity = Boolean()
-
-class Fact(Record):
-    s = String()
-    p = String()
-    o = String()
-
 # extract-definitions:
 #   chunk -> definitions
 # extract-relationships:
@@ -55,5 +35,4 @@ class PromptResponse(Record):
     # JSON encoded
     object = String()
 
-############################################################################
-
+############################################################################