trustgraph-flow 2.7.8__tar.gz → 2.7.11__tar.gz

{trustgraph_flow-2.7.8 → trustgraph_flow-2.7.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-flow
-Version: 2.7.8
+Version: 2.7.11
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_flow-2.7.8 → trustgraph_flow-2.7.11}/trustgraph/direct/cassandra_kg.py

@@ -2,7 +2,7 @@
 import datetime
 import os
 import logging
-
+import re
 from cassandra.cluster import Cluster
 from cassandra.auth import PlainTextAuthProvider
 from cassandra.query import BatchStatement, SimpleStatement
@@ -548,7 +548,7 @@ class EntityCentricKnowledgeGraph:
         if hosts is None:
             hosts = ["localhost"]
 
-        self.keyspace = keyspace
+        self.keyspace = self.sanitize_name(keyspace)
         self.replication_factor = replication_factor
         self.username = username
 
@@ -572,6 +572,13 @@ class EntityCentricKnowledgeGraph:
 
         self.init()
         self.prepare_statements()
+        
+    def sanitize_name(self, name: str) -> str:
+        """Sanitize names for Cassandra compatibility"""
+        safe_name = re.sub(r'[^a-zA-Z0-9_]', '_', name)
+        if safe_name and not safe_name[0].isalpha():
+            safe_name = 'r_' + safe_name
+        return safe_name.lower()
 
     def clear(self):
         self.session.execute(f"""

{trustgraph_flow-2.7.8 → trustgraph_flow-2.7.11}/trustgraph/embeddings/row_embeddings/embeddings.py

@@ -122,10 +122,16 @@ class Processor(CollectionConfigHandler, FlowProcessor):
                     fields=fields
                 )
 
+                vector_indexes = set(schema_def.get("vector-indexes", []))
+                for field in fields:
+                    if field.name in vector_indexes:
+                        field.indexed = True
+
                 ws_schemas[schema_name] = row_schema
                 logger.info(
                     f"Loaded schema: {schema_name} with "
-                    f"{len(fields)} fields for {workspace}"
+                    f"{len(fields)} fields, {len(vector_indexes)} vector-indexed "
+                    f"for {workspace}"
                 )
 
             except Exception as e:
@@ -140,7 +146,7 @@ class Processor(CollectionConfigHandler, FlowProcessor):
         """Get all index names for a schema."""
         index_names = []
         for field in schema.fields:
-            if field.primary or field.indexed:
+            if field.indexed:
                 index_names.append(field.name)
         return index_names
 

trustgraph_flow-2.7.11/trustgraph/flow_version.py

@@ -0,0 +1 @@
+__version__ = "2.7.11"

{trustgraph_flow-2.7.8 → trustgraph_flow-2.7.11}/trustgraph/query/rows/cassandra/service.py

@@ -190,11 +190,17 @@ class Processor(FlowProcessor):
                     fields=fields
                 )
 
+                query_indexes = set(schema_def.get("query-indexes", []))
+                for field in fields:
+                    if field.name in query_indexes:
+                        field.indexed = True
+
                 ws_schemas[schema_name] = row_schema
                 builder.add_schema(schema_name, row_schema)
                 logger.info(
                     f"Loaded schema: {schema_name} with "
-                    f"{len(fields)} fields for {workspace}"
+                    f"{len(fields)} fields, {len(query_indexes)} query-indexed "
+                    f"for {workspace}"
                 )
 
             except Exception as e:

{trustgraph_flow-2.7.8 → trustgraph_flow-2.7.11}/trustgraph/storage/rows/cassandra/write.py

@@ -7,10 +7,13 @@ Uses a single 'rows' table with the schema:
     - schema_name: text
     - index_name: text
     - index_value: frozen<list<text>>
+    - row_id: text
     - data: map<text, text>
     - source: text
 
-Each row is written multiple times - once per indexed field defined in the schema.
+Each row is written multiple times - once per query-indexed field defined
+in the schema. The row_id (primary key value) is included as a clustering
+column to allow multiple rows with the same index value.
 """
 
 import asyncio
@@ -183,10 +186,16 @@ class Processor(CollectionConfigHandler, FlowProcessor):
                     fields=fields
                 )
 
+                query_indexes = set(schema_def.get("query-indexes", []))
+                for field in fields:
+                    if field.name in query_indexes:
+                        field.indexed = True
+
                 ws_schemas[schema_name] = row_schema
                 logger.info(
                     f"Loaded schema: {schema_name} with "
-                    f"{len(fields)} fields for {workspace}"
+                    f"{len(fields)} fields, {len(query_indexes)} query-indexed "
+                    f"for {workspace}"
                 )
 
             except Exception as e:
@@ -263,9 +272,10 @@ class Processor(CollectionConfigHandler, FlowProcessor):
             schema_name text,
             index_name text,
             index_value frozen<list<text>>,
+            row_id text,
             data map<text, text>,
             source text,
-            PRIMARY KEY ((collection, schema_name, index_name), index_value)
+            PRIMARY KEY ((collection, schema_name, index_name), index_value, row_id)
         )
         """
 
@@ -351,6 +361,14 @@ class Processor(CollectionConfigHandler, FlowProcessor):
         self.registered_partitions.add(cache_key)
         logger.info(f"Registered partitions for {collection}/{schema_name}: {index_names}")
 
+    def get_row_id(self, schema: RowSchema, value_map: Dict[str, str]) -> str:
+        """Get the primary key value for a row."""
+        for field in schema.fields:
+            if field.primary:
+                value = value_map.get(field.name)
+                return str(value) if value is not None else ""
+        return ""
+
     def build_index_value(self, value_map: Dict[str, str], index_name: str) -> List[str]:
         """
         Build the index_value list for a given index.
@@ -420,8 +438,8 @@ class Processor(CollectionConfigHandler, FlowProcessor):
         # Prepare insert statement
         insert_cql = f"""
         INSERT INTO {safe_keyspace}.rows
-            (collection, schema_name, index_name, index_value, data, source)
-        VALUES (%s, %s, %s, %s, %s, %s)
+            (collection, schema_name, index_name, index_value, row_id, data, source)
+        VALUES (%s, %s, %s, %s, %s, %s, %s)
         """
 
         # Process each row in the batch
@@ -434,6 +452,8 @@ class Processor(CollectionConfigHandler, FlowProcessor):
                 if raw_value is not None:
                     data_map[field.name] = str(raw_value)
 
+            row_id = self.get_row_id(schema, value_map)
+
             # Write one copy per index
             for index_name in index_names:
                 index_value = self.build_index_value(value_map, index_name)
@@ -450,7 +470,7 @@ class Processor(CollectionConfigHandler, FlowProcessor):
                     await async_execute(
                         self.session,
                         insert_cql,
-                        (collection, schema_name, index_name, index_value, data_map, source),
+                        (collection, schema_name, index_name, index_value, row_id, data_map, source),
                     )
                     rows_written += 1
                 except Exception as e:

{trustgraph_flow-2.7.8 → trustgraph_flow-2.7.11}/trustgraph_flow.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-flow
-Version: 2.7.8
+Version: 2.7.11
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

trustgraph_flow-2.7.8/trustgraph/flow_version.py

@@ -1 +0,0 @@
-__version__ = "2.7.8"