remdb 0.3.180__py3-none-any.whl → 0.3.258__py3-none-any.whl

rem/services/postgres/register_type.py

@@ -94,14 +94,14 @@ def generate_table_schema(
     # Always add id as primary key
     columns.append("id UUID PRIMARY KEY DEFAULT uuid_generate_v4()")
 
-    # Add tenant_id if tenant scoped
+    # Add tenant_id if tenant scoped (nullable - NULL means public/shared)
     if tenant_scoped:
-        columns.append("tenant_id VARCHAR(100) NOT NULL")
-        indexes.append(f"CREATE INDEX idx_{table_name}_tenant ON {table_name} (tenant_id);")
+        columns.append("tenant_id VARCHAR(100)")
+        indexes.append(f"CREATE INDEX IF NOT EXISTS idx_{table_name}_tenant ON {table_name} (tenant_id);")
 
     # Add user_id (owner field)
     columns.append("user_id VARCHAR(256)")
-    indexes.append(f"CREATE INDEX idx_{table_name}_user ON {table_name} (user_id);")
+    indexes.append(f"CREATE INDEX IF NOT EXISTS idx_{table_name}_user ON {table_name} (user_id);")
 
     # Process Pydantic fields (skip system fields)
     for field_name, field_info in model.model_fields.items():
@@ -125,19 +125,19 @@ def generate_table_schema(
     # Add graph_edges JSONB field
     columns.append("graph_edges JSONB DEFAULT '[]'::jsonb")
     indexes.append(
-        f"CREATE INDEX idx_{table_name}_graph_edges ON {table_name} USING GIN (graph_edges);"
+        f"CREATE INDEX IF NOT EXISTS idx_{table_name}_graph_edges ON {table_name} USING GIN (graph_edges);"
     )
 
     # Add metadata JSONB field
     columns.append("metadata JSONB DEFAULT '{}'::jsonb")
     indexes.append(
-        f"CREATE INDEX idx_{table_name}_metadata ON {table_name} USING GIN (metadata);"
+        f"CREATE INDEX IF NOT EXISTS idx_{table_name}_metadata ON {table_name} USING GIN (metadata);"
     )
 
     # Add tags field (TEXT[] for list[str])
     columns.append("tags TEXT[] DEFAULT ARRAY[]::TEXT[]")
     indexes.append(
-        f"CREATE INDEX idx_{table_name}_tags ON {table_name} USING GIN (tags);"
+        f"CREATE INDEX IF NOT EXISTS idx_{table_name}_tags ON {table_name} USING GIN (tags);"
     )
 
     # Generate CREATE TABLE statement
@@ -202,10 +202,10 @@ CREATE TABLE IF NOT EXISTS {embeddings_table} (
 );
 
 -- Index for entity lookup (get all embeddings for entity)
-CREATE INDEX idx_{embeddings_table}_entity ON {embeddings_table} (entity_id);
+CREATE INDEX IF NOT EXISTS idx_{embeddings_table}_entity ON {embeddings_table} (entity_id);
 
 -- Index for field + provider lookup
-CREATE INDEX idx_{embeddings_table}_field_provider ON {embeddings_table} (field_name, provider);
+CREATE INDEX IF NOT EXISTS idx_{embeddings_table}_field_provider ON {embeddings_table} (field_name, provider);
 
 -- HNSW index for vector similarity search (created in background)
 -- Note: This will be created by background thread after data load
@@ -258,6 +258,7 @@ BEGIN
         RETURN OLD;
     ELSIF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN
         -- Upsert to KV_STORE (O(1) lookup by entity_key)
+        -- tenant_id can be NULL (meaning public/shared data)
         INSERT INTO kv_store (
             entity_key,
             entity_type,
@@ -277,7 +278,7 @@ BEGIN
             COALESCE(NEW.graph_edges, '[]'::jsonb),
             CURRENT_TIMESTAMP
         )
-        ON CONFLICT (tenant_id, entity_key)
+        ON CONFLICT (COALESCE(tenant_id, ''), entity_key)
         DO UPDATE SET
             entity_id = EXCLUDED.entity_id,
             user_id = EXCLUDED.user_id,

rem/services/postgres/repository.py

@@ -33,15 +33,15 @@ if TYPE_CHECKING:
 
 def get_postgres_service() -> "PostgresService | None":
     """
-    Get PostgresService instance with connection string from settings.
+    Get PostgresService singleton from parent module.
 
-    Returns None if Postgres is disabled.
+    Uses late import to avoid circular import issues.
+    Previously had a separate _postgres_instance here which caused
+    "pool not connected" errors due to duplicate connection pools.
     """
-    if not settings.postgres.enabled:
-        return None
-    
-    from .service import PostgresService
-    return PostgresService()
+    # Late import to avoid circular import (repository.py imported by __init__.py)
+    from rem.services.postgres import get_postgres_service as _get_singleton
+    return _get_singleton()
 
 T = TypeVar("T", bound=BaseModel)
 
@@ -74,7 +74,7 @@ class Repository(Generic[T]):
         self,
         records: T | list[T],
         embeddable_fields: list[str] | None = None,
-        generate_embeddings: bool = False,
+        generate_embeddings: bool = True,
     ) -> T | list[T]:
         """
         Upsert single record or list of records (create or update on ID conflict).
@@ -84,8 +84,9 @@ class Repository(Generic[T]):
 
         Args:
             records: Single model instance or list of model instances
-            embeddable_fields: Optional list of fields to generate embeddings for
-            generate_embeddings: Whether to queue embedding generation tasks
+            embeddable_fields: Optional list of fields to generate embeddings for.
+                              If None, auto-detects 'content' field if present.
+            generate_embeddings: Whether to queue embedding generation tasks (default: True)
 
         Returns:
             Single record or list of records with generated IDs (matches input type)
@@ -118,25 +119,35 @@ class Repository(Generic[T]):
                     record.id = row["id"]  # type: ignore[attr-defined]
 
         # Queue embedding generation if requested and worker is available
-        if generate_embeddings and embeddable_fields and self.db.embedding_worker:
+        if generate_embeddings and self.db.embedding_worker:
             from rem.services.embeddings import EmbeddingTask
-
-            for record in records_list:
-                for field_name in embeddable_fields:
-                    content = getattr(record, field_name, None)
-                    if content and isinstance(content, str):
-                        task = EmbeddingTask(
-                            task_id=f"{record.id}-{field_name}",  # type: ignore[attr-defined]
-                            entity_id=str(record.id),  # type: ignore[attr-defined]
-                            table_name=self.table_name,
-                            field_name=field_name,
-                            content=content,
-                            provider="openai",  # Default provider
-                            model="text-embedding-3-small",  # Default model
-                        )
-                        await self.db.embedding_worker.queue_task(task)
-
-            logger.debug(f"Queued {len(records_list) * len(embeddable_fields)} embedding tasks")
+            from .register_type import should_embed_field
+
+            # Auto-detect embeddable fields if not specified
+            if embeddable_fields is None:
+                embeddable_fields = [
+                    field_name
+                    for field_name, field_info in self.model_class.model_fields.items()
+                    if should_embed_field(field_name, field_info)
+                ]
+
+            if embeddable_fields:
+                for record in records_list:
+                    for field_name in embeddable_fields:
+                        content = getattr(record, field_name, None)
+                        if content and isinstance(content, str):
+                            task = EmbeddingTask(
+                                task_id=f"{record.id}-{field_name}",  # type: ignore[attr-defined]
+                                entity_id=str(record.id),  # type: ignore[attr-defined]
+                                table_name=self.table_name,
+                                field_name=field_name,
+                                content=content,
+                                provider="openai",  # Default provider
+                                model="text-embedding-3-small",  # Default model
+                            )
+                            await self.db.embedding_worker.queue_task(task)
+
+                logger.debug(f"Queued {len(records_list) * len(embeddable_fields)} embedding tasks")
 
         # Return single item or list to match input type
         return records_list[0] if is_single else records_list

rem/services/postgres/schema_generator.py

@@ -351,10 +351,10 @@ class SchemaGenerator:
 
         Priority:
         1. Field with json_schema_extra={\"entity_key\": True}
-        2. Field named \"name\"
+        2. Field named \"name\" (human-readable identifier)
         3. Field named \"key\"
-        4. Field named \"label\"
-        5. First string field
+        4. Field named \"uri\"
+        5. Field named \"id\" (fallback)
 
         Args:
             model: Pydantic model class
@@ -369,9 +369,9 @@ class SchemaGenerator:
                 if json_extra.get("entity_key"):
                     return field_name
 
-        # Check for key fields in priority order: id -> uri -> key -> name
+        # Check for key fields in priority order: name -> key -> uri -> id
         # (matching sql_builder.get_entity_key convention)
-        for candidate in ["id", "uri", "key", "name"]:
+        for candidate in ["name", "key", "uri", "id"]:
             if candidate in model.model_fields:
                 return candidate
 

rem/services/postgres/sql_builder.py

@@ -35,10 +35,11 @@ def get_natural_key(model: BaseModel) -> str | None:
 
 def get_entity_key(model: BaseModel) -> str:
     """
-    Get entity key for KV store following precedence: id -> uri -> key -> name.
+    Get entity key for KV store following precedence: name -> key -> uri -> id.
 
-    For KV store lookups, we prefer globally unique identifiers first (id),
-    then natural keys (uri/key/name). Always returns a value (id as fallback).
+    For KV store lookups, we prefer human-readable identifiers first (name/key),
+    then URIs, with id as the fallback. This allows users to lookup entities
+    by their natural names like "panic-disorder" instead of UUIDs.
 
     Args:
         model: Pydantic model instance
@@ -46,13 +47,13 @@ def get_entity_key(model: BaseModel) -> str:
     Returns:
         Entity key string (guaranteed to exist)
     """
-    for field in ["id", "uri", "key", "name"]:
+    for field in ["name", "key", "uri", "id"]:
         if hasattr(model, field):
             value = getattr(model, field)
             if value:
                 return str(value)
     # Should never reach here since id always exists in CoreModel
-    raise ValueError(f"Model {type(model)} has no id, uri, key, or name field")
+    raise ValueError(f"Model {type(model)} has no name, key, uri, or id field")
 
 
 def generate_deterministic_id(user_id: str | None, entity_key: str) -> uuid.UUID:

rem/services/rem/README.md

@@ -40,15 +40,16 @@ FuzzyQuery ::= FUZZY <text:string> [THRESHOLD <t:float>] [LIMIT <n:int>]
   available   : Stage 1+
   example     : FUZZY "sara" THRESHOLD 0.5 LIMIT 10
 
-SearchQuery ::= SEARCH <text:string> [TABLE <table:string>] [WHERE <clause:string>] [LIMIT <n:int>]
+SearchQuery ::= SEARCH <text:string> [IN|TABLE <table:string>] [WHERE <clause:string>] [LIMIT <n:int>]
   text        : Semantic query text
-  table       : Target table (default: "resources")
+  table       : Target table (default: "resources"). Use IN or TABLE keyword.
   clause      : Optional PostgreSQL WHERE clause for hybrid filtering (combines vector + structured)
   limit       : Max results (default: 10)
   performance : Indexed (pgvector)
   available   : Stage 3+
   examples    :
-    - SEARCH "database migration" TABLE resources LIMIT 10
+    - SEARCH "database migration" IN resources LIMIT 10
+    - SEARCH "parcel delivery" IN ontologies
     - SEARCH "team discussion" TABLE moments WHERE "moment_type='meeting'" LIMIT 5
     - SEARCH "project updates" WHERE "created_at >= '2024-01-01'" LIMIT 20
     - SEARCH "AI research" WHERE "tags @> ARRAY['machine-learning']" LIMIT 10

rem/services/rem/parser.py

@@ -64,7 +64,7 @@ class RemQueryParser:
             token_upper = token.upper()
 
             # Handle REM keywords that take a value
-            if token_upper in ("LIMIT", "DEPTH", "THRESHOLD", "TYPE", "FROM", "WITH"):
+            if token_upper in ("LIMIT", "DEPTH", "THRESHOLD", "TYPE", "FROM", "WITH", "TABLE", "IN", "WHERE"):
                 if i + 1 < len(tokens):
                     keyword_map = {
                         "LIMIT": "limit",
@@ -73,6 +73,9 @@ class RemQueryParser:
                         "TYPE": "edge_types",
                         "FROM": "initial_query",
                         "WITH": "initial_query",
+                        "TABLE": "table_name",
+                        "IN": "table_name",  # IN is alias for TABLE
+                        "WHERE": "where_clause",
                     }
                     key = keyword_map[token_upper]
                     value = tokens[i + 1]
@@ -161,15 +164,9 @@ class RemQueryParser:
             params["query_text"] = combined_value
 
         elif query_type == QueryType.SEARCH:
-            # SEARCH expects: SEARCH <table> <query_text> [LIMIT n]
-            # First positional arg is table name, rest is query text
-            if len(positional_args) >= 2:
-                params["table_name"] = positional_args[0]
-                params["query_text"] = " ".join(positional_args[1:])
-            elif len(positional_args) == 1:
-                # Could be table name or query text - assume query text if no table
-                params["query_text"] = positional_args[0]
-            # If no positional args, params stays empty
+            # SEARCH expects: SEARCH <text> [TABLE <table>] [WHERE <clause>] [LIMIT n]
+            # All positional args are query_text, TABLE/WHERE/LIMIT are handled as keywords
+            params["query_text"] = combined_value
 
         elif query_type == QueryType.TRAVERSE:
             params["initial_query"] = combined_value

rem/services/rem/service.py

@@ -478,6 +478,53 @@ class RemService:
         parser = RemQueryParser()
         return parser.parse(query_string)
 
+    async def execute_query_string(
+        self, query_string: str, user_id: str | None = None
+    ) -> dict[str, Any]:
+        """
+        Execute a REM dialect query string directly.
+
+        This is the unified entry point for executing REM queries from both
+        the CLI and API. It handles parsing the query string, creating the
+        RemQuery model, and executing it.
+
+        Args:
+            query_string: REM dialect query (e.g., 'LOOKUP "Sarah Chen"',
+                         'SEARCH resources "API design"', 'SELECT * FROM users')
+            user_id: Optional user ID for query isolation
+
+        Returns:
+            Dict with query results and metadata:
+            - query_type: The type of query executed
+            - results: List of result rows
+            - count: Number of results
+            - Additional fields depending on query type
+
+        Raises:
+            ValueError: If the query string is invalid
+            QueryExecutionError: If query execution fails
+
+        Example:
+            >>> result = await rem_service.execute_query_string(
+            ...     'LOOKUP "Sarah Chen"',
+            ...     user_id="user-123"
+            ... )
+            >>> print(result["count"])
+            1
+        """
+        # Parse the query string into type and parameters
+        query_type, parameters = self._parse_query_string(query_string)
+
+        # Create and validate the RemQuery model
+        rem_query = RemQuery.model_validate({
+            "query_type": query_type,
+            "parameters": parameters,
+            "user_id": user_id,
+        })
+
+        # Execute and return results
+        return await self.execute_query(rem_query)
+
     async def ask_rem(
         self, natural_query: str, tenant_id: str, llm_model: str | None = None, plan_mode: bool = False
     ) -> dict[str, Any]:

rem/services/session/__init__.py

@@ -1,6 +1,13 @@
 """Session management services for conversation persistence and compression."""
 
 from .compression import MessageCompressor, SessionMessageStore
+from .pydantic_messages import audit_session_history, session_to_pydantic_messages
 from .reload import reload_session
 
-__all__ = ["MessageCompressor", "SessionMessageStore", "reload_session"]
+__all__ = [
+    "MessageCompressor",
+    "SessionMessageStore",
+    "audit_session_history",
+    "reload_session",
+    "session_to_pydantic_messages",
+]

rem/services/session/compression.py

@@ -65,7 +65,7 @@ def truncate_key(key: str, max_length: int = MAX_ENTITY_KEY_LENGTH) -> str:
     logger.warning(f"Truncated key from {len(key)} to {len(truncated)} chars: {key[:50]}...")
     return truncated
 
-from rem.models.entities import Message
+from rem.models.entities import Message, Session
 from rem.services.postgres import PostgresService, Repository
 from rem.settings import settings
 
@@ -96,7 +96,7 @@ class MessageCompressor:
         Returns:
             Compressed message dict
         """
-        content = message.get("content", "")
+        content = message.get("content") or ""
 
         # Don't compress short messages or system messages
         if (
@@ -177,6 +177,39 @@ class SessionMessageStore:
         self.user_id = user_id
         self.compressor = compressor or MessageCompressor()
         self.repo = Repository(Message)
+        self._session_repo = Repository(Session, table_name="sessions")
+
+    async def _ensure_session_exists(
+        self,
+        session_id: str,
+        user_id: str | None = None,
+    ) -> None:
+        """
+        Ensure session exists, creating it if necessary.
+
+        Args:
+            session_id: Session UUID from X-Session-Id header
+            user_id: Optional user identifier
+        """
+        try:
+            # Check if session already exists by UUID
+            existing = await self._session_repo.get_by_id(session_id)
+            if existing:
+                return  # Session already exists
+
+            # Create new session with the provided UUID as id
+            session = Session(
+                id=session_id,  # Use the provided UUID as session id
+                name=session_id,  # Default name to UUID, can be updated later
+                user_id=user_id or self.user_id,
+                tenant_id=self.user_id,  # tenant_id set to user_id for scoping
+            )
+            await self._session_repo.upsert(session)
+            logger.info(f"Created session {session_id} for user {user_id or self.user_id}")
+
+        except Exception as e:
+            # Log but don't fail - session creation is best-effort
+            logger.warning(f"Failed to ensure session exists: {e}")
 
     async def store_message(
         self,
@@ -209,7 +242,7 @@ class SessionMessageStore:
         # Use pre-generated id from message dict if available (for frontend feedback)
         msg = Message(
             id=message.get("id"),  # Use pre-generated ID if provided
-            content=message.get("content", ""),
+            content=message.get("content") or "",
             message_type=message.get("role", "assistant"),
             session_id=session_id,
             tenant_id=self.user_id,  # Set tenant_id to user_id (application scoped to user)
@@ -283,8 +316,10 @@ class SessionMessageStore:
         """
         Store all session messages and return compressed versions.
 
+        Ensures session exists before storing messages.
+
         Args:
-            session_id: Session identifier
+            session_id: Session UUID
             messages: List of messages to store
             user_id: Optional user identifier
             compress: Whether to compress messages (default: True)
@@ -296,10 +331,13 @@ class SessionMessageStore:
             logger.debug("Postgres disabled, returning messages uncompressed")
             return messages
 
+        # Ensure session exists before storing messages
+        await self._ensure_session_exists(session_id, user_id)
+
         compressed_messages = []
 
         for idx, message in enumerate(messages):
-            content = message.get("content", "")
+            content = message.get("content") or ""
 
             # Only store and compress long assistant responses
             if (
@@ -330,6 +368,8 @@ class SessionMessageStore:
                 }
 
                 # For tool messages, include tool call details in metadata
+                # Note: tool_arguments is stored only when provided (parent tool calls)
+                # For child tool calls (e.g., register_metadata), args are in content as JSON
                 if message.get("role") == "tool":
                     if message.get("tool_call_id"):
                         msg_metadata["tool_call_id"] = message.get("tool_call_id")
@@ -398,6 +438,8 @@ class SessionMessageStore:
                 }
 
                 # For tool messages, reconstruct tool call metadata
+                # Note: tool_arguments may be in metadata (parent calls) or parsed from
+                # content (child calls like register_metadata) by pydantic_messages.py
                 if role == "tool" and msg.metadata:
                     if msg.metadata.get("tool_call_id"):
                         msg_dict["tool_call_id"] = msg.metadata["tool_call_id"]