remdb 0.3.163__py3-none-any.whl → 0.3.181__py3-none-any.whl

rem/services/embeddings/worker.py

@@ -23,6 +23,8 @@ Future:
 import asyncio
 import os
 from typing import Any, Optional
+import hashlib
+import uuid
 from uuid import uuid4
 
 import httpx
@@ -108,6 +110,7 @@ class EmbeddingWorker:
         self.task_queue: asyncio.Queue = asyncio.Queue()
         self.workers: list[asyncio.Task] = []
         self.running = False
+        self._in_flight_count = 0  # Track tasks being processed (not just in queue)
 
         # Store API key for direct HTTP requests
         from ...settings import settings
@@ -143,17 +146,18 @@ class EmbeddingWorker:
             return
 
         queue_size = self.task_queue.qsize()
-        logger.debug(f"Stopping EmbeddingWorker (processing {queue_size} queued tasks first)")
+        in_flight = self._in_flight_count
+        logger.debug(f"Stopping EmbeddingWorker (queue={queue_size}, in_flight={in_flight})")
 
-        # Wait for queue to drain (with timeout)
+        # Wait for both queue to drain AND in-flight tasks to complete
         max_wait = 30  # 30 seconds max
         waited = 0.0
-        while not self.task_queue.empty() and waited < max_wait:
+        while (not self.task_queue.empty() or self._in_flight_count > 0) and waited < max_wait:
             await asyncio.sleep(0.5)
             waited += 0.5
 
-        if not self.task_queue.empty():
-            remaining = self.task_queue.qsize()
+        if not self.task_queue.empty() or self._in_flight_count > 0:
+            remaining = self.task_queue.qsize() + self._in_flight_count
             logger.warning(
                 f"EmbeddingWorker timeout: {remaining} tasks remaining after {max_wait}s"
             )
@@ -205,12 +209,18 @@ class EmbeddingWorker:
                 if not batch:
                     continue
 
-                logger.debug(f"Worker {worker_id} processing batch of {len(batch)} tasks")
+                # Track in-flight tasks
+                self._in_flight_count += len(batch)
 
-                # Generate embeddings for batch
-                await self._process_batch(batch)
+                logger.debug(f"Worker {worker_id} processing batch of {len(batch)} tasks")
 
-                logger.debug(f"Worker {worker_id} completed batch")
+                try:
+                    # Generate embeddings for batch
+                    await self._process_batch(batch)
+                    logger.debug(f"Worker {worker_id} completed batch")
+                finally:
+                    # Always decrement in-flight count, even on error
+                    self._in_flight_count -= len(batch)
 
             except asyncio.CancelledError:
                 logger.debug(f"Worker {worker_id} cancelled")
@@ -373,7 +383,11 @@ class EmbeddingWorker:
         for task, embedding in zip(tasks, embeddings):
             table_name = f"embeddings_{task.table_name}"
 
-            # Build upsert SQL
+            # Generate deterministic ID from key fields (entity_id, field_name, provider)
+            key_string = f"{task.entity_id}:{task.field_name}:{task.provider}"
+            embedding_id = str(uuid.UUID(hashlib.md5(key_string.encode()).hexdigest()))
+
+            # Build upsert SQL - conflict on deterministic ID
             sql = f"""
                 INSERT INTO {table_name} (
                     id,
@@ -386,7 +400,7 @@ class EmbeddingWorker:
                     updated_at
                 )
                 VALUES ($1, $2, $3, $4, $5, $6, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
-                ON CONFLICT (entity_id, field_name, provider)
+                ON CONFLICT (id)
                 DO UPDATE SET
                     model = EXCLUDED.model,
                     embedding = EXCLUDED.embedding,
@@ -400,7 +414,7 @@ class EmbeddingWorker:
                 await self.postgres_service.execute(
                     sql,
                     (
-                        str(uuid4()),
+                        embedding_id,
                         task.entity_id,
                         task.field_name,
                         task.provider,

rem/services/postgres/register_type.py

@@ -268,7 +268,7 @@ BEGIN
             graph_edges,
             updated_at
         ) VALUES (
-            NEW.{entity_key_field}::VARCHAR,
+            normalize_key(NEW.{entity_key_field}::VARCHAR),
             '{table_name}',
             NEW.id,
             NEW.tenant_id,

rem/services/postgres/repository.py

@@ -74,7 +74,7 @@ class Repository(Generic[T]):
         self,
         records: T | list[T],
         embeddable_fields: list[str] | None = None,
-        generate_embeddings: bool = False,
+        generate_embeddings: bool = True,
     ) -> T | list[T]:
         """
         Upsert single record or list of records (create or update on ID conflict).
@@ -84,8 +84,9 @@ class Repository(Generic[T]):
 
         Args:
             records: Single model instance or list of model instances
-            embeddable_fields: Optional list of fields to generate embeddings for
-            generate_embeddings: Whether to queue embedding generation tasks
+            embeddable_fields: Optional list of fields to generate embeddings for.
+                              If None, auto-detects 'content' field if present.
+            generate_embeddings: Whether to queue embedding generation tasks (default: True)
 
         Returns:
             Single record or list of records with generated IDs (matches input type)
@@ -118,25 +119,35 @@ class Repository(Generic[T]):
                     record.id = row["id"]  # type: ignore[attr-defined]
 
         # Queue embedding generation if requested and worker is available
-        if generate_embeddings and embeddable_fields and self.db.embedding_worker:
+        if generate_embeddings and self.db.embedding_worker:
             from rem.services.embeddings import EmbeddingTask
-
-            for record in records_list:
-                for field_name in embeddable_fields:
-                    content = getattr(record, field_name, None)
-                    if content and isinstance(content, str):
-                        task = EmbeddingTask(
-                            task_id=f"{record.id}-{field_name}",  # type: ignore[attr-defined]
-                            entity_id=str(record.id),  # type: ignore[attr-defined]
-                            table_name=self.table_name,
-                            field_name=field_name,
-                            content=content,
-                            provider="openai",  # Default provider
-                            model="text-embedding-3-small",  # Default model
-                        )
-                        await self.db.embedding_worker.queue_task(task)
-
-            logger.debug(f"Queued {len(records_list) * len(embeddable_fields)} embedding tasks")
+            from .register_type import should_embed_field
+
+            # Auto-detect embeddable fields if not specified
+            if embeddable_fields is None:
+                embeddable_fields = [
+                    field_name
+                    for field_name, field_info in self.model_class.model_fields.items()
+                    if should_embed_field(field_name, field_info)
+                ]
+
+            if embeddable_fields:
+                for record in records_list:
+                    for field_name in embeddable_fields:
+                        content = getattr(record, field_name, None)
+                        if content and isinstance(content, str):
+                            task = EmbeddingTask(
+                                task_id=f"{record.id}-{field_name}",  # type: ignore[attr-defined]
+                                entity_id=str(record.id),  # type: ignore[attr-defined]
+                                table_name=self.table_name,
+                                field_name=field_name,
+                                content=content,
+                                provider="openai",  # Default provider
+                                model="text-embedding-3-small",  # Default model
+                            )
+                            await self.db.embedding_worker.queue_task(task)
+
+                logger.debug(f"Queued {len(records_list) * len(embeddable_fields)} embedding tasks")
 
         # Return single item or list to match input type
         return records_list[0] if is_single else records_list

rem/services/postgres/schema_generator.py

@@ -351,10 +351,10 @@ class SchemaGenerator:
 
         Priority:
         1. Field with json_schema_extra={\"entity_key\": True}
-        2. Field named \"name\"
+        2. Field named \"name\" (human-readable identifier)
         3. Field named \"key\"
-        4. Field named \"label\"
-        5. First string field
+        4. Field named \"uri\"
+        5. Field named \"id\" (fallback)
 
         Args:
             model: Pydantic model class
@@ -369,9 +369,9 @@ class SchemaGenerator:
                 if json_extra.get("entity_key"):
                     return field_name
 
-        # Check for key fields in priority order: id -> uri -> key -> name
+        # Check for key fields in priority order: name -> key -> uri -> id
         # (matching sql_builder.get_entity_key convention)
-        for candidate in ["id", "uri", "key", "name"]:
+        for candidate in ["name", "key", "uri", "id"]:
             if candidate in model.model_fields:
                 return candidate
 

rem/services/postgres/sql_builder.py

@@ -35,10 +35,11 @@ def get_natural_key(model: BaseModel) -> str | None:
 
 def get_entity_key(model: BaseModel) -> str:
     """
-    Get entity key for KV store following precedence: id -> uri -> key -> name.
+    Get entity key for KV store following precedence: name -> key -> uri -> id.
 
-    For KV store lookups, we prefer globally unique identifiers first (id),
-    then natural keys (uri/key/name). Always returns a value (id as fallback).
+    For KV store lookups, we prefer human-readable identifiers first (name/key),
+    then URIs, with id as the fallback. This allows users to lookup entities
+    by their natural names like "panic-disorder" instead of UUIDs.
 
     Args:
         model: Pydantic model instance
@@ -46,13 +47,13 @@ def get_entity_key(model: BaseModel) -> str:
     Returns:
         Entity key string (guaranteed to exist)
     """
-    for field in ["id", "uri", "key", "name"]:
+    for field in ["name", "key", "uri", "id"]:
         if hasattr(model, field):
             value = getattr(model, field)
             if value:
                 return str(value)
     # Should never reach here since id always exists in CoreModel
-    raise ValueError(f"Model {type(model)} has no id, uri, key, or name field")
+    raise ValueError(f"Model {type(model)} has no name, key, uri, or id field")
 
 
 def generate_deterministic_id(user_id: str | None, entity_key: str) -> uuid.UUID:

rem/services/user_service.py

@@ -4,7 +4,8 @@ User Service - User account management.
 Handles user creation, profile updates, and session linking.
 """
 
-from datetime import datetime
+from rem.utils.date_utils import utc_now
+from rem.utils.user_id import email_to_user_id
 from typing import Optional
 
 from loguru import logger
@@ -51,22 +52,24 @@ class UserService:
                     updated = True
             
             if updated:
-                user.updated_at = datetime.utcnow()
+                user.updated_at = utc_now()
                 await self.repo.upsert(user)
             
             return user
         
         # Create new user
+        # id and user_id = UUID5 hash of email (deterministic bijection)
+        # name = email (entity_key for LOOKUP by email in KV store)
+        hashed_id = email_to_user_id(email)
         user = User(
+            id=hashed_id,  # Database id = hash of email
             tenant_id=tenant_id,
-            user_id=email, # Use email as user_id for now? Or UUID?
-            # The User model has 'user_id' field but also 'id' UUID.
-            # Usually user_id is the external ID or email.
-            name=name,
+            user_id=hashed_id,  # user_id = hash of email (same as id)
+            name=email,  # Email as entity_key for REM LOOKUP
             email=email,
             tier=UserTier.FREE,
-            created_at=datetime.utcnow(),
-            updated_at=datetime.utcnow(),
+            created_at=utc_now(),
+            updated_at=utc_now(),
             metadata={"avatar_url": avatar_url} if avatar_url else {},
         )
         await self.repo.upsert(user)
@@ -117,7 +120,7 @@ class UserService:
 
         # Add to list
         user.anonymous_ids.append(anon_id)
-        user.updated_at = datetime.utcnow()
+        user.updated_at = utc_now()
         
         # Save
         await self.repo.upsert(user)

rem/settings.py

@@ -77,6 +77,7 @@ class LLMSettings(BaseSettings):
         LLM__ANTHROPIC_API_KEY or ANTHROPIC_API_KEY - Anthropic API key
         LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai)
         LLM__EMBEDDING_MODEL or EMBEDDING_MODEL - Default embedding model name
+        LLM__DEFAULT_STRUCTURED_OUTPUT - Default structured output mode (False = streaming text)
     """
 
     model_config = SettingsConfigDict(
@@ -138,6 +139,11 @@ class LLMSettings(BaseSettings):
         description="Default embedding model (provider-specific model name)",
     )
 
+    default_structured_output: bool = Field(
+        default=False,
+        description="Default structured output mode for agents. False = streaming text (easier), True = JSON schema validation",
+    )
+
     @field_validator("openai_api_key", mode="before")
     @classmethod
     def validate_openai_api_key(cls, v):
@@ -1028,7 +1034,7 @@ class ChatSettings(BaseSettings):
     - Prevents context window bloat while maintaining conversation continuity
 
     User Context (on-demand by default):
-    - Agent system prompt includes: "User ID: {user_id}. To load user profile: Use REM LOOKUP users/{user_id}"
+    - Agent system prompt includes: "User: {email}. To load user profile: Use REM LOOKUP \"{email}\""
     - Agent decides whether to load profile based on query
     - More efficient for queries that don't need personalization
 

rem/sql/background_indexes.sql

@@ -21,6 +21,11 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_moments_vector_hnsw
 ON embeddings_moments
 USING hnsw (embedding vector_cosine_ops);
 
+-- HNSW vector index for embeddings_ontologies
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_ontologies_vector_hnsw
+ON embeddings_ontologies
+USING hnsw (embedding vector_cosine_ops);
+
 -- HNSW vector index for embeddings_ontology_configs
 CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_ontology_configs_vector_hnsw
 ON embeddings_ontology_configs

rem/sql/migrations/001_install.sql

@@ -44,6 +44,33 @@ BEGIN
     RAISE NOTICE '✓ All required extensions installed successfully';
 END $$;
 
+-- ============================================================================
+-- NORMALIZATION HELPER
+-- ============================================================================
+
+-- Normalize entity keys to lower-kebab-case for consistent lookups
+-- "Mood Disorder" -> "mood-disorder"
+-- "mood_disorder" -> "mood-disorder"
+-- "MoodDisorder" -> "mood-disorder"
+CREATE OR REPLACE FUNCTION normalize_key(input TEXT)
+RETURNS TEXT AS $$
+BEGIN
+    RETURN lower(
+        regexp_replace(
+            regexp_replace(
+                regexp_replace(input, '([a-z])([A-Z])', '\1-\2', 'g'),  -- camelCase -> kebab
+                '[_\s]+', '-', 'g'  -- underscores/spaces -> hyphens
+            ),
+            '-+', '-', 'g'  -- collapse multiple hyphens
+        )
+    );
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+COMMENT ON FUNCTION normalize_key IS
+'Normalizes entity keys to lower-kebab-case for consistent lookups.
+Examples: "Mood Disorder" -> "mood-disorder", "mood_disorder" -> "mood-disorder"';
+
 -- ============================================================================
 -- MIGRATION TRACKING
 -- ============================================================================
@@ -237,10 +264,11 @@ BEGIN
 
     -- First lookup in KV store to get entity_type (table name)
     -- Include user-owned AND public (NULL user_id) entries
+    -- Normalize input key for consistent matching
     SELECT kv.entity_type INTO entity_table
     FROM kv_store kv
     WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
-    AND kv.entity_key = p_entity_key
+    AND kv.entity_key = normalize_key(p_entity_key)
     LIMIT 1;
 
     -- If not found, return empty
@@ -414,6 +442,7 @@ BEGIN
     FOR graph_keys IN
         WITH RECURSIVE graph_traversal AS (
             -- Base case: Find starting entity (user-owned OR public)
+            -- Normalize input key for consistent matching
             SELECT
                 0 AS depth,
                 kv.entity_key,
@@ -424,7 +453,7 @@ BEGIN
                 ARRAY[kv.entity_key]::TEXT[] AS path
             FROM kv_store kv
             WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
-            AND kv.entity_key = p_entity_key
+            AND kv.entity_key = normalize_key(p_entity_key)
 
             UNION ALL
 
@@ -441,7 +470,7 @@ BEGIN
             JOIN kv_store source_kv ON source_kv.entity_key = gt.entity_key
                 AND (source_kv.user_id = effective_user_id OR source_kv.user_id IS NULL)
             CROSS JOIN LATERAL jsonb_array_elements(COALESCE(source_kv.graph_edges, '[]'::jsonb)) AS edge
-            JOIN kv_store target_kv ON target_kv.entity_key = (edge->>'dst')::VARCHAR(255)
+            JOIN kv_store target_kv ON target_kv.entity_key = normalize_key((edge->>'dst')::VARCHAR(255))
                 AND (target_kv.user_id = effective_user_id OR target_kv.user_id IS NULL)
             WHERE gt.depth < p_max_depth
             AND (p_rel_type IS NULL OR (edge->>'rel_type')::VARCHAR(100) = p_rel_type)
@@ -657,7 +686,7 @@ BEGIN
         MIN(msg_counts.first_msg)::TIMESTAMP AS first_message_at,
         MAX(msg_counts.last_msg)::TIMESTAMP AS last_message_at
     FROM shared_sessions ss
-    LEFT JOIN users u ON u.user_id = ss.owner_user_id AND u.tenant_id = ss.tenant_id
+    LEFT JOIN users u ON u.id::text = ss.owner_user_id AND u.tenant_id = ss.tenant_id
     LEFT JOIN (
         SELECT
             m.session_id,