remdb 0.3.7__py3-none-any.whl → 0.3.133__py3-none-any.whl

rem/services/embeddings/api.py

@@ -5,13 +5,20 @@ Provides synchronous and async wrappers for embedding generation using
 raw HTTP requests (no OpenAI SDK dependency).
 """
 
-import os
 from typing import Optional, cast
 
 import httpx
 import requests
 from loguru import logger
 
+from rem.utils.constants import DEFAULT_EMBEDDING_DIMS, HTTP_TIMEOUT_DEFAULT
+
+
+def _get_openai_api_key() -> Optional[str]:
+    """Get OpenAI API key from settings."""
+    from rem.settings import settings
+    return settings.llm.openai_api_key
+
 
 def generate_embedding(
     text: str,
@@ -26,19 +33,19 @@ def generate_embedding(
         text: Text to embed
         model: Model name (default: text-embedding-3-small)
         provider: Provider name (default: openai)
-        api_key: API key (defaults to OPENAI_API_KEY env var)
+        api_key: API key (defaults to settings.llm.openai_api_key)
 
     Returns:
         Embedding vector (1536 dimensions for text-embedding-3-small)
     """
     if provider == "openai":
-        api_key = api_key or os.getenv("OPENAI_API_KEY")
+        api_key = api_key or _get_openai_api_key()
         if not api_key:
             logger.warning("No OpenAI API key - returning zero vector")
-            return [0.0] * 1536
+            return [0.0] * DEFAULT_EMBEDDING_DIMS
 
         try:
-            logger.info(f"Generating OpenAI embedding for text using {model}")
+            logger.debug(f"Generating OpenAI embedding for text using {model}")
 
             response = requests.post(
                 "https://api.openai.com/v1/embeddings",
@@ -47,22 +54,22 @@ def generate_embedding(
                     "Content-Type": "application/json",
                 },
                 json={"input": [text], "model": model},
-                timeout=30,
+                timeout=HTTP_TIMEOUT_DEFAULT,
             )
             response.raise_for_status()
 
             data = response.json()
             embedding = data["data"][0]["embedding"]
-            logger.info(f"Successfully generated embedding (dimension: {len(embedding)})")
+            logger.debug(f"Successfully generated embedding (dimension: {len(embedding)})")
             return cast(list[float], embedding)
 
         except Exception as e:
             logger.error(f"Failed to generate embedding from OpenAI: {e}", exc_info=True)
-            return [0.0] * 1536
+            return [0.0] * DEFAULT_EMBEDDING_DIMS
 
     else:
         logger.warning(f"Unsupported provider '{provider}' - returning zero vector")
-        return [0.0] * 1536
+        return [0.0] * DEFAULT_EMBEDDING_DIMS
 
 
 async def generate_embedding_async(
@@ -78,19 +85,19 @@ async def generate_embedding_async(
         text: Text to embed
         model: Model name (default: text-embedding-3-small)
         provider: Provider name (default: openai)
-        api_key: API key (defaults to OPENAI_API_KEY env var)
+        api_key: API key (defaults to settings.llm.openai_api_key)
 
     Returns:
         Embedding vector (1536 dimensions for text-embedding-3-small)
     """
     if provider == "openai":
-        api_key = api_key or os.getenv("OPENAI_API_KEY")
+        api_key = api_key or _get_openai_api_key()
         if not api_key:
             logger.warning("No OpenAI API key - returning zero vector")
-            return [0.0] * 1536
+            return [0.0] * DEFAULT_EMBEDDING_DIMS
 
         try:
-            logger.info(f"Generating OpenAI embedding for text using {model}")
+            logger.debug(f"Generating OpenAI embedding for text using {model}")
 
             async with httpx.AsyncClient() as client:
                 response = await client.post(
@@ -100,21 +107,21 @@ async def generate_embedding_async(
                         "Content-Type": "application/json",
                     },
                     json={"input": [text], "model": model},
-                    timeout=30.0,
+                    timeout=HTTP_TIMEOUT_DEFAULT,
                 )
                 response.raise_for_status()
 
                 data = response.json()
                 embedding = data["data"][0]["embedding"]
-                logger.info(
+                logger.debug(
                     f"Successfully generated embedding (dimension: {len(embedding)})"
                 )
                 return cast(list[float], embedding)
 
         except Exception as e:
             logger.error(f"Failed to generate embedding from OpenAI: {e}", exc_info=True)
-            return [0.0] * 1536
+            return [0.0] * DEFAULT_EMBEDDING_DIMS
 
     else:
         logger.warning(f"Unsupported provider '{provider}' - returning zero vector")
-        return [0.0] * 1536
+        return [0.0] * DEFAULT_EMBEDDING_DIMS

rem/services/embeddings/worker.py

@@ -69,7 +69,7 @@ def get_global_embedding_worker(postgres_service: Any = None) -> "EmbeddingWorke
         if postgres_service is None:
             raise RuntimeError("Must provide postgres_service on first call to get_global_embedding_worker")
         _global_worker = EmbeddingWorker(postgres_service=postgres_service)
-        logger.info("Created global EmbeddingWorker singleton")
+        logger.debug("Created global EmbeddingWorker singleton")
 
     return _global_worker
 
@@ -117,7 +117,7 @@ class EmbeddingWorker:
                 "No OpenAI API key provided - embeddings will use zero vectors"
             )
 
-        logger.info(
+        logger.debug(
             f"Initialized EmbeddingWorker: {num_workers} workers, "
             f"batch_size={batch_size}, timeout={batch_timeout}s"
         )
@@ -125,17 +125,17 @@ class EmbeddingWorker:
     async def start(self) -> None:
         """Start worker pool."""
         if self.running:
-            logger.warning("EmbeddingWorker already running")
+            logger.debug("EmbeddingWorker already running")
             return
 
         self.running = True
-        logger.info(f"Starting {self.num_workers} embedding workers")
+        logger.debug(f"Starting {self.num_workers} embedding workers")
 
         for i in range(self.num_workers):
             worker = asyncio.create_task(self._worker_loop(i))
             self.workers.append(worker)
 
-        logger.info("EmbeddingWorker started")
+        logger.debug("EmbeddingWorker started")
 
     async def stop(self) -> None:
         """Stop worker pool gracefully - processes remaining queue before stopping."""
@@ -143,7 +143,7 @@ class EmbeddingWorker:
             return
 
         queue_size = self.task_queue.qsize()
-        logger.info(f"Stopping EmbeddingWorker (processing {queue_size} queued tasks first)")
+        logger.debug(f"Stopping EmbeddingWorker (processing {queue_size} queued tasks first)")
 
         # Wait for queue to drain (with timeout)
         max_wait = 30  # 30 seconds max
@@ -171,7 +171,7 @@ class EmbeddingWorker:
         await asyncio.gather(*self.workers, return_exceptions=True)
 
         self.workers.clear()
-        logger.info("EmbeddingWorker stopped")
+        logger.debug("EmbeddingWorker stopped")
 
     async def queue_task(self, task: EmbeddingTask) -> None:
         """
@@ -195,7 +195,7 @@ class EmbeddingWorker:
         Args:
             worker_id: Unique worker identifier
         """
-        logger.info(f"Worker {worker_id} started")
+        logger.debug(f"Worker {worker_id} started")
 
         while self.running:
             try:
@@ -205,7 +205,7 @@ class EmbeddingWorker:
                 if not batch:
                     continue
 
-                logger.info(f"Worker {worker_id} processing batch of {len(batch)} tasks")
+                logger.debug(f"Worker {worker_id} processing batch of {len(batch)} tasks")
 
                 # Generate embeddings for batch
                 await self._process_batch(batch)
@@ -213,14 +213,14 @@ class EmbeddingWorker:
                 logger.debug(f"Worker {worker_id} completed batch")
 
             except asyncio.CancelledError:
-                logger.info(f"Worker {worker_id} cancelled")
+                logger.debug(f"Worker {worker_id} cancelled")
                 break
             except Exception as e:
                 logger.error(f"Worker {worker_id} error: {e}", exc_info=True)
                 # Continue processing (don't crash worker on error)
                 await asyncio.sleep(1)
 
-        logger.info(f"Worker {worker_id} stopped")
+        logger.debug(f"Worker {worker_id} stopped")
 
     async def _collect_batch(self) -> list[EmbeddingTask]:
         """
@@ -284,10 +284,10 @@ class EmbeddingWorker:
             )
 
             # Upsert to database
-            logger.info(f"Upserting {len(embeddings)} embeddings to database...")
+            logger.debug(f"Upserting {len(embeddings)} embeddings to database...")
             await self._upsert_embeddings(batch, embeddings)
 
-            logger.info(
+            logger.debug(
                 f"Successfully generated and stored {len(embeddings)} embeddings "
                 f"(provider={provider}, model={model})"
             )
@@ -315,7 +315,7 @@ class EmbeddingWorker:
         """
         if provider == "openai" and self.openai_api_key:
             try:
-                logger.info(
+                logger.debug(
                     f"Generating OpenAI embeddings for {len(texts)} texts using {model}"
                 )
 
@@ -336,7 +336,7 @@ class EmbeddingWorker:
                     data = response.json()
                     embeddings = [item["embedding"] for item in data["data"]]
 
-                    logger.info(
+                    logger.debug(
                         f"Successfully generated {len(embeddings)} embeddings from OpenAI"
                     )
                     return embeddings
@@ -409,7 +409,7 @@ class EmbeddingWorker:
                     ),
                 )
 
-                logger.info(
+                logger.debug(
                     f"Upserted embedding: {task.table_name}.{task.entity_id}.{task.field_name}"
                 )
 

rem/services/phoenix/EXPERIMENT_DESIGN.md

@@ -164,7 +164,7 @@ cp curated-queries.csv experiments/rem-001/validation/production/
 **Option C: Curated Engrams**
 ```bash
 # Generate engrams from REM data
-rem dreaming full --user-id test-user --tenant-id acme --generate-test-cases
+rem dreaming full --user-id test-user  --generate-test-cases
 
 # Review and select high-quality engrams
 rem engram list --quality high --limit 100 --output engrams.csv
@@ -357,7 +357,7 @@ Level 4 (Mature):   Multiple cycles, full query capabilities
 # Generate engrams from REM data
 rem dreaming full \
   --user-id test-user \
-  --tenant-id acme \
+   \
   --generate-test-cases \
   --quality-level 3
 
@@ -1027,7 +1027,7 @@ rem experiments experiment run rem-lookup-ask_rem-golden \
 
 ```bash
 # 1. Generate high-quality engrams
-rem dreaming full --tenant-id acme --generate-test-cases --quality-level 4
+rem dreaming full  --generate-test-cases --quality-level 4
 
 # 2. Export engrams
 rem engram export rem-engrams-mature-mixed --output engrams.csv --format phoenix