remdb 0.3.14__py3-none-any.whl → 0.3.133__py3-none-any.whl

rem/services/embeddings/worker.py

@@ -69,7 +69,7 @@ def get_global_embedding_worker(postgres_service: Any = None) -> "EmbeddingWorke
         if postgres_service is None:
             raise RuntimeError("Must provide postgres_service on first call to get_global_embedding_worker")
         _global_worker = EmbeddingWorker(postgres_service=postgres_service)
-        logger.info("Created global EmbeddingWorker singleton")
+        logger.debug("Created global EmbeddingWorker singleton")
 
     return _global_worker
 
@@ -117,7 +117,7 @@ class EmbeddingWorker:
                 "No OpenAI API key provided - embeddings will use zero vectors"
             )
 
-        logger.info(
+        logger.debug(
             f"Initialized EmbeddingWorker: {num_workers} workers, "
             f"batch_size={batch_size}, timeout={batch_timeout}s"
         )
@@ -125,17 +125,17 @@ class EmbeddingWorker:
     async def start(self) -> None:
         """Start worker pool."""
         if self.running:
-            logger.warning("EmbeddingWorker already running")
+            logger.debug("EmbeddingWorker already running")
             return
 
         self.running = True
-        logger.info(f"Starting {self.num_workers} embedding workers")
+        logger.debug(f"Starting {self.num_workers} embedding workers")
 
         for i in range(self.num_workers):
             worker = asyncio.create_task(self._worker_loop(i))
             self.workers.append(worker)
 
-        logger.info("EmbeddingWorker started")
+        logger.debug("EmbeddingWorker started")
 
     async def stop(self) -> None:
         """Stop worker pool gracefully - processes remaining queue before stopping."""
@@ -143,7 +143,7 @@ class EmbeddingWorker:
             return
 
         queue_size = self.task_queue.qsize()
-        logger.info(f"Stopping EmbeddingWorker (processing {queue_size} queued tasks first)")
+        logger.debug(f"Stopping EmbeddingWorker (processing {queue_size} queued tasks first)")
 
         # Wait for queue to drain (with timeout)
         max_wait = 30  # 30 seconds max
@@ -171,7 +171,7 @@ class EmbeddingWorker:
         await asyncio.gather(*self.workers, return_exceptions=True)
 
         self.workers.clear()
-        logger.info("EmbeddingWorker stopped")
+        logger.debug("EmbeddingWorker stopped")
 
     async def queue_task(self, task: EmbeddingTask) -> None:
         """
@@ -195,7 +195,7 @@ class EmbeddingWorker:
         Args:
             worker_id: Unique worker identifier
         """
-        logger.info(f"Worker {worker_id} started")
+        logger.debug(f"Worker {worker_id} started")
 
         while self.running:
             try:
@@ -205,7 +205,7 @@ class EmbeddingWorker:
                 if not batch:
                     continue
 
-                logger.info(f"Worker {worker_id} processing batch of {len(batch)} tasks")
+                logger.debug(f"Worker {worker_id} processing batch of {len(batch)} tasks")
 
                 # Generate embeddings for batch
                 await self._process_batch(batch)
@@ -213,14 +213,14 @@ class EmbeddingWorker:
                 logger.debug(f"Worker {worker_id} completed batch")
 
             except asyncio.CancelledError:
-                logger.info(f"Worker {worker_id} cancelled")
+                logger.debug(f"Worker {worker_id} cancelled")
                 break
             except Exception as e:
                 logger.error(f"Worker {worker_id} error: {e}", exc_info=True)
                 # Continue processing (don't crash worker on error)
                 await asyncio.sleep(1)
 
-        logger.info(f"Worker {worker_id} stopped")
+        logger.debug(f"Worker {worker_id} stopped")
 
     async def _collect_batch(self) -> list[EmbeddingTask]:
         """
@@ -284,10 +284,10 @@ class EmbeddingWorker:
             )
 
             # Upsert to database
-            logger.info(f"Upserting {len(embeddings)} embeddings to database...")
+            logger.debug(f"Upserting {len(embeddings)} embeddings to database...")
             await self._upsert_embeddings(batch, embeddings)
 
-            logger.info(
+            logger.debug(
                 f"Successfully generated and stored {len(embeddings)} embeddings "
                 f"(provider={provider}, model={model})"
             )
@@ -315,7 +315,7 @@ class EmbeddingWorker:
         """
         if provider == "openai" and self.openai_api_key:
             try:
-                logger.info(
+                logger.debug(
                     f"Generating OpenAI embeddings for {len(texts)} texts using {model}"
                 )
 
@@ -336,7 +336,7 @@ class EmbeddingWorker:
                     data = response.json()
                     embeddings = [item["embedding"] for item in data["data"]]
 
-                    logger.info(
+                    logger.debug(
                         f"Successfully generated {len(embeddings)} embeddings from OpenAI"
                     )
                     return embeddings
@@ -409,7 +409,7 @@ class EmbeddingWorker:
                     ),
                 )
 
-                logger.info(
+                logger.debug(
                     f"Upserted embedding: {task.table_name}.{task.entity_id}.{task.field_name}"
                 )
 

rem/services/phoenix/client.py

@@ -792,29 +792,169 @@ class PhoenixClient:
         label: str | None = None,
         score: float | None = None,
         explanation: str | None = None,
-    ) -> None:
-        """Add feedback annotation to a span.
+        metadata: dict[str, Any] | None = None,
+        trace_id: str | None = None,
+    ) -> str | None:
+        """Add feedback annotation to a span via Phoenix REST API.
+
+        Uses direct HTTP POST to /v1/span_annotations for reliability
+        (Phoenix Python client API changes frequently).
 
         Args:
-            span_id: Span ID to annotate
-            annotation_name: Name of the annotation (e.g., "correctness")
+            span_id: Span ID to annotate (hex string)
+            annotation_name: Name of the annotation (e.g., "correctness", "user_feedback")
             annotator_kind: Type of annotator ("HUMAN", "LLM", "CODE")
-            label: Optional label (e.g., "correct", "incorrect")
+            label: Optional label (e.g., "correct", "incorrect", "helpful")
             score: Optional numeric score (0.0-1.0)
             explanation: Optional explanation text
+            metadata: Optional additional metadata dict
+            trace_id: Optional trace ID (used if span lookup needed)
+
+        Returns:
+            Annotation ID if successful, None otherwise
         """
+        import httpx
+
         try:
-            self._client.add_span_annotation(  # type: ignore[attr-defined]
-                span_id=span_id,
-                name=annotation_name,
-                annotator_kind=annotator_kind,
-                label=label,
-                score=score,
-                explanation=explanation,
-            )
+            # Build annotation payload for Phoenix REST API
+            annotation_data = {
+                "span_id": span_id,
+                "name": annotation_name,
+                "annotator_kind": annotator_kind,
+                "result": {
+                    "label": label,
+                    "score": score,
+                    "explanation": explanation,
+                },
+                "metadata": metadata or {},
+            }
+
+            # Add trace_id if provided
+            if trace_id:
+                annotation_data["trace_id"] = trace_id
+
+            # POST to Phoenix REST API
+            annotations_endpoint = f"{self.config.base_url}/v1/span_annotations"
+            headers = {}
+            if self.config.api_key:
+                headers["Authorization"] = f"Bearer {self.config.api_key}"
+
+            with httpx.Client(timeout=5.0) as client:
+                response = client.post(
+                    annotations_endpoint,
+                    json={"data": [annotation_data]},
+                    headers=headers,
+                )
+                response.raise_for_status()
 
             logger.info(f"Added {annotator_kind} feedback to span {span_id}")
+            return span_id  # Return span_id as annotation reference
 
+        except httpx.HTTPStatusError as e:
+            logger.error(
+                f"Failed to add span feedback (HTTP {e.response.status_code}): "
+                f"{e.response.text if hasattr(e, 'response') else 'N/A'}"
+            )
+            return None
         except Exception as e:
             logger.error(f"Failed to add span feedback: {e}")
-            raise
+            return None
+
+    def sync_user_feedback(
+        self,
+        span_id: str,
+        rating: int | None = None,
+        categories: list[str] | None = None,
+        comment: str | None = None,
+        feedback_id: str | None = None,
+        trace_id: str | None = None,
+    ) -> str | None:
+        """Sync user feedback to Phoenix as a span annotation.
+
+        Convenience method for syncing Feedback entities to Phoenix.
+        Converts REM feedback format to Phoenix annotation format.
+
+        Args:
+            span_id: OTEL span ID to annotate
+            rating: User rating (-1, 1-5 scale)
+            categories: List of feedback categories
+            comment: Free-text comment
+            feedback_id: Optional REM feedback ID for reference
+            trace_id: Optional trace ID for the span
+
+        Returns:
+            Phoenix annotation ID if successful
+
+        Example:
+            >>> client.sync_user_feedback(
+            ...     span_id="abc123",
+            ...     rating=4,
+            ...     categories=["helpful", "accurate"],
+            ...     comment="Great response!"
+            ... )
+        """
+        # Convert rating to 0-1 score
+        # Rating scheme:
+        #   -1 = thumbs down → score 0.0
+        #    1 = thumbs up   → score 1.0
+        #  2-5 = star rating → normalized to 0-1 range
+        score = None
+        if rating is not None:
+            if rating == -1:
+                score = 0.0
+            elif rating == 1:
+                score = 1.0  # Thumbs up
+            elif 2 <= rating <= 5:
+                score = (rating - 1) / 4.0  # 2→0.25, 3→0.5, 4→0.75, 5→1.0
+
+        # Use primary category as label
+        label = categories[0] if categories else None
+
+        # Build explanation from comment and additional categories
+        explanation = comment
+        if categories and len(categories) > 1:
+            cats_str = ", ".join(categories[1:])
+            if explanation:
+                explanation = f"{explanation} [Categories: {cats_str}]"
+            else:
+                explanation = f"Categories: {cats_str}"
+
+        # Build metadata
+        metadata: dict[str, Any] = {
+            "rating": rating,
+            "categories": categories or [],
+        }
+        if feedback_id:
+            metadata["rem_feedback_id"] = feedback_id
+
+        return self.add_span_feedback(
+            span_id=span_id,
+            annotation_name="user_feedback",
+            annotator_kind="HUMAN",
+            label=label,
+            score=score,
+            explanation=explanation,
+            metadata=metadata,
+            trace_id=trace_id,
+        )
+
+    def get_span_annotations(
+        self,
+        span_id: str,
+        annotation_name: str | None = None,
+    ) -> list[dict[str, Any]]:
+        """Get annotations for a span.
+
+        Args:
+            span_id: Span ID to query
+            annotation_name: Optional filter by annotation name
+
+        Returns:
+            List of annotation dicts
+
+        TODO: Implement once Phoenix client exposes this method
+        """
+        # TODO: Phoenix client doesn't expose annotation query yet
+        # This is a stub for future implementation
+        logger.warning("get_span_annotations not yet implemented in Phoenix client")
+        return []

rem/services/postgres/README.md

@@ -348,8 +348,27 @@ results = await service.vector_search(
 
 ### Initialize Service
 
+There are two ways to initialize the PostgresService:
+
+**Option 1: Factory function (recommended for apps using remdb as a library)**
+
+```python
+from rem.services.postgres import get_postgres_service
+
+# Uses POSTGRES__CONNECTION_STRING from environment
+pg = get_postgres_service()
+if pg is None:
+    raise RuntimeError("Database not configured - set POSTGRES__CONNECTION_STRING")
+
+await pg.connect()
+# ... use pg ...
+await pg.disconnect()
+```
+
+**Option 2: Direct instantiation**
+
 ```python
-from rem.services.postgres import PostgresService, Repository
+from rem.services.postgres import PostgresService
 
 service = PostgresService(
     connection_string="postgresql://user:pass@localhost/remdb",
@@ -359,6 +378,9 @@ service = PostgresService(
 await service.connect()
 ```
 
+> **Note**: `get_postgres_service()` returns the service directly. It does NOT support
+> `async with` context manager syntax. Always call `connect()` and `disconnect()` explicitly.
+
 ### Using Repository Pattern
 
 **Generic Repository** for simple CRUD operations:
@@ -514,34 +536,156 @@ results = await service.vector_search(
 - HNSW parameters: `m=16, ef_construction=64` (tunable)
 - Monitor shared_buffers and work_mem
 
-## Migrations
+## Schema Management
 
-Run migrations in order:
+REM uses a **code-as-source-of-truth** approach. Pydantic models define the schema, and the database is kept in sync via diff-based migrations.
 
-```bash
-psql -d remdb -f sql/migrations/001_setup_extensions.sql
-psql -d remdb -f sql/migrations/002_kv_store_cache.sql
-psql -d remdb -f sql/generated_schema.sql
+### File Structure
+
+```
+src/rem/sql/
+├── migrations/
+│   ├── 001_install.sql          # Core infrastructure (manual)
+│   └── 002_install_models.sql   # Entity tables (auto-generated)
+└── background_indexes.sql       # HNSW vector indexes (optional)
 ```
 
-Background indexes (after data load):
+**Key principle**: Only two migration files. No incremental `003_`, `004_` files.
+
+### CLI Commands
 
 ```bash
-psql -d remdb -f sql/background_indexes.sql
+# Apply migrations (installs extensions, core tables, entity tables)
+rem db migrate
+
+# Check migration status
+rem db status
+
+# Generate schema SQL from models (for remdb development)
+rem db schema generate --models src/rem/models/entities
+
+# Validate models for schema generation
+rem db schema validate --models src/rem/models/entities
 ```
 
-## CLI Usage
+### Model Registry
 
-Generate schema from models:
+Models are discovered via the registry:
 
-```bash
-rem schema generate --models src/rem/models/entities --output sql/schema.sql
+```python
+import rem
+from rem.models.core import CoreModel
+
+@rem.register_model
+class MyEntity(CoreModel):
+    name: str
+    description: str  # Auto-embeds
+```
+
+## Using REM as a Library (Downstream Apps)
+
+When building an application that **depends on remdb as a package** (e.g., `pip install remdb`),
+there are important differences from developing remdb itself.
+
+### What Works Out of the Box
+
+1. **All core entity tables** - Resources, Messages, Users, Sessions, etc.
+2. **PostgresService** - Full database access via `get_postgres_service()`
+3. **Repository pattern** - CRUD operations for core entities
+4. **Migrations** - `rem db migrate` applies the bundled SQL files
+
+```python
+# In your downstream app (e.g., myapp/main.py)
+from rem.services.postgres import get_postgres_service
+from rem.models.entities import Message, Resource
+
+pg = get_postgres_service()
+await pg.connect()
+
+# Use core entities - tables already exist
+messages = await pg.query(Message, {"session_id": "abc"})
 ```
 
-Validate models:
+### Custom Models in Downstream Apps
+
+The `@rem.register_model` decorator registers models in the **runtime registry**, which is useful for:
+- Schema introspection at runtime
+- Future tooling that reads the registry
+
+However, **`rem db migrate` only applies SQL files bundled in the remdb package**.
+Custom models from downstream apps do NOT automatically get tables created.
+
+**Options for custom model tables:**
+
+**Option A: Use core entities with metadata**
+
+Store custom data in the `metadata` JSONB field of existing entities:
+
+```python
+resource = Resource(
+    name="my-custom-thing",
+    content="...",
+    metadata={"custom_field": "value", "another": 123}
+)
+```
+
+**Option B: Create tables manually**
+
+Write and apply your own SQL:
+
+```sql
+-- myapp/sql/custom_tables.sql
+CREATE TABLE IF NOT EXISTS conversation_summaries (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    session_ref TEXT NOT NULL,
+    summary TEXT NOT NULL,
+    -- ... include CoreModel fields for compatibility
+    user_id VARCHAR(256),
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+```
 
 ```bash
-rem schema validate --models src/rem/models/entities
+psql $DATABASE_URL -f myapp/sql/custom_tables.sql
+```
+
+**Option C: Contribute upstream**
+
+If your model is generally useful, contribute it to remdb so it's included in
+the next release and `rem db migrate` creates it automatically.
+
+### Example: Downstream App Structure
+
+```
+myapp/
+├── main.py              # Import models, start API
+├── models/
+│   └── __init__.py      # @rem.register_model decorators
+├── sql/
+│   └── custom.sql       # Manual migrations for custom tables
+├── .env                 # POSTGRES__CONNECTION_STRING, LLM keys
+└── pyproject.toml       # dependencies = ["remdb>=0.3.110"]
+```
+
+```python
+# myapp/models/__init__.py
+import rem
+from rem.models.core import CoreModel
+
+@rem.register_model
+class ConversationSummary(CoreModel):
+    """Registered for introspection, but table created via sql/custom.sql"""
+    session_ref: str
+    summary: str
+```
+
+```python
+# myapp/main.py
+import models  # Registers custom models
+
+from rem.api.main import app  # Use REM's FastAPI app
+# Or build your own app using rem.services
 ```
 
 ## Configuration

rem/services/postgres/__init__.py

@@ -2,6 +2,7 @@
 PostgreSQL service for CloudNativePG database operations.
 """
 
+from .diff_service import DiffService, SchemaDiff
 from .repository import Repository
 from .service import PostgresService
 
@@ -20,4 +21,4 @@ def get_postgres_service() -> PostgresService | None:
     return PostgresService()
 
 
-__all__ = ["PostgresService", "get_postgres_service", "Repository"]
+__all__ = ["PostgresService", "get_postgres_service", "Repository", "DiffService", "SchemaDiff"]