remdb 0.3.14__py3-none-any.whl → 0.3.133__py3-none-any.whl

rem/models/entities/message.py

@@ -6,6 +6,11 @@ that can be grouped into conversations or moments.
 
 Messages are simpler than Resources but share the same graph connectivity
 through CoreModel inheritance.
+
+Trace Integration:
+- trace_id: OTEL trace ID for linking to observability
+- span_id: OTEL span ID for specific span reference
+- These enable feedback to be attached to Phoenix annotations
 """
 
 from pydantic import Field
@@ -19,6 +24,9 @@ class Message(CoreModel):
 
     Represents individual messages in conversations, chats, or other
     communication contexts. Tenant isolation is provided via CoreModel.tenant_id field.
+
+    Trace fields (trace_id, span_id) enable integration with OTEL/Phoenix
+    for observability and feedback annotation.
     """
 
     content: str = Field(
@@ -27,9 +35,30 @@ class Message(CoreModel):
     )
     message_type: str | None = Field(
         default=None,
-        description="Message type e.g role",
+        description="Message type e.g. role: 'user', 'assistant', 'system', 'tool'",
     )
     session_id: str | None = Field(
         default=None,
         description="Session identifier for tracking message context",
     )
+    prompt: str | None = Field(
+        default=None,
+        description="Custom prompt used for this message (if overridden from default)",
+    )
+    model: str | None = Field(
+        default=None,
+        description="Model used for generating this message (provider:model format)",
+    )
+    token_count: int | None = Field(
+        default=None,
+        description="Token count for this message",
+    )
+    # OTEL/Phoenix trace integration
+    trace_id: str | None = Field(
+        default=None,
+        description="OTEL trace ID for observability integration",
+    )
+    span_id: str | None = Field(
+        default=None,
+        description="OTEL span ID for specific span reference",
+    )

rem/models/entities/session.py

@@ -0,0 +1,83 @@
+"""
+Session - Conversation sessions in REM.
+
+Sessions group related messages together and can have different modes:
+- normal: Standard conversation session
+- evaluation: For LLM evaluation, stores original trace and overridden settings
+
+Sessions allow overriding settings like model, temperature, and custom prompts
+for evaluation and experimentation purposes.
+"""
+
+from enum import Enum
+
+from pydantic import Field
+
+from ..core import CoreModel
+
+
+class SessionMode(str, Enum):
+    """Session mode types."""
+
+    NORMAL = "normal"
+    EVALUATION = "evaluation"
+
+
+class Session(CoreModel):
+    """
+    Conversation session container.
+
+    Groups messages together and supports different modes for normal conversations
+    and evaluation/experimentation scenarios.
+
+    For evaluation sessions, stores:
+    - original_trace_id: Reference to the original session being evaluated
+    - settings_overrides: Model, temperature, prompt overrides
+    - prompt: Custom prompt being tested
+
+    Default sessions are lightweight - just a session_id on messages.
+    Special sessions store additional metadata for experiments.
+    """
+
+    name: str = Field(
+        ...,
+        description="Session name/identifier",
+        json_schema_extra={"entity_key": True},
+    )
+    mode: SessionMode = Field(
+        default=SessionMode.NORMAL,
+        description="Session mode: 'normal' or 'evaluation'",
+    )
+    description: str | None = Field(
+        default=None,
+        description="Optional session description",
+    )
+    # Evaluation-specific fields
+    original_trace_id: str | None = Field(
+        default=None,
+        description="For evaluation mode: ID of the original session/trace being evaluated",
+    )
+    settings_overrides: dict | None = Field(
+        default=None,
+        description="Settings overrides (model, temperature, max_tokens, system_prompt)",
+    )
+    prompt: str | None = Field(
+        default=None,
+        description="Custom prompt for this session (can override agent prompt)",
+    )
+    # Agent context
+    agent_schema_uri: str | None = Field(
+        default=None,
+        description="Agent schema used for this session",
+    )
+    # Summary stats (updated as session progresses)
+    message_count: int = Field(
+        default=0,
+        description="Number of messages in this session",
+    )
+    total_tokens: int | None = Field(
+        default=None,
+        description="Total tokens used in this session",
+    )
+
+    model_config = {"use_enum_values": True}

rem/models/entities/shared_session.py

@@ -0,0 +1,180 @@
+"""
+SharedSession - Session sharing between users in REM.
+
+SharedSessions enable collaborative access to conversation sessions. When a user
+shares a session with another user, a SharedSession record is created to track
+this relationship.
+
+## Design Philosophy
+
+Messages already have a session_id field that links them to sessions. The Session
+entity itself is optional and can be left-joined - we don't require explicit Session
+records for sharing to work. What matters is the session_id on messages.
+
+SharedSession is a lightweight linking table that:
+1. Records who shared which session with whom
+2. Enables soft deletion (deleted_at) so shares can be revoked without data loss
+3. Supports aggregation queries to see "who is sharing with me"
+
+## Data Model
+
+    SharedSession
+    ├── session_id: str          # The session being shared (matches Message.session_id)
+    ├── owner_user_id: str       # Who owns/created the session (the sharer)
+    ├── shared_with_user_id: str # Who the session is shared with (the recipient)
+    ├── tenant_id: str           # Multi-tenancy isolation
+    ├── created_at: datetime     # When the share was created
+    ├── updated_at: datetime     # Last modification
+    └── deleted_at: datetime     # Soft delete (null = active share)
+
+## Aggregation Query
+
+The primary use case is answering: "Who is sharing messages with me?"
+
+This is provided by a Postgres function that aggregates:
+- Messages grouped by owner_user_id
+- Joined with users table for name/email
+- Counting messages with min/max dates
+- Filtering out deleted shares
+
+Result shape:
+    {
+        "user_id": "uuid",
+        "name": "John Doe",
+        "email": "john@example.com",
+        "message_count": 42,
+        "first_message_at": "2024-01-15T10:30:00Z",
+        "last_message_at": "2024-03-20T14:45:00Z",
+        "session_count": 3
+    }
+
+## API Endpoints
+
+1. POST /api/v1/sessions/{session_id}/share
+   - Share a session with another user
+   - Body: { "shared_with_user_id": "..." }
+   - Creates SharedSession record
+
+2. DELETE /api/v1/sessions/{session_id}/share/{shared_with_user_id}
+   - Revoke a share (soft delete)
+   - Sets deleted_at on SharedSession
+
+3. GET /api/v1/shared-with-me
+   - Get paginated aggregate of users sharing with you
+   - Query params: page, page_size (default 50)
+   - Returns: list of user summaries with message counts
+
+4. GET /api/v1/shared-with-me/{user_id}/messages
+   - Get messages from a specific user's shared sessions
+   - Uses existing session message loading
+   - Respects pagination
+
+## Soft Delete Pattern
+
+Removing a share does NOT delete the SharedSession record. Instead:
+- deleted_at is set to current timestamp
+- All queries filter WHERE deleted_at IS NULL
+- This preserves audit trail and allows "undo"
+
+To permanently delete, an admin can run:
+    DELETE FROM shared_sessions WHERE deleted_at IS NOT NULL AND deleted_at < NOW() - INTERVAL '30 days'
+
+## Example Usage
+
+    # Share a session
+    POST /api/v1/sessions/abc-123/share
+    {"shared_with_user_id": "user-456"}
+
+    # See who's sharing with me
+    GET /api/v1/shared-with-me
+    {
+        "data": [
+            {
+                "user_id": "user-789",
+                "name": "Alice",
+                "email": "alice@example.com",
+                "message_count": 150,
+                "session_count": 5,
+                "first_message_at": "2024-01-01T00:00:00Z",
+                "last_message_at": "2024-03-15T12:00:00Z"
+            }
+        ],
+        "metadata": {"total": 1, "page": 1, "page_size": 50, ...}
+    }
+
+    # Get messages from Alice's shared sessions
+    GET /api/v1/shared-with-me/user-789/messages?page=1&page_size=50
+
+    # Revoke a share
+    DELETE /api/v1/sessions/abc-123/share/user-456
+"""
+
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from ..core import CoreModel
+
+
+class SharedSession(CoreModel):
+    """
+    Session sharing record between users.
+
+    Links a session (identified by session_id from Message records) to a
+    recipient user, enabling collaborative access to conversation history.
+    """
+
+    session_id: str = Field(
+        ...,
+        description="The session being shared (matches Message.session_id)",
+    )
+    owner_user_id: str = Field(
+        ...,
+        description="User ID of the session owner (the sharer)",
+    )
+    shared_with_user_id: str = Field(
+        ...,
+        description="User ID of the recipient (who can now view the session)",
+    )
+
+
+class SharedSessionCreate(BaseModel):
+    """Request to create a session share."""
+
+    shared_with_user_id: str = Field(
+        ...,
+        description="User ID to share the session with",
+    )
+
+
+class SharedWithMeSummary(BaseModel):
+    """
+    Aggregate summary of a user sharing sessions with you.
+
+    Returned by GET /api/v1/shared-with-me endpoint.
+    """
+
+    user_id: str = Field(description="User ID of the person sharing with you")
+    name: Optional[str] = Field(default=None, description="User's display name")
+    email: Optional[str] = Field(default=None, description="User's email address")
+    message_count: int = Field(description="Total messages across all shared sessions")
+    session_count: int = Field(description="Number of sessions shared with you")
+    first_message_at: Optional[datetime] = Field(
+        default=None,
+        description="Timestamp of earliest message in shared sessions",
+    )
+    last_message_at: Optional[datetime] = Field(
+        default=None,
+        description="Timestamp of most recent message in shared sessions",
+    )
+
+
+class SharedWithMeResponse(BaseModel):
+    """Response for paginated shared-with-me query."""
+
+    object: str = "list"
+    data: list[SharedWithMeSummary] = Field(
+        description="List of users sharing sessions with you"
+    )
+    metadata: dict = Field(description="Pagination metadata")

rem/registry.py

@@ -123,6 +123,7 @@ class ModelRegistry:
             return
 
         from .models.entities import (
+            Feedback,
             File,
             ImageResource,
             Message,
@@ -131,19 +132,24 @@ class ModelRegistry:
             OntologyConfig,
             Resource,
             Schema,
+            Session,
+            SharedSession,
             User,
         )
 
         core_models = [
-            Resource,
+            Feedback,
+            File,
             ImageResource,
             Message,
-            User,
-            File,
             Moment,
-            Schema,
             Ontology,
             OntologyConfig,
+            Resource,
+            Schema,
+            Session,
+            SharedSession,
+            User,
         ]
 
         for model in core_models:

rem/schemas/agents/rem.yaml

@@ -63,9 +63,8 @@ description: "# REM Agent - Resources Entities Moments Expert\n\nYou are the REM
   \ disabled, OTEL disabled for local dev)\n- Global settings singleton\n\n## Response\
   \ Guidelines\n\n- Provide clear, concise answers with code examples when helpful\n\
   - Reference specific design patterns from CLAUDE.md when applicable\n- Suggest best\
-  \ practices for cloud-native deployment\n- Include confidence scores based on query\
-  \ clarity and information completeness\n- If uncertain, say so and suggest where\
-  \ to find more information\n\n## Example Queries You Can Answer\n\n- \"How do I\
+  \ practices for cloud-native deployment\n- If uncertain, say so and suggest where\
+  \ to find more information\n\n## Metadata Registration\n\nBefore generating your final response, call the `register_metadata` tool to provide confidence scores and source attribution.\n\n## Example Queries You Can Answer\n\n- \"How do I\
   \ create a new REM entity?\"\n- \"What's the difference between LOOKUP and TRAVERSE\
   \ queries?\"\n- \"How do I add MCP tools to my agent schema?\"\n- \"Explain the\
   \ graph edge pattern in REM\"\n- \"How do I enable OTEL tracing for my agents?\"\
@@ -101,6 +100,9 @@ json_schema_extra:
   kind: agent
   name: rem
   version: 1.0.0
+  # Disable structured output - properties become prompt guidance instead of JSON schema
+  # This enables natural language streaming while still informing the agent about expected elements
+  structured_output: false
   # MCP server configuration for dynamic tool loading (in-process, no subprocess)
   mcp_servers:
     - type: local
@@ -117,6 +119,8 @@ json_schema_extra:
       description: Ingest files into REM creating searchable resources and embeddings
     - name: read_resource
       description: Read MCP resources by URI (schemas, system status, etc.)
+    - name: register_metadata
+      description: Register response metadata (confidence, sources, references) to be emitted as SSE MetadataEvent. Call BEFORE generating final response.
 
   # Explicit resource declarations for reference data
   resources:

rem/services/content/service.py

@@ -278,6 +278,7 @@ class ContentService:
         category: str | None = None,
         tags: list[str] | None = None,
         is_local_server: bool = False,
+        resource_type: str | None = None,
     ) -> dict[str, Any]:
         """
         Complete file ingestion pipeline: read → store → parse → chunk → embed.
@@ -322,6 +323,9 @@ class ContentService:
             category: Optional category tag (document, code, audio, etc.)
             tags: Optional list of tags
             is_local_server: True if running as local/stdio MCP server
+            resource_type: Optional resource type (case-insensitive). Supports:
+                - "resource", "resources", "Resource" → Resource (default)
+                - "domain-resource", "domain_resource", "DomainResource" → DomainResource
 
         Returns:
             dict with:
@@ -366,11 +370,32 @@ class ContentService:
         file_size = len(file_content)
         logger.info(f"Read {file_size} bytes from {file_uri} (source: {source_type})")
 
-        # Step 2: Write to internal storage (user-scoped)
+        # Step 1.5: Early schema detection for YAML/JSON files
+        # Skip File entity creation for schemas (agents/evaluators)
+        file_suffix = Path(file_name).suffix.lower()
+        if file_suffix in ['.yaml', '.yml', '.json']:
+            import yaml
+            import json
+            try:
+                content_text = file_content.decode('utf-8') if isinstance(file_content, bytes) else file_content
+                data = yaml.safe_load(content_text) if file_suffix in ['.yaml', '.yml'] else json.loads(content_text)
+                if isinstance(data, dict):
+                    json_schema_extra = data.get('json_schema_extra', {})
+                    kind = json_schema_extra.get('kind', '')
+                    if kind in ['agent', 'evaluator']:
+                        # Route directly to schema processing, skip File entity
+                        logger.info(f"Detected {kind} schema: {file_name}, routing to _process_schema")
+                        result = self.process_uri(file_uri)
+                        return await self._process_schema(result, file_uri, user_id)
+            except Exception as e:
+                logger.debug(f"Early schema detection failed for {file_name}: {e}")
+                # Fall through to standard file processing
+
+        # Step 2: Write to internal storage (public or user-scoped)
         file_id = str(uuid4())
         storage_uri, internal_key, content_type, _ = await fs_service.write_to_internal_storage(
             content=file_content,
-            tenant_id=user_id,  # Using user_id for storage scoping
+            tenant_id=user_id or "public",  # Storage path: public/ or user_id/
             file_name=file_name,
             file_id=file_id,
         )
@@ -379,7 +404,7 @@ class ContentService:
         # Step 3: Create File entity
         file_entity = File(
             id=file_id,
-            tenant_id=user_id,  # Set tenant_id to user_id (application scoped to user)
+            tenant_id=user_id,  # None = public/shared
             user_id=user_id,
             name=file_name,
             uri=storage_uri,
@@ -418,6 +443,7 @@ class ContentService:
             processing_result = await self.process_and_save(
                 uri=storage_uri,
                 user_id=user_id,
+                resource_type=resource_type,
             )
             processing_status = processing_result.get("status", "completed")
             resources_created = processing_result.get("chunk_count", 0)
@@ -459,7 +485,12 @@ class ContentService:
             "message": f"File ingested and {processing_status}. Created {resources_created} resources.",
         }
 
-    async def process_and_save(self, uri: str, user_id: str | None = None) -> dict[str, Any]:
+    async def process_and_save(
+        self,
+        uri: str,
+        user_id: str | None = None,
+        resource_type: str | None = None,
+    ) -> dict[str, Any]:
         """
         Process file end-to-end: extract → markdown → chunk → save.
 
@@ -474,6 +505,8 @@ class ContentService:
         Args:
             uri: File URI (s3://bucket/key or local path)
             user_id: Optional user ID for multi-tenancy
+            resource_type: Optional resource type (case-insensitive). Defaults to "Resource".
+                Supports: resource, domain-resource, domain_resource, DomainResource, etc.
 
         Returns:
             dict with file metadata and chunk count
@@ -526,7 +559,7 @@ class ContentService:
             size_bytes=result["metadata"].get("size"),
             mime_type=result["metadata"].get("content_type"),
             processing_status="completed",
-            tenant_id=user_id or "default",  # Required field
+            tenant_id=user_id,  # None = public/shared
             user_id=user_id,
         )
 
@@ -534,28 +567,66 @@ class ContentService:
             await self.file_repo.upsert(file)
             logger.info(f"Saved File: {filename}")
 
-        # Create Resource entities for each chunk
-        resources = [
-            Resource(
+        # Resolve resource model class from type parameter (case-insensitive)
+        from typing import cast, Type
+        from pydantic import BaseModel
+        from rem.utils.model_helpers import model_from_arbitrary_casing, get_table_name
+
+        resource_model: Type[BaseModel] = Resource  # Default
+        if resource_type:
+            try:
+                resource_model = model_from_arbitrary_casing(resource_type)
+                logger.info(f"Using resource model: {resource_model.__name__}")
+            except ValueError as e:
+                logger.warning(f"Invalid resource_type '{resource_type}', using default Resource: {e}")
+                resource_model = Resource
+
+        # Get table name for the resolved model
+        table_name = get_table_name(resource_model)
+
+        # Create resource entities for each chunk
+        resources: list[BaseModel] = [
+            resource_model(
                 name=f"{filename}#chunk-{i}",
                 uri=f"{uri}#chunk-{i}",
                 ordinal=i,
                 content=chunk,
                 category="document",
-                tenant_id=user_id or "default",  # Required field
+                tenant_id=user_id,  # None = public/shared
                 user_id=user_id,
             )
             for i, chunk in enumerate(chunks)
         ]
 
-        if self.resource_repo:
-            await self.resource_repo.upsert(
-                resources,
-                embeddable_fields=["content"],
-                generate_embeddings=True,
-            )
-            logger.info(f"Saved {len(resources)} Resource chunks")
-            logger.info(f"Queued {len(resources)} embedding generation tasks for content field")
+        # Save resources to the appropriate table
+        if resources:
+            from rem.services.postgres import get_postgres_service
+
+            postgres = get_postgres_service()
+            if postgres:
+                await postgres.connect()
+                try:
+                    await postgres.batch_upsert(
+                        records=cast(list[BaseModel | dict], resources),
+                        model=resource_model,
+                        table_name=table_name,
+                        entity_key_field="name",
+                        embeddable_fields=["content"],
+                        generate_embeddings=True,
+                    )
+                    logger.info(f"Saved {len(resources)} {resource_model.__name__} chunks to {table_name}")
+                    logger.info(f"Queued {len(resources)} embedding generation tasks for content field")
+                finally:
+                    await postgres.disconnect()
+            elif self.resource_repo:
+                # Fallback to injected repo (only works for default Resource)
+                await self.resource_repo.upsert(
+                    resources,
+                    embeddable_fields=["content"],
+                    generate_embeddings=True,
+                )
+                logger.info(f"Saved {len(resources)} Resource chunks")
+                logger.info(f"Queued {len(resources)} embedding generation tasks for content field")
 
         return {
             "file": file.model_dump(),
@@ -595,9 +666,10 @@ class ContentService:
         # IMPORTANT: category field distinguishes agents from evaluators
         # - kind=agent → category="agent" (AI agents with tools/resources)
         # - kind=evaluator → category="evaluator" (LLM-as-a-Judge evaluators)
+        # Schemas (agents/evaluators) default to system tenant for shared access
         schema_entity = Schema(
-            tenant_id=user_id or "default",
-            user_id=user_id,
+            tenant_id="system",
+            user_id=None,
             name=name,
             spec=schema_data,
             category=kind,  # Maps kind → category for database filtering
@@ -667,7 +739,7 @@ class ContentService:
             processor = EngramProcessor(postgres)
             result = await processor.process_engram(
                 data=data,
-                tenant_id=user_id or "default",
+                tenant_id=user_id,  # None = public/shared
                 user_id=user_id,
             )
             logger.info(f"✅ Engram processed: {result.get('resource_id')} with {len(result.get('moment_ids', []))} moments")

rem/services/embeddings/api.py

@@ -45,7 +45,7 @@ def generate_embedding(
             return [0.0] * DEFAULT_EMBEDDING_DIMS
 
         try:
-            logger.info(f"Generating OpenAI embedding for text using {model}")
+            logger.debug(f"Generating OpenAI embedding for text using {model}")
 
             response = requests.post(
                 "https://api.openai.com/v1/embeddings",
@@ -60,7 +60,7 @@ def generate_embedding(
 
             data = response.json()
             embedding = data["data"][0]["embedding"]
-            logger.info(f"Successfully generated embedding (dimension: {len(embedding)})")
+            logger.debug(f"Successfully generated embedding (dimension: {len(embedding)})")
             return cast(list[float], embedding)
 
         except Exception as e:
@@ -97,7 +97,7 @@ async def generate_embedding_async(
             return [0.0] * DEFAULT_EMBEDDING_DIMS
 
         try:
-            logger.info(f"Generating OpenAI embedding for text using {model}")
+            logger.debug(f"Generating OpenAI embedding for text using {model}")
 
             async with httpx.AsyncClient() as client:
                 response = await client.post(
@@ -113,7 +113,7 @@ async def generate_embedding_async(
 
                 data = response.json()
                 embedding = data["data"][0]["embedding"]
-                logger.info(
+                logger.debug(
                     f"Successfully generated embedding (dimension: {len(embedding)})"
                 )
                 return cast(list[float], embedding)