remdb 0.3.127__py3-none-any.whl → 0.3.172__py3-none-any.whl

rem/cli/dreaming.py

@@ -43,7 +43,7 @@ rem-dreaming full --user-id=user-123 --rem-api-url=http://localhost:8000
 Environment Variables:
 - REM_API_URL: REM API endpoint (default: http://rem-api:8000)
 - REM_EMBEDDING_PROVIDER: Embedding provider (default: text-embedding-3-small)
-- REM_DEFAULT_MODEL: LLM model (default: gpt-4o)
+- REM_DEFAULT_MODEL: LLM model (default: gpt-4.1)
 - REM_LOOKBACK_HOURS: Default lookback window (default: 24)
 - OPENAI_API_KEY: OpenAI API key
 
@@ -83,7 +83,7 @@ def get_worker() -> DreamingWorker:
         embedding_provider=os.getenv(
             "REM_EMBEDDING_PROVIDER", "text-embedding-3-small"
         ),
-        default_model=os.getenv("REM_DEFAULT_MODEL", "gpt-4o"),
+        default_model=os.getenv("REM_DEFAULT_MODEL", "gpt-4.1"),
         lookback_hours=int(os.getenv("REM_LOOKBACK_HOURS", "24")),
     )
 

rem/cli/main.py

@@ -96,6 +96,7 @@ from .commands.serve import register_command as register_serve_command
 from .commands.mcp import register_command as register_mcp_command
 from .commands.scaffold import scaffold as scaffold_command
 from .commands.cluster import register_commands as register_cluster_commands
+from .commands.session import register_command as register_session_command
 
 register_schema_commands(schema)
 register_db_commands(db)
@@ -108,6 +109,7 @@ register_serve_command(cli)
 register_mcp_command(cli)
 cli.add_command(experiments_group)
 cli.add_command(scaffold_command)
+register_session_command(cli)
 
 
 def main():

rem/models/core/experiment.py

@@ -138,18 +138,14 @@ class DatasetReference(BaseModel):
 
     path: str = Field(
         description=(
-            "Path to dataset:\n"
+            "Path to dataset. Format is inferred from file extension.\n"
+            "Supported: .csv, .tsv, .parquet, .json, .jsonl, .xlsx, .ods, .avro, .ipc\n"
             "- Git: Relative path from experiment root (e.g., 'datasets/ground_truth.csv')\n"
-            "- S3: Full S3 URI (e.g., 's3://bucket/experiments/my-exp/datasets/ground_truth.csv')\n"
+            "- S3: Full S3 URI (e.g., 's3://bucket/experiments/my-exp/datasets/data.parquet')\n"
             "- Hybrid: S3 URI for data, Git path for schema"
         )
     )
 
-    format: Literal["csv", "jsonl", "parquet", "json"] = Field(
-        default="csv",
-        description="Dataset file format"
-    )
-
     schema_path: str | None = Field(
         default=None,
         description=(
@@ -262,8 +258,7 @@ class ExperimentConfig(BaseModel):
     datasets:
       ground_truth:
         location: git
-        path: datasets/ground_truth.csv
-        format: csv
+        path: datasets/ground_truth.csv  # format inferred from extension
     results:
       location: git
       base_path: results/
@@ -288,12 +283,10 @@ class ExperimentConfig(BaseModel):
       ground_truth:
         location: s3
         path: s3://rem-prod/experiments/cv-parser-production/datasets/ground_truth.parquet
-        format: parquet
         schema_path: datasets/schema.yaml  # Schema in Git for documentation
       test_cases:
         location: s3
         path: s3://rem-prod/experiments/cv-parser-production/datasets/test_cases.jsonl
-        format: jsonl
     results:
       location: hybrid
       base_path: s3://rem-prod/experiments/cv-parser-production/results/
@@ -318,6 +311,15 @@ class ExperimentConfig(BaseModel):
         )
     )
 
+    task: str = Field(
+        default="general",
+        description=(
+            "Task name for organizing experiments by purpose.\n"
+            "Used with agent name to form directory: {agent}/{task}/\n"
+            "Examples: 'risk-assessment', 'classification', 'general'"
+        )
+    )
+
     description: str = Field(
         description="Human-readable description of experiment purpose and goals"
     )
@@ -410,6 +412,24 @@ class ExperimentConfig(BaseModel):
 
         return v
 
+    @field_validator("task")
+    @classmethod
+    def validate_task(cls, v: str) -> str:
+        """Validate task name follows conventions."""
+        if not v:
+            return "general"  # Default value
+
+        if not v.islower():
+            raise ValueError("Task name must be lowercase")
+
+        if " " in v:
+            raise ValueError("Task name cannot contain spaces (use hyphens)")
+
+        if not all(c.isalnum() or c == "-" for c in v):
+            raise ValueError("Task name can only contain lowercase letters, numbers, and hyphens")
+
+        return v
+
     @field_validator("tags")
     @classmethod
     def validate_tags(cls, v: list[str]) -> list[str]:
@@ -420,6 +440,15 @@ class ExperimentConfig(BaseModel):
         """Get the experiment directory path."""
         return Path(base_path) / self.name
 
+    def get_agent_task_dir(self, base_path: str = ".experiments") -> Path:
+        """
+        Get the experiment directory path organized by agent/task.
+
+        Returns: Path like .experiments/{agent}/{task}/
+        This is the recommended structure for S3 export compatibility.
+        """
+        return Path(base_path) / self.agent_schema_ref.name / self.task
+
     def get_config_path(self, base_path: str = ".experiments") -> Path:
         """Get the path to experiment.yaml file."""
         return self.get_experiment_dir(base_path) / "experiment.yaml"
@@ -428,6 +457,22 @@ class ExperimentConfig(BaseModel):
         """Get the path to README.md file."""
         return self.get_experiment_dir(base_path) / "README.md"
 
+    def get_evaluator_filename(self) -> str:
+        """
+        Get the evaluator filename with task prefix.
+
+        Returns: {agent_name}-{task}.yaml (e.g., siggy-risk-assessment.yaml)
+        """
+        return f"{self.agent_schema_ref.name}-{self.task}.yaml"
+
+    def get_s3_export_path(self, bucket: str, version: str = "v0") -> str:
+        """
+        Get the S3 path for exporting this experiment.
+
+        Returns: s3://{bucket}/{version}/datasets/calibration/experiments/{agent}/{task}/
+        """
+        return f"s3://{bucket}/{version}/datasets/calibration/experiments/{self.agent_schema_ref.name}/{self.task}"
+
     def to_yaml(self) -> str:
         """Export configuration as YAML string."""
         import yaml
@@ -483,6 +528,7 @@ class ExperimentConfig(BaseModel):
 ## Configuration
 
 **Status**: `{self.status.value}`
+**Task**: `{self.task}`
 **Tags**: {', '.join(f'`{tag}`' for tag in self.tags) if self.tags else 'None'}
 
 ## Agent Schema
@@ -494,6 +540,7 @@ class ExperimentConfig(BaseModel):
 ## Evaluator Schema
 
 - **Name**: `{self.evaluator_schema_ref.name}`
+- **File**: `{self.get_evaluator_filename()}`
 - **Type**: `{self.evaluator_schema_ref.type}`
 
 ## Datasets
@@ -504,7 +551,6 @@ class ExperimentConfig(BaseModel):
 
 - **Location**: `{dataset.location.value}`
 - **Path**: `{dataset.path}`
-- **Format**: `{dataset.format}`
 """
             if dataset.description:
                 readme += f"- **Description**: {dataset.description}\n"
@@ -575,7 +621,6 @@ EXAMPLE_SMALL_EXPERIMENT = ExperimentConfig(
         "ground_truth": DatasetReference(
             location=DatasetLocation.GIT,
             path="datasets/ground_truth.csv",
-            format="csv",
             description="10 manually curated test cases"
         )
     },
@@ -605,7 +650,6 @@ EXAMPLE_LARGE_EXPERIMENT = ExperimentConfig(
         "ground_truth": DatasetReference(
             location=DatasetLocation.S3,
             path="s3://rem-prod/experiments/cv-parser-production/datasets/ground_truth.parquet",
-            format="parquet",
             schema_path="datasets/schema.yaml",
             description="10,000 CV/resume pairs with ground truth extractions"
         )

rem/models/entities/__init__.py

@@ -39,6 +39,7 @@ from .shared_session import (
     SharedWithMeResponse,
     SharedWithMeSummary,
 )
+from .subscriber import Subscriber, SubscriberOrigin, SubscriberStatus
 from .user import User, UserTier
 
 __all__ = [
@@ -56,6 +57,9 @@ __all__ = [
     "FeedbackCategory",
     "User",
     "UserTier",
+    "Subscriber",
+    "SubscriberStatus",
+    "SubscriberOrigin",
     "File",
     "Moment",
     "Schema",

rem/models/entities/ontology.py

@@ -129,7 +129,7 @@ class Ontology(CoreModel):
             file_id="file-uuid-456",
             agent_schema_id="contract-parser-v2",
             provider_name="openai",
-            model_name="gpt-4o",
+            model_name="gpt-4.1",
             extracted_data={
                 "contract_type": "supplier_agreement",
                 "parties": [

rem/models/entities/ontology_config.py

@@ -74,7 +74,7 @@ class OntologyConfig(CoreModel):
             priority=200,  # Higher priority = runs first
             enabled=True,
             provider_name="openai",  # Override default provider
-            model_name="gpt-4o",
+            model_name="gpt-4.1",
             tenant_id="acme-corp",
             tags=["legal", "procurement"]
         )

rem/models/entities/subscriber.py

@@ -0,0 +1,175 @@
+"""
+Subscriber - Email subscription management.
+
+This model stores subscribers who sign up via websites/apps.
+Subscribers can be collected before user registration for newsletters,
+updates, and approval-based access control.
+
+Key features:
+- Deterministic UUID from email (same email = same ID)
+- Approval workflow for access control
+- Tags for segmentation
+- Origin tracking for analytics
+"""
+
+import uuid
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Optional
+
+from pydantic import Field, EmailStr, model_validator
+
+from ..core import CoreModel
+
+
+class SubscriberStatus(str, Enum):
+    """Subscription status."""
+
+    ACTIVE = "active"           # Actively subscribed
+    UNSUBSCRIBED = "unsubscribed"  # User unsubscribed
+    BOUNCED = "bounced"         # Email bounced
+    PENDING = "pending"         # Pending confirmation (if double opt-in)
+
+
+class SubscriberOrigin(str, Enum):
+    """Where the subscription originated from."""
+
+    WEBSITE = "website"         # Main website subscribe form
+    LANDING_PAGE = "landing_page"  # Campaign landing page
+    APP = "app"                 # In-app subscription
+    IMPORT = "import"           # Bulk import
+    REFERRAL = "referral"       # Referred by another user
+    OTHER = "other"
+
+
+class Subscriber(CoreModel):
+    """
+    Email subscriber for newsletters and access control.
+
+    This model captures subscribers who sign up via the website, landing pages,
+    or in-app prompts. Uses deterministic UUID from email for natural upserts.
+
+    Access control via `approved` field:
+    - When email auth checks subscriber status, only approved subscribers
+      can complete login (if approval is enabled in settings).
+    - Subscribers can be pre-approved, or approved manually/automatically.
+
+    Usage:
+        from rem.services.postgres import Repository
+        from rem.models.entities import Subscriber, SubscriberStatus
+
+        repo = Repository(Subscriber, db=db)
+
+        # Create subscriber (ID auto-generated from email)
+        subscriber = Subscriber(
+            email="user@example.com",
+            name="John Doe",
+            origin=SubscriberOrigin.WEBSITE,
+        )
+        await repo.upsert(subscriber)
+
+        # Check if approved for login
+        subscriber = await repo.get_by_id(subscriber.id, tenant_id="default")
+        if subscriber and subscriber.approved:
+            # Allow login
+            pass
+    """
+
+    # Required field
+    email: EmailStr = Field(
+        description="Subscriber's email address (unique identifier)"
+    )
+
+    # Optional fields
+    name: Optional[str] = Field(
+        default=None,
+        description="Subscriber's name (optional)"
+    )
+
+    comment: Optional[str] = Field(
+        default=None,
+        max_length=500,
+        description="Optional comment or message from subscriber"
+    )
+
+    status: SubscriberStatus = Field(
+        default=SubscriberStatus.ACTIVE,
+        description="Current subscription status"
+    )
+
+    # Access control
+    approved: bool = Field(
+        default=False,
+        description="Whether subscriber is approved for login (for approval workflows)"
+    )
+
+    approved_at: Optional[datetime] = Field(
+        default=None,
+        description="When the subscriber was approved"
+    )
+
+    approved_by: Optional[str] = Field(
+        default=None,
+        description="Who approved the subscriber (user ID or 'system')"
+    )
+
+    # Origin tracking
+    origin: SubscriberOrigin = Field(
+        default=SubscriberOrigin.WEBSITE,
+        description="Where the subscription originated"
+    )
+
+    origin_detail: Optional[str] = Field(
+        default=None,
+        description="Additional origin context (e.g., campaign name, page URL)"
+    )
+
+    # Timestamps
+    subscribed_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc),
+        description="When the subscription was created"
+    )
+
+    unsubscribed_at: Optional[datetime] = Field(
+        default=None,
+        description="When the user unsubscribed (if applicable)"
+    )
+
+    # Compliance
+    ip_address: Optional[str] = Field(
+        default=None,
+        description="IP address at subscription time (for compliance)"
+    )
+
+    user_agent: Optional[str] = Field(
+        default=None,
+        description="Browser user agent at subscription time"
+    )
+
+    # Segmentation
+    tags: list[str] = Field(
+        default_factory=list,
+        description="Tags for segmentation (e.g., ['early-access', 'beta'])"
+    )
+
+    @staticmethod
+    def email_to_uuid(email: str) -> uuid.UUID:
+        """Generate a deterministic UUID from an email address.
+
+        Uses UUID v5 with DNS namespace for consistency with
+        EmailService.generate_user_id_from_email().
+
+        Args:
+            email: Email address
+
+        Returns:
+            Deterministic UUID
+        """
+        return uuid.uuid5(uuid.NAMESPACE_DNS, email.lower().strip())
+
+    @model_validator(mode="after")
+    def set_id_from_email(self) -> "Subscriber":
+        """Auto-generate deterministic ID from email for natural upsert."""
+        if self.email:
+            self.id = self.email_to_uuid(self.email)
+        return self

rem/models/entities/user.py

@@ -22,6 +22,7 @@ from ..core import CoreModel
 class UserTier(str, Enum):
     """User subscription tier for feature gating."""
 
+    BLOCKED = "blocked"  # User is blocked from logging in
     ANONYMOUS = "anonymous"
     FREE = "free"
     BASIC = "basic"

rem/schemas/agents/core/agent-builder.yaml

@@ -0,0 +1,235 @@
+type: object
+description: |
+  # Agent Builder - Create Custom AI Agents Through Conversation
+
+  You help users create custom AI agents for the REM platform through natural conversation.
+  Guide them step-by-step, gather requirements, show previews, and save when ready.
+
+  ## Your Workflow
+
+  1. **Understand the need**: Ask what they want the agent to do
+  2. **Define personality**: Help them choose tone and communication style
+  3. **Set guardrails**: What should the agent NOT do?
+  4. **Structure outputs**: Define what data the agent captures (optional)
+  5. **Preview**: Show them what the agent will look like
+  6. **Save**: Use `save_agent` tool to persist it
+
+  ## Conversation Style
+
+  Be friendly and helpful. Ask one or two questions at a time.
+  Don't overwhelm with options - guide them step by step.
+
+  ## IMPORTANT: Tool Usage
+
+  - `save_agent` - Use ONLY in Step 6 when user approves the preview
+  - `get_agents_list` - Use if user asks to see existing agents as examples
+  - `get_agent_schema` - Use to load a specific agent (like "rem") as reference
+
+  DO NOT loop on tools. If a user asks for examples, call get_agents_list ONCE,
+  then discuss what you found. This is a conversational workflow.
+
+  ## Step 1: Identity & Purpose
+
+  Ask about:
+  - What should this agent help with? (primary purpose)
+  - What would you like to call it? (suggest kebab-case like "sales-assistant")
+  - What role/persona should it embody?
+
+  ## Step 2: Tone & Communication Style
+
+  Help define tone using this framework:
+
+  | Dimension | Options |
+  |-----------|---------|
+  | Formality | casual, conversational, professional, formal |
+  | Warmth | empathetic, friendly, neutral, businesslike |
+  | Pace | patient, balanced, efficient, direct |
+  | Expertise | peer, guide, expert, authority |
+
+  Ask: "What tone feels right? For example, should it be friendly and casual, or more professional?"
+
+  ## Step 3: Guardrails
+
+  Ask what the agent should NOT do:
+  - Topics to avoid?
+  - Actions it shouldn't take?
+  - Boundaries to respect?
+
+  Example guardrails:
+  - "Never provide medical/legal/financial advice"
+  - "Don't make promises about timelines"
+  - "Always recommend consulting a professional for serious issues"
+
+  ## Step 4: Structured Outputs (Optional)
+
+  Most agents just need an `answer` field. But some use cases benefit from structured data:
+
+  | Field | Type | Description |
+  |-------|------|-------------|
+  | answer | string | Natural language response (always required) |
+  | confidence | number | 0.0-1.0 confidence score |
+  | category | string | Classification of the request |
+  | follow_up_needed | boolean | Whether follow-up is required |
+
+  Field types available:
+  - `string` - text values
+  - `number` - numeric values (can add minimum/maximum)
+  - `boolean` - true/false
+  - `array` - list of items
+  - `string` with `enum` - fixed set of choices
+
+  Only suggest structured outputs if the use case clearly benefits from them.
+
+  ## Step 5: Preview
+
+  Before saving, show a preview:
+
+  ```
+  ## Agent Preview: {name}
+
+  **Purpose:** {brief description}
+
+  **Personality:** {tone and approach}
+
+  **System Prompt:**
+  {the actual prompt that will guide the agent}
+
+  **Guardrails:**
+  - {guardrail 1}
+  - {guardrail 2}
+
+  **Structured Fields:** (if any beyond answer)
+  | Field | Type | Description |
+  |-------|------|-------------|
+  | answer | string | Response to user |
+  ```
+
+  Ask: "Does this look good? I can save it now or adjust anything."
+
+  ## Step 6: Save the Agent
+
+  When the user approves, call `save_agent` with:
+  - `name`: kebab-case name (e.g., "customer-support-bot")
+  - `description`: The full system prompt (this is the most important part!)
+  - `properties`: Structured output fields (optional, defaults to just "answer")
+  - `required`: Which fields are required (optional, defaults to ["answer"])
+  - `tags`: Relevant tags for organization
+
+  ## Writing Great System Prompts
+
+  The `description` field IS the system prompt. Make it:
+
+  1. **Clear about role**: "You are a [role] that helps users [do what]"
+  2. **Specific about behavior**: How should it respond? What's the communication style?
+  3. **Explicit about guardrails**: What should it NOT do?
+  4. **Structured with sections**: Use markdown headers for organization
+
+  Example structure:
+  ```
+  # [Agent Name]
+
+  You are a [role description]. Your purpose is to [main goal].
+
+  ## Communication Style
+  [Tone, approach, language preferences]
+
+  ## Core Responsibilities
+  - [Responsibility 1]
+  - [Responsibility 2]
+
+  ## Guardrails
+  - Never [thing to avoid]
+  - Always [thing to ensure]
+
+  ## How to Respond
+  [Guidelines for responses]
+  ```
+
+  ## Example Conversation
+
+  User: "I want an agent that helps write professional emails"
+
+  You: "Great idea! Let me help you create an email writing assistant.
+
+  First, what tone should it have? For example:
+  - Professional and formal
+  - Friendly but polished
+  - Direct and concise"
+
+  User: "Friendly but polished"
+
+  You: "Perfect! Should it focus just on writing emails, or also help with:
+  - Suggesting subject lines
+  - Adjusting tone for different recipients
+  - Summarizing long email threads"
+
+  User: "All of those would be helpful"
+
+  You: "Great! Any things it should NOT do? For example, some email assistants avoid:
+  - Writing emails that could be seen as legally binding
+  - Making commitments on behalf of the user"
+
+  User: "Yeah, it shouldn't make commitments or promises"
+
+  You: "Perfect! Here's a preview:
+
+  ## Agent Preview: email-assistant
+
+  **Purpose:** Help users write polished, professional emails
+
+  **Personality:** Friendly, helpful, professional writing partner
+
+  **System Prompt:**
+  # Email Writing Assistant
+
+  You are a friendly email writing assistant. Help users craft polished,
+  professional emails while maintaining a warm, approachable tone.
+
+  ## What You Can Do
+  - Write new emails from scratch based on user's intent
+  - Suggest compelling subject lines
+  - Adjust tone for different audiences (colleagues, executives, clients)
+  - Summarize long email threads
+  - Proofread and improve existing drafts
+
+  ## Communication Style
+  Be helpful and collaborative. Suggest improvements but respect the user's voice.
+  Ask clarifying questions when the request is ambiguous.
+
+  ## Guardrails
+  - Never write emails that make commitments or promises on behalf of the user
+  - Don't write anything that could be legally binding
+  - Always let the user review before sending
+
+  Does this look good? I can save it now or adjust anything."
+
+  User: "Looks great, save it!"
+
+  You: *calls save_agent tool*
+  "Done! Your email-assistant is ready to use."
+
+properties:
+  answer:
+    type: string
+    description: Your conversational response to the user
+
+required:
+  - answer
+
+json_schema_extra:
+  kind: agent
+  name: agent-builder
+  version: "1.2.0"
+  tags:
+    - meta
+    - builder
+  structured_output: false  # Stream text responses, don't return JSON
+  mcp_servers: []  # Disable default MCP tools to prevent search_rem looping
+  resources:
+    - uri: rem://agents
+      description: "List all available agent schemas with descriptions"
+    - uri: rem://agents/{agent_name}
+      description: "Load a specific agent schema by name (e.g., 'rem', 'siggy')"
+  tools:
+    - name: save_agent
+      description: "Save the agent schema. Only call when user approves the preview in Step 6."

rem/schemas/agents/examples/contract-analyzer.yaml

@@ -308,7 +308,7 @@ json_schema_extra:
   - provider_name: anthropic
     model_name: claude-sonnet-4-5-20250929
   - provider_name: openai
-    model_name: gpt-4o
+    model_name: gpt-4.1
   embedding_fields:
   - contract_title
   - contract_type

rem/schemas/agents/examples/contract-extractor.yaml

@@ -131,4 +131,4 @@ json_schema_extra:
   - provider_name: anthropic
     model_name: claude-sonnet-4-5-20250929
   - provider_name: openai
-    model_name: gpt-4o
+    model_name: gpt-4.1

rem/schemas/agents/examples/cv-parser.yaml

@@ -255,7 +255,7 @@ json_schema_extra:
     - provider_name: anthropic
       model_name: claude-sonnet-4-5-20250929
     - provider_name: openai
-      model_name: gpt-4o
+      model_name: gpt-4.1
   embedding_fields:
     - candidate_name
     - professional_summary