remdb 0.2.6__py3-none-any.whl

rem/auth/providers/microsoft.py

@@ -0,0 +1,237 @@
+"""
+Microsoft Entra ID (Azure AD) OAuth Provider.
+
+Implements OAuth 2.1 / OIDC for Microsoft authentication.
+
+Configuration:
+1. Register application at https://portal.azure.com/#view/Microsoft_AAD_RegisteredApps
+2. Create client secret under "Certificates & secrets"
+3. Add redirect URI: http://localhost:8000/api/auth/callback (dev)
+4. Set API permissions:
+   - Microsoft Graph: User.Read (delegated)
+   - Optional: email, profile, openid (automatically included)
+5. Set environment variables:
+   - AUTH__MICROSOFT__CLIENT_ID (Application ID)
+   - AUTH__MICROSOFT__CLIENT_SECRET
+   - AUTH__MICROSOFT__TENANT_ID (or "common" for multi-tenant)
+   - AUTH__MICROSOFT__REDIRECT_URI
+
+Microsoft-specific features:
+- Multi-tenant support (common, organizations, consumers)
+- Azure AD B2C support
+- Conditional access policies
+- Token caching with MSAL
+
+Tenant options:
+- common: Multi-tenant + personal Microsoft accounts
+- organizations: Multi-tenant (work/school only)
+- consumers: Personal Microsoft accounts only
+- {tenant-id}: Single tenant (specific organization)
+
+References:
+- Microsoft identity platform: https://learn.microsoft.com/en-us/entra/identity-platform/
+- OAuth 2.0 flow: https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-auth-code-flow
+- OIDC: https://learn.microsoft.com/en-us/entra/identity-platform/v2-protocols-oidc
+- Scopes: https://learn.microsoft.com/en-us/graph/permissions-reference
+"""
+
+from typing import Any
+
+from .base import OAuthProvider, OAuthUserInfo
+
+
+class MicrosoftOAuthProvider(OAuthProvider):
+    """
+    Microsoft Entra ID (Azure AD) OAuth 2.1 / OIDC provider.
+
+    Supports multi-tenant authentication and Microsoft Graph API access.
+    Uses Microsoft identity platform v2.0 endpoints.
+    """
+
+    # Microsoft identity platform v2.0 endpoints
+    # Replace {tenant} with:
+    # - "common" for multi-tenant + personal accounts
+    # - "organizations" for work/school accounts only
+    # - "consumers" for personal Microsoft accounts only
+    # - Tenant ID/domain for single-tenant
+    AUTHORIZATION_ENDPOINT_TEMPLATE = (
+        "https://login.microsoftonline.com/{tenant}/oauth2/v2.0/authorize"
+    )
+    TOKEN_ENDPOINT_TEMPLATE = "https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token"
+    USERINFO_ENDPOINT = "https://graph.microsoft.com/v1.0/me"
+    JWKS_URI_TEMPLATE = "https://login.microsoftonline.com/{tenant}/discovery/v2.0/keys"
+
+    # Microsoft Graph scopes
+    # openid: Required for OIDC
+    # email: User email address
+    # profile: User profile information
+    # User.Read: Read user profile via Microsoft Graph
+    # offline_access: Request refresh token
+    DEFAULT_SCOPES = [
+        "openid",
+        "email",
+        "profile",
+        "User.Read",  # Microsoft Graph: read user profile
+    ]
+
+    def __init__(
+        self,
+        client_id: str,
+        client_secret: str,
+        redirect_uri: str,
+        tenant: str = "common",
+    ):
+        """
+        Initialize Microsoft OAuth provider.
+
+        Args:
+            client_id: Application (client) ID from Azure portal
+            client_secret: Client secret from Azure portal
+            redirect_uri: Redirect URI registered in Azure portal
+            tenant: Tenant ID or "common"/"organizations"/"consumers"
+        """
+        super().__init__(client_id, client_secret, redirect_uri)
+        self.tenant = tenant
+
+    @property
+    def authorization_endpoint(self) -> str:
+        """Microsoft authorization endpoint."""
+        return self.AUTHORIZATION_ENDPOINT_TEMPLATE.format(tenant=self.tenant)
+
+    @property
+    def token_endpoint(self) -> str:
+        """Microsoft token endpoint."""
+        return self.TOKEN_ENDPOINT_TEMPLATE.format(tenant=self.tenant)
+
+    @property
+    def userinfo_endpoint(self) -> str:
+        """Microsoft Graph /me endpoint for user info."""
+        return self.USERINFO_ENDPOINT
+
+    @property
+    def jwks_uri(self) -> str:
+        """Microsoft JWKS URI for token validation."""
+        return self.JWKS_URI_TEMPLATE.format(tenant=self.tenant)
+
+    @property
+    def default_scopes(self) -> list[str]:
+        """Default scopes for Microsoft OAuth."""
+        return self.DEFAULT_SCOPES.copy()
+
+    def normalize_user_info(self, claims: dict[str, Any]) -> OAuthUserInfo:
+        """
+        Normalize Microsoft claims to OAuthUserInfo.
+
+        Microsoft Graph /me response:
+        - id: Unique user ID (stable identifier)
+        - userPrincipalName: User principal name (UPN)
+        - mail: Primary email (may be null)
+        - displayName: Display name
+        - givenName: First name
+        - surname: Last name
+        - preferredLanguage: User locale
+
+        Microsoft ID token claims:
+        - sub: Subject (unique user ID, different from Graph ID)
+        - email: User email
+        - name: Full name
+        - given_name: First name
+        - family_name: Last name
+        - preferred_username: UPN or email
+
+        Args:
+            claims: Raw claims from ID token or Microsoft Graph /me
+
+        Returns:
+            Normalized user information
+        """
+        # Handle both ID token claims and Graph API response
+        # Graph API uses different field names than OIDC claims
+        if "id" in claims:
+            # Microsoft Graph /me response
+            sub = claims["id"]
+            email = claims.get("mail") or claims.get("userPrincipalName")
+            name = claims.get("displayName")
+            given_name = claims.get("givenName")
+            family_name = claims.get("surname")
+            locale = claims.get("preferredLanguage")
+        else:
+            # OIDC ID token claims
+            sub = claims["sub"]
+            email = claims.get("email") or claims.get("preferred_username")
+            name = claims.get("name")
+            given_name = claims.get("given_name")
+            family_name = claims.get("family_name")
+            locale = claims.get("locale")
+
+        return OAuthUserInfo(
+            sub=sub,
+            email=email,
+            email_verified=True,  # Microsoft verifies emails during account creation
+            name=name,
+            given_name=given_name,
+            family_name=family_name,
+            picture=None,  # Microsoft Graph requires separate photo endpoint
+            locale=locale,
+            provider="microsoft",
+            raw_claims=claims,
+        )
+
+    def generate_auth_url_with_prompt(
+        self,
+        state: str,
+        code_challenge: str,
+        prompt: str | None = None,
+        domain_hint: str | None = None,
+        login_hint: str | None = None,
+        scopes: list[str] | None = None,
+        nonce: str | None = None,
+    ) -> str:
+        """
+        Generate authorization URL with Microsoft-specific parameters.
+
+        Args:
+            state: CSRF protection state
+            code_challenge: PKCE code challenge
+            prompt: Authentication behavior (none, login, consent, select_account)
+            domain_hint: Domain hint for faster login (e.g., "contoso.com")
+            login_hint: Login hint (email) to pre-fill sign-in form
+            scopes: OAuth scopes (uses default_scopes if None)
+            nonce: OIDC nonce for ID token replay protection
+
+        Returns:
+            Authorization URL
+
+        Microsoft-specific parameters:
+        - prompt: Authentication behavior
+          - none: Silent authentication (fails if interaction required)
+          - login: Force user to re-authenticate
+          - consent: Force consent screen
+          - select_account: Show account picker
+        - domain_hint: Domain hint for faster login (skip domain discovery)
+        - login_hint: Email to pre-fill sign-in form
+        - response_mode: query (default), form_post, fragment
+        """
+        extra_params: dict[str, str] = {}
+
+        if prompt:
+            extra_params["prompt"] = prompt
+
+        if domain_hint:
+            extra_params["domain_hint"] = domain_hint
+
+        if login_hint:
+            extra_params["login_hint"] = login_hint
+
+        # Add offline_access scope for refresh token
+        scopes_with_offline = scopes or self.default_scopes.copy()
+        if "offline_access" not in scopes_with_offline:
+            scopes_with_offline.append("offline_access")
+
+        return self.generate_auth_url(
+            state=state,
+            code_challenge=code_challenge,
+            scopes=scopes_with_offline,
+            nonce=nonce,
+            extra_params=extra_params,
+        )

rem/cli/README.md

@@ -0,0 +1,455 @@
+# REM CLI - Agent Testing Guide
+
+## Overview
+
+The `rem ask` command provides a CLI interface for testing Pydantic AI agents with YAML-based schemas. It supports both streaming and non-streaming modes, structured output, and optional OTEL/Phoenix instrumentation.
+
+## Installation
+
+```bash
+# Install REM with all dependencies
+cd /Users/sirsh/code/mr_saoirse/remstack/rem
+uv pip install -e .
+
+# Verify installation
+rem --help
+```
+
+## Basic Usage
+
+```bash
+# Simple question (non-streaming by default)
+rem ask simple "What is 2+2?"
+
+# Streaming mode for real-time output
+rem ask simple "What is 2+2?" --stream
+
+# With specific model
+rem ask simple "What is 2+2?" --model openai:gpt-4o-mini
+
+# Structured output
+rem ask query "Find all documents by Sarah" --model openai:gpt-4o-mini
+
+# Process file and save output
+rem ask contract-analyzer -i rem/tests/data/content-examples/service_agreement.txt -o output.yaml
+```
+
+## File Processing
+
+The `--input-file` option allows you to process files directly instead of providing a text query:
+
+```bash
+# Extract data from contract (text file)
+rem ask contract-analyzer \
+  -i rem/tests/data/content-examples/service_agreement.txt \
+  -o output.yaml
+
+# Extract from PDF contract
+rem ask contract-analyzer \
+  -i rem/tests/data/content-examples/pdf/service_contract.pdf \
+  -o output.yaml
+
+# With specific model
+rem ask contract-analyzer \
+  -i rem/tests/data/content-examples/service_agreement.txt \
+  -o output.yaml \
+  -m anthropic:claude-sonnet-4-5-20250929
+
+# Output to console (default)
+rem ask contract-analyzer -i rem/tests/data/content-examples/service_agreement.txt
+
+# Stream output in real-time
+rem ask contract-analyzer -i rem/tests/data/content-examples/service_agreement.txt --stream
+```
+
+**Schema name resolution:**
+- Short names: `contract-analyzer` → `schemas/agents/examples/contract-analyzer.yaml`
+- With folder: `examples/contract-analyzer` → `schemas/agents/examples/contract-analyzer.yaml`
+- Core agents: `moment-builder` → `schemas/agents/core/moment-builder.yaml`
+- Full paths: `schemas/agents/examples/contract-analyzer.yaml` (as-is)
+```
+
+**Supported file types:**
+- Documents: PDF, DOCX, PPTX, XLSX (via Kreuzberg)
+- Text: TXT, MD, Markdown, code files
+- Schemas: YAML, JSON
+- Audio: MP3, WAV, M4A (via Whisper API)
+
+See [examples/README.md](../../../examples/README.md) for complete contract extraction examples.
+```
+
+## Command Options
+
+```
+rem ask NAME [QUERY] [OPTIONS]
+
+Arguments:
+  NAME   Agent schema name (YAML files in schemas/agents/)
+         - Short name: contract-analyzer → schemas/agents/examples/contract-analyzer.yaml
+         - With folder: examples/contract-analyzer → schemas/agents/examples/contract-analyzer.yaml
+         - Core agent: moment-builder → schemas/agents/core/moment-builder.yaml
+         - Full path: schemas/agents/examples/contract-analyzer.yaml
+
+  QUERY  User query to send to the agent (optional if --input-file is used)
+
+Options:
+  --model, -m TEXT            LLM model (default: from settings)
+  --temperature, -t FLOAT     Temperature 0.0-1.0 (not yet implemented)
+  --max-turns INTEGER         Maximum turns for execution (default: 10)
+  --version, -v TEXT          Schema version for registry lookup
+  --stream / --no-stream      Enable/disable streaming (default: disabled)
+  --input-file, -i PATH       Read input from file (PDF, TXT, Markdown, etc.)
+  --output-file, -o PATH      Write output to file (YAML format)
+  --user-id TEXT              User ID for context (default: cli-user)
+  --session-id TEXT           Session ID for context (default: auto-generated)
+```
+
+## Agent Schema Format
+
+Agent schemas are YAML files following JSON Schema with embedded metadata:
+
+```yaml
+type: object
+description: |
+  System prompt for the agent.
+
+  This describes what the agent does and how it should behave.
+
+properties:
+  answer:
+    type: string
+    description: The response to the user's query
+
+  confidence:
+    type: number
+    minimum: 0
+    maximum: 1
+    description: Confidence score for the response
+
+required:
+  - answer
+
+json_schema_extra:
+  fully_qualified_name: "rem.agents.SimpleAgent"
+  version: "1.0.0"
+  tools: []        # MCP tool configurations (future)
+  resources: []    # MCP resource configurations (future)
+```
+
+## Example Schemas
+
+### Simple Agent (`schemas/agents/examples/simple.yaml`)
+
+A basic conversational agent that returns simple text answers:
+
+```yaml
+type: object
+description: |
+  A simple conversational agent that provides helpful, friendly responses.
+
+  You are a helpful AI assistant. Answer questions clearly and concisely.
+  If you don't know something, say so. Be friendly and professional.
+
+properties:
+  answer:
+    type: string
+    description: The response to the user's query
+
+required:
+  - answer
+
+json_schema_extra:
+  fully_qualified_name: "rem.agents.SimpleAgent"
+  version: "1.0.0"
+  tools: []
+  resources: []
+```
+
+### Query Agent (`schemas/agents/examples/query.yaml`)
+
+An agent that provides structured output with confidence scores:
+
+```yaml
+type: object
+description: |
+  REM Query Agent - Converts natural language questions to REM queries.
+
+  You are a specialized agent that understands REM (Resources Entities Moments) queries.
+  Your job is to interpret user questions and provide answers with confidence scores.
+
+properties:
+  answer:
+    type: string
+    description: The answer to the user's query with supporting details
+
+  confidence:
+    type: number
+    minimum: 0
+    maximum: 1
+    description: Confidence score (0.0-1.0) for this answer
+
+  query_type:
+    type: string
+    enum:
+      - LOOKUP
+      - FUZZY
+      - TRAVERSE
+      - UNKNOWN
+    description: The type of REM query that would best answer this question
+
+required:
+  - answer
+  - confidence
+  - query_type
+
+json_schema_extra:
+  fully_qualified_name: "rem.agents.QueryAgent"
+  version: "1.0.0"
+  tools: []
+  resources: []
+```
+
+## Streaming vs Non-Streaming
+
+### Non-Streaming Mode (default)
+
+Uses `agent.run()` to return complete structured result at once:
+
+```bash
+rem ask simple "Explain quantum computing"
+```
+
+Output:
+```json
+{
+  "answer": "Quantum computing uses quantum mechanical phenomena..."
+}
+```
+
+**Best for:**
+- Saving output to files
+- Structured data extraction
+- Processing files with complex schemas
+- Programmatic usage
+
+### Streaming Mode
+
+Uses `agent.iter()` to stream events in real-time:
+- Tool call markers: `[Calling: tool_name]`
+- Text content deltas as they arrive
+- Final structured result after completion
+
+```bash
+rem ask simple "Explain quantum computing" --stream
+```
+
+Output:
+```
+[Calling: final_result]
+Quantum computing uses quantum mechanical phenomena like superposition...
+
+{
+  "answer": "Quantum computing uses quantum mechanical phenomena..."
+}
+```
+
+**Best for:**
+- Interactive conversations
+- Long-running queries where you want to see progress
+- Debugging agent behavior
+
+## Implementation Details
+
+### Architecture
+
+```
+CLI (ask.py)
+  ├── load_schema_from_file() - YAML file loading
+  ├── load_schema_from_registry() - TODO: Database/cache lookup
+  ├── run_agent_streaming() - agent.iter() with event streaming
+  └── run_agent_non_streaming() - agent.run() for complete result
+
+Agent Factory (providers/pydantic_ai.py)
+  ├── create_pydantic_ai_agent() - Main factory
+  ├── _create_model_from_schema() - JSON Schema → Pydantic model
+  └── _create_schema_wrapper() - Strip description for LLM
+
+OTEL (otel/setup.py)
+  ├── setup_instrumentation() - Initialize OTLP exporters
+  └── set_agent_resource_attributes() - Set span attributes
+```
+
+### Design Patterns
+
+1. **JsonSchema to Pydantic Pattern**
+   - Agent schemas are JSON Schema with embedded metadata
+   - `description` field becomes system prompt
+   - `properties` field becomes Pydantic output model
+   - Dynamic model creation using `json-schema-to-pydantic`
+
+2. **Streaming with agent.iter() Pattern**
+   - Use `agent.iter()` for complete execution (not `run_stream()`)
+   - `agent.iter()` captures tool calls, `run_stream()` stops after first output
+   - Stream tool call events with `[Calling: tool_name]` markers
+   - Stream text content deltas as they arrive
+
+3. **Conditional OTEL Instrumentation**
+   - OTEL disabled by default for local development
+   - Enabled in production via `OTEL__ENABLED=true`
+   - Applied at agent creation time: `Agent(..., instrument=settings.otel.enabled)`
+
+## Environment Variables
+
+Set API keys for LLM providers:
+
+```bash
+# In ~/.bash_profile or ~/.zshrc
+export OPENAI_API_KEY="sk-..."
+export ANTHROPIC_API_KEY="sk-ant-..."
+
+# Optional: OTEL/Phoenix configuration
+export OTEL__ENABLED=true
+export OTEL__SERVICE_NAME=rem-cli
+export OTEL__COLLECTOR_ENDPOINT=http://localhost:4318
+export PHOENIX__ENABLED=true
+export PHOENIX__COLLECTOR_ENDPOINT=http://localhost:6006/v1/traces
+```
+
+## Observability (Optional)
+
+### OTEL Configuration
+
+Enable distributed tracing with OpenTelemetry:
+
+```bash
+# Enable OTEL
+export OTEL__ENABLED=true
+export OTEL__SERVICE_NAME=rem-cli
+export OTEL__COLLECTOR_ENDPOINT=http://localhost:4318
+export OTEL__PROTOCOL=http
+
+# Run agent with tracing
+rem ask query "Find documents" --model openai:gpt-4o-mini
+```
+
+### Phoenix Integration
+
+Enable LLM observability with Arize Phoenix:
+
+```bash
+# Start Phoenix locally
+docker run -p 6006:6006 arizephoenix/phoenix:latest
+
+# Enable Phoenix
+export PHOENIX__ENABLED=true
+export PHOENIX__COLLECTOR_ENDPOINT=http://localhost:6006/v1/traces
+export PHOENIX__PROJECT_NAME=rem-cli
+
+# Run agent with Phoenix tracing
+rem ask query "Find documents" --model openai:gpt-4o-mini
+
+# View traces at http://localhost:6006
+```
+
+## Schema Registry (TODO)
+
+The schema registry is stubbed but not yet implemented. To implement:
+
+1. **Database Schema**:
+   ```sql
+   CREATE TABLE agent_schemas (
+     id UUID PRIMARY KEY,
+     name TEXT NOT NULL,
+     version TEXT NOT NULL,
+     schema_json JSONB NOT NULL,
+     created_at TIMESTAMPTZ DEFAULT NOW(),
+     UNIQUE(name, version)
+   );
+   ```
+
+2. **Cache Layer**:
+   - Redis for fast lookups
+   - In-memory cache for CLI
+
+3. **Versioning**:
+   - Semantic versioning (1.0.0, 1.1.0, etc.)
+   - Latest version fallback
+
+Once implemented, you can load agents by name:
+
+```bash
+# Load latest version
+rem ask query "Find documents"
+
+# Load specific version
+rem ask query "Find documents" --version 1.2.0
+```
+
+## Testing
+
+```bash
+# Test simple agent (default non-streaming)
+rem ask simple "What is 2+2?" --model openai:gpt-4o-mini
+
+# Test simple agent (streaming)
+rem ask simple "What is 2+2?" --stream --model openai:gpt-4o-mini
+
+# Test structured output
+rem ask query "Find all documents by Sarah" --model openai:gpt-4o-mini
+
+# Test file processing
+rem ask contract-analyzer -i examples/contract.pdf -o output.yaml
+
+# Test with different models
+rem ask simple "Hello" --model openai:gpt-4o
+rem ask simple "Hello" --model anthropic:claude-sonnet-4-5-20250929
+```
+
+## Troubleshooting
+
+### API Key Not Found
+
+```bash
+# Set API key in environment
+export OPENAI_API_KEY="sk-..."
+
+# Or source your profile
+source ~/.bash_profile
+```
+
+### Schema Registry Not Implemented
+
+```
+Schema registry not implemented yet. Please use a file path instead.
+```
+
+Use file paths until registry is implemented:
+```bash
+rem ask simple "query"
+```
+
+### Model Not Found
+
+Ensure you're using the correct model format:
+- OpenAI: `openai:gpt-4o-mini`, `openai:gpt-4o`
+- Anthropic: `anthropic:claude-sonnet-4-5-20250929`
+
+## Next Steps
+
+1. **Implement Schema Registry**
+   - PostgreSQL table for schema storage
+   - Redis cache for fast lookups
+   - Version management
+
+2. **Add MCP Tool Support**
+   - Dynamic tool loading from schema
+   - MCP server configuration
+
+3. **Temperature Override**
+   - Pass temperature to agent.run()
+   - Model-specific settings
+
+4. **CLI Improvements**
+   - Interactive mode
+   - Multi-turn conversations
+   - Session management

rem/cli/__init__.py

@@ -0,0 +1,8 @@
+"""
+REM CLI - Command-line interface for REM operations.
+
+Commands:
+- schema: Database schema generation and management
+- migrate: Run database migrations
+- dev: Development utilities
+"""