remdb 0.3.242__py3-none-any.whl

rem/models/core/inline_edge.py

@@ -0,0 +1,132 @@
+"""
+InlineEdge - Knowledge graph edge representation.
+
+REM uses human-readable entity labels instead of UUIDs for graph edges,
+enabling natural language queries without schema knowledge.
+
+Key Design Decision:
+- dst field contains LABELS (e.g., "sarah-chen", "tidb-migration-spec")
+- NOT UUIDs (e.g., "550e8400-e29b-41d4-a716-446655440000")
+- This enables LOOKUP operations on labels directly
+- LLMs can query "LOOKUP sarah-chen" without knowing internal IDs
+
+Edge Weight Guidelines:
+- 1.0: Primary/strong relationships (authored_by, owns, part_of)
+- 0.8-0.9: Important relationships (depends_on, reviewed_by, implements)
+- 0.5-0.7: Secondary relationships (references, related_to, inspired_by)
+- 0.3-0.4: Weak relationships (mentions, cites)
+
+Destination Entity Type Convention (CRITICAL - properties.dst_entity_type):
+
+Format: <table_schema>:<category>/<key>
+
+Where:
+- table_schema: Database table (resources, moments, users, etc.)
+- category: Optional entity category within that table
+- key: The actual entity key (must match dst field)
+
+Examples:
+- "resources:managers/bob" → Look up bob in resources table with category="managers"
+- "users:engineers/sarah-chen" → Look up sarah-chen in users table with category="engineers"
+- "moments:meetings/standup-2024-01" → Look up in moments table with category="meetings"
+- "resources/api-design-v2" → Look up api-design-v2 in resources table (no category)
+- "bob" → Defaults to resources table, no category (use sparingly)
+
+IMPORTANT - Upsert Rules:
+1. When upserting referenced entities, parse dst_entity_type to determine:
+   - table_schema → which table to upsert into
+   - category → set the 'category' field in that table
+   - key → match against entity_key_field (usually 'name' or 'id')
+
+2. If dst_entity_type is missing or just a type like "managers":
+   - Default table_schema to "resources"
+   - Set category to the type (e.g., "managers")
+   - Use dst as the key
+
+3. Agents should NEVER guess entity types
+   - If type is unknown, omit dst_entity_type or set to null
+   - Better to have no category than wrong category
+   - System will handle entities without categories
+
+4. Category is optional and can be null - this is perfectly fine
+   - Categories enable filtering but are not required for graph traversal
+   - Use categories when they add semantic value (roles, types, domains)
+
+Edge Type Format Guidelines (rel_type):
+- Use snake_case: "authored_by", "depends_on", "references"
+- Be specific but consistent: "reviewed_by" not "reviewed"
+- Use passive voice for bidirectional clarity: "authored_by" (reverse: "authors")
+"""
+
+from datetime import datetime, timezone
+from typing import Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class InlineEdge(BaseModel):
+    """
+    Knowledge graph edge with human-readable destination labels.
+
+    Stores relationships between entities using natural language labels
+    instead of UUIDs, enabling conversational queries.
+    """
+
+    dst: str = Field(
+        ...,
+        description="Human-readable destination key matching the entity's name/id field (e.g., 'tidb-migration-spec', 'sarah-chen', 'bob')",
+    )
+    rel_type: str = Field(
+        ...,
+        description="Relationship type in snake_case (e.g., 'authored_by', 'depends_on', 'references')",
+    )
+    weight: float = Field(
+        default=0.5,
+        ge=0.0,
+        le=1.0,
+        description="Relationship strength: 1.0=primary, 0.8-0.9=important, 0.5-0.7=secondary, 0.3-0.4=weak",
+    )
+    properties: dict = Field(
+        default_factory=dict,
+        description=(
+            "Rich metadata. CRITICAL field: dst_entity_type with format 'table_schema:category/key' "
+            "(e.g., 'resources:managers/bob', 'users:engineers/sarah-chen'). "
+            "Used to determine upsert target table and category. Can be null/omitted if unknown."
+        ),
+    )
+    created_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None), description="Edge creation timestamp"
+    )
+
+
+class InlineEdges(BaseModel):
+    """
+    Collection of InlineEdge objects.
+
+    Used for structured edge operations and batch processing.
+    """
+
+    edges: list[InlineEdge] = Field(
+        default_factory=list, description="List of graph edges"
+    )
+
+    def add_edge(
+        self,
+        dst: str,
+        rel_type: str,
+        weight: float = 0.5,
+        properties: Optional[dict] = None,
+    ) -> None:
+        """Add a new edge to the collection."""
+        edge = InlineEdge(
+            dst=dst, rel_type=rel_type, weight=weight, properties=properties or {}
+        )
+        self.edges.append(edge)
+
+    def filter_by_rel_type(self, rel_types: list[str]) -> list[InlineEdge]:
+        """Filter edges by relationship types."""
+        return [edge for edge in self.edges if edge.rel_type in rel_types]
+
+    def filter_by_weight(self, min_weight: float = 0.0) -> list[InlineEdge]:
+        """Filter edges by minimum weight threshold."""
+        return [edge for edge in self.edges if edge.weight >= min_weight]

rem/models/core/rem_query.py

@@ -0,0 +1,246 @@
+"""
+REM Query Models
+
+REM provides schema-agnostic query operations optimized for LLM-augmented
+iterated retrieval. Unlike traditional SQL, REM queries work with natural
+language labels instead of UUIDs and support multi-turn exploration.
+
+Query Types (Performance Contract):
+- LOOKUP: O(1) schema-agnostic entity resolution
+- FUZZY: Indexed fuzzy text matching across all entities
+- SEARCH: Indexed semantic vector search
+- SQL: Direct table queries (provider dialect)
+- TRAVERSE: Iterative O(1) lookups on graph edges
+
+Key Design Principles:
+1. Natural language surface area (labels, not UUIDs)
+2. Schema-agnostic operations (no table name required for LOOKUP/FUZZY/TRAVERSE)
+3. Multi-turn iteration with stage tracking and memos
+4. O(1) performance guarantees for entity resolution
+
+Iterated Retrieval Pattern:
+- Stage 1: Find entry point (LOOKUP/SEARCH)
+- Stage 2: Analyze neighborhood (TRAVERSE DEPTH 0 = PLAN mode)
+- Stage 3: Selective traversal (TRAVERSE with edge filters)
+- Stage 4: Refinement based on results
+
+Example Multi-Turn Query:
+```python
+# Turn 1: PLAN mode to analyze edges
+TRAVERSE WITH LOOKUP "sarah chen" DEPTH 0
+
+# Turn 2: Follow specific edge types
+TRAVERSE manages,mentors WITH LOOKUP "sarah chen" DEPTH 2
+
+# Turn 3: Refine based on results
+TRAVERSE authored_by WITH LOOKUP "api-design-v2" DEPTH 1
+```
+
+REM Query Contract (MANDATORY for all providers):
+| Query Type | Performance | Schema | Multi-Match | Required |
+|------------|-------------|--------|-------------|----------|
+| LOOKUP | O(1) | Agnostic | Yes | ✅ |
+| FUZZY | Indexed | Agnostic | Yes | ✅ |
+| SEARCH | Indexed | Specific | Yes | ✅ |
+| SQL | O(n) | Specific | No | ✅ |
+| TRAVERSE | O(k) | Agnostic | Yes | ✅ |
+"""
+
+from enum import Enum
+from typing import Any, Optional, Union
+
+from pydantic import BaseModel, Field
+
+
+class QueryType(str, Enum):
+    """
+    REM query types.
+
+    Each type has specific performance and schema requirements
+    defined in the REM contract.
+    """
+
+    LOOKUP = "LOOKUP"
+    FUZZY = "FUZZY"
+    SEARCH = "SEARCH"
+    SQL = "SQL"
+    TRAVERSE = "TRAVERSE"
+
+
+class LookupParameters(BaseModel):
+    """
+    LOOKUP query parameters.
+
+    Performance: O(1) per key
+    Schema: Agnostic - No table name required
+    Multi-match: Returns entities from ALL tables with matching keys
+    """
+
+    key: Union[str, list[str]] = Field(
+        ..., description="Entity identifier(s) - single key or list of keys (natural language labels)"
+    )
+    user_id: Optional[str] = Field(
+        default=None, description="Optional user ID filter for multi-user tenants"
+    )
+
+
+class FuzzyParameters(BaseModel):
+    """
+    FUZZY query parameters.
+
+    Performance: Indexed - FTS or trigram index required
+    Schema: Agnostic - Searches across all entity names
+    Multi-match: Returns entities from ALL tables matching fuzzy pattern
+    """
+
+    query_text: str = Field(..., description="Fuzzy search text")
+    threshold: float = Field(
+        default=0.5, ge=0.0, le=1.0, description="Similarity threshold"
+    )
+    limit: int = Field(default=5, gt=0, description="Maximum results")
+
+
+class SearchParameters(BaseModel):
+    """
+    SEARCH query parameters.
+
+    Performance: Indexed - Vector index required (IVF, HNSW)
+    Schema: Table-specific - Requires table name
+    """
+
+    query_text: str = Field(..., description="Semantic search query")
+    table_name: str = Field(..., description="Table to search (resources, moments, etc.)")
+    limit: int = Field(default=10, gt=0, description="Maximum results")
+    min_similarity: float = Field(
+        default=0.3, ge=0.0, le=1.0, description="Minimum similarity score (0.3 recommended for general queries)"
+    )
+
+
+class SQLParameters(BaseModel):
+    """
+    SQL query parameters.
+
+    Performance: O(n) - Table scan with optional indexes
+    Schema: Table-specific - Requires table name and column knowledge
+    Provider-specific: Uses native SQL dialect
+
+    Supports two modes:
+    1. Structured: table_name + where_clause + order_by + limit
+    2. Raw: raw_query (full SQL statement like SELECT...)
+    """
+
+    raw_query: Optional[str] = Field(
+        default=None, description="Raw SQL query (e.g., SELECT * FROM resources WHERE...)"
+    )
+    table_name: Optional[str] = Field(default=None, description="Table to query (structured mode)")
+    where_clause: Optional[str] = Field(
+        default=None, description="SQL WHERE clause (structured mode)"
+    )
+    order_by: Optional[str] = Field(default=None, description="SQL ORDER BY clause (structured mode)")
+    limit: Optional[int] = Field(default=None, description="SQL LIMIT (structured mode)")
+
+
+class TraverseParameters(BaseModel):
+    """
+    TRAVERSE query parameters.
+
+    Performance: O(k) where k = number of keys traversed
+    Schema: Agnostic - Follows graph edges across tables
+    Implementation: Iterative LOOKUP calls on edge destinations
+
+    Syntax: TRAVERSE {edge_filter} WITH [REM_QUERY] DEPTH [0-N]
+
+    Depth Modes:
+    - 0: PLAN mode (analyze edges without traversal)
+    - 1: Single-hop traversal (default)
+    - N: Multi-hop traversal (N hops from source)
+
+    Plan Memo:
+    Agent-maintained scratchpad for tracking multi-turn progress.
+    Kept terse for fast token generation.
+    Example: "Goal: org chart. Step 1: find CEO"
+    """
+
+    initial_query: str = Field(
+        ..., description="Initial query to find entry nodes (LOOKUP key, SEARCH text, etc.)"
+    )
+    edge_types: list[str] = Field(
+        default_factory=lambda: ["*"],
+        description="Edge types to follow (e.g., ['manages', 'reports-to']). Default: ['*'] (all)",
+    )
+    max_depth: int = Field(
+        default=1, ge=0, description="Maximum traversal depth. 0 = PLAN mode (no traversal)"
+    )
+    order_by: str = Field(
+        default="edge.created_at DESC",
+        description="Result ordering (edge.created_at, node.name, edge.weight)",
+    )
+    limit: int = Field(default=9, gt=0, description="Maximum nodes to return")
+    plan_memo: Optional[str] = Field(
+        default=None,
+        description="Agent's terse scratchpad for tracking multi-turn progress",
+    )
+
+
+class RemQuery(BaseModel):
+    """
+    REM query plan.
+
+    Combines query type with type-specific parameters.
+    Used by both direct REM queries and ask_rem() natural language interface.
+    """
+
+    query_type: QueryType = Field(..., description="REM query type")
+    parameters: (
+        LookupParameters
+        | FuzzyParameters
+        | SearchParameters
+        | SQLParameters
+        | TraverseParameters
+    ) = Field(..., description="Query parameters")
+    user_id: Optional[str] = Field(
+        default=None,
+        description="User identifier (UUID5 hash of email). None = anonymous (shared/public data only)"
+    )
+
+
+class TraverseStage(BaseModel):
+    """
+    TRAVERSE execution stage information.
+
+    Captures query execution details for LLM interaction and multi-turn planning.
+    """
+
+    depth: int = Field(..., description="Traversal depth for this stage")
+    executed: str = Field(..., description="Query executed at this stage")
+    found: dict[str, int] = Field(
+        ..., description="Discovery stats (nodes, edges counts)"
+    )
+    plan_memo: Optional[str] = Field(
+        default=None, description="Agent's memo echoed from request"
+    )
+
+
+class TraverseResponse(BaseModel):
+    """
+    TRAVERSE query response.
+
+    Returns nodes, execution stages, and metadata for LLM-driven iteration.
+    """
+
+    nodes: list[dict[str, Any]] = Field(
+        default_factory=list, description="Discovered nodes"
+    )
+    stages: list[TraverseStage] = Field(
+        default_factory=list, description="Execution stage information"
+    )
+    source_nodes: list[str] = Field(
+        default_factory=list, description="Initial entry node labels"
+    )
+    edge_summary: list[tuple[str, str, str]] = Field(
+        default_factory=list,
+        description="Edge shorthand tuples (src, rel_type, dst) for analysis",
+    )
+    metadata: dict[str, Any] = Field(
+        default_factory=dict, description="Query metadata (total_nodes, max_depth_reached, etc.)"
+    )

rem/models/entities/__init__.py

@@ -0,0 +1,68 @@
+"""
+REM Entity Models
+
+Core entity types for the REM system:
+- Resources: Base content units (documents, conversations, artifacts)
+- ImageResources: Image-specific resources with CLIP embeddings
+- Messages: Communication content
+- Sessions: Conversation sessions (normal or evaluation mode)
+- SharedSessions: Session sharing between users for collaboration
+- Feedback: User feedback on messages/sessions with trace integration
+- Users: User entities
+- Files: File metadata and tracking
+- Moments: Temporal narratives (meetings, coding sessions, conversations)
+- Schemas: Agent schema definitions (JsonSchema specifications for Pydantic AI)
+- Ontologies: Domain-specific extracted knowledge from files
+- OntologyConfigs: User-defined rules for automatic ontology extraction
+
+All entities inherit from CoreModel and support:
+- Graph connectivity via InlineEdge
+- Temporal tracking
+- Flexible metadata
+- Natural language labels for conversational queries
+"""
+
+from .domain_resource import DomainResource
+from .feedback import Feedback, FeedbackCategory
+from .file import File
+from .image_resource import ImageResource
+from .message import Message
+from .moment import Moment
+from .ontology import Ontology
+from .ontology_config import OntologyConfig
+from .resource import Resource
+from .schema import Schema
+from .session import Session, SessionMode
+from .shared_session import (
+    SharedSession,
+    SharedSessionCreate,
+    SharedWithMeResponse,
+    SharedWithMeSummary,
+)
+from .subscriber import Subscriber, SubscriberOrigin, SubscriberStatus
+from .user import User, UserTier
+
+__all__ = [
+    "Resource",
+    "DomainResource",
+    "ImageResource",
+    "Message",
+    "Session",
+    "SessionMode",
+    "SharedSession",
+    "SharedSessionCreate",
+    "SharedWithMeResponse",
+    "SharedWithMeSummary",
+    "Feedback",
+    "FeedbackCategory",
+    "User",
+    "UserTier",
+    "Subscriber",
+    "SubscriberStatus",
+    "SubscriberOrigin",
+    "File",
+    "Moment",
+    "Schema",
+    "Ontology",
+    "OntologyConfig",
+]

rem/models/entities/domain_resource.py

@@ -0,0 +1,38 @@
+"""
+DomainResource - Curated internal knowledge in REM.
+
+DomainResources are a specialized subclass of Resource for storing curated,
+domain-specific internal knowledge that is not part of general knowledge.
+This includes proprietary information, internal documentation, institutional
+knowledge, and other content that requires more careful curation.
+
+Key Differences from Resource:
+- Intended for curated, internal knowledge (not raw ingested content)
+- Higher quality bar - content is reviewed/vetted before ingestion
+- May contain proprietary or sensitive information
+- Subject to different retention/governance policies
+
+Use Cases:
+- Internal documentation and procedures
+- Proprietary research and analysis
+- Institutional knowledge bases
+- Domain-specific ontologies and taxonomies
+- Curated best practices and guidelines
+"""
+
+from .resource import Resource
+
+
+class DomainResource(Resource):
+    """
+    Curated domain-specific knowledge resource.
+
+    Inherits all fields from Resource but stored in a separate table
+    (domain_resources) to distinguish curated internal knowledge from
+    general ingested content.
+
+    The schema is identical to Resource, allowing seamless migration
+    of content between tables as curation status changes.
+    """
+
+    pass

rem/models/entities/feedback.py

@@ -0,0 +1,123 @@
+"""
+Feedback - User feedback on chat messages and sessions.
+
+Feedback allows users to rate and categorize responses, providing
+data for evaluation and model improvement. Feedback can be attached
+to specific messages or entire sessions.
+
+Trace Integration:
+- Feedback references trace_id/span_id for OTEL/Phoenix integration
+- Can attach annotations to Phoenix spans for unified observability
+
+Predefined Categories (system-defined, extensible):
+- INCOMPLETE: Response lacks expected information
+- INACCURATE: Response contains factual errors
+- POOR_TONE: Inappropriate or unprofessional tone
+- OFF_TOPIC: Response doesn't address the question
+- TOO_VERBOSE: Unnecessarily long response
+- TOO_BRIEF: Insufficiently detailed response
+- HELPFUL: Positive feedback marker
+- EXCELLENT: Exceptionally good response
+"""
+
+from enum import Enum
+from typing import Any
+
+from pydantic import Field
+
+from ..core import CoreModel
+
+
+class FeedbackCategory(str, Enum):
+    """Predefined feedback categories (system-defined)."""
+
+    # Negative categories
+    INCOMPLETE = "incomplete"
+    INACCURATE = "inaccurate"
+    POOR_TONE = "poor_tone"
+    OFF_TOPIC = "off_topic"
+    TOO_VERBOSE = "too_verbose"
+    TOO_BRIEF = "too_brief"
+    CONFUSING = "confusing"
+    UNSAFE = "unsafe"
+
+    # Positive categories
+    HELPFUL = "helpful"
+    EXCELLENT = "excellent"
+    ACCURATE = "accurate"
+    WELL_WRITTEN = "well_written"
+
+    # Neutral/Other
+    OTHER = "other"
+
+
+class Feedback(CoreModel):
+    """
+    User feedback on a message or session.
+
+    Captures structured feedback including:
+    - Rating (1-5 scale or thumbs up/down)
+    - Categories (predefined or custom)
+    - Free-text comment
+    - Trace reference for OTEL/Phoenix integration
+
+    The feedback can be attached to:
+    - A specific message (message_id set)
+    - An entire session (session_id set, message_id null)
+    """
+
+    # Target reference (at least one required)
+    session_id: str = Field(
+        ...,
+        description="Session ID this feedback relates to",
+    )
+    message_id: str | None = Field(
+        default=None,
+        description="Specific message ID (null for session-level feedback)",
+    )
+
+    # Rating (flexible: 1-5, or -1/1 for thumbs)
+    rating: int | None = Field(
+        default=None,
+        ge=-1,
+        le=5,
+        description="Rating: -1 (thumbs down), 1 (thumbs up), or 1-5 scale",
+    )
+
+    # Categories (can select multiple)
+    categories: list[str] = Field(
+        default_factory=list,
+        description="Selected feedback categories (from FeedbackCategory or custom)",
+    )
+
+    # Free-text comment
+    comment: str | None = Field(
+        default=None,
+        description="Optional free-text feedback comment",
+    )
+
+    # Trace reference for OTEL/Phoenix integration
+    trace_id: str | None = Field(
+        default=None,
+        description="OTEL trace ID for linking to observability",
+    )
+    span_id: str | None = Field(
+        default=None,
+        description="OTEL span ID for specific span feedback",
+    )
+
+    # Phoenix annotation status
+    phoenix_synced: bool = Field(
+        default=False,
+        description="Whether feedback has been synced to Phoenix as annotation",
+    )
+    phoenix_annotation_id: str | None = Field(
+        default=None,
+        description="Phoenix annotation ID after sync",
+    )
+
+    # Annotator info
+    annotator_kind: str = Field(
+        default="HUMAN",
+        description="Annotator type: HUMAN, LLM, CODE",
+    )

rem/models/entities/file.py

@@ -0,0 +1,57 @@
+"""
+File - File metadata and tracking in REM.
+
+Files represent uploaded or referenced files (PDFs, images, audio, etc.)
+that are parsed into Resources or used as input to dreaming workflows.
+
+File entities track:
+- File metadata (name, size, mime type)
+- Storage location (URI)
+- Processing status
+- Relationships to derived Resources
+"""
+
+from typing import Optional
+
+from pydantic import Field
+
+from ..core import CoreModel
+
+
+class File(CoreModel):
+    """
+    File metadata and tracking.
+
+    Represents files uploaded to or referenced by the REM system,
+    tracking their metadata and processing status. Tenant isolation
+    is provided via CoreModel.tenant_id field.
+    """
+
+    name: str = Field(
+        ...,
+        description="File name",
+    )
+    uri: str = Field(
+        ...,
+        description="File storage URI (S3, local path, etc.)",
+    )
+    content: Optional[str] = Field(
+        default=None,
+        description="Extracted text content (if applicable)",
+    )
+    timestamp: Optional[str] = Field(
+        default=None,
+        description="File creation/modification timestamp",
+    )
+    size_bytes: Optional[int] = Field(
+        default=None,
+        description="File size in bytes",
+    )
+    mime_type: Optional[str] = Field(
+        default=None,
+        description="File MIME type",
+    )
+    processing_status: Optional[str] = Field(
+        default="pending",
+        description="File processing status (pending, processing, completed, failed)",
+    )