memorisdk 1.0.0__py3-none-any.whl

memoriai/utils/pydantic_models.py

@@ -0,0 +1,258 @@
+"""
+Pydantic Models for Structured Memory Processing
+"""
+
+from datetime import datetime
+from enum import Enum
+from typing import Annotated, Dict, List, Literal, Optional, Tuple
+
+from pydantic import BaseModel, Field
+
+
+class MemoryCategoryType(str, Enum):
+    """Primary memory categories"""
+
+    fact = "fact"
+    preference = "preference"
+    skill = "skill"
+    context = "context"
+    rule = "rule"
+
+
+class RetentionType(str, Enum):
+    """Memory retention types"""
+
+    short_term = "short_term"
+    long_term = "long_term"
+    permanent = "permanent"
+
+
+class EntityType(str, Enum):
+    """Types of entities that can be extracted"""
+
+    person = "person"
+    technology = "technology"
+    topic = "topic"
+    skill = "skill"
+    project = "project"
+    keyword = "keyword"
+
+
+# Define constrained types using Annotated
+ConfidenceScore = Annotated[float, Field(ge=0.0, le=1.0)]
+ImportanceScore = Annotated[float, Field(ge=0.0, le=1.0)]
+RelevanceScore = Annotated[float, Field(ge=0.0, le=1.0)]
+PriorityLevel = Annotated[int, Field(ge=1, le=10)]
+
+
+class MemoryCategory(BaseModel):
+    """Memory categorization with confidence and reasoning"""
+
+    primary_category: MemoryCategoryType
+    confidence_score: ConfidenceScore = Field(
+        description="Confidence in categorization (0.0-1.0)"
+    )
+    reasoning: str = Field(
+        description="Brief explanation for why this category was chosen"
+    )
+
+
+class ExtractedEntity(BaseModel):
+    """Individual extracted entity with metadata"""
+
+    entity_type: EntityType
+    value: str = Field(description="The actual entity value")
+    relevance_score: RelevanceScore = Field(
+        description="How relevant this entity is to the memory"
+    )
+    context: Optional[str] = Field(
+        default=None, description="Additional context about this entity"
+    )
+
+
+class ExtractedEntities(BaseModel):
+    """All entities extracted from a conversation"""
+
+    people: List[str] = Field(
+        default_factory=list, description="Names of people mentioned"
+    )
+    technologies: List[str] = Field(
+        default_factory=list, description="Technologies, tools, libraries mentioned"
+    )
+    topics: List[str] = Field(
+        default_factory=list, description="Main topics or subjects discussed"
+    )
+    skills: List[str] = Field(
+        default_factory=list, description="Skills, abilities, or competencies mentioned"
+    )
+    projects: List[str] = Field(
+        default_factory=list,
+        description="Projects, repositories, or initiatives mentioned",
+    )
+    keywords: List[str] = Field(
+        default_factory=list, description="Important keywords for search"
+    )
+
+    # Structured entities with metadata
+    structured_entities: List[ExtractedEntity] = Field(
+        default_factory=list, description="Detailed entity extraction"
+    )
+
+
+class MemoryImportance(BaseModel):
+    """Importance scoring and retention decisions"""
+
+    importance_score: ImportanceScore = Field(
+        description="Overall importance score (0.0-1.0)"
+    )
+    retention_type: RetentionType = Field(description="Recommended retention type")
+    reasoning: str = Field(
+        description="Explanation for the importance level and retention decision"
+    )
+
+    # Additional scoring factors
+    novelty_score: RelevanceScore = Field(
+        default=0.5, description="How novel/new this information is"
+    )
+    relevance_score: RelevanceScore = Field(
+        default=0.5, description="How relevant to user's interests"
+    )
+    actionability_score: RelevanceScore = Field(
+        default=0.5, description="How actionable this information is"
+    )
+
+
+class ProcessedMemory(BaseModel):
+    """Complete processed memory with all extracted information"""
+
+    # Core categorization
+    category: MemoryCategory
+
+    # Entity extraction
+    entities: ExtractedEntities
+
+    # Importance and retention
+    importance: MemoryImportance
+
+    # Content processing
+    summary: str = Field(description="Concise, searchable summary of the memory")
+    searchable_content: str = Field(
+        description="Content optimized for keyword and semantic search"
+    )
+    key_insights: List[str] = Field(
+        default_factory=list, description="Key insights or takeaways"
+    )
+
+    # Storage decision
+    should_store: bool = Field(description="Whether this memory should be stored")
+    storage_reasoning: str = Field(
+        description="Why this memory should or shouldn't be stored"
+    )
+
+    # Metadata (optional fields)
+    timestamp: Optional[datetime] = Field(
+        default_factory=datetime.now, description="When this memory was processed"
+    )
+    processing_metadata: Optional[Dict[str, str]] = Field(
+        default=None, description="Additional processing metadata"
+    )
+
+
+class MemorySearchQuery(BaseModel):
+    """Structured query for memory search"""
+
+    # Query components
+    query_text: str = Field(description="Original query text")
+    intent: str = Field(description="Interpreted intent of the query")
+
+    # Search parameters
+    entity_filters: List[str] = Field(
+        default_factory=list, description="Specific entities to search for"
+    )
+    category_filters: List[MemoryCategoryType] = Field(
+        default_factory=list, description="Memory categories to include"
+    )
+    time_range: Optional[str] = Field(
+        default=None, description="Time range for search (e.g., 'last_week')"
+    )
+    min_importance: ImportanceScore = Field(
+        default=0.0, description="Minimum importance score"
+    )
+
+    # Search strategy
+    search_strategy: List[str] = Field(
+        default_factory=list, description="Recommended search strategies"
+    )
+    expected_result_types: List[str] = Field(
+        default_factory=list, description="Expected types of results"
+    )
+
+
+class MemoryRelationship(BaseModel):
+    """Relationship between memories"""
+
+    source_memory_id: str
+    target_memory_id: str
+    relationship_type: Literal[
+        "builds_on", "contradicts", "supports", "related_to", "prerequisite"
+    ]
+    strength: RelevanceScore = Field(description="Strength of the relationship")
+    reasoning: str = Field(description="Why these memories are related")
+
+
+class UserRule(BaseModel):
+    """User preferences and rules"""
+
+    rule_text: str = Field(description="The rule or preference in natural language")
+    rule_type: Literal["preference", "instruction", "constraint", "goal"]
+    priority: PriorityLevel = Field(default=5, description="Priority level (1-10)")
+    context: Optional[str] = Field(default=None, description="When this rule applies")
+    active: bool = Field(
+        default=True, description="Whether this rule is currently active"
+    )
+
+
+class ConversationContext(BaseModel):
+    """Context information for memory processing"""
+
+    model_config = {"protected_namespaces": ()}
+
+    user_id: Optional[str] = Field(default=None)
+    session_id: str
+    conversation_id: str
+    model_used: str
+
+    # User context
+    user_preferences: List[str] = Field(default_factory=list)
+    current_projects: List[str] = Field(default_factory=list)
+    relevant_skills: List[str] = Field(default_factory=list)
+
+    # Conversation metadata
+    conversation_length: int = Field(
+        default=1, description="Number of exchanges in this conversation"
+    )
+    topic_thread: Optional[str] = Field(
+        default=None, description="Main topic thread being discussed"
+    )
+
+    # Memory context
+    recent_memories: List[str] = Field(
+        default_factory=list, description="IDs of recently accessed memories"
+    )
+    applied_rules: List[str] = Field(
+        default_factory=list, description="Rules that were applied"
+    )
+
+
+class MemoryStats(BaseModel):
+    """Statistics about stored memories"""
+
+    total_memories: int
+    memories_by_category: Dict[str, int]
+    memories_by_retention: Dict[str, int]
+    average_importance: float
+    total_entities: int
+    most_common_entities: List[Tuple[str, int]]
+    storage_size_mb: float
+    oldest_memory_date: Optional[datetime]
+    newest_memory_date: Optional[datetime]

memoriai/utils/validators.py

@@ -0,0 +1,339 @@
+"""
+Data validation utilities for Memoriai
+"""
+
+import re
+from pathlib import Path
+from typing import Any, Dict, Union
+
+from .exceptions import ValidationError
+
+
+class DataValidator:
+    """Centralized data validation utilities"""
+
+    # Regex patterns
+    UUID_PATTERN = re.compile(
+        r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE
+    )
+    EMAIL_PATTERN = re.compile(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$")
+    API_KEY_PATTERN = re.compile(r"^sk-[a-zA-Z0-9]{48}$")
+
+    @classmethod
+    def validate_uuid(cls, value: str, field_name: str = "UUID") -> str:
+        """Validate UUID format"""
+        if not isinstance(value, str):
+            raise ValidationError(f"{field_name} must be a string")
+
+        if not cls.UUID_PATTERN.match(value):
+            raise ValidationError(f"{field_name} must be a valid UUID format")
+
+        return value
+
+    @classmethod
+    def validate_email(cls, value: str, field_name: str = "email") -> str:
+        """Validate email format"""
+        if not isinstance(value, str):
+            raise ValidationError(f"{field_name} must be a string")
+
+        if not cls.EMAIL_PATTERN.match(value):
+            raise ValidationError(f"{field_name} must be a valid email address")
+
+        return value
+
+    @classmethod
+    def validate_openai_api_key(cls, value: str, field_name: str = "API key") -> str:
+        """Validate OpenAI API key format"""
+        if not isinstance(value, str):
+            raise ValidationError(f"{field_name} must be a string")
+
+        if not value.startswith("sk-"):
+            raise ValidationError(f"{field_name} must start with 'sk-'")
+
+        if len(value) != 51:  # sk- + 48 characters
+            raise ValidationError(f"{field_name} must be 51 characters long")
+
+        return value
+
+    @classmethod
+    def validate_namespace(cls, value: str, field_name: str = "namespace") -> str:
+        """Validate namespace format"""
+        if not isinstance(value, str):
+            raise ValidationError(f"{field_name} must be a string")
+
+        if not value:
+            raise ValidationError(f"{field_name} cannot be empty")
+
+        if len(value) > 64:
+            raise ValidationError(f"{field_name} cannot exceed 64 characters")
+
+        # Allow alphanumeric, underscore, hyphen
+        if not re.match(r"^[a-zA-Z0-9_-]+$", value):
+            raise ValidationError(
+                f"{field_name} can only contain letters, numbers, underscores, and hyphens"
+            )
+
+        return value
+
+    @classmethod
+    def validate_importance_score(
+        cls, value: float, field_name: str = "importance score"
+    ) -> float:
+        """Validate importance score (0.0 to 1.0)"""
+        if not isinstance(value, (int, float)):
+            raise ValidationError(f"{field_name} must be a number")
+
+        if not 0.0 <= value <= 1.0:
+            raise ValidationError(f"{field_name} must be between 0.0 and 1.0")
+
+        return float(value)
+
+    @classmethod
+    def validate_database_url(cls, value: str, field_name: str = "database URL") -> str:
+        """Validate database connection string"""
+        if not isinstance(value, str):
+            raise ValidationError(f"{field_name} must be a string")
+
+        valid_schemes = ["sqlite://", "sqlite:///", "postgresql://", "mysql://"]
+        if not any(value.startswith(scheme) for scheme in valid_schemes):
+            raise ValidationError(f"{field_name} must use a supported database scheme")
+
+        return value
+
+    @classmethod
+    def validate_file_path(
+        cls,
+        value: Union[str, Path],
+        field_name: str = "file path",
+        must_exist: bool = False,
+    ) -> Path:
+        """Validate file path"""
+        if isinstance(value, str):
+            path = Path(value)
+        elif isinstance(value, Path):
+            path = value
+        else:
+            raise ValidationError(f"{field_name} must be a string or Path object")
+
+        if must_exist and not path.exists():
+            raise ValidationError(f"{field_name} does not exist: {path}")
+
+        return path
+
+    @classmethod
+    def validate_json_dict(
+        cls, value: Any, field_name: str = "JSON data"
+    ) -> Dict[str, Any]:
+        """Validate that value is a JSON-serializable dictionary"""
+        if not isinstance(value, dict):
+            raise ValidationError(f"{field_name} must be a dictionary")
+
+        try:
+            import json
+
+            json.dumps(value)  # Test JSON serialization
+        except (TypeError, ValueError) as e:
+            raise ValidationError(f"{field_name} must be JSON serializable: {e}") from e
+
+        return value
+
+    @classmethod
+    def validate_memory_category(
+        cls, value: str, field_name: str = "memory category"
+    ) -> str:
+        """Validate memory category"""
+        if not isinstance(value, str):
+            raise ValidationError(f"{field_name} must be a string")
+
+        valid_categories = ["fact", "preference", "skill", "context", "rule"]
+        if value not in valid_categories:
+            raise ValidationError(
+                f"{field_name} must be one of: {', '.join(valid_categories)}"
+            )
+
+        return value
+
+    @classmethod
+    def validate_retention_type(
+        cls, value: str, field_name: str = "retention type"
+    ) -> str:
+        """Validate retention type"""
+        if not isinstance(value, str):
+            raise ValidationError(f"{field_name} must be a string")
+
+        valid_types = ["short_term", "long_term", "permanent"]
+        if value not in valid_types:
+            raise ValidationError(
+                f"{field_name} must be one of: {', '.join(valid_types)}"
+            )
+
+        return value
+
+    @classmethod
+    def validate_entity_type(cls, value: str, field_name: str = "entity type") -> str:
+        """Validate entity type"""
+        if not isinstance(value, str):
+            raise ValidationError(f"{field_name} must be a string")
+
+        valid_types = [
+            "person",
+            "technology",
+            "topic",
+            "skill",
+            "project",
+            "keyword",
+            "location",
+            "organization",
+        ]
+        if value not in valid_types:
+            raise ValidationError(
+                f"{field_name} must be one of: {', '.join(valid_types)}"
+            )
+
+        return value
+
+    @classmethod
+    def validate_positive_integer(
+        cls, value: int, field_name: str = "value", min_value: int = 1
+    ) -> int:
+        """Validate positive integer"""
+        if not isinstance(value, int):
+            raise ValidationError(f"{field_name} must be an integer")
+
+        if value < min_value:
+            raise ValidationError(f"{field_name} must be at least {min_value}")
+
+        return value
+
+    @classmethod
+    def validate_text_length(
+        cls,
+        value: str,
+        field_name: str = "text",
+        max_length: int = 1000,
+        min_length: int = 1,
+    ) -> str:
+        """Validate text length"""
+        if not isinstance(value, str):
+            raise ValidationError(f"{field_name} must be a string")
+
+        if len(value) < min_length:
+            raise ValidationError(
+                f"{field_name} must be at least {min_length} characters"
+            )
+
+        if len(value) > max_length:
+            raise ValidationError(f"{field_name} cannot exceed {max_length} characters")
+
+        return value
+
+    @classmethod
+    def sanitize_input(cls, value: str, field_name: str = "input") -> str:
+        """Sanitize user input for security"""
+        if not isinstance(value, str):
+            raise ValidationError(f"{field_name} must be a string")
+
+        # Remove potential SQL injection patterns
+        dangerous_patterns = [
+            r"(\b(DROP|DELETE|UPDATE|INSERT|CREATE|ALTER|EXEC|EXECUTE)\b)",
+            r"(--|#|/\*|\*/)",
+            r"(\b(UNION|SELECT)\b.*\b(FROM|WHERE)\b)",
+        ]
+
+        for pattern in dangerous_patterns:
+            if re.search(pattern, value, re.IGNORECASE):
+                raise ValidationError(
+                    f"{field_name} contains potentially dangerous content"
+                )
+
+        # Basic HTML/script tag removal
+        value = re.sub(r"<[^>]*>", "", value)
+
+        return value.strip()
+
+
+class MemoryValidator:
+    """Specialized validator for memory-related data"""
+
+    @classmethod
+    def validate_memory_data(cls, data: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate complete memory data structure"""
+        validated = {}
+
+        # Required fields
+        if "memory_id" in data:
+            validated["memory_id"] = DataValidator.validate_uuid(
+                data["memory_id"], "memory_id"
+            )
+
+        if "namespace" in data:
+            validated["namespace"] = DataValidator.validate_namespace(data["namespace"])
+
+        if "importance_score" in data:
+            validated["importance_score"] = DataValidator.validate_importance_score(
+                data["importance_score"]
+            )
+
+        if "category_primary" in data:
+            validated["category_primary"] = DataValidator.validate_memory_category(
+                data["category_primary"]
+            )
+
+        if "retention_type" in data:
+            validated["retention_type"] = DataValidator.validate_retention_type(
+                data["retention_type"]
+            )
+
+        # Text fields
+        for field in ["summary", "searchable_content"]:
+            if field in data:
+                validated[field] = DataValidator.validate_text_length(
+                    data[field],
+                    field,
+                    max_length=5000 if field == "searchable_content" else 1000,
+                )
+
+        # JSON fields
+        for field in ["processed_data", "metadata"]:
+            if field in data:
+                validated[field] = DataValidator.validate_json_dict(data[field], field)
+
+        return validated
+
+    @classmethod
+    def validate_chat_data(cls, data: Dict[str, Any]) -> Dict[str, Any]:
+        """Validate chat data structure"""
+        validated = {}
+
+        # Required fields
+        required_fields = ["chat_id", "user_input", "ai_output", "model"]
+        for field in required_fields:
+            if field not in data:
+                raise ValidationError(f"Required field missing: {field}")
+
+        validated["chat_id"] = DataValidator.validate_uuid(data["chat_id"], "chat_id")
+        validated["user_input"] = DataValidator.sanitize_input(
+            data["user_input"], "user_input"
+        )
+        validated["ai_output"] = DataValidator.sanitize_input(
+            data["ai_output"], "ai_output"
+        )
+        validated["model"] = DataValidator.validate_text_length(
+            data["model"], "model", max_length=100
+        )
+
+        # Optional fields
+        if "namespace" in data:
+            validated["namespace"] = DataValidator.validate_namespace(data["namespace"])
+
+        if "tokens_used" in data:
+            validated["tokens_used"] = DataValidator.validate_positive_integer(
+                data["tokens_used"], "tokens_used", min_value=0
+            )
+
+        if "metadata" in data:
+            validated["metadata"] = DataValidator.validate_json_dict(
+                data["metadata"], "metadata"
+            )
+
+        return validated