remdb 0.2.6__py3-none-any.whl

rem/utils/examples/embeddings_example.py

@@ -0,0 +1,305 @@
+"""
+Example usage of embeddings utility for generating vector embeddings.
+
+This demonstrates batch processing, error handling with tenacity automatic retries,
+and integration patterns for the PostgresService.
+"""
+
+from rem.utils.embeddings import (
+    EmbeddingError,
+    RateLimitError,
+    generate_embeddings,
+    get_embedding_dimension,
+)
+
+
+def example_single_embedding():
+    """Generate embedding for a single text."""
+    print("=" * 80)
+    print("SINGLE EMBEDDING EXAMPLE")
+    print("=" * 80)
+
+    text = "What is the meaning of life?"
+    embedding_provider = "openai:text-embedding-3-small"
+
+    try:
+        # Generate embedding
+        embedding = generate_embeddings(embedding_provider, text)
+
+        # Check dimensions
+        dimension = get_embedding_dimension(embedding_provider)
+
+        print(f"\nText: {text}")
+        print(f"Provider: {embedding_provider}")
+        print(f"Embedding dimension: {dimension}")
+        print(f"Actual length: {len(embedding)}")
+        print(f"First 5 values: {embedding[:5]}")
+
+    except EmbeddingError as e:
+        print(f"Error: {e}")
+
+
+def example_batch_embeddings():
+    """Generate embeddings for multiple texts in a single API call."""
+    print("\n" + "=" * 80)
+    print("BATCH EMBEDDING EXAMPLE")
+    print("=" * 80)
+
+    texts = [
+        "What is the meaning of life?",
+        "How do I bake a chocolate cake?",
+        "Explain quantum physics in simple terms",
+        "Write a haiku about programming",
+        "What is the capital of France?",
+    ]
+
+    embedding_provider = "openai:text-embedding-3-small"
+
+    try:
+        # Generate embeddings in batch (more efficient than individual calls)
+        embeddings = generate_embeddings(embedding_provider, texts)
+
+        print(f"\nGenerated {len(embeddings)} embeddings")
+        print(f"Provider: {embedding_provider}\n")
+
+        for i, (text, embedding) in enumerate(zip(texts, embeddings)):
+            print(f"{i+1}. {text[:50]}...")
+            print(f"   Dimension: {len(embedding)}")
+            print(f"   First 3 values: {embedding[:3]}")
+
+    except RateLimitError as e:
+        print(f"Rate limit exceeded: {e}")
+        print("Tenacity automatic retry failed. Consider reducing batch size or waiting.")
+    except EmbeddingError as e:
+        print(f"Error: {e}")
+
+
+def example_multiple_providers():
+    """Compare embeddings from different providers."""
+    print("\n" + "=" * 80)
+    print("MULTIPLE PROVIDERS EXAMPLE")
+    print("=" * 80)
+
+    text = "Machine learning is transforming software development"
+
+    providers = [
+        "openai:text-embedding-3-small",
+        "openai:text-embedding-3-large",
+        "openai:text-embedding-ada-002",
+    ]
+
+    print(f"\nText: {text}\n")
+
+    for provider in providers:
+        try:
+            embedding = generate_embeddings(provider, text)
+            dimension = get_embedding_dimension(provider)
+
+            print(f"Provider: {provider}")
+            print(f"  Dimension: {dimension}")
+            print(f"  First 3 values: {embedding[:3]}\n")
+
+        except EmbeddingError as e:
+            print(f"Provider: {provider}")
+            print(f"  Error: {e}\n")
+
+
+def example_error_handling():
+    """Demonstrate error handling and retries."""
+    print("\n" + "=" * 80)
+    print("ERROR HANDLING EXAMPLE")
+    print("=" * 80)
+
+    # Invalid provider format
+    try:
+        generate_embeddings("invalid_format", "test")
+    except ValueError as e:
+        print(f"\nInvalid format error (expected): {e}")
+
+    # Empty text
+    try:
+        generate_embeddings("openai:text-embedding-3-small", [])
+    except ValueError as e:
+        print(f"\nEmpty input error (expected): {e}")
+
+    # Unknown model
+    try:
+        get_embedding_dimension("openai:unknown-model")
+    except ValueError as e:
+        print(f"\nUnknown model error (expected): {e}")
+
+
+def example_postgres_integration():
+    """
+    Example pattern for PostgresService integration.
+
+    This shows how to use embeddings utility in a PostgresService method.
+    """
+    print("\n" + "=" * 80)
+    print("POSTGRES INTEGRATION PATTERN")
+    print("=" * 80)
+
+    print(
+        """
+# In PostgresService class:
+
+async def generate_and_store_embedding(
+    self,
+    table_name: str,
+    record_id: str,
+    text_content: str,
+    embedding_provider: str = "openai:text-embedding-3-small"
+) -> None:
+    '''
+    Generate embedding for text content and store in database.
+
+    Args:
+        table_name: Table containing the record
+        record_id: ID of the record to update
+        text_content: Text to embed
+        embedding_provider: Provider and model for embeddings
+    '''
+    from rem.utils.embeddings import generate_embeddings, get_embedding_dimension
+
+    # Generate embedding
+    embedding = generate_embeddings(embedding_provider, text_content)
+
+    # Get dimension for vector column
+    dimension = get_embedding_dimension(embedding_provider)
+
+    # Ensure vector column exists
+    await self.execute(f'''
+        ALTER TABLE {table_name}
+        ADD COLUMN IF NOT EXISTS embedding vector({dimension})
+    ''')
+
+    # Store embedding
+    await self.execute(
+        f'''
+        UPDATE {table_name}
+        SET embedding = $1::vector
+        WHERE id = $2
+        ''',
+        embedding,
+        record_id
+    )
+
+
+async def batch_generate_embeddings(
+    self,
+    table_name: str,
+    text_column: str = "content",
+    embedding_provider: str = "openai:text-embedding-3-small",
+    batch_size: int = 100
+) -> None:
+    '''
+    Generate embeddings for all records in a table (batch processing).
+
+    Args:
+        table_name: Table to process
+        text_column: Column containing text to embed
+        embedding_provider: Provider and model for embeddings
+        batch_size: Number of records to process per batch
+    '''
+    from rem.utils.embeddings import generate_embeddings, get_embedding_dimension
+
+    # Get dimension
+    dimension = get_embedding_dimension(embedding_provider)
+
+    # Ensure vector column exists
+    await self.execute(f'''
+        ALTER TABLE {table_name}
+        ADD COLUMN IF NOT EXISTS embedding vector({dimension})
+    ''')
+
+    # Get all records without embeddings
+    records = await self.fetch_all(f'''
+        SELECT id, {text_column}
+        FROM {table_name}
+        WHERE embedding IS NULL
+        LIMIT {batch_size}
+    ''')
+
+    if not records:
+        return
+
+    # Extract texts and IDs
+    texts = [record[text_column] for record in records]
+    ids = [record['id'] for record in records]
+
+    # Generate embeddings in batch
+    embeddings = generate_embeddings(embedding_provider, texts)
+
+    # Store embeddings
+    for record_id, embedding in zip(ids, embeddings):
+        await self.execute(
+            f'''
+            UPDATE {table_name}
+            SET embedding = $1::vector
+            WHERE id = $2
+            ''',
+            embedding,
+            record_id
+        )
+"""
+    )
+
+
+if __name__ == "__main__":
+    # Run examples
+    # NOTE: Requires OPENAI_API_KEY or LLM__OPENAI_API_KEY environment variable
+
+    # Check if API key is available
+    import os
+
+    if not (os.getenv("OPENAI_API_KEY") or os.getenv("LLM__OPENAI_API_KEY")):
+        print("=" * 80)
+        print("SETUP REQUIRED")
+        print("=" * 80)
+        print("\nTo run these examples, set your OpenAI API key:")
+        print("  export OPENAI_API_KEY='sk-...'")
+        print("  # OR")
+        print("  export LLM__OPENAI_API_KEY='sk-...'")
+        print("\nThen run:")
+        print("  python embeddings_example.py")
+        exit(1)
+
+    # Run examples (comment out if you don't want to make API calls)
+    example_single_embedding()
+    example_batch_embeddings()
+    example_multiple_providers()
+    example_error_handling()
+    example_postgres_integration()
+
+    print("\n" + "=" * 80)
+    print("BEST PRACTICES")
+    print("=" * 80)
+    print(
+        """
+1. Batch Processing:
+   - Process multiple texts in a single API call (up to 2048 for OpenAI)
+   - Reduces API overhead and stays within rate limits (RPM)
+   - Example: generate_embeddings(provider, [text1, text2, ...])
+
+2. Rate Limit Handling:
+   - Uses tenacity library for automatic exponential backoff (default: 1 retry)
+   - Adjust max_retries parameter if needed (default: 1)
+   - Monitor your usage and adjust batch_size accordingly
+   - Consider implementing a queue for large-scale processing
+
+3. Error Handling:
+   - Catch EmbeddingError for general API errors
+   - Catch RateLimitError for rate limit specific handling
+   - Validate embedding_provider format before batch processing
+
+4. Cost Optimization:
+   - OpenAI text-embedding-3-small: $0.02 / 1M tokens
+   - OpenAI text-embedding-3-large: $0.13 / 1M tokens
+   - Use smaller models unless you need higher accuracy
+
+5. PostgreSQL Integration:
+   - Use vector({dimension}) column type with pgvector extension
+   - Create indexes: CREATE INDEX ON table USING ivfflat (embedding vector_cosine_ops)
+   - For similarity search: ORDER BY embedding <=> query_vector LIMIT 10
+"""
+    )

rem/utils/examples/sql_types_example.py

@@ -0,0 +1,202 @@
+"""
+Example usage of sql_types utility for generating PostgreSQL schema from Pydantic models.
+
+This demonstrates how REM entity models are mapped to PostgreSQL types.
+"""
+
+from datetime import datetime
+from uuid import UUID
+
+from pydantic import BaseModel, Field
+
+from rem.utils.sql_types import (
+    get_column_definition,
+    get_sql_type,
+    model_to_create_table,
+    model_to_upsert,
+)
+
+
+# Example 1: CoreModel with various field types
+class CoreModel(BaseModel):
+    """Base model demonstrating all common field types."""
+
+    # ID - Union type, should prefer UUID
+    id: UUID | str = Field(..., description="Unique identifier")
+
+    # Timestamps
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+    updated_at: datetime = Field(default_factory=datetime.utcnow)
+
+    # Optional tenant/user fields
+    tenant_id: str | None = Field(default=None, description="Tenant identifier")
+    user_id: str | None = Field(default=None, description="User identifier")
+
+    # JSONB fields
+    graph_edges: list[dict] = Field(default_factory=list, description="Graph edges")
+    metadata: dict = Field(default_factory=dict, description="Flexible metadata")
+
+    # Array fields
+    tags: list[str] = Field(default_factory=list, description="Tags")
+
+    # Database schema metadata
+    column: dict = Field(default_factory=dict, description="Column metadata")
+
+
+# Example 2: Resource with content fields
+class Resource(BaseModel):
+    """Resource entity with long-form text fields."""
+
+    id: str
+    name: str  # VARCHAR(256)
+    uri: str | None = None  # VARCHAR(256), nullable
+    content: str = ""  # TEXT (long-form field name)
+    description: str | None = None  # TEXT (long-form field name)
+    category: str | None = None  # VARCHAR(256)
+    related_entities: list[dict] = Field(default_factory=list)  # JSONB
+
+
+# Example 3: Schema with embedding provider
+class Schema(BaseModel):
+    """Schema with embedding field."""
+
+    id: str
+    name: str
+    content: str = Field(
+        default="",
+        json_schema_extra={
+            "embedding_provider": "openai:text-embedding-3-small"  # Forces TEXT
+        },
+    )
+    spec: dict = Field(..., description="JSON schema specification")  # JSONB
+    category: str | None = None
+
+
+# Example 4: Custom SQL type override
+class CustomModel(BaseModel):
+    """Model with custom SQL type specification."""
+
+    id: str
+    vector_data: list[float] = Field(
+        default_factory=list,
+        json_schema_extra={"sql_type": "vector(1536)"},  # Custom pgvector type
+    )
+    json_data: dict = Field(default_factory=dict)
+
+
+def demonstrate_field_mapping():
+    """Show how individual fields map to SQL types."""
+    print("=" * 80)
+    print("FIELD TYPE MAPPING EXAMPLES")
+    print("=" * 80)
+
+    examples = [
+        (CoreModel.model_fields["id"], "id", "Union[UUID, str] -> UUID (prefers UUID in unions)"),
+        (CoreModel.model_fields["created_at"], "created_at", "datetime -> TIMESTAMP"),
+        (CoreModel.model_fields["tenant_id"], "tenant_id", "str | None -> VARCHAR(256)"),
+        (CoreModel.model_fields["graph_edges"], "graph_edges", "list[dict] -> JSONB"),
+        (CoreModel.model_fields["metadata"], "metadata", "dict -> JSONB"),
+        (CoreModel.model_fields["tags"], "tags", "list[str] -> TEXT[]"),
+        (Resource.model_fields["content"], "content", "str (field name 'content') -> TEXT"),
+        (Resource.model_fields["name"], "name", "str -> VARCHAR(256)"),
+        (
+            Schema.model_fields["content"],
+            "content",
+            "str with embedding_provider (openai:text-embedding-3-small) -> TEXT",
+        ),
+        (
+            CustomModel.model_fields["vector_data"],
+            "vector_data",
+            "list[float] with sql_type -> vector(1536)",
+        ),
+    ]
+
+    for field_info, field_name, description in examples:
+        sql_type = get_sql_type(field_info, field_name)
+        print(f"\n{description}")
+        print(f"  SQL Type: {sql_type}")
+
+
+def demonstrate_column_definitions():
+    """Show complete column definitions."""
+    print("\n" + "=" * 80)
+    print("COLUMN DEFINITION EXAMPLES")
+    print("=" * 80)
+
+    examples = [
+        (CoreModel.model_fields["id"], "id", False, True, "Primary key"),
+        (CoreModel.model_fields["created_at"], "created_at", False, False, "Required timestamp"),
+        (CoreModel.model_fields["tenant_id"], "tenant_id", True, False, "Optional tenant"),
+        (CoreModel.model_fields["metadata"], "metadata", False, False, "JSONB with default"),
+        (CoreModel.model_fields["tags"], "tags", False, False, "Array with default"),
+    ]
+
+    for field_info, field_name, nullable, is_pk, description in examples:
+        col_def = get_column_definition(field_info, field_name, nullable, is_pk)
+        print(f"\n{description}:")
+        print(f"  {col_def}")
+
+
+def demonstrate_create_table():
+    """Generate CREATE TABLE statements."""
+    print("\n" + "=" * 80)
+    print("CREATE TABLE EXAMPLES")
+    print("=" * 80)
+
+    # Generate for Resource model
+    print("\n-- Resource Table")
+    print(model_to_create_table(Resource, "resources"))
+
+    # Generate for Schema model
+    print("\n\n-- Schema Table")
+    print(model_to_create_table(Schema, "schemas"))
+
+
+def demonstrate_upsert():
+    """Generate UPSERT statements."""
+    print("\n" + "=" * 80)
+    print("UPSERT EXAMPLES")
+    print("=" * 80)
+
+    print("\n-- Resource Upsert")
+    print(model_to_upsert(Resource, "resources"))
+
+    print("\n-- Schema Upsert")
+    print(model_to_upsert(Schema, "schemas"))
+
+
+if __name__ == "__main__":
+    demonstrate_field_mapping()
+    demonstrate_column_definitions()
+    demonstrate_create_table()
+    demonstrate_upsert()
+
+    print("\n" + "=" * 80)
+    print("USAGE IN CODE")
+    print("=" * 80)
+    print(
+        """
+# Generate schema for all REM entities
+from rem.models.entities import Resource, Message, User, File, Moment, Schema
+from rem.utils.sql_types import model_to_create_table
+
+for model, table_name in [
+    (Resource, "resources"),
+    (Message, "messages"),
+    (User, "users"),
+    (File, "files"),
+    (Moment, "moments"),
+    (Schema, "schemas"),
+]:
+    sql = model_to_create_table(model, table_name)
+    print(sql)
+    print()
+
+# Generate upsert for inserting/updating entities
+from rem.utils.sql_types import model_to_upsert
+
+upsert_sql = model_to_upsert(Resource, "resources")
+# Use with psycopg:
+# cursor.execute(upsert_sql, (id, name, uri, content, ...))
+"""
+    )

rem/utils/markdown.py

@@ -0,0 +1,16 @@
+"""Markdown conversion utilities for document processing."""
+
+
+def to_markdown(content: str, filename: str) -> str:
+    """
+    Convert extracted content to structured markdown.
+
+    Args:
+        content: Extracted text content
+        filename: Source filename
+
+    Returns:
+        Structured markdown string with header
+    """
+    lines = [f"# {filename}\n", content]
+    return "\n".join(lines)