django-cfg 1.1.81__py3-none-any.whl → 1.2.0__py3-none-any.whl

django_cfg/apps/knowbase/services/embedding/batch_result.py

@@ -0,0 +1,61 @@
+"""
+Batch processing result builder.
+
+This module provides a clean way to build BatchProcessingResult
+from individual batch operations without using raw dicts.
+"""
+
+from typing import List
+from .models import BatchProcessingResult, EmbeddingResult
+
+
+class BatchResultBuilder:
+    """Builder for BatchProcessingResult to avoid raw dict usage."""
+    
+    def __init__(self, total_chunks: int):
+        self.total_chunks = total_chunks
+        self.successful_chunks = 0
+        self.failed_chunks = 0
+        self.total_tokens = 0
+        self.total_cost = 0.0
+        self.errors: List[str] = []
+        self.start_time: float = 0.0
+    
+    def add_successful_result(self, result: EmbeddingResult) -> None:
+        """Add a successful embedding result."""
+        if result.success:
+            self.successful_chunks += 1
+            self.total_tokens += result.tokens
+            self.total_cost += result.cost
+        else:
+            self.add_failed_result(result.error or "Unknown error")
+    
+    def add_failed_result(self, error: str) -> None:
+        """Add a failed result."""
+        self.failed_chunks += 1
+        self.errors.append(error)
+    
+    def add_batch_results(self, results: List[EmbeddingResult]) -> None:
+        """Add multiple results from a batch."""
+        for result in results:
+            if result.success:
+                self.add_successful_result(result)
+            else:
+                self.add_failed_result(result.error or "Unknown error")
+    
+    def add_batch_error(self, error: str, chunk_count: int) -> None:
+        """Add an error that affected an entire batch."""
+        self.failed_chunks += chunk_count
+        self.errors.append(error)
+    
+    def build(self, processing_time: float) -> BatchProcessingResult:
+        """Build the final BatchProcessingResult."""
+        return BatchProcessingResult(
+            total_chunks=self.total_chunks,
+            successful_chunks=self.successful_chunks,
+            failed_chunks=self.failed_chunks,
+            total_tokens=self.total_tokens,
+            total_cost=self.total_cost,
+            processing_time=processing_time,
+            errors=self.errors
+        )

django_cfg/apps/knowbase/services/embedding/models.py

@@ -0,0 +1,229 @@
+"""
+Data models for embedding processing.
+
+This module defines the core data structures used throughout
+the embedding processing pipeline using Pydantic for type safety.
+"""
+
+from typing import List, Dict, Any, Optional
+from pydantic import BaseModel, Field, validator
+from enum import Enum
+
+
+class ChunkType(str, Enum):
+    """Supported chunk types."""
+    DOCUMENT = "document"
+    ARCHIVE = "archive"
+    EXTERNAL_DATA = "external_data"
+    UNKNOWN = "unknown"
+
+
+class ChunkData(BaseModel):
+    """Unified chunk data structure for processing."""
+    id: str = Field(..., description="Unique chunk identifier")
+    content: str = Field(..., min_length=1, description="Chunk content text")
+    context_metadata: Optional[Dict[str, Any]] = Field(
+        default=None, 
+        description="Additional context metadata for the chunk"
+    )
+    parent_id: Optional[str] = Field(
+        default=None, 
+        description="ID of the parent document or archive"
+    )
+    parent_type: ChunkType = Field(
+        default=ChunkType.UNKNOWN, 
+        description="Type of parent content"
+    )
+    
+    @validator('content')
+    def content_must_not_be_empty(cls, v):
+        if not v or not v.strip():
+            raise ValueError('Content cannot be empty')
+        return v.strip()
+    
+    class Config:
+        use_enum_values = True
+
+
+class EmbeddingResult(BaseModel):
+    """Result of embedding generation."""
+    chunk_id: str = Field(..., description="ID of the processed chunk")
+    embedding: List[float] = Field(
+        default_factory=list, 
+        description="Generated embedding vector"
+    )
+    tokens: int = Field(
+        default=0, 
+        ge=0, 
+        description="Number of tokens used"
+    )
+    cost: float = Field(
+        default=0.0, 
+        ge=0.0, 
+        description="Processing cost in USD"
+    )
+    success: bool = Field(
+        default=True, 
+        description="Whether embedding generation was successful"
+    )
+    error: Optional[str] = Field(
+        default=None, 
+        description="Error message if processing failed"
+    )
+    processing_time: Optional[float] = Field(
+        default=None, 
+        ge=0.0, 
+        description="Time taken to process this chunk in seconds"
+    )
+    
+    @validator('embedding')
+    def validate_embedding_dimension(cls, v):
+        if v is not None and len(v) > 0 and len(v) not in [1536, 3072]:  # Common OpenAI embedding dimensions
+            # Warning, not error - allow different dimensions
+            pass
+        return v
+    
+    class Config:
+        validate_assignment = True
+
+
+class BatchProcessingResult(BaseModel):
+    """Result of batch processing."""
+    total_chunks: int = Field(
+        ..., 
+        ge=0, 
+        description="Total number of chunks processed"
+    )
+    successful_chunks: int = Field(
+        ..., 
+        ge=0, 
+        description="Number of successfully processed chunks"
+    )
+    failed_chunks: int = Field(
+        ..., 
+        ge=0, 
+        description="Number of failed chunks"
+    )
+    total_tokens: int = Field(
+        default=0, 
+        ge=0, 
+        description="Total tokens used across all chunks"
+    )
+    total_cost: float = Field(
+        default=0.0, 
+        ge=0.0, 
+        description="Total processing cost in USD"
+    )
+    processing_time: float = Field(
+        ..., 
+        ge=0.0, 
+        description="Total processing time in seconds"
+    )
+    errors: List[str] = Field(
+        default_factory=list, 
+        description="List of error messages"
+    )
+    
+    # Computed properties
+    @property
+    def success_rate(self) -> float:
+        """Calculate success rate as percentage."""
+        if self.total_chunks == 0:
+            return 0.0
+        return (self.successful_chunks / self.total_chunks) * 100.0
+    
+    @property
+    def chunks_per_second(self) -> float:
+        """Calculate processing speed."""
+        if self.processing_time == 0:
+            return 0.0
+        return self.total_chunks / self.processing_time
+    
+    @property
+    def average_cost_per_chunk(self) -> float:
+        """Calculate average cost per successfully processed chunk."""
+        if self.successful_chunks == 0:
+            return 0.0
+        return self.total_cost / self.successful_chunks
+    
+    @property
+    def average_tokens_per_chunk(self) -> float:
+        """Calculate average tokens per successfully processed chunk."""
+        if self.successful_chunks == 0:
+            return 0.0
+        return self.total_tokens / self.successful_chunks
+    
+    @validator('successful_chunks', 'failed_chunks')
+    def validate_chunk_counts(cls, v, values):
+        if 'total_chunks' in values:
+            total = values['total_chunks']
+            if v > total:
+                raise ValueError(f'Chunk count cannot exceed total chunks ({total})')
+        return v
+    
+    @validator('failed_chunks')
+    def validate_total_consistency(cls, v, values):
+        if 'total_chunks' in values and 'successful_chunks' in values:
+            expected_failed = values['total_chunks'] - values['successful_chunks']
+            if v != expected_failed:
+                raise ValueError(
+                    f'Failed chunks ({v}) + successful chunks ({values["successful_chunks"]}) '
+                    f'must equal total chunks ({values["total_chunks"]})'
+                )
+        return v
+    
+    class Config:
+        validate_assignment = True
+        
+    def model_dump_summary(self) -> Dict[str, Any]:
+        """Get a summary dict for logging."""
+        return {
+            "total_chunks": self.total_chunks,
+            "successful": self.successful_chunks,
+            "failed": self.failed_chunks,
+            "success_rate": f"{self.success_rate:.1f}%",
+            "total_tokens": self.total_tokens,
+            "total_cost": f"${self.total_cost:.4f}",
+            "processing_time": f"{self.processing_time:.2f}s",
+            "chunks_per_second": f"{self.chunks_per_second:.1f}",
+            "avg_cost_per_chunk": f"${self.average_cost_per_chunk:.4f}",
+            "error_count": len(self.errors)
+        }
+
+
+class ProcessingConfig(BaseModel):
+    """Configuration for embedding processing."""
+    batch_size: int = Field(
+        default=100, 
+        ge=1, 
+        le=2048, 
+        description="Number of chunks to process in one batch"
+    )
+    embedding_model: str = Field(
+        default="text-embedding-ada-002",
+        description="OpenAI embedding model to use"
+    )
+    max_retries: int = Field(
+        default=3, 
+        ge=0, 
+        le=10, 
+        description="Maximum number of retries for failed requests"
+    )
+    retry_delay: float = Field(
+        default=1.0, 
+        ge=0.0, 
+        description="Delay between retries in seconds"
+    )
+    rate_limit_delay: float = Field(
+        default=0.5, 
+        ge=0.0, 
+        description="Delay between batches to respect rate limits"
+    )
+    timeout_seconds: int = Field(
+        default=60, 
+        ge=1, 
+        description="Timeout for API requests in seconds"
+    )
+    
+    class Config:
+        validate_assignment = True

django_cfg/apps/knowbase/services/embedding/processors.py

@@ -0,0 +1,148 @@
+"""
+Chunk processors for different content types.
+
+This module provides specialized processors for handling
+document and archive chunks with their specific requirements.
+"""
+
+import logging
+from typing import Protocol
+
+from django_cfg.apps.knowbase.models import DocumentChunk, ArchiveItemChunk, ExternalDataChunk
+from .models import ChunkData, EmbeddingResult
+
+logger = logging.getLogger(__name__)
+
+
+class ChunkProcessor(Protocol):
+    """Protocol for chunk processors."""
+    
+    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
+        """Prepare chunk content for embedding generation."""
+        ...
+    
+    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
+        """Save embedding result to database."""
+        ...
+
+
+class DocumentChunkProcessor:
+    """Processor for document chunks."""
+    
+    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
+        """Prepare document chunk content for embedding."""
+        return chunk.content.strip()
+    
+    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
+        """Save embedding result for document chunk."""
+        try:
+            logger.debug(f"🔍 Looking for document chunk with id: {chunk_id}")
+            chunk = DocumentChunk.objects.get(id=chunk_id)
+            logger.debug(f"📄 Found document chunk: {chunk.id}, current embedding length: {len(chunk.embedding) if chunk.embedding is not None and len(chunk.embedding) > 0 else 0}")
+            
+            chunk.embedding = result.embedding
+            chunk.token_count = result.tokens
+            chunk.embedding_cost = result.cost
+            chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
+            
+            logger.info(f"✅ Document chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
+            
+        except DocumentChunk.DoesNotExist:
+            logger.error(f"❌ Document chunk {chunk_id} not found")
+            raise
+        except Exception as e:
+            logger.error(f"❌ Error saving document chunk {chunk_id}: {e}")
+            raise
+
+
+class ArchiveChunkProcessor:
+    """Processor for archive chunks."""
+    
+    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
+        """Prepare archive chunk content for embedding with context."""
+        content = chunk.content
+        context = chunk.context_metadata or {}
+        
+        # Build context prefix for better embeddings
+        context_parts = []
+        
+        if context.get('file_path'):
+            context_parts.append(f"File: {context['file_path']}")
+        if context.get('function_name'):
+            context_parts.append(f"Function: {context['function_name']}")
+        if context.get('class_name'):
+            context_parts.append(f"Class: {context['class_name']}")
+        if context.get('language'):
+            context_parts.append(f"Language: {context['language']}")
+        
+        if context_parts:
+            context_prefix = " | ".join(context_parts)
+            enhanced_content = f"{context_prefix}\n\n{content}"
+        else:
+            enhanced_content = content
+        
+        # Ensure content is not too long for embedding model
+        max_length = 8000  # Conservative limit
+        if len(enhanced_content) > max_length:
+            if context_parts:
+                context_prefix_len = len(context_prefix) + 2  # +2 for \n\n
+                available_content_len = max_length - context_prefix_len
+                if available_content_len > 100:  # Ensure we have meaningful content
+                    truncated_content = content[:available_content_len] + "..."
+                    enhanced_content = f"{context_prefix}\n\n{truncated_content}"
+                else:
+                    enhanced_content = content[:max_length] + "..."
+            else:
+                enhanced_content = content[:max_length] + "..."
+        
+        return enhanced_content.strip()
+    
+    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
+        """Save embedding result for archive chunk."""
+        try:
+            chunk = ArchiveItemChunk.objects.select_related('item').get(id=chunk_id)
+            chunk.embedding = result.embedding
+            chunk.token_count = result.tokens
+            chunk.embedding_cost = result.cost
+            chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
+            
+            # Update parent item statistics
+            item = chunk.item
+            item.total_tokens += result.tokens
+            item.processing_cost += result.cost
+            item.save(update_fields=['total_tokens', 'processing_cost'])
+            
+            logger.debug(f"✅ Archive chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
+            
+        except ArchiveItemChunk.DoesNotExist:
+            logger.error(f"❌ Archive chunk {chunk_id} not found")
+            raise
+
+
+class ExternalDataChunkProcessor:
+    """Processor for external data chunks."""
+    
+    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
+        """Prepare external data chunk content for embedding."""
+        return chunk.content.strip()
+    
+    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
+        """Save embedding result for external data chunk."""
+        try:
+            logger.debug(f"🔍 Looking for external data chunk with id: {chunk_id}")
+            chunk = ExternalDataChunk.objects.get(id=chunk_id)
+            logger.debug(f"🔗 Found external data chunk: {chunk.id}, current embedding length: {len(chunk.embedding) if chunk.embedding is not None and len(chunk.embedding) > 0 else 0}")
+            
+            chunk.embedding = result.embedding
+            chunk.token_count = result.tokens
+            chunk.embedding_cost = result.cost
+            chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
+            
+            logger.info(f"✅ External data chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
+            
+        except ExternalDataChunk.DoesNotExist:
+            logger.error(f"❌ External data chunk {chunk_id} not found")
+            raise
+        except Exception as e:
+            logger.error(f"❌ Error saving external data chunk {chunk_id}: {e}")
+            raise

django_cfg/apps/knowbase/services/embedding/utils.py

@@ -0,0 +1,176 @@
+"""
+Utility functions for embedding processing.
+
+This module provides convenient wrapper functions for common
+embedding processing operations.
+"""
+
+import logging
+from typing import List, Union
+
+from django_cfg.apps.knowbase.models import DocumentChunk, ArchiveItemChunk, ExternalDataChunk
+
+from .models import ChunkData, BatchProcessingResult, ChunkType
+from .batch_processor import OptimizedEmbeddingProcessor
+from .async_processor import AsyncOptimizedEmbeddingProcessor
+
+logger = logging.getLogger(__name__)
+
+
+def process_document_chunks_optimized(document_chunks: List[DocumentChunk]) -> BatchProcessingResult:
+    """Process document chunks with optimized batch operations."""
+    
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            parent_id=str(chunk.document_id),
+            parent_type=ChunkType.DOCUMENT
+        )
+        for chunk in document_chunks
+        if chunk.content and chunk.content.strip()
+    ]
+    
+    processor = OptimizedEmbeddingProcessor()
+    return processor.process_chunks_batch(chunk_data)
+
+
+def process_archive_chunks_optimized(archive_chunks: List[ArchiveItemChunk]) -> BatchProcessingResult:
+    """Process archive chunks with optimized batch operations."""
+    
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            context_metadata=chunk.context_metadata,
+            parent_id=str(chunk.item_id),
+            parent_type=ChunkType.ARCHIVE
+        )
+        for chunk in archive_chunks
+        if chunk.content and chunk.content.strip()
+    ]
+    
+    processor = OptimizedEmbeddingProcessor()
+    return processor.process_chunks_batch(chunk_data)
+
+
+async def aprocess_document_chunks_optimized(document_chunks: List[DocumentChunk]) -> BatchProcessingResult:
+    """Async version of document chunk processing."""
+    
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            parent_id=str(chunk.document_id),
+            parent_type=ChunkType.DOCUMENT
+        )
+        for chunk in document_chunks
+        if chunk.content and chunk.content.strip()
+    ]
+    
+    processor = AsyncOptimizedEmbeddingProcessor()
+    return await processor.aprocess_chunks_batch(chunk_data)
+
+
+async def aprocess_archive_chunks_optimized(archive_chunks: List[ArchiveItemChunk]) -> BatchProcessingResult:
+    """Async version of archive chunk processing."""
+    
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            context_metadata=chunk.context_metadata,
+            parent_id=str(chunk.item_id),
+            parent_type=ChunkType.ARCHIVE
+        )
+        for chunk in archive_chunks
+        if chunk.content and chunk.content.strip()
+    ]
+    
+    processor = AsyncOptimizedEmbeddingProcessor()
+    return await processor.aprocess_chunks_batch(chunk_data)
+
+
+def process_chunks_context_aware(chunks: Union[List[DocumentChunk], List[ArchiveItemChunk]]) -> BatchProcessingResult:
+    """
+    Context-aware chunk processing that works in both sync and async environments.
+    
+    This function automatically detects the execution context and uses appropriate methods.
+    """
+    if not chunks:
+        return BatchProcessingResult(
+            total_chunks=0,
+            successful_chunks=0,
+            failed_chunks=0,
+            total_tokens=0,
+            total_cost=0.0,
+            processing_time=0.0,
+            errors=[]
+        )
+    
+    # Determine chunk type
+    first_chunk = chunks[0]
+    if isinstance(first_chunk, DocumentChunk):
+        chunk_data = [
+            ChunkData(
+                id=str(chunk.id),
+                content=chunk.content,
+                parent_id=str(chunk.document_id),
+                parent_type=ChunkType.DOCUMENT
+            )
+            for chunk in chunks
+            if chunk.content and chunk.content.strip()
+        ]
+    elif isinstance(first_chunk, ArchiveItemChunk):
+        chunk_data = [
+            ChunkData(
+                id=str(chunk.id),
+                content=chunk.content,
+                context_metadata=chunk.context_metadata,
+                parent_id=str(chunk.item_id),
+                parent_type=ChunkType.ARCHIVE
+            )
+            for chunk in chunks
+            if chunk.content and chunk.content.strip()
+        ]
+    else:
+        raise ValueError(f"Unsupported chunk type: {type(first_chunk)}")
+    
+    processor = AsyncOptimizedEmbeddingProcessor()
+    return processor.process_chunks_batch_context_aware(chunk_data)
+
+
+def process_external_data_chunks_optimized(external_data_chunks: List[ExternalDataChunk]) -> BatchProcessingResult:
+    """Process external data chunks with optimized batch operations."""
+    
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            context_metadata=chunk.chunk_metadata,
+            parent_id=str(chunk.external_data.id),
+            parent_type=ChunkType.EXTERNAL_DATA
+        )
+        for chunk in external_data_chunks
+    ]
+    
+    processor = OptimizedEmbeddingProcessor()
+    return processor.process_chunks_batch(chunk_data)
+
+
+async def aprocess_external_data_chunks_optimized(external_data_chunks: List[ExternalDataChunk]) -> BatchProcessingResult:
+    """Async version of external data chunk processing."""
+    
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            context_metadata=chunk.chunk_metadata,
+            parent_id=str(chunk.external_data.id),
+            parent_type=ChunkType.EXTERNAL_DATA
+        )
+        for chunk in external_data_chunks
+    ]
+    
+    processor = AsyncOptimizedEmbeddingProcessor()
+    return await processor.aprocess_chunks_batch(chunk_data)