django-cfg 1.4.48__py3-none-any.whl → 1.5.7__py3-none-any.whl

django_cfg/apps/business/knowbase/services/embedding/batch_processor.py

@@ -0,0 +1,253 @@
+"""
+High-performance batch embedding processor.
+
+This module provides the main batch processing engine for generating
+embeddings with optimized API calls and database operations.
+"""
+
+import logging
+import time
+from typing import List, Optional
+
+from django_cfg.apps.business.knowbase.config.settings import get_cache_settings
+from django_cfg.apps.business.knowbase.utils.chunk_settings import (
+    get_embedding_batch_size,
+    get_embedding_model,
+)
+from django_cfg.modules.django_llm.llm.client import LLMClient
+
+from .batch_result import BatchResultBuilder
+from .models import BatchProcessingResult, ChunkData, ChunkType, EmbeddingResult
+from .processors import ArchiveChunkProcessor, DocumentChunkProcessor, ExternalDataChunkProcessor
+
+logger = logging.getLogger(__name__)
+
+
+class OptimizedEmbeddingProcessor:
+    """High-performance embedding processor with batch operations."""
+
+    def __init__(self, batch_size: Optional[int] = None, embedding_model: Optional[str] = None):
+        """
+        Initialize the processor.
+        
+        Args:
+            batch_size: Number of chunks to process in one API call (uses Constance setting if None)
+            embedding_model: Embedding model to use (uses Constance setting if None)
+        """
+        # Use Constance settings if not provided
+        self.batch_size = min(batch_size or get_embedding_batch_size(), 100)  # Conservative limit for stability
+        self.embedding_model = embedding_model or get_embedding_model()
+
+        # Initialize LLM client with OpenAI only for embeddings
+        # OpenRouter doesn't support embedding models, so we use OpenAI directly
+        # Use auto-configured LLMClient with explicit OpenAI preference for embeddings
+        # Get cache settings from configuration (directory is auto-created)
+        cache_settings = get_cache_settings()
+        self.llm_client = LLMClient(
+            preferred_provider="openai",  # Force OpenAI for embeddings
+            cache_dir=cache_settings.cache_dir,
+            cache_ttl=cache_settings.cache_ttl,
+            max_cache_size=cache_settings.max_cache_size
+        )
+
+        # Processors for different chunk types
+        self.processors = {
+            ChunkType.DOCUMENT: DocumentChunkProcessor(),
+            ChunkType.ARCHIVE: ArchiveChunkProcessor(),
+            ChunkType.EXTERNAL_DATA: ExternalDataChunkProcessor()
+        }
+
+        logger.info(f"🚀 OptimizedEmbeddingProcessor initialized: batch_size={self.batch_size}, model={self.embedding_model}")
+
+    def process_chunks_batch(self, chunks: List[ChunkData]) -> BatchProcessingResult:
+        """
+        Process multiple chunks with optimized batch operations.
+        
+        Args:
+            chunks: List of chunks to process
+            
+        Returns:
+            BatchProcessingResult with processing statistics
+        """
+        start_time = time.time()
+        total_chunks = len(chunks)
+        result_builder = BatchResultBuilder(total_chunks)
+
+        logger.info(f"🔮 Starting batch processing of {total_chunks} chunks")
+
+        # Process in batches
+        for i in range(0, total_chunks, self.batch_size):
+            batch = chunks[i:i + self.batch_size]
+            batch_num = (i // self.batch_size) + 1
+            total_batches = (total_chunks + self.batch_size - 1) // self.batch_size
+
+            logger.info(f"🔮 Processing batch {batch_num}/{total_batches} ({len(batch)} chunks)")
+
+            try:
+                batch_results = self._process_single_batch(batch)
+                result_builder.add_batch_results(batch_results)
+
+                # Small delay between batches to respect rate limits
+                if i + self.batch_size < total_chunks:
+                    time.sleep(0.5)
+
+            except Exception as e:
+                error_msg = f"Batch {batch_num} failed: {str(e)}"
+                logger.error(f"❌ {error_msg}")
+                result_builder.add_batch_error(error_msg, len(batch))
+
+        processing_time = time.time() - start_time
+        result = result_builder.build(processing_time)
+
+        # Log using Pydantic model's summary
+        summary = result.model_dump_summary()
+        logger.info(f"🎉 Batch processing completed: {summary}")
+        logger.info(f"📊 Performance: {summary['chunks_per_second']} chunks/sec, {summary['avg_cost_per_chunk']} per chunk")
+
+        return result
+
+    def _process_single_batch(self, batch: List[ChunkData]) -> List[EmbeddingResult]:
+        """Process a single batch of chunks and return list of EmbeddingResult."""
+
+        # Prepare content for all chunks
+        prepared_contents = []
+        chunk_mapping = {}  # Map index to chunk
+
+        for idx, chunk in enumerate(batch):
+            processor = self.processors.get(chunk.parent_type)
+            if not processor:
+                logger.warning(f"⚠️ Unknown chunk type: {chunk.parent_type}")
+                continue
+
+            try:
+                content = processor.prepare_content_for_embedding(chunk)
+                if content and content.strip():
+                    prepared_contents.append(content)
+                    chunk_mapping[len(prepared_contents) - 1] = chunk
+                else:
+                    logger.warning(f"⚠️ Empty content for chunk {chunk.id}")
+            except Exception as e:
+                logger.error(f"❌ Failed to prepare content for chunk {chunk.id}: {e}")
+
+        if not prepared_contents:
+            # Return failed results for all chunks
+            return [
+                EmbeddingResult(
+                    chunk_id=chunk.id,
+                    success=False,
+                    error="No valid content to process"
+                )
+                for chunk in batch
+            ]
+
+        # Generate embeddings in batch
+        try:
+            embedding_results = self._generate_batch_embeddings(prepared_contents)
+
+            # Process and save results
+            final_results = []
+
+            for idx, embedding_result in enumerate(embedding_results):
+                if idx not in chunk_mapping:
+                    continue
+
+                chunk = chunk_mapping[idx]
+                processor = self.processors[chunk.parent_type]
+
+                # Set the chunk_id in the result
+                embedding_result.chunk_id = chunk.id
+
+                if embedding_result.success:
+                    try:
+                        logger.debug(f"🔄 Attempting to save embedding for chunk {chunk.id} (type: {chunk.parent_type})")
+                        processor.save_embedding_result(chunk.id, embedding_result)
+                        logger.info(f"✅ Successfully saved embedding for chunk {chunk.id}")
+                        final_results.append(embedding_result)
+                    except Exception as e:
+                        error_msg = f"Failed to save embedding for chunk {chunk.id}: {e}"
+                        logger.error(f"❌ {error_msg}")
+                        failed_result = EmbeddingResult(
+                            chunk_id=chunk.id,
+                            success=False,
+                            error=error_msg
+                        )
+                        final_results.append(failed_result)
+                else:
+                    final_results.append(embedding_result)
+
+            return final_results
+
+        except Exception as e:
+            error_msg = f"Batch embedding generation failed: {e}"
+            logger.error(f"❌ {error_msg}")
+            # Return failed results for all chunks
+            return [
+                EmbeddingResult(
+                    chunk_id=chunk.id,
+                    success=False,
+                    error=error_msg
+                )
+                for chunk in batch
+            ]
+
+    def _generate_batch_embeddings(self, contents: List[str]) -> List[EmbeddingResult]:
+        """Generate embeddings for multiple contents using LLMClient."""
+
+        results = []
+
+        try:
+            # Use LLMClient's generate_embedding method for each content
+            # This handles both OpenAI and OpenRouter properly
+            for idx, content in enumerate(contents):
+                try:
+                    # Use LLMClient's method which handles provider differences
+                    embedding_response = self.llm_client.generate_embedding(
+                        text=content,
+                        model=self.embedding_model
+                    )
+
+                    results.append(EmbeddingResult(
+                        chunk_id="",  # Will be set by caller
+                        embedding=embedding_response.embedding,
+                        tokens=embedding_response.tokens,
+                        cost=embedding_response.cost,
+                        success=True
+                    ))
+
+                except Exception as e:
+                    logger.error(f"❌ Failed to generate embedding for content {idx}: {e}")
+                    results.append(EmbeddingResult(
+                        chunk_id="",
+                        embedding=[],
+                        tokens=0,
+                        cost=0.0,
+                        success=False,
+                        error=str(e)
+                    ))
+
+            successful_count = len([r for r in results if r.success])
+            logger.info(f"🎯 Generated {successful_count}/{len(results)} embeddings successfully")
+
+            # Log details of each result
+            for i, result in enumerate(results):
+                if result.success:
+                    logger.debug(f"  ✅ Result {i}: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
+                else:
+                    logger.debug(f"  ❌ Result {i}: {result.error}")
+
+            return results
+
+        except Exception as e:
+            logger.error(f"❌ Batch embedding generation failed: {e}")
+            # Return failed results for all contents
+            return [
+                EmbeddingResult(
+                    chunk_id="",
+                    embedding=[],
+                    tokens=0,
+                    cost=0.0,
+                    success=False,
+                    error=str(e)
+                )
+                for _ in contents
+            ]

django_cfg/apps/business/knowbase/services/embedding/processors.py

@@ -0,0 +1,152 @@
+"""
+Chunk processors for different content types.
+
+This module provides specialized processors for handling
+document and archive chunks with their specific requirements.
+"""
+
+import logging
+from typing import Protocol
+
+from django_cfg.apps.business.knowbase.models import ArchiveItemChunk, DocumentChunk, ExternalDataChunk
+
+from .models import ChunkData, EmbeddingResult
+
+logger = logging.getLogger(__name__)
+
+
+class ChunkProcessor(Protocol):
+    """Protocol for chunk processors."""
+
+    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
+        """Prepare chunk content for embedding generation."""
+        ...
+
+    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
+        """Save embedding result to database."""
+        ...
+
+
+class DocumentChunkProcessor:
+    """Processor for document chunks."""
+
+    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
+        """Prepare document chunk content for embedding."""
+        return chunk.content.strip()
+
+    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
+        """Save embedding result for document chunk."""
+        try:
+            logger.debug(f"🔍 Looking for document chunk with id: {chunk_id}")
+            chunk = DocumentChunk.objects.get(id=chunk_id)
+            logger.debug(f"📄 Found document chunk: {chunk.id}, current embedding length: {len(chunk.embedding) if chunk.embedding is not None and len(chunk.embedding) > 0 else 0}")
+
+            chunk.embedding = result.embedding
+            chunk.token_count = result.tokens
+            chunk.embedding_cost = result.cost
+            chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
+
+            logger.info(f"✅ Document chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
+
+        except DocumentChunk.DoesNotExist:
+            logger.error(f"❌ Document chunk {chunk_id} not found")
+            raise
+        except Exception as e:
+            logger.error(f"❌ Error saving document chunk {chunk_id}: {e}")
+            raise
+
+
+class ArchiveChunkProcessor:
+    """Processor for archive chunks."""
+
+    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
+        """Prepare archive chunk content for embedding with context."""
+        content = chunk.content
+        context = chunk.context_metadata or {}
+
+        # Build context prefix for better embeddings
+        context_parts = []
+
+        if context.get('file_path'):
+            context_parts.append(f"File: {context['file_path']}")
+        if context.get('function_name'):
+            context_parts.append(f"Function: {context['function_name']}")
+        if context.get('class_name'):
+            context_parts.append(f"Class: {context['class_name']}")
+        if context.get('language'):
+            context_parts.append(f"Language: {context['language']}")
+
+        if context_parts:
+            context_prefix = " | ".join(context_parts)
+            enhanced_content = f"{context_prefix}\n\n{content}"
+        else:
+            enhanced_content = content
+
+        # Ensure content is not too long for embedding model
+        max_length = 8000  # Conservative limit
+        if len(enhanced_content) > max_length:
+            if context_parts:
+                context_prefix_len = len(context_prefix) + 2  # +2 for \n\n
+                # Account for "..." suffix (3 chars)
+                available_content_len = max_length - context_prefix_len - 3
+                if available_content_len > 100:  # Ensure we have meaningful content
+                    truncated_content = content[:available_content_len] + "..."
+                    enhanced_content = f"{context_prefix}\n\n{truncated_content}"
+                else:
+                    # Subtract 3 for "..." suffix
+                    enhanced_content = content[:max_length - 3] + "..."
+            else:
+                # Subtract 3 for "..." suffix
+                enhanced_content = content[:max_length - 3] + "..."
+
+        return enhanced_content.strip()
+
+    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
+        """Save embedding result for archive chunk."""
+        try:
+            chunk = ArchiveItemChunk.objects.select_related('item').get(id=chunk_id)
+            chunk.embedding = result.embedding
+            chunk.token_count = result.tokens
+            chunk.embedding_cost = result.cost
+            chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
+
+            # Update parent item statistics
+            item = chunk.item
+            item.total_tokens += result.tokens
+            item.processing_cost += result.cost
+            item.save(update_fields=['total_tokens', 'processing_cost'])
+
+            logger.debug(f"✅ Archive chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
+
+        except ArchiveItemChunk.DoesNotExist:
+            logger.error(f"❌ Archive chunk {chunk_id} not found")
+            raise
+
+
+class ExternalDataChunkProcessor:
+    """Processor for external data chunks."""
+
+    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
+        """Prepare external data chunk content for embedding."""
+        return chunk.content.strip()
+
+    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
+        """Save embedding result for external data chunk."""
+        try:
+            logger.debug(f"🔍 Looking for external data chunk with id: {chunk_id}")
+            chunk = ExternalDataChunk.objects.get(id=chunk_id)
+            logger.debug(f"🔗 Found external data chunk: {chunk.id}, current embedding length: {len(chunk.embedding) if chunk.embedding is not None and len(chunk.embedding) > 0 else 0}")
+
+            chunk.embedding = result.embedding
+            chunk.token_count = result.tokens
+            chunk.embedding_cost = result.cost
+            chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
+
+            logger.info(f"✅ External data chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
+
+        except ExternalDataChunk.DoesNotExist:
+            logger.error(f"❌ External data chunk {chunk_id} not found")
+            raise
+        except Exception as e:
+            logger.error(f"❌ Error saving external data chunk {chunk_id}: {e}")
+            raise

django_cfg/apps/business/knowbase/services/embedding/utils.py

@@ -0,0 +1,176 @@
+"""
+Utility functions for embedding processing.
+
+This module provides convenient wrapper functions for common
+embedding processing operations.
+"""
+
+import logging
+from typing import List, Union
+
+from django_cfg.apps.business.knowbase.models import ArchiveItemChunk, DocumentChunk, ExternalDataChunk
+
+from .async_processor import AsyncOptimizedEmbeddingProcessor
+from .batch_processor import OptimizedEmbeddingProcessor
+from .models import BatchProcessingResult, ChunkData, ChunkType
+
+logger = logging.getLogger(__name__)
+
+
+def process_document_chunks_optimized(document_chunks: List[DocumentChunk]) -> BatchProcessingResult:
+    """Process document chunks with optimized batch operations."""
+
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            parent_id=str(chunk.document_id),
+            parent_type=ChunkType.DOCUMENT
+        )
+        for chunk in document_chunks
+        if chunk.content and chunk.content.strip()
+    ]
+
+    processor = OptimizedEmbeddingProcessor()
+    return processor.process_chunks_batch(chunk_data)
+
+
+def process_archive_chunks_optimized(archive_chunks: List[ArchiveItemChunk]) -> BatchProcessingResult:
+    """Process archive chunks with optimized batch operations."""
+
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            context_metadata=chunk.context_metadata,
+            parent_id=str(chunk.item_id),
+            parent_type=ChunkType.ARCHIVE
+        )
+        for chunk in archive_chunks
+        if chunk.content and chunk.content.strip()
+    ]
+
+    processor = OptimizedEmbeddingProcessor()
+    return processor.process_chunks_batch(chunk_data)
+
+
+async def aprocess_document_chunks_optimized(document_chunks: List[DocumentChunk]) -> BatchProcessingResult:
+    """Async version of document chunk processing."""
+
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            parent_id=str(chunk.document_id),
+            parent_type=ChunkType.DOCUMENT
+        )
+        for chunk in document_chunks
+        if chunk.content and chunk.content.strip()
+    ]
+
+    processor = AsyncOptimizedEmbeddingProcessor()
+    return await processor.aprocess_chunks_batch(chunk_data)
+
+
+async def aprocess_archive_chunks_optimized(archive_chunks: List[ArchiveItemChunk]) -> BatchProcessingResult:
+    """Async version of archive chunk processing."""
+
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            context_metadata=chunk.context_metadata,
+            parent_id=str(chunk.item_id),
+            parent_type=ChunkType.ARCHIVE
+        )
+        for chunk in archive_chunks
+        if chunk.content and chunk.content.strip()
+    ]
+
+    processor = AsyncOptimizedEmbeddingProcessor()
+    return await processor.aprocess_chunks_batch(chunk_data)
+
+
+def process_chunks_context_aware(chunks: Union[List[DocumentChunk], List[ArchiveItemChunk]]) -> BatchProcessingResult:
+    """
+    Context-aware chunk processing that works in both sync and async environments.
+    
+    This function automatically detects the execution context and uses appropriate methods.
+    """
+    if not chunks:
+        return BatchProcessingResult(
+            total_chunks=0,
+            successful_chunks=0,
+            failed_chunks=0,
+            total_tokens=0,
+            total_cost=0.0,
+            processing_time=0.0,
+            errors=[]
+        )
+
+    # Determine chunk type
+    first_chunk = chunks[0]
+    if isinstance(first_chunk, DocumentChunk):
+        chunk_data = [
+            ChunkData(
+                id=str(chunk.id),
+                content=chunk.content,
+                parent_id=str(chunk.document_id),
+                parent_type=ChunkType.DOCUMENT
+            )
+            for chunk in chunks
+            if chunk.content and chunk.content.strip()
+        ]
+    elif isinstance(first_chunk, ArchiveItemChunk):
+        chunk_data = [
+            ChunkData(
+                id=str(chunk.id),
+                content=chunk.content,
+                context_metadata=chunk.context_metadata,
+                parent_id=str(chunk.item_id),
+                parent_type=ChunkType.ARCHIVE
+            )
+            for chunk in chunks
+            if chunk.content and chunk.content.strip()
+        ]
+    else:
+        raise ValueError(f"Unsupported chunk type: {type(first_chunk)}")
+
+    processor = AsyncOptimizedEmbeddingProcessor()
+    return processor.process_chunks_batch_context_aware(chunk_data)
+
+
+def process_external_data_chunks_optimized(external_data_chunks: List[ExternalDataChunk]) -> BatchProcessingResult:
+    """Process external data chunks with optimized batch operations."""
+
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            context_metadata=chunk.chunk_metadata,
+            parent_id=str(chunk.external_data.id),
+            parent_type=ChunkType.EXTERNAL_DATA
+        )
+        for chunk in external_data_chunks
+    ]
+
+    processor = OptimizedEmbeddingProcessor()
+    return processor.process_chunks_batch(chunk_data)
+
+
+async def aprocess_external_data_chunks_optimized(external_data_chunks: List[ExternalDataChunk]) -> BatchProcessingResult:
+    """Async version of external data chunk processing."""
+
+    chunk_data = [
+        ChunkData(
+            id=str(chunk.id),
+            content=chunk.content,
+            context_metadata=chunk.chunk_metadata,
+            parent_id=str(chunk.external_data.id),
+            parent_type=ChunkType.EXTERNAL_DATA
+        )
+        for chunk in external_data_chunks
+    ]
+
+    processor = AsyncOptimizedEmbeddingProcessor()
+    return await processor.aprocess_chunks_batch(chunk_data)