django-cfg 1.4.48__py3-none-any.whl → 1.5.7__py3-none-any.whl

django_cfg/apps/knowbase/services/embedding/batch_processor.py

@@ -1,253 +0,0 @@
-"""
-High-performance batch embedding processor.
-
-This module provides the main batch processing engine for generating
-embeddings with optimized API calls and database operations.
-"""
-
-import logging
-import time
-from typing import List, Optional
-
-from django_cfg.apps.knowbase.config.settings import get_cache_settings
-from django_cfg.apps.knowbase.utils.chunk_settings import (
-    get_embedding_batch_size,
-    get_embedding_model,
-)
-from django_cfg.modules.django_llm.llm.client import LLMClient
-
-from .batch_result import BatchResultBuilder
-from .models import BatchProcessingResult, ChunkData, ChunkType, EmbeddingResult
-from .processors import ArchiveChunkProcessor, DocumentChunkProcessor, ExternalDataChunkProcessor
-
-logger = logging.getLogger(__name__)
-
-
-class OptimizedEmbeddingProcessor:
-    """High-performance embedding processor with batch operations."""
-
-    def __init__(self, batch_size: Optional[int] = None, embedding_model: Optional[str] = None):
-        """
-        Initialize the processor.
-        
-        Args:
-            batch_size: Number of chunks to process in one API call (uses Constance setting if None)
-            embedding_model: Embedding model to use (uses Constance setting if None)
-        """
-        # Use Constance settings if not provided
-        self.batch_size = min(batch_size or get_embedding_batch_size(), 100)  # Conservative limit for stability
-        self.embedding_model = embedding_model or get_embedding_model()
-
-        # Initialize LLM client with OpenAI only for embeddings
-        # OpenRouter doesn't support embedding models, so we use OpenAI directly
-        # Use auto-configured LLMClient with explicit OpenAI preference for embeddings
-        # Get cache settings from configuration (directory is auto-created)
-        cache_settings = get_cache_settings()
-        self.llm_client = LLMClient(
-            preferred_provider="openai",  # Force OpenAI for embeddings
-            cache_dir=cache_settings.cache_dir,
-            cache_ttl=cache_settings.cache_ttl,
-            max_cache_size=cache_settings.max_cache_size
-        )
-
-        # Processors for different chunk types
-        self.processors = {
-            ChunkType.DOCUMENT: DocumentChunkProcessor(),
-            ChunkType.ARCHIVE: ArchiveChunkProcessor(),
-            ChunkType.EXTERNAL_DATA: ExternalDataChunkProcessor()
-        }
-
-        logger.info(f"🚀 OptimizedEmbeddingProcessor initialized: batch_size={self.batch_size}, model={self.embedding_model}")
-
-    def process_chunks_batch(self, chunks: List[ChunkData]) -> BatchProcessingResult:
-        """
-        Process multiple chunks with optimized batch operations.
-        
-        Args:
-            chunks: List of chunks to process
-            
-        Returns:
-            BatchProcessingResult with processing statistics
-        """
-        start_time = time.time()
-        total_chunks = len(chunks)
-        result_builder = BatchResultBuilder(total_chunks)
-
-        logger.info(f"🔮 Starting batch processing of {total_chunks} chunks")
-
-        # Process in batches
-        for i in range(0, total_chunks, self.batch_size):
-            batch = chunks[i:i + self.batch_size]
-            batch_num = (i // self.batch_size) + 1
-            total_batches = (total_chunks + self.batch_size - 1) // self.batch_size
-
-            logger.info(f"🔮 Processing batch {batch_num}/{total_batches} ({len(batch)} chunks)")
-
-            try:
-                batch_results = self._process_single_batch(batch)
-                result_builder.add_batch_results(batch_results)
-
-                # Small delay between batches to respect rate limits
-                if i + self.batch_size < total_chunks:
-                    time.sleep(0.5)
-
-            except Exception as e:
-                error_msg = f"Batch {batch_num} failed: {str(e)}"
-                logger.error(f"❌ {error_msg}")
-                result_builder.add_batch_error(error_msg, len(batch))
-
-        processing_time = time.time() - start_time
-        result = result_builder.build(processing_time)
-
-        # Log using Pydantic model's summary
-        summary = result.model_dump_summary()
-        logger.info(f"🎉 Batch processing completed: {summary}")
-        logger.info(f"📊 Performance: {summary['chunks_per_second']} chunks/sec, {summary['avg_cost_per_chunk']} per chunk")
-
-        return result
-
-    def _process_single_batch(self, batch: List[ChunkData]) -> List[EmbeddingResult]:
-        """Process a single batch of chunks and return list of EmbeddingResult."""
-
-        # Prepare content for all chunks
-        prepared_contents = []
-        chunk_mapping = {}  # Map index to chunk
-
-        for idx, chunk in enumerate(batch):
-            processor = self.processors.get(chunk.parent_type)
-            if not processor:
-                logger.warning(f"⚠️ Unknown chunk type: {chunk.parent_type}")
-                continue
-
-            try:
-                content = processor.prepare_content_for_embedding(chunk)
-                if content and content.strip():
-                    prepared_contents.append(content)
-                    chunk_mapping[len(prepared_contents) - 1] = chunk
-                else:
-                    logger.warning(f"⚠️ Empty content for chunk {chunk.id}")
-            except Exception as e:
-                logger.error(f"❌ Failed to prepare content for chunk {chunk.id}: {e}")
-
-        if not prepared_contents:
-            # Return failed results for all chunks
-            return [
-                EmbeddingResult(
-                    chunk_id=chunk.id,
-                    success=False,
-                    error="No valid content to process"
-                )
-                for chunk in batch
-            ]
-
-        # Generate embeddings in batch
-        try:
-            embedding_results = self._generate_batch_embeddings(prepared_contents)
-
-            # Process and save results
-            final_results = []
-
-            for idx, embedding_result in enumerate(embedding_results):
-                if idx not in chunk_mapping:
-                    continue
-
-                chunk = chunk_mapping[idx]
-                processor = self.processors[chunk.parent_type]
-
-                # Set the chunk_id in the result
-                embedding_result.chunk_id = chunk.id
-
-                if embedding_result.success:
-                    try:
-                        logger.debug(f"🔄 Attempting to save embedding for chunk {chunk.id} (type: {chunk.parent_type})")
-                        processor.save_embedding_result(chunk.id, embedding_result)
-                        logger.info(f"✅ Successfully saved embedding for chunk {chunk.id}")
-                        final_results.append(embedding_result)
-                    except Exception as e:
-                        error_msg = f"Failed to save embedding for chunk {chunk.id}: {e}"
-                        logger.error(f"❌ {error_msg}")
-                        failed_result = EmbeddingResult(
-                            chunk_id=chunk.id,
-                            success=False,
-                            error=error_msg
-                        )
-                        final_results.append(failed_result)
-                else:
-                    final_results.append(embedding_result)
-
-            return final_results
-
-        except Exception as e:
-            error_msg = f"Batch embedding generation failed: {e}"
-            logger.error(f"❌ {error_msg}")
-            # Return failed results for all chunks
-            return [
-                EmbeddingResult(
-                    chunk_id=chunk.id,
-                    success=False,
-                    error=error_msg
-                )
-                for chunk in batch
-            ]
-
-    def _generate_batch_embeddings(self, contents: List[str]) -> List[EmbeddingResult]:
-        """Generate embeddings for multiple contents using LLMClient."""
-
-        results = []
-
-        try:
-            # Use LLMClient's generate_embedding method for each content
-            # This handles both OpenAI and OpenRouter properly
-            for idx, content in enumerate(contents):
-                try:
-                    # Use LLMClient's method which handles provider differences
-                    embedding_response = self.llm_client.generate_embedding(
-                        text=content,
-                        model=self.embedding_model
-                    )
-
-                    results.append(EmbeddingResult(
-                        chunk_id="",  # Will be set by caller
-                        embedding=embedding_response.embedding,
-                        tokens=embedding_response.tokens,
-                        cost=embedding_response.cost,
-                        success=True
-                    ))
-
-                except Exception as e:
-                    logger.error(f"❌ Failed to generate embedding for content {idx}: {e}")
-                    results.append(EmbeddingResult(
-                        chunk_id="",
-                        embedding=[],
-                        tokens=0,
-                        cost=0.0,
-                        success=False,
-                        error=str(e)
-                    ))
-
-            successful_count = len([r for r in results if r.success])
-            logger.info(f"🎯 Generated {successful_count}/{len(results)} embeddings successfully")
-
-            # Log details of each result
-            for i, result in enumerate(results):
-                if result.success:
-                    logger.debug(f"  ✅ Result {i}: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
-                else:
-                    logger.debug(f"  ❌ Result {i}: {result.error}")
-
-            return results
-
-        except Exception as e:
-            logger.error(f"❌ Batch embedding generation failed: {e}")
-            # Return failed results for all contents
-            return [
-                EmbeddingResult(
-                    chunk_id="",
-                    embedding=[],
-                    tokens=0,
-                    cost=0.0,
-                    success=False,
-                    error=str(e)
-                )
-                for _ in contents
-            ]

django_cfg/apps/knowbase/services/embedding/processors.py

@@ -1,152 +0,0 @@
-"""
-Chunk processors for different content types.
-
-This module provides specialized processors for handling
-document and archive chunks with their specific requirements.
-"""
-
-import logging
-from typing import Protocol
-
-from django_cfg.apps.knowbase.models import ArchiveItemChunk, DocumentChunk, ExternalDataChunk
-
-from .models import ChunkData, EmbeddingResult
-
-logger = logging.getLogger(__name__)
-
-
-class ChunkProcessor(Protocol):
-    """Protocol for chunk processors."""
-
-    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
-        """Prepare chunk content for embedding generation."""
-        ...
-
-    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
-        """Save embedding result to database."""
-        ...
-
-
-class DocumentChunkProcessor:
-    """Processor for document chunks."""
-
-    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
-        """Prepare document chunk content for embedding."""
-        return chunk.content.strip()
-
-    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
-        """Save embedding result for document chunk."""
-        try:
-            logger.debug(f"🔍 Looking for document chunk with id: {chunk_id}")
-            chunk = DocumentChunk.objects.get(id=chunk_id)
-            logger.debug(f"📄 Found document chunk: {chunk.id}, current embedding length: {len(chunk.embedding) if chunk.embedding is not None and len(chunk.embedding) > 0 else 0}")
-
-            chunk.embedding = result.embedding
-            chunk.token_count = result.tokens
-            chunk.embedding_cost = result.cost
-            chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
-
-            logger.info(f"✅ Document chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
-
-        except DocumentChunk.DoesNotExist:
-            logger.error(f"❌ Document chunk {chunk_id} not found")
-            raise
-        except Exception as e:
-            logger.error(f"❌ Error saving document chunk {chunk_id}: {e}")
-            raise
-
-
-class ArchiveChunkProcessor:
-    """Processor for archive chunks."""
-
-    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
-        """Prepare archive chunk content for embedding with context."""
-        content = chunk.content
-        context = chunk.context_metadata or {}
-
-        # Build context prefix for better embeddings
-        context_parts = []
-
-        if context.get('file_path'):
-            context_parts.append(f"File: {context['file_path']}")
-        if context.get('function_name'):
-            context_parts.append(f"Function: {context['function_name']}")
-        if context.get('class_name'):
-            context_parts.append(f"Class: {context['class_name']}")
-        if context.get('language'):
-            context_parts.append(f"Language: {context['language']}")
-
-        if context_parts:
-            context_prefix = " | ".join(context_parts)
-            enhanced_content = f"{context_prefix}\n\n{content}"
-        else:
-            enhanced_content = content
-
-        # Ensure content is not too long for embedding model
-        max_length = 8000  # Conservative limit
-        if len(enhanced_content) > max_length:
-            if context_parts:
-                context_prefix_len = len(context_prefix) + 2  # +2 for \n\n
-                # Account for "..." suffix (3 chars)
-                available_content_len = max_length - context_prefix_len - 3
-                if available_content_len > 100:  # Ensure we have meaningful content
-                    truncated_content = content[:available_content_len] + "..."
-                    enhanced_content = f"{context_prefix}\n\n{truncated_content}"
-                else:
-                    # Subtract 3 for "..." suffix
-                    enhanced_content = content[:max_length - 3] + "..."
-            else:
-                # Subtract 3 for "..." suffix
-                enhanced_content = content[:max_length - 3] + "..."
-
-        return enhanced_content.strip()
-
-    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
-        """Save embedding result for archive chunk."""
-        try:
-            chunk = ArchiveItemChunk.objects.select_related('item').get(id=chunk_id)
-            chunk.embedding = result.embedding
-            chunk.token_count = result.tokens
-            chunk.embedding_cost = result.cost
-            chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
-
-            # Update parent item statistics
-            item = chunk.item
-            item.total_tokens += result.tokens
-            item.processing_cost += result.cost
-            item.save(update_fields=['total_tokens', 'processing_cost'])
-
-            logger.debug(f"✅ Archive chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
-
-        except ArchiveItemChunk.DoesNotExist:
-            logger.error(f"❌ Archive chunk {chunk_id} not found")
-            raise
-
-
-class ExternalDataChunkProcessor:
-    """Processor for external data chunks."""
-
-    def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
-        """Prepare external data chunk content for embedding."""
-        return chunk.content.strip()
-
-    def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
-        """Save embedding result for external data chunk."""
-        try:
-            logger.debug(f"🔍 Looking for external data chunk with id: {chunk_id}")
-            chunk = ExternalDataChunk.objects.get(id=chunk_id)
-            logger.debug(f"🔗 Found external data chunk: {chunk.id}, current embedding length: {len(chunk.embedding) if chunk.embedding is not None and len(chunk.embedding) > 0 else 0}")
-
-            chunk.embedding = result.embedding
-            chunk.token_count = result.tokens
-            chunk.embedding_cost = result.cost
-            chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
-
-            logger.info(f"✅ External data chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
-
-        except ExternalDataChunk.DoesNotExist:
-            logger.error(f"❌ External data chunk {chunk_id} not found")
-            raise
-        except Exception as e:
-            logger.error(f"❌ Error saving external data chunk {chunk_id}: {e}")
-            raise

django_cfg/apps/knowbase/services/embedding/utils.py

@@ -1,176 +0,0 @@
-"""
-Utility functions for embedding processing.
-
-This module provides convenient wrapper functions for common
-embedding processing operations.
-"""
-
-import logging
-from typing import List, Union
-
-from django_cfg.apps.knowbase.models import ArchiveItemChunk, DocumentChunk, ExternalDataChunk
-
-from .async_processor import AsyncOptimizedEmbeddingProcessor
-from .batch_processor import OptimizedEmbeddingProcessor
-from .models import BatchProcessingResult, ChunkData, ChunkType
-
-logger = logging.getLogger(__name__)
-
-
-def process_document_chunks_optimized(document_chunks: List[DocumentChunk]) -> BatchProcessingResult:
-    """Process document chunks with optimized batch operations."""
-
-    chunk_data = [
-        ChunkData(
-            id=str(chunk.id),
-            content=chunk.content,
-            parent_id=str(chunk.document_id),
-            parent_type=ChunkType.DOCUMENT
-        )
-        for chunk in document_chunks
-        if chunk.content and chunk.content.strip()
-    ]
-
-    processor = OptimizedEmbeddingProcessor()
-    return processor.process_chunks_batch(chunk_data)
-
-
-def process_archive_chunks_optimized(archive_chunks: List[ArchiveItemChunk]) -> BatchProcessingResult:
-    """Process archive chunks with optimized batch operations."""
-
-    chunk_data = [
-        ChunkData(
-            id=str(chunk.id),
-            content=chunk.content,
-            context_metadata=chunk.context_metadata,
-            parent_id=str(chunk.item_id),
-            parent_type=ChunkType.ARCHIVE
-        )
-        for chunk in archive_chunks
-        if chunk.content and chunk.content.strip()
-    ]
-
-    processor = OptimizedEmbeddingProcessor()
-    return processor.process_chunks_batch(chunk_data)
-
-
-async def aprocess_document_chunks_optimized(document_chunks: List[DocumentChunk]) -> BatchProcessingResult:
-    """Async version of document chunk processing."""
-
-    chunk_data = [
-        ChunkData(
-            id=str(chunk.id),
-            content=chunk.content,
-            parent_id=str(chunk.document_id),
-            parent_type=ChunkType.DOCUMENT
-        )
-        for chunk in document_chunks
-        if chunk.content and chunk.content.strip()
-    ]
-
-    processor = AsyncOptimizedEmbeddingProcessor()
-    return await processor.aprocess_chunks_batch(chunk_data)
-
-
-async def aprocess_archive_chunks_optimized(archive_chunks: List[ArchiveItemChunk]) -> BatchProcessingResult:
-    """Async version of archive chunk processing."""
-
-    chunk_data = [
-        ChunkData(
-            id=str(chunk.id),
-            content=chunk.content,
-            context_metadata=chunk.context_metadata,
-            parent_id=str(chunk.item_id),
-            parent_type=ChunkType.ARCHIVE
-        )
-        for chunk in archive_chunks
-        if chunk.content and chunk.content.strip()
-    ]
-
-    processor = AsyncOptimizedEmbeddingProcessor()
-    return await processor.aprocess_chunks_batch(chunk_data)
-
-
-def process_chunks_context_aware(chunks: Union[List[DocumentChunk], List[ArchiveItemChunk]]) -> BatchProcessingResult:
-    """
-    Context-aware chunk processing that works in both sync and async environments.
-    
-    This function automatically detects the execution context and uses appropriate methods.
-    """
-    if not chunks:
-        return BatchProcessingResult(
-            total_chunks=0,
-            successful_chunks=0,
-            failed_chunks=0,
-            total_tokens=0,
-            total_cost=0.0,
-            processing_time=0.0,
-            errors=[]
-        )
-
-    # Determine chunk type
-    first_chunk = chunks[0]
-    if isinstance(first_chunk, DocumentChunk):
-        chunk_data = [
-            ChunkData(
-                id=str(chunk.id),
-                content=chunk.content,
-                parent_id=str(chunk.document_id),
-                parent_type=ChunkType.DOCUMENT
-            )
-            for chunk in chunks
-            if chunk.content and chunk.content.strip()
-        ]
-    elif isinstance(first_chunk, ArchiveItemChunk):
-        chunk_data = [
-            ChunkData(
-                id=str(chunk.id),
-                content=chunk.content,
-                context_metadata=chunk.context_metadata,
-                parent_id=str(chunk.item_id),
-                parent_type=ChunkType.ARCHIVE
-            )
-            for chunk in chunks
-            if chunk.content and chunk.content.strip()
-        ]
-    else:
-        raise ValueError(f"Unsupported chunk type: {type(first_chunk)}")
-
-    processor = AsyncOptimizedEmbeddingProcessor()
-    return processor.process_chunks_batch_context_aware(chunk_data)
-
-
-def process_external_data_chunks_optimized(external_data_chunks: List[ExternalDataChunk]) -> BatchProcessingResult:
-    """Process external data chunks with optimized batch operations."""
-
-    chunk_data = [
-        ChunkData(
-            id=str(chunk.id),
-            content=chunk.content,
-            context_metadata=chunk.chunk_metadata,
-            parent_id=str(chunk.external_data.id),
-            parent_type=ChunkType.EXTERNAL_DATA
-        )
-        for chunk in external_data_chunks
-    ]
-
-    processor = OptimizedEmbeddingProcessor()
-    return processor.process_chunks_batch(chunk_data)
-
-
-async def aprocess_external_data_chunks_optimized(external_data_chunks: List[ExternalDataChunk]) -> BatchProcessingResult:
-    """Async version of external data chunk processing."""
-
-    chunk_data = [
-        ChunkData(
-            id=str(chunk.id),
-            content=chunk.content,
-            context_metadata=chunk.chunk_metadata,
-            parent_id=str(chunk.external_data.id),
-            parent_type=ChunkType.EXTERNAL_DATA
-        )
-        for chunk in external_data_chunks
-    ]
-
-    processor = AsyncOptimizedEmbeddingProcessor()
-    return await processor.aprocess_chunks_batch(chunk_data)