django-cfg 1.4.21__py3-none-any.whl → 1.4.23__py3-none-any.whl

django_cfg/apps/knowbase/services/embedding/batch_processor.py

@@ -5,24 +5,27 @@ This module provides the main batch processing engine for generating
 embeddings with optimized API calls and database operations.
 """
 
-import time
 import logging
-from typing import List, Dict, Any, Optional
+import time
+from typing import List, Optional
 
-from django_cfg.apps.knowbase.utils.chunk_settings import get_embedding_batch_size, get_embedding_model
-from django_cfg.apps.knowbase.config.settings import get_openai_api_key, get_cache_settings
+from django_cfg.apps.knowbase.config.settings import get_cache_settings
+from django_cfg.apps.knowbase.utils.chunk_settings import (
+    get_embedding_batch_size,
+    get_embedding_model,
+)
 from django_cfg.modules.django_llm.llm.client import LLMClient
 
-from .models import ChunkData, EmbeddingResult, BatchProcessingResult, ProcessingConfig, ChunkType
-from .processors import DocumentChunkProcessor, ArchiveChunkProcessor, ExternalDataChunkProcessor
 from .batch_result import BatchResultBuilder
+from .models import BatchProcessingResult, ChunkData, ChunkType, EmbeddingResult
+from .processors import ArchiveChunkProcessor, DocumentChunkProcessor, ExternalDataChunkProcessor
 
 logger = logging.getLogger(__name__)
 
 
 class OptimizedEmbeddingProcessor:
     """High-performance embedding processor with batch operations."""
-    
+
     def __init__(self, batch_size: Optional[int] = None, embedding_model: Optional[str] = None):
         """
         Initialize the processor.
@@ -34,7 +37,7 @@ class OptimizedEmbeddingProcessor:
         # Use Constance settings if not provided
         self.batch_size = min(batch_size or get_embedding_batch_size(), 100)  # Conservative limit for stability
         self.embedding_model = embedding_model or get_embedding_model()
-        
+
         # Initialize LLM client with OpenAI only for embeddings
         # OpenRouter doesn't support embedding models, so we use OpenAI directly
         # Use auto-configured LLMClient with explicit OpenAI preference for embeddings
@@ -46,16 +49,16 @@ class OptimizedEmbeddingProcessor:
             cache_ttl=cache_settings.cache_ttl,
             max_cache_size=cache_settings.max_cache_size
         )
-        
+
         # Processors for different chunk types
         self.processors = {
             ChunkType.DOCUMENT: DocumentChunkProcessor(),
             ChunkType.ARCHIVE: ArchiveChunkProcessor(),
             ChunkType.EXTERNAL_DATA: ExternalDataChunkProcessor()
         }
-        
+
         logger.info(f"🚀 OptimizedEmbeddingProcessor initialized: batch_size={self.batch_size}, model={self.embedding_model}")
-    
+
     def process_chunks_batch(self, chunks: List[ChunkData]) -> BatchProcessingResult:
         """
         Process multiple chunks with optimized batch operations.
@@ -69,53 +72,53 @@ class OptimizedEmbeddingProcessor:
         start_time = time.time()
         total_chunks = len(chunks)
         result_builder = BatchResultBuilder(total_chunks)
-        
+
         logger.info(f"🔮 Starting batch processing of {total_chunks} chunks")
-        
+
         # Process in batches
         for i in range(0, total_chunks, self.batch_size):
             batch = chunks[i:i + self.batch_size]
             batch_num = (i // self.batch_size) + 1
             total_batches = (total_chunks + self.batch_size - 1) // self.batch_size
-            
+
             logger.info(f"🔮 Processing batch {batch_num}/{total_batches} ({len(batch)} chunks)")
-            
+
             try:
                 batch_results = self._process_single_batch(batch)
                 result_builder.add_batch_results(batch_results)
-                
+
                 # Small delay between batches to respect rate limits
                 if i + self.batch_size < total_chunks:
                     time.sleep(0.5)
-                    
+
             except Exception as e:
                 error_msg = f"Batch {batch_num} failed: {str(e)}"
                 logger.error(f"❌ {error_msg}")
                 result_builder.add_batch_error(error_msg, len(batch))
-        
+
         processing_time = time.time() - start_time
         result = result_builder.build(processing_time)
-        
+
         # Log using Pydantic model's summary
         summary = result.model_dump_summary()
         logger.info(f"🎉 Batch processing completed: {summary}")
         logger.info(f"📊 Performance: {summary['chunks_per_second']} chunks/sec, {summary['avg_cost_per_chunk']} per chunk")
-        
+
         return result
-    
+
     def _process_single_batch(self, batch: List[ChunkData]) -> List[EmbeddingResult]:
         """Process a single batch of chunks and return list of EmbeddingResult."""
-        
+
         # Prepare content for all chunks
         prepared_contents = []
         chunk_mapping = {}  # Map index to chunk
-        
+
         for idx, chunk in enumerate(batch):
             processor = self.processors.get(chunk.parent_type)
             if not processor:
                 logger.warning(f"⚠️ Unknown chunk type: {chunk.parent_type}")
                 continue
-            
+
             try:
                 content = processor.prepare_content_for_embedding(chunk)
                 if content and content.strip():
@@ -125,7 +128,7 @@ class OptimizedEmbeddingProcessor:
                     logger.warning(f"⚠️ Empty content for chunk {chunk.id}")
             except Exception as e:
                 logger.error(f"❌ Failed to prepare content for chunk {chunk.id}: {e}")
-        
+
         if not prepared_contents:
             # Return failed results for all chunks
             return [
@@ -136,24 +139,24 @@ class OptimizedEmbeddingProcessor:
                 )
                 for chunk in batch
             ]
-        
+
         # Generate embeddings in batch
         try:
             embedding_results = self._generate_batch_embeddings(prepared_contents)
-            
+
             # Process and save results
             final_results = []
-            
+
             for idx, embedding_result in enumerate(embedding_results):
                 if idx not in chunk_mapping:
                     continue
-                
+
                 chunk = chunk_mapping[idx]
                 processor = self.processors[chunk.parent_type]
-                
+
                 # Set the chunk_id in the result
                 embedding_result.chunk_id = chunk.id
-                
+
                 if embedding_result.success:
                     try:
                         logger.debug(f"🔄 Attempting to save embedding for chunk {chunk.id} (type: {chunk.parent_type})")
@@ -171,9 +174,9 @@ class OptimizedEmbeddingProcessor:
                         final_results.append(failed_result)
                 else:
                     final_results.append(embedding_result)
-            
+
             return final_results
-            
+
         except Exception as e:
             error_msg = f"Batch embedding generation failed: {e}"
             logger.error(f"❌ {error_msg}")
@@ -186,12 +189,12 @@ class OptimizedEmbeddingProcessor:
                 )
                 for chunk in batch
             ]
-    
+
     def _generate_batch_embeddings(self, contents: List[str]) -> List[EmbeddingResult]:
         """Generate embeddings for multiple contents using LLMClient."""
-        
+
         results = []
-        
+
         try:
             # Use LLMClient's generate_embedding method for each content
             # This handles both OpenAI and OpenRouter properly
@@ -202,7 +205,7 @@ class OptimizedEmbeddingProcessor:
                         text=content,
                         model=self.embedding_model
                     )
-                    
+
                     results.append(EmbeddingResult(
                         chunk_id="",  # Will be set by caller
                         embedding=embedding_response.embedding,
@@ -210,7 +213,7 @@ class OptimizedEmbeddingProcessor:
                         cost=embedding_response.cost,
                         success=True
                     ))
-                    
+
                 except Exception as e:
                     logger.error(f"❌ Failed to generate embedding for content {idx}: {e}")
                     results.append(EmbeddingResult(
@@ -221,19 +224,19 @@ class OptimizedEmbeddingProcessor:
                         success=False,
                         error=str(e)
                     ))
-            
+
             successful_count = len([r for r in results if r.success])
             logger.info(f"🎯 Generated {successful_count}/{len(results)} embeddings successfully")
-            
+
             # Log details of each result
             for i, result in enumerate(results):
                 if result.success:
                     logger.debug(f"  ✅ Result {i}: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
                 else:
                     logger.debug(f"  ❌ Result {i}: {result.error}")
-            
+
             return results
-            
+
         except Exception as e:
             logger.error(f"❌ Batch embedding generation failed: {e}")
             # Return failed results for all contents

django_cfg/apps/knowbase/services/embedding/batch_result.py

@@ -6,12 +6,13 @@ from individual batch operations without using raw dicts.
 """
 
 from typing import List
+
 from .models import BatchProcessingResult, EmbeddingResult
 
 
 class BatchResultBuilder:
     """Builder for BatchProcessingResult to avoid raw dict usage."""
-    
+
     def __init__(self, total_chunks: int):
         self.total_chunks = total_chunks
         self.successful_chunks = 0
@@ -20,7 +21,7 @@ class BatchResultBuilder:
         self.total_cost = 0.0
         self.errors: List[str] = []
         self.start_time: float = 0.0
-    
+
     def add_successful_result(self, result: EmbeddingResult) -> None:
         """Add a successful embedding result."""
         if result.success:
@@ -29,12 +30,12 @@ class BatchResultBuilder:
             self.total_cost += result.cost
         else:
             self.add_failed_result(result.error or "Unknown error")
-    
+
     def add_failed_result(self, error: str) -> None:
         """Add a failed result."""
         self.failed_chunks += 1
         self.errors.append(error)
-    
+
     def add_batch_results(self, results: List[EmbeddingResult]) -> None:
         """Add multiple results from a batch."""
         for result in results:
@@ -42,12 +43,12 @@ class BatchResultBuilder:
                 self.add_successful_result(result)
             else:
                 self.add_failed_result(result.error or "Unknown error")
-    
+
     def add_batch_error(self, error: str, chunk_count: int) -> None:
         """Add an error that affected an entire batch."""
         self.failed_chunks += chunk_count
         self.errors.append(error)
-    
+
     def build(self, processing_time: float) -> BatchProcessingResult:
         """Build the final BatchProcessingResult."""
         return BatchProcessingResult(

django_cfg/apps/knowbase/services/embedding/models.py

@@ -5,9 +5,10 @@ This module defines the core data structures used throughout
 the embedding processing pipeline using Pydantic for type safety.
 """
 
-from typing import List, Dict, Any, Optional
-from pydantic import BaseModel, Field, field_validator, ValidationInfo
 from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field, ValidationInfo, field_validator
 
 
 class ChunkType(str, Enum):
@@ -23,25 +24,25 @@ class ChunkData(BaseModel):
     id: str = Field(..., description="Unique chunk identifier")
     content: str = Field(..., min_length=1, description="Chunk content text")
     context_metadata: Optional[Dict[str, Any]] = Field(
-        default=None, 
+        default=None,
         description="Additional context metadata for the chunk"
     )
     parent_id: Optional[str] = Field(
-        default=None, 
+        default=None,
         description="ID of the parent document or archive"
     )
     parent_type: ChunkType = Field(
-        default=ChunkType.UNKNOWN, 
+        default=ChunkType.UNKNOWN,
         description="Type of parent content"
     )
-    
+
     @field_validator('content')
     @classmethod
     def content_must_not_be_empty(cls, v):
         if not v or not v.strip():
             raise ValueError('Content cannot be empty')
         return v.strip()
-    
+
     class Config:
         use_enum_values = True
 
@@ -50,33 +51,33 @@ class EmbeddingResult(BaseModel):
     """Result of embedding generation."""
     chunk_id: str = Field(..., description="ID of the processed chunk")
     embedding: List[float] = Field(
-        default_factory=list, 
+        default_factory=list,
         description="Generated embedding vector"
     )
     tokens: int = Field(
-        default=0, 
-        ge=0, 
+        default=0,
+        ge=0,
         description="Number of tokens used"
     )
     cost: float = Field(
-        default=0.0, 
-        ge=0.0, 
+        default=0.0,
+        ge=0.0,
         description="Processing cost in USD"
     )
     success: bool = Field(
-        default=True, 
+        default=True,
         description="Whether embedding generation was successful"
     )
     error: Optional[str] = Field(
-        default=None, 
+        default=None,
         description="Error message if processing failed"
     )
     processing_time: Optional[float] = Field(
-        default=None, 
-        ge=0.0, 
+        default=None,
+        ge=0.0,
         description="Time taken to process this chunk in seconds"
     )
-    
+
     @field_validator('embedding')
     @classmethod
     def validate_embedding_dimension(cls, v):
@@ -84,7 +85,7 @@ class EmbeddingResult(BaseModel):
             # Warning, not error - allow different dimensions
             pass
         return v
-    
+
     class Config:
         validate_assignment = True
 
@@ -92,40 +93,40 @@ class EmbeddingResult(BaseModel):
 class BatchProcessingResult(BaseModel):
     """Result of batch processing."""
     total_chunks: int = Field(
-        ..., 
-        ge=0, 
+        ...,
+        ge=0,
         description="Total number of chunks processed"
     )
     successful_chunks: int = Field(
-        ..., 
-        ge=0, 
+        ...,
+        ge=0,
         description="Number of successfully processed chunks"
     )
     failed_chunks: int = Field(
-        ..., 
-        ge=0, 
+        ...,
+        ge=0,
         description="Number of failed chunks"
     )
     total_tokens: int = Field(
-        default=0, 
-        ge=0, 
+        default=0,
+        ge=0,
         description="Total tokens used across all chunks"
     )
     total_cost: float = Field(
-        default=0.0, 
-        ge=0.0, 
+        default=0.0,
+        ge=0.0,
         description="Total processing cost in USD"
     )
     processing_time: float = Field(
-        ..., 
-        ge=0.0, 
+        ...,
+        ge=0.0,
         description="Total processing time in seconds"
     )
     errors: List[str] = Field(
-        default_factory=list, 
+        default_factory=list,
         description="List of error messages"
     )
-    
+
     # Computed properties
     @property
     def success_rate(self) -> float:
@@ -133,28 +134,28 @@ class BatchProcessingResult(BaseModel):
         if self.total_chunks == 0:
             return 0.0
         return (self.successful_chunks / self.total_chunks) * 100.0
-    
+
     @property
     def chunks_per_second(self) -> float:
         """Calculate processing speed."""
         if self.processing_time == 0:
             return 0.0
         return self.total_chunks / self.processing_time
-    
+
     @property
     def average_cost_per_chunk(self) -> float:
         """Calculate average cost per successfully processed chunk."""
         if self.successful_chunks == 0:
             return 0.0
         return self.total_cost / self.successful_chunks
-    
+
     @property
     def average_tokens_per_chunk(self) -> float:
         """Calculate average tokens per successfully processed chunk."""
         if self.successful_chunks == 0:
             return 0.0
         return self.total_tokens / self.successful_chunks
-    
+
     @field_validator('successful_chunks', 'failed_chunks')
     @classmethod
     def validate_chunk_counts(cls, v, info: ValidationInfo):
@@ -175,10 +176,10 @@ class BatchProcessingResult(BaseModel):
                     f'must equal total chunks ({info.data["total_chunks"]})'
                 )
         return v
-    
+
     class Config:
         validate_assignment = True
-        
+
     def model_dump_summary(self) -> Dict[str, Any]:
         """Get a summary dict for logging."""
         return {
@@ -198,9 +199,9 @@ class BatchProcessingResult(BaseModel):
 class ProcessingConfig(BaseModel):
     """Configuration for embedding processing."""
     batch_size: int = Field(
-        default=100, 
-        ge=1, 
-        le=2048, 
+        default=100,
+        ge=1,
+        le=2048,
         description="Number of chunks to process in one batch"
     )
     embedding_model: str = Field(
@@ -208,26 +209,26 @@ class ProcessingConfig(BaseModel):
         description="OpenAI embedding model to use"
     )
     max_retries: int = Field(
-        default=3, 
-        ge=0, 
-        le=10, 
+        default=3,
+        ge=0,
+        le=10,
         description="Maximum number of retries for failed requests"
     )
     retry_delay: float = Field(
-        default=1.0, 
-        ge=0.0, 
+        default=1.0,
+        ge=0.0,
         description="Delay between retries in seconds"
     )
     rate_limit_delay: float = Field(
-        default=0.5, 
-        ge=0.0, 
+        default=0.5,
+        ge=0.0,
         description="Delay between batches to respect rate limits"
     )
     timeout_seconds: int = Field(
-        default=60, 
-        ge=1, 
+        default=60,
+        ge=1,
         description="Timeout for API requests in seconds"
     )
-    
+
     class Config:
         validate_assignment = True

django_cfg/apps/knowbase/services/embedding/processors.py

@@ -8,7 +8,8 @@ document and archive chunks with their specific requirements.
 import logging
 from typing import Protocol
 
-from django_cfg.apps.knowbase.models import DocumentChunk, ArchiveItemChunk, ExternalDataChunk
+from django_cfg.apps.knowbase.models import ArchiveItemChunk, DocumentChunk, ExternalDataChunk
+
 from .models import ChunkData, EmbeddingResult
 
 logger = logging.getLogger(__name__)
@@ -16,11 +17,11 @@ logger = logging.getLogger(__name__)
 
 class ChunkProcessor(Protocol):
     """Protocol for chunk processors."""
-    
+
     def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
         """Prepare chunk content for embedding generation."""
         ...
-    
+
     def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
         """Save embedding result to database."""
         ...
@@ -28,25 +29,25 @@ class ChunkProcessor(Protocol):
 
 class DocumentChunkProcessor:
     """Processor for document chunks."""
-    
+
     def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
         """Prepare document chunk content for embedding."""
         return chunk.content.strip()
-    
+
     def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
         """Save embedding result for document chunk."""
         try:
             logger.debug(f"🔍 Looking for document chunk with id: {chunk_id}")
             chunk = DocumentChunk.objects.get(id=chunk_id)
             logger.debug(f"📄 Found document chunk: {chunk.id}, current embedding length: {len(chunk.embedding) if chunk.embedding is not None and len(chunk.embedding) > 0 else 0}")
-            
+
             chunk.embedding = result.embedding
             chunk.token_count = result.tokens
             chunk.embedding_cost = result.cost
             chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
-            
+
             logger.info(f"✅ Document chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
-            
+
         except DocumentChunk.DoesNotExist:
             logger.error(f"❌ Document chunk {chunk_id} not found")
             raise
@@ -57,15 +58,15 @@ class DocumentChunkProcessor:
 
 class ArchiveChunkProcessor:
     """Processor for archive chunks."""
-    
+
     def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
         """Prepare archive chunk content for embedding with context."""
         content = chunk.content
         context = chunk.context_metadata or {}
-        
+
         # Build context prefix for better embeddings
         context_parts = []
-        
+
         if context.get('file_path'):
             context_parts.append(f"File: {context['file_path']}")
         if context.get('function_name'):
@@ -74,13 +75,13 @@ class ArchiveChunkProcessor:
             context_parts.append(f"Class: {context['class_name']}")
         if context.get('language'):
             context_parts.append(f"Language: {context['language']}")
-        
+
         if context_parts:
             context_prefix = " | ".join(context_parts)
             enhanced_content = f"{context_prefix}\n\n{content}"
         else:
             enhanced_content = content
-        
+
         # Ensure content is not too long for embedding model
         max_length = 8000  # Conservative limit
         if len(enhanced_content) > max_length:
@@ -97,9 +98,9 @@ class ArchiveChunkProcessor:
             else:
                 # Subtract 3 for "..." suffix
                 enhanced_content = content[:max_length - 3] + "..."
-        
+
         return enhanced_content.strip()
-    
+
     def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
         """Save embedding result for archive chunk."""
         try:
@@ -108,15 +109,15 @@ class ArchiveChunkProcessor:
             chunk.token_count = result.tokens
             chunk.embedding_cost = result.cost
             chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
-            
+
             # Update parent item statistics
             item = chunk.item
             item.total_tokens += result.tokens
             item.processing_cost += result.cost
             item.save(update_fields=['total_tokens', 'processing_cost'])
-            
+
             logger.debug(f"✅ Archive chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
-            
+
         except ArchiveItemChunk.DoesNotExist:
             logger.error(f"❌ Archive chunk {chunk_id} not found")
             raise
@@ -124,25 +125,25 @@ class ArchiveChunkProcessor:
 
 class ExternalDataChunkProcessor:
     """Processor for external data chunks."""
-    
+
     def prepare_content_for_embedding(self, chunk: ChunkData) -> str:
         """Prepare external data chunk content for embedding."""
         return chunk.content.strip()
-    
+
     def save_embedding_result(self, chunk_id: str, result: EmbeddingResult) -> None:
         """Save embedding result for external data chunk."""
         try:
             logger.debug(f"🔍 Looking for external data chunk with id: {chunk_id}")
             chunk = ExternalDataChunk.objects.get(id=chunk_id)
             logger.debug(f"🔗 Found external data chunk: {chunk.id}, current embedding length: {len(chunk.embedding) if chunk.embedding is not None and len(chunk.embedding) > 0 else 0}")
-            
+
             chunk.embedding = result.embedding
             chunk.token_count = result.tokens
             chunk.embedding_cost = result.cost
             chunk.save(update_fields=['embedding', 'token_count', 'embedding_cost'])
-            
+
             logger.info(f"✅ External data chunk {chunk_id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
-            
+
         except ExternalDataChunk.DoesNotExist:
             logger.error(f"❌ External data chunk {chunk_id} not found")
             raise