django-cfg 1.1.81__py3-none-any.whl → 1.2.0__py3-none-any.whl

django_cfg/apps/knowbase/services/document_service.py

@@ -0,0 +1,144 @@
+"""
+Document management service.
+"""
+
+from typing import List, Optional, Dict, Any
+from django.db import transaction, models
+from django.utils import timezone
+from ..models import Document, DocumentChunk, ProcessingStatus
+from .base import BaseService
+
+
+class DocumentService(BaseService):
+    """Service for document management and processing."""
+    
+    def create_document(
+        self,
+        title: str,
+        content: str,
+        file_type: str = "text/plain",
+        metadata: Optional[Dict[str, Any]] = None
+    ) -> Document:
+        """Create document and trigger async processing."""
+        
+        # Generate content hash for duplicate detection
+        content_hash = self._generate_content_hash(content)
+        
+        # Check for duplicates
+        existing = Document.objects.filter(
+            user=self.user,
+            content_hash=content_hash
+        ).first()
+        
+        if existing:
+            raise ValueError(f"Document with same content already exists: {existing.title}")
+        
+        # Create document (async processing will be triggered by post_save signal)
+        document = Document.objects.create(
+            user=self.user,
+            title=title,
+            content=content,
+            content_hash=content_hash,
+            file_type=file_type,
+            file_size=len(content.encode('utf-8')),
+            metadata=metadata or {},
+            processing_status=ProcessingStatus.PENDING
+        )
+        
+        return document
+    
+    def get_document(self, document_id: str) -> Optional[Document]:
+        """Get document by ID with user access check."""
+        try:
+            document = Document.objects.get(
+                id=document_id,
+                user=self.user
+            )
+            return document
+        except Document.DoesNotExist:
+            return None
+    
+    def get_user_documents(self, status: Optional[str] = None):
+        """Get user documents queryset with filtering."""
+        queryset = Document.objects.filter(user=self.user)
+        
+        if status:
+            queryset = queryset.filter(processing_status=status)
+        
+        return queryset.order_by('-created_at')
+    
+    def list_documents(
+        self,
+        status: Optional[str] = None,
+        limit: int = 20,
+        offset: int = 0
+    ) -> List[Document]:
+        """List user documents with filtering."""
+        queryset = self.get_user_documents(status)
+        return list(queryset[offset:offset + limit])
+    
+    def delete_document(self, document_id: str) -> bool:
+        """Delete document and all associated chunks."""
+        try:
+            with transaction.atomic():
+                document = Document.objects.get(
+                    id=document_id,
+                    user=self.user
+                )
+                
+                # Delete associated chunks first
+                DocumentChunk.objects.filter(document=document).delete()
+                
+                # Delete document
+                document.delete()
+                
+                return True
+        except Document.DoesNotExist:
+            return False
+    
+    def get_processing_stats(self) -> Dict[str, Any]:
+        """Get user's document processing statistics."""
+        
+        from django.db.models import Count, Sum, Avg
+        
+        stats = Document.objects.filter(user=self.user).aggregate(
+            total_documents=Count('id'),
+            completed_documents=Count('id', filter=models.Q(processing_status=ProcessingStatus.COMPLETED)),
+            total_chunks=Sum('chunks_count'),
+            total_tokens=Sum('total_tokens'),
+            total_cost=Sum('total_cost_usd'),
+        )
+        
+        return {
+            'total_documents': stats['total_documents'] or 0,
+            'completed_documents': stats['completed_documents'] or 0,
+            'processing_success_rate': (
+                (stats['completed_documents'] / stats['total_documents'] * 100)
+                if stats['total_documents'] > 0 else 0
+            ),
+            'total_chunks': stats['total_chunks'] or 0,
+            'total_tokens': stats['total_tokens'] or 0,
+            'total_cost_usd': float(stats['total_cost'] or 0),
+            'avg_processing_time_seconds': 0.0  # Calculated separately if needed
+        }
+    
+    def reprocess_document(self, document_id: str) -> bool:
+        """Trigger document reprocessing."""
+        try:
+            document = Document.objects.get(
+                id=document_id,
+                user=self.user
+            )
+            
+            # Reset processing status
+            document.processing_status = ProcessingStatus.PENDING
+            document.processing_error = ""
+            document.save()
+            
+            # Trigger async reprocessing
+            from ..tasks import reprocess_document_chunks
+            reprocess_document_chunks.send(str(document.id))
+            
+            return True
+        except Document.DoesNotExist:
+            return False

django_cfg/apps/knowbase/services/embedding/__init__.py

@@ -0,0 +1,43 @@
+"""
+Embedding processing services.
+
+This package provides high-performance embedding generation services
+for documents and archives with batch processing and async support.
+"""
+
+from .models import ChunkData, EmbeddingResult, BatchProcessingResult, ProcessingConfig, ChunkType
+from .processors import DocumentChunkProcessor, ArchiveChunkProcessor, ExternalDataChunkProcessor
+from .batch_processor import OptimizedEmbeddingProcessor
+from .async_processor import AsyncOptimizedEmbeddingProcessor
+from .batch_result import BatchResultBuilder
+from .utils import (
+    process_document_chunks_optimized,
+    process_archive_chunks_optimized,
+    process_external_data_chunks_optimized,
+    process_chunks_context_aware,
+)
+
+__all__ = [
+    # Data models
+    "ChunkData",
+    "EmbeddingResult", 
+    "BatchProcessingResult",
+    "ProcessingConfig",
+    "ChunkType",
+    
+    # Processors
+    "DocumentChunkProcessor",
+    "ArchiveChunkProcessor",
+    "ExternalDataChunkProcessor",
+    "OptimizedEmbeddingProcessor",
+    "AsyncOptimizedEmbeddingProcessor",
+    
+    # Utilities
+    "BatchResultBuilder",
+    
+    # Convenience functions
+    "process_document_chunks_optimized",
+    "process_archive_chunks_optimized",
+    "process_external_data_chunks_optimized",
+    "process_chunks_context_aware",
+]

django_cfg/apps/knowbase/services/embedding/async_processor.py

@@ -0,0 +1,244 @@
+"""
+Async-compatible embedding processor for Django 5.2.
+
+This module provides async/sync compatibility for embedding generation,
+following Django 5.2 async best practices.
+"""
+
+import asyncio
+import time
+import logging
+from typing import List, Dict, Any, Optional, Union
+
+from asgiref.sync import sync_to_async, async_to_sync
+from django.db import transaction
+
+from django_cfg.apps.knowbase.models import DocumentChunk, ArchiveItemChunk
+from django_cfg.apps.knowbase.utils.chunk_settings import get_embedding_batch_size, get_embedding_model
+
+from .models import ChunkData, BatchProcessingResult, EmbeddingResult
+from .batch_processor import OptimizedEmbeddingProcessor
+
+logger = logging.getLogger(__name__)
+
+
+def is_async_context() -> bool:
+    """Detect current execution context."""
+    try:
+        asyncio.get_running_loop()
+        return True
+    except RuntimeError:
+        return False
+
+
+class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
+    """
+    Async-compatible embedding processor that works in both sync and async contexts.
+    
+    Based on Django 5.2 async patterns:
+    - Context-aware operations
+    - Proper async/sync method selection
+    - Compatible with both WSGI and ASGI
+    """
+    
+    def __init__(self, batch_size: Optional[int] = None, embedding_model: Optional[str] = None):
+        """Initialize async-compatible processor."""
+        super().__init__(batch_size, embedding_model)
+        logger.info(f"🚀 AsyncOptimizedEmbeddingProcessor initialized: async_context={is_async_context()}")
+    
+    async def aprocess_chunks_batch(self, chunks: List[ChunkData]) -> BatchProcessingResult:
+        """
+        Async version of batch processing.
+        
+        Uses Django 5.2 async ORM methods (a-prefixed) for database operations.
+        """
+        start_time = time.time()
+        total_chunks = len(chunks)
+        successful_chunks = 0
+        failed_chunks = 0
+        total_tokens = 0
+        total_cost = 0.0
+        errors = []
+        
+        logger.info(f"🔮 Starting async batch processing of {total_chunks} chunks")
+        
+        # Process in batches
+        for i in range(0, total_chunks, self.batch_size):
+            batch = chunks[i:i + self.batch_size]
+            batch_num = (i // self.batch_size) + 1
+            total_batches = (total_chunks + self.batch_size - 1) // self.batch_size
+            
+            logger.info(f"🔮 Processing async batch {batch_num}/{total_batches} ({len(batch)} chunks)")
+            
+            try:
+                batch_result = await self._aprocess_single_batch(batch)
+                
+                successful_chunks += batch_result['successful']
+                failed_chunks += batch_result['failed']
+                total_tokens += batch_result['tokens']
+                total_cost += batch_result['cost']
+                errors.extend(batch_result['errors'])
+                
+                # Small async delay between batches
+                if i + self.batch_size < total_chunks:
+                    await asyncio.sleep(0.5)
+                    
+            except Exception as e:
+                error_msg = f"Async batch {batch_num} failed: {str(e)}"
+                logger.error(f"❌ {error_msg}")
+                errors.append(error_msg)
+                failed_chunks += len(batch)
+        
+        processing_time = time.time() - start_time
+        
+        result = BatchProcessingResult(
+            total_chunks=total_chunks,
+            successful_chunks=successful_chunks,
+            failed_chunks=failed_chunks,
+            total_tokens=total_tokens,
+            total_cost=total_cost,
+            processing_time=processing_time,
+            errors=errors
+        )
+        
+        logger.info(
+            f"🎉 Async batch processing completed: {successful_chunks}/{total_chunks} successful, "
+            f"{total_tokens} tokens, ${total_cost:.4f} cost, {processing_time:.2f}s"
+        )
+        
+        return result
+    
+    async def _aprocess_single_batch(self, batch: List[ChunkData]) -> Dict[str, Any]:
+        """Async version of single batch processing."""
+        
+        # Prepare content for all chunks (sync operation)
+        prepared_contents = []
+        chunk_mapping = {}
+        
+        for idx, chunk in enumerate(batch):
+            processor = self.processors.get(chunk.parent_type)
+            if not processor:
+                logger.warning(f"⚠️ Unknown chunk type: {chunk.parent_type}")
+                continue
+            
+            try:
+                content = processor.prepare_content_for_embedding(chunk)
+                if content and content.strip():
+                    prepared_contents.append(content)
+                    chunk_mapping[len(prepared_contents) - 1] = chunk
+                else:
+                    logger.warning(f"⚠️ Empty content for chunk {chunk.id}")
+            except Exception as e:
+                logger.error(f"❌ Failed to prepare content for chunk {chunk.id}: {e}")
+        
+        if not prepared_contents:
+            return {
+                'successful': 0,
+                'failed': len(batch),
+                'tokens': 0,
+                'cost': 0.0,
+                'errors': ['No valid content to process']
+            }
+        
+        # Generate embeddings (sync operation - OpenAI client is sync)
+        try:
+            embedding_results = self._generate_batch_embeddings(prepared_contents)
+            
+            # Save results using async database operations
+            successful = 0
+            failed = 0
+            total_tokens = 0
+            total_cost = 0.0
+            errors = []
+            
+            for idx, embedding_result in enumerate(embedding_results):
+                if idx not in chunk_mapping:
+                    continue
+                
+                chunk = chunk_mapping[idx]
+                
+                if embedding_result.success:
+                    try:
+                        await self._asave_embedding_result(chunk, embedding_result)
+                        successful += 1
+                        total_tokens += embedding_result.tokens
+                        total_cost += embedding_result.cost
+                    except Exception as e:
+                        error_msg = f"Failed to save async embedding for chunk {chunk.id}: {e}"
+                        logger.error(f"❌ {error_msg}")
+                        errors.append(error_msg)
+                        failed += 1
+                else:
+                    errors.append(embedding_result.error or f"Failed to generate embedding for chunk {chunk.id}")
+                    failed += 1
+            
+            return {
+                'successful': successful,
+                'failed': failed,
+                'tokens': total_tokens,
+                'cost': total_cost,
+                'errors': errors
+            }
+            
+        except Exception as e:
+            error_msg = f"Async batch embedding generation failed: {e}"
+            logger.error(f"❌ {error_msg}")
+            return {
+                'successful': 0,
+                'failed': len(batch),
+                'tokens': 0,
+                'cost': 0.0,
+                'errors': [error_msg]
+            }
+    
+    async def _asave_embedding_result(self, chunk: ChunkData, result: EmbeddingResult) -> None:
+        """Save embedding result using async database operations."""
+        
+        try:
+            if chunk.parent_type == "document":
+                # Use Django 5.2 async ORM methods
+                chunk_obj = await DocumentChunk.objects.aget(id=chunk.id)
+                chunk_obj.embedding = result.embedding
+                chunk_obj.token_count = result.tokens
+                chunk_obj.embedding_cost = result.cost
+                await chunk_obj.asave(update_fields=['embedding', 'token_count', 'embedding_cost'])
+                
+                logger.debug(f"✅ Async document chunk {chunk.id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
+                
+            elif chunk.parent_type == "archive":
+                # Use async ORM with select_related
+                chunk_obj = await ArchiveItemChunk.objects.select_related('item').aget(id=chunk.id)
+                chunk_obj.embedding = result.embedding
+                chunk_obj.token_count = result.tokens
+                chunk_obj.embedding_cost = result.cost
+                await chunk_obj.asave(update_fields=['embedding', 'token_count', 'embedding_cost'])
+                
+                # Update parent item statistics
+                item = chunk_obj.item
+                item.total_tokens += result.tokens
+                item.processing_cost += result.cost
+                await item.asave(update_fields=['total_tokens', 'processing_cost'])
+                
+                logger.debug(f"✅ Async archive chunk {chunk.id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
+            else:
+                raise ValueError(f"Unknown chunk type: {chunk.parent_type}")
+                
+        except Exception as e:
+            logger.error(f"❌ Failed to save async embedding for chunk {chunk.id}: {e}")
+            raise
+    
+    def process_chunks_batch_context_aware(self, chunks: List[ChunkData]) -> BatchProcessingResult:
+        """
+        Context-aware processing that works in both sync and async contexts.
+        
+        Based on Django 5.2 async patterns.
+        """
+        if is_async_context():
+            # We're in async context - use async methods
+            logger.info("🔮 Detected async context - using async processing")
+            # Convert async method to sync for compatibility
+            return async_to_sync(self.aprocess_chunks_batch)(chunks)
+        else:
+            # We're in sync context - use sync methods
+            logger.info("🔮 Detected sync context - using sync processing")
+            return super().process_chunks_batch(chunks)

django_cfg/apps/knowbase/services/embedding/batch_processor.py

@@ -0,0 +1,250 @@
+"""
+High-performance batch embedding processor.
+
+This module provides the main batch processing engine for generating
+embeddings with optimized API calls and database operations.
+"""
+
+import time
+import logging
+from typing import List, Dict, Any, Optional
+
+from django_cfg.apps.knowbase.utils.chunk_settings import get_embedding_batch_size, get_embedding_model
+from django_cfg.apps.knowbase.config.settings import get_openai_api_key, get_cache_settings
+from django_cfg.modules.django_llm.llm.client import LLMClient
+
+from .models import ChunkData, EmbeddingResult, BatchProcessingResult, ProcessingConfig, ChunkType
+from .processors import DocumentChunkProcessor, ArchiveChunkProcessor, ExternalDataChunkProcessor
+from .batch_result import BatchResultBuilder
+
+logger = logging.getLogger(__name__)
+
+
+class OptimizedEmbeddingProcessor:
+    """High-performance embedding processor with batch operations."""
+    
+    def __init__(self, batch_size: Optional[int] = None, embedding_model: Optional[str] = None):
+        """
+        Initialize the processor.
+        
+        Args:
+            batch_size: Number of chunks to process in one API call (uses Constance setting if None)
+            embedding_model: Embedding model to use (uses Constance setting if None)
+        """
+        # Use Constance settings if not provided
+        self.batch_size = min(batch_size or get_embedding_batch_size(), 100)  # Conservative limit for stability
+        self.embedding_model = embedding_model or get_embedding_model()
+        
+        # Initialize LLM client with OpenAI only for embeddings
+        # OpenRouter doesn't support embedding models, so we use OpenAI directly
+        # Use auto-configured LLMClient with explicit OpenAI preference for embeddings
+        # Get cache settings from configuration (directory is auto-created)
+        cache_settings = get_cache_settings()
+        self.llm_client = LLMClient(
+            preferred_provider="openai",  # Force OpenAI for embeddings
+            cache_dir=cache_settings.cache_dir,
+            cache_ttl=cache_settings.cache_ttl,
+            max_cache_size=cache_settings.max_cache_size
+        )
+        
+        # Processors for different chunk types
+        self.processors = {
+            ChunkType.DOCUMENT: DocumentChunkProcessor(),
+            ChunkType.ARCHIVE: ArchiveChunkProcessor(),
+            ChunkType.EXTERNAL_DATA: ExternalDataChunkProcessor()
+        }
+        
+        logger.info(f"🚀 OptimizedEmbeddingProcessor initialized: batch_size={self.batch_size}, model={self.embedding_model}")
+    
+    def process_chunks_batch(self, chunks: List[ChunkData]) -> BatchProcessingResult:
+        """
+        Process multiple chunks with optimized batch operations.
+        
+        Args:
+            chunks: List of chunks to process
+            
+        Returns:
+            BatchProcessingResult with processing statistics
+        """
+        start_time = time.time()
+        total_chunks = len(chunks)
+        result_builder = BatchResultBuilder(total_chunks)
+        
+        logger.info(f"🔮 Starting batch processing of {total_chunks} chunks")
+        
+        # Process in batches
+        for i in range(0, total_chunks, self.batch_size):
+            batch = chunks[i:i + self.batch_size]
+            batch_num = (i // self.batch_size) + 1
+            total_batches = (total_chunks + self.batch_size - 1) // self.batch_size
+            
+            logger.info(f"🔮 Processing batch {batch_num}/{total_batches} ({len(batch)} chunks)")
+            
+            try:
+                batch_results = self._process_single_batch(batch)
+                result_builder.add_batch_results(batch_results)
+                
+                # Small delay between batches to respect rate limits
+                if i + self.batch_size < total_chunks:
+                    time.sleep(0.5)
+                    
+            except Exception as e:
+                error_msg = f"Batch {batch_num} failed: {str(e)}"
+                logger.error(f"❌ {error_msg}")
+                result_builder.add_batch_error(error_msg, len(batch))
+        
+        processing_time = time.time() - start_time
+        result = result_builder.build(processing_time)
+        
+        # Log using Pydantic model's summary
+        summary = result.model_dump_summary()
+        logger.info(f"🎉 Batch processing completed: {summary}")
+        logger.info(f"📊 Performance: {summary['chunks_per_second']} chunks/sec, {summary['avg_cost_per_chunk']} per chunk")
+        
+        return result
+    
+    def _process_single_batch(self, batch: List[ChunkData]) -> List[EmbeddingResult]:
+        """Process a single batch of chunks and return list of EmbeddingResult."""
+        
+        # Prepare content for all chunks
+        prepared_contents = []
+        chunk_mapping = {}  # Map index to chunk
+        
+        for idx, chunk in enumerate(batch):
+            processor = self.processors.get(chunk.parent_type)
+            if not processor:
+                logger.warning(f"⚠️ Unknown chunk type: {chunk.parent_type}")
+                continue
+            
+            try:
+                content = processor.prepare_content_for_embedding(chunk)
+                if content and content.strip():
+                    prepared_contents.append(content)
+                    chunk_mapping[len(prepared_contents) - 1] = chunk
+                else:
+                    logger.warning(f"⚠️ Empty content for chunk {chunk.id}")
+            except Exception as e:
+                logger.error(f"❌ Failed to prepare content for chunk {chunk.id}: {e}")
+        
+        if not prepared_contents:
+            # Return failed results for all chunks
+            return [
+                EmbeddingResult(
+                    chunk_id=chunk.id,
+                    success=False,
+                    error="No valid content to process"
+                )
+                for chunk in batch
+            ]
+        
+        # Generate embeddings in batch
+        try:
+            embedding_results = self._generate_batch_embeddings(prepared_contents)
+            
+            # Process and save results
+            final_results = []
+            
+            for idx, embedding_result in enumerate(embedding_results):
+                if idx not in chunk_mapping:
+                    continue
+                
+                chunk = chunk_mapping[idx]
+                processor = self.processors[chunk.parent_type]
+                
+                # Set the chunk_id in the result
+                embedding_result.chunk_id = chunk.id
+                
+                if embedding_result.success:
+                    try:
+                        logger.debug(f"🔄 Attempting to save embedding for chunk {chunk.id} (type: {chunk.parent_type})")
+                        processor.save_embedding_result(chunk.id, embedding_result)
+                        logger.info(f"✅ Successfully saved embedding for chunk {chunk.id}")
+                        final_results.append(embedding_result)
+                    except Exception as e:
+                        error_msg = f"Failed to save embedding for chunk {chunk.id}: {e}"
+                        logger.error(f"❌ {error_msg}")
+                        failed_result = EmbeddingResult(
+                            chunk_id=chunk.id,
+                            success=False,
+                            error=error_msg
+                        )
+                        final_results.append(failed_result)
+                else:
+                    final_results.append(embedding_result)
+            
+            return final_results
+            
+        except Exception as e:
+            error_msg = f"Batch embedding generation failed: {e}"
+            logger.error(f"❌ {error_msg}")
+            # Return failed results for all chunks
+            return [
+                EmbeddingResult(
+                    chunk_id=chunk.id,
+                    success=False,
+                    error=error_msg
+                )
+                for chunk in batch
+            ]
+    
+    def _generate_batch_embeddings(self, contents: List[str]) -> List[EmbeddingResult]:
+        """Generate embeddings for multiple contents using LLMClient."""
+        
+        results = []
+        
+        try:
+            # Use LLMClient's generate_embedding method for each content
+            # This handles both OpenAI and OpenRouter properly
+            for idx, content in enumerate(contents):
+                try:
+                    # Use LLMClient's method which handles provider differences
+                    embedding_response = self.llm_client.generate_embedding(
+                        text=content,
+                        model=self.embedding_model
+                    )
+                    
+                    results.append(EmbeddingResult(
+                        chunk_id="",  # Will be set by caller
+                        embedding=embedding_response.embedding,
+                        tokens=embedding_response.tokens,
+                        cost=embedding_response.cost,
+                        success=True
+                    ))
+                    
+                except Exception as e:
+                    logger.error(f"❌ Failed to generate embedding for content {idx}: {e}")
+                    results.append(EmbeddingResult(
+                        chunk_id="",
+                        embedding=[],
+                        tokens=0,
+                        cost=0.0,
+                        success=False,
+                        error=str(e)
+                    ))
+            
+            successful_count = len([r for r in results if r.success])
+            logger.info(f"🎯 Generated {successful_count}/{len(results)} embeddings successfully")
+            
+            # Log details of each result
+            for i, result in enumerate(results):
+                if result.success:
+                    logger.debug(f"  ✅ Result {i}: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
+                else:
+                    logger.debug(f"  ❌ Result {i}: {result.error}")
+            
+            return results
+            
+        except Exception as e:
+            logger.error(f"❌ Batch embedding generation failed: {e}")
+            # Return failed results for all contents
+            return [
+                EmbeddingResult(
+                    chunk_id="",
+                    embedding=[],
+                    tokens=0,
+                    cost=0.0,
+                    success=False,
+                    error=str(e)
+                )
+                for _ in contents
+            ]