django-cfg 1.4.46__py3-none-any.whl → 1.5.181__py3-none-any.whl

django_cfg/apps/knowbase/services/document_service.py

@@ -1,145 +0,0 @@
-"""
-Document management service.
-"""
-
-from typing import Any, Dict, List, Optional
-
-from django.db import models, transaction
-
-from ..models import Document, DocumentChunk, ProcessingStatus
-from .base import BaseService
-
-
-class DocumentService(BaseService):
-    """Service for document management and processing."""
-
-    def create_document(
-        self,
-        title: str,
-        content: str,
-        file_type: str = "text/plain",
-        metadata: Optional[Dict[str, Any]] = None
-    ) -> Document:
-        """Create document and trigger async processing."""
-
-        # Generate content hash for duplicate detection
-        content_hash = self._generate_content_hash(content)
-
-        # Check for duplicates
-        existing = Document.objects.filter(
-            user=self.user,
-            content_hash=content_hash
-        ).first()
-
-        if existing:
-            raise ValueError(f"Document with same content already exists: {existing.title}")
-
-        # Create document (async processing will be triggered by post_save signal)
-        document = Document.objects.create(
-            user=self.user,
-            title=title,
-            content=content,
-            content_hash=content_hash,
-            file_type=file_type,
-            file_size=len(content.encode('utf-8')),
-            metadata=metadata or {},
-            processing_status=ProcessingStatus.PENDING
-        )
-
-        return document
-
-    def get_document(self, document_id: str) -> Optional[Document]:
-        """Get document by ID with user access check."""
-        try:
-            document = Document.objects.get(
-                id=document_id,
-                user=self.user
-            )
-            return document
-        except Document.DoesNotExist:
-            return None
-
-    def get_user_documents(self, status: Optional[str] = None):
-        """Get user documents queryset with filtering."""
-        queryset = Document.objects.filter(user=self.user)
-
-        if status:
-            queryset = queryset.filter(processing_status=status)
-
-        return queryset.order_by('-created_at')
-
-    def list_documents(
-        self,
-        status: Optional[str] = None,
-        limit: int = 20,
-        offset: int = 0
-    ) -> List[Document]:
-        """List user documents with filtering."""
-        queryset = self.get_user_documents(status)
-        return list(queryset[offset:offset + limit])
-
-    def delete_document(self, document_id: str) -> bool:
-        """Delete document and all associated chunks."""
-        try:
-            with transaction.atomic():
-                document = Document.objects.get(
-                    id=document_id,
-                    user=self.user
-                )
-
-                # Delete associated chunks first
-                DocumentChunk.objects.filter(document=document).delete()
-
-                # Delete document
-                document.delete()
-
-                return True
-        except Document.DoesNotExist:
-            return False
-
-    def get_processing_stats(self) -> Dict[str, Any]:
-        """Get user's document processing statistics."""
-
-        from django.db.models import Count, Sum
-
-        stats = Document.objects.filter(user=self.user).aggregate(
-            total_documents=Count('id'),
-            completed_documents=Count('id', filter=models.Q(processing_status=ProcessingStatus.COMPLETED)),
-            total_chunks=Sum('chunks_count'),
-            total_tokens=Sum('total_tokens'),
-            total_cost=Sum('total_cost_usd'),
-        )
-
-        return {
-            'total_documents': stats['total_documents'] or 0,
-            'completed_documents': stats['completed_documents'] or 0,
-            'processing_success_rate': (
-                (stats['completed_documents'] / stats['total_documents'] * 100)
-                if stats['total_documents'] > 0 else 0
-            ),
-            'total_chunks': stats['total_chunks'] or 0,
-            'total_tokens': stats['total_tokens'] or 0,
-            'total_cost_usd': float(stats['total_cost'] or 0),
-            'avg_processing_time_seconds': 0.0  # Calculated separately if needed
-        }
-
-    def reprocess_document(self, document_id: str) -> bool:
-        """Trigger document reprocessing."""
-        try:
-            document = Document.objects.get(
-                id=document_id,
-                user=self.user
-            )
-
-            # Reset processing status
-            document.processing_status = ProcessingStatus.PENDING
-            document.processing_error = ""
-            document.save()
-
-            # Trigger async reprocessing
-            from ..tasks import reprocess_document_chunks
-            reprocess_document_chunks.send(str(document.id))
-
-            return True
-        except Document.DoesNotExist:
-            return False

django_cfg/apps/knowbase/services/embedding/__init__.py

@@ -1,43 +0,0 @@
-"""
-Embedding processing services.
-
-This package provides high-performance embedding generation services
-for documents and archives with batch processing and async support.
-"""
-
-from .async_processor import AsyncOptimizedEmbeddingProcessor
-from .batch_processor import OptimizedEmbeddingProcessor
-from .batch_result import BatchResultBuilder
-from .models import BatchProcessingResult, ChunkData, ChunkType, EmbeddingResult, ProcessingConfig
-from .processors import ArchiveChunkProcessor, DocumentChunkProcessor, ExternalDataChunkProcessor
-from .utils import (
-    process_archive_chunks_optimized,
-    process_chunks_context_aware,
-    process_document_chunks_optimized,
-    process_external_data_chunks_optimized,
-)
-
-__all__ = [
-    # Data models
-    "ChunkData",
-    "EmbeddingResult",
-    "BatchProcessingResult",
-    "ProcessingConfig",
-    "ChunkType",
-
-    # Processors
-    "DocumentChunkProcessor",
-    "ArchiveChunkProcessor",
-    "ExternalDataChunkProcessor",
-    "OptimizedEmbeddingProcessor",
-    "AsyncOptimizedEmbeddingProcessor",
-
-    # Utilities
-    "BatchResultBuilder",
-
-    # Convenience functions
-    "process_document_chunks_optimized",
-    "process_archive_chunks_optimized",
-    "process_external_data_chunks_optimized",
-    "process_chunks_context_aware",
-]

django_cfg/apps/knowbase/services/embedding/async_processor.py

@@ -1,242 +0,0 @@
-"""
-Async-compatible embedding processor for Django 5.2.
-
-This module provides async/sync compatibility for embedding generation,
-following Django 5.2 async best practices.
-"""
-
-import asyncio
-import logging
-import time
-from typing import Any, Dict, List, Optional
-
-from asgiref.sync import async_to_sync
-
-from django_cfg.apps.knowbase.models import ArchiveItemChunk, DocumentChunk
-
-from .batch_processor import OptimizedEmbeddingProcessor
-from .models import BatchProcessingResult, ChunkData, EmbeddingResult
-
-logger = logging.getLogger(__name__)
-
-
-def is_async_context() -> bool:
-    """Detect current execution context."""
-    try:
-        asyncio.get_running_loop()
-        return True
-    except RuntimeError:
-        return False
-
-
-class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
-    """
-    Async-compatible embedding processor that works in both sync and async contexts.
-    
-    Based on Django 5.2 async patterns:
-    - Context-aware operations
-    - Proper async/sync method selection
-    - Compatible with both WSGI and ASGI
-    """
-
-    def __init__(self, batch_size: Optional[int] = None, embedding_model: Optional[str] = None):
-        """Initialize async-compatible processor."""
-        super().__init__(batch_size, embedding_model)
-        logger.info(f"🚀 AsyncOptimizedEmbeddingProcessor initialized: async_context={is_async_context()}")
-
-    async def aprocess_chunks_batch(self, chunks: List[ChunkData]) -> BatchProcessingResult:
-        """
-        Async version of batch processing.
-        
-        Uses Django 5.2 async ORM methods (a-prefixed) for database operations.
-        """
-        start_time = time.time()
-        total_chunks = len(chunks)
-        successful_chunks = 0
-        failed_chunks = 0
-        total_tokens = 0
-        total_cost = 0.0
-        errors = []
-
-        logger.info(f"🔮 Starting async batch processing of {total_chunks} chunks")
-
-        # Process in batches
-        for i in range(0, total_chunks, self.batch_size):
-            batch = chunks[i:i + self.batch_size]
-            batch_num = (i // self.batch_size) + 1
-            total_batches = (total_chunks + self.batch_size - 1) // self.batch_size
-
-            logger.info(f"🔮 Processing async batch {batch_num}/{total_batches} ({len(batch)} chunks)")
-
-            try:
-                batch_result = await self._aprocess_single_batch(batch)
-
-                successful_chunks += batch_result['successful']
-                failed_chunks += batch_result['failed']
-                total_tokens += batch_result['tokens']
-                total_cost += batch_result['cost']
-                errors.extend(batch_result['errors'])
-
-                # Small async delay between batches
-                if i + self.batch_size < total_chunks:
-                    await asyncio.sleep(0.5)
-
-            except Exception as e:
-                error_msg = f"Async batch {batch_num} failed: {str(e)}"
-                logger.error(f"❌ {error_msg}")
-                errors.append(error_msg)
-                failed_chunks += len(batch)
-
-        processing_time = time.time() - start_time
-
-        result = BatchProcessingResult(
-            total_chunks=total_chunks,
-            successful_chunks=successful_chunks,
-            failed_chunks=failed_chunks,
-            total_tokens=total_tokens,
-            total_cost=total_cost,
-            processing_time=processing_time,
-            errors=errors
-        )
-
-        logger.info(
-            f"🎉 Async batch processing completed: {successful_chunks}/{total_chunks} successful, "
-            f"{total_tokens} tokens, ${total_cost:.4f} cost, {processing_time:.2f}s"
-        )
-
-        return result
-
-    async def _aprocess_single_batch(self, batch: List[ChunkData]) -> Dict[str, Any]:
-        """Async version of single batch processing."""
-
-        # Prepare content for all chunks (sync operation)
-        prepared_contents = []
-        chunk_mapping = {}
-
-        for idx, chunk in enumerate(batch):
-            processor = self.processors.get(chunk.parent_type)
-            if not processor:
-                logger.warning(f"⚠️ Unknown chunk type: {chunk.parent_type}")
-                continue
-
-            try:
-                content = processor.prepare_content_for_embedding(chunk)
-                if content and content.strip():
-                    prepared_contents.append(content)
-                    chunk_mapping[len(prepared_contents) - 1] = chunk
-                else:
-                    logger.warning(f"⚠️ Empty content for chunk {chunk.id}")
-            except Exception as e:
-                logger.error(f"❌ Failed to prepare content for chunk {chunk.id}: {e}")
-
-        if not prepared_contents:
-            return {
-                'successful': 0,
-                'failed': len(batch),
-                'tokens': 0,
-                'cost': 0.0,
-                'errors': ['No valid content to process']
-            }
-
-        # Generate embeddings (sync operation - OpenAI client is sync)
-        try:
-            embedding_results = self._generate_batch_embeddings(prepared_contents)
-
-            # Save results using async database operations
-            successful = 0
-            failed = 0
-            total_tokens = 0
-            total_cost = 0.0
-            errors = []
-
-            for idx, embedding_result in enumerate(embedding_results):
-                if idx not in chunk_mapping:
-                    continue
-
-                chunk = chunk_mapping[idx]
-
-                if embedding_result.success:
-                    try:
-                        await self._asave_embedding_result(chunk, embedding_result)
-                        successful += 1
-                        total_tokens += embedding_result.tokens
-                        total_cost += embedding_result.cost
-                    except Exception as e:
-                        error_msg = f"Failed to save async embedding for chunk {chunk.id}: {e}"
-                        logger.error(f"❌ {error_msg}")
-                        errors.append(error_msg)
-                        failed += 1
-                else:
-                    errors.append(embedding_result.error or f"Failed to generate embedding for chunk {chunk.id}")
-                    failed += 1
-
-            return {
-                'successful': successful,
-                'failed': failed,
-                'tokens': total_tokens,
-                'cost': total_cost,
-                'errors': errors
-            }
-
-        except Exception as e:
-            error_msg = f"Async batch embedding generation failed: {e}"
-            logger.error(f"❌ {error_msg}")
-            return {
-                'successful': 0,
-                'failed': len(batch),
-                'tokens': 0,
-                'cost': 0.0,
-                'errors': [error_msg]
-            }
-
-    async def _asave_embedding_result(self, chunk: ChunkData, result: EmbeddingResult) -> None:
-        """Save embedding result using async database operations."""
-
-        try:
-            if chunk.parent_type == "document":
-                # Use Django 5.2 async ORM methods
-                chunk_obj = await DocumentChunk.objects.aget(id=chunk.id)
-                chunk_obj.embedding = result.embedding
-                chunk_obj.token_count = result.tokens
-                chunk_obj.embedding_cost = result.cost
-                await chunk_obj.asave(update_fields=['embedding', 'token_count', 'embedding_cost'])
-
-                logger.debug(f"✅ Async document chunk {chunk.id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
-
-            elif chunk.parent_type == "archive":
-                # Use async ORM with select_related
-                chunk_obj = await ArchiveItemChunk.objects.select_related('item').aget(id=chunk.id)
-                chunk_obj.embedding = result.embedding
-                chunk_obj.token_count = result.tokens
-                chunk_obj.embedding_cost = result.cost
-                await chunk_obj.asave(update_fields=['embedding', 'token_count', 'embedding_cost'])
-
-                # Update parent item statistics
-                item = chunk_obj.item
-                item.total_tokens += result.tokens
-                item.processing_cost += result.cost
-                await item.asave(update_fields=['total_tokens', 'processing_cost'])
-
-                logger.debug(f"✅ Async archive chunk {chunk.id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
-            else:
-                raise ValueError(f"Unknown chunk type: {chunk.parent_type}")
-
-        except Exception as e:
-            logger.error(f"❌ Failed to save async embedding for chunk {chunk.id}: {e}")
-            raise
-
-    def process_chunks_batch_context_aware(self, chunks: List[ChunkData]) -> BatchProcessingResult:
-        """
-        Context-aware processing that works in both sync and async contexts.
-        
-        Based on Django 5.2 async patterns.
-        """
-        if is_async_context():
-            # We're in async context - use async methods
-            logger.info("🔮 Detected async context - using async processing")
-            # Convert async method to sync for compatibility
-            return async_to_sync(self.aprocess_chunks_batch)(chunks)
-        else:
-            # We're in sync context - use sync methods
-            logger.info("🔮 Detected sync context - using sync processing")
-            return super().process_chunks_batch(chunks)

django_cfg/apps/knowbase/services/embedding/batch_processor.py

@@ -1,253 +0,0 @@
-"""
-High-performance batch embedding processor.
-
-This module provides the main batch processing engine for generating
-embeddings with optimized API calls and database operations.
-"""
-
-import logging
-import time
-from typing import List, Optional
-
-from django_cfg.apps.knowbase.config.settings import get_cache_settings
-from django_cfg.apps.knowbase.utils.chunk_settings import (
-    get_embedding_batch_size,
-    get_embedding_model,
-)
-from django_cfg.modules.django_llm.llm.client import LLMClient
-
-from .batch_result import BatchResultBuilder
-from .models import BatchProcessingResult, ChunkData, ChunkType, EmbeddingResult
-from .processors import ArchiveChunkProcessor, DocumentChunkProcessor, ExternalDataChunkProcessor
-
-logger = logging.getLogger(__name__)
-
-
-class OptimizedEmbeddingProcessor:
-    """High-performance embedding processor with batch operations."""
-
-    def __init__(self, batch_size: Optional[int] = None, embedding_model: Optional[str] = None):
-        """
-        Initialize the processor.
-        
-        Args:
-            batch_size: Number of chunks to process in one API call (uses Constance setting if None)
-            embedding_model: Embedding model to use (uses Constance setting if None)
-        """
-        # Use Constance settings if not provided
-        self.batch_size = min(batch_size or get_embedding_batch_size(), 100)  # Conservative limit for stability
-        self.embedding_model = embedding_model or get_embedding_model()
-
-        # Initialize LLM client with OpenAI only for embeddings
-        # OpenRouter doesn't support embedding models, so we use OpenAI directly
-        # Use auto-configured LLMClient with explicit OpenAI preference for embeddings
-        # Get cache settings from configuration (directory is auto-created)
-        cache_settings = get_cache_settings()
-        self.llm_client = LLMClient(
-            preferred_provider="openai",  # Force OpenAI for embeddings
-            cache_dir=cache_settings.cache_dir,
-            cache_ttl=cache_settings.cache_ttl,
-            max_cache_size=cache_settings.max_cache_size
-        )
-
-        # Processors for different chunk types
-        self.processors = {
-            ChunkType.DOCUMENT: DocumentChunkProcessor(),
-            ChunkType.ARCHIVE: ArchiveChunkProcessor(),
-            ChunkType.EXTERNAL_DATA: ExternalDataChunkProcessor()
-        }
-
-        logger.info(f"🚀 OptimizedEmbeddingProcessor initialized: batch_size={self.batch_size}, model={self.embedding_model}")
-
-    def process_chunks_batch(self, chunks: List[ChunkData]) -> BatchProcessingResult:
-        """
-        Process multiple chunks with optimized batch operations.
-        
-        Args:
-            chunks: List of chunks to process
-            
-        Returns:
-            BatchProcessingResult with processing statistics
-        """
-        start_time = time.time()
-        total_chunks = len(chunks)
-        result_builder = BatchResultBuilder(total_chunks)
-
-        logger.info(f"🔮 Starting batch processing of {total_chunks} chunks")
-
-        # Process in batches
-        for i in range(0, total_chunks, self.batch_size):
-            batch = chunks[i:i + self.batch_size]
-            batch_num = (i // self.batch_size) + 1
-            total_batches = (total_chunks + self.batch_size - 1) // self.batch_size
-
-            logger.info(f"🔮 Processing batch {batch_num}/{total_batches} ({len(batch)} chunks)")
-
-            try:
-                batch_results = self._process_single_batch(batch)
-                result_builder.add_batch_results(batch_results)
-
-                # Small delay between batches to respect rate limits
-                if i + self.batch_size < total_chunks:
-                    time.sleep(0.5)
-
-            except Exception as e:
-                error_msg = f"Batch {batch_num} failed: {str(e)}"
-                logger.error(f"❌ {error_msg}")
-                result_builder.add_batch_error(error_msg, len(batch))
-
-        processing_time = time.time() - start_time
-        result = result_builder.build(processing_time)
-
-        # Log using Pydantic model's summary
-        summary = result.model_dump_summary()
-        logger.info(f"🎉 Batch processing completed: {summary}")
-        logger.info(f"📊 Performance: {summary['chunks_per_second']} chunks/sec, {summary['avg_cost_per_chunk']} per chunk")
-
-        return result
-
-    def _process_single_batch(self, batch: List[ChunkData]) -> List[EmbeddingResult]:
-        """Process a single batch of chunks and return list of EmbeddingResult."""
-
-        # Prepare content for all chunks
-        prepared_contents = []
-        chunk_mapping = {}  # Map index to chunk
-
-        for idx, chunk in enumerate(batch):
-            processor = self.processors.get(chunk.parent_type)
-            if not processor:
-                logger.warning(f"⚠️ Unknown chunk type: {chunk.parent_type}")
-                continue
-
-            try:
-                content = processor.prepare_content_for_embedding(chunk)
-                if content and content.strip():
-                    prepared_contents.append(content)
-                    chunk_mapping[len(prepared_contents) - 1] = chunk
-                else:
-                    logger.warning(f"⚠️ Empty content for chunk {chunk.id}")
-            except Exception as e:
-                logger.error(f"❌ Failed to prepare content for chunk {chunk.id}: {e}")
-
-        if not prepared_contents:
-            # Return failed results for all chunks
-            return [
-                EmbeddingResult(
-                    chunk_id=chunk.id,
-                    success=False,
-                    error="No valid content to process"
-                )
-                for chunk in batch
-            ]
-
-        # Generate embeddings in batch
-        try:
-            embedding_results = self._generate_batch_embeddings(prepared_contents)
-
-            # Process and save results
-            final_results = []
-
-            for idx, embedding_result in enumerate(embedding_results):
-                if idx not in chunk_mapping:
-                    continue
-
-                chunk = chunk_mapping[idx]
-                processor = self.processors[chunk.parent_type]
-
-                # Set the chunk_id in the result
-                embedding_result.chunk_id = chunk.id
-
-                if embedding_result.success:
-                    try:
-                        logger.debug(f"🔄 Attempting to save embedding for chunk {chunk.id} (type: {chunk.parent_type})")
-                        processor.save_embedding_result(chunk.id, embedding_result)
-                        logger.info(f"✅ Successfully saved embedding for chunk {chunk.id}")
-                        final_results.append(embedding_result)
-                    except Exception as e:
-                        error_msg = f"Failed to save embedding for chunk {chunk.id}: {e}"
-                        logger.error(f"❌ {error_msg}")
-                        failed_result = EmbeddingResult(
-                            chunk_id=chunk.id,
-                            success=False,
-                            error=error_msg
-                        )
-                        final_results.append(failed_result)
-                else:
-                    final_results.append(embedding_result)
-
-            return final_results
-
-        except Exception as e:
-            error_msg = f"Batch embedding generation failed: {e}"
-            logger.error(f"❌ {error_msg}")
-            # Return failed results for all chunks
-            return [
-                EmbeddingResult(
-                    chunk_id=chunk.id,
-                    success=False,
-                    error=error_msg
-                )
-                for chunk in batch
-            ]
-
-    def _generate_batch_embeddings(self, contents: List[str]) -> List[EmbeddingResult]:
-        """Generate embeddings for multiple contents using LLMClient."""
-
-        results = []
-
-        try:
-            # Use LLMClient's generate_embedding method for each content
-            # This handles both OpenAI and OpenRouter properly
-            for idx, content in enumerate(contents):
-                try:
-                    # Use LLMClient's method which handles provider differences
-                    embedding_response = self.llm_client.generate_embedding(
-                        text=content,
-                        model=self.embedding_model
-                    )
-
-                    results.append(EmbeddingResult(
-                        chunk_id="",  # Will be set by caller
-                        embedding=embedding_response.embedding,
-                        tokens=embedding_response.tokens,
-                        cost=embedding_response.cost,
-                        success=True
-                    ))
-
-                except Exception as e:
-                    logger.error(f"❌ Failed to generate embedding for content {idx}: {e}")
-                    results.append(EmbeddingResult(
-                        chunk_id="",
-                        embedding=[],
-                        tokens=0,
-                        cost=0.0,
-                        success=False,
-                        error=str(e)
-                    ))
-
-            successful_count = len([r for r in results if r.success])
-            logger.info(f"🎯 Generated {successful_count}/{len(results)} embeddings successfully")
-
-            # Log details of each result
-            for i, result in enumerate(results):
-                if result.success:
-                    logger.debug(f"  ✅ Result {i}: {result.tokens} tokens, ${result.cost:.4f}, embedding_len={len(result.embedding)}")
-                else:
-                    logger.debug(f"  ❌ Result {i}: {result.error}")
-
-            return results
-
-        except Exception as e:
-            logger.error(f"❌ Batch embedding generation failed: {e}")
-            # Return failed results for all contents
-            return [
-                EmbeddingResult(
-                    chunk_id="",
-                    embedding=[],
-                    tokens=0,
-                    cost=0.0,
-                    success=False,
-                    error=str(e)
-                )
-                for _ in contents
-            ]