django-cfg 1.4.48__py3-none-any.whl → 1.5.7__py3-none-any.whl

django_cfg/apps/knowbase/services/archive/vectorization_service.py

@@ -1,365 +0,0 @@
-"""
-Archive vectorization service.
-
-Handles embedding generation for archive chunks with batch processing.
-"""
-
-import logging
-from typing import Any, Dict, List, Optional
-
-from django.contrib.auth import get_user_model
-from django.db import models, transaction
-
-from django_cfg.modules.django_llm.llm.models import EmbeddingResponse
-
-from ...models.archive import ArchiveItemChunk
-from ..base import BaseService
-from ..embedding import process_archive_chunks_optimized
-from .exceptions import VectorizationError
-
-User = get_user_model()
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
-
-class VectorizationResult:
-    """Result of vectorization operation."""
-
-    def __init__(self):
-        self.vectorized_count: int = 0
-        self.failed_count: int = 0
-        self.total_tokens: int = 0
-        self.total_cost: float = 0.0
-        self.errors: List[str] = []
-
-
-class ArchiveVectorizationService(BaseService):
-    """Service for vectorizing archive chunks."""
-
-    def __init__(self, user: User):
-        super().__init__(user)
-        self.batch_size = 10  # Process chunks in batches
-
-    def vectorize_chunks_batch(self, chunks: List[ArchiveItemChunk]) -> Dict[str, Any]:
-        """Vectorize chunks using optimized batch processing."""
-
-        if not chunks:
-            logger.warning("🔮 No chunks provided for vectorization")
-            return {
-                'vectorized_count': 0,
-                'failed_count': 0,
-                'total_tokens': 0,
-                'total_cost': 0.0,
-                'success_rate': 0.0,
-                'errors': []
-            }
-
-        logger.info(f"🔮 Starting optimized vectorization of {len(chunks)} chunks")
-
-        try:
-            # Use optimized batch processor
-            result = process_archive_chunks_optimized(chunks)
-
-            logger.info(
-                f"🔮 Optimized vectorization completed: {result.successful_chunks}/{result.total_chunks} chunks, "
-                f"{result.failed_chunks} failed, {result.total_tokens} tokens, ${result.total_cost:.4f} cost, "
-                f"{result.processing_time:.2f}s"
-            )
-
-            return {
-                'vectorized_count': result.successful_chunks,
-                'failed_count': result.failed_chunks,
-                'total_tokens': result.total_tokens,
-                'total_cost': result.total_cost,
-                'success_rate': result.successful_chunks / result.total_chunks if result.total_chunks > 0 else 0.0,
-                'errors': result.errors
-            }
-
-        except Exception as e:
-            logger.error(f"❌ Optimized vectorization failed: {e}")
-            raise VectorizationError(
-                message=f"Optimized vectorization failed: {str(e)}",
-                code="OPTIMIZED_VECTORIZATION_FAILED",
-                details={
-                    "total_chunks": len(chunks),
-                    "error": str(e)
-                }
-            ) from e
-
-    def _vectorize_chunk_batch(self, chunks: List[ArchiveItemChunk]) -> VectorizationResult:
-        """Vectorize a single batch of chunks."""
-
-        result = VectorizationResult()
-
-        for chunk in chunks:
-            try:
-                # Skip if already vectorized
-                if chunk.embedding is not None and len(chunk.embedding) > 0:
-                    logger.debug(f"🔮 Chunk {chunk.id} already vectorized, skipping")
-                    continue
-
-                logger.debug(f"🔮 Generating embedding for chunk {chunk.id} ({chunk.item.item_name})")
-
-                # Generate embedding
-                embedding_result = self._generate_chunk_embedding(chunk)
-
-                if embedding_result:
-                    # Update chunk with embedding
-                    with transaction.atomic():
-                        chunk.embedding = embedding_result.embedding
-                        chunk.token_count = embedding_result.tokens
-                        chunk.embedding_cost = embedding_result.cost
-                        chunk.save()
-
-                        # Update item statistics
-                        item = chunk.item
-                        item.total_tokens += embedding_result.tokens
-                        item.processing_cost += embedding_result.cost
-                        item.save()
-
-                    result.vectorized_count += 1
-                    result.total_tokens += embedding_result.tokens
-                    logger.debug(f"✅ Chunk {chunk.id} vectorized successfully: {embedding_result.tokens} tokens, ${embedding_result.cost:.4f}")
-                    result.total_cost += embedding_result.cost
-                else:
-                    result.failed_count += 1
-                    error_msg = f"Failed to generate embedding for chunk {chunk.id}"
-                    result.errors.append(error_msg)
-                    logger.error(f"❌ {error_msg}")
-
-            except Exception as e:
-                result.failed_count += 1
-                error_msg = f"Error processing chunk {chunk.id}: {str(e)}"
-                result.errors.append(error_msg)
-                logger.error(f"❌ {error_msg}")
-                continue
-
-        return result
-
-    def _generate_chunk_embedding(self, chunk: ArchiveItemChunk) -> Optional[EmbeddingResponse]:
-        """Generate embedding for a single chunk."""
-
-        if not chunk.content or not chunk.content.strip():
-            return None
-
-        try:
-            # Prepare content for embedding
-            content_for_embedding = self._prepare_content_for_embedding(chunk)
-
-            logger.debug(f"🔮 Prepared content for embedding: {len(content_for_embedding)} chars")
-
-            # Generate embedding using LLM client with specified model
-            from django_cfg.apps.knowbase.utils.chunk_settings import get_embedding_model
-            embedding_model = get_embedding_model()
-            embedding_result = self.llm_client.generate_embedding(
-                text=content_for_embedding,
-                model=embedding_model
-            )
-
-            if embedding_result:
-                logger.debug(f"🔮 Embedding generated successfully for chunk {chunk.id}")
-                logger.debug(f"🔮 Embedding result structure: {list(embedding_result.keys()) if isinstance(embedding_result, dict) else type(embedding_result)}")
-            else:
-                logger.warning(f"🔮 Embedding generation returned None for chunk {chunk.id}")
-
-            return embedding_result
-
-        except Exception as e:
-            # Log error but don't raise - we want to continue with other chunks
-            logger.error(f"🔮 Error generating embedding for chunk {chunk.id}: {str(e)}", exc_info=True)
-            return None
-
-    def _prepare_content_for_embedding(self, chunk: ArchiveItemChunk) -> str:
-        """Prepare chunk content for embedding generation."""
-
-        content = chunk.content
-        context = chunk.context_metadata
-
-        # Add context information to improve embedding quality
-        context_prefix = self._build_context_prefix(context)
-
-        # Combine context and content
-        if context_prefix:
-            enhanced_content = f"{context_prefix}\n\n{content}"
-        else:
-            enhanced_content = content
-
-        # Ensure content is not too long for embedding model
-        max_length = 8000  # Conservative limit for most embedding models
-        if len(enhanced_content) > max_length:
-            # Truncate but keep context prefix
-            if context_prefix:
-                available_length = max_length - len(context_prefix) - 4  # Account for separators
-                truncated_content = content[:available_length] + "..."
-                enhanced_content = f"{context_prefix}\n\n{truncated_content}"
-            else:
-                enhanced_content = content[:max_length] + "..."
-
-        return enhanced_content
-
-    def _build_context_prefix(self, context: Dict[str, Any]) -> str:
-        """Build context prefix to enhance embedding quality."""
-
-        prefix_parts = []
-
-        # Archive context
-        archive_info = context.get('archive_info', {})
-        if archive_info.get('title'):
-            prefix_parts.append(f"Archive: {archive_info['title']}")
-
-        # Item context
-        item_info = context.get('item_info', {})
-        if item_info.get('relative_path'):
-            prefix_parts.append(f"File: {item_info['relative_path']}")
-
-        if item_info.get('content_type'):
-            prefix_parts.append(f"Type: {item_info['content_type']}")
-
-        if item_info.get('language'):
-            prefix_parts.append(f"Language: {item_info['language']}")
-
-        # Structure context
-        structure_info = context.get('structure_info', {})
-        if structure_info.get('element_name'):
-            prefix_parts.append(f"Element: {structure_info['element_name']}")
-
-        if structure_info.get('section_title'):
-            prefix_parts.append(f"Section: {structure_info['section_title']}")
-
-        # Semantic context
-        semantic_info = context.get('semantic_info', {})
-        if semantic_info.get('content_purpose'):
-            prefix_parts.append(f"Purpose: {semantic_info['content_purpose']}")
-
-        return " | ".join(prefix_parts) if prefix_parts else ""
-
-    def vectorize_single_chunk(self, chunk_id: str) -> Dict[str, Any]:
-        """Vectorize a single chunk by ID."""
-
-        try:
-            chunk = ArchiveItemChunk.objects.get(id=chunk_id, user=self.user)
-        except ArchiveItemChunk.DoesNotExist:
-            raise VectorizationError(
-                message=f"Chunk not found: {chunk_id}",
-                code="CHUNK_NOT_FOUND",
-                details={"chunk_id": chunk_id}
-            )
-
-        # Check if already vectorized
-        if chunk.embedding is not None and len(chunk.embedding) > 0:
-            return {
-                'status': 'already_vectorized',
-                'chunk_id': chunk_id,
-                'token_count': chunk.token_count,
-                'cost': chunk.embedding_cost
-            }
-
-        # Generate embedding
-        embedding_result = self._generate_chunk_embedding(chunk)
-
-        if not embedding_result:
-            raise VectorizationError(
-                message=f"Failed to generate embedding for chunk {chunk_id}",
-                code="EMBEDDING_GENERATION_FAILED",
-                details={"chunk_id": chunk_id}
-            )
-
-        # Update chunk
-        with transaction.atomic():
-            chunk.embedding = embedding_result.embedding
-            chunk.token_count = embedding_result.tokens
-            chunk.embedding_cost = embedding_result.cost
-            chunk.save()
-
-            # Update item statistics
-            item = chunk.item
-            item.total_tokens += embedding_result.tokens
-            item.processing_cost += embedding_result.cost
-            item.save()
-
-        return {
-            'status': 'vectorized',
-            'chunk_id': chunk_id,
-            'token_count': embedding_result.tokens,
-            'cost': embedding_result.cost
-        }
-
-    def get_vectorization_statistics(self, archive_id: Optional[str] = None) -> Dict[str, Any]:
-        """Get vectorization statistics for user's chunks."""
-
-        queryset = ArchiveItemChunk.objects.filter(user=self.user)
-
-        if archive_id:
-            queryset = queryset.filter(archive_id=archive_id)
-
-        total_chunks = queryset.count()
-        vectorized_chunks = queryset.filter(embedding__isnull=False).count()
-        pending_chunks = total_chunks - vectorized_chunks
-
-        # Aggregate statistics
-        stats = queryset.aggregate(
-            total_tokens=models.Sum('token_count'),
-            total_cost=models.Sum('embedding_cost'),
-            avg_tokens_per_chunk=models.Avg('token_count'),
-            avg_cost_per_chunk=models.Avg('embedding_cost')
-        )
-
-        return {
-            'total_chunks': total_chunks,
-            'vectorized_chunks': vectorized_chunks,
-            'pending_chunks': pending_chunks,
-            'vectorization_rate': vectorized_chunks / total_chunks if total_chunks > 0 else 0.0,
-            'total_tokens': stats['total_tokens'] or 0,
-            'total_cost': stats['total_cost'] or 0.0,
-            'avg_tokens_per_chunk': stats['avg_tokens_per_chunk'] or 0.0,
-            'avg_cost_per_chunk': stats['avg_cost_per_chunk'] or 0.0
-        }
-
-    def revectorize_chunks(
-        self,
-        chunk_ids: List[str],
-        force: bool = False
-    ) -> Dict[str, Any]:
-        """Re-vectorize specific chunks."""
-
-        chunks = ArchiveItemChunk.objects.filter(
-            id__in=chunk_ids,
-            user=self.user
-        )
-
-        if not force:
-            # Only re-vectorize chunks that don't have embeddings
-            chunks = chunks.filter(embedding__isnull=True)
-
-        return self.vectorize_chunks_batch(list(chunks))
-
-    def cleanup_failed_vectorizations(self) -> Dict[str, Any]:
-        """Clean up chunks that failed vectorization."""
-
-        # Find chunks without embeddings that are older than 1 hour
-        from datetime import timedelta
-
-        from django.utils import timezone
-
-        cutoff_time = timezone.now() - timedelta(hours=1)
-
-        failed_chunks = ArchiveItemChunk.objects.filter(
-            user=self.user,
-            embedding__isnull=True,
-            created_at__lt=cutoff_time
-        )
-
-        failed_count = failed_chunks.count()
-
-        # Attempt to re-vectorize
-        if failed_count > 0:
-            result = self.vectorize_chunks_batch(list(failed_chunks))
-
-            return {
-                'found_failed_chunks': failed_count,
-                'retry_result': result
-            }
-
-        return {
-            'found_failed_chunks': 0,
-            'retry_result': None
-        }

django_cfg/apps/knowbase/services/embedding/async_processor.py

@@ -1,242 +0,0 @@
-"""
-Async-compatible embedding processor for Django 5.2.
-
-This module provides async/sync compatibility for embedding generation,
-following Django 5.2 async best practices.
-"""
-
-import asyncio
-import logging
-import time
-from typing import Any, Dict, List, Optional
-
-from asgiref.sync import async_to_sync
-
-from django_cfg.apps.knowbase.models import ArchiveItemChunk, DocumentChunk
-
-from .batch_processor import OptimizedEmbeddingProcessor
-from .models import BatchProcessingResult, ChunkData, EmbeddingResult
-
-logger = logging.getLogger(__name__)
-
-
-def is_async_context() -> bool:
-    """Detect current execution context."""
-    try:
-        asyncio.get_running_loop()
-        return True
-    except RuntimeError:
-        return False
-
-
-class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
-    """
-    Async-compatible embedding processor that works in both sync and async contexts.
-    
-    Based on Django 5.2 async patterns:
-    - Context-aware operations
-    - Proper async/sync method selection
-    - Compatible with both WSGI and ASGI
-    """
-
-    def __init__(self, batch_size: Optional[int] = None, embedding_model: Optional[str] = None):
-        """Initialize async-compatible processor."""
-        super().__init__(batch_size, embedding_model)
-        logger.info(f"🚀 AsyncOptimizedEmbeddingProcessor initialized: async_context={is_async_context()}")
-
-    async def aprocess_chunks_batch(self, chunks: List[ChunkData]) -> BatchProcessingResult:
-        """
-        Async version of batch processing.
-        
-        Uses Django 5.2 async ORM methods (a-prefixed) for database operations.
-        """
-        start_time = time.time()
-        total_chunks = len(chunks)
-        successful_chunks = 0
-        failed_chunks = 0
-        total_tokens = 0
-        total_cost = 0.0
-        errors = []
-
-        logger.info(f"🔮 Starting async batch processing of {total_chunks} chunks")
-
-        # Process in batches
-        for i in range(0, total_chunks, self.batch_size):
-            batch = chunks[i:i + self.batch_size]
-            batch_num = (i // self.batch_size) + 1
-            total_batches = (total_chunks + self.batch_size - 1) // self.batch_size
-
-            logger.info(f"🔮 Processing async batch {batch_num}/{total_batches} ({len(batch)} chunks)")
-
-            try:
-                batch_result = await self._aprocess_single_batch(batch)
-
-                successful_chunks += batch_result['successful']
-                failed_chunks += batch_result['failed']
-                total_tokens += batch_result['tokens']
-                total_cost += batch_result['cost']
-                errors.extend(batch_result['errors'])
-
-                # Small async delay between batches
-                if i + self.batch_size < total_chunks:
-                    await asyncio.sleep(0.5)
-
-            except Exception as e:
-                error_msg = f"Async batch {batch_num} failed: {str(e)}"
-                logger.error(f"❌ {error_msg}")
-                errors.append(error_msg)
-                failed_chunks += len(batch)
-
-        processing_time = time.time() - start_time
-
-        result = BatchProcessingResult(
-            total_chunks=total_chunks,
-            successful_chunks=successful_chunks,
-            failed_chunks=failed_chunks,
-            total_tokens=total_tokens,
-            total_cost=total_cost,
-            processing_time=processing_time,
-            errors=errors
-        )
-
-        logger.info(
-            f"🎉 Async batch processing completed: {successful_chunks}/{total_chunks} successful, "
-            f"{total_tokens} tokens, ${total_cost:.4f} cost, {processing_time:.2f}s"
-        )
-
-        return result
-
-    async def _aprocess_single_batch(self, batch: List[ChunkData]) -> Dict[str, Any]:
-        """Async version of single batch processing."""
-
-        # Prepare content for all chunks (sync operation)
-        prepared_contents = []
-        chunk_mapping = {}
-
-        for idx, chunk in enumerate(batch):
-            processor = self.processors.get(chunk.parent_type)
-            if not processor:
-                logger.warning(f"⚠️ Unknown chunk type: {chunk.parent_type}")
-                continue
-
-            try:
-                content = processor.prepare_content_for_embedding(chunk)
-                if content and content.strip():
-                    prepared_contents.append(content)
-                    chunk_mapping[len(prepared_contents) - 1] = chunk
-                else:
-                    logger.warning(f"⚠️ Empty content for chunk {chunk.id}")
-            except Exception as e:
-                logger.error(f"❌ Failed to prepare content for chunk {chunk.id}: {e}")
-
-        if not prepared_contents:
-            return {
-                'successful': 0,
-                'failed': len(batch),
-                'tokens': 0,
-                'cost': 0.0,
-                'errors': ['No valid content to process']
-            }
-
-        # Generate embeddings (sync operation - OpenAI client is sync)
-        try:
-            embedding_results = self._generate_batch_embeddings(prepared_contents)
-
-            # Save results using async database operations
-            successful = 0
-            failed = 0
-            total_tokens = 0
-            total_cost = 0.0
-            errors = []
-
-            for idx, embedding_result in enumerate(embedding_results):
-                if idx not in chunk_mapping:
-                    continue
-
-                chunk = chunk_mapping[idx]
-
-                if embedding_result.success:
-                    try:
-                        await self._asave_embedding_result(chunk, embedding_result)
-                        successful += 1
-                        total_tokens += embedding_result.tokens
-                        total_cost += embedding_result.cost
-                    except Exception as e:
-                        error_msg = f"Failed to save async embedding for chunk {chunk.id}: {e}"
-                        logger.error(f"❌ {error_msg}")
-                        errors.append(error_msg)
-                        failed += 1
-                else:
-                    errors.append(embedding_result.error or f"Failed to generate embedding for chunk {chunk.id}")
-                    failed += 1
-
-            return {
-                'successful': successful,
-                'failed': failed,
-                'tokens': total_tokens,
-                'cost': total_cost,
-                'errors': errors
-            }
-
-        except Exception as e:
-            error_msg = f"Async batch embedding generation failed: {e}"
-            logger.error(f"❌ {error_msg}")
-            return {
-                'successful': 0,
-                'failed': len(batch),
-                'tokens': 0,
-                'cost': 0.0,
-                'errors': [error_msg]
-            }
-
-    async def _asave_embedding_result(self, chunk: ChunkData, result: EmbeddingResult) -> None:
-        """Save embedding result using async database operations."""
-
-        try:
-            if chunk.parent_type == "document":
-                # Use Django 5.2 async ORM methods
-                chunk_obj = await DocumentChunk.objects.aget(id=chunk.id)
-                chunk_obj.embedding = result.embedding
-                chunk_obj.token_count = result.tokens
-                chunk_obj.embedding_cost = result.cost
-                await chunk_obj.asave(update_fields=['embedding', 'token_count', 'embedding_cost'])
-
-                logger.debug(f"✅ Async document chunk {chunk.id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
-
-            elif chunk.parent_type == "archive":
-                # Use async ORM with select_related
-                chunk_obj = await ArchiveItemChunk.objects.select_related('item').aget(id=chunk.id)
-                chunk_obj.embedding = result.embedding
-                chunk_obj.token_count = result.tokens
-                chunk_obj.embedding_cost = result.cost
-                await chunk_obj.asave(update_fields=['embedding', 'token_count', 'embedding_cost'])
-
-                # Update parent item statistics
-                item = chunk_obj.item
-                item.total_tokens += result.tokens
-                item.processing_cost += result.cost
-                await item.asave(update_fields=['total_tokens', 'processing_cost'])
-
-                logger.debug(f"✅ Async archive chunk {chunk.id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
-            else:
-                raise ValueError(f"Unknown chunk type: {chunk.parent_type}")
-
-        except Exception as e:
-            logger.error(f"❌ Failed to save async embedding for chunk {chunk.id}: {e}")
-            raise
-
-    def process_chunks_batch_context_aware(self, chunks: List[ChunkData]) -> BatchProcessingResult:
-        """
-        Context-aware processing that works in both sync and async contexts.
-        
-        Based on Django 5.2 async patterns.
-        """
-        if is_async_context():
-            # We're in async context - use async methods
-            logger.info("🔮 Detected async context - using async processing")
-            # Convert async method to sync for compatibility
-            return async_to_sync(self.aprocess_chunks_batch)(chunks)
-        else:
-            # We're in sync context - use sync methods
-            logger.info("🔮 Detected sync context - using sync processing")
-            return super().process_chunks_batch(chunks)