django-cfg 1.4.21__py3-none-any.whl → 1.4.23__py3-none-any.whl

django_cfg/apps/knowbase/services/archive/vectorization_service.py

@@ -4,24 +4,26 @@ Archive vectorization service.
 Handles embedding generation for archive chunks with batch processing.
 """
 
-from typing import List, Dict, Any, Optional
-from django.contrib.auth import get_user_model
-from django.db import transaction, models
 import logging
-from ...models.archive import ArchiveItemChunk
+from typing import Any, Dict, List, Optional
+
+from django.contrib.auth import get_user_model
+from django.db import models, transaction
+
 from django_cfg.modules.django_llm.llm.models import EmbeddingResponse
+
+from ...models.archive import ArchiveItemChunk
 from ..base import BaseService
 from ..embedding import process_archive_chunks_optimized
 from .exceptions import VectorizationError
 
-
 User = get_user_model()
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 
 class VectorizationResult:
     """Result of vectorization operation."""
-    
+
     def __init__(self):
         self.vectorized_count: int = 0
         self.failed_count: int = 0
@@ -32,14 +34,14 @@ class VectorizationResult:
 
 class ArchiveVectorizationService(BaseService):
     """Service for vectorizing archive chunks."""
-    
+
     def __init__(self, user: User):
         super().__init__(user)
         self.batch_size = 10  # Process chunks in batches
-    
+
     def vectorize_chunks_batch(self, chunks: List[ArchiveItemChunk]) -> Dict[str, Any]:
         """Vectorize chunks using optimized batch processing."""
-        
+
         if not chunks:
             logger.warning("🔮 No chunks provided for vectorization")
             return {
@@ -50,19 +52,19 @@ class ArchiveVectorizationService(BaseService):
                 'success_rate': 0.0,
                 'errors': []
             }
-        
+
         logger.info(f"🔮 Starting optimized vectorization of {len(chunks)} chunks")
-        
+
         try:
             # Use optimized batch processor
             result = process_archive_chunks_optimized(chunks)
-            
+
             logger.info(
                 f"🔮 Optimized vectorization completed: {result.successful_chunks}/{result.total_chunks} chunks, "
                 f"{result.failed_chunks} failed, {result.total_tokens} tokens, ${result.total_cost:.4f} cost, "
                 f"{result.processing_time:.2f}s"
             )
-            
+
             return {
                 'vectorized_count': result.successful_chunks,
                 'failed_count': result.failed_chunks,
@@ -71,7 +73,7 @@ class ArchiveVectorizationService(BaseService):
                 'success_rate': result.successful_chunks / result.total_chunks if result.total_chunks > 0 else 0.0,
                 'errors': result.errors
             }
-            
+
         except Exception as e:
             logger.error(f"❌ Optimized vectorization failed: {e}")
             raise VectorizationError(
@@ -82,24 +84,24 @@ class ArchiveVectorizationService(BaseService):
                     "error": str(e)
                 }
             ) from e
-    
+
     def _vectorize_chunk_batch(self, chunks: List[ArchiveItemChunk]) -> VectorizationResult:
         """Vectorize a single batch of chunks."""
-        
+
         result = VectorizationResult()
-        
+
         for chunk in chunks:
             try:
                 # Skip if already vectorized
                 if chunk.embedding is not None and len(chunk.embedding) > 0:
                     logger.debug(f"🔮 Chunk {chunk.id} already vectorized, skipping")
                     continue
-                
+
                 logger.debug(f"🔮 Generating embedding for chunk {chunk.id} ({chunk.item.item_name})")
-                
+
                 # Generate embedding
                 embedding_result = self._generate_chunk_embedding(chunk)
-                
+
                 if embedding_result:
                     # Update chunk with embedding
                     with transaction.atomic():
@@ -107,13 +109,13 @@ class ArchiveVectorizationService(BaseService):
                         chunk.token_count = embedding_result.tokens
                         chunk.embedding_cost = embedding_result.cost
                         chunk.save()
-                        
+
                         # Update item statistics
                         item = chunk.item
                         item.total_tokens += embedding_result.tokens
                         item.processing_cost += embedding_result.cost
                         item.save()
-                    
+
                     result.vectorized_count += 1
                     result.total_tokens += embedding_result.tokens
                     logger.debug(f"✅ Chunk {chunk.id} vectorized successfully: {embedding_result.tokens} tokens, ${embedding_result.cost:.4f}")
@@ -123,28 +125,28 @@ class ArchiveVectorizationService(BaseService):
                     error_msg = f"Failed to generate embedding for chunk {chunk.id}"
                     result.errors.append(error_msg)
                     logger.error(f"❌ {error_msg}")
-            
+
             except Exception as e:
                 result.failed_count += 1
                 error_msg = f"Error processing chunk {chunk.id}: {str(e)}"
                 result.errors.append(error_msg)
                 logger.error(f"❌ {error_msg}")
                 continue
-        
+
         return result
-    
+
     def _generate_chunk_embedding(self, chunk: ArchiveItemChunk) -> Optional[EmbeddingResponse]:
         """Generate embedding for a single chunk."""
-        
+
         if not chunk.content or not chunk.content.strip():
             return None
-        
+
         try:
             # Prepare content for embedding
             content_for_embedding = self._prepare_content_for_embedding(chunk)
-            
+
             logger.debug(f"🔮 Prepared content for embedding: {len(content_for_embedding)} chars")
-            
+
             # Generate embedding using LLM client with specified model
             from django_cfg.apps.knowbase.utils.chunk_settings import get_embedding_model
             embedding_model = get_embedding_model()
@@ -152,35 +154,35 @@ class ArchiveVectorizationService(BaseService):
                 text=content_for_embedding,
                 model=embedding_model
             )
-            
+
             if embedding_result:
                 logger.debug(f"🔮 Embedding generated successfully for chunk {chunk.id}")
                 logger.debug(f"🔮 Embedding result structure: {list(embedding_result.keys()) if isinstance(embedding_result, dict) else type(embedding_result)}")
             else:
                 logger.warning(f"🔮 Embedding generation returned None for chunk {chunk.id}")
-            
+
             return embedding_result
-            
+
         except Exception as e:
             # Log error but don't raise - we want to continue with other chunks
             logger.error(f"🔮 Error generating embedding for chunk {chunk.id}: {str(e)}", exc_info=True)
             return None
-    
+
     def _prepare_content_for_embedding(self, chunk: ArchiveItemChunk) -> str:
         """Prepare chunk content for embedding generation."""
-        
+
         content = chunk.content
         context = chunk.context_metadata
-        
+
         # Add context information to improve embedding quality
         context_prefix = self._build_context_prefix(context)
-        
+
         # Combine context and content
         if context_prefix:
             enhanced_content = f"{context_prefix}\n\n{content}"
         else:
             enhanced_content = content
-        
+
         # Ensure content is not too long for embedding model
         max_length = 8000  # Conservative limit for most embedding models
         if len(enhanced_content) > max_length:
@@ -191,48 +193,48 @@ class ArchiveVectorizationService(BaseService):
                 enhanced_content = f"{context_prefix}\n\n{truncated_content}"
             else:
                 enhanced_content = content[:max_length] + "..."
-        
+
         return enhanced_content
-    
+
     def _build_context_prefix(self, context: Dict[str, Any]) -> str:
         """Build context prefix to enhance embedding quality."""
-        
+
         prefix_parts = []
-        
+
         # Archive context
         archive_info = context.get('archive_info', {})
         if archive_info.get('title'):
             prefix_parts.append(f"Archive: {archive_info['title']}")
-        
+
         # Item context
         item_info = context.get('item_info', {})
         if item_info.get('relative_path'):
             prefix_parts.append(f"File: {item_info['relative_path']}")
-        
+
         if item_info.get('content_type'):
             prefix_parts.append(f"Type: {item_info['content_type']}")
-        
+
         if item_info.get('language'):
             prefix_parts.append(f"Language: {item_info['language']}")
-        
+
         # Structure context
         structure_info = context.get('structure_info', {})
         if structure_info.get('element_name'):
             prefix_parts.append(f"Element: {structure_info['element_name']}")
-        
+
         if structure_info.get('section_title'):
             prefix_parts.append(f"Section: {structure_info['section_title']}")
-        
+
         # Semantic context
         semantic_info = context.get('semantic_info', {})
         if semantic_info.get('content_purpose'):
             prefix_parts.append(f"Purpose: {semantic_info['content_purpose']}")
-        
+
         return " | ".join(prefix_parts) if prefix_parts else ""
-    
+
     def vectorize_single_chunk(self, chunk_id: str) -> Dict[str, Any]:
         """Vectorize a single chunk by ID."""
-        
+
         try:
             chunk = ArchiveItemChunk.objects.get(id=chunk_id, user=self.user)
         except ArchiveItemChunk.DoesNotExist:
@@ -241,7 +243,7 @@ class ArchiveVectorizationService(BaseService):
                 code="CHUNK_NOT_FOUND",
                 details={"chunk_id": chunk_id}
             )
-        
+
         # Check if already vectorized
         if chunk.embedding is not None and len(chunk.embedding) > 0:
             return {
@@ -250,49 +252,49 @@ class ArchiveVectorizationService(BaseService):
                 'token_count': chunk.token_count,
                 'cost': chunk.embedding_cost
             }
-        
+
         # Generate embedding
         embedding_result = self._generate_chunk_embedding(chunk)
-        
+
         if not embedding_result:
             raise VectorizationError(
                 message=f"Failed to generate embedding for chunk {chunk_id}",
                 code="EMBEDDING_GENERATION_FAILED",
                 details={"chunk_id": chunk_id}
             )
-        
+
         # Update chunk
         with transaction.atomic():
             chunk.embedding = embedding_result.embedding
             chunk.token_count = embedding_result.tokens
             chunk.embedding_cost = embedding_result.cost
             chunk.save()
-            
+
             # Update item statistics
             item = chunk.item
             item.total_tokens += embedding_result.tokens
             item.processing_cost += embedding_result.cost
             item.save()
-        
+
         return {
             'status': 'vectorized',
             'chunk_id': chunk_id,
             'token_count': embedding_result.tokens,
             'cost': embedding_result.cost
         }
-    
+
     def get_vectorization_statistics(self, archive_id: Optional[str] = None) -> Dict[str, Any]:
         """Get vectorization statistics for user's chunks."""
-        
+
         queryset = ArchiveItemChunk.objects.filter(user=self.user)
-        
+
         if archive_id:
             queryset = queryset.filter(archive_id=archive_id)
-        
+
         total_chunks = queryset.count()
         vectorized_chunks = queryset.filter(embedding__isnull=False).count()
         pending_chunks = total_chunks - vectorized_chunks
-        
+
         # Aggregate statistics
         stats = queryset.aggregate(
             total_tokens=models.Sum('token_count'),
@@ -300,7 +302,7 @@ class ArchiveVectorizationService(BaseService):
             avg_tokens_per_chunk=models.Avg('token_count'),
             avg_cost_per_chunk=models.Avg('embedding_cost')
         )
-        
+
         return {
             'total_chunks': total_chunks,
             'vectorized_chunks': vectorized_chunks,
@@ -311,51 +313,52 @@ class ArchiveVectorizationService(BaseService):
             'avg_tokens_per_chunk': stats['avg_tokens_per_chunk'] or 0.0,
             'avg_cost_per_chunk': stats['avg_cost_per_chunk'] or 0.0
         }
-    
+
     def revectorize_chunks(
-        self, 
-        chunk_ids: List[str], 
+        self,
+        chunk_ids: List[str],
         force: bool = False
     ) -> Dict[str, Any]:
         """Re-vectorize specific chunks."""
-        
+
         chunks = ArchiveItemChunk.objects.filter(
             id__in=chunk_ids,
             user=self.user
         )
-        
+
         if not force:
             # Only re-vectorize chunks that don't have embeddings
             chunks = chunks.filter(embedding__isnull=True)
-        
+
         return self.vectorize_chunks_batch(list(chunks))
-    
+
     def cleanup_failed_vectorizations(self) -> Dict[str, Any]:
         """Clean up chunks that failed vectorization."""
-        
+
         # Find chunks without embeddings that are older than 1 hour
-        from django.utils import timezone
         from datetime import timedelta
-        
+
+        from django.utils import timezone
+
         cutoff_time = timezone.now() - timedelta(hours=1)
-        
+
         failed_chunks = ArchiveItemChunk.objects.filter(
             user=self.user,
             embedding__isnull=True,
             created_at__lt=cutoff_time
         )
-        
+
         failed_count = failed_chunks.count()
-        
+
         # Attempt to re-vectorize
         if failed_count > 0:
             result = self.vectorize_chunks_batch(list(failed_chunks))
-            
+
             return {
                 'found_failed_chunks': failed_count,
                 'retry_result': result
             }
-        
+
         return {
             'found_failed_chunks': 0,
             'retry_result': None

django_cfg/apps/knowbase/services/base.py

@@ -2,19 +2,21 @@
 Base service classes and protocols.
 """
 
-from typing import Protocol, Optional, List, Dict, Any
+from abc import ABC
+from typing import Any, Dict, List, Optional, Protocol
+
 from django.contrib.auth import get_user_model
+
 from django_cfg.modules.django_llm.llm.client import LLMClient
-from django.conf import settings
-from abc import ABC
-from ..config.settings import get_openai_api_key, get_openrouter_api_key, get_cache_settings
+
+from ..config.settings import get_cache_settings
 
 User = get_user_model()
 
 
 class LLMServiceProtocol(Protocol):
     """Protocol for LLM service dependency injection."""
-    
+
     def generate_embedding(self, text: str) -> List[float]: ...
     def chat_completion(self, messages: List[Dict[str, str]]) -> Dict[str, Any]: ...
     def count_tokens(self, text: str, model: str = None) -> int: ...
@@ -23,7 +25,7 @@ class LLMServiceProtocol(Protocol):
 
 class CacheServiceProtocol(Protocol):
     """Protocol for cache service."""
-    
+
     def get(self, key: str) -> Optional[str]: ...
     def set(self, key: str, value: str, ttl: int = 3600) -> None: ...
     def delete(self, key: str) -> None: ...
@@ -31,7 +33,7 @@ class CacheServiceProtocol(Protocol):
 
 class BaseService(ABC):
     """Base service with common functionality."""
-    
+
     def __init__(self, user: User):
         self.user = user
         # Initialize LLM client with configuration
@@ -41,12 +43,12 @@ class BaseService(ABC):
             cache_ttl=cache_settings.cache_ttl,
             max_cache_size=cache_settings.max_cache_size
         )
-    
+
     def _ensure_user_access(self, obj) -> None:
         """Ensure user has access to object."""
         if hasattr(obj, 'user') and obj.user != self.user:
             raise PermissionError(f"User {self.user.id} cannot access this resource")
-    
+
     def _generate_content_hash(self, content: str) -> str:
         """Generate SHA-256 hash for content."""
         import hashlib