django-cfg 1.4.21__py3-none-any.whl → 1.4.24__py3-none-any.whl

django_cfg/apps/knowbase/services/chat_service.py

@@ -2,23 +2,24 @@
 RAG-powered chat service.
 """
 
-from typing import List, Dict, Any, Optional
-from django.utils import timezone
+from typing import Any, Dict, List
+
 from django_cfg.modules.django_llm.llm.models import ChatCompletionResponse
-from ..models import ChatSession, ChatMessage, DocumentChunk
+
+from ..models import ChatMessage, ChatSession
 from ..utils.validation import clean_search_results, safe_float
 from .base import BaseService
-from .search_service import SearchService
 from .prompt_builder import SystemPromptBuilder
+from .search_service import SearchService
 
 
 class ChatService(BaseService):
     """RAG-powered chat service with context management."""
-    
+
     def __init__(self, user):
         super().__init__(user)
         self.search_service = SearchService(user)
-    
+
     def create_session(
         self,
         title: str = "",
@@ -27,7 +28,7 @@ class ChatService(BaseService):
         max_context_chunks: int = 5
     ) -> ChatSession:
         """Create new chat session."""
-        
+
         session = ChatSession.objects.create(
             user=self.user,
             title=title or "New Chat Session",
@@ -36,9 +37,9 @@ class ChatService(BaseService):
             max_context_chunks=max_context_chunks,
             is_active=True
         )
-        
+
         return session
-    
+
     def process_query(
         self,
         session_id: str,
@@ -48,14 +49,14 @@ class ChatService(BaseService):
         enable_diagrams: bool = False
     ) -> Dict[str, Any]:
         """Process chat query with RAG context."""
-        
+
         # Get session
         session = ChatSession.objects.get(
             id=session_id,
             user=self.user,
             is_active=True
         )
-        
+
         # Perform universal semantic search for context (documents + archives + external data)
         # Using type-specific thresholds automatically
         raw_search_results = self.search_service.semantic_search_universal(
@@ -66,10 +67,10 @@ class ChatService(BaseService):
             include_archives=True,
             include_external=True
         )
-        
+
         # Clean search results to remove invalid similarity scores
         search_results = clean_search_results(raw_search_results)
-        
+
         # Build context messages
         context_messages = self._build_context_messages(
             session=session,
@@ -77,7 +78,7 @@ class ChatService(BaseService):
             search_results=search_results,
             enable_diagrams=enable_diagrams
         )
-        
+
         # Generate LLM response (now returns ChatCompletionResponse Pydantic model)
         response: ChatCompletionResponse = self.llm_client.chat_completion(
             messages=context_messages,
@@ -85,7 +86,7 @@ class ChatService(BaseService):
             temperature=session.temperature,
             max_tokens=max_tokens
         )
-        
+
         # Save user message
         context_chunk_ids = []
         for result in search_results:
@@ -95,7 +96,7 @@ class ChatService(BaseService):
                 context_chunk_ids.append(f"archive:{result['chunk'].id}")
             elif result['type'] == 'external_data':
                 context_chunk_ids.append(f"external:{result['chunk'].id}")
-        
+
         user_message = ChatMessage.objects.create(
             session=session,
             user=self.user,
@@ -103,7 +104,7 @@ class ChatService(BaseService):
             content=query,
             context_chunks=context_chunk_ids
         )
-        
+
         # Save assistant response
         assistant_message = ChatMessage.objects.create(
             session=session,
@@ -116,17 +117,17 @@ class ChatService(BaseService):
             model_name=session.model_name,
             finish_reason=response.finish_reason
         )
-        
+
         # Update session statistics (messages_count is handled by signals)
         session.total_tokens_used += response.tokens_used
         session.total_cost_usd = safe_float(session.total_cost_usd, 0.0) + safe_float(response.cost_usd, 0.0)
         session.save()
-        
+
         # Auto-generate session title if empty
         if not session.title or session.title == "New Chat Session":
             session.title = query[:50] + "..." if len(query) > 50 else query
             session.save()
-        
+
         result = {
             'message_id': str(assistant_message.id),
             'content': response.content,
@@ -135,7 +136,7 @@ class ChatService(BaseService):
             'processing_time_ms': int(response.processing_time * 1000),
             'model_used': session.model_name
         }
-        
+
         if include_sources:
             # Search results are already cleaned by clean_search_results()
             result['sources'] = [
@@ -148,9 +149,9 @@ class ChatService(BaseService):
                 }
                 for search_result in search_results
             ]
-        
+
         return result
-    
+
     def _build_context_messages(
         self,
         session: ChatSession,
@@ -159,9 +160,9 @@ class ChatService(BaseService):
         enable_diagrams: bool = False
     ) -> List[Dict[str, str]]:
         """Build context messages for LLM."""
-        
+
         messages = []
-        
+
         # Build system message using SystemPromptBuilder
         if enable_diagrams:
             system_message = SystemPromptBuilder.build_diagram_enhanced_prompt(
@@ -171,61 +172,61 @@ class ChatService(BaseService):
             system_message = SystemPromptBuilder.build_conversation_prompt(
                 search_results=search_results if search_results else None
             )
-        
+
         messages.append({
             "role": "system",
             "content": system_message
         })
-        
+
         # Add recent conversation history (last 5 messages)
         recent_messages = list(ChatMessage.objects.filter(
             session=session
         ).order_by('-created_at')[:5])
-        
+
         # Reverse to get chronological order
         for message in reversed(recent_messages):
             messages.append({
                 "role": message.role,
                 "content": message.content
             })
-        
+
         # Add current query
         messages.append({
             "role": "user",
             "content": query
         })
-        
+
         return messages
-    
+
     def get_session_history(
         self,
         session_id: str,
         limit: int = 50
     ) -> List[ChatMessage]:
         """Get chat session message history."""
-        
+
         # Verify session access
         session = ChatSession.objects.get(
             id=session_id,
             user=self.user
         )
-        
+
         messages = ChatMessage.objects.filter(
             session=session
         ).order_by('created_at')[:limit]
-        
+
         return list(messages)
-    
+
     def list_sessions(self, active_only: bool = True) -> List[ChatSession]:
         """List user chat sessions."""
-        
+
         queryset = ChatSession.objects.filter(user=self.user)
-        
+
         if active_only:
             queryset = queryset.filter(is_active=True)
-        
+
         return list(queryset.order_by('-created_at'))
-    
+
     def delete_session(self, session_id: str) -> bool:
         """Delete chat session and all messages."""
         try:

django_cfg/apps/knowbase/services/document_service.py

@@ -2,16 +2,17 @@
 Document management service.
 """
 
-from typing import List, Optional, Dict, Any
-from django.db import transaction, models
-from django.utils import timezone
+from typing import Any, Dict, List, Optional
+
+from django.db import models, transaction
+
 from ..models import Document, DocumentChunk, ProcessingStatus
 from .base import BaseService
 
 
 class DocumentService(BaseService):
     """Service for document management and processing."""
-    
+
     def create_document(
         self,
         title: str,
@@ -20,19 +21,19 @@ class DocumentService(BaseService):
         metadata: Optional[Dict[str, Any]] = None
     ) -> Document:
         """Create document and trigger async processing."""
-        
+
         # Generate content hash for duplicate detection
         content_hash = self._generate_content_hash(content)
-        
+
         # Check for duplicates
         existing = Document.objects.filter(
             user=self.user,
             content_hash=content_hash
         ).first()
-        
+
         if existing:
             raise ValueError(f"Document with same content already exists: {existing.title}")
-        
+
         # Create document (async processing will be triggered by post_save signal)
         document = Document.objects.create(
             user=self.user,
@@ -44,9 +45,9 @@ class DocumentService(BaseService):
             metadata=metadata or {},
             processing_status=ProcessingStatus.PENDING
         )
-        
+
         return document
-    
+
     def get_document(self, document_id: str) -> Optional[Document]:
         """Get document by ID with user access check."""
         try:
@@ -57,16 +58,16 @@ class DocumentService(BaseService):
             return document
         except Document.DoesNotExist:
             return None
-    
+
     def get_user_documents(self, status: Optional[str] = None):
         """Get user documents queryset with filtering."""
         queryset = Document.objects.filter(user=self.user)
-        
+
         if status:
             queryset = queryset.filter(processing_status=status)
-        
+
         return queryset.order_by('-created_at')
-    
+
     def list_documents(
         self,
         status: Optional[str] = None,
@@ -76,7 +77,7 @@ class DocumentService(BaseService):
         """List user documents with filtering."""
         queryset = self.get_user_documents(status)
         return list(queryset[offset:offset + limit])
-    
+
     def delete_document(self, document_id: str) -> bool:
         """Delete document and all associated chunks."""
         try:
@@ -85,22 +86,22 @@ class DocumentService(BaseService):
                     id=document_id,
                     user=self.user
                 )
-                
+
                 # Delete associated chunks first
                 DocumentChunk.objects.filter(document=document).delete()
-                
+
                 # Delete document
                 document.delete()
-                
+
                 return True
         except Document.DoesNotExist:
             return False
-    
+
     def get_processing_stats(self) -> Dict[str, Any]:
         """Get user's document processing statistics."""
-        
-        from django.db.models import Count, Sum, Avg
-        
+
+        from django.db.models import Count, Sum
+
         stats = Document.objects.filter(user=self.user).aggregate(
             total_documents=Count('id'),
             completed_documents=Count('id', filter=models.Q(processing_status=ProcessingStatus.COMPLETED)),
@@ -108,7 +109,7 @@ class DocumentService(BaseService):
             total_tokens=Sum('total_tokens'),
             total_cost=Sum('total_cost_usd'),
         )
-        
+
         return {
             'total_documents': stats['total_documents'] or 0,
             'completed_documents': stats['completed_documents'] or 0,
@@ -121,7 +122,7 @@ class DocumentService(BaseService):
             'total_cost_usd': float(stats['total_cost'] or 0),
             'avg_processing_time_seconds': 0.0  # Calculated separately if needed
         }
-    
+
     def reprocess_document(self, document_id: str) -> bool:
         """Trigger document reprocessing."""
         try:
@@ -129,16 +130,16 @@ class DocumentService(BaseService):
                 id=document_id,
                 user=self.user
             )
-            
+
             # Reset processing status
             document.processing_status = ProcessingStatus.PENDING
             document.processing_error = ""
             document.save()
-            
+
             # Trigger async reprocessing
             from ..tasks import reprocess_document_chunks
             reprocess_document_chunks.send(str(document.id))
-            
+
             return True
         except Document.DoesNotExist:
             return False

django_cfg/apps/knowbase/services/embedding/__init__.py

@@ -5,36 +5,36 @@ This package provides high-performance embedding generation services
 for documents and archives with batch processing and async support.
 """
 
-from .models import ChunkData, EmbeddingResult, BatchProcessingResult, ProcessingConfig, ChunkType
-from .processors import DocumentChunkProcessor, ArchiveChunkProcessor, ExternalDataChunkProcessor
-from .batch_processor import OptimizedEmbeddingProcessor
 from .async_processor import AsyncOptimizedEmbeddingProcessor
+from .batch_processor import OptimizedEmbeddingProcessor
 from .batch_result import BatchResultBuilder
+from .models import BatchProcessingResult, ChunkData, ChunkType, EmbeddingResult, ProcessingConfig
+from .processors import ArchiveChunkProcessor, DocumentChunkProcessor, ExternalDataChunkProcessor
 from .utils import (
-    process_document_chunks_optimized,
     process_archive_chunks_optimized,
-    process_external_data_chunks_optimized,
     process_chunks_context_aware,
+    process_document_chunks_optimized,
+    process_external_data_chunks_optimized,
 )
 
 __all__ = [
     # Data models
     "ChunkData",
-    "EmbeddingResult", 
+    "EmbeddingResult",
     "BatchProcessingResult",
     "ProcessingConfig",
     "ChunkType",
-    
+
     # Processors
     "DocumentChunkProcessor",
     "ArchiveChunkProcessor",
     "ExternalDataChunkProcessor",
     "OptimizedEmbeddingProcessor",
     "AsyncOptimizedEmbeddingProcessor",
-    
+
     # Utilities
     "BatchResultBuilder",
-    
+
     # Convenience functions
     "process_document_chunks_optimized",
     "process_archive_chunks_optimized",

django_cfg/apps/knowbase/services/embedding/async_processor.py

@@ -6,18 +6,16 @@ following Django 5.2 async best practices.
 """
 
 import asyncio
-import time
 import logging
-from typing import List, Dict, Any, Optional, Union
+import time
+from typing import Any, Dict, List, Optional
 
-from asgiref.sync import sync_to_async, async_to_sync
-from django.db import transaction
+from asgiref.sync import async_to_sync
 
-from django_cfg.apps.knowbase.models import DocumentChunk, ArchiveItemChunk
-from django_cfg.apps.knowbase.utils.chunk_settings import get_embedding_batch_size, get_embedding_model
+from django_cfg.apps.knowbase.models import ArchiveItemChunk, DocumentChunk
 
-from .models import ChunkData, BatchProcessingResult, EmbeddingResult
 from .batch_processor import OptimizedEmbeddingProcessor
+from .models import BatchProcessingResult, ChunkData, EmbeddingResult
 
 logger = logging.getLogger(__name__)
 
@@ -40,12 +38,12 @@ class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
     - Proper async/sync method selection
     - Compatible with both WSGI and ASGI
     """
-    
+
     def __init__(self, batch_size: Optional[int] = None, embedding_model: Optional[str] = None):
         """Initialize async-compatible processor."""
         super().__init__(batch_size, embedding_model)
         logger.info(f"🚀 AsyncOptimizedEmbeddingProcessor initialized: async_context={is_async_context()}")
-    
+
     async def aprocess_chunks_batch(self, chunks: List[ChunkData]) -> BatchProcessingResult:
         """
         Async version of batch processing.
@@ -59,38 +57,38 @@ class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
         total_tokens = 0
         total_cost = 0.0
         errors = []
-        
+
         logger.info(f"🔮 Starting async batch processing of {total_chunks} chunks")
-        
+
         # Process in batches
         for i in range(0, total_chunks, self.batch_size):
             batch = chunks[i:i + self.batch_size]
             batch_num = (i // self.batch_size) + 1
             total_batches = (total_chunks + self.batch_size - 1) // self.batch_size
-            
+
             logger.info(f"🔮 Processing async batch {batch_num}/{total_batches} ({len(batch)} chunks)")
-            
+
             try:
                 batch_result = await self._aprocess_single_batch(batch)
-                
+
                 successful_chunks += batch_result['successful']
                 failed_chunks += batch_result['failed']
                 total_tokens += batch_result['tokens']
                 total_cost += batch_result['cost']
                 errors.extend(batch_result['errors'])
-                
+
                 # Small async delay between batches
                 if i + self.batch_size < total_chunks:
                     await asyncio.sleep(0.5)
-                    
+
             except Exception as e:
                 error_msg = f"Async batch {batch_num} failed: {str(e)}"
                 logger.error(f"❌ {error_msg}")
                 errors.append(error_msg)
                 failed_chunks += len(batch)
-        
+
         processing_time = time.time() - start_time
-        
+
         result = BatchProcessingResult(
             total_chunks=total_chunks,
             successful_chunks=successful_chunks,
@@ -100,27 +98,27 @@ class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
             processing_time=processing_time,
             errors=errors
         )
-        
+
         logger.info(
             f"🎉 Async batch processing completed: {successful_chunks}/{total_chunks} successful, "
             f"{total_tokens} tokens, ${total_cost:.4f} cost, {processing_time:.2f}s"
         )
-        
+
         return result
-    
+
     async def _aprocess_single_batch(self, batch: List[ChunkData]) -> Dict[str, Any]:
         """Async version of single batch processing."""
-        
+
         # Prepare content for all chunks (sync operation)
         prepared_contents = []
         chunk_mapping = {}
-        
+
         for idx, chunk in enumerate(batch):
             processor = self.processors.get(chunk.parent_type)
             if not processor:
                 logger.warning(f"⚠️ Unknown chunk type: {chunk.parent_type}")
                 continue
-            
+
             try:
                 content = processor.prepare_content_for_embedding(chunk)
                 if content and content.strip():
@@ -130,7 +128,7 @@ class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
                     logger.warning(f"⚠️ Empty content for chunk {chunk.id}")
             except Exception as e:
                 logger.error(f"❌ Failed to prepare content for chunk {chunk.id}: {e}")
-        
+
         if not prepared_contents:
             return {
                 'successful': 0,
@@ -139,24 +137,24 @@ class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
                 'cost': 0.0,
                 'errors': ['No valid content to process']
             }
-        
+
         # Generate embeddings (sync operation - OpenAI client is sync)
         try:
             embedding_results = self._generate_batch_embeddings(prepared_contents)
-            
+
             # Save results using async database operations
             successful = 0
             failed = 0
             total_tokens = 0
             total_cost = 0.0
             errors = []
-            
+
             for idx, embedding_result in enumerate(embedding_results):
                 if idx not in chunk_mapping:
                     continue
-                
+
                 chunk = chunk_mapping[idx]
-                
+
                 if embedding_result.success:
                     try:
                         await self._asave_embedding_result(chunk, embedding_result)
@@ -171,7 +169,7 @@ class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
                 else:
                     errors.append(embedding_result.error or f"Failed to generate embedding for chunk {chunk.id}")
                     failed += 1
-            
+
             return {
                 'successful': successful,
                 'failed': failed,
@@ -179,7 +177,7 @@ class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
                 'cost': total_cost,
                 'errors': errors
             }
-            
+
         except Exception as e:
             error_msg = f"Async batch embedding generation failed: {e}"
             logger.error(f"❌ {error_msg}")
@@ -190,10 +188,10 @@ class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
                 'cost': 0.0,
                 'errors': [error_msg]
             }
-    
+
     async def _asave_embedding_result(self, chunk: ChunkData, result: EmbeddingResult) -> None:
         """Save embedding result using async database operations."""
-        
+
         try:
             if chunk.parent_type == "document":
                 # Use Django 5.2 async ORM methods
@@ -202,9 +200,9 @@ class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
                 chunk_obj.token_count = result.tokens
                 chunk_obj.embedding_cost = result.cost
                 await chunk_obj.asave(update_fields=['embedding', 'token_count', 'embedding_cost'])
-                
+
                 logger.debug(f"✅ Async document chunk {chunk.id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
-                
+
             elif chunk.parent_type == "archive":
                 # Use async ORM with select_related
                 chunk_obj = await ArchiveItemChunk.objects.select_related('item').aget(id=chunk.id)
@@ -212,21 +210,21 @@ class AsyncOptimizedEmbeddingProcessor(OptimizedEmbeddingProcessor):
                 chunk_obj.token_count = result.tokens
                 chunk_obj.embedding_cost = result.cost
                 await chunk_obj.asave(update_fields=['embedding', 'token_count', 'embedding_cost'])
-                
+
                 # Update parent item statistics
                 item = chunk_obj.item
                 item.total_tokens += result.tokens
                 item.processing_cost += result.cost
                 await item.asave(update_fields=['total_tokens', 'processing_cost'])
-                
+
                 logger.debug(f"✅ Async archive chunk {chunk.id} embedding saved: {result.tokens} tokens, ${result.cost:.4f}")
             else:
                 raise ValueError(f"Unknown chunk type: {chunk.parent_type}")
-                
+
         except Exception as e:
             logger.error(f"❌ Failed to save async embedding for chunk {chunk.id}: {e}")
             raise
-    
+
     def process_chunks_batch_context_aware(self, chunks: List[ChunkData]) -> BatchProcessingResult:
         """
         Context-aware processing that works in both sync and async contexts.