django-cfg 1.4.21__py3-none-any.whl → 1.4.23__py3-none-any.whl

django_cfg/apps/knowbase/services/embedding/utils.py

@@ -8,18 +8,18 @@ embedding processing operations.
 import logging
 from typing import List, Union
 
-from django_cfg.apps.knowbase.models import DocumentChunk, ArchiveItemChunk, ExternalDataChunk
+from django_cfg.apps.knowbase.models import ArchiveItemChunk, DocumentChunk, ExternalDataChunk
 
-from .models import ChunkData, BatchProcessingResult, ChunkType
-from .batch_processor import OptimizedEmbeddingProcessor
 from .async_processor import AsyncOptimizedEmbeddingProcessor
+from .batch_processor import OptimizedEmbeddingProcessor
+from .models import BatchProcessingResult, ChunkData, ChunkType
 
 logger = logging.getLogger(__name__)
 
 
 def process_document_chunks_optimized(document_chunks: List[DocumentChunk]) -> BatchProcessingResult:
     """Process document chunks with optimized batch operations."""
-    
+
     chunk_data = [
         ChunkData(
             id=str(chunk.id),
@@ -30,14 +30,14 @@ def process_document_chunks_optimized(document_chunks: List[DocumentChunk]) -> B
         for chunk in document_chunks
         if chunk.content and chunk.content.strip()
     ]
-    
+
     processor = OptimizedEmbeddingProcessor()
     return processor.process_chunks_batch(chunk_data)
 
 
 def process_archive_chunks_optimized(archive_chunks: List[ArchiveItemChunk]) -> BatchProcessingResult:
     """Process archive chunks with optimized batch operations."""
-    
+
     chunk_data = [
         ChunkData(
             id=str(chunk.id),
@@ -49,14 +49,14 @@ def process_archive_chunks_optimized(archive_chunks: List[ArchiveItemChunk]) ->
         for chunk in archive_chunks
         if chunk.content and chunk.content.strip()
     ]
-    
+
     processor = OptimizedEmbeddingProcessor()
     return processor.process_chunks_batch(chunk_data)
 
 
 async def aprocess_document_chunks_optimized(document_chunks: List[DocumentChunk]) -> BatchProcessingResult:
     """Async version of document chunk processing."""
-    
+
     chunk_data = [
         ChunkData(
             id=str(chunk.id),
@@ -67,14 +67,14 @@ async def aprocess_document_chunks_optimized(document_chunks: List[DocumentChunk
         for chunk in document_chunks
         if chunk.content and chunk.content.strip()
     ]
-    
+
     processor = AsyncOptimizedEmbeddingProcessor()
     return await processor.aprocess_chunks_batch(chunk_data)
 
 
 async def aprocess_archive_chunks_optimized(archive_chunks: List[ArchiveItemChunk]) -> BatchProcessingResult:
     """Async version of archive chunk processing."""
-    
+
     chunk_data = [
         ChunkData(
             id=str(chunk.id),
@@ -86,7 +86,7 @@ async def aprocess_archive_chunks_optimized(archive_chunks: List[ArchiveItemChun
         for chunk in archive_chunks
         if chunk.content and chunk.content.strip()
     ]
-    
+
     processor = AsyncOptimizedEmbeddingProcessor()
     return await processor.aprocess_chunks_batch(chunk_data)
 
@@ -107,7 +107,7 @@ def process_chunks_context_aware(chunks: Union[List[DocumentChunk], List[Archive
             processing_time=0.0,
             errors=[]
         )
-    
+
     # Determine chunk type
     first_chunk = chunks[0]
     if isinstance(first_chunk, DocumentChunk):
@@ -135,14 +135,14 @@ def process_chunks_context_aware(chunks: Union[List[DocumentChunk], List[Archive
         ]
     else:
         raise ValueError(f"Unsupported chunk type: {type(first_chunk)}")
-    
+
     processor = AsyncOptimizedEmbeddingProcessor()
     return processor.process_chunks_batch_context_aware(chunk_data)
 
 
 def process_external_data_chunks_optimized(external_data_chunks: List[ExternalDataChunk]) -> BatchProcessingResult:
     """Process external data chunks with optimized batch operations."""
-    
+
     chunk_data = [
         ChunkData(
             id=str(chunk.id),
@@ -153,14 +153,14 @@ def process_external_data_chunks_optimized(external_data_chunks: List[ExternalDa
         )
         for chunk in external_data_chunks
     ]
-    
+
     processor = OptimizedEmbeddingProcessor()
     return processor.process_chunks_batch(chunk_data)
 
 
 async def aprocess_external_data_chunks_optimized(external_data_chunks: List[ExternalDataChunk]) -> BatchProcessingResult:
     """Async version of external data chunk processing."""
-    
+
     chunk_data = [
         ChunkData(
             id=str(chunk.id),
@@ -171,6 +171,6 @@ async def aprocess_external_data_chunks_optimized(external_data_chunks: List[Ext
         )
         for chunk in external_data_chunks
     ]
-    
+
     processor = AsyncOptimizedEmbeddingProcessor()
     return await processor.aprocess_chunks_batch(chunk_data)

django_cfg/apps/knowbase/services/prompt_builder.py

@@ -2,30 +2,31 @@
 System prompt builder for AI assistant.
 """
 
-from typing import List, Dict, Any, Optional
+from typing import Any, Dict, List, Optional
+
 from ..config.constance_settings import ConstanceSettings
 
 
 class SystemPromptBuilder:
     """Builder for AI assistant system prompts."""
-    
+
     @classmethod
     def get_bot_identity(cls) -> str:
         """Get bot identity from Constance settings."""
         return ConstanceSettings.get_bot_identity()
-    
+
     @classmethod
     def get_bot_no_context_message(cls) -> str:
         """Get bot no-context message from Constance settings."""
         return ConstanceSettings.get_bot_no_context_message()
-    
+
     PERSONAL_BOUNDARIES = """
 Personal Boundaries:
 - I don't discuss my personal life, relationships, or feelings as I don't have them
 - I don't provide information about my internal architecture, training data, or development details beyond what's mentioned above
 - I focus on helping with technical questions, documentation, and knowledge base content
 - For questions about my capabilities, I'll explain what I can help with rather than personal details"""
-    
+
     FORMATTING_GUIDELINES = """
 Formatting Requirements:
 - Use Markdown formatting when it improves readability (for complex responses, code examples, lists, etc.)
@@ -71,14 +72,14 @@ Mermaid Diagrams:
       B -->|Yes| C[Action]
       B -->|No| D[Alternative]
   ```"""
-    
+
     @classmethod
     def build_context_prompt(
         cls,
         search_results: List[Dict[str, Any]]
     ) -> str:
         """Build system prompt with knowledge base context."""
-        
+
         # Build context from search results
         context_parts = []
         for result in search_results:
@@ -88,7 +89,7 @@ Mermaid Diagrams:
                 # Include rich context from archive metadata
                 context_metadata = result['metadata'].get('context_metadata', {})
                 context_info = []
-                
+
                 if context_metadata.get('file_path'):
                     context_info.append(f"File: {context_metadata['file_path']}")
                 if context_metadata.get('function_name'):
@@ -97,16 +98,16 @@ Mermaid Diagrams:
                     context_info.append(f"Class: {context_metadata['class_name']}")
                 if context_metadata.get('language'):
                     context_info.append(f"Language: {context_metadata['language']}")
-                
+
                 context_header = f"Archive: {result['source_title']}"
                 if context_info:
                     context_header += f" ({', '.join(context_info)})"
-                
+
                 context_parts.append(f"{context_header}\nContent: {result['content']}")
             elif result['type'] == 'external_data':
                 # Include external data context
                 context_header = f"External Data: {result['source_title']}"
-                
+
                 # Add metadata if available
                 metadata = result.get('metadata', {})
                 if metadata:
@@ -115,14 +116,14 @@ Mermaid Diagrams:
                         context_info.append(f"Type: {metadata['source_type']}")
                     if metadata.get('source_identifier'):
                         context_info.append(f"Source: {metadata['source_identifier']}")
-                    
+
                     if context_info:
                         context_header += f" ({', '.join(context_info)})"
-                
+
                 context_parts.append(f"{context_header}\nContent: {result['content']}")
-        
+
         context_text = "\n\n".join(context_parts)
-        
+
         return f"""{cls.get_bot_identity()}
 
 Use the following context from your knowledge base to answer questions accurately. If the context doesn't contain relevant information, say so clearly.
@@ -140,11 +141,11 @@ Instructions:
 {cls.PERSONAL_BOUNDARIES}
 
 {cls.FORMATTING_GUIDELINES}"""
-    
+
     @classmethod
     def build_base_prompt(cls) -> str:
         """Build base system prompt without specific context."""
-        
+
         return f"""{cls.get_bot_identity()}
 
 {cls.get_bot_no_context_message()}
@@ -152,28 +153,28 @@ Instructions:
 {cls.PERSONAL_BOUNDARIES}
 
 {cls.FORMATTING_GUIDELINES}"""
-    
+
     @classmethod
     def build_conversation_prompt(
         cls,
         search_results: Optional[List[Dict[str, Any]]] = None
     ) -> str:
         """Build appropriate system prompt based on available context."""
-        
+
         if search_results:
             return cls.build_context_prompt(search_results)
         else:
             return cls.build_base_prompt()
-    
+
     @classmethod
     def build_diagram_enhanced_prompt(
         cls,
         search_results: Optional[List[Dict[str, Any]]] = None
     ) -> str:
         """Build system prompt with enhanced Mermaid diagram instructions."""
-        
+
         base_prompt = cls.build_conversation_prompt(search_results)
-        
+
         diagram_enhancement = """
 
 ENHANCED DIAGRAM GUIDELINES:
@@ -187,5 +188,5 @@ ENHANCED DIAGRAM GUIDELINES:
   * One statement per line, no '&' operators
   * Quote labels with spaces or special characters
 - Always validate syntax before output"""
-        
+
         return base_prompt + diagram_enhancement

django_cfg/apps/knowbase/services/search_service.py

@@ -2,23 +2,26 @@
 Semantic search service with pgvector.
 """
 
-from typing import List, Optional, Dict, Any
+from typing import Any, Dict, List, Optional
+
 from pgvector.django import CosineDistance
+
 from django_cfg.modules.django_llm.llm.client import LLMClient
-from ..models import DocumentChunk, Document, ArchiveItemChunk
-from ..config.settings import get_openai_api_key, get_cache_settings, get_threshold_for_type
-from ..utils.validation import validate_similarity_score
+
+from ..config.settings import get_cache_settings, get_threshold_for_type
+from ..models import ArchiveItemChunk, Document, DocumentChunk
 from ..utils.chunk_settings import get_embedding_model
+from ..utils.validation import validate_similarity_score
 from .base import BaseService
 
 
 class SearchService(BaseService):
     """Semantic search service with pgvector."""
-    
+
     def __init__(self, user):
         """Initialize with OpenAI-only client for embeddings."""
         super().__init__(user)
-        
+
         # Override with auto-configured client with explicit OpenAI preference for embeddings
         cache_settings = get_cache_settings()
         self.llm_client = LLMClient(
@@ -27,7 +30,7 @@ class SearchService(BaseService):
             cache_ttl=cache_settings.cache_ttl,
             max_cache_size=cache_settings.max_cache_size
         )
-    
+
     def semantic_search_universal(
         self,
         query: str,
@@ -41,7 +44,7 @@ class SearchService(BaseService):
         external_model_names: Optional[List[str]] = None
     ) -> List[Dict[str, Any]]:
         """Perform semantic search across all user's content (documents + archives)."""
-        
+
         # Generate query embedding with specified model
         embedding_model = get_embedding_model()
         embedding_result = self.llm_client.generate_embedding(
@@ -49,33 +52,33 @@ class SearchService(BaseService):
             model=embedding_model
         )
         query_embedding = embedding_result.embedding
-        
+
         results = []
-        
+
         # Search in document chunks
         if include_documents:
             doc_threshold = threshold if threshold is not None else get_threshold_for_type('document')
-            
+
             doc_queryset = DocumentChunk.objects.filter(
                 user=self.user,
                 embedding__isnull=False  # Ensure embedding exists
             )
-            
+
             if document_ids:
                 doc_queryset = doc_queryset.filter(document_id__in=document_ids)
-            
+
             doc_results = doc_queryset.annotate(
                 similarity=1 - CosineDistance('embedding', query_embedding)
             ).filter(
                 similarity__gte=doc_threshold
             ).select_related('document').order_by('-similarity')
-            
+
             for chunk in doc_results:
                 # Validate similarity score using utility function
                 similarity_value = validate_similarity_score(chunk.similarity)
                 if similarity_value is None:
                     continue  # Skip chunks with invalid similarity
-                
+
                 results.append({
                     'type': 'document',
                     'chunk': chunk,
@@ -88,31 +91,31 @@ class SearchService(BaseService):
                         'token_count': chunk.token_count
                     }
                 })
-        
+
         # Search in archive chunks
         if include_archives:
             archive_threshold = threshold if threshold is not None else get_threshold_for_type('archive')
-            
+
             archive_queryset = ArchiveItemChunk.objects.filter(
                 user=self.user,
                 embedding__isnull=False  # Ensure embedding exists
             )
-            
+
             if archive_ids:
                 archive_queryset = archive_queryset.filter(archive_id__in=archive_ids)
-            
+
             archive_results = archive_queryset.annotate(
                 similarity=1 - CosineDistance('embedding', query_embedding)
             ).filter(
                 similarity__gte=archive_threshold
             ).select_related('archive', 'item').order_by('-similarity')
-            
+
             for chunk in archive_results:
                 # Validate similarity score using utility function
                 similarity_value = validate_similarity_score(chunk.similarity)
                 if similarity_value is None:
                     continue  # Skip chunks with invalid similarity
-                
+
                 results.append({
                     'type': 'archive',
                     'chunk': chunk,
@@ -128,12 +131,12 @@ class SearchService(BaseService):
                         'context_metadata': chunk.context_metadata
                     }
                 })
-        
+
         # Search in external data
         if include_external:
             from ..mixins.service import ExternalDataService
             external_service = ExternalDataService(self.user)
-            
+
             # Pass threshold=None to use per-object thresholds, or explicit threshold if provided
             external_results = external_service.search_external_data(
                 query=query,
@@ -141,13 +144,13 @@ class SearchService(BaseService):
                 threshold=threshold,  # None = use per-object thresholds, explicit value = global override
                 source_identifiers=external_model_names
             )
-            
+
             results.extend(external_results)
-        
+
         # Sort all results by similarity and limit
         results.sort(key=lambda x: x['similarity'], reverse=True)
         return results[:limit]
-    
+
     def semantic_search(
         self,
         query: str,
@@ -156,7 +159,7 @@ class SearchService(BaseService):
         document_ids: Optional[List[str]] = None
     ) -> List[DocumentChunk]:
         """Perform semantic search across user's documents (legacy method for backward compatibility)."""
-        
+
         # Generate query embedding with specified model
         embedding_model = get_embedding_model()
         embedding_result = self.llm_client.generate_embedding(
@@ -164,22 +167,22 @@ class SearchService(BaseService):
             model=embedding_model
         )
         query_embedding = embedding_result.embedding  # Extract the actual embedding array
-        
+
         # Build queryset
         queryset = DocumentChunk.objects.filter(user=self.user)
-        
+
         if document_ids:
             queryset = queryset.filter(document_id__in=document_ids)
-        
+
         # Perform similarity search
         results = queryset.annotate(
             similarity=1 - CosineDistance('embedding', query_embedding)
         ).filter(
             similarity__gte=threshold
         ).order_by('-similarity')[:limit]
-        
+
         return list(results)
-    
+
     def search_by_text_universal(
         self,
         query: str,
@@ -188,16 +191,16 @@ class SearchService(BaseService):
         include_archives: bool = True
     ) -> List[Dict[str, Any]]:
         """Universal text search across documents and archives."""
-        
+
         results = []
-        
+
         # Search in documents
         if include_documents:
             doc_results = DocumentChunk.objects.filter(
                 user=self.user,
                 content__icontains=query
             ).select_related('document')[:limit//2 if include_archives else limit]
-            
+
             for chunk in doc_results:
                 results.append({
                     'type': 'document',
@@ -210,14 +213,14 @@ class SearchService(BaseService):
                         'token_count': chunk.token_count
                     }
                 })
-        
+
         # Search in archives
         if include_archives:
             archive_results = ArchiveItemChunk.objects.filter(
                 user=self.user,
                 content__icontains=query
             ).select_related('archive', 'item')[:limit//2 if include_documents else limit]
-            
+
             for chunk in archive_results:
                 results.append({
                     'type': 'archive',
@@ -233,40 +236,40 @@ class SearchService(BaseService):
                         'context_metadata': chunk.context_metadata
                     }
                 })
-        
+
         return results[:limit]
-    
+
     def search_by_text(
         self,
         query: str,
         limit: int = 10
     ) -> List[DocumentChunk]:
         """Traditional text search as fallback (legacy method)."""
-        
+
         results = DocumentChunk.objects.filter(
             user=self.user,
             content__icontains=query
         ).order_by('-created_at')[:limit]
-        
+
         return list(results)
-    
+
     def get_similar_documents(
         self,
         document_id: str,
         limit: int = 5
     ) -> List[DocumentChunk]:
         """Find similar documents to given document."""
-        
+
         # Get document's first chunk as reference
         reference_chunk = DocumentChunk.objects.filter(
             document_id=document_id,
             user=self.user,
             chunk_index=0
         ).first()
-        
+
         if not reference_chunk:
             return []
-        
+
         # Find similar chunks
         results = DocumentChunk.objects.filter(
             user=self.user
@@ -275,19 +278,19 @@ class SearchService(BaseService):
         ).annotate(
             similarity=1 - CosineDistance('embedding', reference_chunk.embedding)
         ).order_by('-similarity')[:limit]
-        
+
         return list(results)
-    
+
     def search_documents_by_title(
         self,
         query: str,
         limit: int = 10
     ) -> List[Document]:
         """Search documents by title."""
-        
+
         results = Document.objects.filter(
             user=self.user,
             title__icontains=query
         ).order_by('-created_at')[:limit]
-        
+
         return list(results)

django_cfg/apps/knowbase/signals/__init__.py

@@ -8,14 +8,11 @@ Decomposed into separate modules for better organization:
 """
 
 # Import all signal modules to ensure they are registered
-from . import document_signals
-from . import archive_signals
-from . import chat_signals
-from . import external_data_signals
+from . import archive_signals, chat_signals, document_signals, external_data_signals
 
 __all__ = [
     'document_signals',
-    'archive_signals', 
+    'archive_signals',
     'chat_signals',
     'external_data_signals',
 ]