django-cfg 1.1.82__py3-none-any.whl → 1.2.1__py3-none-any.whl

django_cfg/apps/knowbase/services/prompt_builder.py

@@ -0,0 +1,191 @@
+"""
+System prompt builder for AI assistant.
+"""
+
+from typing import List, Dict, Any, Optional
+from ..config.constance_settings import ConstanceSettings
+
+
+class SystemPromptBuilder:
+    """Builder for AI assistant system prompts."""
+    
+    @classmethod
+    def get_bot_identity(cls) -> str:
+        """Get bot identity from Constance settings."""
+        return ConstanceSettings.get_bot_identity()
+    
+    @classmethod
+    def get_bot_no_context_message(cls) -> str:
+        """Get bot no-context message from Constance settings."""
+        return ConstanceSettings.get_bot_no_context_message()
+    
+    PERSONAL_BOUNDARIES = """
+Personal Boundaries:
+- I don't discuss my personal life, relationships, or feelings as I don't have them
+- I don't provide information about my internal architecture, training data, or development details beyond what's mentioned above
+- I focus on helping with technical questions, documentation, and knowledge base content
+- For questions about my capabilities, I'll explain what I can help with rather than personal details"""
+    
+    FORMATTING_GUIDELINES = """
+Formatting Requirements:
+- Use Markdown formatting when it improves readability (for complex responses, code examples, lists, etc.)
+- For code blocks, always specify the language for proper syntax highlighting:
+  ```python
+  # Python code here
+  ```
+  ```javascript
+  // JavaScript code here
+  ```
+  ```typescript
+  // TypeScript code here
+  ```
+  ```sql
+  -- SQL code here
+  ```
+  ```json
+  {"key": "value"}
+  ```
+  ```bash
+  # Shell commands here
+  ```
+  ```yaml
+  # YAML configuration here
+  ```
+- Use appropriate language tags: python, javascript, typescript, java, go, rust, php, sql, json, yaml, xml, html, css, bash, shell, dockerfile, etc.
+- Use **bold** for important terms and *italic* for emphasis when helpful
+- Use `inline code` for variable names, function names, and short code snippets
+- Use proper headings (##, ###) to structure complex responses
+- Use bullet points (-) or numbered lists (1.) when organizing multiple items
+
+Mermaid Diagrams:
+- When appropriate, you can include Mermaid diagrams to visualize processes, flows, or relationships
+- Always use proper syntax: start with diagram type (flowchart TD, graph LR, sequenceDiagram)
+- Node syntax: A[Label] (rectangles), A{Label} (diamonds), A((Label)) (circles)
+- Connections: A --> B (arrows), A --- B (lines), A -.-> B (dotted)
+- Never use 'end' as lowercase node ID, avoid IDs starting with 'o' or 'x'
+- Quote labels with spaces, one statement per line, specify direction (TD/LR/RL/BT)
+- Example:
+  ```mermaid
+  flowchart TD
+      A[Start] --> B{Decision}
+      B -->|Yes| C[Action]
+      B -->|No| D[Alternative]
+  ```"""
+    
+    @classmethod
+    def build_context_prompt(
+        cls,
+        search_results: List[Dict[str, Any]]
+    ) -> str:
+        """Build system prompt with knowledge base context."""
+        
+        # Build context from search results
+        context_parts = []
+        for result in search_results:
+            if result['type'] == 'document':
+                context_parts.append(f"Document: {result['source_title']}\nContent: {result['content']}")
+            elif result['type'] == 'archive':
+                # Include rich context from archive metadata
+                context_metadata = result['metadata'].get('context_metadata', {})
+                context_info = []
+                
+                if context_metadata.get('file_path'):
+                    context_info.append(f"File: {context_metadata['file_path']}")
+                if context_metadata.get('function_name'):
+                    context_info.append(f"Function: {context_metadata['function_name']}")
+                if context_metadata.get('class_name'):
+                    context_info.append(f"Class: {context_metadata['class_name']}")
+                if context_metadata.get('language'):
+                    context_info.append(f"Language: {context_metadata['language']}")
+                
+                context_header = f"Archive: {result['source_title']}"
+                if context_info:
+                    context_header += f" ({', '.join(context_info)})"
+                
+                context_parts.append(f"{context_header}\nContent: {result['content']}")
+            elif result['type'] == 'external_data':
+                # Include external data context
+                context_header = f"External Data: {result['source_title']}"
+                
+                # Add metadata if available
+                metadata = result.get('metadata', {})
+                if metadata:
+                    context_info = []
+                    if metadata.get('source_type'):
+                        context_info.append(f"Type: {metadata['source_type']}")
+                    if metadata.get('source_identifier'):
+                        context_info.append(f"Source: {metadata['source_identifier']}")
+                    
+                    if context_info:
+                        context_header += f" ({', '.join(context_info)})"
+                
+                context_parts.append(f"{context_header}\nContent: {result['content']}")
+        
+        context_text = "\n\n".join(context_parts)
+        
+        return f"""{cls.get_bot_identity()}
+
+Use the following context from your knowledge base to answer questions accurately. If the context doesn't contain relevant information, say so clearly.
+
+Context:
+{context_text}
+
+Instructions:
+- Answer based on the provided context
+- Be concise and accurate
+- If context is insufficient, acknowledge this
+- Cite specific documents or files when possible
+- For code-related questions, reference the specific files and functions
+
+{cls.PERSONAL_BOUNDARIES}
+
+{cls.FORMATTING_GUIDELINES}"""
+    
+    @classmethod
+    def build_base_prompt(cls) -> str:
+        """Build base system prompt without specific context."""
+        
+        return f"""{cls.get_bot_identity()}
+
+{cls.get_bot_no_context_message()}
+
+{cls.PERSONAL_BOUNDARIES}
+
+{cls.FORMATTING_GUIDELINES}"""
+    
+    @classmethod
+    def build_conversation_prompt(
+        cls,
+        search_results: Optional[List[Dict[str, Any]]] = None
+    ) -> str:
+        """Build appropriate system prompt based on available context."""
+        
+        if search_results:
+            return cls.build_context_prompt(search_results)
+        else:
+            return cls.build_base_prompt()
+    
+    @classmethod
+    def build_diagram_enhanced_prompt(
+        cls,
+        search_results: Optional[List[Dict[str, Any]]] = None
+    ) -> str:
+        """Build system prompt with enhanced Mermaid diagram instructions."""
+        
+        base_prompt = cls.build_conversation_prompt(search_results)
+        
+        diagram_enhancement = """
+
+ENHANCED DIAGRAM GUIDELINES:
+- Prioritize visual explanations when describing processes, architectures, or workflows
+- Use Mermaid diagrams for: system flows, decision trees, database schemas, API interactions, class relationships
+- Critical syntax rules:
+  * Start: flowchart TD/LR, graph TD/LR, sequenceDiagram, classDiagram
+  * Nodes: A[Rectangle], A{Diamond}, A((Circle)), A>Flag], A[/Parallelogram/]
+  * Never use lowercase 'end' as node ID → use 'End' or 'END'
+  * Avoid node IDs starting with 'o' or 'x'
+  * One statement per line, no '&' operators
+  * Quote labels with spaces or special characters
+- Always validate syntax before output"""
+        
+        return base_prompt + diagram_enhancement

django_cfg/apps/knowbase/services/search_service.py

@@ -0,0 +1,293 @@
+"""
+Semantic search service with pgvector.
+"""
+
+from typing import List, Optional, Dict, Any
+from pgvector.django import CosineDistance
+from django_cfg.modules.django_llm.llm.client import LLMClient
+from ..models import DocumentChunk, Document, ArchiveItemChunk
+from ..config.settings import get_openai_api_key, get_cache_settings, get_threshold_for_type
+from ..utils.validation import validate_similarity_score
+from ..utils.chunk_settings import get_embedding_model
+from .base import BaseService
+
+
+class SearchService(BaseService):
+    """Semantic search service with pgvector."""
+    
+    def __init__(self, user):
+        """Initialize with OpenAI-only client for embeddings."""
+        super().__init__(user)
+        
+        # Override with auto-configured client with explicit OpenAI preference for embeddings
+        cache_settings = get_cache_settings()
+        self.llm_client = LLMClient(
+            preferred_provider="openai",  # Force OpenAI for embeddings
+            cache_dir=cache_settings.cache_dir,
+            cache_ttl=cache_settings.cache_ttl,
+            max_cache_size=cache_settings.max_cache_size
+        )
+    
+    def semantic_search_universal(
+        self,
+        query: str,
+        limit: int = 5,
+        threshold: Optional[float] = None,  # Now optional, will use type-specific thresholds
+        document_ids: Optional[List[str]] = None,
+        archive_ids: Optional[List[str]] = None,
+        include_documents: bool = True,
+        include_archives: bool = True,
+        include_external: bool = True,
+        external_model_names: Optional[List[str]] = None
+    ) -> List[Dict[str, Any]]:
+        """Perform semantic search across all user's content (documents + archives)."""
+        
+        # Generate query embedding with specified model
+        embedding_model = get_embedding_model()
+        embedding_result = self.llm_client.generate_embedding(
+            text=query,
+            model=embedding_model
+        )
+        query_embedding = embedding_result.embedding
+        
+        results = []
+        
+        # Search in document chunks
+        if include_documents:
+            doc_threshold = threshold if threshold is not None else get_threshold_for_type('document')
+            
+            doc_queryset = DocumentChunk.objects.filter(
+                user=self.user,
+                embedding__isnull=False  # Ensure embedding exists
+            )
+            
+            if document_ids:
+                doc_queryset = doc_queryset.filter(document_id__in=document_ids)
+            
+            doc_results = doc_queryset.annotate(
+                similarity=1 - CosineDistance('embedding', query_embedding)
+            ).filter(
+                similarity__gte=doc_threshold
+            ).select_related('document').order_by('-similarity')
+            
+            for chunk in doc_results:
+                # Validate similarity score using utility function
+                similarity_value = validate_similarity_score(chunk.similarity)
+                if similarity_value is None:
+                    continue  # Skip chunks with invalid similarity
+                
+                results.append({
+                    'type': 'document',
+                    'chunk': chunk,
+                    'similarity': similarity_value,
+                    'source_title': chunk.document.title,
+                    'content': chunk.content,
+                    'metadata': {
+                        'document_id': str(chunk.document.id),
+                        'chunk_index': chunk.chunk_index,
+                        'token_count': chunk.token_count
+                    }
+                })
+        
+        # Search in archive chunks
+        if include_archives:
+            archive_threshold = threshold if threshold is not None else get_threshold_for_type('archive')
+            
+            archive_queryset = ArchiveItemChunk.objects.filter(
+                user=self.user,
+                embedding__isnull=False  # Ensure embedding exists
+            )
+            
+            if archive_ids:
+                archive_queryset = archive_queryset.filter(archive_id__in=archive_ids)
+            
+            archive_results = archive_queryset.annotate(
+                similarity=1 - CosineDistance('embedding', query_embedding)
+            ).filter(
+                similarity__gte=archive_threshold
+            ).select_related('archive', 'item').order_by('-similarity')
+            
+            for chunk in archive_results:
+                # Validate similarity score using utility function
+                similarity_value = validate_similarity_score(chunk.similarity)
+                if similarity_value is None:
+                    continue  # Skip chunks with invalid similarity
+                
+                results.append({
+                    'type': 'archive',
+                    'chunk': chunk,
+                    'similarity': similarity_value,
+                    'source_title': f"{chunk.archive.title} / {chunk.item.item_name}",
+                    'content': chunk.content,
+                    'metadata': {
+                        'archive_id': str(chunk.archive.id),
+                        'item_id': str(chunk.item.id),
+                        'chunk_index': chunk.chunk_index,
+                        'token_count': chunk.token_count,
+                        'chunk_type': chunk.chunk_type,
+                        'context_metadata': chunk.context_metadata
+                    }
+                })
+        
+        # Search in external data
+        if include_external:
+            from ..mixins.service import ExternalDataService
+            external_service = ExternalDataService(self.user)
+            
+            # Pass threshold=None to use per-object thresholds, or explicit threshold if provided
+            external_results = external_service.search_external_data(
+                query=query,
+                limit=limit,
+                threshold=threshold,  # None = use per-object thresholds, explicit value = global override
+                source_identifiers=external_model_names
+            )
+            
+            results.extend(external_results)
+        
+        # Sort all results by similarity and limit
+        results.sort(key=lambda x: x['similarity'], reverse=True)
+        return results[:limit]
+    
+    def semantic_search(
+        self,
+        query: str,
+        limit: int = 5,
+        threshold: float = 0.7,
+        document_ids: Optional[List[str]] = None
+    ) -> List[DocumentChunk]:
+        """Perform semantic search across user's documents (legacy method for backward compatibility)."""
+        
+        # Generate query embedding with specified model
+        embedding_model = get_embedding_model()
+        embedding_result = self.llm_client.generate_embedding(
+            text=query,
+            model=embedding_model
+        )
+        query_embedding = embedding_result.embedding  # Extract the actual embedding array
+        
+        # Build queryset
+        queryset = DocumentChunk.objects.filter(user=self.user)
+        
+        if document_ids:
+            queryset = queryset.filter(document_id__in=document_ids)
+        
+        # Perform similarity search
+        results = queryset.annotate(
+            similarity=1 - CosineDistance('embedding', query_embedding)
+        ).filter(
+            similarity__gte=threshold
+        ).order_by('-similarity')[:limit]
+        
+        return list(results)
+    
+    def search_by_text_universal(
+        self,
+        query: str,
+        limit: int = 10,
+        include_documents: bool = True,
+        include_archives: bool = True
+    ) -> List[Dict[str, Any]]:
+        """Universal text search across documents and archives."""
+        
+        results = []
+        
+        # Search in documents
+        if include_documents:
+            doc_results = DocumentChunk.objects.filter(
+                user=self.user,
+                content__icontains=query
+            ).select_related('document')[:limit//2 if include_archives else limit]
+            
+            for chunk in doc_results:
+                results.append({
+                    'type': 'document',
+                    'chunk': chunk,
+                    'source_title': chunk.document.title,
+                    'content': chunk.content,
+                    'metadata': {
+                        'document_id': str(chunk.document.id),
+                        'chunk_index': chunk.chunk_index,
+                        'token_count': chunk.token_count
+                    }
+                })
+        
+        # Search in archives
+        if include_archives:
+            archive_results = ArchiveItemChunk.objects.filter(
+                user=self.user,
+                content__icontains=query
+            ).select_related('archive', 'item')[:limit//2 if include_documents else limit]
+            
+            for chunk in archive_results:
+                results.append({
+                    'type': 'archive',
+                    'chunk': chunk,
+                    'source_title': f"{chunk.archive.title} / {chunk.item.item_name}",
+                    'content': chunk.content,
+                    'metadata': {
+                        'archive_id': str(chunk.archive.id),
+                        'item_id': str(chunk.item.id),
+                        'chunk_index': chunk.chunk_index,
+                        'token_count': chunk.token_count,
+                        'chunk_type': chunk.chunk_type,
+                        'context_metadata': chunk.context_metadata
+                    }
+                })
+        
+        return results[:limit]
+    
+    def search_by_text(
+        self,
+        query: str,
+        limit: int = 10
+    ) -> List[DocumentChunk]:
+        """Traditional text search as fallback (legacy method)."""
+        
+        results = DocumentChunk.objects.filter(
+            user=self.user,
+            content__icontains=query
+        ).order_by('-created_at')[:limit]
+        
+        return list(results)
+    
+    def get_similar_documents(
+        self,
+        document_id: str,
+        limit: int = 5
+    ) -> List[DocumentChunk]:
+        """Find similar documents to given document."""
+        
+        # Get document's first chunk as reference
+        reference_chunk = DocumentChunk.objects.filter(
+            document_id=document_id,
+            user=self.user,
+            chunk_index=0
+        ).first()
+        
+        if not reference_chunk:
+            return []
+        
+        # Find similar chunks
+        results = DocumentChunk.objects.filter(
+            user=self.user
+        ).exclude(
+            document_id=document_id
+        ).annotate(
+            similarity=1 - CosineDistance('embedding', reference_chunk.embedding)
+        ).order_by('-similarity')[:limit]
+        
+        return list(results)
+    
+    def search_documents_by_title(
+        self,
+        query: str,
+        limit: int = 10
+    ) -> List[Document]:
+        """Search documents by title."""
+        
+        results = Document.objects.filter(
+            user=self.user,
+            title__icontains=query
+        ).order_by('-created_at')[:limit]
+        
+        return list(results)

django_cfg/apps/knowbase/signals/__init__.py

@@ -0,0 +1,21 @@
+"""
+Django signals for knowledge base events.
+
+Decomposed into separate modules for better organization:
+- document_signals: Document and DocumentChunk related signals
+- archive_signals: Archive processing signals  
+- chat_signals: Chat and messaging signals
+"""
+
+# Import all signal modules to ensure they are registered
+from . import document_signals
+from . import archive_signals
+from . import chat_signals
+from . import external_data_signals
+
+__all__ = [
+    'document_signals',
+    'archive_signals', 
+    'chat_signals',
+    'external_data_signals',
+]