django-cfg 1.1.82__py3-none-any.whl → 1.2.0__py3-none-any.whl

django_cfg/apps/knowbase/signals/archive_signals.py

@@ -0,0 +1,211 @@
+"""
+Archive processing signals.
+"""
+
+from django.db.models.signals import post_save, post_delete
+from django.dispatch import receiver
+from django.core.cache import cache
+import logging
+
+from ..models import DocumentArchive, ArchiveItem, ArchiveItemChunk
+
+logger = logging.getLogger(__name__)
+
+
+@receiver(post_save, sender=DocumentArchive)
+def archive_post_save(sender, instance, created, **kwargs):
+    """Handle archive creation and updates."""
+    
+    # Clear user's archive cache on any archive change
+    cache_key = f"user_archives:{instance.user.id}"
+    cache.delete(cache_key)
+    
+    if created and instance.archive_file:
+        # New archive with file - start processing
+        logger.info(f"📦 New archive created: {instance.title} (ID: {instance.id})")
+        _start_archive_processing(instance)
+        
+    elif not created:
+        # Archive update - check what changed
+        update_fields = kwargs.get('update_fields')
+        
+        # Define fields that, if updated alone, should NOT trigger reprocessing
+        processing_fields = {
+            'processing_status', 'processed_at', 'processing_duration_ms',
+            'processing_error', 'total_items', 'processed_items', 'total_chunks',
+            'vectorized_chunks', 'total_tokens', 'total_cost_usd'
+        }
+        
+        # If specific fields were updated, check if they are only processing-related
+        if update_fields is not None:
+            update_fields_set = set(update_fields) if update_fields else set()
+            
+            if update_fields_set and update_fields_set.issubset(processing_fields):
+                # This is just a processing status update - don't reprocess
+                logger.debug(f"📊 Archive stats updated: {instance.title}")
+                return
+            
+            # Check if archive file was updated
+            if 'archive_file' in update_fields_set and instance.archive_file:
+                logger.info(f"📦 Archive file updated: {instance.title} (ID: {instance.id})")
+                
+                # Clear existing items and reset processing state
+                _reset_archive_processing(instance)
+                
+                # Start new processing
+                _start_archive_processing(instance)
+            else:
+                logger.debug(f"📊 Archive non-file update: {instance.title} (fields: {update_fields_set})")
+        else:
+            # Full save without update_fields - check if file exists and process if needed
+            if instance.archive_file and instance.processing_status == 'pending':
+                logger.info(f"📦 Archive saved, checking if processing needed: {instance.title}")
+                _start_archive_processing(instance)
+
+
+def _reset_archive_processing(archive):
+    """Reset archive processing state and clear items."""
+    logger.debug(f"🧹 Clearing items for archive: {archive.title}")
+    
+    # Delete existing items (cascades to chunks)
+    archive.items.all().delete()
+    
+    # Reset processing fields
+    archive.processing_status = "pending"
+    archive.processed_at = None
+    archive.processing_duration_ms = 0
+    archive.processing_error = ""
+    archive.total_items = 0
+    archive.processed_items = 0
+    archive.total_chunks = 0
+    archive.vectorized_chunks = 0
+    archive.total_tokens = 0
+    archive.total_cost_usd = 0
+    
+    # Save with explicit update_fields to avoid triggering this signal again
+    archive.save(update_fields=[
+        'processing_status', 'processed_at', 'processing_duration_ms',
+        'processing_error', 'total_items', 'processed_items', 'total_chunks',
+        'vectorized_chunks', 'total_tokens', 'total_cost_usd'
+    ])
+
+
+def _start_archive_processing(archive):
+    """Start async archive processing."""
+    try:
+        # Lazy import to avoid middleware initialization issues
+        from ..tasks.archive_tasks import process_archive_task
+        process_archive_task.send(str(archive.id), str(archive.user.id))
+        logger.info(f"🚀 Started async processing for archive: {archive.id}")
+        
+    except Exception as e:
+        logger.error(f"❌ Failed to start archive processing: {e}")
+        
+        # Update archive status to failed
+        archive.processing_status = "failed"
+        archive.processing_error = f"Failed to start processing: {e}"
+        archive.save(update_fields=['processing_status', 'processing_error'])
+
+
+@receiver(post_save, sender=ArchiveItem)
+def archive_item_post_save(sender, instance, created, **kwargs):
+    """Handle archive item creation."""
+    if created:
+        logger.debug(f"📄 New archive item created: {instance.archive.title} - {instance.item_name}")
+        
+        # Update archive item count
+        archive = instance.archive
+        archive.total_items = archive.items.count()
+        archive.save(update_fields=['total_items'])
+
+
+@receiver(post_delete, sender=ArchiveItem)
+def archive_item_post_delete(sender, instance, **kwargs):
+    """Handle archive item deletion."""
+    try:
+        # Safely get archive - it might be deleted already due to cascade
+        try:
+            archive = instance.archive
+            archive_title = archive.title
+        except (AttributeError, DocumentArchive.DoesNotExist):
+            logger.debug("⚠️ Archive already deleted, skipping item count update")
+            return
+            
+        logger.debug(f"🗑️ Archive item deleted: {archive_title} - {instance.item_name}")
+        
+        # Update archive item count
+        archive.total_items = archive.items.count()
+        archive.save(update_fields=['total_items'])
+        
+    except DocumentArchive.DoesNotExist:
+        # Archive was already deleted
+        logger.debug("Archive already deleted, skipping item count update")
+    except Exception as e:
+        logger.error(f"❌ Error in archive item post-delete signal: {e}")
+        # Don't re-raise to avoid breaking deletion
+
+
+@receiver(post_save, sender=ArchiveItemChunk)
+def archive_chunk_post_save(sender, instance, created, **kwargs):
+    """Handle archive chunk creation."""
+    if created:
+        logger.debug(f"🧩 New archive chunk created: {instance.item.item_name} chunk {instance.chunk_index}")
+        
+        # Update archive chunk count
+        archive = instance.archive
+        archive.total_chunks = ArchiveItemChunk.objects.filter(archive=archive).count()
+        
+        # Update vectorized chunks count
+        archive.vectorized_chunks = ArchiveItemChunk.objects.filter(
+            archive=archive, 
+            embedding__isnull=False
+        ).count()
+        
+        archive.save(update_fields=['total_chunks', 'vectorized_chunks'])
+
+
+@receiver(post_delete, sender=ArchiveItemChunk)
+def archive_chunk_post_delete(sender, instance, **kwargs):
+    """Handle archive chunk deletion."""
+    try:
+        # Safely get item name - item might be deleted already due to cascade
+        try:
+            item_name = instance.item.item_name if hasattr(instance, 'item') and instance.item else "unknown"
+        except (AttributeError, ArchiveItem.DoesNotExist):
+            item_name = "unknown"
+            
+        logger.debug(f"🗑️ Archive chunk deleted: {item_name} chunk {instance.chunk_index}")
+        
+        # Update archive chunk counts - get archive through item if available
+        try:
+            if hasattr(instance, 'item') and instance.item:
+                archive = instance.item.archive
+            else:
+                # If item is already deleted, we can't update counts safely
+                logger.debug("⚠️ Cannot update chunk counts - parent item already deleted")
+                return
+                
+            archive.total_chunks = ArchiveItemChunk.objects.filter(item__archive=archive).count()
+            archive.vectorized_chunks = ArchiveItemChunk.objects.filter(
+                item__archive=archive, 
+                embedding__isnull=False
+            ).count()
+            
+            archive.save(update_fields=['total_chunks', 'vectorized_chunks'])
+            
+        except (AttributeError, ArchiveItem.DoesNotExist, DocumentArchive.DoesNotExist):
+            logger.debug("⚠️ Cannot update chunk counts - related objects already deleted")
+        
+    except Exception as e:
+        logger.error(f"❌ Error in archive chunk post-delete signal: {e}")
+        # Don't re-raise to avoid breaking deletion
+
+
+@receiver(post_delete, sender=DocumentArchive)
+def archive_post_delete(sender, instance, **kwargs):
+    """Handle archive deletion cleanup."""
+    # Clear user's archive cache
+    cache_key = f"user_archives:{instance.user.id}"
+    cache.delete(cache_key)
+    
+    logger.info(f"🗑️ Archive deleted: {instance.title} (ID: {instance.id})")

django_cfg/apps/knowbase/signals/chat_signals.py

@@ -0,0 +1,37 @@
+"""
+Chat and messaging related signals.
+"""
+
+from django.db.models.signals import post_save, post_delete
+from django.db import models
+from django.dispatch import receiver
+import logging
+
+from ..models import ChatSession, ChatMessage
+
+logger = logging.getLogger(__name__)
+
+
+@receiver(post_save, sender=ChatMessage)
+def message_post_save(sender, instance, created, **kwargs):
+    """Handle chat message creation."""
+    if created:
+        logger.debug(f"💬 New message: {instance.session.title} - {instance.role}")
+        
+        # Update session statistics
+        session = instance.session
+        session.messages_count = session.messages.count()
+        session.total_tokens_used = session.messages.aggregate(
+            total=models.Sum('tokens_used')
+        )['total'] or 0
+        session.total_cost_usd = session.messages.aggregate(
+            total=models.Sum('cost_usd')
+        )['total'] or 0
+        
+        session.save(update_fields=['messages_count', 'total_tokens_used', 'total_cost_usd'])
+
+
+@receiver(post_delete, sender=ChatSession)
+def session_post_delete(sender, instance, **kwargs):
+    """Handle chat session deletion."""
+    logger.info(f"🗑️ Chat session deleted: {instance.title} (ID: {instance.id})")

django_cfg/apps/knowbase/signals/document_signals.py

@@ -0,0 +1,143 @@
+"""
+Document and DocumentChunk related signals.
+"""
+
+from django.db.models.signals import post_save, post_delete
+from django.db import models
+from django.dispatch import receiver
+from django.core.cache import cache
+import logging
+
+from ..models import Document, DocumentChunk, ProcessingStatus
+
+logger = logging.getLogger(__name__)
+
+
+@receiver(post_save, sender=Document)
+def document_post_save(sender, instance, created, **kwargs):
+    """Handle document creation and updates."""
+    
+    # Clear user's document cache on any document change
+    cache_key = f"user_documents:{instance.user.id}"
+    cache.delete(cache_key)
+    
+    if created:
+        # New document - always process
+        logger.info(f"📄 New document created: {instance.title} (ID: {instance.id})")
+        _start_document_processing(instance)
+        
+    else:
+        # Document update - check what changed
+        update_fields = kwargs.get('update_fields')
+        
+        # Define fields that, if updated alone, should NOT trigger reprocessing
+        processing_fields = {
+            'processing_status', 'processing_started_at', 'processing_completed_at',
+            'processing_error', 'chunks_count', 'total_tokens', 'total_cost_usd'
+        }
+        
+        # If specific fields were updated, check if they are only processing-related
+        if update_fields is not None:
+            update_fields_set = set(update_fields) if update_fields else set()
+            
+            if update_fields_set and update_fields_set.issubset(processing_fields):
+                # This is just a processing status update - don't reprocess
+                logger.debug(f"📊 Document stats updated: {instance.title}")
+                return
+            
+            # Check if content actually changed
+            content_fields = {'content', 'content_hash'}
+            if content_fields.intersection(update_fields_set):
+                logger.info(f"📝 Document content updated: {instance.title} (ID: {instance.id})")
+                
+                # Clear existing chunks and reset processing state
+                _reset_document_processing(instance)
+                
+                # Start new processing
+                _start_document_processing(instance)
+            else:
+                logger.debug(f"📊 Document non-content update: {instance.title} (fields: {update_fields_set})")
+        else:
+            # Full save without update_fields specified (e.g., from admin save button or manual save())
+            # Assume content might have changed and reprocess
+            logger.info(f"📝 Document saved without specific fields, assuming content update: {instance.title} (ID: {instance.id})")
+            _reset_document_processing(instance)
+            _start_document_processing(instance)
+
+
+def _reset_document_processing(document):
+    """Reset document processing state and clear chunks."""
+    logger.debug(f"🧹 Clearing chunks for document: {document.title}")
+    
+    # Delete existing chunks
+    document.chunks.all().delete()
+    
+    # Reset processing fields
+    document.processing_status = "pending"
+    document.processing_started_at = None
+    document.processing_completed_at = None
+    document.processing_error = ""
+    document.chunks_count = 0
+    document.total_tokens = 0
+    document.total_cost_usd = 0
+    
+    # Save with explicit update_fields to avoid triggering this signal again
+    document.save(update_fields=[
+        'processing_status', 'processing_started_at', 'processing_completed_at',
+        'processing_error', 'chunks_count', 'total_tokens', 'total_cost_usd'
+    ])
+
+
+def _start_document_processing(document):
+    """Start async document processing."""
+    try:
+        # Lazy import to avoid middleware initialization issues
+        from ..tasks.document_processing import process_document_async
+        process_document_async.send(str(document.id))
+        logger.info(f"🚀 Started async processing for document: {document.id}")
+        
+    except Exception as e:
+        logger.error(f"❌ Failed to start document processing: {e}")
+        
+        # Update document status to failed
+        document.processing_status = ProcessingStatus.FAILED
+        document.processing_error = f"Failed to start processing: {e}"
+        document.save(update_fields=['processing_status', 'processing_error'])
+
+
+@receiver(post_save, sender=DocumentChunk)
+def chunk_post_save(sender, instance, created, **kwargs):
+    """Handle chunk creation."""
+    if created:
+        logger.debug(f"🧩 New chunk created: {instance.document.title} chunk {instance.chunk_index}")
+        
+        # Update document chunk count (this will trigger document save with update_fields)
+        document = instance.document
+        document.chunks_count = document.chunks.count()
+        document.save(update_fields=['chunks_count'])
+
+
+@receiver(post_delete, sender=DocumentChunk)
+def chunk_post_delete(sender, instance, **kwargs):
+    """Handle chunk deletion."""
+    try:
+        logger.debug(f"🗑️ Chunk deleted: {instance.document.title} chunk {instance.chunk_index}")
+        
+        # Update document chunk count
+        document = instance.document
+        document.chunks_count = document.chunks.count()
+        document.save(update_fields=['chunks_count'])
+        
+    except Document.DoesNotExist:
+        # Document was already deleted
+        logger.debug("Document already deleted, skipping chunk count update")
+
+
+@receiver(post_delete, sender=Document)
+def document_post_delete(sender, instance, **kwargs):
+    """Handle document deletion cleanup."""
+    # Clear user's document cache
+    cache_key = f"user_documents:{instance.user.id}"
+    cache.delete(cache_key)
+    
+    logger.info(f"🗑️ Document deleted: {instance.title} (ID: {instance.id})")

django_cfg/apps/knowbase/signals/external_data_signals.py

@@ -0,0 +1,157 @@
+"""
+External Data and ExternalDataChunk related signals.
+"""
+
+from django.db.models.signals import post_save, post_delete
+from django.db import models
+from django.dispatch import receiver
+from django.core.cache import cache
+import logging
+
+from ..models.external_data import ExternalData, ExternalDataChunk, ExternalDataStatus
+
+logger = logging.getLogger(__name__)
+
+
+@receiver(post_save, sender=ExternalData)
+def external_data_post_save(sender, instance, created, **kwargs):
+    """Handle external data creation and updates."""
+    
+    # Clear user's external data cache on any change
+    cache_key = f"user_external_data:{instance.user.id}"
+    cache.delete(cache_key)
+    
+    if created:
+        # New external data - process if has content
+        logger.info(f"🔗 New external data created: {instance.title} (ID: {instance.id})")
+        if instance.content and instance.content.strip():
+            _start_external_data_processing(instance)
+        else:
+            logger.debug(f"📝 External data created without content: {instance.title}")
+        
+    else:
+        # External data update - check what changed
+        update_fields = kwargs.get('update_fields')
+        
+        # Define fields that, if updated alone, should NOT trigger reprocessing
+        processing_fields = {
+            'status', 'processed_at', 'source_updated_at', 'processing_error',
+            'total_chunks', 'total_tokens', 'processing_cost'
+        }
+        
+        # If specific fields were updated, check if they are only processing-related
+        if update_fields is not None:
+            update_fields_set = set(update_fields) if update_fields else set()
+            
+            if update_fields_set and update_fields_set.issubset(processing_fields):
+                # This is just a processing status update - don't reprocess
+                logger.debug(f"📊 External data stats updated: {instance.title}")
+                return
+            
+            # Check if content actually changed
+            content_fields = {'content', 'content_hash', 'source_config', 'chunk_size', 'overlap_size', 'embedding_model'}
+            if content_fields.intersection(update_fields_set):
+                logger.info(f"📝 External data content updated: {instance.title} (ID: {instance.id})")
+                
+                # Only reprocess if there's content
+                if instance.content and instance.content.strip():
+                    # Clear existing chunks and reset processing state
+                    _reset_external_data_processing(instance)
+                    
+                    # Start new processing
+                    _start_external_data_processing(instance)
+                else:
+                    logger.debug(f"📝 External data updated but no content: {instance.title}")
+            else:
+                logger.debug(f"📊 External data non-content update: {instance.title} (fields: {update_fields_set})")
+        else:
+            # Full save without update_fields - check if content exists and processing is needed
+            if instance.content and instance.content.strip():
+                # Check if content hash changed (indicating content update)
+                if hasattr(instance, '_original_content_hash'):
+                    if instance.content_hash != instance._original_content_hash:
+                        logger.info(f"🔮 External data content changed (hash mismatch), reprocessing: {instance.title}")
+                        _reset_external_data_processing(instance)
+                        _start_external_data_processing(instance)
+                    else:
+                        logger.debug(f"📊 External data saved but content unchanged: {instance.title}")
+                elif instance.status == ExternalDataStatus.PENDING:
+                    logger.info(f"🔮 External data saved, checking if processing needed: {instance.title}")
+                    _start_external_data_processing(instance)
+
+
+def _reset_external_data_processing(external_data):
+    """Reset external data processing state and clear chunks."""
+    logger.debug(f"🧹 Clearing chunks for external data: {external_data.title}")
+    
+    # Delete existing chunks
+    external_data.chunks.all().delete()
+    
+    # Reset processing fields
+    external_data.status = ExternalDataStatus.PENDING
+    external_data.processed_at = None
+    external_data.processing_error = ""
+    external_data.total_chunks = 0
+    external_data.total_tokens = 0
+    external_data.processing_cost = 0.0
+    
+    # Save with explicit update_fields to avoid triggering this signal again
+    external_data.save(update_fields=[
+        'status', 'processed_at', 'processing_error',
+        'total_chunks', 'total_tokens', 'processing_cost'
+    ])
+
+
+def _start_external_data_processing(external_data):
+    """Start async external data processing."""
+    try:
+        # Lazy import to avoid middleware initialization issues
+        from ..tasks.external_data_tasks import process_external_data_async
+        process_external_data_async.send(str(external_data.id))
+        logger.info(f"🚀 Started async processing for external data: {external_data.id}")
+        
+    except Exception as e:
+        logger.error(f"❌ Failed to start external data processing: {e}")
+        
+        # Update external data status to failed
+        external_data.status = ExternalDataStatus.FAILED
+        external_data.processing_error = f"Failed to start processing: {e}"
+        external_data.save(update_fields=['status', 'processing_error'])
+
+
+@receiver(post_save, sender=ExternalDataChunk)
+def external_data_chunk_post_save(sender, instance, created, **kwargs):
+    """Handle external data chunk creation."""
+    if created:
+        logger.debug(f"🧩 New external data chunk created: {instance.external_data.title} chunk {instance.chunk_index}")
+        
+        # Update external data chunk count (this will trigger external data save with update_fields)
+        external_data = instance.external_data
+        external_data.total_chunks = external_data.chunks.count()
+        external_data.save(update_fields=['total_chunks'])
+
+
+@receiver(post_delete, sender=ExternalDataChunk)
+def external_data_chunk_post_delete(sender, instance, **kwargs):
+    """Handle external data chunk deletion."""
+    try:
+        logger.debug(f"🗑️ External data chunk deleted: {instance.external_data.title} chunk {instance.chunk_index}")
+        
+        # Update external data chunk count
+        external_data = instance.external_data
+        external_data.total_chunks = external_data.chunks.count()
+        external_data.save(update_fields=['total_chunks'])
+        
+    except ExternalData.DoesNotExist:
+        # External data was already deleted
+        logger.debug("External data already deleted, skipping chunk count update")
+
+
+@receiver(post_delete, sender=ExternalData)
+def external_data_post_delete(sender, instance, **kwargs):
+    """Handle external data deletion cleanup."""
+    # Clear user's external data cache
+    cache_key = f"user_external_data:{instance.user.id}"
+    cache.delete(cache_key)
+    
+    logger.info(f"🗑️ External data deleted: {instance.title} (ID: {instance.id})")

django_cfg/apps/knowbase/tasks/__init__.py

@@ -0,0 +1,39 @@
+"""
+Knowledge Base Background Tasks
+
+Dramatiq tasks for document processing and maintenance.
+"""
+
+from .document_processing import *
+from .maintenance import *
+from .archive_tasks import *
+from .external_data_tasks import *
+
+
+__all__ = [
+    # Document processing
+    'process_document_async',
+    'reprocess_document_chunks',
+    'generate_embeddings_batch',
+    
+    # Archive processing
+    'process_archive_task',
+    'vectorize_archive_items_task',
+    'cleanup_failed_archives_task',
+    'generate_archive_statistics_task',
+    'archive_health_check_task',
+    'test_archive_task',
+    
+    # External data processing
+    'process_external_data_async',
+    'bulk_process_external_data_async',
+    'cleanup_failed_external_data_async',
+    
+    # Maintenance
+    'cleanup_old_embeddings',
+    'optimize_vector_indexes',
+    'health_check_knowledge_base',
+    
+    # Test tasks
+    'test_simple_task',
+]