django-cfg 1.4.21__py3-none-any.whl → 1.4.24__py3-none-any.whl

django_cfg/apps/knowbase/tasks/archive_tasks.py

@@ -2,20 +2,20 @@
 Archive processing tasks with Dramatiq.
 """
 
-import dramatiq
 import logging
 import time
-from typing import Dict, Any
-from django.db import transaction
-from django.utils import timezone
+from typing import Any, Dict
+
+import dramatiq
 from django.contrib.auth import get_user_model
+from django.utils import timezone
 
-from ..models.archive import DocumentArchive, ArchiveItem, ArchiveItemChunk
+from ..models.archive import ArchiveItem, ArchiveItemChunk, DocumentArchive
 from ..models.base import ProcessingStatus
 from ..services.archive import (
-    DocumentArchiveService,
+    ArchiveProcessingError,
     ArchiveVectorizationService,
-    ArchiveProcessingError
+    DocumentArchiveService,
 )
 
 logger = logging.getLogger(__name__)
@@ -44,43 +44,43 @@ def process_archive_task(archive_id: str, user_id: str) -> bool:
         ArchiveProcessingError: If processing fails
     """
     logger.info(f"Starting archive processing for archive {archive_id}")
-    
+
     try:
         # Get archive and user
         archive = DocumentArchive.objects.all_users().get(pk=archive_id)
         user = User.objects.get(pk=user_id)
-        
+
         # Debug logging
         logger.info(f"Retrieved archive: {archive}, type: {type(archive)}")
         logger.info(f"Archive ID: {archive.id if archive else 'None'}")
         logger.info(f"Archive file: {archive.archive_file if archive else 'None'}")
-        
+
         if not archive:
             raise ArchiveProcessingError(
                 message=f"Archive {archive_id} not found or is None",
                 code="ARCHIVE_NOT_FOUND"
             )
-        
+
         # Verify user owns the archive
         if archive.user_id != user.id:
             raise ArchiveProcessingError(
                 message=f"User {user_id} does not own archive {archive_id}",
                 code="UNAUTHORIZED_ACCESS"
             )
-        
+
         # Initialize services
         service = DocumentArchiveService(user=user)
-        
+
         # Process the archive (remove transaction.atomic to avoid nested transaction conflicts)
         success = service.process_archive(archive)
-            
+
         if success:
             logger.info(f"Successfully processed archive {archive_id}")
             return True
         else:
             logger.error(f"Failed to process archive {archive_id}")
             return False
-            
+
     except DocumentArchive.DoesNotExist:
         logger.error(f"Archive {archive_id} not found")
         raise
@@ -117,28 +117,28 @@ def vectorize_archive_items_task(archive_id: str, user_id: str) -> int:
         ArchiveProcessingError: If vectorization fails
     """
     logger.info(f"Starting vectorization for archive {archive_id}")
-    
+
     try:
         # Get archive and user
         archive = DocumentArchive.objects.all_users().get(pk=archive_id)
         user = User.objects.get(pk=user_id)
-        
+
         # Verify user owns the archive
         if archive.user_id != user.id:
             raise ArchiveProcessingError(
                 message=f"User {user_id} does not own archive {archive_id}",
                 code="UNAUTHORIZED_ACCESS"
             )
-        
+
         # Initialize vectorization service
         service = ArchiveVectorizationService(user=user)
-        
+
         # Vectorize archive items
         vectorized_count = service.vectorize_archive_items(archive)
-        
+
         logger.info(f"Successfully vectorized {vectorized_count} items for archive {archive_id}")
         return vectorized_count
-        
+
     except DocumentArchive.DoesNotExist:
         logger.error(f"Archive {archive_id} not found")
         raise
@@ -169,24 +169,24 @@ def cleanup_failed_archives_task(days_old: int = 7) -> int:
         Number of archives cleaned up
     """
     logger.info(f"Starting cleanup of failed archives older than {days_old} days")
-    
+
     try:
         cutoff_date = timezone.now() - timezone.timedelta(days=days_old)
-        
+
         # Find failed archives older than cutoff
         failed_archives = DocumentArchive.objects.filter(
             processing_status=ProcessingStatus.FAILED,
             created_at__lt=cutoff_date
         )
-        
+
         count = failed_archives.count()
-        
+
         # Delete the archives (cascade will handle related objects)
         deleted_count, _ = failed_archives.delete()
-        
+
         logger.info(f"Cleaned up {deleted_count} failed archives")
         return deleted_count
-        
+
     except Exception as e:
         logger.error(f"Error during archive cleanup: {str(e)}")
         raise
@@ -208,13 +208,13 @@ def generate_archive_statistics_task(user_id: str) -> Dict[str, Any]:
         Dictionary with archive statistics
     """
     logger.info(f"Generating archive statistics for user {user_id}")
-    
+
     try:
         user = User.objects.get(pk=user_id)
-        
+
         # Get user's archives
         archives = DocumentArchive.objects.filter(user=user)
-        
+
         # Calculate statistics
         stats = {
             'total_archives': archives.count(),
@@ -226,10 +226,10 @@ def generate_archive_statistics_task(user_id: str) -> Dict[str, Any]:
             'total_chunks': sum(archive.total_chunks for archive in archives),
             'total_cost': sum(archive.total_cost_usd for archive in archives),
         }
-        
+
         logger.info(f"Generated statistics for user {user_id}: {stats}")
         return stats
-        
+
     except User.DoesNotExist:
         logger.error(f"User {user_id} not found")
         raise
@@ -251,32 +251,32 @@ def archive_health_check_task() -> Dict[str, Any]:
         Dictionary with health check results
     """
     logger.info("Starting archive system health check")
-    
+
     try:
         # Check database connectivity
         total_archives = DocumentArchive.objects.count()
-        
+
         # Check for orphaned items
         orphaned_items = ArchiveItem.objects.filter(archive__isnull=True).count()
-        
+
         # Check for orphaned chunks
         orphaned_chunks = ArchiveItemChunk.objects.filter(item__isnull=True).count()
-        
+
         # Check processing status distribution
         status_counts = {}
         for status in ProcessingStatus:
             count = DocumentArchive.objects.filter(processing_status=status).count()
             status_counts[status.value] = count
-        
+
         # Check for archives with missing files
         archives_with_files = DocumentArchive.objects.exclude(file_path__isnull=True).exclude(file_path='')
         unhealthy_archives = 0
-        
+
         for archive in archives_with_files:
             import os
             if not os.path.exists(archive.file_path):
                 unhealthy_archives += 1
-        
+
         health_data = {
             'total_checked': total_archives,
             'healthy_archives': total_archives - unhealthy_archives,
@@ -286,10 +286,10 @@ def archive_health_check_task() -> Dict[str, Any]:
             'status_distribution': status_counts,
             'timestamp': timezone.now().isoformat()
         }
-        
+
         logger.info(f"Health check completed: {health_data}")
         return health_data
-        
+
     except Exception as e:
         logger.error(f"Error during health check: {str(e)}")
         raise

django_cfg/apps/knowbase/tasks/document_processing.py

@@ -2,19 +2,20 @@
 Document processing tasks with Dramatiq.
 """
 
-import dramatiq
 import logging
 import time
-from typing import Dict, List, Any, Tuple, Optional
+from typing import Any, Dict, List, Optional, Tuple
+
+import dramatiq
 from django.db import transaction
 from django.utils import timezone
+
 from django_cfg.modules.django_llm.llm.client import LLMClient
-from django.conf import settings
 
 from ..models import Document, DocumentChunk, ProcessingStatus
-from ..utils.text_processing import TextProcessor, SemanticChunker
 from ..services.embedding import process_document_chunks_optimized
 from ..utils.chunk_settings import get_chunking_params_for_type, get_embedding_model
+from ..utils.text_processing import SemanticChunker, TextProcessor
 
 logger = logging.getLogger(__name__)
 
@@ -45,44 +46,44 @@ def process_document_async(
         Processing results with statistics
     """
     start_time = time.time()
-    
+
     try:
         with transaction.atomic():
             # Load document
             document = Document.objects.select_for_update().get(
                 id=document_id
             )
-            
+
             # Update processing status
             document.processing_status = ProcessingStatus.PROCESSING
             document.processing_started_at = timezone.now()
             document.save(update_fields=['processing_status', 'processing_started_at'])
-            
+
             logger.info(f"Starting document processing: {document_id}")
-            
+
             # Get dynamic settings from Constance
             chunking_params = get_chunking_params_for_type('document')
             final_chunk_size = chunk_size or chunking_params['chunk_size']
             final_chunk_overlap = chunk_overlap or chunking_params['overlap']
             final_embedding_model = embedding_model or get_embedding_model()
-            
+
             logger.info(f"Using dynamic settings: chunk_size={final_chunk_size}, overlap={final_chunk_overlap}, model={final_embedding_model}")
-            
+
             # Initialize services
             text_processor = TextProcessor()
             chunker = SemanticChunker(
                 chunk_size=final_chunk_size,
                 overlap=final_chunk_overlap
             )
-            
+
             # Step 1: Clean and preprocess text
             cleaned_content = text_processor.clean_text(document.content)
-            
+
             # Step 2: Create semantic chunks
             chunks = chunker.create_chunks(cleaned_content)
-            
+
             logger.info(f"Created {len(chunks)} chunks for document {document_id}")
-            
+
             # Step 3: Create chunks without embeddings first
             chunk_objects = []
             for idx, chunk_text in enumerate(chunks):
@@ -101,30 +102,30 @@ def process_document_async(
                     }
                 )
                 chunk_objects.append(chunk)
-            
+
             # Bulk create chunks for performance
             DocumentChunk.objects.bulk_create(
                 chunk_objects,
                 batch_size=100
             )
-            
+
             # Step 4: Generate embeddings using optimized processor
             created_chunks = DocumentChunk.objects.filter(document=document).order_by('chunk_index')
             chunks_list = list(created_chunks)
             logger.info(f"🔍 About to process {len(chunks_list)} chunks for embeddings")
-            
+
             embedding_result = process_document_chunks_optimized(chunks_list)
-            
+
             logger.info(f"🔍 Embedding result: {embedding_result.successful_chunks}/{embedding_result.total_chunks}")
-            
+
             total_tokens = embedding_result.total_tokens
             total_cost = embedding_result.total_cost
-            
+
             logger.info(
                 f"Optimized embedding processing: {embedding_result.successful_chunks}/{embedding_result.total_chunks} chunks, "
                 f"{total_tokens} tokens, ${total_cost:.4f} cost, {embedding_result.processing_time:.2f}s"
             )
-            
+
             # Step 5: Update document status
             processing_time = time.time() - start_time
 
@@ -144,7 +145,7 @@ def process_document_async(
                 'processing_status', 'processing_completed_at', 'processing_error',
                 'chunks_count', 'total_tokens', 'total_cost_usd'
             ])
-            
+
             return {
                 "document_id": str(document.id),
                 "status": document.processing_status.value,
@@ -154,7 +155,7 @@ def process_document_async(
                 "processing_time": processing_time,
                 "errors": embedding_result.errors
             }
-            
+
     except Document.DoesNotExist:
         logger.error(f"Document {document_id} not found.")
         return {
@@ -192,7 +193,11 @@ def generate_embeddings_batch(
         List of (chunk_text, embedding, tokens, cost) tuples
     """
     try:
-        from django_cfg.apps.knowbase.config.settings import get_openai_api_key, get_openrouter_api_key, get_cache_settings
+        from django_cfg.apps.knowbase.config.settings import (
+            get_cache_settings,
+            get_openai_api_key,
+            get_openrouter_api_key,
+        )
         cache_settings = get_cache_settings()
         llm_service = LLMClient(
             apikey_openai=get_openai_api_key(),
@@ -202,45 +207,45 @@ def generate_embeddings_batch(
             max_cache_size=cache_settings.max_cache_size
         )
         results = []
-        
+
         # Process in batches to avoid rate limits
         for i in range(0, len(chunks), batch_size):
             batch = chunks[i:i + batch_size]
-            
+
             for chunk_text in batch:
                 # Generate embedding (sync call for simplicity)
                 embedding_response = llm_service.generate_embedding(chunk_text, embedding_model)
-                
+
                 # Extract embedding vector from response
                 embedding_vector = embedding_response.embedding if embedding_response else []
-                
+
                 # Use tokens and cost from embedding response if available
                 tokens = embedding_response.tokens if embedding_response else 0
                 cost = embedding_response.cost if embedding_response else 0.0
-                
+
                 # Fallback to manual calculation if needed
                 if tokens == 0:
                     tokens = llm_service.count_tokens(chunk_text, embedding_model)
                 if cost == 0.0:
                     cost = llm_service.estimate_cost(embedding_model, tokens, 0)
-                
+
                 results.append((
                     chunk_text,
                     embedding_vector,
                     tokens,
                     cost
                 ))
-                
+
                 # Small delay between requests to respect rate limits
                 time.sleep(0.1)
-            
+
             # Longer delay between batches
             if i + batch_size < len(chunks):
                 time.sleep(1.0)
-        
+
         logger.info(f"Generated {len(results)} embeddings for document {document_id}")
         return results
-        
+
     except Exception as exc:
         logger.error(f"Batch embedding generation failed: {exc}")
         raise
@@ -270,12 +275,12 @@ def reprocess_document_chunks(
     try:
         with transaction.atomic():
             document = Document.objects.get(id=document_id)
-            
+
             # Delete existing chunks
             DocumentChunk.objects.filter(
                 document=document
             ).delete()
-            
+
             # Reset document status
             document.processing_status = ProcessingStatus.PENDING
             document.chunks_count = 0
@@ -285,14 +290,14 @@ def reprocess_document_chunks(
                 'processing_status', 'processing_started_at', 'processing_completed_at',
                 'processing_error', 'chunks_count', 'total_tokens', 'total_cost_usd'
             ])
-            
+
             # Trigger reprocessing
             return process_document_async(
                 document_id=document_id,
                 chunk_size=new_chunk_size or 1000,
                 embedding_model=new_embedding_model or "text-embedding-ada-002"
             )
-            
+
     except Exception as exc:
         logger.error(f"Reprocessing failed for {document_id}: {exc}")
         raise
@@ -316,11 +321,11 @@ def optimize_document_embeddings(document_id: str) -> Dict[str, Any]:
     try:
         # Update vector index statistics
         from django.db import connection
-        
+
         with connection.cursor() as cursor:
             # Always analyze the table
             cursor.execute("ANALYZE django_cfg_knowbase_document_chunks;")
-            
+
             # Check if index exists before trying to reindex
             cursor.execute("""
                 SELECT EXISTS (
@@ -329,21 +334,21 @@ def optimize_document_embeddings(document_id: str) -> Dict[str, Any]:
                 );
             """)
             index_exists = cursor.fetchone()[0]
-            
+
             if index_exists:
                 cursor.execute("REINDEX INDEX embedding_cosine_idx;")
                 logger.debug("Reindexed embedding_cosine_idx")
             else:
                 logger.warning("embedding_cosine_idx index does not exist, skipping reindex")
-        
+
         logger.info(f"Optimized embeddings for document {document_id}")
-        
+
         return {
             "status": "optimized",
             "document_id": document_id,
             "timestamp": timezone.now().isoformat()
         }
-        
+
     except Exception as exc:
         logger.error(f"Embedding optimization failed for {document_id}: {exc}")
         raise