django-cfg 1.4.20__py3-none-any.whl → 1.4.23__py3-none-any.whl

django_cfg/apps/knowbase/tasks/external_data_tasks.py

@@ -2,15 +2,16 @@
 External Data processing tasks with Dramatiq.
 """
 
-import dramatiq
 import logging
 import time
-from typing import Dict, List, Any, Tuple, Optional
+from typing import Any, Dict, List, Optional
+
+import dramatiq
 from django.db import transaction
 from django.utils import timezone
 
-from ..models.external_data import ExternalData, ExternalDataChunk, ExternalDataStatus
 from ..mixins.service import ExternalDataService
+from ..models.external_data import ExternalData, ExternalDataStatus
 
 logger = logging.getLogger(__name__)
 
@@ -37,12 +38,12 @@ def process_external_data_async(
         Processing results with statistics
     """
     start_time = time.time()
-    
+
     try:
         with transaction.atomic():
             # Get external data instance
             external_data = ExternalData.objects.select_for_update().get(id=external_data_id)
-            
+
             # Check if already processing
             if external_data.status == ExternalDataStatus.PROCESSING:
                 logger.warning(f"External data {external_data_id} is already being processed")
@@ -51,7 +52,7 @@ def process_external_data_async(
                     'error': 'Already processing',
                     'external_data_id': external_data_id
                 }
-            
+
             # Check if already completed and not forcing reprocess
             if external_data.status == ExternalDataStatus.COMPLETED and not force_reprocess:
                 logger.info(f"External data {external_data_id} already completed, skipping")
@@ -63,7 +64,7 @@ def process_external_data_async(
                     'total_tokens': external_data.total_tokens,
                     'processing_cost': external_data.processing_cost
                 }
-            
+
             # Check if has content
             if not external_data.content or not external_data.content.strip():
                 logger.warning(f"External data {external_data_id} has no content to process")
@@ -73,7 +74,7 @@ def process_external_data_async(
                 external_data.total_tokens = 0
                 external_data.processing_cost = 0.0
                 external_data.save(update_fields=[
-                    'status', 'processed_at', 'total_chunks', 
+                    'status', 'processed_at', 'total_chunks',
                     'total_tokens', 'processing_cost'
                 ])
                 return {
@@ -84,30 +85,30 @@ def process_external_data_async(
                     'processing_cost': 0.0,
                     'processing_time': time.time() - start_time
                 }
-            
+
             logger.info(f"🚀 Starting external data processing: {external_data.title} (ID: {external_data_id})")
-            
+
             # Update status to processing
             external_data.status = ExternalDataStatus.PROCESSING
             external_data.processing_error = ""
             external_data.save(update_fields=['status', 'processing_error'])
-        
+
         # Process external data using service (outside transaction for long-running operations)
         service = ExternalDataService(external_data.user)
         success = service.vectorize_external_data(external_data)
-        
+
         # Refresh from database to get updated statistics
         external_data.refresh_from_db()
-        
+
         processing_time = time.time() - start_time
-        
+
         if success:
             logger.info(
                 f"✅ External data processing completed: {external_data.title} "
                 f"({external_data.total_chunks} chunks, {external_data.total_tokens} tokens, "
                 f"${external_data.processing_cost:.6f}, {processing_time:.2f}s)"
             )
-            
+
             return {
                 'success': True,
                 'external_data_id': external_data_id,
@@ -125,7 +126,7 @@ def process_external_data_async(
                 'error': external_data.processing_error or 'Unknown processing error',
                 'processing_time': processing_time
             }
-            
+
     except ExternalData.DoesNotExist:
         error_msg = f"External data {external_data_id} not found"
         logger.error(f"❌ {error_msg}")
@@ -134,12 +135,12 @@ def process_external_data_async(
             'error': error_msg,
             'external_data_id': external_data_id
         }
-        
+
     except Exception as e:
         processing_time = time.time() - start_time
         error_msg = f"Unexpected error processing external data {external_data_id}: {e}"
         logger.error(f"❌ {error_msg}", exc_info=True)
-        
+
         # Try to update status to failed
         try:
             external_data = ExternalData.objects.get(id=external_data_id)
@@ -148,7 +149,7 @@ def process_external_data_async(
             external_data.save(update_fields=['status', 'processing_error'])
         except Exception as save_error:
             logger.error(f"❌ Failed to update external data status: {save_error}")
-        
+
         return {
             'success': False,
             'error': error_msg,
@@ -181,12 +182,12 @@ def bulk_process_external_data_async(
         Bulk processing results with statistics
     """
     start_time = time.time()
-    
+
     try:
         from django.contrib.auth import get_user_model
         User = get_user_model()
         user = User.objects.get(id=user_id)
-        
+
         # Get external data to process
         if external_data_ids:
             external_data_queryset = ExternalData.objects.filter(
@@ -199,10 +200,10 @@ def bulk_process_external_data_async(
                 user=user,
                 status=ExternalDataStatus.PENDING
             )
-        
+
         external_data_list = list(external_data_queryset)
         total_count = len(external_data_list)
-        
+
         if total_count == 0:
             logger.info(f"No external data to process for user {user_id}")
             return {
@@ -214,22 +215,22 @@ def bulk_process_external_data_async(
                 'skipped_count': 0,
                 'processing_time': time.time() - start_time
             }
-        
+
         logger.info(f"🚀 Starting bulk external data processing for user {user_id}: {total_count} items")
-        
+
         # Process each external data
         processed_count = 0
         failed_count = 0
         skipped_count = 0
         results = []
-        
+
         for external_data in external_data_list:
             try:
                 result = process_external_data_async.send(
                     str(external_data.id),
                     force_reprocess=force_reprocess
                 )
-                
+
                 if result.get('success'):
                     if result.get('skipped'):
                         skipped_count += 1
@@ -237,9 +238,9 @@ def bulk_process_external_data_async(
                         processed_count += 1
                 else:
                     failed_count += 1
-                
+
                 results.append(result)
-                
+
             except Exception as e:
                 logger.error(f"❌ Failed to queue external data {external_data.id}: {e}")
                 failed_count += 1
@@ -248,15 +249,15 @@ def bulk_process_external_data_async(
                     'external_data_id': str(external_data.id),
                     'error': f'Failed to queue: {e}'
                 })
-        
+
         processing_time = time.time() - start_time
-        
+
         logger.info(
             f"✅ Bulk external data processing queued for user {user_id}: "
             f"{processed_count} processed, {failed_count} failed, "
             f"{skipped_count} skipped out of {total_count} total ({processing_time:.2f}s)"
         )
-        
+
         return {
             'success': True,
             'user_id': user_id,
@@ -267,12 +268,12 @@ def bulk_process_external_data_async(
             'processing_time': processing_time,
             'results': results
         }
-        
+
     except Exception as e:
         processing_time = time.time() - start_time
         error_msg = f"Unexpected error in bulk external data processing for user {user_id}: {e}"
         logger.error(f"❌ {error_msg}", exc_info=True)
-        
+
         return {
             'success': False,
             'error': error_msg,
@@ -297,21 +298,22 @@ def cleanup_failed_external_data_async(older_than_days: int = 7) -> Dict[str, An
         Cleanup results
     """
     start_time = time.time()
-    
+
     try:
-        from django.utils import timezone
         from datetime import timedelta
-        
+
+        from django.utils import timezone
+
         cutoff_date = timezone.now() - timedelta(days=older_than_days)
-        
+
         # Find failed external data older than cutoff
         failed_external_data = ExternalData.objects.filter(
             status=ExternalDataStatus.FAILED,
             updated_at__lt=cutoff_date
         )
-        
+
         count = failed_external_data.count()
-        
+
         if count > 0:
             logger.info(f"🧹 Cleaning up {count} failed external data older than {older_than_days} days")
             deleted_count, _ = failed_external_data.delete()
@@ -319,21 +321,21 @@ def cleanup_failed_external_data_async(older_than_days: int = 7) -> Dict[str, An
         else:
             logger.info(f"No failed external data older than {older_than_days} days found")
             deleted_count = 0
-        
+
         processing_time = time.time() - start_time
-        
+
         return {
             'success': True,
             'deleted_count': deleted_count,
             'older_than_days': older_than_days,
             'processing_time': processing_time
         }
-        
+
     except Exception as e:
         processing_time = time.time() - start_time
         error_msg = f"Error cleaning up failed external data: {e}"
         logger.error(f"❌ {error_msg}", exc_info=True)
-        
+
         return {
             'success': False,
             'error': error_msg,

django_cfg/apps/knowbase/tasks/maintenance.py

@@ -2,15 +2,17 @@
 Maintenance and cleanup tasks.
 """
 
-import dramatiq
 import logging
-from typing import Dict, Any
+from datetime import timedelta
+from typing import Any, Dict
+
+import dramatiq
+from django.conf import settings
+from django.core.cache import cache
 from django.db import connection
 from django.utils import timezone
-from datetime import timedelta
+
 from django_cfg.modules.django_llm.llm.client import LLMClient
-from django.core.cache import cache
-from django.conf import settings
 
 from ..models import Document, DocumentChunk, ProcessingStatus
 
@@ -34,14 +36,14 @@ def cleanup_old_embeddings(days_old: int = 90) -> Dict[str, Any]:
     """
     try:
         cutoff_date = timezone.now() - timedelta(days=days_old)
-        
+
         # Find orphaned chunks (documents deleted but chunks remain)
         orphaned_chunks = DocumentChunk.objects.filter(
             document__isnull=True
         )
         orphaned_count = orphaned_chunks.count()
         orphaned_chunks.delete()
-        
+
         # Find very old, unused chunks
         old_chunks = DocumentChunk.objects.filter(
             created_at__lt=cutoff_date,
@@ -49,12 +51,12 @@ def cleanup_old_embeddings(days_old: int = 90) -> Dict[str, Any]:
         )
         old_count = old_chunks.count()
         old_chunks.delete()
-        
+
         # Vacuum and analyze tables
         with connection.cursor() as cursor:
             cursor.execute("VACUUM ANALYZE django_cfg_knowbase_document_chunks;")
             cursor.execute("VACUUM ANALYZE django_cfg_knowbase_documents;")
-        
+
         result = {
             "orphaned_chunks_deleted": orphaned_count,
             "old_chunks_deleted": old_count,
@@ -62,10 +64,10 @@ def cleanup_old_embeddings(days_old: int = 90) -> Dict[str, Any]:
             "cutoff_date": cutoff_date.isoformat(),
             "timestamp": timezone.now().isoformat()
         }
-        
+
         logger.info(f"Cleanup completed: {result}")
         return result
-        
+
     except Exception as exc:
         logger.error(f"Cleanup failed: {exc}")
         raise
@@ -87,10 +89,10 @@ def optimize_vector_indexes() -> Dict[str, Any]:
         with connection.cursor() as cursor:
             # Reindex vector indexes (match model definition)
             cursor.execute("REINDEX INDEX CONCURRENTLY embedding_cosine_idx;")
-            
+
             # Update table statistics
             cursor.execute("ANALYZE django_cfg_knowbase_document_chunks;")
-            
+
             # Get index usage statistics
             cursor.execute("""
                 SELECT 
@@ -102,17 +104,17 @@ def optimize_vector_indexes() -> Dict[str, Any]:
                 FROM pg_stat_user_indexes 
                 WHERE tablename = 'django_cfg_knowbase_document_chunks';
             """)
-            
+
             index_stats = cursor.fetchall()
-        
+
         logger.info("Vector indexes optimized successfully")
-        
+
         return {
             "status": "optimized",
             "index_stats": index_stats,
             "timestamp": timezone.now().isoformat()
         }
-        
+
     except Exception as exc:
         logger.error(f"Index optimization failed: {exc}")
         raise
@@ -139,18 +141,18 @@ def health_check_knowledge_base() -> Dict[str, Any]:
             "redis_cache": "unknown",
             "statistics": {}
         }
-        
+
         # Check database connectivity
         try:
             total_docs = Document.objects.count()
             total_chunks = DocumentChunk.objects.count()
-            
+
             health_status["database"] = "healthy"
             health_status["statistics"]["total_documents"] = total_docs
             health_status["statistics"]["total_chunks"] = total_chunks
         except Exception:
             health_status["database"] = "unhealthy"
-        
+
         # Check LLM service
         try:
             llm_service = LLMClient(
@@ -165,7 +167,7 @@ def health_check_knowledge_base() -> Dict[str, Any]:
                 health_status["llm_service"] = "not_configured"
         except Exception:
             health_status["llm_service"] = "unhealthy"
-        
+
         # Check Redis cache
         try:
             cache.set("health_check", "ok", 10)
@@ -175,7 +177,7 @@ def health_check_knowledge_base() -> Dict[str, Any]:
                 health_status["redis_cache"] = "unhealthy"
         except Exception:
             health_status["redis_cache"] = "unhealthy"
-        
+
         # Overall health
         health_status["overall"] = (
             "healthy" if all(
@@ -186,10 +188,10 @@ def health_check_knowledge_base() -> Dict[str, Any]:
                 ]
             ) else "degraded"
         )
-        
+
         logger.info(f"Health check completed: {health_status['overall']}")
         return health_status
-        
+
     except Exception as exc:
         logger.error(f"Health check failed: {exc}")
         raise

django_cfg/apps/knowbase/urls.py

@@ -2,9 +2,10 @@
 Knowledge Base URL Configuration
 """
 
-from django.urls import path, include
+from django.urls import include, path
 from rest_framework.routers import DefaultRouter
-from .views.public_views import PublicDocumentViewSet, PublicCategoryViewSet
+
+from .views.public_views import PublicCategoryViewSet, PublicDocumentViewSet
 
 # Public router for client access
 public_router = DefaultRouter()

django_cfg/apps/knowbase/urls_admin.py

@@ -2,10 +2,13 @@
 Knowledge Base URL Configuration
 """
 
-from django.urls import path, include
+from django.urls import include, path
 from rest_framework.routers import DefaultRouter
+
 from .views import (
-    DocumentViewSet, ChatViewSet, ChatSessionViewSet,
+    ChatSessionViewSet,
+    ChatViewSet,
+    DocumentViewSet,
 )
 
 # Create router and register viewsets
@@ -20,4 +23,4 @@ urlpatterns = [
     path('', include(router.urls)),
 ]
 
-app_name = 'cfg_knowbase_admin'
+app_name = 'cfg_knowbase_admin'

django_cfg/apps/knowbase/urls_system.py

@@ -2,11 +2,10 @@
 Knowledge Base URL Configuration
 """
 
-from django.urls import path, include
+from django.urls import include, path
 from rest_framework.routers import DefaultRouter
-from .views import (
-    DocumentArchiveViewSet, ArchiveItemViewSet, ArchiveItemChunkViewSet
-)
+
+from .views import ArchiveItemChunkViewSet, ArchiveItemViewSet, DocumentArchiveViewSet
 
 # Archive router for authenticated users
 archive_router = DefaultRouter()
@@ -19,7 +18,7 @@ urlpatterns = [
 
     # Archive API endpoints (require authentication)
     path('', include(archive_router.urls)),
-    
+
 ]
 
 # Add app name for namespacing

django_cfg/apps/knowbase/utils/chunk_settings.py

@@ -6,8 +6,10 @@ settings with type safety and validation.
 """
 
 import logging
-from typing import Dict, Any, Optional
+from typing import Any, Dict
+
 from pydantic import BaseModel, Field, validator
+
 from ..config.constance_settings import ConstanceSettings
 
 logger = logging.getLogger(__name__)
@@ -15,29 +17,29 @@ logger = logging.getLogger(__name__)
 
 class ChunkSettings(BaseModel):
     """Pydantic model for chunk processing settings."""
-    
+
     chunk_size: int = Field(
         ge=100,
         le=8000,
         description="Size of each chunk in characters"
     )
-    
+
     chunk_overlap: int = Field(
         ge=0,
         description="Overlap between chunks in characters"
     )
-    
+
     embedding_batch_size: int = Field(
         ge=1,
         le=100,
         description="Number of chunks to process in one embedding batch"
     )
-    
+
     embedding_model: str = Field(
         min_length=1,
         description="OpenAI embedding model name"
     )
-    
+
     @validator('chunk_overlap')
     def validate_overlap(cls, v, values):
         """Ensure overlap is less than chunk_size."""
@@ -45,7 +47,7 @@ class ChunkSettings(BaseModel):
         if chunk_size and v >= chunk_size:
             raise ValueError(f"Chunk overlap ({v}) must be less than chunk size ({chunk_size})")
         return v
-    
+
     class Config:
         """Pydantic configuration."""
         validate_assignment = True
@@ -53,7 +55,7 @@ class ChunkSettings(BaseModel):
 
 class ChunkSettingsManager:
     """Manager for dynamic chunk settings using Pydantic configuration."""
-    
+
     @classmethod
     def get_document_settings(cls) -> ChunkSettings:
         """Get chunk settings for document processing."""
@@ -65,7 +67,7 @@ class ChunkSettingsManager:
             embedding_batch_size=ConstanceSettings.get_embedding_batch_size(),
             embedding_model=ConstanceSettings.get_embedding_model()
         )
-    
+
     @classmethod
     def get_archive_settings(cls) -> ChunkSettings:
         """Get chunk settings for archive processing."""
@@ -77,7 +79,7 @@ class ChunkSettingsManager:
             embedding_batch_size=ConstanceSettings.get_embedding_batch_size(),
             embedding_model=ConstanceSettings.get_embedding_model()
         )
-    
+
     @classmethod
     def get_settings_for_type(cls, content_type: str) -> ChunkSettings:
         """
@@ -96,7 +98,7 @@ class ChunkSettingsManager:
         else:
             logger.warning(f"Unknown content type: {content_type}, using document settings")
             return cls.get_document_settings()
-    
+
     @classmethod
     def get_all_settings(cls) -> Dict[str, ChunkSettings]:
         """Get all chunk settings as dictionary."""
@@ -104,7 +106,7 @@ class ChunkSettingsManager:
             'document': cls.get_document_settings(),
             'archive': cls.get_archive_settings()
         }
-    
+
     @classmethod
     def validate_settings(cls, settings: ChunkSettings) -> bool:
         """
@@ -119,37 +121,37 @@ class ChunkSettingsManager:
         if settings.chunk_size <= 0:
             logger.error(f"Invalid chunk_size: {settings.chunk_size}")
             return False
-        
+
         if settings.chunk_overlap < 0:
             logger.error(f"Invalid chunk_overlap: {settings.chunk_overlap}")
             return False
-        
+
         if settings.chunk_overlap >= settings.chunk_size:
             logger.error(f"Chunk overlap ({settings.chunk_overlap}) must be less than chunk size ({settings.chunk_size})")
             return False
-        
+
         if settings.embedding_batch_size <= 0 or settings.embedding_batch_size > 2048:
             logger.error(f"Invalid embedding_batch_size: {settings.embedding_batch_size} (must be 1-2048)")
             return False
-        
+
         if not settings.embedding_model or not settings.embedding_model.strip():
             logger.error("Embedding model cannot be empty")
             return False
-        
+
         return True
-    
+
     @classmethod
     def log_current_settings(cls) -> None:
         """Log current settings for debugging."""
         try:
             doc_settings = cls.get_document_settings()
             archive_settings = cls.get_archive_settings()
-            
+
             logger.info("📊 Current Chunk Settings:")
             logger.info(f"  📄 Documents: size={doc_settings.chunk_size}, overlap={doc_settings.chunk_overlap}")
             logger.info(f"  📦 Archives: size={archive_settings.chunk_size}, overlap={archive_settings.chunk_overlap}")
             logger.info(f"  🔮 Embedding: batch_size={doc_settings.embedding_batch_size}, model={doc_settings.embedding_model}")
-            
+
         except Exception as e:
             logger.error(f"Failed to log current settings: {e}")