django-cfg 1.4.21__py3-none-any.whl → 1.4.24__py3-none-any.whl

django_cfg/apps/knowbase/services/archive/archive_service.py

@@ -4,29 +4,26 @@ Main document archive service.
 Orchestrates the complete archive processing pipeline with synchronous processing.
 """
 
-import os
-import time
-import tempfile
 import hashlib
 import logging
-from typing import List, Dict, Any, Optional
-from django.core.files.uploadedfile import UploadedFile
+import os
+import tempfile
+import time
+from typing import Any, Dict, List, Optional
+
 from django.contrib.auth import get_user_model
+from django.core.files.uploadedfile import UploadedFile
 from django.db import transaction
 from django.utils import timezone
 from pydantic import BaseModel, Field, ValidationError
 
-from ...models.archive import DocumentArchive, ArchiveType, ContentType
-from ...models.document import DocumentCategory
+from ...models.archive import ArchiveType, DocumentArchive
 from ...models.base import ProcessingStatus
+from ...models.document import DocumentCategory
 from ..base import BaseService
-from .exceptions import (
-    ArchiveValidationError, 
-    ArchiveProcessingError,
-    ProcessingTimeoutError
-)
-from .extraction_service import ArchiveExtractionService, ExtractedItemData
 from .chunking_service import ContextualChunkingService
+from .exceptions import ArchiveProcessingError, ArchiveValidationError, ProcessingTimeoutError
+from .extraction_service import ArchiveExtractionService, ExtractedItemData
 from .vectorization_service import ArchiveVectorizationService
 
 User = get_user_model()
@@ -35,20 +32,20 @@ logger = logging.getLogger(__name__)
 
 class ArchiveUploadRequest(BaseModel):
     """Pydantic model for archive upload validation."""
-    
+
     title: str = Field(..., min_length=1, max_length=512)
     description: Optional[str] = Field(None, max_length=2000)
     category_ids: List[str] = Field(default_factory=list)
     is_public: bool = Field(default=True)
     process_immediately: bool = Field(default=True)
-    
+
     class Config:
         str_strip_whitespace = True
 
 
 class ArchiveProcessingResult(BaseModel):
     """Result of archive processing operation."""
-    
+
     archive_id: str
     status: str
     processing_time_ms: int
@@ -61,25 +58,25 @@ class ArchiveProcessingResult(BaseModel):
 
 class DocumentArchiveService(BaseService):
     """Main service for document archive operations."""
-    
+
     # Processing limits
     MAX_ARCHIVE_SIZE = 200 * 1024 * 1024  # 200MB
     MAX_ITEMS_COUNT = 2000
     MAX_PROCESSING_TIME = 120  # 2 minutes
-    
+
     def __init__(self, user: User):
         super().__init__(user)
         self.extraction_service = ArchiveExtractionService()
         self.chunking_service = ContextualChunkingService(user)
         self.vectorization_service = ArchiveVectorizationService(user)
-    
+
     def create_and_process_archive(
         self,
         uploaded_file: UploadedFile,
         request_data: Dict[str, Any]
     ) -> ArchiveProcessingResult:
         """Create archive and process it synchronously."""
-        
+
         # Validate request data
         try:
             validated_request = ArchiveUploadRequest(**request_data)
@@ -89,10 +86,10 @@ class DocumentArchiveService(BaseService):
                 code="INVALID_REQUEST",
                 details={"validation_errors": e.errors()}
             )
-        
+
         # Create archive record
         archive = self._create_archive_record(uploaded_file, validated_request)
-        
+
         # Process synchronously if requested
         if validated_request.process_immediately:
             return self._process_archive_sync(archive, uploaded_file)
@@ -106,72 +103,72 @@ class DocumentArchiveService(BaseService):
                 vectorized_chunks=0,
                 total_cost_usd=0.0
             )
-    
+
     def process_archive(self, archive: DocumentArchive) -> bool:
         """Process an existing archive by its stored file."""
-        
+
         # Debug logging
         logger.info(f"process_archive called with archive: {archive}, type: {type(archive)}")
-        
+
         if not archive:
             raise ArchiveProcessingError(
                 message="Archive object is None",
                 code="ARCHIVE_IS_NONE"
             )
-        
+
         if not archive.archive_file:
             raise ArchiveProcessingError(
                 message="Archive has no file to process",
                 code="NO_FILE"
             )
-        
+
         start_time = time.time()
-        
+
         try:
             # Update status
             archive.processing_status = ProcessingStatus.PROCESSING
             archive.save()
-            
+
             # Get file path from the archive_file field
             file_path = archive.archive_file.path
-            
+
             # Extract archive
             extracted_items = self.extraction_service.extract_archive(
-                file_path, 
+                file_path,
                 archive.archive_type
             )
-            
+
             # Check processing time
             self._check_processing_timeout(start_time)
-            
+
             # Create item records
             items = self._create_item_records(archive, extracted_items)
-            
+
             # Check processing time again
             self._check_processing_timeout(start_time)
-            
+
             # Generate chunks
             chunks = self._generate_chunks_for_items(items)
-            
+
             # Check processing time again
             self._check_processing_timeout(start_time)
-            
+
             # Vectorize chunks
             vectorization_result = self._vectorize_chunks(chunks)
-            
+
             # Update archive statistics
             self._update_archive_statistics(archive, items, chunks, vectorization_result)
-            
+
             # Mark as completed
             processing_time_ms = int((time.time() - start_time) * 1000)
             archive.processing_status = ProcessingStatus.COMPLETED
             archive.processed_at = timezone.now()
             archive.processing_duration_ms = processing_time_ms
             archive.save()
-            
+
             logger.info(f"Successfully processed archive {archive.id} in {processing_time_ms}ms")
             return True
-            
+
         except ProcessingTimeoutError:
             processing_time_ms = int((time.time() - start_time) * 1000)
             archive.processing_status = ProcessingStatus.FAILED
@@ -180,7 +177,7 @@ class DocumentArchiveService(BaseService):
             archive.save()
             logger.error(f"Archive processing timeout for {archive.id}")
             return False
-            
+
         except Exception as e:
             processing_time_ms = int((time.time() - start_time) * 1000)
             archive.processing_status = ProcessingStatus.FAILED
@@ -196,29 +193,29 @@ class DocumentArchiveService(BaseService):
         request: ArchiveUploadRequest
     ) -> DocumentArchive:
         """Create initial archive record."""
-        
+
         # Validate file
         self._validate_uploaded_file(uploaded_file)
-        
+
         # Generate content hash
         content_hash = self._generate_file_hash(uploaded_file)
-        
+
         # Check for duplicates
         existing = DocumentArchive.objects.filter(
             user=self.user,
             content_hash=content_hash
         ).first()
-        
+
         if existing:
             raise ArchiveValidationError(
                 message=f"Archive already exists: {existing.title}",
                 code="DUPLICATE_ARCHIVE",
                 details={"existing_archive_id": str(existing.id)}
             )
-        
+
         # Detect archive type
         archive_type = self._detect_archive_type(uploaded_file.name)
-        
+
         with transaction.atomic():
             # Create archive record
             archive = DocumentArchive.objects.create(
@@ -232,73 +229,73 @@ class DocumentArchiveService(BaseService):
                 is_public=request.is_public,
                 processing_status=ProcessingStatus.PENDING
             )
-            
+
             # Add categories
             if request.category_ids:
                 categories = DocumentCategory.objects.filter(
                     id__in=request.category_ids
                 )
                 archive.categories.set(categories)
-        
+
         return archive
-    
+
     def _process_archive_sync(
         self,
         archive: DocumentArchive,
         uploaded_file: UploadedFile
     ) -> ArchiveProcessingResult:
         """Process archive synchronously with time limits."""
-        
+
         start_time = time.time()
-        
+
         try:
             # Update status
             archive.processing_status = ProcessingStatus.PROCESSING
             archive.save()
-            
+
             # Save file temporarily
             temp_file_path = self._save_temp_file(uploaded_file, archive.id)
-            
+
             try:
                 # Extract archive
                 extracted_items = self.extraction_service.extract_archive(
-                    temp_file_path, 
+                    temp_file_path,
                     archive.archive_type
                 )
-                
+
                 # Check processing time
                 self._check_processing_timeout(start_time)
-                
+
                 # Create item records
                 items = self._create_item_records(archive, extracted_items)
-                
+
                 # Check processing time
                 self._check_processing_timeout(start_time)
-                
+
                 # Generate chunks
                 all_chunks = self._generate_chunks_for_items(items)
-                
+
                 # Check processing time
                 self._check_processing_timeout(start_time)
-                
+
                 # Vectorize chunks
                 vectorization_result = self._vectorize_chunks(all_chunks)
-                
+
                 # Update archive statistics
                 self._update_archive_statistics(
-                    archive, 
-                    items, 
-                    all_chunks, 
+                    archive,
+                    items,
+                    all_chunks,
                     vectorization_result
                 )
-                
+
                 # Mark as completed
                 processing_time_ms = int((time.time() - start_time) * 1000)
                 archive.processing_status = ProcessingStatus.COMPLETED
                 archive.processed_at = timezone.now()
                 archive.processing_duration_ms = processing_time_ms
                 archive.save()
-                
+
                 return ArchiveProcessingResult(
                     archive_id=str(archive.id),
                     status=archive.processing_status,
@@ -308,11 +305,11 @@ class DocumentArchiveService(BaseService):
                     vectorized_chunks=vectorization_result['vectorized_count'],
                     total_cost_usd=vectorization_result['total_cost']
                 )
-                
+
             finally:
                 # Always cleanup temp file
                 self._cleanup_temp_file(temp_file_path)
-        
+
         except Exception as e:
             # Mark as failed
             processing_time_ms = int((time.time() - start_time) * 1000)
@@ -320,7 +317,7 @@ class DocumentArchiveService(BaseService):
             archive.processing_error = str(e)
             archive.processing_duration_ms = processing_time_ms
             archive.save()
-            
+
             return ArchiveProcessingResult(
                 archive_id=str(archive.id),
                 status=archive.processing_status,
@@ -331,10 +328,10 @@ class DocumentArchiveService(BaseService):
                 total_cost_usd=0.0,
                 error_message=str(e)
             )
-    
+
     def _validate_uploaded_file(self, uploaded_file: UploadedFile) -> None:
         """Validate uploaded archive file."""
-        
+
         # Size check
         if uploaded_file.size > self.MAX_ARCHIVE_SIZE:
             raise ArchiveValidationError(
@@ -345,7 +342,7 @@ class DocumentArchiveService(BaseService):
                     "max_size": self.MAX_ARCHIVE_SIZE
                 }
             )
-        
+
         # Type check
         archive_type = self._detect_archive_type(uploaded_file.name)
         if not archive_type:
@@ -354,11 +351,11 @@ class DocumentArchiveService(BaseService):
                 code="UNSUPPORTED_FORMAT",
                 details={"filename": uploaded_file.name}
             )
-    
+
     def _detect_archive_type(self, filename: str) -> Optional[str]:
         """Detect archive type from filename."""
         filename_lower = filename.lower()
-        
+
         if filename_lower.endswith('.zip'):
             return ArchiveType.ZIP
         elif filename_lower.endswith(('.tar.gz', '.tgz')):
@@ -367,47 +364,47 @@ class DocumentArchiveService(BaseService):
             return ArchiveType.TAR_BZ2
         elif filename_lower.endswith('.tar'):
             return ArchiveType.TAR
-        
+
         return None
-    
+
     def _generate_file_hash(self, uploaded_file: UploadedFile) -> str:
         """Generate SHA-256 hash of uploaded file."""
         hash_sha256 = hashlib.sha256()
-        
+
         # Reset file pointer
         uploaded_file.seek(0)
-        
+
         for chunk in uploaded_file.chunks():
             hash_sha256.update(chunk)
-        
+
         # Reset file pointer again
         uploaded_file.seek(0)
-        
+
         return hash_sha256.hexdigest()
-    
+
     def _save_temp_file(self, uploaded_file: UploadedFile, archive_id: str) -> str:
         """Save uploaded file to temporary location."""
         temp_dir = tempfile.mkdtemp(prefix=f'archive_{archive_id}_')
         temp_path = os.path.join(temp_dir, uploaded_file.name)
-        
+
         with open(temp_path, 'wb') as f:
             for chunk in uploaded_file.chunks():
                 f.write(chunk)
-        
+
         return temp_path
-    
+
     def _cleanup_temp_file(self, temp_file_path: str) -> None:
         """Clean up temporary file and directory."""
         if os.path.exists(temp_file_path):
             os.unlink(temp_file_path)
-            
+
             # Remove directory if empty
             temp_dir = os.path.dirname(temp_file_path)
             try:
                 os.rmdir(temp_dir)
             except OSError:
                 pass  # Directory not empty or other error
-    
+
     def _check_processing_timeout(self, start_time: float) -> None:
         """Check if processing has exceeded time limit."""
         elapsed = time.time() - start_time
@@ -420,15 +417,15 @@ class DocumentArchiveService(BaseService):
                     "max_seconds": self.MAX_PROCESSING_TIME
                 }
             )
-    
+
     def _create_item_records(
-        self, 
-        archive: DocumentArchive, 
+        self,
+        archive: DocumentArchive,
         extracted_items: List[ExtractedItemData]
     ) -> List:
         """Create ArchiveItem records from extracted data."""
         from ...models.archive import ArchiveItem
-        
+
         if len(extracted_items) > self.MAX_ITEMS_COUNT:
             raise ArchiveValidationError(
                 message=f"Too many items: {len(extracted_items)}",
@@ -438,11 +435,11 @@ class DocumentArchiveService(BaseService):
                     "max_count": self.MAX_ITEMS_COUNT
                 }
             )
-        
+
         items = []
-        
+
         # Note: Items should already be cleared by reprocess method
-        
+
         with transaction.atomic():
             for item_data in extracted_items:
                 item = ArchiveItem.objects.create(
@@ -456,33 +453,33 @@ class DocumentArchiveService(BaseService):
                     metadata=item_data.metadata
                 )
                 items.append(item)
-            
+
             # Update archive statistics
             archive.total_items = len(items)
             archive.processed_items = len(items)
             archive.save()
-        
+
         return items
-    
+
     def _generate_chunks_for_items(self, items: List) -> List:
         """Generate chunks for all processable items."""
         all_chunks = []
-        
+
         for item in items:
             if item.is_processable and item.raw_content:
                 chunks = self.chunking_service.create_chunks_with_context(item)
                 all_chunks.extend(chunks)
-                
+
                 # Update item statistics
                 item.chunks_count = len(chunks)
                 item.save()
-        
+
         return all_chunks
-    
+
     def _vectorize_chunks(self, chunks: List) -> Dict[str, Any]:
         """Vectorize all chunks."""
         return self.vectorization_service.vectorize_chunks_batch(chunks)
-    
+
     def _update_archive_statistics(
         self,
         archive: DocumentArchive,
@@ -491,16 +488,16 @@ class DocumentArchiveService(BaseService):
         vectorization_result: Dict[str, Any]
     ) -> None:
         """Update archive with final statistics."""
-        
+
         total_tokens = sum(item.total_tokens for item in items)
         total_cost = sum(item.processing_cost for item in items)
-        
+
         archive.total_chunks = len(chunks)
         archive.vectorized_chunks = vectorization_result['vectorized_count']
         archive.total_tokens = total_tokens
         archive.total_cost_usd = total_cost
         archive.save()
-    
+
     def get_archive_by_id(self, archive_id: str) -> Optional[DocumentArchive]:
         """Get archive by ID with user access check."""
         try:
@@ -508,7 +505,7 @@ class DocumentArchiveService(BaseService):
             return archive
         except DocumentArchive.DoesNotExist:
             return None
-    
+
     def list_user_archives(
         self,
         limit: int = 20,
@@ -516,21 +513,21 @@ class DocumentArchiveService(BaseService):
         status_filter: Optional[str] = None
     ) -> Dict[str, Any]:
         """List user's archives with pagination."""
-        
+
         queryset = DocumentArchive.objects.filter(user=self.user)
-        
+
         if status_filter:
             queryset = queryset.filter(processing_status=status_filter)
-        
+
         total_count = queryset.count()
         archives = list(queryset.order_by('-created_at')[offset:offset + limit])
-        
+
         return {
             'archives': archives,
             'total_count': total_count,
             'has_more': offset + limit < total_count
         }
-    
+
     def delete_archive(self, archive_id: str) -> bool:
         """Delete archive and all related data."""
         try:

django_cfg/apps/knowbase/services/archive/chunking/__init__.py

@@ -5,10 +5,10 @@ Language-specific and content-aware chunking implementations.
 """
 
 from .base import BaseChunker
-from .text_chunker import TextChunker
-from .python_chunker import PythonChunker
-from .markdown_chunker import MarkdownChunker
 from .json_chunker import JsonChunker
+from .markdown_chunker import MarkdownChunker
+from .python_chunker import PythonChunker
+from .text_chunker import TextChunker
 
 __all__ = [
     'BaseChunker',

django_cfg/apps/knowbase/services/archive/chunking/base.py

@@ -6,6 +6,7 @@ Abstract base class for content-specific chunking strategies.
 
 from abc import ABC, abstractmethod
 from typing import List
+
 from ....models.archive import ArchiveItem
 from ..context.models import ChunkData
 

django_cfg/apps/knowbase/services/archive/chunking/json_chunker.py

@@ -6,10 +6,11 @@ Chunks JSON by object structure.
 
 import json
 from typing import List
-from .base import BaseChunker
-from ..context.models import ChunkData
-from ..context.builders import ChunkContextBuilder
+
 from ....models.archive import ArchiveItem, ChunkType
+from ..context.builders import ChunkContextBuilder
+from ..context.models import ChunkData
+from .base import BaseChunker
 
 
 class JsonChunker(BaseChunker):

django_cfg/apps/knowbase/services/archive/chunking/markdown_chunker.py

@@ -4,11 +4,12 @@ Markdown document chunker.
 Chunks markdown by headings and sections.
 """
 
-from typing import List, Dict, Any
-from .base import BaseChunker
-from ..context.models import ChunkData
-from ..context.builders import ChunkContextBuilder
+from typing import Any, Dict, List
+
 from ....models.archive import ArchiveItem, ChunkType
+from ..context.builders import ChunkContextBuilder
+from ..context.models import ChunkData
+from .base import BaseChunker
 
 
 class MarkdownChunker(BaseChunker):

django_cfg/apps/knowbase/services/archive/chunking/python_chunker.py

@@ -5,11 +5,12 @@ Chunks Python code by logical boundaries (classes, functions, imports).
 """
 
 import ast
-from typing import List, Optional, Dict, Any
-from .base import BaseChunker
-from ..context.models import ChunkData
-from ..context.builders import ChunkContextBuilder
+from typing import Any, Dict, List, Optional
+
 from ....models.archive import ArchiveItem, ChunkType
+from ..context.builders import ChunkContextBuilder
+from ..context.models import ChunkData
+from .base import BaseChunker
 
 
 class PythonChunker(BaseChunker):
@@ -97,7 +98,7 @@ class PythonChunker(BaseChunker):
         content = '\n'.join(lines[start_line:end_line])
 
         # Analyze code structure
-        from ..analyzers import ComplexityAnalyzer, QualityAnalyzer, PurposeDetector
+        from ..analyzers import ComplexityAnalyzer, PurposeDetector, QualityAnalyzer
         code_info = self._analyze_structure(
             node, content,
             ComplexityAnalyzer, QualityAnalyzer, PurposeDetector

django_cfg/apps/knowbase/services/archive/chunking/text_chunker.py

@@ -5,10 +5,11 @@ Simple text chunking with fixed size and overlap.
 """
 
 from typing import List
-from .base import BaseChunker
-from ..context.models import ChunkData
-from ..context.builders import ChunkContextBuilder
+
 from ....models.archive import ArchiveItem, ChunkType
+from ..context.builders import ChunkContextBuilder
+from ..context.models import ChunkData
+from .base import BaseChunker
 
 
 class TextChunker(BaseChunker):