django-cfg 1.4.21__py3-none-any.whl → 1.4.23__py3-none-any.whl

django_cfg/apps/knowbase/services/archive/chunking_service.py

@@ -7,20 +7,16 @@ Uses specialized chunkers for different content types.
 
 import logging
 from typing import List, Optional
+
 from django.contrib.auth import get_user_model
 
 from ...models.archive import ArchiveItem, ArchiveItemChunk
 from ...utils.chunk_settings import get_chunking_params_for_type
 from ..base import BaseService
-from .exceptions import ChunkingError
 
 # Import chunkers
-from .chunking import (
-    PythonChunker,
-    MarkdownChunker,
-    JsonChunker,
-    TextChunker
-)
+from .chunking import JsonChunker, MarkdownChunker, PythonChunker, TextChunker
+from .exceptions import ChunkingError
 
 User = get_user_model()
 logger = logging.getLogger(__name__)

django_cfg/apps/knowbase/services/archive/context/__init__.py

@@ -4,8 +4,8 @@ Context building for chunks.
 Models and builders for chunk context metadata.
 """
 
-from .models import ChunkContextMetadata, ChunkData
 from .builders import ChunkContextBuilder
+from .models import ChunkContextMetadata, ChunkData
 
 __all__ = [
     'ChunkContextMetadata',

django_cfg/apps/knowbase/services/archive/context/builders.py

@@ -4,7 +4,8 @@ Context metadata builders for chunks.
 Builds rich context metadata for different types of content chunks.
 """
 
-from typing import Dict, Any, Optional, List
+from typing import Any, Dict, Optional
+
 from ....models.archive import ArchiveItem
 
 

django_cfg/apps/knowbase/services/archive/context/models.py

@@ -4,8 +4,9 @@ Pydantic models for chunking context metadata.
 Data structures for chunk data and context metadata.
 """
 
+from typing import Any, Dict
+
 from pydantic import BaseModel
-from typing import Dict, Any
 
 
 class ChunkContextMetadata(BaseModel):

django_cfg/apps/knowbase/services/archive/exceptions.py

@@ -4,16 +4,16 @@ Archive processing exceptions.
 Custom exception hierarchy for archive processing operations.
 """
 
-from typing import Optional, Dict, Any
+from typing import Any, Dict, Optional
 
 
 class ArchiveProcessingError(Exception):
     """Base exception for archive processing errors."""
-    
+
     def __init__(
-        self, 
-        message: str, 
-        code: str, 
+        self,
+        message: str,
+        code: str,
         details: Optional[Dict[str, Any]] = None
     ):
         self.message = message

django_cfg/apps/knowbase/services/archive/extraction_service.py

@@ -4,24 +4,25 @@ Archive extraction services.
 Handles extraction of different archive formats and content processing.
 """
 
+import hashlib
+import mimetypes
 import os
-import zipfile
+import shutil
 import tarfile
 import tempfile
-import shutil
-import hashlib
-import mimetypes
+import zipfile
 from pathlib import Path
-from typing import List, Dict, Any, Optional, Set
+from typing import Any, Dict, List, Optional, Set
+
 from pydantic import BaseModel
 
 from ...models.archive import ArchiveType, ContentType
-from .exceptions import ExtractionError, ContentTypeDetectionError
+from .exceptions import ExtractionError
 
 
 class ExtractedItemData(BaseModel):
     """Data structure for extracted archive item."""
-    
+
     relative_path: str
     item_name: str
     file_size: int
@@ -35,10 +36,10 @@ class ExtractedItemData(BaseModel):
 
 class ArchiveExtractionService:
     """Service for extracting archives and processing content."""
-    
+
     # File size limits
     MAX_FILE_SIZE = 10 * 1024 * 1024  # 10MB per file
-    
+
     # Text file extensions
     TEXT_EXTENSIONS: Set[str] = {
         '.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.go', '.rs',
@@ -51,57 +52,57 @@ class ArchiveExtractionService:
         '.dockerfile', '.makefile', '.gitignore', '.env',
         '.tf', '.hcl'
     }
-    
+
     def extract_archive(
-        self, 
-        archive_path: str, 
+        self,
+        archive_path: str,
         archive_type: str
     ) -> List[ExtractedItemData]:
         """Extract archive and return processed item data."""
-        
+
         extract_dir = tempfile.mkdtemp(prefix='extracted_')
-        
+
         try:
             # Extract based on type
             file_list = self._extract_by_type(archive_path, archive_type, extract_dir)
-            
+
             # Process extracted files
             extracted_items = []
-            
+
             for relative_path in file_list:
                 full_path = os.path.join(extract_dir, relative_path)
-                
+
                 # Skip directories
                 if os.path.isdir(full_path):
                     continue
-                
+
                 # Skip unwanted files
                 if self._should_skip_file(relative_path):
                     continue
-                
+
                 try:
                     item_data = self._process_extracted_file(full_path, relative_path)
                     if item_data:
                         extracted_items.append(item_data)
-                except Exception as e:
+                except Exception:
                     # Log error but continue with other files
                     continue
-            
+
             return extracted_items
-            
+
         finally:
             # Always cleanup extraction directory
             if os.path.exists(extract_dir):
                 shutil.rmtree(extract_dir)
-    
+
     def _extract_by_type(
-        self, 
-        archive_path: str, 
-        archive_type: str, 
+        self,
+        archive_path: str,
+        archive_type: str,
         extract_dir: str
     ) -> List[str]:
         """Extract archive based on its type."""
-        
+
         try:
             if archive_type == ArchiveType.ZIP:
                 return self._extract_zip(archive_path, extract_dir)
@@ -116,53 +117,53 @@ class ArchiveExtractionService:
         except Exception as e:
             if isinstance(e, ExtractionError):
                 raise
-            
+
             raise ExtractionError(
                 message=f"Failed to extract archive: {str(e)}",
                 code="EXTRACTION_FAILED",
                 details={"archive_path": archive_path, "error": str(e)}
             ) from e
-    
+
     def _extract_zip(self, archive_path: str, extract_dir: str) -> List[str]:
         """Extract ZIP archive."""
         with zipfile.ZipFile(archive_path, 'r') as zip_file:
             # Check for zip bomb
             self._check_zip_bomb(zip_file)
-            
+
             zip_file.extractall(extract_dir)
             return zip_file.namelist()
-    
+
     def _extract_tar(
-        self, 
-        archive_path: str, 
-        archive_type: str, 
+        self,
+        archive_path: str,
+        archive_type: str,
         extract_dir: str
     ) -> List[str]:
         """Extract TAR archive (including compressed variants)."""
-        
+
         mode_map = {
             ArchiveType.TAR: 'r',
             ArchiveType.TAR_GZ: 'r:gz',
             ArchiveType.TAR_BZ2: 'r:bz2'
         }
-        
+
         with tarfile.open(archive_path, mode_map[archive_type]) as tar_file:
             # Security check for path traversal
             self._check_tar_security(tar_file)
-            
+
             tar_file.extractall(extract_dir)
             return tar_file.getnames()
-    
+
     def _check_zip_bomb(self, zip_file: zipfile.ZipFile) -> None:
         """Check for zip bomb attacks."""
-        
+
         total_uncompressed = 0
         total_compressed = 0
-        
+
         for info in zip_file.infolist():
             total_uncompressed += info.file_size
             total_compressed += info.compress_size
-        
+
         # Check compression ratio
         if total_compressed > 0:
             ratio = total_uncompressed / total_compressed
@@ -175,7 +176,7 @@ class ArchiveExtractionService:
                         "uncompressed_size": total_uncompressed
                     }
                 )
-        
+
         # Check total uncompressed size
         if total_uncompressed > 1024 * 1024 * 1024:  # 1GB limit
             raise ExtractionError(
@@ -183,10 +184,10 @@ class ArchiveExtractionService:
                 code="ARCHIVE_TOO_LARGE_UNCOMPRESSED",
                 details={"uncompressed_size": total_uncompressed}
             )
-    
+
     def _check_tar_security(self, tar_file: tarfile.TarFile) -> None:
         """Check TAR file for security issues."""
-        
+
         for member in tar_file.getmembers():
             # Check for path traversal
             if os.path.isabs(member.name) or ".." in member.name:
@@ -195,7 +196,7 @@ class ArchiveExtractionService:
                     code="PATH_TRAVERSAL_DETECTED",
                     details={"member_name": member.name}
                 )
-            
+
             # Check for suspicious file sizes
             if member.size > self.MAX_FILE_SIZE * 10:  # 100MB limit per file
                 raise ExtractionError(
@@ -206,89 +207,89 @@ class ArchiveExtractionService:
                         "file_size": member.size
                     }
                 )
-    
+
     def _should_skip_file(self, relative_path: str) -> bool:
         """Check if file should be skipped during processing."""
-        
+
         # Normalize path for consistent checking
         relative_path_lower = relative_path.lower()
         path_parts = relative_path_lower.split('/')
-        
+
         # Check for hidden files (starting with dot)
         for part in path_parts:
             if part.startswith('.') and part not in ['.', '..']:
                 return True
-        
+
         # Check for system/build directories
         skip_directories = {
             '__pycache__', 'node_modules', 'dist', 'build', 'target',
             '.git', '.svn', '.hg', '.vscode', '.idea', '.eclipse'
         }
-        
+
         for part in path_parts:
             if part in skip_directories:
                 return True
-        
+
         # Check file extensions
         skip_extensions = {
             '.pyc', '.pyo', '.tmp', '.temp', '.swp', '.bak',
             '.exe', '.dll', '.so', '.dylib', '.jar', '.war', '.ear', '.iso', '.dmg'
         }
-        
+
         for ext in skip_extensions:
             if relative_path_lower.endswith(ext):
                 return True
-        
+
         # Check specific filenames
         filename = path_parts[-1] if path_parts else ''
         skip_filenames = {'.ds_store', 'thumbs.db'}
-        
+
         if filename in skip_filenames:
             return True
-        
+
         # Skip very deep paths (potential zip bomb)
         if relative_path.count('/') > 10:
             return True
-        
+
         return False
-    
+
     def _process_extracted_file(
-        self, 
-        full_path: str, 
+        self,
+        full_path: str,
         relative_path: str
     ) -> Optional[ExtractedItemData]:
         """Process individual extracted file."""
-        
+
         try:
             stat = os.stat(full_path)
             file_size = stat.st_size
-            
+
             # Skip very large files
             if file_size > self.MAX_FILE_SIZE:
                 return None
-            
+
             item_name = os.path.basename(relative_path)
-            
+
             # Detect content type and processability
             content_type = self._detect_content_type(item_name, full_path)
             is_text_file = self._is_text_file(item_name, full_path)
             is_processable = is_text_file and content_type in [
-                ContentType.DOCUMENT, 
-                ContentType.CODE, 
+                ContentType.DOCUMENT,
+                ContentType.CODE,
                 ContentType.DATA
             ]
-            
+
             # Extract content for processable files
             content = None
             if is_processable:
                 content = self._extract_text_content(full_path)
-            
+
             # Generate content hash
             content_hash = self._generate_content_hash(full_path, content)
-            
+
             # Detect language
             language = self._detect_language(item_name, content_type)
-            
+
             # Build metadata
             metadata = {
                 'mime_type': mimetypes.guess_type(item_name)[0] or 'application/octet-stream',
@@ -296,7 +297,7 @@ class ArchiveExtractionService:
                 'extraction_method': 'direct_read' if is_text_file else 'binary_skip',
                 'file_extension': Path(item_name).suffix.lower(),
             }
-            
+
             return ExtractedItemData(
                 relative_path=relative_path,
                 item_name=item_name,
@@ -308,44 +309,44 @@ class ArchiveExtractionService:
                 language=language,
                 metadata=metadata
             )
-            
-        except Exception as e:
+
+        except Exception:
             # Return None for problematic files
             return None
-    
+
     def _detect_content_type(self, item_name: str, full_path: str) -> str:
         """Detect content type from file extension and content."""
-        
+
         file_path = Path(item_name)
         extension = file_path.suffix.lower()
-        
+
         # Code files
         code_extensions = {
             '.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.go', '.rs',
             '.cpp', '.c', '.h', '.hpp', '.php', '.rb', '.cs', '.swift',
             '.kt', '.scala', '.clj', '.hs', '.ml', '.fs', '.elm'
         }
-        
+
         # Document files
         document_extensions = {
             '.md', '.txt', '.rst', '.adoc', '.pdf', '.docx', '.doc'
         }
-        
+
         # Data files
         data_extensions = {
             '.json', '.csv', '.xml', '.yml', '.yaml', '.toml', '.ini'
         }
-        
+
         # Image files
         image_extensions = {
             '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.webp'
         }
-        
+
         # Archive files
         archive_extensions = {
             '.zip', '.tar', '.gz', '.bz2', '.7z', '.rar'
         }
-        
+
         if extension in code_extensions:
             return ContentType.CODE
         elif extension in document_extensions:
@@ -358,78 +359,78 @@ class ArchiveExtractionService:
             return ContentType.ARCHIVE
         else:
             return ContentType.UNKNOWN
-    
+
     def _is_text_file(self, item_name: str, full_path: str) -> bool:
         """Check if file is a text file."""
-        
+
         # Check by extension first
         file_path = Path(item_name)
         extension = file_path.suffix.lower()
-        
+
         if extension in self.TEXT_EXTENSIONS:
             return True
-        
+
         # Special filenames
         special_names = {
             'dockerfile', 'makefile', 'readme', 'license', 'changelog',
             '.gitignore', '.dockerignore', '.env', '.settings.example'
         }
-        
+
         if file_path.name.lower() in special_names:
             return True
-        
+
         # Try to detect by content (sample first 1KB)
         try:
             with open(full_path, 'rb') as f:
                 sample = f.read(1024)
-            
+
             # Check for null bytes (binary indicator)
             if b'\x00' in sample:
                 return False
-            
+
             # Try to decode as UTF-8
             try:
                 sample.decode('utf-8')
                 return True
             except UnicodeDecodeError:
                 return False
-                
+
         except Exception:
             return False
-    
+
     def _extract_text_content(self, full_path: str) -> Optional[str]:
         """Extract text content from file."""
-        
+
         try:
             # Try different encodings
             encodings = ['utf-8', 'utf-16', 'latin1', 'cp1252']
-            
+
             for encoding in encodings:
                 try:
-                    with open(full_path, 'r', encoding=encoding) as f:
+                    with open(full_path, encoding=encoding) as f:
                         content = f.read()
-                        
+
                     # Validate content is reasonable
                     if len(content) > 0 and len(content) < 1024 * 1024:  # Max 1MB text
                         return content
-                        
+
                 except UnicodeDecodeError:
                     continue
                 except Exception:
                     break
-            
+
             return None
-            
+
         except Exception:
             return None
-    
+
     def _generate_content_hash(
-        self, 
-        full_path: str, 
+        self,
+        full_path: str,
         content: Optional[str]
     ) -> str:
         """Generate SHA-256 hash of file content."""
-        
+
         if content:
             return hashlib.sha256(content.encode()).hexdigest()
         else:
@@ -443,13 +444,13 @@ class ArchiveExtractionService:
             except Exception:
                 # Fallback to path-based hash
                 return hashlib.sha256(full_path.encode()).hexdigest()
-    
+
     def _detect_language(self, item_name: str, content_type: str) -> Optional[str]:
         """Detect programming language from file extension."""
-        
+
         if content_type != ContentType.CODE:
             return None
-        
+
         language_map = {
             '.py': 'python',
             '.js': 'javascript',
@@ -473,34 +474,34 @@ class ArchiveExtractionService:
             '.fs': 'fsharp',
             '.elm': 'elm',
         }
-        
+
         file_path = Path(item_name)
         extension = file_path.suffix.lower()
-        
+
         # Special cases
         if file_path.name.lower() in ['dockerfile']:
             return 'dockerfile'
         elif file_path.name.lower() in ['makefile']:
             return 'makefile'
-        
+
         return language_map.get(extension)
 
 
 class ContentExtractionService:
     """Service for extracting content from specific file types."""
-    
+
     def extract_pdf_content(self, file_path: str) -> Optional[str]:
         """Extract text from PDF file."""
         # TODO: Implement PDF text extraction
         # Could use PyPDF2, pdfplumber, or similar
         return None
-    
+
     def extract_docx_content(self, file_path: str) -> Optional[str]:
         """Extract text from DOCX file."""
         # TODO: Implement DOCX text extraction
         # Could use python-docx
         return None
-    
+
     def extract_image_text(self, file_path: str) -> Optional[str]:
         """Extract text from image using OCR."""
         # TODO: Implement OCR text extraction