django-cfg 1.1.81__py3-none-any.whl → 1.2.0__py3-none-any.whl

django_cfg/apps/knowbase/utils/chunk_settings.py

@@ -0,0 +1,261 @@
+"""
+Dynamic chunk settings using Pydantic configuration.
+
+This module provides utilities for accessing and managing chunk processing
+settings with type safety and validation.
+"""
+
+import logging
+from typing import Dict, Any, Optional
+from pydantic import BaseModel, Field, validator
+from ..config.constance_settings import ConstanceSettings
+
+logger = logging.getLogger(__name__)
+
+
+class ChunkSettings(BaseModel):
+    """Pydantic model for chunk processing settings."""
+    
+    chunk_size: int = Field(
+        ge=100,
+        le=8000,
+        description="Size of each chunk in characters"
+    )
+    
+    chunk_overlap: int = Field(
+        ge=0,
+        description="Overlap between chunks in characters"
+    )
+    
+    embedding_batch_size: int = Field(
+        ge=1,
+        le=100,
+        description="Number of chunks to process in one embedding batch"
+    )
+    
+    embedding_model: str = Field(
+        min_length=1,
+        description="OpenAI embedding model name"
+    )
+    
+    @validator('chunk_overlap')
+    def validate_overlap(cls, v, values):
+        """Ensure overlap is less than chunk_size."""
+        chunk_size = values.get('chunk_size')
+        if chunk_size and v >= chunk_size:
+            raise ValueError(f"Chunk overlap ({v}) must be less than chunk size ({chunk_size})")
+        return v
+    
+    class Config:
+        """Pydantic configuration."""
+        validate_assignment = True
+
+
+class ChunkSettingsManager:
+    """Manager for dynamic chunk settings using Pydantic configuration."""
+    
+    @classmethod
+    def get_document_settings(cls) -> ChunkSettings:
+        """Get chunk settings for document processing."""
+        from ..config.settings import get_config
+        config = get_config()
+        return ChunkSettings(
+            chunk_size=ConstanceSettings.get_document_chunk_size(),
+            chunk_overlap=config.chunking.document_chunk_overlap,
+            embedding_batch_size=ConstanceSettings.get_embedding_batch_size(),
+            embedding_model=ConstanceSettings.get_embedding_model()
+        )
+    
+    @classmethod
+    def get_archive_settings(cls) -> ChunkSettings:
+        """Get chunk settings for archive processing."""
+        from ..config.settings import get_config
+        config = get_config()
+        return ChunkSettings(
+            chunk_size=ConstanceSettings.get_archive_chunk_size(),
+            chunk_overlap=config.chunking.archive_chunk_overlap,
+            embedding_batch_size=ConstanceSettings.get_embedding_batch_size(),
+            embedding_model=ConstanceSettings.get_embedding_model()
+        )
+    
+    @classmethod
+    def get_settings_for_type(cls, content_type: str) -> ChunkSettings:
+        """
+        Get chunk settings for specific content type.
+        
+        Args:
+            content_type: Either 'document' or 'archive'
+            
+        Returns:
+            ChunkSettings object with appropriate settings
+        """
+        if content_type == 'document':
+            return cls.get_document_settings()
+        elif content_type == 'archive':
+            return cls.get_archive_settings()
+        else:
+            logger.warning(f"Unknown content type: {content_type}, using document settings")
+            return cls.get_document_settings()
+    
+    @classmethod
+    def get_all_settings(cls) -> Dict[str, ChunkSettings]:
+        """Get all chunk settings as dictionary."""
+        return {
+            'document': cls.get_document_settings(),
+            'archive': cls.get_archive_settings()
+        }
+    
+    @classmethod
+    def validate_settings(cls, settings: ChunkSettings) -> bool:
+        """
+        Validate chunk settings.
+        
+        Args:
+            settings: ChunkSettings to validate
+            
+        Returns:
+            True if settings are valid, False otherwise
+        """
+        if settings.chunk_size <= 0:
+            logger.error(f"Invalid chunk_size: {settings.chunk_size}")
+            return False
+        
+        if settings.chunk_overlap < 0:
+            logger.error(f"Invalid chunk_overlap: {settings.chunk_overlap}")
+            return False
+        
+        if settings.chunk_overlap >= settings.chunk_size:
+            logger.error(f"Chunk overlap ({settings.chunk_overlap}) must be less than chunk size ({settings.chunk_size})")
+            return False
+        
+        if settings.embedding_batch_size <= 0 or settings.embedding_batch_size > 2048:
+            logger.error(f"Invalid embedding_batch_size: {settings.embedding_batch_size} (must be 1-2048)")
+            return False
+        
+        if not settings.embedding_model or not settings.embedding_model.strip():
+            logger.error("Embedding model cannot be empty")
+            return False
+        
+        return True
+    
+    @classmethod
+    def log_current_settings(cls) -> None:
+        """Log current settings for debugging."""
+        try:
+            doc_settings = cls.get_document_settings()
+            archive_settings = cls.get_archive_settings()
+            
+            logger.info("📊 Current Chunk Settings:")
+            logger.info(f"  📄 Documents: size={doc_settings.chunk_size}, overlap={doc_settings.chunk_overlap}")
+            logger.info(f"  📦 Archives: size={archive_settings.chunk_size}, overlap={archive_settings.chunk_overlap}")
+            logger.info(f"  🔮 Embedding: batch_size={doc_settings.embedding_batch_size}, model={doc_settings.embedding_model}")
+            
+        except Exception as e:
+            logger.error(f"Failed to log current settings: {e}")
+
+
+# Convenience functions for easy access (using new Pydantic config)
+def get_document_chunk_size() -> int:
+    """Get document chunk size."""
+    from ..config.settings import get_document_chunk_size
+    return get_document_chunk_size()
+
+
+def get_document_chunk_overlap() -> int:
+    """Get document chunk overlap."""
+    from ..config.settings import get_document_chunk_overlap
+    return get_document_chunk_overlap()
+
+
+def get_archive_chunk_size() -> int:
+    """Get archive chunk size."""
+    from ..config.settings import get_archive_chunk_size
+    return get_archive_chunk_size()
+
+
+def get_archive_chunk_overlap() -> int:
+    """Get archive chunk overlap."""
+    from ..config.settings import get_archive_chunk_overlap
+    return get_archive_chunk_overlap()
+
+
+def get_embedding_batch_size() -> int:
+    """Get embedding batch size."""
+    from ..config.settings import get_embedding_batch_size
+    return get_embedding_batch_size()
+
+
+def get_embedding_model() -> str:
+    """Get embedding model."""
+    from ..config.settings import get_embedding_model
+    return get_embedding_model()
+
+
+# Additional convenience functions using new Pydantic config
+def get_search_results_limit() -> int:
+    """Get search results limit."""
+    from ..config.settings import get_search_results_limit
+    return get_search_results_limit()
+
+
+def get_search_similarity_threshold() -> float:
+    """Get search similarity threshold."""
+    from ..config.settings import get_search_similarity_threshold
+    return get_search_similarity_threshold()
+
+
+def get_chat_context_chunks() -> int:
+    """Get number of chunks for chat context."""
+    from ..config.settings import get_chat_context_chunks
+    return get_chat_context_chunks()
+
+
+def get_chat_max_tokens() -> int:
+    """Get maximum tokens for chat completion."""
+    from ..config.settings import get_chat_max_tokens
+    return get_chat_max_tokens()
+
+
+def get_chat_temperature() -> float:
+    """Get chat completion temperature."""
+    from ..config.settings import get_chat_temperature
+    return get_chat_temperature()
+
+
+def get_max_archive_size_mb() -> int:
+    """Get maximum archive size in MB."""
+    from ..config.settings import get_max_archive_size_mb
+    return get_max_archive_size_mb()
+
+
+def get_max_document_size_mb() -> int:
+    """Get maximum document size in MB."""
+    from ..config.settings import get_max_document_size_mb
+    return get_max_document_size_mb()
+
+
+def get_processing_timeout_minutes() -> int:
+    """Get processing timeout in minutes."""
+    from ..config.settings import get_processing_timeout_minutes
+    return get_processing_timeout_minutes()
+
+
+def get_chunking_params_for_type(content_type: str) -> Dict[str, Any]:
+    """
+    Get chunking parameters for SemanticChunker.
+    
+    Args:
+        content_type: Either 'document' or 'archive'
+        
+    Returns:
+        Dictionary with chunk_size and overlap parameters
+    """
+    from ..config.settings import get_chunking_params_for_type
+    return get_chunking_params_for_type(content_type)
+
+
+# Initialize settings logging on module import
+try:
+    ChunkSettingsManager.log_current_settings()
+except Exception as e:
+    logger.debug(f"Could not log settings on import: {e}")

django_cfg/apps/knowbase/utils/text_processing.py

@@ -0,0 +1,375 @@
+"""
+Text processing utilities for document chunking and cleaning.
+"""
+
+import re
+from typing import List, Optional
+from pydantic import BaseModel, Field, validator
+from bs4 import BeautifulSoup, NavigableString
+
+
+class ChunkConfig(BaseModel):
+    """Pydantic configuration for text chunking."""
+    
+    chunk_size: int = Field(
+        default=1000,
+        ge=100,
+        le=8000,
+        description="Size of each text chunk in characters"
+    )
+    
+    overlap: int = Field(
+        default=200,
+        ge=0,
+        description="Overlap between consecutive chunks in characters"
+    )
+    
+    separators: List[str] = Field(
+        default_factory=lambda: ["\n\n", "\n", ". ", " "],
+        description="List of separators for text splitting in order of preference"
+    )
+    
+    @validator('overlap')
+    def validate_overlap(cls, v, values):
+        """Ensure overlap is less than chunk_size."""
+        chunk_size = values.get('chunk_size', 1000)
+        if v >= chunk_size:
+            raise ValueError(f"Overlap ({v}) must be less than chunk_size ({chunk_size})")
+        return v
+    
+    class Config:
+        """Pydantic configuration."""
+        validate_assignment = True
+
+
+class TextProcessor:
+    """Text cleaning and preprocessing utilities."""
+    
+    def clean_text(self, text: str) -> str:
+        """Clean and normalize text content."""
+        
+        # First, check if content contains HTML and clean it
+        if self.is_html_content(text):
+            text = self.clean_html_content(text)
+        
+        # Remove excessive whitespace
+        text = re.sub(r'\s+', ' ', text)
+        
+        # Remove special characters but keep punctuation
+        text = re.sub(r'[^\w\s\.\,\!\?\;\:\-\(\)\[\]\{\}\"\']+', '', text)
+        
+        # Normalize quotes
+        text = re.sub(r'["""]', '"', text)
+        text = re.sub(r"[''']", "'", text)
+        
+        # Remove extra spaces around punctuation
+        text = re.sub(r'\s+([\.,:;!?])', r'\1', text)
+        text = re.sub(r'([\.,:;!?])\s+', r'\1 ', text)
+        
+        # Strip and normalize
+        text = text.strip()
+        
+        return text
+    
+    def is_html_content(self, text: str) -> bool:
+        """Detect if content contains HTML tags."""
+        html_pattern = re.compile(r'<[^>]+>')
+        return bool(html_pattern.search(text))
+    
+    def clean_html_content(self, html_content: str) -> str:
+        """
+        Convert HTML content to clean text while preserving structure.
+        
+        Args:
+            html_content: Raw HTML content
+            
+        Returns:
+            Clean text with preserved structure
+        """
+        try:
+            # Parse HTML with BeautifulSoup
+            soup = BeautifulSoup(html_content, 'lxml')
+            
+            # Remove unwanted elements
+            self._remove_unwanted_elements(soup)
+            
+            # Convert to structured text
+            text = self._extract_structured_text(soup)
+            
+            return text
+            
+        except Exception as e:
+            # Fallback to simple tag removal if parsing fails
+            text = re.sub(r'<[^>]+>', '', html_content)
+            return text
+    
+    def _remove_unwanted_elements(self, soup: BeautifulSoup) -> None:
+        """Remove unwanted HTML elements."""
+        
+        # Remove script and style elements
+        for element in soup(['script', 'style', 'meta', 'link']):
+            element.decompose()
+        
+        # Remove comments
+        from bs4 import Comment
+        for comment in soup.find_all(string=lambda text: isinstance(text, Comment)):
+            comment.extract()
+        
+        # Remove empty elements
+        for element in soup.find_all():
+            if not element.get_text(strip=True) and not element.name in ['br', 'hr', 'img']:
+                element.decompose()
+    
+    def _extract_structured_text(self, soup: BeautifulSoup) -> str:
+        """
+        Extract text while preserving document structure.
+        
+        Args:
+            soup: BeautifulSoup parsed HTML
+            
+        Returns:
+            Structured text content
+        """
+        # Start with basic text extraction with proper spacing
+        text = soup.get_text(separator=' ', strip=True)
+        
+        # Process specific elements for better structure
+        for element in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
+            header_text = element.get_text(strip=True)
+            if header_text and header_text in text:
+                text = text.replace(header_text, f'\n\n{header_text}\n')
+        
+        for element in soup.find_all(['li']):
+            li_text = element.get_text(strip=True)
+            if li_text and li_text in text:
+                text = text.replace(li_text, f'\n• {li_text}')
+        
+        # Clean up excessive whitespace and newlines
+        text = re.sub(r'\s+', ' ', text)  # Multiple spaces to single space
+        text = re.sub(r'\n\s*\n', '\n\n', text)  # Multiple newlines to double newline
+        text = re.sub(r'\n{3,}', '\n\n', text)  # More than 2 newlines to 2
+        text = text.strip()
+        
+        return text
+    
+    def extract_metadata(self, text: str) -> dict:
+        """Extract basic metadata from text."""
+        
+        lines = text.split('\n')
+        
+        metadata = {
+            'character_count': len(text),
+            'word_count': len(text.split()),
+            'line_count': len(lines),
+            'paragraph_count': len([line for line in lines if line.strip()]),
+            'has_code': bool(re.search(r'```|`[^`]+`', text)),
+            'has_urls': bool(re.search(r'https?://\S+', text)),
+            'has_emails': bool(re.search(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', text)),
+            'is_html': self.is_html_content(text)
+        }
+        
+        # Add HTML-specific metadata if content is HTML
+        if metadata['is_html']:
+            html_metadata = self._extract_html_metadata(text)
+            metadata.update(html_metadata)
+        
+        return metadata
+    
+    def _extract_html_metadata(self, html_content: str) -> dict:
+        """Extract HTML-specific metadata."""
+        try:
+            soup = BeautifulSoup(html_content, 'lxml')
+            
+            # Count different HTML elements
+            headers = soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])
+            paragraphs = soup.find_all('p')
+            lists = soup.find_all(['ul', 'ol'])
+            list_items = soup.find_all('li')
+            links = soup.find_all('a')
+            images = soup.find_all('img')
+            tables = soup.find_all('table')
+            
+            return {
+                'html_headers_count': len(headers),
+                'html_paragraphs_count': len(paragraphs),
+                'html_lists_count': len(lists),
+                'html_list_items_count': len(list_items),
+                'html_links_count': len(links),
+                'html_images_count': len(images),
+                'html_tables_count': len(tables),
+                'html_has_forms': bool(soup.find('form')),
+                'html_has_media': bool(soup.find_all(['img', 'video', 'audio'])),
+            }
+        except:
+            return {
+                'html_parsing_error': True
+            }
+
+
+class SemanticChunker:
+    """Intelligent text chunking with semantic awareness."""
+    
+    def __init__(
+        self,
+        chunk_size: int = 1000,
+        overlap: int = 200,
+        separators: Optional[List[str]] = None
+    ):
+        # Handle None separators for backward compatibility
+        if separators is None:
+            separators = ["\n\n", "\n", ". ", " "]
+            
+        self.config = ChunkConfig(
+            chunk_size=chunk_size,
+            overlap=overlap,
+            separators=separators
+        )
+    
+    def create_chunks(self, text: str) -> List[str]:
+        """Split text into semantic chunks."""
+        
+        if len(text) <= self.config.chunk_size:
+            return [text]
+        
+        chunks = []
+        current_chunk = ""
+        
+        # Split by separators in order of preference
+        segments = self._split_by_separators(text, self.config.separators)
+        
+        for segment in segments:
+            # If segment alone is too big, split it further
+            if len(segment) > self.config.chunk_size:
+                # Split large segment
+                sub_chunks = self._split_large_segment(segment)
+                
+                # Add current chunk if it exists
+                if current_chunk:
+                    chunks.append(current_chunk.strip())
+                    current_chunk = ""
+                
+                # Add all but last sub-chunk
+                chunks.extend(sub_chunks[:-1])
+                current_chunk = sub_chunks[-1] if sub_chunks else ""
+                
+            # If adding segment would exceed chunk size
+            elif len(current_chunk) + len(segment) > self.config.chunk_size:
+                if current_chunk:
+                    chunks.append(current_chunk.strip())
+                current_chunk = segment
+            else:
+                current_chunk += segment
+        
+        # Add final chunk
+        if current_chunk:
+            chunks.append(current_chunk.strip())
+        
+        # Add overlap between chunks
+        if self.config.overlap > 0:
+            chunks = self._add_overlap(chunks)
+        
+        return [chunk for chunk in chunks if chunk.strip()]
+    
+    def _split_by_separators(self, text: str, separators: List[str]) -> List[str]:
+        """Split text by separators in order of preference."""
+        
+        segments = [text]
+        
+        for separator in separators:
+            new_segments = []
+            for segment in segments:
+                if separator in segment:
+                    parts = segment.split(separator)
+                    for i, part in enumerate(parts):
+                        if i > 0:
+                            new_segments.append(separator + part)
+                        else:
+                            new_segments.append(part)
+                else:
+                    new_segments.append(segment)
+            segments = new_segments
+        
+        return segments
+    
+    def _split_large_segment(self, segment: str) -> List[str]:
+        """Split a segment that's too large."""
+        
+        chunks = []
+        start = 0
+        
+        while start < len(segment):
+            end = start + self.config.chunk_size
+            
+            if end >= len(segment):
+                chunks.append(segment[start:])
+                break
+            
+            # Try to find a good breaking point
+            break_point = self._find_break_point(segment, start, end)
+            
+            chunks.append(segment[start:break_point])
+            start = break_point - self.config.overlap if break_point > self.config.overlap else break_point
+        
+        return chunks
+    
+    def _find_break_point(self, text: str, start: int, end: int) -> int:
+        """Find a good breaking point near the end position."""
+        
+        # Look for sentence endings
+        for i in range(end - 1, start + self.config.chunk_size // 2, -1):
+            if text[i] in '.!?':
+                return i + 1
+        
+        # Look for paragraph breaks
+        for i in range(end - 1, start + self.config.chunk_size // 2, -1):
+            if text[i] == '\n':
+                return i + 1
+        
+        # Look for word boundaries
+        for i in range(end - 1, start + self.config.chunk_size // 2, -1):
+            if text[i] == ' ':
+                return i + 1
+        
+        # No good break point found, use hard limit
+        return end
+    
+    def _add_overlap(self, chunks: List[str]) -> List[str]:
+        """Add overlap between consecutive chunks."""
+        
+        if len(chunks) <= 1:
+            return chunks
+        
+        overlapped_chunks = [chunks[0]]
+        
+        for i in range(1, len(chunks)):
+            prev_chunk = chunks[i - 1]
+            current_chunk = chunks[i]
+            
+            # Get overlap from previous chunk
+            overlap_text = prev_chunk[-self.config.overlap:] if len(prev_chunk) > self.config.overlap else prev_chunk
+            
+            # Add overlap to current chunk
+            overlapped_chunk = overlap_text + " " + current_chunk
+            overlapped_chunks.append(overlapped_chunk)
+        
+        return overlapped_chunks
+
+
+# Convenience function for backward compatibility
+def chunk_text(text: str, chunk_size: int = 500, overlap: int = 100) -> List[str]:
+    """
+    Convenience function to chunk text with default settings.
+    
+    Args:
+        text: Text to chunk
+        chunk_size: Size of each chunk
+        overlap: Overlap between chunks
+        
+    Returns:
+        List of text chunks
+    """
+    chunker = SemanticChunker(
+        chunk_size=chunk_size,
+        overlap=overlap
+    )
+    return chunker.create_chunks(text)