django-cfg 1.4.21__py3-none-any.whl → 1.4.24__py3-none-any.whl

django_cfg/apps/knowbase/utils/text_processing.py

@@ -4,31 +4,32 @@ Text processing utilities for document chunking and cleaning.
 
 import re
 from typing import List, Optional
+
+from bs4 import BeautifulSoup
 from pydantic import BaseModel, Field, validator
-from bs4 import BeautifulSoup, NavigableString
 
 
 class ChunkConfig(BaseModel):
     """Pydantic configuration for text chunking."""
-    
+
     chunk_size: int = Field(
         default=1000,
         ge=100,
         le=8000,
         description="Size of each text chunk in characters"
     )
-    
+
     overlap: int = Field(
         default=200,
         ge=0,
         description="Overlap between consecutive chunks in characters"
     )
-    
+
     separators: List[str] = Field(
         default_factory=lambda: ["\n\n", "\n", ". ", " "],
         description="List of separators for text splitting in order of preference"
     )
-    
+
     @validator('overlap')
     def validate_overlap(cls, v, values):
         """Ensure overlap is less than chunk_size."""
@@ -36,7 +37,7 @@ class ChunkConfig(BaseModel):
         if v >= chunk_size:
             raise ValueError(f"Overlap ({v}) must be less than chunk_size ({chunk_size})")
         return v
-    
+
     class Config:
         """Pydantic configuration."""
         validate_assignment = True
@@ -44,38 +45,38 @@ class ChunkConfig(BaseModel):
 
 class TextProcessor:
     """Text cleaning and preprocessing utilities."""
-    
+
     def clean_text(self, text: str) -> str:
         """Clean and normalize text content."""
-        
+
         # First, check if content contains HTML and clean it
         if self.is_html_content(text):
             text = self.clean_html_content(text)
-        
+
         # Remove excessive whitespace
         text = re.sub(r'\s+', ' ', text)
-        
+
         # Remove special characters but keep punctuation
         text = re.sub(r'[^\w\s\.\,\!\?\;\:\-\(\)\[\]\{\}\"\']+', '', text)
-        
+
         # Normalize quotes
         text = re.sub(r'["""]', '"', text)
         text = re.sub(r"[''']", "'", text)
-        
+
         # Remove extra spaces around punctuation
         text = re.sub(r'\s+([\.,:;!?])', r'\1', text)
         text = re.sub(r'([\.,:;!?])\s+', r'\1 ', text)
-        
+
         # Strip and normalize
         text = text.strip()
-        
+
         return text
-    
+
     def is_html_content(self, text: str) -> bool:
         """Detect if content contains HTML tags."""
         html_pattern = re.compile(r'<[^>]+>')
         return bool(html_pattern.search(text))
-    
+
     def clean_html_content(self, html_content: str) -> str:
         """
         Convert HTML content to clean text while preserving structure.
@@ -89,37 +90,37 @@ class TextProcessor:
         try:
             # Parse HTML with BeautifulSoup
             soup = BeautifulSoup(html_content, 'lxml')
-            
+
             # Remove unwanted elements
             self._remove_unwanted_elements(soup)
-            
+
             # Convert to structured text
             text = self._extract_structured_text(soup)
-            
+
             return text
-            
-        except Exception as e:
+
+        except Exception:
             # Fallback to simple tag removal if parsing fails
             text = re.sub(r'<[^>]+>', '', html_content)
             return text
-    
+
     def _remove_unwanted_elements(self, soup: BeautifulSoup) -> None:
         """Remove unwanted HTML elements."""
-        
+
         # Remove script and style elements
         for element in soup(['script', 'style', 'meta', 'link']):
             element.decompose()
-        
+
         # Remove comments
         from bs4 import Comment
         for comment in soup.find_all(string=lambda text: isinstance(text, Comment)):
             comment.extract()
-        
+
         # Remove empty elements
         for element in soup.find_all():
-            if not element.get_text(strip=True) and not element.name in ['br', 'hr', 'img']:
+            if not element.get_text(strip=True) and element.name not in ['br', 'hr', 'img']:
                 element.decompose()
-    
+
     def _extract_structured_text(self, soup: BeautifulSoup) -> str:
         """
         Extract text while preserving document structure.
@@ -132,31 +133,31 @@ class TextProcessor:
         """
         # Start with basic text extraction with proper spacing
         text = soup.get_text(separator=' ', strip=True)
-        
+
         # Process specific elements for better structure
         for element in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
             header_text = element.get_text(strip=True)
             if header_text and header_text in text:
                 text = text.replace(header_text, f'\n\n{header_text}\n')
-        
+
         for element in soup.find_all(['li']):
             li_text = element.get_text(strip=True)
             if li_text and li_text in text:
                 text = text.replace(li_text, f'\n• {li_text}')
-        
+
         # Clean up excessive whitespace and newlines
         text = re.sub(r'\s+', ' ', text)  # Multiple spaces to single space
         text = re.sub(r'\n\s*\n', '\n\n', text)  # Multiple newlines to double newline
         text = re.sub(r'\n{3,}', '\n\n', text)  # More than 2 newlines to 2
         text = text.strip()
-        
+
         return text
-    
+
     def extract_metadata(self, text: str) -> dict:
         """Extract basic metadata from text."""
-        
+
         lines = text.split('\n')
-        
+
         metadata = {
             'character_count': len(text),
             'word_count': len(text.split()),
@@ -167,19 +168,19 @@ class TextProcessor:
             'has_emails': bool(re.search(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', text)),
             'is_html': self.is_html_content(text)
         }
-        
+
         # Add HTML-specific metadata if content is HTML
         if metadata['is_html']:
             html_metadata = self._extract_html_metadata(text)
             metadata.update(html_metadata)
-        
+
         return metadata
-    
+
     def _extract_html_metadata(self, html_content: str) -> dict:
         """Extract HTML-specific metadata."""
         try:
             soup = BeautifulSoup(html_content, 'lxml')
-            
+
             # Count different HTML elements
             headers = soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])
             paragraphs = soup.find_all('p')
@@ -188,7 +189,7 @@ class TextProcessor:
             links = soup.find_all('a')
             images = soup.find_all('img')
             tables = soup.find_all('table')
-            
+
             return {
                 'html_headers_count': len(headers),
                 'html_paragraphs_count': len(paragraphs),
@@ -208,7 +209,7 @@ class TextProcessor:
 
 class SemanticChunker:
     """Intelligent text chunking with semantic awareness."""
-    
+
     def __init__(
         self,
         chunk_size: int = 1000,
@@ -218,40 +219,40 @@ class SemanticChunker:
         # Handle None separators for backward compatibility
         if separators is None:
             separators = ["\n\n", "\n", ". ", " "]
-            
+
         self.config = ChunkConfig(
             chunk_size=chunk_size,
             overlap=overlap,
             separators=separators
         )
-    
+
     def create_chunks(self, text: str) -> List[str]:
         """Split text into semantic chunks."""
-        
+
         if len(text) <= self.config.chunk_size:
             return [text]
-        
+
         chunks = []
         current_chunk = ""
-        
+
         # Split by separators in order of preference
         segments = self._split_by_separators(text, self.config.separators)
-        
+
         for segment in segments:
             # If segment alone is too big, split it further
             if len(segment) > self.config.chunk_size:
                 # Split large segment
                 sub_chunks = self._split_large_segment(segment)
-                
+
                 # Add current chunk if it exists
                 if current_chunk:
                     chunks.append(current_chunk.strip())
                     current_chunk = ""
-                
+
                 # Add all but last sub-chunk
                 chunks.extend(sub_chunks[:-1])
                 current_chunk = sub_chunks[-1] if sub_chunks else ""
-                
+
             # If adding segment would exceed chunk size
             elif len(current_chunk) + len(segment) > self.config.chunk_size:
                 if current_chunk:
@@ -259,22 +260,22 @@ class SemanticChunker:
                 current_chunk = segment
             else:
                 current_chunk += segment
-        
+
         # Add final chunk
         if current_chunk:
             chunks.append(current_chunk.strip())
-        
+
         # Add overlap between chunks
         if self.config.overlap > 0:
             chunks = self._add_overlap(chunks)
-        
+
         return [chunk for chunk in chunks if chunk.strip()]
-    
+
     def _split_by_separators(self, text: str, separators: List[str]) -> List[str]:
         """Split text by separators in order of preference."""
-        
+
         segments = [text]
-        
+
         for separator in separators:
             new_segments = []
             for segment in segments:
@@ -288,70 +289,70 @@ class SemanticChunker:
                 else:
                     new_segments.append(segment)
             segments = new_segments
-        
+
         return segments
-    
+
     def _split_large_segment(self, segment: str) -> List[str]:
         """Split a segment that's too large."""
-        
+
         chunks = []
         start = 0
-        
+
         while start < len(segment):
             end = start + self.config.chunk_size
-            
+
             if end >= len(segment):
                 chunks.append(segment[start:])
                 break
-            
+
             # Try to find a good breaking point
             break_point = self._find_break_point(segment, start, end)
-            
+
             chunks.append(segment[start:break_point])
             start = break_point - self.config.overlap if break_point > self.config.overlap else break_point
-        
+
         return chunks
-    
+
     def _find_break_point(self, text: str, start: int, end: int) -> int:
         """Find a good breaking point near the end position."""
-        
+
         # Look for sentence endings
         for i in range(end - 1, start + self.config.chunk_size // 2, -1):
             if text[i] in '.!?':
                 return i + 1
-        
+
         # Look for paragraph breaks
         for i in range(end - 1, start + self.config.chunk_size // 2, -1):
             if text[i] == '\n':
                 return i + 1
-        
+
         # Look for word boundaries
         for i in range(end - 1, start + self.config.chunk_size // 2, -1):
             if text[i] == ' ':
                 return i + 1
-        
+
         # No good break point found, use hard limit
         return end
-    
+
     def _add_overlap(self, chunks: List[str]) -> List[str]:
         """Add overlap between consecutive chunks."""
-        
+
         if len(chunks) <= 1:
             return chunks
-        
+
         overlapped_chunks = [chunks[0]]
-        
+
         for i in range(1, len(chunks)):
             prev_chunk = chunks[i - 1]
             current_chunk = chunks[i]
-            
+
             # Get overlap from previous chunk
             overlap_text = prev_chunk[-self.config.overlap:] if len(prev_chunk) > self.config.overlap else prev_chunk
-            
+
             # Add overlap to current chunk
             overlapped_chunk = overlap_text + " " + current_chunk
             overlapped_chunks.append(overlapped_chunk)
-        
+
         return overlapped_chunks
 
 

django_cfg/apps/knowbase/utils/validation.py

@@ -3,7 +3,7 @@ Validation utilities for the knowbase app.
 """
 
 import math
-from typing import Union, Optional
+from typing import Optional, Union
 
 
 def is_valid_float(value: Union[float, int, None]) -> bool:
@@ -18,7 +18,7 @@ def is_valid_float(value: Union[float, int, None]) -> bool:
     """
     if value is None:
         return False
-    
+
     try:
         float_value = float(value)
         return not (math.isnan(float_value) or math.isinf(float_value))
@@ -39,7 +39,7 @@ def safe_float(value: Union[float, int, None], default: float = 0.0) -> float:
     """
     if value is None:
         return default
-    
+
     try:
         float_value = float(value)
         if math.isnan(float_value) or math.isinf(float_value):
@@ -61,15 +61,15 @@ def validate_similarity_score(similarity: Union[float, int, None]) -> Optional[f
     """
     if not is_valid_float(similarity):
         return None
-    
+
     score = float(similarity)
-    
+
     # Clamp to valid range [0.0, 1.0] for similarity scores
     if score < 0.0:
         return 0.0
     elif score > 1.0:
         return 1.0
-    
+
     return score
 
 
@@ -84,16 +84,16 @@ def clean_search_results(results: list) -> list:
         Cleaned list with valid similarity scores
     """
     cleaned_results = []
-    
+
     for result in results:
         if 'similarity' not in result:
             continue
-            
+
         similarity = validate_similarity_score(result['similarity'])
         if similarity is not None:
             # Create a copy and update similarity
             cleaned_result = result.copy()
             cleaned_result['similarity'] = similarity
             cleaned_results.append(cleaned_result)
-    
+
     return cleaned_results

django_cfg/apps/knowbase/views/__init__.py

@@ -4,23 +4,23 @@ Knowledge Base API Views
 DRF ViewSets for REST API endpoints.
 """
 
-from .document_views import *
+from .archive_views import *
 from .chat_views import *
+from .document_views import *
 from .public_views import *
-from .archive_views import *
 
 __all__ = [
     # Document views
     'DocumentViewSet',
-    
+
     # Public views
     'PublicDocumentViewSet',
     'PublicCategoryViewSet',
-    
+
     # Chat views
     'ChatViewSet',
     'ChatSessionViewSet',
-    
+
     # Archive views
     'DocumentArchiveViewSet',
     'ArchiveItemViewSet',