django-cfg 1.3.11__py3-none-any.whl → 1.4.0__py3-none-any.whl

django_cfg/apps/knowbase/services/archive/analyzers/tag_generator.py

@@ -0,0 +1,103 @@
+"""
+Tag generation for chunks.
+
+Generates semantic tags for code and document chunks.
+"""
+
+import re
+from typing import List, Dict, Any
+
+
+class TagGenerator:
+    """Generate tags for different content types."""
+
+    @staticmethod
+    def generate_code_tags(content: str, code_info: Dict[str, Any]) -> List[str]:
+        """
+        Generate technical tags for code content.
+
+        Args:
+            content: Code content
+            code_info: Code analysis info (element_name, element_type, etc.)
+
+        Returns:
+            List of technical tags
+        """
+        tags = []
+
+        # Element type tags
+        if code_info.get('element_type'):
+            tags.append(f"contains:{code_info['element_type']}")
+
+        # Async tag
+        if code_info.get('is_async'):
+            tags.append('async')
+
+        # Pattern detection
+        if 'import ' in content or 'from ' in content:
+            tags.append('contains:imports')
+
+        if 'class ' in content:
+            tags.append('contains:class_definition')
+
+        if 'def ' in content:
+            tags.append('contains:function_definition')
+
+        if 'test' in code_info.get('element_name', '').lower():
+            tags.append('purpose:testing')
+
+        return tags
+
+    @staticmethod
+    def generate_document_tags(content: str) -> List[str]:
+        """
+        Generate topic tags for document content.
+
+        Args:
+            content: Document content
+
+        Returns:
+            List of topic tags
+        """
+        tags = []
+
+        # Detect headings
+        if content.strip().startswith('#'):
+            tags.append('contains:heading')
+
+        # Detect lists
+        if re.search(r'^\s*[-*+]\s', content, re.MULTILINE):
+            tags.append('contains:list')
+
+        # Detect code blocks
+        if '```' in content or '    ' in content:
+            tags.append('contains:code_block')
+
+        return tags
+
+    @staticmethod
+    def detect_code_purpose(element_name: str, content: str) -> str:
+        """
+        Detect purpose of code element.
+
+        Args:
+            element_name: Name of code element
+            content: Code content
+
+        Returns:
+            Purpose string (test, configuration, initialization, etc.)
+        """
+        name_lower = element_name.lower()
+
+        if name_lower.startswith('test_'):
+            return 'test'
+        elif name_lower.startswith('_'):
+            return 'private_method'
+        elif 'config' in name_lower:
+            return 'configuration'
+        elif 'init' in name_lower:
+            return 'initialization'
+        elif 'main' in name_lower:
+            return 'main_function'
+        else:
+            return 'implementation'

django_cfg/apps/knowbase/services/archive/chunking/__init__.py

@@ -0,0 +1,19 @@
+"""
+Chunking strategies for different content types.
+
+Language-specific and content-aware chunking implementations.
+"""
+
+from .base import BaseChunker
+from .text_chunker import TextChunker
+from .python_chunker import PythonChunker
+from .markdown_chunker import MarkdownChunker
+from .json_chunker import JsonChunker
+
+__all__ = [
+    'BaseChunker',
+    'TextChunker',
+    'PythonChunker',
+    'MarkdownChunker',
+    'JsonChunker',
+]

django_cfg/apps/knowbase/services/archive/chunking/base.py

@@ -0,0 +1,81 @@
+"""
+Base chunker interface.
+
+Abstract base class for content-specific chunking strategies.
+"""
+
+from abc import ABC, abstractmethod
+from typing import List
+from ....models.archive import ArchiveItem
+from ..context.models import ChunkData
+
+
+class BaseChunker(ABC):
+    """
+    Base class for content chunkers.
+
+    Provides common interface and utilities for chunking strategies.
+    """
+
+    def __init__(self, chunk_size: int = 1000, overlap: int = 200):
+        """
+        Initialize chunker.
+
+        Args:
+            chunk_size: Default chunk size in characters
+            overlap: Overlap between chunks in characters
+        """
+        self.chunk_size = chunk_size
+        self.overlap = overlap
+
+    @abstractmethod
+    def can_handle(self, item: ArchiveItem) -> bool:
+        """
+        Check if this chunker can handle the item.
+
+        Args:
+            item: Archive item to check
+
+        Returns:
+            True if this chunker can process the item
+        """
+        pass
+
+    @abstractmethod
+    def chunk(self, item: ArchiveItem) -> List[ChunkData]:
+        """
+        Create chunks from item content.
+
+        Args:
+            item: Archive item to chunk
+
+        Returns:
+            List of ChunkData objects
+        """
+        pass
+
+    def _find_good_break_point(self, content: str, start: int, end: int) -> int:
+        """
+        Find good break point for text chunking.
+
+        Tries to break at sentence endings or word boundaries.
+
+        Args:
+            content: Content being chunked
+            start: Start position
+            end: Desired end position
+
+        Returns:
+            Actual break point position
+        """
+        # Look for sentence endings
+        for i in range(end - 1, start, -1):
+            if content[i] in '.!?\n':
+                return i + 1
+
+        # Look for word boundaries
+        for i in range(end - 1, start, -1):
+            if content[i].isspace():
+                return i
+
+        return end

django_cfg/apps/knowbase/services/archive/chunking/json_chunker.py

@@ -0,0 +1,62 @@
+"""
+JSON data chunker.
+
+Chunks JSON by object structure.
+"""
+
+import json
+from typing import List
+from .base import BaseChunker
+from ..context.models import ChunkData
+from ..context.builders import ChunkContextBuilder
+from ....models.archive import ArchiveItem, ChunkType
+
+
+class JsonChunker(BaseChunker):
+    """
+    Chunk JSON by object structure.
+
+    Splits JSON documents by top-level keys.
+    """
+
+    def can_handle(self, item: ArchiveItem) -> bool:
+        """Can handle JSON files."""
+        return item.language == 'json'
+
+    def chunk(self, item: ArchiveItem) -> List[ChunkData]:
+        """
+        Chunk JSON by object structure.
+
+        Args:
+            item: Archive item to chunk
+
+        Returns:
+            List of ChunkData objects
+        """
+        try:
+            data = json.loads(item.raw_content)
+            chunks = []
+
+            if isinstance(data, dict):
+                # Chunk by top-level keys
+                for key, value in data.items():
+                    chunk_content = json.dumps({key: value}, indent=2)
+
+                    context = ChunkContextBuilder.build_data_context(
+                        item, len(chunks), chunk_content, 'json_object', key
+                    )
+
+                    chunks.append(ChunkData(
+                        content=chunk_content,
+                        chunk_index=len(chunks),
+                        chunk_type=ChunkType.METADATA,
+                        context_metadata=context
+                    ))
+
+            return chunks
+
+        except json.JSONDecodeError:
+            # Fallback to text chunking on parse errors
+            from .text_chunker import TextChunker
+            text_chunker = TextChunker(self.chunk_size, self.overlap)
+            return text_chunker.chunk(item)

django_cfg/apps/knowbase/services/archive/chunking/markdown_chunker.py

@@ -0,0 +1,107 @@
+"""
+Markdown document chunker.
+
+Chunks markdown by headings and sections.
+"""
+
+from typing import List, Dict, Any
+from .base import BaseChunker
+from ..context.models import ChunkData
+from ..context.builders import ChunkContextBuilder
+from ....models.archive import ArchiveItem, ChunkType
+
+
+class MarkdownChunker(BaseChunker):
+    """
+    Chunk markdown by headings and sections.
+
+    Splits markdown documents into logical sections based on heading structure.
+    """
+
+    def can_handle(self, item: ArchiveItem) -> bool:
+        """Can handle markdown files."""
+        return item.language == 'markdown'
+
+    def chunk(self, item: ArchiveItem) -> List[ChunkData]:
+        """
+        Chunk markdown by headings and sections.
+
+        Args:
+            item: Archive item to chunk
+
+        Returns:
+            List of ChunkData objects
+        """
+        content = item.raw_content
+        lines = content.split('\n')
+        chunks = []
+
+        current_section = {'title': '', 'level': 0, 'start_line': 0}
+
+        for i, line in enumerate(lines):
+            if line.startswith('#'):
+                # New section found
+                if current_section['start_line'] < i:
+                    # Create chunk for previous section
+                    chunk = self._create_section_chunk(
+                        lines[current_section['start_line']:i],
+                        current_section,
+                        item,
+                        len(chunks)
+                    )
+                    chunks.append(chunk)
+
+                # Start new section
+                level = len(line) - len(line.lstrip('#'))
+                current_section = {
+                    'title': line.lstrip('# ').strip(),
+                    'level': level,
+                    'start_line': i
+                }
+
+        # Handle last section
+        if current_section['start_line'] < len(lines):
+            chunk = self._create_section_chunk(
+                lines[current_section['start_line']:],
+                current_section,
+                item,
+                len(chunks)
+            )
+            chunks.append(chunk)
+
+        return chunks
+
+    def _create_section_chunk(
+        self,
+        section_lines: List[str],
+        section_info: Dict[str, Any],
+        item: ArchiveItem,
+        chunk_index: int
+    ) -> ChunkData:
+        """
+        Create chunk for markdown section.
+
+        Args:
+            section_lines: Lines of the section
+            section_info: Section metadata (title, level)
+            item: Archive item
+            chunk_index: Index of this chunk
+
+        Returns:
+            ChunkData for the section
+        """
+        content = '\n'.join(section_lines)
+
+        # Build context metadata
+        context = ChunkContextBuilder.build_document_context(
+            item, chunk_index, content, section_info
+        )
+
+        chunk_type = ChunkType.HEADING if section_info['title'] else ChunkType.TEXT
+
+        return ChunkData(
+            content=content,
+            chunk_index=chunk_index,
+            chunk_type=chunk_type,
+            context_metadata=context
+        )

django_cfg/apps/knowbase/services/archive/chunking/python_chunker.py

@@ -0,0 +1,248 @@
+"""
+Python code chunker using AST parsing.
+
+Chunks Python code by logical boundaries (classes, functions, imports).
+"""
+
+import ast
+from typing import List, Optional, Dict, Any
+from .base import BaseChunker
+from ..context.models import ChunkData
+from ..context.builders import ChunkContextBuilder
+from ....models.archive import ArchiveItem, ChunkType
+
+
+class PythonChunker(BaseChunker):
+    """
+    Chunk Python code by logical boundaries using AST.
+
+    Extracts:
+    - Import statements
+    - Class definitions
+    - Function definitions
+    - Module-level code
+    """
+
+    def can_handle(self, item: ArchiveItem) -> bool:
+        """Can handle Python code files."""
+        return item.language == 'python'
+
+    def chunk(self, item: ArchiveItem) -> List[ChunkData]:
+        """
+        Chunk Python code by classes and functions.
+
+        Args:
+            item: Archive item to chunk
+
+        Returns:
+            List of ChunkData objects
+        """
+        content = item.raw_content
+        lines = content.split('\n')
+        chunks = []
+
+        try:
+            tree = ast.parse(content)
+
+            # Extract imports first
+            imports_chunk = self._extract_imports(tree, lines, item, 0)
+            if imports_chunk:
+                chunks.append(imports_chunk)
+
+            # Extract classes and functions
+            for node in ast.walk(tree):
+                if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
+                    chunk = self._create_element_chunk(
+                        node, lines, item, len(chunks)
+                    )
+                    chunks.append(chunk)
+
+            # Handle module-level code
+            remaining_chunk = self._extract_remaining_code(
+                tree, lines, item, len(chunks)
+            )
+            if remaining_chunk:
+                chunks.append(remaining_chunk)
+
+        except SyntaxError:
+            # Fallback to text chunking on syntax errors
+            from .text_chunker import TextChunker
+            text_chunker = TextChunker(self.chunk_size, self.overlap)
+            return text_chunker.chunk(item)
+
+        return chunks
+
+    def _create_element_chunk(
+        self,
+        node: ast.AST,
+        lines: List[str],
+        item: ArchiveItem,
+        chunk_index: int
+    ) -> ChunkData:
+        """
+        Create chunk for Python code element.
+
+        Args:
+            node: AST node (ClassDef, FunctionDef, etc.)
+            lines: File lines
+            item: Archive item
+            chunk_index: Index of this chunk
+
+        Returns:
+            ChunkData for the element
+        """
+        start_line = node.lineno - 1
+        end_line = self._find_block_end(node, lines)
+
+        content = '\n'.join(lines[start_line:end_line])
+
+        # Analyze code structure
+        from ..analyzers import ComplexityAnalyzer, QualityAnalyzer, PurposeDetector
+        code_info = self._analyze_structure(
+            node, content,
+            ComplexityAnalyzer, QualityAnalyzer, PurposeDetector
+        )
+
+        # Build context metadata
+        context = ChunkContextBuilder.build_code_context(
+            item, chunk_index, content, start_line, end_line, code_info
+        )
+
+        return ChunkData(
+            content=content,
+            chunk_index=chunk_index,
+            chunk_type=ChunkType.CODE,
+            context_metadata=context
+        )
+
+    def _analyze_structure(
+        self,
+        node: ast.AST,
+        content: str,
+        complexity_analyzer,
+        quality_analyzer,
+        purpose_detector
+    ) -> Dict[str, Any]:
+        """
+        Analyze Python code structure for context.
+
+        Args:
+            node: AST node
+            content: Code content
+            complexity_analyzer: Analyzer for complexity
+            quality_analyzer: Analyzer for quality
+            purpose_detector: Detector for purpose
+
+        Returns:
+            Dictionary with code analysis info
+        """
+        info = {
+            'element_name': node.name,
+            'element_type': 'class' if isinstance(node, ast.ClassDef) else 'function',
+            'is_async': isinstance(node, ast.AsyncFunctionDef),
+            'docstring': ast.get_docstring(node),
+            'decorators': [d.id for d in getattr(node, 'decorator_list', []) if hasattr(d, 'id')],
+            'complexity_score': complexity_analyzer.calculate_code_complexity(content),
+            'quality_score': quality_analyzer.assess_code_quality(content),
+            'purpose': purpose_detector.detect_code_purpose(node.name, content),
+        }
+
+        # Extract function/method arguments
+        if hasattr(node, 'args'):
+            info['arguments'] = [arg.arg for arg in node.args.args]
+
+        # Extract class bases
+        if isinstance(node, ast.ClassDef):
+            info['base_classes'] = [base.id for base in node.bases if hasattr(base, 'id')]
+
+        return info
+
+    def _find_block_end(self, node: ast.AST, lines: List[str]) -> int:
+        """
+        Find end line of Python code block.
+
+        Args:
+            node: AST node
+            lines: File lines
+
+        Returns:
+            End line number
+        """
+        # Start from the node's end line
+        start_line = getattr(node, 'end_lineno', node.lineno) or node.lineno
+
+        # Look for the actual end by checking indentation
+        for i in range(start_line, len(lines)):
+            line = lines[i]
+            if line.strip() and not line.startswith(' ') and not line.startswith('\t'):
+                return i
+
+        return len(lines)
+
+    def _extract_imports(
+        self,
+        tree: ast.AST,
+        lines: List[str],
+        item: ArchiveItem,
+        chunk_index: int
+    ) -> Optional[ChunkData]:
+        """
+        Extract imports as separate chunk.
+
+        Args:
+            tree: Parsed AST
+            lines: File lines
+            item: Archive item
+            chunk_index: Chunk index
+
+        Returns:
+            ChunkData for imports or None
+        """
+        import_lines = []
+
+        for node in ast.walk(tree):
+            if isinstance(node, (ast.Import, ast.ImportFrom)):
+                import_lines.append(node.lineno - 1)
+
+        if not import_lines:
+            return None
+
+        # Get all import lines
+        import_content = '\n'.join(lines[min(import_lines):max(import_lines) + 1])
+
+        context = ChunkContextBuilder.build_code_context(
+            item, chunk_index, import_content,
+            min(import_lines), max(import_lines) + 1,
+            {'element_name': 'imports', 'element_type': 'imports', 'purpose': 'imports'}
+        )
+
+        return ChunkData(
+            content=import_content,
+            chunk_index=chunk_index,
+            chunk_type=ChunkType.METADATA,
+            context_metadata=context
+        )
+
+    def _extract_remaining_code(
+        self,
+        tree: ast.AST,
+        lines: List[str],
+        item: ArchiveItem,
+        chunk_index: int
+    ) -> Optional[ChunkData]:
+        """
+        Extract remaining module-level code.
+
+        Args:
+            tree: Parsed AST
+            lines: File lines
+            item: Archive item
+            chunk_index: Chunk index
+
+        Returns:
+            ChunkData for module-level code or None
+        """
+        # Simplified implementation - skip module-level code for now
+        # Could be enhanced to identify module-level statements
+        # that aren't part of classes or functions
+        return None

django_cfg/apps/knowbase/services/archive/chunking/text_chunker.py

@@ -0,0 +1,70 @@
+"""
+Generic text chunker.
+
+Simple text chunking with fixed size and overlap.
+"""
+
+from typing import List
+from .base import BaseChunker
+from ..context.models import ChunkData
+from ..context.builders import ChunkContextBuilder
+from ....models.archive import ArchiveItem, ChunkType
+
+
+class TextChunker(BaseChunker):
+    """
+    Generic text chunker with overlap.
+
+    Uses fixed-size chunking with overlap and smart break points.
+    """
+
+    def can_handle(self, item: ArchiveItem) -> bool:
+        """Can handle any content as fallback."""
+        return True
+
+    def chunk(self, item: ArchiveItem) -> List[ChunkData]:
+        """
+        Chunk text with fixed size and overlap.
+
+        Args:
+            item: Archive item to chunk
+
+        Returns:
+            List of ChunkData objects
+        """
+        content = item.raw_content
+        chunks = []
+
+        # Simple text splitting with overlap
+        start = 0
+        chunk_index = 0
+
+        while start < len(content):
+            end = start + self.chunk_size
+
+            # Try to break at word boundary
+            if end < len(content):
+                break_point = self._find_good_break_point(content, start, end)
+                if break_point > start:
+                    end = break_point
+
+            chunk_content = content[start:end].strip()
+
+            if chunk_content:
+                context = ChunkContextBuilder.build_generic_context(
+                    item, chunk_index, chunk_content, start, end
+                )
+
+                chunks.append(ChunkData(
+                    content=chunk_content,
+                    chunk_index=chunk_index,
+                    chunk_type=ChunkType.TEXT,
+                    context_metadata=context
+                ))
+
+                chunk_index += 1
+
+            # Move start position with overlap
+            start = max(start + self.chunk_size - self.overlap, end)
+
+        return chunks