agent_os_kernel 3.1.0__py3-none-any.whl

caas/ingestion/processors.py

@@ -0,0 +1,253 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+"""
+Data ingestion module for processing different file formats.
+"""
+
+import re
+from abc import ABC, abstractmethod
+from typing import Dict, Any, List
+from io import BytesIO
+
+from caas.models import Document, ContentFormat, DocumentType, Section
+
+
+class BaseProcessor(ABC):
+    """Base class for document processors."""
+    
+    @abstractmethod
+    def process(self, content: bytes, metadata: Dict[str, Any]) -> Document:
+        """Process raw content into a Document."""
+        pass
+    
+    def _extract_sections(self, text: str) -> List[Section]:
+        """Extract sections from text based on common patterns."""
+        sections = []
+        
+        # Pattern for headers (markdown-style or numbered)
+        header_pattern = r'(?:^|\n)(#{1,6}\s+.+|[A-Z][^\n]{5,80}:|\d+\.\s+[A-Z][^\n]+)'
+        matches = list(re.finditer(header_pattern, text))
+        
+        if not matches:
+            # No clear sections, treat as single section
+            return [Section(
+                title="Main Content",
+                content=text,
+                start_pos=0,
+                end_pos=len(text)
+            )]
+        
+        for i, match in enumerate(matches):
+            start = match.start()
+            end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+            title = match.group(1).strip('#: ')
+            content = text[start:end].strip()
+            
+            sections.append(Section(
+                title=title,
+                content=content,
+                start_pos=start,
+                end_pos=end
+            ))
+        
+        return sections
+
+
+class PDFProcessor(BaseProcessor):
+    """Processor for PDF documents."""
+    
+    def process(self, content: bytes, metadata: Dict[str, Any]) -> Document:
+        """Process PDF content."""
+        try:
+            from pypdf import PdfReader
+        except ImportError:
+            raise ImportError("pypdf is required for PDF processing")
+        
+        pdf_file = BytesIO(content)
+        reader = PdfReader(pdf_file)
+        
+        text = ""
+        for page in reader.pages:
+            text += page.extract_text() + "\n"
+        
+        sections = self._extract_sections(text)
+        
+        return Document(
+            id=metadata.get("id", ""),
+            title=metadata.get("title", "Untitled PDF"),
+            content=text,
+            format=ContentFormat.PDF,
+            detected_type=DocumentType.UNKNOWN,
+            sections=sections,
+            metadata=metadata
+        )
+
+
+class HTMLProcessor(BaseProcessor):
+    """Processor for HTML documents."""
+    
+    def process(self, content: bytes, metadata: Dict[str, Any]) -> Document:
+        """Process HTML content."""
+        try:
+            from bs4 import BeautifulSoup
+        except ImportError:
+            raise ImportError("beautifulsoup4 is required for HTML processing")
+        
+        soup = BeautifulSoup(content, 'lxml')
+        
+        # Extract title
+        title = soup.title.string if soup.title else "Untitled HTML"
+        
+        # Remove script and style elements
+        for element in soup(['script', 'style', 'nav', 'footer']):
+            element.decompose()
+        
+        # Extract sections based on headers with hierarchy tracking
+        sections = []
+        current_h1 = None  # Track current chapter (H1)
+        current_h2 = None  # Track current parent section (H2)
+        
+        for header in soup.find_all(['h1', 'h2', 'h3', 'h4']):
+            section_title = header.get_text().strip()
+            header_level = header.name  # 'h1', 'h2', etc.
+            
+            # Update hierarchy tracking BEFORE processing content
+            if header_level == 'h1':
+                current_h1 = section_title
+                current_h2 = None
+            elif header_level == 'h2':
+                current_h2 = section_title
+            
+            # Get content until next header or end
+            content_parts = []
+            for sibling in header.find_next_siblings():
+                if sibling.name in ['h1', 'h2', 'h3', 'h4']:
+                    break
+                content_parts.append(sibling.get_text())
+            
+            section_content = '\n'.join(content_parts).strip()
+            if section_content:
+                # Assign hierarchy based on current tracking
+                chapter = None
+                parent_section = None
+                
+                if header_level == 'h1':
+                    # H1 sections don't have chapter/parent
+                    pass
+                elif header_level == 'h2':
+                    chapter = current_h1
+                elif header_level in ['h3', 'h4']:
+                    chapter = current_h1
+                    parent_section = current_h2
+                
+                sections.append(Section(
+                    title=section_title,
+                    content=section_content,
+                    start_pos=0,
+                    end_pos=len(section_content),
+                    chapter=chapter,
+                    parent_section=parent_section
+                ))
+        
+        # Get all text
+        text = soup.get_text(separator='\n', strip=True)
+        
+        if not sections:
+            sections = self._extract_sections(text)
+        
+        return Document(
+            id=metadata.get("id", ""),
+            title=title,
+            content=text,
+            format=ContentFormat.HTML,
+            detected_type=DocumentType.UNKNOWN,
+            sections=sections,
+            metadata=metadata
+        )
+
+
+class CodeProcessor(BaseProcessor):
+    """Processor for source code files."""
+    
+    def process(self, content: bytes, metadata: Dict[str, Any]) -> Document:
+        """Process source code content."""
+        text = content.decode('utf-8', errors='ignore')
+        
+        # Detect programming language from metadata or content
+        language = metadata.get("language", self._detect_language(text))
+        
+        # Extract sections (classes, functions, etc.)
+        sections = self._extract_code_sections(text, language)
+        
+        if not sections:
+            sections = [Section(
+                title="Source Code",
+                content=text,
+                start_pos=0,
+                end_pos=len(text)
+            )]
+        
+        return Document(
+            id=metadata.get("id", ""),
+            title=metadata.get("title", "Source Code"),
+            content=text,
+            format=ContentFormat.CODE,
+            detected_type=DocumentType.SOURCE_CODE,
+            sections=sections,
+            metadata={**metadata, "language": language}
+        )
+    
+    def _detect_language(self, text: str) -> str:
+        """Simple language detection based on syntax."""
+        if 'def ' in text and ':' in text:
+            return 'python'
+        elif 'function' in text and '{' in text:
+            return 'javascript'
+        elif 'public class' in text or 'private class' in text:
+            return 'java'
+        elif '#include' in text:
+            return 'c++'
+        return 'unknown'
+    
+    def _extract_code_sections(self, text: str, language: str) -> List[Section]:
+        """Extract code sections (functions, classes, etc.)."""
+        sections = []
+        
+        if language == 'python':
+            # Match class and function definitions
+            pattern = r'(?:^|\n)((?:class|def)\s+\w+[^\n]*:)'
+            matches = list(re.finditer(pattern, text, re.MULTILINE))
+            
+            for i, match in enumerate(matches):
+                start = match.start()
+                end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+                title = match.group(1).strip()
+                content = text[start:end].strip()
+                
+                sections.append(Section(
+                    title=title,
+                    content=content,
+                    start_pos=start,
+                    end_pos=end
+                ))
+        
+        return sections
+
+
+class ProcessorFactory:
+    """Factory for creating appropriate processors."""
+    
+    @staticmethod
+    def get_processor(format: ContentFormat) -> BaseProcessor:
+        """Get processor for given format."""
+        processors = {
+            ContentFormat.PDF: PDFProcessor(),
+            ContentFormat.HTML: HTMLProcessor(),
+            ContentFormat.CODE: CodeProcessor(),
+        }
+        
+        processor = processors.get(format)
+        if not processor:
+            raise ValueError(f"No processor available for format: {format}")
+        
+        return processor

caas/ingestion/structure_parser.py

@@ -0,0 +1,188 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""
+Structure-Aware Parser for assigning content tiers.
+
+Implements hierarchical structure parsing that assigns weights based on 
+content importance tiers as described in the "Flat Chunk Fallacy".
+"""
+
+import re
+from typing import List
+from caas.models import Section, ContentTier, DocumentType
+
+
+class StructureParser:
+    """
+    Parses document structure and assigns content tiers.
+    
+    Tier 1 (High Weight): Titles, Headers, Class Definitions, API Contracts
+    Tier 2 (Medium Weight): Body text, Function logic
+    Tier 3 (Low Weight): Footnotes, Comments, Disclaimers
+    """
+    
+    # Patterns for identifying Tier 1 content (High Value)
+    TIER_1_PATTERNS = {
+        DocumentType.SOURCE_CODE: [
+            # Matches: public class MyClass, private interface IAuth, protected enum Status
+            r'^(public|private|protected)?\s*(class|interface|enum)\s+\w+',
+            # Matches: public void login(...) { (Java/C-style API methods)
+            r'^(public|private|protected)?\s*\w+\s+\w+\s*\([^)]*\)\s*{',
+            # Matches: def login(self, username, password): (Python top-level functions)
+            r'^\s*def\s+\w+\s*\([^)]*\)\s*:',
+            # Matches: export function authenticate, async function getData
+            r'^\s*(export\s+)?(async\s+)?function\s+\w+',
+            # Matches: @api, @Api, @API decorators/annotations
+            r'@(api|Api|API)',
+        ],
+        DocumentType.TECHNICAL_DOCUMENTATION: [
+            r'^#{1,2}\s+',  # H1, H2 headers (markdown)
+            r'^(API|Endpoint|Request|Response|Authentication|Authorization)',  # API sections
+            r'^\s*(GET|POST|PUT|DELETE|PATCH)\s+/',  # HTTP methods
+        ],
+        DocumentType.LEGAL_CONTRACT: [
+            r'^#{1,2}\s+',  # Main headers
+            r'^(DEFINITIONS?|TERMS?|OBLIGATIONS?|LIABILITY|INDEMNITY)',  # Key legal sections
+            r'^\d+\.\s+[A-Z][^:]+:',  # Numbered main clauses
+        ],
+        DocumentType.RESEARCH_PAPER: [
+            r'^(ABSTRACT|INTRODUCTION|CONCLUSION|RESULTS)',  # Key sections
+            r'^#{1,2}\s+',  # Main headers
+        ],
+        DocumentType.API_DOCUMENTATION: [
+            r'^#{1,2}\s+',  # H1, H2 headers
+            r'^\s*(GET|POST|PUT|DELETE|PATCH)\s+/',  # Endpoints
+            r'^(Authentication|Authorization|Endpoint)',  # Critical API sections
+        ],
+    }
+    
+    # Patterns for identifying Tier 3 content (Low Value)
+    TIER_3_PATTERNS = [
+        r'^\s*#.*$',  # Comments (code)
+        r'^\s*//.*$',  # Single-line comments
+        r'^\s*/\*.*\*/',  # Multi-line comments
+        r'TODO:|FIXME:|XXX:|HACK:',  # Comment markers
+        r'^\s*\*\s+Note:',  # Footnotes
+        r'^\s*\*\s+Disclaimer:',  # Disclaimers
+        r'^Footnote[s]?:',  # Footnotes
+        r'^Disclaimer[s]?:',  # Disclaimers
+        r'^Note[s]?:',  # Notes
+    ]
+    
+    def parse_and_assign_tiers(
+        self, 
+        sections: List[Section], 
+        doc_type: DocumentType,
+        content: str = ""
+    ) -> List[Section]:
+        """
+        Parse sections and assign content tiers.
+        
+        Args:
+            sections: List of document sections
+            doc_type: Type of document
+            content: Full document content (optional, for context)
+            
+        Returns:
+            List of sections with assigned tiers
+        """
+        for section in sections:
+            tier = self._determine_tier(section, doc_type)
+            section.tier = tier
+        
+        return sections
+    
+    def _determine_tier(self, section: Section, doc_type: DocumentType) -> ContentTier:
+        """
+        Determine the content tier for a section.
+        
+        Args:
+            section: The section to classify
+            doc_type: Type of document
+            
+        Returns:
+            The assigned content tier
+        """
+        content = section.content
+        title = section.title
+        
+        # Check for Tier 3 (Low Value) first
+        if self._is_tier_3_content(content, title):
+            return ContentTier.TIER_3_LOW
+        
+        # Check for Tier 1 (High Value)
+        if self._is_tier_1_content(content, title, doc_type):
+            return ContentTier.TIER_1_HIGH
+        
+        # Default to Tier 2 (Medium Value)
+        return ContentTier.TIER_2_MEDIUM
+    
+    def _is_tier_1_content(self, content: str, title: str, doc_type: DocumentType) -> bool:
+        """Check if content is Tier 1 (High Value)."""
+        combined_text = f"{title}\n{content}"
+        
+        # Check title for high-value indicators
+        title_lower = title.lower()
+        high_value_title_keywords = [
+            'definition', 'api', 'class', 'interface', 'contract', 
+            'authentication', 'authorization', 'endpoint', 'abstract',
+            'introduction', 'conclusion', 'overview', 'summary'
+        ]
+        
+        if any(keyword in title_lower for keyword in high_value_title_keywords):
+            return True
+        
+        # Check doc-type specific patterns
+        tier_1_patterns = self.TIER_1_PATTERNS.get(doc_type, [])
+        for pattern in tier_1_patterns:
+            if re.search(pattern, combined_text, re.MULTILINE | re.IGNORECASE):
+                return True
+        
+        # Check for API contracts (general)
+        if re.search(r'(contract|interface|protocol|specification)', combined_text, re.IGNORECASE):
+            return True
+        
+        return False
+    
+    def _is_tier_3_content(self, content: str, title: str) -> bool:
+        """Check if content is Tier 3 (Low Value)."""
+        combined_text = f"{title}\n{content}"
+        
+        # Check title for low-value indicators
+        title_lower = title.lower()
+        low_value_keywords = [
+            'footnote', 'disclaimer', 'note', 'comment', 'appendix',
+            'acknowledgment', 'copyright', 'license'
+        ]
+        
+        if any(keyword in title_lower for keyword in low_value_keywords):
+            return True
+        
+        # Check patterns
+        for pattern in self.TIER_3_PATTERNS:
+            if re.search(pattern, combined_text, re.MULTILINE | re.IGNORECASE):
+                # Make sure it's substantial (not just one comment line in a large section)
+                comment_lines = len(re.findall(pattern, combined_text, re.MULTILINE | re.IGNORECASE))
+                total_lines = len(combined_text.split('\n'))
+                if total_lines > 0 and comment_lines / total_lines > 0.5:
+                    return True
+        
+        return False
+    
+    def get_tier_base_weight(self, tier: ContentTier) -> float:
+        """
+        Get the base weight multiplier for a tier.
+        
+        Args:
+            tier: The content tier
+            
+        Returns:
+            Base weight multiplier
+        """
+        tier_weights = {
+            ContentTier.TIER_1_HIGH: 2.0,    # High value content gets 2x base weight
+            ContentTier.TIER_2_MEDIUM: 1.0,   # Medium value gets 1x base weight
+            ContentTier.TIER_3_LOW: 0.5,      # Low value gets 0.5x base weight
+        }
+        return tier_weights.get(tier, 1.0)