agent_os_kernel 3.1.0__py3-none-any.whl

caas/detection/detector.py

@@ -0,0 +1,238 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+"""
+Auto-detection module for identifying document types and structures.
+"""
+
+import re
+from typing import Dict, List, Tuple, Any
+
+from caas.models import Document, DocumentType, Section
+
+
+class DocumentTypeDetector:
+    """Detects document type based on content analysis."""
+    
+    # Keywords and patterns for different document types
+    PATTERNS = {
+        DocumentType.LEGAL_CONTRACT: [
+            r'\bwhereas\b',
+            r'\bparty\b.*\bparty\b',
+            r'\bagreement\b',
+            r'\bcontract\b',
+            r'\btherefore\b',
+            r'\bhereby\b',
+            r'\bindemnify\b',
+            r'\bliability\b',
+            r'\btermination\b',
+            r'\bgoverning law\b',
+        ],
+        DocumentType.TECHNICAL_DOCUMENTATION: [
+            r'\bAPI\b',
+            r'\binstallation\b',
+            r'\bconfiguration\b',
+            r'\bparameter[s]?\b',
+            r'\bmethod[s]?\b',
+            r'\breturn[s]?\b',
+            r'\bexample[s]?\b',
+            r'\busage\b',
+            r'\bsyntax\b',
+        ],
+        DocumentType.SOURCE_CODE: [
+            r'\bfunction\b',
+            r'\bclass\b',
+            r'\bdef\b',
+            r'\bimport\b',
+            r'\breturn\b',
+            r'\bif\b.*\belse\b',
+            r'\bfor\b.*\bin\b',
+            r'\bwhile\b',
+        ],
+        DocumentType.RESEARCH_PAPER: [
+            r'\babstract\b',
+            r'\bintroduction\b',
+            r'\bmethodology\b',
+            r'\bresults\b',
+            r'\bconclusion\b',
+            r'\breferences\b',
+            r'\bcitation[s]?\b',
+            r'\bhypothesis\b',
+        ],
+        DocumentType.TUTORIAL: [
+            r'\bstep[s]?\b',
+            r'\btutorial\b',
+            r'\bhow to\b',
+            r'\bguide\b',
+            r'\bbeginners?\b',
+            r'\blesson\b',
+            r'\bexercise[s]?\b',
+        ],
+        DocumentType.API_DOCUMENTATION: [
+            r'\bendpoint[s]?\b',
+            r'\bGET\b.*\bPOST\b',
+            r'\bHTTP\b',
+            r'\brequest\b.*\bresponse\b',
+            r'\bauthentication\b',
+            r'\bheader[s]?\b',
+            r'\bstatus code[s]?\b',
+        ],
+    }
+    
+    def detect(self, document: Document) -> DocumentType:
+        """
+        Detect the document type based on content analysis.
+        
+        Args:
+            document: The document to analyze
+            
+        Returns:
+            The detected document type
+        """
+        if document.format == "code":
+            return DocumentType.SOURCE_CODE
+        
+        content = document.content.lower()
+        scores: Dict[DocumentType, int] = {}
+        
+        # Score each document type based on pattern matches
+        for doc_type, patterns in self.PATTERNS.items():
+            score = 0
+            for pattern in patterns:
+                matches = len(re.findall(pattern, content, re.IGNORECASE))
+                score += matches
+            scores[doc_type] = score
+        
+        # Get the type with highest score
+        if scores:
+            max_score = max(scores.values())
+            if max_score > 0:
+                detected_type = max(scores, key=scores.get)
+                return detected_type
+        
+        return DocumentType.UNKNOWN
+    
+    def detect_structure(self, document: Document) -> Dict[str, Any]:
+        """
+        Detect the structural characteristics of a document.
+        
+        Returns information about:
+        - Section hierarchy
+        - Key sections
+        - Document organization
+        """
+        structure = {
+            "has_clear_sections": len(document.sections) > 1,
+            "section_count": len(document.sections),
+            "section_titles": [s.title for s in document.sections],
+            "key_sections": self._identify_key_sections(document),
+        }
+        
+        return structure
+    
+    def _identify_key_sections(self, document: Document) -> List[str]:
+        """Identify which sections are likely most important."""
+        key_patterns = [
+            r'definition[s]?',
+            r'summary',
+            r'conclusion',
+            r'abstract',
+            r'introduction',
+            r'overview',
+            r'getting started',
+            r'quick start',
+            r'main',
+        ]
+        
+        key_sections = []
+        for section in document.sections:
+            title_lower = section.title.lower()
+            for pattern in key_patterns:
+                if re.search(pattern, title_lower):
+                    key_sections.append(section.title)
+                    break
+        
+        return key_sections
+
+
+class StructureAnalyzer:
+    """Analyzes document structure for optimization."""
+    
+    def analyze(self, document: Document) -> Dict[str, Any]:
+        """
+        Analyze document structure and return insights.
+        
+        Returns:
+            Analysis results including section importance and relationships
+        """
+        analysis = {
+            "document_type": document.detected_type,
+            "section_analysis": [],
+            "content_density": self._calculate_density(document),
+            "structure_quality": self._assess_structure_quality(document),
+        }
+        
+        # Analyze each section
+        for section in document.sections:
+            section_info = {
+                "title": section.title,
+                "length": len(section.content),
+                "complexity": self._estimate_complexity(section.content),
+                "keyword_density": self._calculate_keyword_density(section.content),
+            }
+            analysis["section_analysis"].append(section_info)
+        
+        return analysis
+    
+    def _calculate_density(self, document: Document) -> float:
+        """Calculate content density (information per character)."""
+        if not document.content:
+            return 0.0
+        
+        words = len(document.content.split())
+        chars = len(document.content)
+        return words / chars if chars > 0 else 0.0
+    
+    def _assess_structure_quality(self, document: Document) -> str:
+        """Assess the quality of document structure."""
+        if len(document.sections) == 0:
+            return "poor"
+        elif len(document.sections) < 3:
+            return "basic"
+        elif len(document.sections) < 8:
+            return "good"
+        else:
+            return "excellent"
+    
+    def _estimate_complexity(self, text: str) -> float:
+        """Estimate text complexity based on various factors."""
+        if not text:
+            return 0.0
+        
+        words = text.split()
+        if not words:
+            return 0.0
+        
+        # Average word length
+        avg_word_len = sum(len(w) for w in words) / len(words)
+        
+        # Sentence count
+        sentences = len(re.split(r'[.!?]+', text))
+        words_per_sentence = len(words) / sentences if sentences > 0 else 0
+        
+        # Complexity score (normalized 0-1)
+        complexity = min(1.0, (avg_word_len / 10 + words_per_sentence / 30) / 2)
+        return complexity
+    
+    def _calculate_keyword_density(self, text: str) -> float:
+        """Calculate density of important keywords."""
+        important_words = set([
+            'important', 'critical', 'must', 'required', 'essential',
+            'key', 'primary', 'main', 'core', 'fundamental'
+        ])
+        
+        words = text.lower().split()
+        if not words:
+            return 0.0
+        
+        keyword_count = sum(1 for w in words if w in important_words)
+        return keyword_count / len(words)

caas/enrichment.py

@@ -0,0 +1,130 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+"""
+Metadata enrichment module for contextual injection.
+
+Solves the "Flat Chunk Fallacy" by enriching chunks with parent metadata.
+Instead of storing isolated chunks like "It increased by 5%", we store:
+"[Document: Q3 Earnings] [Chapter: Revenue] [Section: North America] It increased by 5%."
+
+This ensures the vector carries the weight of its context.
+"""
+
+from typing import List, Optional
+from caas.models import Section, Document, DocumentType
+
+
+class MetadataEnricher:
+    """
+    Enriches sections with parent metadata for contextual awareness.
+    
+    Transforms isolated chunks into context-aware chunks by injecting
+    hierarchical metadata (document, chapter, section).
+    """
+    
+    def enrich_sections(self, document: Document) -> List[Section]:
+        """
+        Enrich all sections in a document with metadata prefixes.
+        
+        Args:
+            document: Document with sections to enrich
+            
+        Returns:
+            List of enriched sections
+        """
+        enriched_sections = []
+        
+        for section in document.sections:
+            enriched_section = self._enrich_section(section, document)
+            enriched_sections.append(enriched_section)
+        
+        return enriched_sections
+    
+    def _enrich_section(self, section: Section, document: Document) -> Section:
+        """
+        Enrich a single section with metadata prefix.
+        
+        Args:
+            section: Section to enrich
+            document: Parent document
+            
+        Returns:
+            Section with enriched content
+        """
+        # Build metadata prefix
+        metadata_parts = []
+        
+        # Add document title
+        metadata_parts.append(f"[Document: {document.title}]")
+        
+        # Add document type if meaningful
+        if document.detected_type and document.detected_type.value != "unknown":
+            doc_type_display = document.detected_type.value.replace("_", " ").title()
+            metadata_parts.append(f"[Type: {doc_type_display}]")
+        
+        # Add chapter/parent section if available
+        if section.chapter:
+            metadata_parts.append(f"[Chapter: {section.chapter}]")
+        elif section.parent_section:
+            metadata_parts.append(f"[Parent: {section.parent_section}]")
+        
+        # Add current section
+        metadata_parts.append(f"[Section: {section.title}]")
+        
+        # Build enriched content
+        metadata_prefix = " ".join(metadata_parts)
+        enriched_content = f"{metadata_prefix} {section.content}"
+        
+        # Create a new section with enriched content
+        # We preserve the original section but update the content
+        # Note: Using model_copy() from Pydantic v2 (we're on v2.5.0)
+        enriched_section = section.model_copy()
+        enriched_section.content = enriched_content
+        
+        return enriched_section
+    
+    def get_enriched_chunk(
+        self, 
+        section: Section, 
+        document_title: str,
+        document_type: Optional[DocumentType] = None,
+        include_type: bool = True
+    ) -> str:
+        """
+        Get an enriched chunk string for a section.
+        
+        Useful for building enriched context on-the-fly without modifying
+        the stored section.
+        
+        Args:
+            section: Section to enrich
+            document_title: Title of parent document
+            document_type: Type of document (optional)
+            include_type: Whether to include document type in prefix
+            
+        Returns:
+            Enriched chunk string
+        """
+        metadata_parts = []
+        
+        # Add document title
+        metadata_parts.append(f"[Document: {document_title}]")
+        
+        # Add document type if requested and available
+        if include_type and document_type and document_type.value != "unknown":
+            doc_type_display = document_type.value.replace("_", " ").title()
+            metadata_parts.append(f"[Type: {doc_type_display}]")
+        
+        # Add hierarchical context
+        if section.chapter:
+            metadata_parts.append(f"[Chapter: {section.chapter}]")
+        elif section.parent_section:
+            metadata_parts.append(f"[Parent: {section.parent_section}]")
+        
+        # Add current section
+        metadata_parts.append(f"[Section: {section.title}]")
+        
+        # Build and return enriched content
+        metadata_prefix = " ".join(metadata_parts)
+        return f"{metadata_prefix} {section.content}"

caas/gateway/__init__.py

@@ -0,0 +1,27 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+"""
+Trust Gateway: Enterprise-Grade Private Cloud Router
+
+The enterprise-ready gateway that can be deployed on-premises or in private cloud
+to address CISO concerns about data security and privacy.
+"""
+
+from caas.gateway.trust_gateway import (
+    TrustGateway,
+    DeploymentMode,
+    SecurityPolicy,
+    SecurityLevel,
+    AuditLog,
+    DataRetentionPolicy
+)
+
+__all__ = [
+    "TrustGateway",
+    "DeploymentMode",
+    "SecurityPolicy",
+    "SecurityLevel",
+    "AuditLog",
+    "DataRetentionPolicy"
+]
+