aiecs 1.0.8__py3-none-any.whl → 1.1.0__py3-none-any.whl

aiecs/tools/docs/document_parser_tool.py

@@ -0,0 +1,904 @@
+import os
+import re
+import mimetypes
+import logging
+import asyncio
+from typing import Dict, Any, List, Optional, Union, Tuple
+from enum import Enum
+from urllib.parse import urlparse
+from pathlib import Path
+import tempfile
+
+import httpx
+from pydantic import BaseModel, Field, ValidationError, ConfigDict
+from pydantic_settings import BaseSettings
+
+from aiecs.tools.base_tool import BaseTool
+from aiecs.tools import register_tool
+
+
+class DocumentType(str, Enum):
+    """Supported document types for parsing"""
+    PDF = "pdf"
+    DOCX = "docx"
+    XLSX = "xlsx"
+    PPTX = "pptx"
+    TXT = "txt"
+    HTML = "html"
+    RTF = "rtf"
+    CSV = "csv"
+    JSON = "json"
+    XML = "xml"
+    MARKDOWN = "md"
+    IMAGE = "image"
+    UNKNOWN = "unknown"
+
+
+class ParsingStrategy(str, Enum):
+    """Document parsing strategies"""
+    TEXT_ONLY = "text_only"
+    STRUCTURED = "structured"
+    FULL_CONTENT = "full_content"
+    METADATA_ONLY = "metadata_only"
+
+
+class OutputFormat(str, Enum):
+    """Output formats for parsed content"""
+    TEXT = "text"
+    JSON = "json"
+    MARKDOWN = "markdown"
+    HTML = "html"
+
+
+class DocumentParserSettings(BaseSettings):
+    """Configuration for DocumentParserTool"""
+    user_agent: str = "DocumentParser/1.0"
+    max_file_size: int = 50 * 1024 * 1024  # 50MB
+    temp_dir: str = os.path.join(tempfile.gettempdir(), 'document_parser')
+    default_encoding: str = "utf-8"
+    timeout: int = 30
+    max_pages: int = 1000  # For large PDF files
+    
+    # Cloud storage settings
+    enable_cloud_storage: bool = True
+    gcs_bucket_name: str = "aiecs-documents"
+    gcs_project_id: Optional[str] = None
+    
+    model_config = ConfigDict(env_prefix="DOC_PARSER_")
+
+
+class DocumentParserError(Exception):
+    """Base exception for document parser errors"""
+    pass
+
+
+class UnsupportedDocumentError(DocumentParserError):
+    """Raised when document type is not supported"""
+    pass
+
+
+class DownloadError(DocumentParserError):
+    """Raised when document download fails"""
+    pass
+
+
+class ParseError(DocumentParserError):
+    """Raised when document parsing fails"""
+    pass
+
+
+@register_tool("document_parser")
+class DocumentParserTool(BaseTool):
+    """
+    Modern high-performance document parsing component that can:
+    1. Auto-detect document types from URLs or files
+    2. Download documents from URLs
+    3. Parse various document formats using existing atomic tools
+    4. Output structured content for AI consumption
+    
+    Leverages existing tools:
+    - ScraperTool for URL downloading
+    - OfficeTool for Office document parsing
+    - ImageTool for image OCR
+    """
+    
+    def __init__(self, config: Optional[Dict] = None):
+        """Initialize DocumentParserTool with settings"""
+        super().__init__(config)
+        # Initialize settings with config if provided
+        if config:
+            try:
+                # For BaseSettings, use dictionary unpacking
+                self.settings = DocumentParserSettings(**config)
+            except ValidationError as e:
+                raise ValueError(f"Invalid settings: {e}")
+        else:
+            self.settings = DocumentParserSettings()
+        
+        self.logger = logging.getLogger(__name__)
+        os.makedirs(self.settings.temp_dir, exist_ok=True)
+        
+        # Initialize dependent tools
+        self._init_dependent_tools()
+        
+        # Initialize cloud storage
+        self._init_cloud_storage()
+    
+    def _init_dependent_tools(self):
+        """Initialize dependent tools for document processing"""
+        try:
+            from aiecs.tools.task_tools.scraper_tool import ScraperTool
+            self.scraper_tool = ScraperTool()
+        except ImportError:
+            self.logger.warning("ScraperTool not available")
+            self.scraper_tool = None
+            
+        try:
+            from aiecs.tools.task_tools.office_tool import OfficeTool
+            self.office_tool = OfficeTool()
+        except ImportError:
+            self.logger.warning("OfficeTool not available")
+            self.office_tool = None
+            
+        try:
+            from aiecs.tools.task_tools.image_tool import ImageTool
+            self.image_tool = ImageTool()
+        except ImportError:
+            self.logger.warning("ImageTool not available")
+            self.image_tool = None
+    
+    def _init_cloud_storage(self):
+        """Initialize cloud storage for document retrieval"""
+        self.file_storage = None
+        
+        if self.settings.enable_cloud_storage:
+            try:
+                from aiecs.infrastructure.persistence.file_storage import FileStorage
+                
+                storage_config = {
+                    'gcs_bucket_name': self.settings.gcs_bucket_name,
+                    'gcs_project_id': self.settings.gcs_project_id,
+                    'enable_local_fallback': True,
+                    'local_storage_path': self.settings.temp_dir
+                }
+                
+                self.file_storage = FileStorage(storage_config)
+                asyncio.create_task(self._init_storage_async())
+                
+            except ImportError:
+                self.logger.warning("FileStorage not available, cloud storage disabled")
+            except Exception as e:
+                self.logger.warning(f"Failed to initialize cloud storage: {e}")
+    
+    async def _init_storage_async(self):
+        """Async initialization of file storage"""
+        try:
+            if self.file_storage:
+                await self.file_storage.initialize()
+                self.logger.info("Cloud storage initialized successfully")
+        except Exception as e:
+            self.logger.warning(f"Cloud storage initialization failed: {e}")
+            self.file_storage = None
+    
+    # Schema definitions
+    class ParseDocumentSchema(BaseModel):
+        """Schema for parse_document operation"""
+        source: str = Field(description="URL or file path to the document")
+        strategy: ParsingStrategy = Field(default=ParsingStrategy.FULL_CONTENT, description="Parsing strategy")
+        output_format: OutputFormat = Field(default=OutputFormat.JSON, description="Output format")
+        force_type: Optional[DocumentType] = Field(default=None, description="Force document type detection")
+        extract_metadata: bool = Field(default=True, description="Whether to extract metadata")
+        chunk_size: Optional[int] = Field(default=None, description="Chunk size for large documents")
+    
+    class DetectTypeSchema(BaseModel):
+        """Schema for detect_document_type operation"""
+        source: str = Field(description="URL or file path to analyze")
+        download_sample: bool = Field(default=True, description="Download sample for content-based detection")
+    
+    def detect_document_type(self, source: str, download_sample: bool = True) -> Dict[str, Any]:
+        """
+        Detect document type from URL or file path
+        
+        Args:
+            source: URL or file path
+            download_sample: Whether to download sample for content analysis
+            
+        Returns:
+            Dict containing detected type and confidence
+        """
+        try:
+            result = {
+                "source": source,
+                "is_url": self._is_url(source),
+                "detected_type": DocumentType.UNKNOWN,
+                "confidence": 0.0,
+                "mime_type": None,
+                "file_extension": None,
+                "file_size": None,
+                "detection_methods": []
+            }
+            
+            # Method 1: File extension analysis
+            extension_type, ext_confidence = self._detect_by_extension(source)
+            if extension_type != DocumentType.UNKNOWN:
+                result["detected_type"] = extension_type
+                result["confidence"] = ext_confidence
+                result["file_extension"] = Path(source).suffix.lower()
+                result["detection_methods"].append("file_extension")
+            
+            # Method 2: MIME type detection (for URLs)
+            if self._is_url(source) and download_sample:
+                mime_type, mime_confidence = self._detect_by_mime_type(source)
+                if mime_type != DocumentType.UNKNOWN and mime_confidence > result["confidence"]:
+                    result["detected_type"] = mime_type
+                    result["confidence"] = mime_confidence
+                    result["detection_methods"].append("mime_type")
+            
+            # Method 3: Content-based detection
+            if download_sample:
+                content_type, content_confidence = self._detect_by_content(source)
+                if content_type != DocumentType.UNKNOWN and content_confidence > result["confidence"]:
+                    result["detected_type"] = content_type
+                    result["confidence"] = content_confidence
+                    result["detection_methods"].append("content_analysis")
+            
+            return result
+            
+        except Exception as e:
+            raise DocumentParserError(f"Document type detection failed: {str(e)}")
+    
+    def parse_document(self, 
+                      source: str, 
+                      strategy: ParsingStrategy = ParsingStrategy.FULL_CONTENT,
+                      output_format: OutputFormat = OutputFormat.JSON,
+                      force_type: Optional[DocumentType] = None,
+                      extract_metadata: bool = True,
+                      chunk_size: Optional[int] = None) -> Dict[str, Any]:
+        """
+        Parse document from URL or file path
+        
+        Args:
+            source: URL or file path to document
+            strategy: Parsing strategy to use
+            output_format: Format for output content
+            force_type: Force specific document type
+            extract_metadata: Whether to extract metadata
+            chunk_size: Chunk size for large documents
+            
+        Returns:
+            Dict containing parsed content and metadata
+        """
+        try:
+            # Step 1: Detect document type
+            if force_type:
+                doc_type = force_type
+                confidence = 1.0
+            else:
+                detection_result = self.detect_document_type(source)
+                doc_type = detection_result["detected_type"]
+                confidence = detection_result["confidence"]
+                
+                if confidence < 0.5:
+                    raise UnsupportedDocumentError(f"Unable to reliably detect document type for: {source}")
+            
+            # Step 2: Download document if it's a URL
+            local_path = self._ensure_local_file(source)
+            
+            # Step 3: Parse document based on type and strategy
+            content = self._parse_by_type(local_path, doc_type, strategy)
+            
+            # Step 4: Extract metadata if requested
+            metadata = {}
+            if extract_metadata:
+                metadata = self._extract_metadata(local_path, doc_type)
+            
+            # Step 5: Format output
+            result = {
+                "source": source,
+                "document_type": doc_type,
+                "detection_confidence": confidence,
+                "parsing_strategy": strategy,
+                "metadata": metadata,
+                "content": content,
+                "content_stats": self._calculate_content_stats(content),
+                "chunks": []
+            }
+            
+            # Step 6: Create chunks if requested
+            if chunk_size and isinstance(content, str):
+                result["chunks"] = self._create_chunks(content, chunk_size)
+            
+            # Step 7: Format output according to requested format
+            if output_format == OutputFormat.TEXT:
+                return {"text": self._format_as_text(result)}
+            elif output_format == OutputFormat.MARKDOWN:
+                return {"markdown": self._format_as_markdown(result)}
+            elif output_format == OutputFormat.HTML:
+                return {"html": self._format_as_html(result)}
+            else:
+                return result
+                
+        except Exception as e:
+            if isinstance(e, DocumentParserError):
+                raise
+            raise ParseError(f"Document parsing failed: {str(e)}")
+        finally:
+            # Cleanup temporary files
+            self._cleanup_temp_files(source)
+    
+    async def parse_document_async(self,
+                                    source: str,
+                                    strategy: ParsingStrategy = ParsingStrategy.FULL_CONTENT,
+                                    output_format: OutputFormat = OutputFormat.JSON,
+                                    force_type: Optional[DocumentType] = None,
+                                    extract_metadata: bool = True,
+                                    chunk_size: Optional[int] = None) -> Dict[str, Any]:
+        """Async version of parse_document"""
+        return await asyncio.to_thread(
+            self.parse_document,
+            source=source,
+            strategy=strategy,
+            output_format=output_format,
+            force_type=force_type,
+            extract_metadata=extract_metadata,
+            chunk_size=chunk_size
+        )
+    
+    def _is_url(self, source: str) -> bool:
+        """Check if source is a URL"""
+        try:
+            result = urlparse(source)
+            return bool(result.scheme and result.netloc)
+        except:
+            return False
+    
+    def _is_cloud_storage_path(self, source: str) -> bool:
+        """Check if source is a cloud storage path"""
+        # Support various cloud storage path formats:
+        # - gs://bucket/path/file.pdf (Google Cloud Storage)
+        # - s3://bucket/path/file.pdf (AWS S3)
+        # - azure://container/path/file.pdf (Azure Blob Storage)
+        # - cloud://path/file.pdf (Generic cloud storage)
+        cloud_schemes = ['gs', 's3', 'azure', 'cloud']
+        try:
+            parsed = urlparse(source)
+            return parsed.scheme in cloud_schemes
+        except:
+            return False
+    
+    def _is_storage_id(self, source: str) -> bool:
+        """Check if source is a storage ID (UUID-like identifier)"""
+        # Check for UUID patterns or other storage ID formats
+        import re
+        uuid_pattern = r'^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$'
+        storage_id_pattern = r'^[a-zA-Z0-9_-]{10,}$'  # Generic storage ID
+        
+        return bool(re.match(uuid_pattern, source, re.IGNORECASE) or 
+                   re.match(storage_id_pattern, source))
+    
+    def _detect_by_extension(self, source: str) -> Tuple[DocumentType, float]:
+        """Detect document type by file extension"""
+        try:
+            path = Path(source)
+            ext = path.suffix.lower()
+            
+            extension_map = {
+                '.pdf': DocumentType.PDF,
+                '.docx': DocumentType.DOCX,
+                '.doc': DocumentType.DOCX,
+                '.xlsx': DocumentType.XLSX,
+                '.xls': DocumentType.XLSX,
+                '.pptx': DocumentType.PPTX,
+                '.ppt': DocumentType.PPTX,
+                '.txt': DocumentType.TXT,
+                '.html': DocumentType.HTML,
+                '.htm': DocumentType.HTML,
+                '.rtf': DocumentType.RTF,
+                '.csv': DocumentType.CSV,
+                '.json': DocumentType.JSON,
+                '.xml': DocumentType.XML,
+                '.md': DocumentType.MARKDOWN,
+                '.markdown': DocumentType.MARKDOWN,
+                '.jpg': DocumentType.IMAGE,
+                '.jpeg': DocumentType.IMAGE,
+                '.png': DocumentType.IMAGE,
+                '.gif': DocumentType.IMAGE,
+                '.bmp': DocumentType.IMAGE,
+                '.tiff': DocumentType.IMAGE,
+            }
+            
+            doc_type = extension_map.get(ext, DocumentType.UNKNOWN)
+            confidence = 0.8 if doc_type != DocumentType.UNKNOWN else 0.0
+            
+            return doc_type, confidence
+            
+        except Exception:
+            return DocumentType.UNKNOWN, 0.0
+    
+    def _detect_by_mime_type(self, url: str) -> Tuple[DocumentType, float]:
+        """Detect document type by MIME type from URL"""
+        try:
+            if not self.scraper_tool:
+                return DocumentType.UNKNOWN, 0.0
+                
+            # Get headers only
+            response = asyncio.run(self.scraper_tool.get_httpx(
+                url, method="HEAD", verify_ssl=False
+            ))
+            
+            content_type = response.get('headers', {}).get('content-type', '').lower()
+            
+            mime_map = {
+                'application/pdf': DocumentType.PDF,
+                'application/vnd.openxmlformats-officedocument.wordprocessingml.document': DocumentType.DOCX,
+                'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': DocumentType.XLSX,
+                'application/vnd.openxmlformats-officedocument.presentationml.presentation': DocumentType.PPTX,
+                'text/plain': DocumentType.TXT,
+                'text/html': DocumentType.HTML,
+                'application/rtf': DocumentType.RTF,
+                'text/csv': DocumentType.CSV,
+                'application/json': DocumentType.JSON,
+                'application/xml': DocumentType.XML,
+                'text/xml': DocumentType.XML,
+                'text/markdown': DocumentType.MARKDOWN,
+                'image/jpeg': DocumentType.IMAGE,
+                'image/png': DocumentType.IMAGE,
+                'image/gif': DocumentType.IMAGE,
+                'image/bmp': DocumentType.IMAGE,
+                'image/tiff': DocumentType.IMAGE,
+            }
+            
+            for mime_pattern, doc_type in mime_map.items():
+                if mime_pattern in content_type:
+                    return doc_type, 0.9
+                    
+            return DocumentType.UNKNOWN, 0.0
+            
+        except Exception:
+            return DocumentType.UNKNOWN, 0.0
+    
+    def _detect_by_content(self, source: str) -> Tuple[DocumentType, float]:
+        """Detect document type by content analysis"""
+        try:
+            # Download a small sample for analysis
+            if self._is_url(source):
+                sample_path = self._download_sample(source, max_size=1024)  # 1KB sample
+            else:
+                sample_path = source
+            
+            with open(sample_path, 'rb') as f:
+                header = f.read(512)  # Read first 512 bytes
+            
+            # Magic number detection
+            if header.startswith(b'%PDF'):
+                return DocumentType.PDF, 0.95
+            elif header.startswith(b'PK\x03\x04'):  # ZIP-based formats
+                if b'word/' in header or b'document.xml' in header:
+                    return DocumentType.DOCX, 0.9
+                elif b'xl/' in header or b'workbook.xml' in header:
+                    return DocumentType.XLSX, 0.9
+                elif b'ppt/' in header or b'presentation.xml' in header:
+                    return DocumentType.PPTX, 0.9
+            elif header.startswith(b'{\rtf'):
+                return DocumentType.RTF, 0.95
+            elif header.startswith((b'\xff\xd8\xff', b'\x89PNG', b'GIF8')):
+                return DocumentType.IMAGE, 0.95
+            elif header.startswith(b'<?xml'):
+                return DocumentType.XML, 0.9
+            elif header.startswith((b'{', b'[')):
+                # Try to parse as JSON
+                try:
+                    import json
+                    json.loads(header.decode('utf-8', errors='ignore'))
+                    return DocumentType.JSON, 0.85
+                except:
+                    pass
+            
+            # Text-based detection
+            try:
+                text_content = header.decode('utf-8', errors='ignore')
+                if re.match(r'^#\s+.*$', text_content, re.MULTILINE):
+                    return DocumentType.MARKDOWN, 0.7
+                elif '<html' in text_content.lower() or '<!doctype html' in text_content.lower():
+                    return DocumentType.HTML, 0.85
+                elif ',' in text_content and '\n' in text_content:
+                    # Simple CSV detection
+                    lines = text_content.split('\n')[:5]
+                    if all(',' in line for line in lines if line.strip()):
+                        return DocumentType.CSV, 0.6
+            except:
+                pass
+                
+            return DocumentType.UNKNOWN, 0.0
+            
+        except Exception:
+            return DocumentType.UNKNOWN, 0.0
+    
+    def _ensure_local_file(self, source: str) -> str:
+        """Ensure we have a local file, download/retrieve if necessary"""
+        # Check source type and handle accordingly
+        if self._is_cloud_storage_path(source) or self._is_storage_id(source):
+            # Download from cloud storage
+            return asyncio.run(self._download_from_cloud_storage(source))
+        elif self._is_url(source):
+            # Download from URL
+            return self._download_document(source)
+        else:
+            # Local file path
+            if not os.path.exists(source):
+                raise FileNotFoundError(f"File not found: {source}")
+            return source
+    
+    def _download_document(self, url: str) -> str:
+        """Download document from URL"""
+        try:
+            if not self.scraper_tool:
+                raise DownloadError("ScraperTool not available for URL download")
+            
+            # Generate temp file path
+            parsed_url = urlparse(url)
+            filename = os.path.basename(parsed_url.path) or "document"
+            temp_path = os.path.join(self.settings.temp_dir, f"download_{hash(url)}_{filename}")
+            
+            # Download using scraper tool
+            result = asyncio.run(self.scraper_tool.get_httpx(
+                url, 
+                content_type="binary",
+                output_path=temp_path,
+                verify_ssl=False
+            ))
+            
+            if isinstance(result, dict) and 'saved_to' in result:
+                return result['saved_to']
+            else:
+                # Fallback: save content manually
+                with open(temp_path, 'wb') as f:
+                    if isinstance(result, dict) and 'content' in result:
+                        f.write(result['content'])
+                    else:
+                        f.write(result)
+                return temp_path
+                
+        except Exception as e:
+            raise DownloadError(f"Failed to download document from {url}: {str(e)}")
+    
+    async def _download_from_cloud_storage(self, source: str) -> str:
+        """Download document from cloud storage"""
+        if not self.file_storage:
+            raise DownloadError("Cloud storage not available")
+        
+        try:
+            # Parse the cloud storage path
+            storage_path = self._parse_cloud_storage_path(source)
+            
+            # Generate local temp file path
+            temp_filename = f"cloud_download_{hash(source)}_{Path(storage_path).name}"
+            temp_path = os.path.join(self.settings.temp_dir, temp_filename)
+            
+            self.logger.info(f"Downloading from cloud storage: {source} -> {temp_path}")
+            
+            # Retrieve file from cloud storage
+            file_data = await self.file_storage.retrieve(storage_path)
+            
+            # Save to local temp file
+            if isinstance(file_data, bytes):
+                with open(temp_path, 'wb') as f:
+                    f.write(file_data)
+            elif isinstance(file_data, str):
+                with open(temp_path, 'w', encoding='utf-8') as f:
+                    f.write(file_data)
+            else:
+                # Handle other data types (e.g., dict, list)
+                import json
+                with open(temp_path, 'w', encoding='utf-8') as f:
+                    json.dump(file_data, f)
+            
+            self.logger.info(f"Successfully downloaded file to: {temp_path}")
+            return temp_path
+            
+        except Exception as e:
+            raise DownloadError(f"Failed to download from cloud storage {source}: {str(e)}")
+    
+    def _parse_cloud_storage_path(self, source: str) -> str:
+        """Parse cloud storage path to get the storage key"""
+        try:
+            if self._is_storage_id(source):
+                # Direct storage ID
+                return source
+            elif self._is_cloud_storage_path(source):
+                parsed = urlparse(source)
+                if parsed.scheme == 'gs':
+                    # Google Cloud Storage: gs://bucket/path/file.pdf -> path/file.pdf
+                    return parsed.path.lstrip('/')
+                elif parsed.scheme == 's3':
+                    # AWS S3: s3://bucket/path/file.pdf -> path/file.pdf  
+                    return parsed.path.lstrip('/')
+                elif parsed.scheme == 'azure':
+                    # Azure Blob: azure://container/path/file.pdf -> path/file.pdf
+                    return parsed.path.lstrip('/')
+                elif parsed.scheme == 'cloud':
+                    # Generic cloud: cloud://path/file.pdf -> path/file.pdf
+                    return parsed.path.lstrip('/')
+                else:
+                    return parsed.path.lstrip('/')
+            else:
+                # Assume it's already a storage path
+                return source
+        except Exception as e:
+            self.logger.warning(f"Failed to parse cloud storage path {source}: {e}")
+            return source
+    
+    def _download_sample(self, url: str, max_size: int = 1024) -> str:
+        """Download a small sample of the document for analysis"""
+        # This is a simplified version - in practice, you'd implement range requests
+        return self._download_document(url)
+    
+    def _parse_by_type(self, file_path: str, doc_type: DocumentType, strategy: ParsingStrategy) -> Union[str, Dict[str, Any]]:
+        """Parse document based on its type and strategy"""
+        try:
+            if doc_type == DocumentType.PDF:
+                return self._parse_pdf(file_path, strategy)
+            elif doc_type in [DocumentType.DOCX, DocumentType.XLSX, DocumentType.PPTX]:
+                return self._parse_office_document(file_path, doc_type, strategy)
+            elif doc_type == DocumentType.IMAGE:
+                return self._parse_image(file_path, strategy)
+            elif doc_type in [DocumentType.TXT, DocumentType.HTML, DocumentType.CSV, 
+                             DocumentType.JSON, DocumentType.XML, DocumentType.MARKDOWN]:
+                return self._parse_text_document(file_path, doc_type, strategy)
+            else:
+                raise UnsupportedDocumentError(f"Unsupported document type: {doc_type}")
+                
+        except Exception as e:
+            raise ParseError(f"Failed to parse {doc_type} document: {str(e)}")
+    
+    def _parse_pdf(self, file_path: str, strategy: ParsingStrategy) -> Union[str, Dict[str, Any]]:
+        """Parse PDF document"""
+        if self.office_tool:
+            try:
+                text_content = self.office_tool.extract_text(file_path)
+                
+                if strategy == ParsingStrategy.TEXT_ONLY:
+                    return text_content
+                elif strategy == ParsingStrategy.STRUCTURED:
+                    # Try to extract structure from PDF
+                    return {
+                        "text": text_content,
+                        "structure": self._extract_pdf_structure(text_content)
+                    }
+                else:
+                    return {
+                        "text": text_content,
+                        "pages": self._split_into_pages(text_content)
+                    }
+            except Exception as e:
+                self.logger.warning(f"OfficeTool PDF parsing failed: {e}")
+        
+        # Fallback to simple text extraction
+        return self._extract_text_fallback(file_path)
+    
+    def _parse_office_document(self, file_path: str, doc_type: DocumentType, strategy: ParsingStrategy) -> Union[str, Dict[str, Any]]:
+        """Parse Office documents (DOCX, XLSX, PPTX)"""
+        if not self.office_tool:
+            raise UnsupportedDocumentError("OfficeTool not available for Office document parsing")
+        
+        try:
+            text_content = self.office_tool.extract_text(file_path)
+            
+            if strategy == ParsingStrategy.TEXT_ONLY:
+                return text_content
+            elif strategy == ParsingStrategy.STRUCTURED:
+                return {
+                    "text": text_content,
+                    "structure": self._extract_office_structure(file_path, doc_type)
+                }
+            else:
+                return {
+                    "text": text_content,
+                    "raw_content": text_content
+                }
+                
+        except Exception as e:
+            raise ParseError(f"Failed to parse Office document: {str(e)}")
+    
+    def _parse_image(self, file_path: str, strategy: ParsingStrategy) -> Union[str, Dict[str, Any]]:
+        """Parse image document using OCR"""
+        if not self.image_tool:
+            raise UnsupportedDocumentError("ImageTool not available for image OCR")
+        
+        try:
+            # Use image tool for OCR
+            ocr_result = self.image_tool.ocr_image(file_path)
+            
+            if strategy == ParsingStrategy.TEXT_ONLY:
+                return ocr_result.get('text', '')
+            else:
+                return ocr_result
+                
+        except Exception as e:
+            raise ParseError(f"Failed to parse image document: {str(e)}")
+    
+    def _parse_text_document(self, file_path: str, doc_type: DocumentType, strategy: ParsingStrategy) -> Union[str, Dict[str, Any]]:
+        """Parse text-based documents"""
+        try:
+            with open(file_path, 'r', encoding=self.settings.default_encoding, errors='ignore') as f:
+                content = f.read()
+            
+            if strategy == ParsingStrategy.TEXT_ONLY:
+                return content
+            elif strategy == ParsingStrategy.STRUCTURED:
+                return self._extract_text_structure(content, doc_type)
+            else:
+                return {
+                    "text": content,
+                    "lines": content.split('\n'),
+                    "word_count": len(content.split())
+                }
+                
+        except Exception as e:
+            raise ParseError(f"Failed to parse text document: {str(e)}")
+    
+    def _extract_metadata(self, file_path: str, doc_type: DocumentType) -> Dict[str, Any]:
+        """Extract metadata from document"""
+        metadata = {
+            "file_path": file_path,
+            "file_size": os.path.getsize(file_path),
+            "file_type": doc_type.value,
+            "created_at": os.path.getctime(file_path),
+            "modified_at": os.path.getmtime(file_path)
+        }
+        
+        # Add type-specific metadata extraction here
+        # This could leverage existing tools' metadata extraction capabilities
+        
+        return metadata
+    
+    def _calculate_content_stats(self, content: Union[str, Dict[str, Any]]) -> Dict[str, Any]:
+        """Calculate statistics about the parsed content"""
+        if isinstance(content, str):
+            return {
+                "character_count": len(content),
+                "word_count": len(content.split()),
+                "line_count": len(content.split('\n')),
+                "paragraph_count": len([p for p in content.split('\n\n') if p.strip()])
+            }
+        else:
+            # For structured content, calculate stats on text portion
+            text_content = content.get('text', '')
+            return self._calculate_content_stats(text_content)
+    
+    def _create_chunks(self, content: str, chunk_size: int) -> List[Dict[str, Any]]:
+        """Create chunks from content for better AI processing"""
+        chunks = []
+        words = content.split()
+        
+        for i in range(0, len(words), chunk_size):
+            chunk_words = words[i:i + chunk_size]
+            chunk_text = ' '.join(chunk_words)
+            
+            chunks.append({
+                "index": len(chunks),
+                "text": chunk_text,
+                "word_count": len(chunk_words),
+                "start_word": i,
+                "end_word": min(i + chunk_size, len(words))
+            })
+        
+        return chunks
+    
+    def _format_as_text(self, result: Dict[str, Any]) -> str:
+        """Format result as plain text"""
+        content = result.get('content', '')
+        if isinstance(content, dict):
+            return content.get('text', str(content))
+        return str(content)
+    
+    def _format_as_markdown(self, result: Dict[str, Any]) -> str:
+        """Format result as Markdown"""
+        content = result.get('content', '')
+        metadata = result.get('metadata', {})
+        
+        md_content = f"# Document: {result.get('source', 'Unknown')}\n\n"
+        md_content += f"**Type:** {result.get('document_type', 'Unknown')}\n"
+        md_content += f"**Detection Confidence:** {result.get('detection_confidence', 0):.2f}\n\n"
+        
+        if isinstance(content, dict):
+            md_content += content.get('text', str(content))
+        else:
+            md_content += str(content)
+        
+        return md_content
+    
+    def _format_as_html(self, result: Dict[str, Any]) -> str:
+        """Format result as HTML"""
+        content = result.get('content', '')
+        
+        html_content = f"""
+        <html>
+        <head><title>Parsed Document</title></head>
+        <body>
+        <h1>Document: {result.get('source', 'Unknown')}</h1>
+        <p><strong>Type:</strong> {result.get('document_type', 'Unknown')}</p>
+        <p><strong>Detection Confidence:</strong> {result.get('detection_confidence', 0):.2f}</p>
+        <div class="content">
+        """
+        
+        if isinstance(content, dict):
+            html_content += f"<pre>{content.get('text', str(content))}</pre>"
+        else:
+            html_content += f"<pre>{str(content)}</pre>"
+        
+        html_content += "</div></body></html>"
+        return html_content
+    
+    def _cleanup_temp_files(self, source: str):
+        """Clean up temporary files"""
+        import glob
+        
+        if self._is_url(source):
+            # Clean up URL downloaded files
+            temp_pattern = os.path.join(self.settings.temp_dir, f"download_{hash(source)}_*")
+            for temp_file in glob.glob(temp_pattern):
+                try:
+                    os.remove(temp_file)
+                    self.logger.debug(f"Cleaned up temp file: {temp_file}")
+                except Exception as e:
+                    self.logger.warning(f"Failed to clean up temp file {temp_file}: {e}")
+        
+        elif self._is_cloud_storage_path(source) or self._is_storage_id(source):
+            # Clean up cloud storage downloaded files
+            temp_pattern = os.path.join(self.settings.temp_dir, f"cloud_download_{hash(source)}_*")
+            for temp_file in glob.glob(temp_pattern):
+                try:
+                    os.remove(temp_file)
+                    self.logger.debug(f"Cleaned up cloud temp file: {temp_file}")
+                except Exception as e:
+                    self.logger.warning(f"Failed to clean up cloud temp file {temp_file}: {e}")
+    
+    # Helper methods for structure extraction
+    def _extract_pdf_structure(self, text: str) -> Dict[str, Any]:
+        """Extract structure from PDF text"""
+        # Implement PDF structure extraction logic
+        return {"sections": [], "headings": []}
+    
+    def _extract_office_structure(self, file_path: str, doc_type: DocumentType) -> Dict[str, Any]:
+        """Extract structure from Office documents"""
+        # Implement Office document structure extraction
+        return {"sections": [], "tables": [], "images": []}
+    
+    def _extract_text_structure(self, content: str, doc_type: DocumentType) -> Dict[str, Any]:
+        """Extract structure from text documents"""
+        result = {"text": content}
+        
+        if doc_type == DocumentType.MARKDOWN:
+            # Extract markdown structure
+            headings = re.findall(r'^(#{1,6})\s+(.+)$', content, re.MULTILINE)
+            result["headings"] = [{"level": len(h[0]), "text": h[1]} for h in headings]
+        elif doc_type == DocumentType.HTML:
+            # Extract HTML structure (simplified)
+            from bs4 import BeautifulSoup
+            soup = BeautifulSoup(content, 'html.parser')
+            result["title"] = soup.title.string if soup.title else ""
+            result["headings"] = [{"tag": h.name, "text": h.get_text()} for h in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])]
+        elif doc_type == DocumentType.JSON:
+            import json
+            try:
+                result["json_data"] = json.loads(content)
+            except:
+                pass
+        
+        return result
+    
+    def _split_into_pages(self, text: str) -> List[str]:
+        """Split text into pages (simplified)"""
+        # This is a simple implementation - could be enhanced
+        pages = text.split('\f')  # Form feed character often indicates page break
+        return [page.strip() for page in pages if page.strip()]
+    
+    def _extract_text_fallback(self, file_path: str) -> str:
+        """Fallback text extraction method"""
+        try:
+            with open(file_path, 'r', encoding=self.settings.default_encoding, errors='ignore') as f:
+                return f.read()
+        except:
+            with open(file_path, 'rb') as f:
+                return f.read().decode('utf-8', errors='ignore')