realtimex-deeptutor 0.5.0.post1__py3-none-any.whl

src/tools/tex_downloader.py

@@ -0,0 +1,253 @@
+# -*- coding: utf-8 -*-
+"""
+TeX Downloader - LaTeX source code download tool
+
+Features:
+1. Download LaTeX source from ArXiv
+2. Extract and locate main tex file
+3. Read tex content
+
+Author: DeepTutor Team
+Version: v1.0
+Based on: TODO.md specification
+"""
+
+import os
+from pathlib import Path
+import re
+import shutil
+import tarfile
+import tempfile
+import zipfile
+
+import requests
+
+
+class TexDownloadResult:
+    """LaTeX download result"""
+
+    def __init__(
+        self,
+        success: bool,
+        tex_path: str | None = None,
+        tex_content: str | None = None,
+        error: str | None = None,
+    ):
+        self.success = success
+        self.tex_path = tex_path
+        self.tex_content = tex_content
+        self.error = error
+
+
+class TexDownloader:
+    """LaTeX source code download tool"""
+
+    def __init__(self, workspace_dir: str):
+        """
+        Initialize downloader
+
+        Args:
+            workspace_dir: Workspace directory (for saving downloaded files)
+        """
+        self.workspace_dir = Path(workspace_dir)
+        self.workspace_dir.mkdir(parents=True, exist_ok=True)
+
+    def download_arxiv_source(
+        self, arxiv_url: str, arxiv_id: str | None = None
+    ) -> TexDownloadResult:
+        """
+        Download LaTeX source from ArXiv
+
+        Args:
+            arxiv_url: ArXiv paper URL
+            arxiv_id: ArXiv ID (optional, if not in URL)
+
+        Returns:
+            TexDownloadResult object
+        """
+        # Extract ArXiv ID
+        if not arxiv_id:
+            arxiv_id = self._extract_arxiv_id(arxiv_url)
+
+        if not arxiv_id:
+            return TexDownloadResult(success=False, error="Unable to extract ArXiv ID")
+
+        try:
+            # Build source download URL
+            source_url = f"https://arxiv.org/e-print/{arxiv_id}"
+
+            # Download source package
+            print(f"  Downloading source: {source_url}")
+            response = requests.get(source_url, timeout=30)
+            response.raise_for_status()
+
+            # Create temporary directory
+            temp_dir = tempfile.mkdtemp(dir=self.workspace_dir)
+
+            # Save source package
+            source_file = Path(temp_dir) / f"{arxiv_id}_source"
+            with open(source_file, "wb") as f:
+                f.write(response.content)
+
+            # Extract source package
+            extract_dir = Path(temp_dir) / "extracted"
+            extract_dir.mkdir(exist_ok=True)
+
+            if self._is_tar_file(source_file):
+                self._extract_tar(source_file, extract_dir)
+            elif self._is_zip_file(source_file):
+                self._extract_zip(source_file, extract_dir)
+            else:
+                # Might be a single tex file
+                shutil.copy(source_file, extract_dir / f"{arxiv_id}.tex")
+
+            # Find main tex file
+            main_tex = self._find_main_tex(extract_dir)
+
+            if not main_tex:
+                return TexDownloadResult(success=False, error="Main tex file not found")
+
+            # Read tex content
+            tex_content = self._read_tex_file(main_tex)
+
+            # Move to permanent location
+            paper_dir = self.workspace_dir / f"paper_{arxiv_id}"
+            paper_dir.mkdir(exist_ok=True)
+
+            final_tex_path = paper_dir / "main.tex"
+            shutil.copy(main_tex, final_tex_path)
+
+            # Clean up temporary directory
+            shutil.rmtree(temp_dir, ignore_errors=True)
+
+            return TexDownloadResult(
+                success=True, tex_path=str(final_tex_path), tex_content=tex_content
+            )
+
+        except requests.exceptions.RequestException as e:
+            return TexDownloadResult(success=False, error=f"Download failed: {e!s}")
+        except Exception as e:
+            return TexDownloadResult(success=False, error=f"Processing failed: {e!s}")
+
+    def _extract_arxiv_id(self, url: str) -> str | None:
+        """Extract ArXiv ID from URL"""
+        match = re.search(r"arxiv\.org/(?:abs|pdf)/(\d+\.\d+)", url)
+        if match:
+            return match.group(1)
+        return None
+
+    def _is_tar_file(self, file_path: Path) -> bool:
+        """Check if file is a tar file"""
+        try:
+            with tarfile.open(file_path, "r:*") as tar:
+                return True
+        except:
+            return False
+
+    def _is_zip_file(self, file_path: Path) -> bool:
+        """Check if file is a zip file"""
+        try:
+            with zipfile.ZipFile(file_path, "r") as zip_file:
+                return True
+        except:
+            return False
+
+    def _extract_tar(self, tar_path: Path, extract_dir: Path):
+        """Extract tar file safely (prevent ZipSlip/TarSlip)"""
+        with tarfile.open(tar_path, "r:*") as tar:
+            # Safe extraction filter
+            def is_within_directory(directory, target):
+                abs_directory = os.path.abspath(directory)
+                abs_target = os.path.abspath(target)
+                prefix = os.path.commonprefix([abs_directory, abs_target])
+                return prefix == abs_directory
+
+            def safe_members(members):
+                for member in members:
+                    member_path = os.path.join(extract_dir, member.name)
+                    if not is_within_directory(extract_dir, member_path):
+                        print(f"Suspicious file path in tar: {member.name}. Skipping.")
+                        continue
+                    yield member
+
+            tar.extractall(extract_dir, members=safe_members(tar))
+
+    def _extract_zip(self, zip_path: Path, extract_dir: Path):
+        """Extract zip file"""
+        with zipfile.ZipFile(zip_path, "r") as zip_file:
+            zip_file.extractall(extract_dir)
+
+    def _find_main_tex(self, directory: Path) -> Path | None:
+        """
+        Find main tex file
+
+        Priority:
+        1. main.tex
+        2. paper.tex
+        3. Tex file containing \\documentclass
+        4. Largest tex file
+        """
+        tex_files = list(directory.rglob("*.tex"))
+
+        if not tex_files:
+            return None
+
+        # 1. Find main.tex or paper.tex
+        for name in ["main.tex", "paper.tex", "manuscript.tex"]:
+            for tex_file in tex_files:
+                if tex_file.name.lower() == name:
+                    return tex_file
+
+        # 2. Find file containing \documentclass
+        for tex_file in tex_files:
+            try:
+                content = tex_file.read_text(encoding="utf-8", errors="ignore")
+                if r"\documentclass" in content:
+                    return tex_file
+            except:
+                continue
+
+        # 3. Return largest tex file
+        largest_tex = max(tex_files, key=lambda f: f.stat().st_size)
+        return largest_tex
+
+    def _read_tex_file(self, tex_path: Path) -> str:
+        """Read tex file content"""
+        try:
+            return tex_path.read_text(encoding="utf-8", errors="ignore")
+        except Exception as e:
+            raise Exception(f"Failed to read tex file: {e!s}")
+
+
+def read_tex_file(tex_path: str) -> str:
+    """
+    Read tex file content (convenience function)
+
+    Args:
+        tex_path: tex file path
+
+    Returns:
+        tex content
+    """
+    return Path(tex_path).read_text(encoding="utf-8", errors="ignore")
+
+
+# ========== Usage Example ==========
+
+if __name__ == "__main__":
+    # Test download
+    downloader = TexDownloader(workspace_dir="./test_workspace")
+
+    # Test an ArXiv paper
+    result = downloader.download_arxiv_source(
+        arxiv_url="https://arxiv.org/abs/1706.03762",  # Attention is All You Need
+        arxiv_id="1706.03762",
+    )
+
+    if result.success:
+        print("✓ Download successful!")
+        print(f"  File path: {result.tex_path}")
+        print(f"  Content length: {len(result.tex_content)} characters")
+        print(f"  Content preview: {result.tex_content[:500]}...")
+    else:
+        print(f"✗ Download failed: {result.error}")

src/tools/web_search.py

@@ -0,0 +1,71 @@
+"""
+Web Search Tool - Simple entry point for agents
+
+This module provides a simple interface to the web search service.
+All search logic is implemented in src/services/search/.
+
+Usage:
+    from src.tools.web_search import web_search
+
+    # Simple usage
+    result = web_search("What is AI?")
+
+    # With provider
+    result = web_search("What is AI?", provider="tavily")
+
+Environment Variables:
+    - SEARCH_PROVIDER: Default search provider (default: perplexity)
+    - SEARCH_API_KEY: Unified API key for all providers
+
+Available Providers:
+    - perplexity: AI-powered search (default)
+    - baidu: Baidu AI Search
+    - tavily: Research-focused with optional answers
+    - exa: Neural/embeddings search with summaries
+    - serper: Google SERP results
+    - jina: SERP with full content extraction
+"""
+
+# Re-export from services layer
+from src.services.search import (
+    CONSOLIDATION_TYPES,
+    PROVIDER_TEMPLATES,
+    SEARCH_API_KEY_ENV,
+    AnswerConsolidator,
+    BaseSearchProvider,
+    Citation,
+    SearchProvider,
+    SearchResult,
+    WebSearchResponse,
+    get_available_providers,
+    get_current_config,
+    get_default_provider,
+    get_provider,
+    get_providers_info,
+    list_providers,
+    web_search,
+)
+
+__all__ = [
+    # Main function
+    "web_search",
+    "get_current_config",
+    # Provider management
+    "get_provider",
+    "list_providers",
+    "get_available_providers",
+    "get_default_provider",
+    "get_providers_info",
+    # Types
+    "WebSearchResponse",
+    "Citation",
+    "SearchResult",
+    # Consolidation
+    "AnswerConsolidator",
+    "CONSOLIDATION_TYPES",
+    "PROVIDER_TEMPLATES",
+    # Base class
+    "BaseSearchProvider",
+    "SearchProvider",
+    "SEARCH_API_KEY_ENV",
+]

src/utils/config_manager.py

@@ -0,0 +1,206 @@
+import logging
+import os
+from pathlib import Path
+import tempfile
+from threading import Lock
+from typing import Any, Dict, List, Optional
+
+from dotenv import dotenv_values, load_dotenv
+from pydantic import ValidationError
+import yaml
+
+from ..config.defaults import DEFAULTS
+
+# Use package-relative imports to avoid PYTHONPATH issues
+from ..config.schema import AppConfig, migrate_config
+from ..core.errors import ConfigError
+
+logger = logging.getLogger(__name__)
+
+
+class ConfigManager:
+    """
+    Thread-safe manager for reading and writing configuration files.
+    Primarily manages config/main.yaml and reads .env.
+
+    Governance additions:
+    - Schema validation via pydantic (AppConfig); invalid configs are rejected.
+    - Versioned migrations via migrate_config.
+    - Atomic writes with temp file and os.replace; creates main.yaml.bak.
+    - Single lock guards mtime read, load, and save.
+    - Deterministic YAML dumps; returns deep copies.
+    - Layered env: .env, then .env.local (override), then process env.
+    """
+
+    _instance: Optional["ConfigManager"] = None
+    _lock = Lock()
+
+    def __new__(cls, project_root: Optional[Path] = None):
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super(ConfigManager, cls).__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+
+    def __init__(self, project_root: Optional[Path] = None):
+        if getattr(self, "_initialized", False):
+            return
+
+        self.project_root = project_root or Path(__file__).parent.parent.parent
+        self.config_path = self.project_root / "config" / "main.yaml"
+        self._config_cache: Dict[str, Any] = {}
+        self._last_mtime: float = 0.0
+        self._initialized = True
+
+        # Layered env loading
+        load_dotenv(dotenv_path=self.project_root / ".env", override=False)
+        load_dotenv(dotenv_path=self.project_root / ".env.local", override=True)
+
+    def _load_env_file(self, path: Path) -> Dict[str, str]:
+        """Load a .env file and return non-None values as strings."""
+        if not path.exists():
+            return {}
+        return {k: str(v) for k, v in dotenv_values(path).items() if v is not None}
+
+    def _read_yaml(self) -> Dict[str, Any]:
+        """Read the main YAML configuration file safely."""
+        if not self.config_path.exists():
+            return {}
+        with open(self.config_path, "r", encoding="utf-8") as f:
+            return yaml.safe_load(f) or {}
+
+    def _deep_update(self, target: Dict[str, Any], source: Dict[str, Any]) -> None:
+        for key, value in source.items():
+            if isinstance(value, dict) and isinstance(target.get(key), dict):
+                self._deep_update(target[key], value)
+            else:
+                target[key] = value
+
+    def _validate_and_migrate(self, raw: Dict[str, Any]) -> Dict[str, Any]:
+        merged: Dict[str, Any] = {}
+        self._deep_update(merged, DEFAULTS)
+        self._deep_update(merged, raw)
+        migrated = migrate_config(merged)
+        try:
+            return AppConfig(**migrated).dict()
+        except ValidationError as e:
+            raise ConfigError("Config validation failed", details={"errors": e.errors()})
+
+    def load_config(self, force_reload: bool = False) -> Dict[str, Any]:
+        """
+        Load configuration from main.yaml.
+        Uses caching based on file modification time and validates against schema.
+        """
+        with self._lock:
+            if not self.config_path.exists():
+                logger.info("Config not found at %s", self.config_path)
+                self._config_cache = {}
+                self._last_mtime = 0
+                return {}
+
+            current_mtime = self.config_path.stat().st_mtime
+            if not self._config_cache or force_reload or current_mtime > self._last_mtime:
+                try:
+                    raw = self._read_yaml()
+                    validated = self._validate_and_migrate(raw)
+                    self._config_cache = validated
+                    self._last_mtime = current_mtime
+                except ConfigError as ce:
+                    logger.error("%s", ce, extra={"context": getattr(ce, "context", {})})
+                    return {}
+                except Exception as e:
+                    logger.exception("Error loading config: %s", e)
+                    return {}
+
+            # deep copy via dump/load for immutability
+            return yaml.safe_load(yaml.safe_dump(self._config_cache, sort_keys=False)) or {}
+
+    def save_config(self, config: Dict[str, Any]) -> bool:
+        """
+        Save configuration to main.yaml.
+        Deep-merges provided config with existing one; writes atomically.
+        Rejects invalid configs per schema.
+        """
+        try:
+            with self._lock:
+                current = self.load_config(force_reload=True)
+                self._deep_update(current, config)
+                validated = self._validate_and_migrate(current)
+
+                self.config_path.parent.mkdir(parents=True, exist_ok=True)
+                yaml_str = yaml.safe_dump(
+                    validated,
+                    default_flow_style=False,
+                    allow_unicode=True,
+                    sort_keys=False,
+                )
+
+                # Atomic write with backup
+                fd, tmp_path = tempfile.mkstemp(
+                    prefix="main.yaml.", dir=str(self.config_path.parent)
+                )
+                try:
+                    with os.fdopen(fd, "w", encoding="utf-8") as tmp:
+                        tmp.write(yaml_str)
+                        tmp.flush()
+                        os.fsync(tmp.fileno())
+                    backup_path = self.config_path.with_suffix(".yaml.bak")
+                    if self.config_path.exists():
+                        try:
+                            os.replace(self.config_path, backup_path)
+                        except Exception:
+                            logger.debug("Backup replace failed; continuing.")
+                    os.replace(tmp_path, self.config_path)
+                    self._config_cache = validated
+                    self._last_mtime = self.config_path.stat().st_mtime
+                    return True
+                finally:
+                    if os.path.exists(tmp_path):
+                        try:
+                            os.remove(tmp_path)
+                        except Exception:
+                            pass
+        except ConfigError as ce:
+            logger.error(
+                "Refusing to save invalid config: %s",
+                ce,
+                extra={"context": getattr(ce, "context", {})},
+            )
+            return False
+        except Exception as e:
+            logger.exception("Error saving config: %s", e)
+            return False
+
+    def get_env_info(self) -> Dict[str, str]:
+        """
+        Read relevant environment variables using layered .env files and process env.
+        Returns only non-sensitive metadata.
+        """
+        env_path = self.project_root / ".env"
+        local_path = self.project_root / ".env.local"
+        parsed_env = self._load_env_file(env_path)
+        parsed_env.update(self._load_env_file(local_path))
+
+        def _get(key: str, default: str = "") -> str:
+            return str(parsed_env.get(key) or os.environ.get(key, default))
+
+        return {
+            "model": _get("LLM_MODEL", DEFAULTS.get("llm", {}).get("model", "Pro/Flash")),
+        }
+
+    def validate_required_env(self, keys: List[str]) -> Dict[str, List[str]]:
+        env_path = self.project_root / ".env"
+        local_path = self.project_root / ".env.local"
+        parsed_env = self._load_env_file(env_path)
+        parsed_env.update(self._load_env_file(local_path))
+        missing = [k for k in keys if not (parsed_env.get(k) or os.environ.get(k))]
+        if missing:
+            logger.warning("Missing required env keys", extra={"missing": missing})
+        return {"missing": missing}
+
+    @classmethod
+    def reset_for_tests(cls) -> None:
+        """Reset singleton to allow re-initialization in tests with a different project_root."""
+        with cls._lock:
+            cls._instance = None

src/utils/document_validator.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Document Validator - Validation utilities for document uploads
+"""
+
+import mimetypes
+import os
+import re
+from typing import ClassVar
+
+
+class DocumentValidator:
+    """Document validation utilities"""
+
+    # Maximum file size in bytes (100MB)
+    MAX_FILE_SIZE: ClassVar[int] = 100 * 1024 * 1024
+
+    # Maximum file size for PDF processing (50MB to prevent resource exhaustion)
+    MAX_PDF_SIZE: ClassVar[int] = 50 * 1024 * 1024
+
+    # Allowed file extensions
+    ALLOWED_EXTENSIONS: ClassVar[set[str]] = {
+        ".pdf",
+        ".txt",
+        ".md",
+        ".doc",
+        ".docx",
+        ".rtf",
+        ".html",
+        ".htm",
+        ".xml",
+        ".json",
+        ".csv",
+        ".xlsx",
+        ".xls",
+        ".pptx",
+        ".ppt",
+    }
+
+    # MIME type mapping for additional validation
+    ALLOWED_MIME_TYPES: ClassVar[set[str]] = {
+        "application/pdf",
+        "text/plain",
+        "text/markdown",
+        "application/msword",
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "application/rtf",
+        "text/html",
+        "application/xml",
+        "text/xml",
+        "application/json",
+        "text/csv",
+        "application/vnd.ms-excel",
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "application/vnd.ms-powerpoint",
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+    }
+
+    @staticmethod
+    def validate_upload_safety(
+        filename: str, file_size: int | None, allowed_extensions: set[str] | None = None
+    ) -> str:
+        """
+        Validate file upload safety
+
+        Args:
+            filename: Name of the file
+            file_size: Size of the file in bytes, or None to skip size validation
+            allowed_extensions: Optional override for allowed extensions
+
+        Returns:
+            Sanitized filename safe for filesystem use
+
+        Raises:
+            ValueError: If validation fails
+        """
+        # Check file size (skip if size is None)
+        if file_size is not None and file_size > DocumentValidator.MAX_FILE_SIZE:
+            raise ValueError(
+                f"File too large: {file_size} bytes. Maximum allowed: {DocumentValidator.MAX_FILE_SIZE} bytes"
+            )
+
+        # Additional size check for PDFs to prevent resource exhaustion
+        _, ext = os.path.splitext(filename.lower())
+        if ext == ".pdf" and file_size is not None and file_size > DocumentValidator.MAX_PDF_SIZE:
+            raise ValueError(
+                f"PDF file too large: {file_size} bytes. Maximum allowed for PDFs: {DocumentValidator.MAX_PDF_SIZE} bytes"
+            )
+
+        # Sanitize filename - remove path components and dangerous characters
+        # Extract just the filename, removing any path components
+        safe_name = os.path.basename(filename)
+        # Remove null bytes and other control characters
+        safe_name = re.sub(r"[\x00-\x1f\x7f]", "", safe_name)
+        # Replace problematic characters
+        safe_name = re.sub(r'[<>:"/\\|?*]', "_", safe_name)
+
+        if not safe_name or safe_name in (".", "..") or safe_name.strip("_") == "":
+            raise ValueError("Invalid filename")
+
+        # Check file extension
+        exts_to_check = allowed_extensions or DocumentValidator.ALLOWED_EXTENSIONS
+        if ext not in exts_to_check:
+            raise ValueError(
+                f"Unsupported file type: {ext}. Allowed types: {', '.join(exts_to_check)}"
+            )
+
+        # Additional MIME type validation for security
+        guessed_mime, _ = mimetypes.guess_type(filename)
+        if guessed_mime and guessed_mime not in DocumentValidator.ALLOWED_MIME_TYPES:
+            raise ValueError(
+                f"MIME type validation failed: {guessed_mime}. File may be malicious or corrupted."
+            )
+
+        return safe_name
+
+    @staticmethod
+    def get_file_info(filename: str, file_size: int) -> dict:
+        """
+        Get file information
+
+        Args:
+            filename: Name of the file
+            file_size: Size of the file in bytes
+
+        Returns:
+            Dictionary with file information
+        """
+        _, ext = os.path.splitext(filename.lower())
+        return {
+            "filename": filename,
+            "extension": ext,
+            "size_bytes": file_size,
+            "size_mb": round(file_size / (1024 * 1024), 2),
+            "is_allowed": ext in DocumentValidator.ALLOWED_EXTENSIONS,
+        }
+
+    @staticmethod
+    def validate_file(path: str) -> dict:
+        """
+        Validate that a file exists, is readable, and has valid content.
+
+        Args:
+            path: Path to the file to validate
+
+        Returns:
+            File info dictionary
+
+        Raises:
+            ValueError: If file is missing or validation fails
+        """
+        if not os.path.exists(path):
+            raise ValueError(f"File not found: {path}")
+
+        if not os.path.isfile(path):
+            raise ValueError(f"Not a file: {path}")
+
+        if not os.access(path, os.R_OK):
+            raise ValueError(f"File not readable: {path}")
+
+        size = os.path.getsize(path)
+        filename = os.path.basename(path)
+
+        # Validate using validate_upload_safety
+        safe_name = DocumentValidator.validate_upload_safety(filename, size)
+
+        return DocumentValidator.get_file_info(safe_name, size)