janito 2.33.0__py3-none-any.whl → 3.1.0__py3-none-any.whl

janito/plugins/web/webtools/tools/fetch_url.py

@@ -0,0 +1,458 @@
+import requests
+import time
+import os
+import json
+from pathlib import Path
+from bs4 import BeautifulSoup
+from typing import Dict, Any, Optional
+from janito.tools.adapters.local.adapter import register_local_tool
+from janito.tools.tool_base import ToolBase, ToolPermissions
+from janito.report_events import ReportAction
+from janito.i18n import tr
+from janito.tools.tool_utils import pluralize
+from janito.tools.loop_protection_decorator import protect_against_loops
+
+
+@register_local_tool
+class FetchUrlTool(ToolBase):
+    """
+    Fetch the content of a web page and extract its text.
+
+    This tool implements a **session-based caching mechanism** that provides
+    **in-memory caching** for the lifetime of the tool instance. URLs are cached
+    in RAM during the session, providing instant access to previously fetched
+    content without making additional HTTP requests.
+
+    **Session Cache Behavior:**
+    - **Lifetime**: Cache exists for the lifetime of the FetchUrlTool instance
+    - **Scope**: In-memory (RAM) cache, not persisted to disk
+    - **Storage**: Successful responses are cached as raw HTML content
+    - **Key**: Cache key is the exact URL string
+    - **Invalidation**: Cache is automatically cleared when the tool instance is destroyed
+    - **Performance**: Subsequent requests for the same URL return instantly
+
+    **Error Cache Behavior:**
+    - HTTP 403 errors: Cached for 24 hours (more permanent)
+    - HTTP 404 errors: Cached for 1 hour (temporary)
+    - Other 4xx errors: Cached for 30 minutes
+    - 5xx errors: Not cached (retried on each request)
+
+    Args:
+        url (str): The URL of the web page to fetch.
+        search_strings (list[str], optional): Strings to search for in the page content.
+        max_length (int, optional): Maximum number of characters to return. Defaults to 5000.
+        max_lines (int, optional): Maximum number of lines to return. Defaults to 200.
+        context_chars (int, optional): Characters of context around search matches. Defaults to 400.
+        timeout (int, optional): Timeout in seconds for the HTTP request. Defaults to 10.
+        save_to_file (str, optional): File path to save the full resource content. If provided,
+            the complete response will be saved to this file instead of being processed.
+        headers (Dict[str, str], optional): Custom HTTP headers to send with the request.
+        cookies (Dict[str, str], optional): Custom cookies to send with the request.
+        follow_redirects (bool, optional): Whether to follow HTTP redirects. Defaults to True.
+    Returns:
+        str: Extracted text content from the web page, or a warning message. Example:
+            - "<main text content...>"
+            - "No lines found for the provided search strings."
+            - "Warning: Empty URL provided. Operation skipped."
+    """
+
+    permissions = ToolPermissions(read=True)
+    tool_name = "fetch_url"
+
+    def __init__(self):
+        super().__init__()
+        self.cache_dir = Path.home() / ".janito" / "cache" / "fetch_url"
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.cache_file = self.cache_dir / "error_cache.json"
+        self.session_cache = (
+            {}
+        )  # In-memory session cache - lifetime matches tool instance
+        self._load_cache()
+
+        # Browser-like session with cookies and headers
+        self.session = requests.Session()
+        self.session.headers.update(
+            {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+                "Accept-Language": "en-US,en;q=0.5",
+                "Accept-Encoding": "gzip, deflate, br",
+                "DNT": "1",
+                "Connection": "keep-alive",
+                "Upgrade-Insecure-Requests": "1",
+            }
+        )
+
+        # Load cookies from disk if they exist
+        self.cookies_file = self.cache_dir / "cookies.json"
+        self._load_cookies()
+
+    def _load_cache(self):
+        """Load error cache from disk."""
+        if self.cache_file.exists():
+            try:
+                with open(self.cache_file, "r", encoding="utf-8") as f:
+                    self.error_cache = json.load(f)
+            except (json.JSONDecodeError, IOError):
+                self.error_cache = {}
+        else:
+            self.error_cache = {}
+
+    def _save_cache(self):
+        """Save error cache to disk."""
+        try:
+            with open(self.cache_file, "w", encoding="utf-8") as f:
+                json.dump(self.error_cache, f, indent=2)
+        except IOError:
+            pass  # Silently fail if we can't write cache
+
+    def _load_cookies(self):
+        """Load cookies from disk into session."""
+        if self.cookies_file.exists():
+            try:
+                with open(self.cookies_file, "r", encoding="utf-8") as f:
+                    cookies_data = json.load(f)
+                    for cookie in cookies_data:
+                        self.session.cookies.set(**cookie)
+            except (json.JSONDecodeError, IOError):
+                pass  # Silently fail if we can't load cookies
+
+    def _save_cookies(self):
+        """Save session cookies to disk."""
+        try:
+            cookies_data = []
+            for cookie in self.session.cookies:
+                cookies_data.append(
+                    {
+                        "name": cookie.name,
+                        "value": cookie.value,
+                        "domain": cookie.domain,
+                        "path": cookie.path,
+                    }
+                )
+            with open(self.cookies_file, "w", encoding="utf-8") as f:
+                json.dump(cookies_data, f, indent=2)
+        except IOError:
+            pass  # Silently fail if we can't write cookies
+
+    def _get_cached_error(self, url: str) -> tuple[str, bool]:
+        """
+        Check if we have a cached error for this URL.
+        Returns (error_message, is_cached) tuple.
+        """
+        if url not in self.error_cache:
+            return None, False
+
+        entry = self.error_cache[url]
+        current_time = time.time()
+
+        # Different expiration times for different status codes
+        if entry["status_code"] == 403:
+            # Cache 403 errors for 24 hours (more permanent)
+            expiration_time = 24 * 3600
+        elif entry["status_code"] == 404:
+            # Cache 404 errors for 1 hour (more temporary)
+            expiration_time = 3600
+        else:
+            # Cache other 4xx errors for 30 minutes
+            expiration_time = 1800
+
+        if current_time - entry["timestamp"] > expiration_time:
+            # Cache expired, remove it
+            del self.error_cache[url]
+            self._save_cache()
+            return None, False
+
+        return entry["message"], True
+
+    def _cache_error(self, url: str, status_code: int, message: str):
+        """Cache an HTTP error response."""
+        self.error_cache[url] = {
+            "status_code": status_code,
+            "message": message,
+            "timestamp": time.time(),
+        }
+        self._save_cache()
+
+    def _fetch_url_content(
+        self,
+        url: str,
+        timeout: int = 10,
+        headers: Optional[Dict[str, str]] = None,
+        cookies: Optional[Dict[str, str]] = None,
+        follow_redirects: bool = True,
+    ) -> str:
+        """Fetch URL content and handle HTTP errors.
+
+        Implements two-tier caching:
+        1. Session cache: In-memory cache for successful responses (lifetime = tool instance)
+        2. Error cache: Persistent disk cache for HTTP errors with different expiration times
+
+        Also implements URL whitelist checking and browser-like behavior.
+        """
+        # Check URL whitelist
+        from janito.tools.url_whitelist import get_url_whitelist_manager
+
+        whitelist_manager = get_url_whitelist_manager()
+
+        if not whitelist_manager.is_url_allowed(url):
+            error_message = tr("Blocked")
+            self.report_error(
+                tr("❗ Blocked"),
+                ReportAction.READ,
+            )
+            return error_message
+
+        # Check session cache first
+        if url in self.session_cache:
+            return self.session_cache[url]
+
+        # Check persistent cache for known errors
+        cached_error, is_cached = self._get_cached_error(url)
+        if cached_error:
+            self.report_warning(
+                tr(
+                    "ℹ️ Using cached HTTP error for URL: {url}",
+                    url=url,
+                ),
+                ReportAction.READ,
+            )
+            return cached_error
+
+        try:
+            # Merge custom headers with default ones
+            request_headers = self.session.headers.copy()
+            if headers:
+                request_headers.update(headers)
+
+            # Merge custom cookies
+            if cookies:
+                self.session.cookies.update(cookies)
+
+            response = self.session.get(
+                url,
+                timeout=timeout,
+                headers=request_headers,
+                allow_redirects=follow_redirects,
+            )
+            response.raise_for_status()
+            content = response.text
+
+            # Save cookies after successful request
+            self._save_cookies()
+
+            # Cache successful responses in session cache
+            self.session_cache[url] = content
+            return content
+        except requests.exceptions.HTTPError as http_err:
+            status_code = http_err.response.status_code if http_err.response else None
+
+            # Map status codes to descriptions
+            status_descriptions = {
+                400: "Bad Request",
+                401: "Unauthorized",
+                403: "Forbidden",
+                404: "Not Found",
+                405: "Method Not Allowed",
+                408: "Request Timeout",
+                409: "Conflict",
+                410: "Gone",
+                413: "Payload Too Large",
+                414: "URI Too Long",
+                415: "Unsupported Media Type",
+                429: "Too Many Requests",
+                500: "Internal Server Error",
+                501: "Not Implemented",
+                502: "Bad Gateway",
+                503: "Service Unavailable",
+                504: "Gateway Timeout",
+                505: "HTTP Version Not Supported",
+            }
+
+            if status_code and 400 <= status_code < 500:
+                description = status_descriptions.get(status_code, "Client Error")
+                error_message = f"HTTP {status_code} {description}"
+                # Cache 403 and 404 errors
+                if status_code in [403, 404]:
+                    self._cache_error(url, status_code, error_message)
+
+                self.report_error(
+                    f"❗ HTTP {status_code} {description}",
+                    ReportAction.READ,
+                )
+                return error_message
+            else:
+                status_code_str = str(status_code) if status_code else "Error"
+                description = status_descriptions.get(
+                    status_code,
+                    (
+                        "Server Error"
+                        if status_code and status_code >= 500
+                        else "Client Error"
+                    ),
+                )
+                self.report_error(
+                    f"❗ HTTP {status_code_str} {description}",
+                    ReportAction.READ,
+                )
+                return f"HTTP {status_code_str} {description}"
+        except requests.exceptions.ConnectionError as conn_err:
+            self.report_error(
+                "❗ Network Error",
+                ReportAction.READ,
+            )
+            return f"Network Error: Failed to connect to {url}"
+        except requests.exceptions.Timeout as timeout_err:
+            self.report_error(
+                "❗ Timeout Error",
+                ReportAction.READ,
+            )
+            return f"Timeout Error: Request timed out after {timeout} seconds"
+        except requests.exceptions.RequestException as req_err:
+            self.report_error(
+                "❗ Request Error",
+                ReportAction.READ,
+            )
+            return f"Request Error: {str(req_err)}"
+        except Exception as err:
+            self.report_error(
+                "❗ Error fetching URL",
+                ReportAction.READ,
+            )
+            return f"Error: {str(err)}"
+
+    def _extract_and_clean_text(self, html_content: str) -> str:
+        """Extract and clean text from HTML content."""
+        soup = BeautifulSoup(html_content, "html.parser")
+        text = soup.get_text(separator="\n")
+
+        # Clean up excessive whitespace
+        lines = [line.strip() for line in text.splitlines() if line.strip()]
+        return "\n".join(lines)
+
+    def _filter_by_search_strings(
+        self, text: str, search_strings: list[str], context_chars: int
+    ) -> str:
+        """Filter text by search strings with context."""
+        filtered = []
+        for s in search_strings:
+            idx = text.find(s)
+            if idx != -1:
+                start = max(0, idx - context_chars)
+                end = min(len(text), idx + len(s) + context_chars)
+                snippet = text[start:end]
+                filtered.append(snippet)
+
+        if filtered:
+            return "\n...\n".join(filtered)
+        else:
+            return tr("No lines found for the provided search strings.")
+
+    def _apply_limits(self, text: str, max_length: int, max_lines: int) -> str:
+        """Apply length and line limits to text."""
+        # Apply length limit
+        if len(text) > max_length:
+            text = text[:max_length] + "\n... (content truncated due to length limit)"
+
+        # Apply line limit
+        lines = text.splitlines()
+        if len(lines) > max_lines:
+            text = (
+                "\n".join(lines[:max_lines])
+                + "\n... (content truncated due to line limit)"
+            )
+
+        return text
+
+    @protect_against_loops(max_calls=5, time_window=10.0, key_field="url")
+    def run(
+        self,
+        url: str,
+        search_strings: list[str] = None,
+        max_length: int = 5000,
+        max_lines: int = 200,
+        context_chars: int = 400,
+        timeout: int = 10,
+        save_to_file: str = None,
+        headers: Dict[str, str] = None,
+        cookies: Dict[str, str] = None,
+        follow_redirects: bool = True,
+    ) -> str:
+        if not url.strip():
+            self.report_warning(tr("ℹ️ Empty URL provided."), ReportAction.READ)
+            return tr("Warning: Empty URL provided. Operation skipped.")
+
+        self.report_action(tr("🌐 Fetch URL '{url}' ...", url=url), ReportAction.READ)
+
+        # Check if we should save to file
+        if save_to_file:
+            html_content = self._fetch_url_content(
+                url,
+                timeout=timeout,
+                headers=headers,
+                cookies=cookies,
+                follow_redirects=follow_redirects,
+            )
+            if (
+                html_content.startswith("HTTP Error ")
+                or html_content == "Error"
+                or html_content == "Blocked"
+            ):
+                return html_content
+
+            try:
+                with open(save_to_file, "w", encoding="utf-8") as f:
+                    f.write(html_content)
+                file_size = len(html_content)
+                self.report_success(
+                    tr(
+                        "✅ Saved {size} bytes to {file}",
+                        size=file_size,
+                        file=save_to_file,
+                    ),
+                    ReportAction.READ,
+                )
+                return tr("Successfully saved content to: {file}", file=save_to_file)
+            except IOError as e:
+                error_msg = tr("Error saving to file: {error}", error=str(e))
+                self.report_error(error_msg, ReportAction.READ)
+                return error_msg
+
+        # Normal processing path
+        html_content = self._fetch_url_content(
+            url,
+            timeout=timeout,
+            headers=headers,
+            cookies=cookies,
+            follow_redirects=follow_redirects,
+        )
+        if (
+            html_content.startswith("HTTP Error ")
+            or html_content == "Error"
+            or html_content == "Blocked"
+        ):
+            return html_content
+
+        # Extract and clean text
+        text = self._extract_and_clean_text(html_content)
+
+        # Filter by search strings if provided
+        if search_strings:
+            text = self._filter_by_search_strings(text, search_strings, context_chars)
+
+        # Apply limits
+        text = self._apply_limits(text, max_length, max_lines)
+
+        # Report success
+        num_lines = len(text.splitlines())
+        total_chars = len(text)
+        self.report_success(
+            tr(
+                "✅ {num_lines} {line_word}, {chars} chars",
+                num_lines=num_lines,
+                line_word=pluralize("line", num_lines),
+                chars=total_chars,
+            ),
+            ReportAction.READ,
+        )
+        return text

janito/tools/__init__.py

@@ -1,6 +1,6 @@
-from janito.tools.adapters.local import (
-    local_tools_adapter as _internal_local_tools_adapter,
-    LocalToolsAdapter,
+from janito.tools.adapters.local import LocalToolsAdapter
+from janito.tools.adapters.local.adapter import (
+    LocalToolsAdapter as _internal_local_tools_adapter,
 )
 
 
@@ -11,8 +11,10 @@ def get_local_tools_adapter(workdir=None, allowed_permissions=None):
     if workdir is not None and not os.path.exists(workdir):
         os.makedirs(workdir, exist_ok=True)
     # Permissions are now managed globally; ignore allowed_permissions argument except for backward compatibility
-    # Reuse the singleton adapter defined in janito.tools.adapters.local to maintain tool registrations
-    registry = _internal_local_tools_adapter
+    # Create and initialize adapter
+    from janito.tools.initialize import initialize_tools
+
+    registry = initialize_tools(LocalToolsAdapter(workdir=workdir))
     # Change workdir if requested
     if workdir is not None:
         try:
@@ -27,10 +29,32 @@ def get_local_tools_adapter(workdir=None, allowed_permissions=None):
     return registry
 
 
-local_tools_adapter = _internal_local_tools_adapter
+# Initialize the global adapter - defer import to avoid circular dependencies
+local_tools_adapter = None
+
+
+def get_local_tools_adapter(workdir=None, allowed_permissions=None):
+    """Get the global tools adapter, initializing on first use."""
+    global local_tools_adapter
+    if local_tools_adapter is None:
+        from janito.tools.initialize import initialize_tools
+
+        adapter = LocalToolsAdapter(workdir=workdir)
+        local_tools_adapter = initialize_tools(adapter)
+
+    # Handle workdir if provided
+    if workdir is not None and local_tools_adapter is not None:
+        import os
+
+        if not os.path.exists(workdir):
+            os.makedirs(workdir, exist_ok=True)
+        os.chdir(workdir)
+        local_tools_adapter.workdir = workdir
+
+    return local_tools_adapter
+
 
 __all__ = [
     "LocalToolsAdapter",
     "get_local_tools_adapter",
-    "local_tools_adapter",
 ]

janito/tools/adapters/__init__.py

@@ -1 +1,6 @@
-# Tools providers package: for plug-and-play tool collections, integrations, and adapters.
+"""
+Tools providers package: for plug-and-play tool collections, integrations, and adapters.
+
+This package contains the core adapter infrastructure for managing tools,
+while the actual tool implementations have been moved to the plugins system.
+"""

janito/tools/adapters/local/__init__.py

@@ -1,73 +1,10 @@
-from .adapter import LocalToolsAdapter
-
-from .ask_user import AskUserTool
-from .copy_file import CopyFileTool
-from .create_directory import CreateDirectoryTool
-from .create_file import CreateFileTool
-from .fetch_url import FetchUrlTool
-from .find_files import FindFilesTool
-from .view_file import ViewFileTool
-from .read_files import ReadFilesTool
-from .move_file import MoveFileTool
-from .open_url import OpenUrlTool
-from .open_html_in_browser import OpenHtmlInBrowserTool
-from .python_code_run import PythonCodeRunTool
-from .python_command_run import PythonCommandRunTool
-from .python_file_run import PythonFileRunTool
-from .remove_directory import RemoveDirectoryTool
-from .remove_file import RemoveFileTool
-from .replace_text_in_file import ReplaceTextInFileTool
-from .run_bash_command import RunBashCommandTool
-from .run_powershell_command import RunPowershellCommandTool
-from .get_file_outline.core import GetFileOutlineTool
-from .get_file_outline.search_outline import SearchOutlineTool
-from .search_text.core import SearchTextTool
-from .validate_file_syntax.core import ValidateFileSyntaxTool
-from .read_chart import ReadChartTool
-from .show_image import ShowImageTool
-from .show_image_grid import ShowImageGridTool
-
-from janito.tools.tool_base import ToolPermissions
-import os
-from janito.tools.permissions import get_global_allowed_permissions
-
-# Singleton tools adapter with all standard tools registered
-local_tools_adapter = LocalToolsAdapter(workdir=os.getcwd())
+"""
+Local tools adapter for janito.
 
+This package provides the LocalToolsAdapter class which manages tool registration
+and execution for local, in-process tools.
+"""
 
-def get_local_tools_adapter(workdir=None):
-    return LocalToolsAdapter(workdir=workdir or os.getcwd())
-
-
-# Register tools
-for tool_class in [
-    AskUserTool,
-    CopyFileTool,
-    CreateDirectoryTool,
-    CreateFileTool,
-    FetchUrlTool,
-    FindFilesTool,
-    ViewFileTool,
-    ReadFilesTool,
-    MoveFileTool,
-    OpenUrlTool,
-    OpenHtmlInBrowserTool,
-    PythonCodeRunTool,
-    PythonCommandRunTool,
-    PythonFileRunTool,
-    RemoveDirectoryTool,
-    RemoveFileTool,
-    ReplaceTextInFileTool,
-    RunBashCommandTool,
-    RunPowershellCommandTool,
-    GetFileOutlineTool,
-    SearchOutlineTool,
-    SearchTextTool,
-    ValidateFileSyntaxTool,
-    ReadChartTool,
-    ShowImageTool,
-    ShowImageGridTool,
-]:
-    local_tools_adapter.register_tool(tool_class)
+from .adapter import LocalToolsAdapter
 
-# DEBUG: Print registered tools at startup
+__all__ = ["LocalToolsAdapter"]

janito/tools/cli_initializer.py

@@ -0,0 +1,88 @@
+"""
+CLI-specific tool initialization for janito.
+
+This module provides functions to initialize tools specifically for CLI usage,
+handling circular imports and ensuring proper registration.
+"""
+
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+# Ensure current directory is in path
+sys.path.insert(0, str(Path.cwd()))
+
+
+def initialize_cli_tools():
+    """Initialize tools for CLI usage, avoiding circular imports."""
+    try:
+        from janito.tools.adapters.local.adapter import LocalToolsAdapter
+        from janito.plugin_system.core_loader_fixed import load_core_plugin
+        from janito.tools.permissions import set_global_allowed_permissions
+        from janito.tools.tool_base import ToolPermissions
+
+        # Create adapter
+        adapter = LocalToolsAdapter()
+
+        # Set permissions for CLI
+        set_global_allowed_permissions(
+            ToolPermissions(read=True, write=True, execute=True)
+        )
+
+        # Core plugins to load
+        core_plugins = [
+            "core.filemanager",
+            "core.codeanalyzer",
+            "core.system",
+            "core.imagedisplay",
+            "dev.pythondev",
+            "dev.visualization",
+        ]
+
+        loaded_count = 0
+        for plugin_name in core_plugins:
+            plugin = load_core_plugin(plugin_name)
+            if plugin:
+                tools = plugin.get_tools()
+                for tool_class in tools:
+                    try:
+                        adapter.register_tool(tool_class)
+                        loaded_count += 1
+                    except ValueError:
+                        # Tool already registered, skip
+                        pass
+
+        return adapter, loaded_count
+
+    except Exception as e:
+        print(f"Error initializing CLI tools: {e}", file=sys.stderr)
+        return None, 0
+
+
+def get_cli_tools_adapter():
+    """Get a CLI-initialized tools adapter."""
+    adapter, count = initialize_cli_tools()
+    if adapter and count > 0:
+        return adapter
+    return None
+
+
+def list_cli_tools():
+    """List all available CLI tools."""
+    adapter = get_cli_tools_adapter()
+    if not adapter:
+        return []
+
+    return adapter.list_tools()
+
+
+if __name__ == "__main__":
+    adapter, count = initialize_cli_tools()
+    if adapter:
+        tools = adapter.list_tools()
+        print(f"CLI initialized {count} tools")
+        print(f"Available tools: {len(tools)}")
+        for tool in sorted(tools):
+            print(f"  - {tool}")
+    else:
+        print("Failed to initialize CLI tools")