janito 2.20.1__py3-none-any.whl → 2.22.0__py3-none-any.whl

janito/providers/ibm/provider.py

@@ -0,0 +1,149 @@
+"""IBM WatsonX AI Provider implementation."""
+
+from janito.llm.provider import LLMProvider
+from janito.llm.auth import LLMAuthManager
+from janito.llm.driver_config import LLMDriverConfig
+from janito.tools import get_local_tools_adapter
+from janito.providers.registry import LLMProviderRegistry
+from .model_info import MODEL_SPECS
+
+try:
+    from janito.drivers.openai.driver import OpenAIModelDriver
+
+    available = True
+    unavailable_reason = None
+except ImportError as e:
+    available = False
+    unavailable_reason = str(e)
+
+
+class IBMProvider(LLMProvider):
+    """IBM WatsonX AI Provider for accessing IBM's AI services."""
+
+    name = "ibm"
+    NAME = "ibm"
+    MAINTAINER = "João Pinto <janito@ikignosis.org>"
+    MODEL_SPECS = MODEL_SPECS
+    DEFAULT_MODEL = "ibm/granite-3-3-8b-instruct"
+
+    def __init__(
+        self, auth_manager: LLMAuthManager = None, config: LLMDriverConfig = None
+    ):
+        self._tools_adapter = get_local_tools_adapter()
+        self._driver = None
+
+        if not self.available:
+            return
+
+        self._initialize_config(auth_manager, config)
+        self._setup_model_config()
+
+    def _initialize_config(self, auth_manager, config):
+        """Initialize configuration and API credentials."""
+        self.auth_manager = auth_manager or LLMAuthManager()
+
+        # IBM WatsonX uses multiple credentials
+        self._api_key = self.auth_manager.get_credentials(type(self).NAME)
+        if not self._api_key:
+            from janito.llm.auth_utils import handle_missing_api_key
+
+            handle_missing_api_key(self.name, "WATSONX_API_KEY")
+
+        # Get project ID for WatsonX
+        self._project_id = self.auth_manager.get_credentials(
+            f"{type(self).NAME}_project_id"
+        )
+        if not self._project_id:
+            from janito.llm.auth_utils import handle_missing_api_key
+
+            handle_missing_api_key(self.name, "WATSONX_PROJECT_ID")
+
+        # Get region/space ID
+        self._space_id = self.auth_manager.get_credentials(
+            f"{type(self).NAME}_space_id"
+        )
+
+        self._driver_config = config or LLMDriverConfig(model=None)
+        if not self._driver_config.model:
+            self._driver_config.model = self.DEFAULT_MODEL
+        if not self._driver_config.api_key:
+            self._driver_config.api_key = self._api_key
+
+    def _setup_model_config(self):
+        """Configure token limits based on model specifications."""
+        model_name = self._driver_config.model
+        model_spec = self.MODEL_SPECS.get(model_name)
+
+        # Reset token parameters
+        if hasattr(self._driver_config, "max_tokens"):
+            self._driver_config.max_tokens = None
+        if hasattr(self._driver_config, "max_completion_tokens"):
+            self._driver_config.max_completion_tokens = None
+
+        if model_spec:
+            if getattr(model_spec, "thinking_supported", False):
+                max_cot = getattr(model_spec, "max_cot", None)
+                if max_cot and max_cot != "N/A":
+                    self._driver_config.max_completion_tokens = int(max_cot)
+            else:
+                max_response = getattr(model_spec, "max_response", None)
+                if max_response and max_response != "N/A":
+                    self._driver_config.max_tokens = int(max_response)
+
+        # Set IBM WatsonX specific parameters
+        self._driver_config.base_url = "https://us-south.ml.cloud.ibm.com"
+        self._driver_config.project_id = self._project_id
+        if self._space_id:
+            self._driver_config.space_id = self._space_id
+
+        self.fill_missing_device_info(self._driver_config)
+
+    @property
+    def driver(self):
+        if not self.available:
+            raise ImportError(f"IBMProvider unavailable: {self.unavailable_reason}")
+        return self._driver
+
+    @property
+    def available(self):
+        return available
+
+    @property
+    def unavailable_reason(self):
+        return unavailable_reason
+
+    def create_driver(self):
+        """
+        Creates and returns a new OpenAIModelDriver instance configured for IBM WatsonX.
+        IBM WatsonX uses OpenAI-compatible API format.
+        """
+        driver = OpenAIModelDriver(
+            tools_adapter=self._tools_adapter, provider_name=self.NAME
+        )
+        driver.config = self._driver_config
+        return driver
+
+    def create_agent(self, tools_adapter=None, agent_name: str = None, **kwargs):
+        from janito.llm.agent import LLMAgent
+
+        if tools_adapter is None:
+            tools_adapter = get_local_tools_adapter()
+        raise NotImplementedError(
+            "create_agent must be constructed via new factory using input/output queues and config."
+        )
+
+    @property
+    def model_name(self):
+        return self._driver_config.model
+
+    @property
+    def driver_config(self):
+        """Public, read-only access to the provider's LLMDriverConfig object."""
+        return self._driver_config
+
+    def execute_tool(self, tool_name: str, event_bus, *args, **kwargs):
+        self._tools_adapter.event_bus = event_bus
+        return self._tools_adapter.execute_by_name(tool_name, *args, **kwargs)
+
+
+LLMProviderRegistry.register(IBMProvider.NAME, IBMProvider)

janito/tools/adapters/local/fetch_url.py

@@ -1,4 +1,8 @@
 import requests
+import time
+import os
+import json
+from pathlib import Path
 from bs4 import BeautifulSoup
 from janito.tools.adapters.local.adapter import register_local_tool
 from janito.tools.tool_base import ToolBase, ToolPermissions
@@ -15,6 +19,10 @@ class FetchUrlTool(ToolBase):
     Args:
         url (str): The URL of the web page to fetch.
         search_strings (list[str], optional): Strings to search for in the page content.
+        max_length (int, optional): Maximum number of characters to return. Defaults to 5000.
+        max_lines (int, optional): Maximum number of lines to return. Defaults to 200.
+        context_chars (int, optional): Characters of context around search matches. Defaults to 400.
+        timeout (int, optional): Timeout in seconds for the HTTP request. Defaults to 10.
     Returns:
         str: Extracted text content from the web page, or a warning message. Example:
             - "<main text content...>"
@@ -25,17 +33,101 @@ class FetchUrlTool(ToolBase):
     permissions = ToolPermissions(read=True)
     tool_name = "fetch_url"
 
-    def run(self, url: str, search_strings: list[str] = None) -> str:
-        if not url.strip():
-            self.report_warning(tr("ℹ️ Empty URL provided."), ReportAction.READ)
-            return tr("Warning: Empty URL provided. Operation skipped.")
-        self.report_action(tr("🌐 Fetch URL '{url}' ...", url=url), ReportAction.READ)
+    def __init__(self):
+        super().__init__()
+        self.cache_dir = Path.home() / ".janito" / "cache" / "fetch_url"
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.cache_file = self.cache_dir / "error_cache.json"
+        self._load_cache()
+
+    def _load_cache(self):
+        """Load error cache from disk."""
+        if self.cache_file.exists():
+            try:
+                with open(self.cache_file, 'r', encoding='utf-8') as f:
+                    self.error_cache = json.load(f)
+            except (json.JSONDecodeError, IOError):
+                self.error_cache = {}
+        else:
+            self.error_cache = {}
+
+    def _save_cache(self):
+        """Save error cache to disk."""
+        try:
+            with open(self.cache_file, 'w', encoding='utf-8') as f:
+                json.dump(self.error_cache, f, indent=2)
+        except IOError:
+            pass  # Silently fail if we can't write cache
+
+    def _get_cached_error(self, url: str) -> tuple[str, bool]:
+        """
+        Check if we have a cached error for this URL.
+        Returns (error_message, is_cached) tuple.
+        """
+        if url not in self.error_cache:
+            return None, False
+        
+        entry = self.error_cache[url]
+        current_time = time.time()
+        
+        # Different expiration times for different status codes
+        if entry['status_code'] == 403:
+            # Cache 403 errors for 24 hours (more permanent)
+            expiration_time = 24 * 3600
+        elif entry['status_code'] == 404:
+            # Cache 404 errors for 1 hour (more temporary)
+            expiration_time = 3600
+        else:
+            # Cache other 4xx errors for 30 minutes
+            expiration_time = 1800
+            
+        if current_time - entry['timestamp'] > expiration_time:
+            # Cache expired, remove it
+            del self.error_cache[url]
+            self._save_cache()
+            return None, False
+            
+        return entry['message'], True
+
+    def _cache_error(self, url: str, status_code: int, message: str):
+        """Cache an HTTP error response."""
+        self.error_cache[url] = {
+            'status_code': status_code,
+            'message': message,
+            'timestamp': time.time()
+        }
+        self._save_cache()
+
+    def _fetch_url_content(self, url: str, timeout: int = 10) -> str:
+        """Fetch URL content and handle HTTP errors."""
+        # Check cache first for known errors
+        cached_error, is_cached = self._get_cached_error(url)
+        if cached_error:
+            self.report_warning(
+                tr(
+                    "ℹ️ Using cached HTTP error for URL: {url}",
+                    url=url,
+                ),
+                ReportAction.READ,
+            )
+            return cached_error
+
         try:
-            response = requests.get(url, timeout=10)
+            response = requests.get(url, timeout=timeout)
             response.raise_for_status()
+            return response.text
         except requests.exceptions.HTTPError as http_err:
             status_code = http_err.response.status_code if http_err.response else None
             if status_code and 400 <= status_code < 500:
+                error_message = tr(
+                    "Warning: HTTP {status_code} error for URL: {url}",
+                    status_code=status_code,
+                    url=url,
+                )
+                # Cache 403 and 404 errors
+                if status_code in [403, 404]:
+                    self._cache_error(url, status_code, error_message)
+                
                 self.report_error(
                     tr(
                         "❗ HTTP {status_code} error for URL: {url}",
@@ -44,11 +136,7 @@ class FetchUrlTool(ToolBase):
                     ),
                     ReportAction.READ,
                 )
-                return tr(
-                    "Warning: HTTP {status_code} error for URL: {url}",
-                    status_code=status_code,
-                    url=url,
-                )
+                return error_message
             else:
                 self.report_error(
                     tr(
@@ -71,27 +159,89 @@ class FetchUrlTool(ToolBase):
             return tr(
                 "Warning: Error fetching URL: {url}: {err}", url=url, err=str(err)
             )
-        soup = BeautifulSoup(response.text, "html.parser")
+
+    def _extract_and_clean_text(self, html_content: str) -> str:
+        """Extract and clean text from HTML content."""
+        soup = BeautifulSoup(html_content, "html.parser")
         text = soup.get_text(separator="\n")
+
+        # Clean up excessive whitespace
+        lines = [line.strip() for line in text.splitlines() if line.strip()]
+        return "\n".join(lines)
+
+    def _filter_by_search_strings(
+        self, text: str, search_strings: list[str], context_chars: int
+    ) -> str:
+        """Filter text by search strings with context."""
+        filtered = []
+        for s in search_strings:
+            idx = text.find(s)
+            if idx != -1:
+                start = max(0, idx - context_chars)
+                end = min(len(text), idx + len(s) + context_chars)
+                snippet = text[start:end]
+                filtered.append(snippet)
+
+        if filtered:
+            return "\n...\n".join(filtered)
+        else:
+            return tr("No lines found for the provided search strings.")
+
+    def _apply_limits(self, text: str, max_length: int, max_lines: int) -> str:
+        """Apply length and line limits to text."""
+        # Apply length limit
+        if len(text) > max_length:
+            text = text[:max_length] + "\n... (content truncated due to length limit)"
+
+        # Apply line limit
+        lines = text.splitlines()
+        if len(lines) > max_lines:
+            text = (
+                "\n".join(lines[:max_lines])
+                + "\n... (content truncated due to line limit)"
+            )
+
+        return text
+
+    def run(
+        self,
+        url: str,
+        search_strings: list[str] = None,
+        max_length: int = 5000,
+        max_lines: int = 200,
+        context_chars: int = 400,
+        timeout: int = 10,
+    ) -> str:
+        if not url.strip():
+            self.report_warning(tr("ℹ️ Empty URL provided."), ReportAction.READ)
+            return tr("Warning: Empty URL provided. Operation skipped.")
+
+        self.report_action(tr("🌐 Fetch URL '{url}' ...", url=url), ReportAction.READ)
+
+        # Fetch URL content
+        html_content = self._fetch_url_content(url, timeout=timeout)
+        if html_content.startswith("Warning:"):
+            return html_content
+
+        # Extract and clean text
+        text = self._extract_and_clean_text(html_content)
+
+        # Filter by search strings if provided
         if search_strings:
-            filtered = []
-            for s in search_strings:
-                idx = text.find(s)
-                if idx != -1:
-                    start = max(0, idx - 200)
-                    end = min(len(text), idx + len(s) + 200)
-                    snippet = text[start:end]
-                    filtered.append(snippet)
-            if filtered:
-                text = "\n...\n".join(filtered)
-            else:
-                text = tr("No lines found for the provided search strings.")
+            text = self._filter_by_search_strings(text, search_strings, context_chars)
+
+        # Apply limits
+        text = self._apply_limits(text, max_length, max_lines)
+
+        # Report success
         num_lines = len(text.splitlines())
+        total_chars = len(text)
         self.report_success(
             tr(
-                "✅ {num_lines} {line_word}",
+                "✅ {num_lines} {line_word}, {chars} chars",
                 num_lines=num_lines,
                 line_word=pluralize("line", num_lines),
+                chars=total_chars,
             ),
             ReportAction.READ,
         )