janito 0.10.1__py3-none-any.whl → 0.12.0__py3-none-any.whl

janito/tools/decorators.py

@@ -1,84 +1,90 @@
-"""
-Decorators for janito tools.
-"""
-import functools
-import inspect
-import string
-from typing import Any, Callable, Dict, Optional, Tuple
-
-
-class ToolMetaFormatter(string.Formatter):
-    """Custom string formatter that handles conditional expressions in format strings."""
-    
-    def get_value(self, key, args, kwargs):
-        """Override to handle conditional expressions."""
-        if key in kwargs:
-            return kwargs[key]
-        
-        # Try to evaluate the key as a Python expression
-        try:
-            # Create a safe local namespace with only the parameters
-            return eval(key, {"__builtins__": {}}, kwargs)
-        except Exception:
-            return f"[{key}]"
-
-
-def tool_meta(label: str):
-    """
-    Decorator to add metadata to a tool function.
-    
-    Args:
-        label: A format string that can reference function parameters.
-              Example: "Finding files {pattern}, on {root_dir}"
-              
-    Returns:
-        Decorated function with metadata attached
-    """
-    def decorator(func: Callable):
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            return func(*args, **kwargs)
-        
-        # Attach metadata to the function
-        wrapper._tool_meta = {
-            'label': label
-        }
-        
-        return wrapper
-    
-    return decorator
-
-
-def format_tool_label(func: Callable, tool_input: Dict[str, Any]) -> Optional[str]:
-    """
-    Format the tool label using the function's parameters.
-    
-    Args:
-        func: The tool function
-        tool_input: Input parameters for the tool
-        
-    Returns:
-        Formatted label string or None if no label is defined
-    """
-    if not hasattr(func, '_tool_meta') or 'label' not in func._tool_meta:
-        return None
-    
-    # Get the label template
-    label_template = func._tool_meta['label']
-    
-    # Special handling for str_replace_editor which uses **kwargs
-    if func.__name__ == 'str_replace_editor':
-        # Extract command and file_path from tool_input if they exist
-        command = tool_input.get('command', 'unknown')
-        file_path = tool_input.get('file_path', '')
-        
-        # Simple string replacement for the common case
-        if '{command}' in label_template and '{file_path}' in label_template:
-            return label_template.replace('{command}', command).replace('{file_path}', file_path)
-    
-    # Format the label with the parameters
-    try:
-        formatter = ToolMetaFormatter()
-        return formatter.format(label_template, **tool_input)
-    except Exception as e:
-        return f"{func.__name__}"
+"""
+Decorators for janito tools.
+"""
+import functools
+import string
+from typing import Any, Callable, Dict, Optional
+
+
+class ToolMetaFormatter(string.Formatter):
+    """Custom string formatter that handles conditional expressions in format strings."""
+    
+    def get_value(self, key, args, kwargs):
+        """Override to handle conditional expressions."""
+        if key in kwargs:
+            return kwargs[key]
+        
+        # Try to evaluate the key as a Python expression
+        try:
+            # Create a safe local namespace with only the parameters
+            return eval(key, {"__builtins__": {}}, kwargs)
+        except Exception:
+            return f"[{key}]"
+
+
+def tool_meta(label: str):
+    """
+    Decorator to add metadata to a tool function.
+    
+    Args:
+        label: A format string that can reference function parameters.
+              Example: "Finding files {pattern}, on {root_dir}"
+              
+    Returns:
+        Decorated function with metadata attached
+    """
+    def decorator(func: Callable):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            return func(*args, **kwargs)
+        
+        # Attach metadata to the function
+        wrapper._tool_meta = {
+            'label': label
+        }
+        
+        return wrapper
+    
+    return decorator
+
+
+def tool(func: Callable):
+    """
+    Basic decorator for tool functions.
+    
+    This decorator marks a function as a tool and can be used for
+    simpler tools that don't need additional metadata.
+    
+    Returns:
+        Decorated function
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+    
+    return wrapper
+
+
+def format_tool_label(func: Callable, tool_input: Dict[str, Any]) -> Optional[str]:
+    """
+    Format the tool label using the function's parameters.
+    
+    Args:
+        func: The tool function
+        tool_input: Input parameters for the tool
+        
+    Returns:
+        Formatted label string or None if no label is defined
+    """
+    if not hasattr(func, '_tool_meta') or 'label' not in func._tool_meta:
+        return None
+    
+    # Get the label template
+    label_template = func._tool_meta['label']
+    
+    # Format the label with the parameters
+    try:
+        formatter = ToolMetaFormatter()
+        return formatter.format(label_template, **tool_input)
+    except Exception:
+        return f"{func.__name__}"

janito/tools/delete_file.py

@@ -1,44 +1,65 @@
-"""
-Tool for deleting files through the claudine agent.
-"""
-import os
-from pathlib import Path
-from typing import Dict, Any, Tuple
-from janito.config import get_config
-from janito.tools.str_replace_editor.utils import normalize_path
-from janito.tools.decorators import tool_meta
-
-
-@tool_meta(label="Deleting file {file_path}")
-def delete_file(
-    file_path: str,
-) -> Tuple[str, bool]:
-    """
-    Delete an existing file.
-    
-    Args:
-        file_path: Path to the file to delete, relative to the workspace directory
-        
-    Returns:
-        A tuple containing (message, is_error)
-    """
-    # Normalize the file path
-    path = normalize_path(file_path)
-    
-    # Convert to Path object for better path handling
-    path_obj = Path(path)
-    
-    # Check if the file exists
-    if not path_obj.exists():
-        return (f"File {path} does not exist.", True)
-    
-    # Check if it's a directory
-    if path_obj.is_dir():
-        return (f"{path} is a directory, not a file. Use delete_directory for directories.", True)
-    
-    # Delete the file
-    try:
-        path_obj.unlink()
-        return (f"Successfully deleted file {path}", False)
-    except Exception as e:
-        return (f"Error deleting file {path}: {str(e)}", True)
+"""
+Tool for deleting files through the claudine agent.
+"""
+from pathlib import Path
+from typing import Tuple
+from janito.tools.str_replace_editor.utils import normalize_path
+from janito.tools.rich_console import print_info, print_success, print_error
+from janito.tools.usage_tracker import track_usage, get_tracker
+
+
+@track_usage('files_deleted')
+def delete_file(
+    file_path: str,
+) -> Tuple[str, bool]:
+    """
+    Delete an existing file.
+    
+    Args:
+        file_path: Path to the file to delete, relative to the workspace directory
+        
+    Returns:
+        A tuple containing (message, is_error)
+    """
+    print_info(f"Deleting file {file_path}", "Delete Operation")
+    # Store the original path for display purposes
+    original_path = file_path
+    
+    # Normalize the file path (converts to absolute path)
+    path = normalize_path(file_path)
+    
+    # Convert to Path object for better path handling
+    path_obj = Path(path)
+    
+    # Check if the file exists
+    if not path_obj.exists():
+        error_msg = f"File {original_path} does not exist."
+        print_error(error_msg, "Error")
+        return (error_msg, True)
+    
+    # Check if it's a directory
+    if path_obj.is_dir():
+        error_msg = f"{original_path} is a directory, not a file. Use delete_directory for directories."
+        print_error(error_msg, "Error")
+        return (error_msg, True)
+    
+    # Delete the file
+    try:
+        # Count the number of lines in the file before deleting
+        try:
+            with open(path_obj, 'r', encoding='utf-8') as f:
+                line_count = len(f.readlines())
+            # Track negative line delta for deleted file
+            get_tracker().increment('lines_delta', -line_count)
+        except Exception:
+            # If we can't read the file, we can't count lines
+            pass
+            
+        path_obj.unlink()
+        success_msg = f"Successfully deleted file {original_path}"
+        print_success("", "Success")
+        return (success_msg, False)
+    except Exception as e:
+        error_msg = f"Error deleting file {original_path}: {str(e)}"
+        print_error(error_msg, "Error")
+        return (error_msg, True)

janito/tools/fetch_webpage/__init__.py

@@ -0,0 +1,34 @@
+"""
+Webpage Content Extractor Package
+
+A comprehensive tool for extracting clean, relevant content from web pages
+for processing with LLMs. Features include:
+- General content extraction with multiple methods
+- Specialized handling for news aggregator sites
+- Targeted extraction based on specific search strings
+- Chunking for large content
+- Structured content extraction
+
+Dependencies:
+- requests
+- beautifulsoup4
+- trafilatura
+- newspaper3k
+
+Author: Claude (Anthropic)
+"""
+
+from janito.tools.fetch_webpage.core import fetch_webpage, fetch_and_extract
+from janito.tools.fetch_webpage.news import fetch_and_extract_news_aggregator
+from janito.tools.fetch_webpage.extractors import extract_clean_text, extract_targeted_content, extract_structured_content
+from janito.tools.fetch_webpage.chunking import chunk_large_content
+
+__all__ = [
+    'fetch_webpage',
+    'fetch_and_extract',
+    'fetch_and_extract_news_aggregator',
+    'extract_clean_text',
+    'extract_targeted_content',
+    'extract_structured_content',
+    'chunk_large_content'
+]

janito/tools/fetch_webpage/chunking.py

@@ -0,0 +1,76 @@
+"""
+Functions for chunking large content into manageable pieces.
+"""
+
+from typing import List
+from janito.tools.rich_console import print_info, print_success
+
+
+def chunk_large_content(text: str, chunk_size: int = 4000, overlap: int = 500) -> List[str]:
+    """
+    Split very large text content into manageable chunks suitable for LLM processing.
+    
+    Args:
+        text: The text to chunk
+        chunk_size: Target size for each chunk in characters
+        overlap: Number of characters to overlap between chunks
+        
+    Returns:
+        List of text chunks
+    """
+    if not text or len(text) <= chunk_size:
+        return [text] if text else []
+    
+    print_info(f"Chunking {len(text)} characters of text into ~{chunk_size} character chunks", "Content Chunking")
+    
+    # Try to split on paragraph breaks first
+    paragraphs = text.split('\n\n')
+    chunks = []
+    current_chunk = ""
+    
+    for para in paragraphs:
+        # If adding this paragraph would exceed chunk size
+        if len(current_chunk) + len(para) + 2 > chunk_size:
+            # If current chunk is not empty, add it to chunks
+            if current_chunk:
+                chunks.append(current_chunk)
+                # Start new chunk with overlap from previous chunk
+                if overlap > 0 and len(current_chunk) > overlap:
+                    current_chunk = current_chunk[-overlap:] + "\n\n" + para
+                else:
+                    current_chunk = para
+            else:
+                # If paragraph itself is bigger than chunk size, split it
+                if len(para) > chunk_size:
+                    words = para.split()
+                    temp_chunk = ""
+                    for word in words:
+                        if len(temp_chunk) + len(word) + 1 > chunk_size:
+                            chunks.append(temp_chunk)
+                            # Start new chunk with overlap
+                            if overlap > 0 and len(temp_chunk) > overlap:
+                                temp_chunk = temp_chunk[-overlap:] + " " + word
+                            else:
+                                temp_chunk = word
+                        else:
+                            if temp_chunk:
+                                temp_chunk += " " + word
+                            else:
+                                temp_chunk = word
+                    if temp_chunk:
+                        current_chunk = temp_chunk
+                else:
+                    chunks.append(para)
+        else:
+            # Add paragraph to current chunk
+            if current_chunk:
+                current_chunk += "\n\n" + para
+            else:
+                current_chunk = para
+    
+    # Don't forget the last chunk
+    if current_chunk:
+        chunks.append(current_chunk)
+    
+    print_success(f"Text chunked into {len(chunks)} segments", "Content Chunking")
+    return chunks

janito/tools/fetch_webpage/core.py

@@ -0,0 +1,155 @@
+"""
+Core functionality for fetching web pages and extracting content.
+"""
+
+import requests
+from typing import Tuple, List, Optional
+from urllib.parse import urlparse
+from janito.tools.rich_console import print_info, print_success, print_error, print_warning
+from janito.tools.usage_tracker import track_usage
+
+from janito.tools.fetch_webpage.extractors import extract_clean_text
+# Import moved to fetch_and_extract function to avoid circular imports
+from janito.tools.fetch_webpage.utils import SITE_SPECIFIC_STRATEGIES
+
+
+@track_usage('web_requests')
+def fetch_webpage(url: str, headers: dict = None, timeout: int = 30, max_size: int = 5000000, 
+                 target_strings: List[str] = None) -> Tuple[str, bool]:
+    """
+    Fetch the content of a web page from a given URL.
+    
+    Args:
+        url: The URL of the web page to fetch
+        headers: Optional HTTP headers to include in the request (default: None)
+        timeout: Request timeout in seconds (default: 30)
+        max_size: Maximum size in bytes to download (default: 5MB)
+        target_strings: Optional list of strings to target specific content sections
+        
+    Returns:
+        A tuple containing (message, is_error)
+    """
+    print_info(f"Fetching content from URL: {url}", "Web Fetch")
+    
+    try:
+        # Set default headers if none provided
+        if headers is None:
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+            }
+        
+        # Make the HTTP request with streaming enabled
+        response = requests.get(url, headers=headers, timeout=timeout, stream=True)
+        
+        # Raise an exception for HTTP errors
+        response.raise_for_status()
+        
+        # Check content length before downloading fully
+        content_length = response.headers.get('Content-Length')
+        if content_length and int(content_length) > max_size:
+            warning_msg = f"Web Fetch: Content size ({int(content_length)/1000000:.1f}MB) exceeds max size ({max_size/1000000:.1f}MB). Aborting download."
+            print_warning(warning_msg)
+            return warning_msg, True
+            
+        # Download content with size limit
+        content_bytes = b''
+        for chunk in response.iter_content(chunk_size=1024 * 1024):  # 1MB chunks
+            content_bytes += chunk
+            if len(content_bytes) > max_size:
+                warning_msg = f"Web Fetch: Download exceeded max size ({max_size/1000000:.1f}MB). Truncating."
+                print_warning(warning_msg)
+                break
+                
+        # Get the content
+        content = content_bytes.decode('utf-8', errors='replace')
+        
+        # If target strings are provided, extract only the relevant sections
+        if target_strings and len(target_strings) > 0:
+            print_info(f"Targeting specific content using {len(target_strings)} search strings", "Web Fetch")
+            from janito.tools.fetch_webpage.extractors import extract_targeted_content
+            targeted_content = extract_targeted_content(content, target_strings)
+            
+            if targeted_content:
+                print_success(f"Successfully targeted specific content based on search strings", "Web Fetch")
+                # Create a summary with first 300 chars of targeted content
+                content_preview = targeted_content[:300] + "..." if len(targeted_content) > 300 else targeted_content
+                summary = f"Successfully fetched targeted content from {url}\n\nContent preview:\n{content_preview}"
+                print_success(f"Successfully fetched targeted content from {url} ({len(targeted_content)} bytes)", "Web Fetch")
+                return targeted_content, False
+            else:
+                print_warning(f"Web Fetch: Could not find content matching the target strings. Returning full content.")
+        
+        # Create a summary message with first 300 chars of content
+        content_preview = content[:300] + "..." if len(content) > 300 else content
+        
+        print_success(f"({len(content)} bytes)", "Web Fetch")
+        
+        # Return the full content
+        return content, False
+        
+    except requests.exceptions.RequestException as e:
+        error_msg = f"Error fetching web page: {str(e)}"
+        print_error(error_msg, "Web Fetch Error")
+        return error_msg, True
+
+
+@track_usage('web_content')
+def fetch_and_extract(url: str, extract_method: str = 'trafilatura', 
+                     max_length: int = 10000,
+                     target_strings: List[str] = None) -> Tuple[str, bool]:
+    """
+    Fetch a webpage and extract its main content in a format suitable for LLM processing.
+    
+    Args:
+        url: The URL to fetch
+        extract_method: Content extraction method ('trafilatura', 'newspaper', 'beautifulsoup', 'all')
+        max_length: Maximum length of text to return
+        target_strings: Optional list of strings to target specific content sections
+        
+    Returns:
+        A tuple containing (extracted_content, is_error)
+    """
+    # Check if this is a news aggregator site that needs special handling
+    domain = urlparse(url).netloc
+    for site_domain in SITE_SPECIFIC_STRATEGIES.keys():
+        if site_domain in domain:
+            print_info(f"Detected news aggregator site: {domain}. Using specialized extraction.", "Content Extraction")
+            # Import here to avoid circular imports
+            from janito.tools.fetch_webpage.news import fetch_and_extract_news_aggregator
+            return fetch_and_extract_news_aggregator(url)
+    
+    # If target strings are provided, pass them directly to fetch_webpage for efficiency
+    if target_strings and len(target_strings) > 0:
+        html_content, is_error = fetch_webpage(url, target_strings=target_strings)
+    else:
+        html_content, is_error = fetch_webpage(url)
+    
+    if is_error:
+        return html_content, True
+    
+    extracted_text = extract_clean_text(html_content, method=extract_method, url=url)
+    
+    if not extracted_text or len(extracted_text) < 100:
+        return f"Could not extract meaningful content from {url}", True
+        
+    # If target strings were provided but not already handled by fetch_webpage
+    if target_strings and len(target_strings) > 0 and not any(target in extracted_text for target in target_strings if len(target) > 3):
+        from janito.tools.fetch_webpage.extractors import extract_targeted_content
+        targeted_content = extract_targeted_content(html_content, target_strings)
+        if targeted_content:
+            print_success(f"Successfully extracted targeted content based on {len(target_strings)} search strings", 
+                         "Targeted Extraction")
+            extracted_text = targeted_content
+    
+    # Truncate if needed
+    if len(extracted_text) > max_length:
+        print_info(f"Truncating content from {len(extracted_text)} to {max_length} characters", "Content Extraction")
+        extracted_text = extracted_text[:max_length] + "..."
+        
+    # Check if the content is still too large for an LLM (rough estimate)
+    estimated_tokens = len(extracted_text.split())
+    if estimated_tokens > 10000:  # Conservative estimate for token limits
+        print_warning(f"Content Extraction: Extracted content still very large (~{estimated_tokens} words). Consider using chunk_large_content()")
+    
+    print_success(f"Successfully extracted {len(extracted_text)} characters of content", "Content Extraction")
+    return extracted_text, False