aiecs 1.0.1__py3-none-any.whl → 1.7.6__py3-none-any.whl

aiecs/tools/task_tools/scraper_tool.py

@@ -2,23 +2,24 @@ import os
 import json
 import time
 import logging
-import asyncio
 import tempfile
 import subprocess
-from typing import Dict, Any, List, Optional, Union, Tuple
+from typing import Dict, Any, List, Optional, Tuple, Union
+import csv
 from enum import Enum
-from urllib.parse import urlparse, urljoin
 
 import httpx
 from bs4 import BeautifulSoup
 from urllib import request as urllib_request
-from pydantic import BaseModel, ValidationError, ConfigDict
-from pydantic_settings import BaseSettings
+from pydantic import BaseModel, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from aiecs.tools.base_tool import BaseTool
 from aiecs.tools import register_tool
 
 # Enums for configuration options
+
+
 class HttpMethod(str, Enum):
     GET = "get"
     POST = "post"
@@ -28,12 +29,14 @@ class HttpMethod(str, Enum):
     OPTIONS = "options"
     PATCH = "patch"
 
+
 class ContentType(str, Enum):
     HTML = "html"
     JSON = "json"
     TEXT = "text"
     BINARY = "binary"
 
+
 class OutputFormat(str, Enum):
     TEXT = "text"
     JSON = "json"
@@ -41,68 +44,44 @@ class OutputFormat(str, Enum):
     MARKDOWN = "markdown"
     CSV = "csv"
 
+
 class RenderEngine(str, Enum):
     NONE = "none"
     PLAYWRIGHT = "playwright"
 
-# Global settings
-class ScraperSettings(BaseSettings):
-    """
-    Configuration for ScraperTool.
-
-    Attributes:
-        user_agent (str): User agent for HTTP requests.
-        max_content_length (int): Maximum content length in bytes.
-        output_dir (str): Directory for output files.
-        scrapy_command (str): Command to run Scrapy.
-        allowed_domains (List[str]): Allowed domains for scraping.
-        blocked_domains (List[str]): Blocked domains for scraping.
-        playwright_available (bool): Whether Playwright is available.
-        env_prefix (str): Environment variable prefix.
-    """
-    user_agent: str = "PythonMiddlewareScraper/2.0"
-    max_content_length: int = 10 * 1024 * 1024  # 10MB
-    output_dir: str = os.path.join(tempfile.gettempdir(), 'scraper_outputs')
-    scrapy_command: str = "scrapy"
-    allowed_domains: List[str] = []
-    blocked_domains: List[str] = []
-    playwright_available: bool = False
-    env_prefix: str = "SCRAPER_TOOL_"
-
-    model_config = ConfigDict(env_prefix="SCRAPER_TOOL_")
 
 # Exceptions
 class ScraperToolError(Exception):
     """Base exception for ScraperTool errors."""
-    pass
+
 
 class HttpError(ScraperToolError):
     """Raised when HTTP requests fail."""
-    pass
+
 
 class TimeoutError(ScraperToolError):
     """Raised when operations time out."""
-    pass
+
 
 class RateLimitError(ScraperToolError):
     """Raised when rate limits are exceeded."""
-    pass
+
 
 class ParsingError(ScraperToolError):
     """Raised when HTML parsing fails."""
-    pass
+
 
 class RenderingError(ScraperToolError):
     """Raised when rendering fails."""
-    pass
+
 
 class ExternalToolError(ScraperToolError):
     """Raised when external tools fail."""
-    pass
+
 
 class FileOperationError(ScraperToolError):
     """Raised when file operations fail."""
-    pass
+
 
 @register_tool("scraper")
 class ScraperTool(BaseTool):
@@ -117,65 +96,128 @@ class ScraperTool(BaseTool):
     - Scrapy integration for advanced crawling
     - Output in various formats: text, JSON, HTML, Markdown, CSV
     """
-    def __init__(self, config: Optional[Dict] = None):
+
+    # Configuration schema
+    class Config(BaseSettings):
+        """Configuration for the scraper tool
+        
+        Automatically reads from environment variables with SCRAPER_TOOL_ prefix.
+        Example: SCRAPER_TOOL_USER_AGENT -> user_agent
+        """
+
+        model_config = SettingsConfigDict(env_prefix="SCRAPER_TOOL_")
+
+        user_agent: str = Field(
+            default="PythonMiddlewareScraper/2.0",
+            description="User agent for HTTP requests",
+        )
+        max_content_length: int = Field(
+            default=10 * 1024 * 1024,
+            description="Maximum content length in bytes",
+        )
+        output_dir: str = Field(
+            default=os.path.join(tempfile.gettempdir(), "scraper_outputs"),
+            description="Directory for output files",
+        )
+        scrapy_command: str = Field(default="scrapy", description="Command to run Scrapy")
+        allowed_domains: List[str] = Field(default=[], description="Allowed domains for scraping")
+        blocked_domains: List[str] = Field(default=[], description="Blocked domains for scraping")
+        playwright_available: bool = Field(
+            default=False,
+            description="Whether Playwright is available (auto-detected)",
+        )
+
+    # Schema definitions
+    class Get_httpxSchema(BaseModel):
+        """Schema for get_httpx operation"""
+
+        url: str = Field(description="URL to scrape")
+        method: HttpMethod = Field(default=HttpMethod.GET, description="HTTP method to use: GET, POST, PUT, DELETE, HEAD, OPTIONS, or PATCH")
+        params: Optional[Dict[str, str]] = Field(default=None, description="Optional query parameters as dictionary")
+        data: Optional[Dict[str, Any]] = Field(default=None, description="Optional form data as dictionary. Mutually exclusive with json_data")
+        json_data: Optional[Dict[str, Any]] = Field(default=None, description="Optional JSON data as dictionary. Mutually exclusive with data")
+        cookies: Optional[Dict[str, str]] = Field(default=None, description="Optional cookies as dictionary")
+        auth: Optional[Tuple[str, str]] = Field(default=None, description="Optional authentication credentials as (username, password) tuple")
+        verify_ssl: Optional[bool] = Field(default=None, description="Optional SSL certificate verification. If None, defaults to True")
+        allow_redirects: bool = Field(default=True, description="Whether to allow HTTP redirects")
+        content_type: ContentType = Field(default=ContentType.TEXT, description="Expected content type: TEXT, JSON, HTML, or BINARY")
+        headers: Optional[Dict[str, str]] = Field(default=None, description="Optional custom HTTP headers as dictionary")
+        output_format: Optional[OutputFormat] = Field(default=None, description="Optional output format for saving: TEXT, JSON, HTML, MARKDOWN, or CSV")
+        output_path: Optional[str] = Field(default=None, description="Optional path to save output file. Requires output_format to be specified")
+        async_mode: bool = Field(default=True, description="Whether to use async HTTP client. If False, uses synchronous client")
+
+    class Parse_htmlSchema(BaseModel):
+        """Schema for parse_html operation"""
+
+        html: str = Field(description="HTML content string to parse")
+        selector: str = Field(description="CSS selector or XPath expression to find elements")
+        selector_type: str = Field(default="css", description="Selector type: 'css' for CSS selectors or 'xpath' for XPath expressions")
+        extract_attr: Optional[str] = Field(default=None, description="Optional attribute name to extract from matched elements (e.g., 'href', 'src')")
+        extract_text: bool = Field(default=True, description="Whether to extract text content from matched elements. Ignored if extract_attr is specified")
+
+    def __init__(self, config: Optional[Dict] = None, **kwargs):
         """
         Initialize ScraperTool with settings and resources.
 
         Args:
-            config (Dict, optional): Configuration overrides for ScraperSettings.
+            config (Dict, optional): Configuration overrides for ScraperTool.
+            **kwargs: Additional arguments passed to BaseTool (e.g., tool_name)
 
         Raises:
             ValueError: If config contains invalid settings.
+
+        Configuration is automatically loaded by BaseTool from:
+        1. Explicit config dict (highest priority)
+        2. YAML config files (config/tools/scraper.yaml)
+        3. Environment variables (via dotenv from .env files)
+        4. Tool defaults (lowest priority)
         """
-        super().__init__(config)
-        self.settings = ScraperSettings()
-        if config:
-            try:
-                self.settings = self.settings.model_validate({**self.settings.model_dump(), **config})
-            except ValidationError as e:
-                raise ValueError(f"Invalid settings: {e}")
+        super().__init__(config, **kwargs)
+
+        # Configuration is automatically loaded by BaseTool into self._config_obj
+        # Access config via self._config_obj (BaseSettings instance)
+        self.config = self._config_obj if self._config_obj else self.Config()
+
         self.logger = logging.getLogger(__name__)
         if not self.logger.handlers:
             handler = logging.StreamHandler()
-            handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
+            handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
             self.logger.addHandler(handler)
         self.logger.setLevel(logging.INFO)
-        os.makedirs(self.settings.output_dir, exist_ok=True)
+        os.makedirs(self.config.output_dir, exist_ok=True)
         self._check_external_tools()
 
     def _check_external_tools(self):
         """Check if external tools are available."""
         try:
-            import playwright
-            self.settings.playwright_available = True
+            self.config.playwright_available = True
         except ImportError:
-            self.settings.playwright_available = False
-
+            self.config.playwright_available = False
 
     async def _save_output(self, content: Any, path: str, format: OutputFormat) -> None:
         """Save content to file in the specified format."""
         try:
             os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
             if format == OutputFormat.TEXT:
-                with open(path, 'w', encoding='utf-8') as f:
+                with open(path, "w", encoding="utf-8") as f:
                     if isinstance(content, dict):
                         f.write(json.dumps(content, indent=2))
                     else:
                         f.write(str(content))
             elif format == OutputFormat.JSON:
-                with open(path, 'w', encoding='utf-8') as f:
+                with open(path, "w", encoding="utf-8") as f:
                     if isinstance(content, dict):
                         json.dump(content, f, indent=2)
                     else:
                         json.dump({"content": content}, f, indent=2)
             elif format == OutputFormat.HTML:
-                with open(path, 'w', encoding='utf-8') as f:
-                    if isinstance(content, dict) and 'html' in content:
-                        f.write(content['html'])
+                with open(path, "w", encoding="utf-8") as f:
+                    if isinstance(content, dict) and "html" in content:
+                        f.write(content["html"])
                     else:
                         f.write(str(content))
             elif format == OutputFormat.MARKDOWN:
-                with open(path, 'w', encoding='utf-8') as f:
+                with open(path, "w", encoding="utf-8") as f:
                     if isinstance(content, dict):
                         f.write("# Scraper Results\n\n")
                         for key, value in content.items():
@@ -186,7 +228,9 @@ class ScraperTool(BaseTool):
                         f.write(str(content))
             elif format == OutputFormat.CSV:
                 import csv
-                with open(path, 'w', newline='', encoding='utf-8') as f:
+
+                with open(path, "w", newline="", encoding="utf-8") as f:
+                    writer: Union[Any, Any]  # csv.writer or csv.DictWriter instance
                     if isinstance(content, dict):
                         writer = csv.writer(f)
                         writer.writerow(content.keys())
@@ -203,7 +247,23 @@ class ScraperTool(BaseTool):
         except Exception as e:
             raise FileOperationError(f"Error saving output: {str(e)}")
 
-    async def get_httpx(self, url: str, method: HttpMethod = HttpMethod.GET, params: Optional[Dict[str, str]] = None, data: Optional[Dict[str, Any]] = None, json_data: Optional[Dict[str, Any]] = None, cookies: Optional[Dict[str, str]] = None, auth: Optional[Tuple[str, str]] = None, verify_ssl: Optional[bool] = None, allow_redirects: bool = True, content_type: ContentType = ContentType.TEXT, headers: Optional[Dict[str, str]] = None, output_format: Optional[OutputFormat] = None, output_path: Optional[str] = None, async_mode: bool = True) -> Any:
+    async def get_httpx(
+        self,
+        url: str,
+        method: HttpMethod = HttpMethod.GET,
+        params: Optional[Dict[str, str]] = None,
+        data: Optional[Dict[str, Any]] = None,
+        json_data: Optional[Dict[str, Any]] = None,
+        cookies: Optional[Dict[str, str]] = None,
+        auth: Optional[Tuple[str, str]] = None,
+        verify_ssl: Optional[bool] = None,
+        allow_redirects: bool = True,
+        content_type: ContentType = ContentType.TEXT,
+        headers: Optional[Dict[str, str]] = None,
+        output_format: Optional[OutputFormat] = None,
+        output_path: Optional[str] = None,
+        async_mode: bool = True,
+    ) -> Any:
         """
         Execute HTTP request using httpx library (supports both sync and async).
 
@@ -231,21 +291,21 @@ class ScraperTool(BaseTool):
         """
         try:
             headers = headers or {}
-            if 'User-Agent' not in headers:
-                headers['User-Agent'] = self.settings.user_agent
-            kwargs = {
-                'params': params,
-                'headers': headers,
-                'follow_redirects': allow_redirects,
+            if "User-Agent" not in headers:
+                headers["User-Agent"] = self.config.user_agent
+            kwargs: Dict[str, Any] = {
+                "params": params,
+                "headers": headers,
+                "follow_redirects": allow_redirects,
             }
             if auth:
-                kwargs['auth'] = auth
+                kwargs["auth"] = auth  # httpx accepts Tuple[str, str] for auth
             if cookies:
-                kwargs['cookies'] = cookies
+                kwargs["cookies"] = cookies
             if json_data:
-                kwargs['json'] = json_data
+                kwargs["json"] = json_data
             elif data:
-                kwargs['data'] = data
+                kwargs["data"] = data
 
             if async_mode:
                 async with httpx.AsyncClient(verify=verify_ssl if verify_ssl is not None else True) as client:
@@ -260,30 +320,47 @@ class ScraperTool(BaseTool):
                 resp.raise_for_status()
             except httpx.HTTPStatusError as e:
                 raise HttpError(f"HTTP {e.response.status_code}: {e.response.reason_phrase} for {url}")
-            
-            if len(resp.content) > self.settings.max_content_length:
+
+            if len(resp.content) > self.config.max_content_length:
                 raise HttpError(f"Response content too large: {len(resp.content)} bytes")
 
             if content_type == ContentType.JSON:
                 result = resp.json()
             elif content_type == ContentType.HTML:
-                result = {'html': resp.text, 'url': str(resp.url), 'status': resp.status_code}
+                result = {
+                    "html": resp.text,
+                    "url": str(resp.url),
+                    "status": resp.status_code,
+                }
             elif content_type == ContentType.BINARY:
-                result = {'content': resp.content, 'url': str(resp.url), 'status': resp.status_code}
+                result = {
+                    "content": resp.content,
+                    "url": str(resp.url),
+                    "status": resp.status_code,
+                }
             else:
                 result = resp.text
 
             if output_format and output_path:
                 await self._save_output(result, output_path, output_format)
                 if isinstance(result, dict):
-                    result['saved_to'] = output_path
+                    result["saved_to"] = output_path
                 else:
-                    result = {'content': result, 'saved_to': output_path}
+                    result = {"content": result, "saved_to": output_path}
             return result
         except httpx.RequestError as e:
             raise HttpError(f"Request failed: {str(e)}")
 
-    async def get_urllib(self, url: str, method: HttpMethod = HttpMethod.GET, data: Optional[Dict[str, Any]] = None, content_type: ContentType = ContentType.TEXT, headers: Optional[Dict[str, str]] = None, output_format: Optional[OutputFormat] = None, output_path: Optional[str] = None) -> Any:
+    async def get_urllib(
+        self,
+        url: str,
+        method: HttpMethod = HttpMethod.GET,
+        data: Optional[Dict[str, Any]] = None,
+        content_type: ContentType = ContentType.TEXT,
+        headers: Optional[Dict[str, str]] = None,
+        output_format: Optional[OutputFormat] = None,
+        output_path: Optional[str] = None,
+    ) -> Any:
         """
         Execute HTTP request using urllib.
 
@@ -305,10 +382,10 @@ class ScraperTool(BaseTool):
         try:
             import urllib.parse
             import urllib.error
-            
+
             headers = headers or {}
-            if 'User-Agent' not in headers:
-                headers['User-Agent'] = self.settings.user_agent
+            if "User-Agent" not in headers:
+                headers["User-Agent"] = self.config.user_agent
             data_bytes = None
             if data:
                 data_bytes = urllib.parse.urlencode(data).encode()
@@ -316,42 +393,122 @@ class ScraperTool(BaseTool):
                 str(url),
                 data=data_bytes,
                 headers=headers,
-                method=method.value.upper()
+                method=method.value.upper(),
             )
             with urllib_request.urlopen(req) as resp:
-                content_length = resp.getheader('Content-Length')
-                if content_length and int(content_length) > self.settings.max_content_length:
+                content_length = resp.getheader("Content-Length")
+                if content_length and int(content_length) > self.config.max_content_length:
                     raise HttpError(f"Response content too large: {content_length} bytes")
                 content = resp.read()
-                charset = resp.headers.get_content_charset() or 'utf-8'
+                charset = resp.headers.get_content_charset() or "utf-8"
                 if content_type == ContentType.JSON:
-                    result = json.loads(content.decode(charset, errors='ignore'))
+                    result = json.loads(content.decode(charset, errors="ignore"))
                 elif content_type == ContentType.HTML:
-                    result = {'html': content.decode(charset, errors='ignore'), 'url': resp.url, 'status': resp.status}
+                    result = {
+                        "html": content.decode(charset, errors="ignore"),
+                        "url": resp.url,
+                        "status": resp.status,
+                    }
                 elif content_type == ContentType.BINARY:
-                    result = {'content': content, 'url': resp.url, 'status': resp.status}
+                    result = {
+                        "content": content,
+                        "url": resp.url,
+                        "status": resp.status,
+                    }
                 else:
-                    result = content.decode(charset, errors='ignore')
+                    result = content.decode(charset, errors="ignore")
                 if output_format and output_path:
                     await self._save_output(result, output_path, output_format)
                     if isinstance(result, dict):
-                        result['saved_to'] = output_path
+                        result["saved_to"] = output_path
                     else:
-                        result = {'content': result, 'saved_to': output_path}
+                        result = {"content": result, "saved_to": output_path}
                 return result
         except urllib.error.URLError as e:
             raise HttpError(f"Request failed: {str(e)}")
 
     # Legacy method names for backward compatibility
-    async def get_requests(self, url: str, method: HttpMethod = HttpMethod.GET, params: Optional[Dict[str, str]] = None, data: Optional[Dict[str, Any]] = None, json_data: Optional[Dict[str, Any]] = None, cookies: Optional[Dict[str, str]] = None, auth: Optional[Tuple[str, str]] = None, verify_ssl: Optional[bool] = None, allow_redirects: bool = True, content_type: ContentType = ContentType.TEXT, headers: Optional[Dict[str, str]] = None, output_format: Optional[OutputFormat] = None, output_path: Optional[str] = None) -> Any:
+    async def get_requests(
+        self,
+        url: str,
+        method: HttpMethod = HttpMethod.GET,
+        params: Optional[Dict[str, str]] = None,
+        data: Optional[Dict[str, Any]] = None,
+        json_data: Optional[Dict[str, Any]] = None,
+        cookies: Optional[Dict[str, str]] = None,
+        auth: Optional[Tuple[str, str]] = None,
+        verify_ssl: Optional[bool] = None,
+        allow_redirects: bool = True,
+        content_type: ContentType = ContentType.TEXT,
+        headers: Optional[Dict[str, str]] = None,
+        output_format: Optional[OutputFormat] = None,
+        output_path: Optional[str] = None,
+    ) -> Any:
         """Legacy method - now uses httpx in sync mode."""
-        return await self.get_httpx(url, method, params, data, json_data, cookies, auth, verify_ssl, allow_redirects, content_type, headers, output_format, output_path, async_mode=False)
-
-    async def get_aiohttp(self, url: str, method: HttpMethod = HttpMethod.GET, params: Optional[Dict[str, str]] = None, data: Optional[Dict[str, Any]] = None, json_data: Optional[Dict[str, Any]] = None, cookies: Optional[Dict[str, str]] = None, auth: Optional[Tuple[str, str]] = None, verify_ssl: Optional[bool] = None, allow_redirects: bool = True, content_type: ContentType = ContentType.TEXT, headers: Optional[Dict[str, str]] = None, output_format: Optional[OutputFormat] = None, output_path: Optional[str] = None) -> Any:
+        return await self.get_httpx(
+            url,
+            method,
+            params,
+            data,
+            json_data,
+            cookies,
+            auth,
+            verify_ssl,
+            allow_redirects,
+            content_type,
+            headers,
+            output_format,
+            output_path,
+            async_mode=False,
+        )
+
+    async def get_aiohttp(
+        self,
+        url: str,
+        method: HttpMethod = HttpMethod.GET,
+        params: Optional[Dict[str, str]] = None,
+        data: Optional[Dict[str, Any]] = None,
+        json_data: Optional[Dict[str, Any]] = None,
+        cookies: Optional[Dict[str, str]] = None,
+        auth: Optional[Tuple[str, str]] = None,
+        verify_ssl: Optional[bool] = None,
+        allow_redirects: bool = True,
+        content_type: ContentType = ContentType.TEXT,
+        headers: Optional[Dict[str, str]] = None,
+        output_format: Optional[OutputFormat] = None,
+        output_path: Optional[str] = None,
+    ) -> Any:
         """Legacy method - now uses httpx in async mode."""
-        return await self.get_httpx(url, method, params, data, json_data, cookies, auth, verify_ssl, allow_redirects, content_type, headers, output_format, output_path, async_mode=True)
-
-    async def render(self, url: str, engine: RenderEngine = RenderEngine.PLAYWRIGHT, wait_time: int = 5, wait_selector: Optional[str] = None, scroll_to_bottom: bool = False, screenshot: bool = False, screenshot_path: Optional[str] = None, headers: Optional[Dict[str, str]] = None, output_format: Optional[OutputFormat] = None, output_path: Optional[str] = None) -> Dict[str, Any]:
+        return await self.get_httpx(
+            url,
+            method,
+            params,
+            data,
+            json_data,
+            cookies,
+            auth,
+            verify_ssl,
+            allow_redirects,
+            content_type,
+            headers,
+            output_format,
+            output_path,
+            async_mode=True,
+        )
+
+    async def render(
+        self,
+        url: str,
+        engine: RenderEngine = RenderEngine.PLAYWRIGHT,
+        wait_time: int = 5,
+        wait_selector: Optional[str] = None,
+        scroll_to_bottom: bool = False,
+        screenshot: bool = False,
+        screenshot_path: Optional[str] = None,
+        headers: Optional[Dict[str, str]] = None,
+        output_format: Optional[OutputFormat] = None,
+        output_path: Optional[str] = None,
+    ) -> Dict[str, Any]:
         """
         Render a web page using a headless browser (Playwright).
 
@@ -375,56 +532,85 @@ class ScraperTool(BaseTool):
         """
         try:
             if engine == RenderEngine.PLAYWRIGHT:
-                if not self.settings.playwright_available:
+                if not self.config.playwright_available:
                     raise RenderingError("Playwright is not available. Install with 'pip install playwright'")
-                result = await self._render_with_playwright(url, wait_time, wait_selector, scroll_to_bottom, screenshot, screenshot_path)
+                result = await self._render_with_playwright(
+                    url,
+                    wait_time,
+                    wait_selector,
+                    scroll_to_bottom,
+                    screenshot,
+                    screenshot_path,
+                )
             else:
                 raise RenderingError(f"Unsupported rendering engine: {engine}. Only PLAYWRIGHT is supported.")
             if output_format and output_path:
                 await self._save_output(result, output_path, output_format)
-                result['saved_to'] = output_path
+                result["saved_to"] = output_path
             return result
         except Exception as e:
             raise RenderingError(f"Failed to render page: {str(e)}")
 
-    async def _render_with_playwright(self, url: str, wait_time: int, wait_selector: Optional[str], scroll_to_bottom: bool, screenshot: bool, screenshot_path: Optional[str]) -> Dict[str, Any]:
+    async def _render_with_playwright(
+        self,
+        url: str,
+        wait_time: int,
+        wait_selector: Optional[str],
+        scroll_to_bottom: bool,
+        screenshot: bool,
+        screenshot_path: Optional[str],
+    ) -> Dict[str, Any]:
         """Render a web page using Playwright with async API."""
         from playwright.async_api import async_playwright
+
         async with async_playwright() as p:
             browser = await p.chromium.launch()
             page = await browser.new_page(
-                user_agent=self.settings.user_agent,
-                viewport={'width': 1280, 'height': 800}
+                user_agent=self.config.user_agent,
+                viewport={"width": 1280, "height": 800},
             )
             try:
                 await page.goto(url)
                 if wait_selector:
                     await page.wait_for_selector(wait_selector)
                 else:
-                    await page.wait_for_load_state('networkidle')
+                    await page.wait_for_load_state("networkidle")
                 if scroll_to_bottom:
-                    await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
+                    await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
                     await page.wait_for_timeout(1000)
                 screenshot_result = None
                 if screenshot:
-                    screenshot_path = screenshot_path or os.path.join(self.settings.output_dir, f"screenshot_{int(time.time())}.png")
-                    os.makedirs(os.path.dirname(os.path.abspath(screenshot_path)), exist_ok=True)
+                    screenshot_path = screenshot_path or os.path.join(
+                        self.config.output_dir,
+                        f"screenshot_{int(time.time())}.png",
+                    )
+                    os.makedirs(
+                        os.path.dirname(os.path.abspath(screenshot_path)),
+                        exist_ok=True,
+                    )
                     await page.screenshot(path=screenshot_path)
                     screenshot_result = screenshot_path
                 html = await page.content()
                 title = await page.title()
                 result = {
-                    'html': html,
-                    'title': title,
-                    'url': page.url,
-                    'screenshot': screenshot_result
+                    "html": html,
+                    "title": title,
+                    "url": page.url,
+                    "screenshot": screenshot_result,
                 }
                 return result
             finally:
                 await browser.close()
 
-
-    def crawl_scrapy(self, project_path: str, spider_name: str, output_path: str, spider_args: Optional[Dict[str, str]] = None, headers: Optional[Dict[str, str]] = None, output_format: Optional[OutputFormat] = None) -> Dict[str, Any]:
+    def crawl_scrapy(
+        self,
+        project_path: str,
+        spider_name: str,
+        output_path: str,
+        spider_args: Optional[Dict[str, str]] = None,
+        headers: Optional[Dict[str, str]] = None,
+        output_format: Optional[OutputFormat] = None,
+    ) -> Dict[str, Any]:
         """
         Execute a Scrapy spider in an existing project and output results to a file.
 
@@ -447,21 +633,25 @@ class ScraperTool(BaseTool):
             start_time = time.time()
             os.makedirs(os.path.dirname(os.path.abspath(output_path)), exist_ok=True)
             cmd = [
-                self.settings.scrapy_command,
-                'crawl', spider_name,
-                '-o', output_path,
-                '-s', f'USER_AGENT={self.settings.user_agent}',
-                '-s', 'LOG_LEVEL=INFO'
+                self.config.scrapy_command,
+                "crawl",
+                spider_name,
+                "-o",
+                output_path,
+                "-s",
+                f"USER_AGENT={self.config.user_agent}",
+                "-s",
+                "LOG_LEVEL=INFO",
             ]
             if spider_args:
                 for k, v in spider_args.items():
-                    cmd += ['-a', f"{k}={v}"]
+                    cmd += ["-a", f"{k}={v}"]
             process = subprocess.run(
                 cmd,
                 cwd=project_path,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
-                text=True
+                text=True,
             )
             if process.returncode != 0:
                 error_msg = process.stderr.strip()
@@ -470,19 +660,26 @@ class ScraperTool(BaseTool):
                 raise ExternalToolError(f"Scrapy crawl did not create output file: {output_path}")
             file_size = os.path.getsize(output_path)
             result = {
-                'output_path': output_path,
-                'execution_time': time.time() - start_time,
-                'file_size': file_size,
-                'stdout': process.stdout.strip(),
-                'stderr': process.stderr.strip()
+                "output_path": output_path,
+                "execution_time": time.time() - start_time,
+                "file_size": file_size,
+                "stdout": process.stdout.strip(),
+                "stderr": process.stderr.strip(),
             }
             return result
         except subprocess.TimeoutExpired:
-            raise TimeoutError(f"Scrapy crawl timed out")
+            raise TimeoutError("Scrapy crawl timed out")
         except Exception as e:
             raise ExternalToolError(f"Error running Scrapy: {str(e)}")
 
-    def parse_html(self, html: str, selector: str, selector_type: str = "css", extract_attr: Optional[str] = None, extract_text: bool = True) -> Dict[str, Any]:
+    def parse_html(
+        self,
+        html: str,
+        selector: str,
+        selector_type: str = "css",
+        extract_attr: Optional[str] = None,
+        extract_text: bool = True,
+    ) -> Dict[str, Any]:
         """
         Parse HTML content using BeautifulSoup.
 
@@ -500,36 +697,37 @@ class ScraperTool(BaseTool):
             ParsingError: If parsing fails.
         """
         try:
-            soup = BeautifulSoup(html, 'html.parser')
-            if selector_type == 'css':
+            soup = BeautifulSoup(html, "html.parser")
+            if selector_type == "css":
                 elements = soup.select(selector)
             else:
                 from lxml.html import fromstring
                 from lxml.etree import XPath
+
                 root = fromstring(html)
                 xpath = XPath(selector)
                 elements = xpath(root)
             results = []
             for element in elements:
                 if extract_attr:
-                    value = element.get(extract_attr) if hasattr(element, 'get') else element.get(extract_attr)
+                    value = element.get(extract_attr) if hasattr(element, "get") else element.get(extract_attr)
                     if value is not None:
                         results.append(value)
                 elif extract_text:
-                    if hasattr(element, 'text_content') and callable(getattr(element, 'text_content')):
+                    if hasattr(element, "text_content") and callable(getattr(element, "text_content")):
                         # lxml element
-                        text = element.text_content()
+                        text = element.text_content()  # type: ignore[misc]
                     else:
                         # BeautifulSoup element
-                        text = element.get_text()
-                    
-                    if text and text.strip():
-                        results.append(text.strip())
+                        text = element.get_text()  # type: ignore[misc]
+
+                    if text and text.strip():  # type: ignore[misc]
+                        results.append(text.strip())  # type: ignore[misc]
             return {
-                'selector': selector,
-                'selector_type': selector_type,
-                'count': len(results),
-                'results': results
+                "selector": selector,
+                "selector_type": selector_type,
+                "count": len(results),
+                "results": results,
             }
         except Exception as e:
             raise ParsingError(f"Error parsing HTML: {str(e)}")
@@ -542,7 +740,3 @@ class ScraperTool(BaseTool):
     head = get_httpx
     options = get_httpx
     patch = get_httpx
-    
-    # Legacy method aliases
-    get_requests = get_httpx
-    get_aiohttp = get_httpx