teddy-cli 0.1.0__py3-none-any.whl

teddy_executor/adapters/outbound/web_scraper_adapter.py

@@ -0,0 +1,346 @@
+from teddy_executor.core.ports.outbound.web_scraper import WebScraper
+from teddy_executor.core.ports.outbound.config_service import IConfigService
+
+
+MIN_GITHUB_CONTENT_LENGTH = 10
+HTTP_BAD_REQUEST = 400
+HTTP_FORBIDDEN = 403
+HTTP_NOT_ACCEPTABLE = 406
+HTTP_TOO_MANY_REQUESTS = 429
+HTTP_INTERNAL_SERVER_ERROR = 500
+
+
+class WebScraperAdapter(WebScraper):
+    """
+    An adapter that implements the WebScraper port using requests and trafilatura.
+    """
+
+    def __init__(self, config_service: IConfigService = None):  # type: ignore
+        self._config_service = config_service
+
+    def _get_trafilatura(self):
+        """Lazy-load trafilatura to keep CLI startup fast."""
+        import trafilatura
+
+        return trafilatura
+
+    def _get_bs4(self):
+        """Lazy-load BeautifulSoup to keep CLI startup fast."""
+        from bs4 import BeautifulSoup
+
+        return BeautifulSoup
+
+    def _extract_github_conversation(self, html: str) -> str:
+        """
+        Extracts issue or pull request content and comments from GitHub HTML.
+        Uses a hybrid strategy: JSON-embedded data (primary) and CSS selectors (fallback).
+        """
+        import json
+
+        soup = self._get_bs4()(html, "html.parser")
+
+        # 1. Primary: Try high-fidelity extraction from embedded JSON data
+        scripts = soup.find_all("script", type="application/json")
+        for script in scripts:
+            if not script.string:
+                continue
+            try:
+                data = json.loads(script.string)
+                result = self._parse_github_json(data)
+                if result:
+                    return result
+            except (json.JSONDecodeError, TypeError, AttributeError):
+                continue
+
+        # 2. Fallback: CSS-based scraping
+        return self._scrape_github_html(soup)
+
+    def _find_key_recursive(self, obj, target_key):
+        """Recursively search for a key in a nested dictionary/list."""
+        if isinstance(obj, dict):
+            if target_key in obj:
+                return obj[target_key]
+            for v in obj.values():
+                res = self._find_key_recursive(v, target_key)
+                if res:
+                    return res
+        elif isinstance(obj, list):
+            for item in obj:
+                res = self._find_key_recursive(item, target_key)
+                if res:
+                    return res
+        return None
+
+    def _gather_edges_recursive(self, obj, edges_out: list):
+        """Recursively gather all 'edges' lists into the output list."""
+        if isinstance(obj, dict):
+            if "edges" in obj and isinstance(obj["edges"], list):
+                edges_out.extend(obj["edges"])
+            for v in obj.values():
+                self._gather_edges_recursive(v, edges_out)
+        elif isinstance(obj, list):
+            for i in obj:
+                self._gather_edges_recursive(i, edges_out)
+
+    def _parse_github_json(self, data: dict) -> str | None:
+        """Helper to recursively find and parse issue/PR data from JSON."""
+        container = self._find_key_recursive(data, "issue") or self._find_key_recursive(
+            data, "pullRequest"
+        )
+        if not container or not isinstance(container, dict):
+            return None
+
+        title = (
+            container.get("title")
+            or container.get("titleHtml")
+            or container.get("titleText")
+            or "Unknown Title"
+        )
+        body = container.get("body") or container.get("bodyHTML", "")
+
+        all_edges: list[dict] = []
+        self._gather_edges_recursive(data, all_edges)
+
+        comments = []
+        seen_ids = set()
+        for edge in all_edges:
+            node = edge.get("node", {}) if isinstance(edge, dict) else {}
+            node_id = node.get("id")
+            if node_id and node_id not in seen_ids:
+                if node.get("__typename") in [
+                    "IssueComment",
+                    "PullRequestReview",
+                    "PullRequestReviewComment",
+                ]:
+                    seen_ids.add(node_id)
+                    author = node.get("author", {}).get("login", "unknown")
+                    c_body = node.get("body") or node.get("bodyHTML") or ""
+                    comments.append(
+                        f"### {node.get('__typename')} by {author}\n{c_body}\n\n"
+                    )
+
+        return f"# {title}\n\n## Description\n{body}\n\n" + "".join(comments)
+
+    def _scrape_github_html(self, soup) -> str:
+        """Helper for CSS-based fallback scraping."""
+        title_elem = soup.select_one(".markdown-title") or soup.select_one(
+            ".gh-header-title"
+        )
+        title = title_elem.get_text(strip=True) if title_elem else "GitHub Content"
+
+        bodies = soup.select(".markdown-body")
+        content_blocks = []
+        for i, block in enumerate(bodies):
+            text = block.get_text(separator="\n", strip=True)
+            if len(text) > MIN_GITHUB_CONTENT_LENGTH:
+                label = "Description" if i == 0 else f"Comment {i}"
+                content_blocks.append(f"## {label}\n{text}\n\n")
+
+        return f"# {title}\n\n" + "".join(content_blocks)
+
+    def _is_retryable_error(self, status_code: int | None) -> bool:
+        """Determines if an HTTP error is transient and should be retried."""
+        if not status_code:
+            return True
+        # Retry on 5xx (server errors) or 429 (Too Many Requests)
+        return (
+            status_code >= HTTP_INTERNAL_SERVER_ERROR
+            or status_code == HTTP_TOO_MANY_REQUESTS
+        )
+
+    def _fetch_with_ua(self, url: str, ua: str, max_retries: int) -> str | None:
+        """Internal helper to attempt fetch with a specific User-Agent and retries."""
+        import requests
+        import time
+
+        headers = {
+            "User-Agent": ua,
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Accept-Encoding": "gzip, deflate, br",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+        }
+
+        for attempt in range(max_retries):
+            try:
+                response = requests.get(url, headers=headers, timeout=20)
+                response.raise_for_status()
+                return response.text
+            except requests.exceptions.HTTPError as e:
+                status_code = getattr(e.response, "status_code", None)
+
+                # 403/406 signal a need for UA rotation, not same-UA retry
+                if status_code in [HTTP_FORBIDDEN, HTTP_NOT_ACCEPTABLE]:
+                    return None
+
+                if self._is_retryable_error(status_code) and attempt < max_retries - 1:
+                    time.sleep(2**attempt)
+                    continue
+                raise
+            except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
+                if attempt < max_retries - 1:
+                    time.sleep(2**attempt)
+                    continue
+                return None
+            except Exception:
+                return None
+        return None
+
+    def _fetch_with_rotation(self, url: str) -> str | None:
+        """Attempts to fetch HTML content using a rotating pool of User-Agents and retries."""
+        user_agents = [
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, Gecko) Chrome/124.0.0.0 Safari/537.36",
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, Gecko) Chrome/121.0.0.0 Safari/537.36",
+            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, Gecko) Chrome/122.0.0.0 Safari/537.36",
+            "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, Gecko) Version/17.4.1 Mobile/15E148 Safari/604.1",
+        ]
+
+        max_retries = 3
+        if self._config_service:
+            val = self._config_service.get_setting("research.max_scraper_retries", 3)
+            if val is not None:
+                max_retries = int(val)
+
+        last_error: Exception | None = None
+
+        for ua in user_agents:
+            try:
+                html_content = self._fetch_with_ua(url, ua, max_retries)
+                if html_content:
+                    return html_content
+            except Exception as e:
+                last_error = e
+                continue
+
+        # Final Resort: Trafilatura's internal fetcher
+        html_content = self._get_trafilatura().fetch_url(url)
+        if not html_content and last_error:
+            raise last_error
+
+        return html_content
+
+    def _handle_github_raw(self, url: str) -> str | None:
+        """Handles specialized fetching for GitHub raw content with retries."""
+        import requests
+        import time
+
+        is_raw_github = url.startswith("https://raw.githubusercontent.com/")
+        is_github_blob = url.startswith("https://github.com/") and "/blob/" in url
+
+        if not (is_raw_github or is_github_blob):
+            return None
+
+        target_url = (
+            url.replace("github.com", "raw.githubusercontent.com").replace(
+                "/blob/", "/"
+            )
+            if is_github_blob
+            else url
+        )
+
+        max_retries = 3
+        if self._config_service:
+            val = self._config_service.get_setting("research.max_scraper_retries", 3)
+            if val is not None:
+                max_retries = int(val)
+
+        for attempt in range(max_retries):
+            try:
+                response = requests.get(
+                    target_url,
+                    headers={
+                        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, Gecko) Chrome/124.0.0.0 Safari/537.36"
+                    },
+                    timeout=30,
+                )
+                response.raise_for_status()
+                return response.text
+            except (
+                requests.exceptions.HTTPError,
+                requests.exceptions.ConnectionError,
+                requests.exceptions.Timeout,
+            ) as e:
+                if attempt < max_retries - 1:
+                    status_code = getattr(
+                        getattr(e, "response", None), "status_code", None
+                    )
+                    # Don't retry on most 4xx
+                    if (
+                        status_code
+                        and HTTP_BAD_REQUEST <= status_code < HTTP_INTERNAL_SERVER_ERROR
+                        and status_code not in [HTTP_FORBIDDEN, HTTP_TOO_MANY_REQUESTS]
+                    ):
+                        raise
+                    time.sleep(2**attempt)
+                    continue
+                raise
+        return None
+
+    def _truncate_content(self, markdown_content: str) -> str:
+        """Truncates markdown content based on read.max_lines configuration."""
+        if not self._config_service or not markdown_content:
+            return markdown_content
+
+        max_lines = self._config_service.get_setting("read.max_lines", None)
+        if max_lines is not None:
+            try:
+                max_lines_int = int(max_lines)
+                lines = markdown_content.splitlines()
+                if len(lines) > max_lines_int:
+                    return "\n".join(lines[:max_lines_int])
+            except (ValueError, TypeError):
+                pass
+        return markdown_content
+
+    def get_content(self, url: str, **_kwargs) -> str:
+        """
+        Fetches and extracts the content from the given URL.
+        Employs a multi-stage stealth rotation to bypass automated blocking.
+
+        Args:
+            url: The URL to fetch content from.
+            **_kwargs: Optional extraction hints.
+
+        Returns:
+            The extracted text content.
+        """
+        # 1. Specialized handling for GitHub raw content
+        raw_github_content = self._handle_github_raw(url)
+        if raw_github_content is not None:
+            return raw_github_content
+
+        # 2. Multi-stage Stealth Rotation for general URLs
+        html_content = self._fetch_with_rotation(url)
+        if not html_content:
+            return ""
+
+        # 3. Routing: Use specialized extractor for GitHub Issues and Pull Requests
+        is_github_domain = (
+            "github.com" in url or "localhost" in url or "127.0.0.1" in url
+        )
+        if is_github_domain and ("/issues/" in url or "/pull/" in url):
+            github_content = self._extract_github_conversation(html_content)
+            if github_content:
+                return github_content
+
+        trafilatura = self._get_trafilatura()
+        markdown_content = trafilatura.extract(
+            html_content,
+            output_format="markdown",
+            include_links=True,
+            include_formatting=True,
+            favor_recall=False,
+            include_comments=False,
+            include_tables=True,
+        )
+
+        if not markdown_content:
+            return ""
+
+        return self._truncate_content(markdown_content)

teddy_executor/adapters/outbound/web_searcher_adapter.py

@@ -0,0 +1,122 @@
+import logging
+from typing import Any, Callable, List, Optional
+from teddy_executor.core.domain.models import (
+    QueryResult,
+    SearchResult,
+    WebSearchError,
+    WebSearchResults,
+)
+from teddy_executor.core.ports.outbound import IConfigService, IWebSearcher
+
+
+logger = logging.getLogger(__name__)
+
+
+class WebSearcherAdapter(IWebSearcher):
+    """
+    An adapter that uses the ddgs library to perform web searches.
+    """
+
+    def __init__(
+        self,
+        config_service: IConfigService,
+        ddgs_factory: Optional[Callable[..., Any]] = None,
+    ):
+        self._config_service = config_service
+        self._ddgs_factory = ddgs_factory
+
+    def _apply_ddgs_monkeypatch(self) -> None:
+        """Applies a structural patch to DDGS to preserve word boundaries."""
+        from ddgs.base import BaseSearchEngine
+
+        def patched_extract_results(self, html_text: str):
+            html_text = self.pre_process_html(html_text)
+            tree = self.extract_tree(html_text)
+            items = tree.xpath(self.items_xpath)
+            results = []
+            for item in items:
+                result = self.result_type()
+                for key, value in self.elements_xpath.items():
+                    parts = (x.strip() for x in item.xpath(value))
+                    # JOIN WITH SPACE instead of empty string to preserve boundaries
+                    data = " ".join(" ".join(parts).split())
+                    result.__setattr__(key, data)
+                results.append(result)
+            return results
+
+        # Apply the structural patch to the base class
+        BaseSearchEngine.extract_results = patched_extract_results  # type: ignore[method-assign]
+
+    def _clean_snippet(self, text: str) -> str:
+        """Fixes missing spaces after punctuation in raw text."""
+        import re
+
+        if not text:
+            return ""
+        # Fix missing space after period, comma, or colon followed by a letter
+        return re.sub(r"([.,:])([A-Za-z])", r"\1 \2", text)
+
+    def _execute_single_query(
+        self, ddgs_client: Any, query: str, total_queries: int
+    ) -> QueryResult:
+        """Executes a single search query and maps results."""
+        try:
+            max_results = self._config_service.get_setting("research.max_results", 5)
+            # DDGS.text returns a generator, so we convert it to a list
+            results = list(ddgs_client.text(query, max_results=max_results))
+
+            search_results_for_query: List[SearchResult] = []
+            for res in results:
+                url = res.get("href", "")
+                # Map library 'body' to our 'description'
+                item: SearchResult = {
+                    "title": res.get("title", ""),
+                    "href": url,
+                    "description": self._clean_snippet(res.get("body", "")),
+                }
+
+                search_results_for_query.append(item)
+
+            return {
+                "query": query,
+                "results": search_results_for_query,
+            }
+        except Exception as e:
+            # Log the individual query failure but continue with other queries.
+            # This prevents one failing query from sabotaging the entire action.
+            logger.warning(f"Search query '{query}' failed: {e}")
+
+            # If this is the ONLY query, we still want to raise the error
+            # to maintain failure transparency (Stop the Line).
+            if total_queries == 1:
+                raise WebSearchError(f"Failed to execute search: {e}") from e
+
+            return {
+                "query": query,
+                "results": [],
+            }
+
+    def search(self, queries: List[str]) -> WebSearchResults:
+        """
+        Performs a web search for each query and maps the results.
+        """
+        from ddgs import DDGS
+
+        self._apply_ddgs_monkeypatch()
+        all_query_results: List[QueryResult] = []
+        factory = self._ddgs_factory or DDGS
+
+        # Globally disable logging (CRITICAL and below) to silence noisy
+        # third-party HTTP clients (urllib3, httpx, curl_cffi) used by DDGS.
+        logging.disable(logging.CRITICAL)
+        try:
+            with factory() as ddgs_client:
+                for query in queries:
+                    result = self._execute_single_query(
+                        ddgs_client, query, len(queries)
+                    )
+                    all_query_results.append(result)
+
+            return {"query_results": all_query_results}
+        finally:
+            logging.disable(logging.NOTSET)

teddy_executor/adapters/outbound/yaml_config_adapter.py

@@ -0,0 +1,105 @@
+import os
+from importlib import resources
+from typing import Any, Dict, Optional
+import yaml
+from teddy_executor.core.ports.outbound.config_service import IConfigService
+
+
+class YamlConfigAdapter(IConfigService):
+    """
+    Implements IConfigService by reading configuration from a YAML file.
+    """
+
+    def __init__(
+        self, config_path: str = ".teddy/config.yaml", root_dir: Optional[str] = None
+    ):
+        if root_dir:
+            self._config_path = os.path.join(root_dir, config_path)
+        else:
+            self._config_path = config_path
+        self._config: Dict[str, Any] = self._load_layered_config()
+
+    def _load_layered_config(self) -> Dict[str, Any]:
+        """Loads the baseline config and merges it with the user config."""
+        # 1. Load Bundled Baseline
+        config = self._load_baseline()
+
+        # 2. Load User Overrides
+        user_config = self._load_user_config()
+
+        # 3. Simple Deep Merge (Layered)
+        self._merge_dicts(config, user_config)
+
+        return config
+
+    def _load_baseline(self) -> Dict[str, Any]:
+        """Loads the bundled baseline config from package resources."""
+        try:
+            resource_path = resources.files("teddy_executor.resources.config").joinpath(
+                "config.yaml"
+            )
+            with resource_path.open("r", encoding="utf-8") as f:
+                data = yaml.safe_load(f)
+                return data if isinstance(data, dict) else {}
+        except (yaml.YAMLError, OSError, ImportError, AttributeError):
+            return {}
+
+    def _load_user_config(self) -> Dict[str, Any]:
+        """Loads the user-specific YAML configuration file if it exists."""
+        if not os.path.exists(self._config_path):
+            return {}
+
+        try:
+            with open(self._config_path, "r", encoding="utf-8") as f:
+                data = yaml.safe_load(f)
+                return data if isinstance(data, dict) else {}
+        except (yaml.YAMLError, OSError):
+            return {}
+
+    def _merge_dicts(self, base: Dict[str, Any], overrides: Dict[str, Any]) -> None:
+        """Recursively merges overrides into base. Prunes keys set to None."""
+        for key, value in overrides.items():
+            if value is None:
+                if key in base:
+                    del base[key]
+            elif isinstance(value, dict):
+                if key not in base or not isinstance(base[key], dict):
+                    base[key] = {}
+                self._merge_dicts(base[key], value)
+            else:
+                base[key] = value
+
+    def get_setting(self, key: str, default: Optional[Any] = None) -> Optional[Any]:
+        """
+        Retrieves a configuration value by its key from the loaded YAML.
+        Supports nested keys using dot notation (e.g., 'outer.inner').
+        """
+        if not key:
+            return default
+
+        # 1. Try exact match first (highest priority: top-level user overrides)
+        if key in self._config:
+            return self._config[key]
+
+        # 2. Try nested resolution (standard hierarchical structure)
+        parts = key.split(".")
+        result = self._resolve_nested(parts)
+
+        if result is not None:
+            return result
+
+        return default
+
+    def get_config_path(self) -> str:
+        """Returns the path to the configuration file."""
+        return self._config_path
+
+    def _resolve_nested(self, parts: list[str]) -> Optional[Any]:
+        """Iteratively resolves nested keys."""
+        current = self._config
+        for part in parts:
+            if isinstance(current, dict) and part in current:
+                current = current[part]
+            else:
+                return None
+        return current