search-parser 0.0.1__py3-none-any.whl

search_engine_parser/parsers/base.py

@@ -0,0 +1,79 @@
+"""Abstract base class for search engine parsers."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from bs4 import BeautifulSoup, Tag
+
+from search_engine_parser.core.models import SearchResults
+
+
+class BaseParser(ABC):
+    """Abstract base class for search engine parsers.
+
+    All search engine parsers must implement this interface.
+    To add a new search engine, subclass this and implement the
+    abstract methods, then register in parsers/__init__.py.
+    """
+
+    @property
+    @abstractmethod
+    def engine_name(self) -> str:
+        """Name of the search engine (e.g., 'google')."""
+        ...
+
+    @abstractmethod
+    def parse(self, html: str) -> SearchResults:
+        """Parse HTML and extract search results.
+
+        Args:
+            html: Raw HTML string from the search engine.
+
+        Returns:
+            Parsed search results.
+        """
+        ...
+
+    @abstractmethod
+    def can_parse(self, soup: BeautifulSoup) -> float:
+        """Check if this parser can handle the given HTML.
+
+        Args:
+            soup: Parsed BeautifulSoup object.
+
+        Returns:
+            Confidence score from 0.0 (cannot parse) to 1.0 (certain).
+        """
+        ...
+
+    def extract_query(self, soup: BeautifulSoup) -> str | None:
+        """Extract the search query from the HTML if possible.
+
+        Args:
+            soup: Parsed BeautifulSoup object.
+
+        Returns:
+            The search query string, or None if not found.
+        """
+        # Common: check <input> with name="q"
+        q_input = soup.find("input", attrs={"name": "q"})
+        if isinstance(q_input, Tag):
+            value = q_input.get("value")
+            if value:
+                return str(value)
+
+        # Check title tag
+        title_tag = soup.find("title")
+        if isinstance(title_tag, Tag) and title_tag.string:
+            text = title_tag.string.strip()
+            for suffix in [
+                " - Google Search",
+                " - Bing",
+                " - Search",
+                " at DuckDuckGo",
+            ]:
+                if text.endswith(suffix):
+                    return text[: -len(suffix)]
+
+        return None

search_engine_parser/parsers/bing.py

@@ -0,0 +1,132 @@
+"""Bing search results parser."""
+
+from __future__ import annotations
+
+import logging
+
+from bs4 import BeautifulSoup, Tag
+
+from search_engine_parser.core.models import SearchResult, SearchResults
+from search_engine_parser.parsers.base import BaseParser
+from search_engine_parser.utils import clean_text, make_soup
+
+logger = logging.getLogger(__name__)
+
+
+class BingParser(BaseParser):
+    """Parser for Bing search result pages."""
+
+    @property
+    def engine_name(self) -> str:
+        return "bing"
+
+    def can_parse(self, soup: BeautifulSoup) -> float:
+        """Check if this HTML is from Bing."""
+        confidence = 0.0
+
+        ms_app = soup.find("meta", attrs={"name": "ms.application"})
+        if isinstance(ms_app, Tag) and "bing" in str(ms_app.get("content", "")).lower():
+            confidence = max(confidence, 0.95)
+
+        if soup.find_all("li", class_="b_algo"):
+            confidence = max(confidence, 0.85)
+
+        for meta in soup.find_all("meta"):
+            content = str(meta.get("content", "")).lower()
+            if "bing" in content:
+                confidence = max(confidence, 0.7)
+
+        return confidence
+
+    def parse(self, html: str) -> SearchResults:
+        """Parse Bing search results HTML."""
+        soup = make_soup(html)
+        results: list[SearchResult] = []
+        position = 1
+
+        # Extract featured snippet
+        featured = self._extract_featured_snippet(soup)
+        if featured:
+            results.append(featured)
+
+        # Extract organic results
+        for item in soup.find_all("li", class_="b_algo"):
+            if not isinstance(item, Tag):
+                continue
+            result = self._parse_organic_result(item, position)
+            if result:
+                results.append(result)
+                position += 1
+
+        query = self.extract_query(soup)
+        confidence = self.can_parse(soup)
+
+        return SearchResults(
+            search_engine=self.engine_name,
+            query=query,
+            results=results,
+            detection_confidence=confidence,
+        )
+
+    def _parse_organic_result(self, item: Tag, position: int) -> SearchResult | None:
+        """Parse a single Bing organic result."""
+        h2 = item.find("h2")
+        if not isinstance(h2, Tag):
+            return None
+
+        link = h2.find("a")
+        if not isinstance(link, Tag):
+            return None
+
+        title = clean_text(link.get_text())
+        url = str(link.get("href", ""))
+
+        if not title or not url:
+            return None
+
+        # Find description
+        desc_div = item.find("div", class_="b_caption")
+        description = None
+        if isinstance(desc_div, Tag):
+            p = desc_div.find("p")
+            if isinstance(p, Tag):
+                description = clean_text(p.get_text())
+
+        return SearchResult(
+            title=title,
+            url=url,
+            description=description,
+            position=position,
+            result_type="organic",
+        )
+
+    def _extract_featured_snippet(self, soup: BeautifulSoup) -> SearchResult | None:
+        """Extract featured snippet if present."""
+        snippet = soup.find("div", class_="b_ans")
+        if not isinstance(snippet, Tag):
+            return None
+
+        h2 = snippet.find("h2")
+        link = snippet.find("a")
+        if not isinstance(h2, Tag) or not isinstance(link, Tag):
+            return None
+
+        title = clean_text(h2.get_text())
+        url = str(link.get("href", ""))
+        if not title or not url:
+            return None
+
+        # Get snippet body text
+        body = snippet.find("div", class_="b_rich")
+        if not isinstance(body, Tag):
+            body = snippet.find("p")
+        description = clean_text(body.get_text()) if isinstance(body, Tag) else None
+
+        return SearchResult(
+            title=title,
+            url=url,
+            description=description,
+            position=0,
+            result_type="featured_snippet",
+            metadata={"snippet_type": "paragraph"},
+        )

search_engine_parser/parsers/duckduckgo.py

@@ -0,0 +1,137 @@
+"""DuckDuckGo search results parser."""
+
+from __future__ import annotations
+
+import logging
+
+from bs4 import BeautifulSoup, Tag
+
+from search_engine_parser.core.models import SearchResult, SearchResults
+from search_engine_parser.parsers.base import BaseParser
+from search_engine_parser.utils import clean_text, make_soup
+
+logger = logging.getLogger(__name__)
+
+
+class DuckDuckGoParser(BaseParser):
+    """Parser for DuckDuckGo search result pages."""
+
+    @property
+    def engine_name(self) -> str:
+        return "duckduckgo"
+
+    def can_parse(self, soup: BeautifulSoup) -> float:
+        """Check if this HTML is from DuckDuckGo."""
+        confidence = 0.0
+
+        for meta in soup.find_all("meta"):
+            content = str(meta.get("content", "")).lower()
+            if "duckduckgo" in content:
+                confidence = max(confidence, 0.95)
+
+        if soup.find_all("article", attrs={"data-testid": "result"}):
+            confidence = max(confidence, 0.85)
+
+        if soup.find_all("div", class_="result"):
+            confidence = max(confidence, 0.5)
+
+        # Check for DDG-specific link patterns
+        for link in soup.find_all("link"):
+            href = str(link.get("href", "")).lower()
+            if "duckduckgo.com" in href:
+                confidence = max(confidence, 0.8)
+
+        return confidence
+
+    def parse(self, html: str) -> SearchResults:
+        """Parse DuckDuckGo search results HTML."""
+        soup = make_soup(html)
+        results: list[SearchResult] = []
+        position = 1
+
+        # Try article-based results (newer DDG layout)
+        articles = soup.find_all("article", attrs={"data-testid": "result"})
+        if articles:
+            for article in articles:
+                if not isinstance(article, Tag):
+                    continue
+                result = self._parse_article_result(article, position)
+                if result:
+                    results.append(result)
+                    position += 1
+        else:
+            # Fallback to div.result layout
+            for item in soup.find_all("div", class_="result"):
+                if not isinstance(item, Tag):
+                    continue
+                result = self._parse_div_result(item, position)
+                if result:
+                    results.append(result)
+                    position += 1
+
+        query = self.extract_query(soup)
+        confidence = self.can_parse(soup)
+
+        return SearchResults(
+            search_engine=self.engine_name,
+            query=query,
+            results=results,
+            detection_confidence=confidence,
+        )
+
+    def _parse_article_result(self, article: Tag, position: int) -> SearchResult | None:
+        """Parse a DDG article-based result."""
+        h2 = article.find("h2")
+        if not isinstance(h2, Tag):
+            return None
+
+        link = h2.find("a")
+        if not isinstance(link, Tag):
+            return None
+
+        title = clean_text(link.get_text())
+        url = str(link.get("href", ""))
+
+        if not title or not url:
+            return None
+
+        # Find description
+        desc = article.find("span", attrs={"data-testid": "result-snippet"})
+        if not isinstance(desc, Tag):
+            desc = article.find("div", class_="snippet")
+        description = clean_text(desc.get_text()) if isinstance(desc, Tag) else None
+
+        return SearchResult(
+            title=title,
+            url=url,
+            description=description,
+            position=position,
+            result_type="organic",
+        )
+
+    def _parse_div_result(self, item: Tag, position: int) -> SearchResult | None:
+        """Parse a DDG div.result-based result."""
+        link = item.find("a", class_="result__a")
+        if not isinstance(link, Tag):
+            link = item.find("a")
+        if not isinstance(link, Tag):
+            return None
+
+        title = clean_text(link.get_text())
+        url = str(link.get("href", ""))
+
+        if not title or not url:
+            return None
+
+        desc = item.find("a", class_="result__snippet")
+        if not isinstance(desc, Tag):
+            desc = item.find("div", class_="result__snippet")
+        description = clean_text(desc.get_text()) if isinstance(desc, Tag) else None
+
+        return SearchResult(
+            title=title,
+            url=url,
+            description=description,
+            position=position,
+            result_type="organic",
+        )

search_engine_parser/parsers/google.py

@@ -0,0 +1,226 @@
+"""Google search results parser."""
+
+from __future__ import annotations
+
+import logging
+
+from bs4 import BeautifulSoup, Tag
+
+from search_engine_parser.core.models import SearchResult, SearchResults
+from search_engine_parser.parsers.base import BaseParser
+from search_engine_parser.utils import clean_text, make_soup
+
+logger = logging.getLogger(__name__)
+
+
+class GoogleParser(BaseParser):
+    """Parser for Google search result pages."""
+
+    @property
+    def engine_name(self) -> str:
+        return "google"
+
+    def can_parse(self, soup: BeautifulSoup) -> float:
+        """Check if this HTML is from Google."""
+        confidence = 0.0
+
+        og_site = soup.find("meta", attrs={"property": "og:site_name"})
+        if isinstance(og_site, Tag) and str(og_site.get("content", "")).lower() == "google":
+            confidence = max(confidence, 0.95)
+
+        if soup.find("div", id="search"):
+            confidence = max(confidence, 0.8)
+
+        if soup.find_all("div", class_="g"):
+            confidence = max(confidence, 0.7)
+
+        return confidence
+
+    def parse(self, html: str) -> SearchResults:
+        """Parse Google search results HTML."""
+        soup = make_soup(html)
+        results: list[SearchResult] = []
+        position = 1
+
+        # Extract sponsored results
+        sponsored = self._extract_sponsored_results(soup)
+        results.extend(sponsored)
+
+        # Extract featured snippet
+        featured = self._extract_featured_snippet(soup)
+        if featured:
+            results.append(featured)
+
+        # Extract organic results
+        for item in self._find_organic_results(soup):
+            result = self._parse_organic_result(item, position)
+            if result:
+                results.append(result)
+                position += 1
+
+        query = self.extract_query(soup)
+        confidence = self.can_parse(soup)
+
+        return SearchResults(
+            search_engine=self.engine_name,
+            query=query,
+            results=results,
+            detection_confidence=confidence,
+        )
+
+    def _find_organic_results(self, soup: BeautifulSoup) -> list[Tag]:
+        """Find all organic result containers."""
+        search_div = soup.find("div", id="search")
+        if isinstance(search_div, Tag):
+            g_divs = [
+                t
+                for t in search_div.find_all("div", class_="g", recursive=True)
+                if isinstance(t, Tag)
+            ]
+            if g_divs:
+                return g_divs
+            # Fallback: locate results by their yuRUbf (title/link) sections
+            # and walk up to find a container that also holds the description.
+            return self._find_results_by_title_links(search_div)
+        return [t for t in soup.find_all("div", class_="g") if isinstance(t, Tag)]
+
+    def _find_results_by_title_links(self, root: Tag) -> list[Tag]:
+        """Find result containers by locating yuRUbf divs and their ancestors."""
+        containers: list[Tag] = []
+        seen: set[int] = set()
+        for yu in root.find_all("div", class_="yuRUbf", recursive=True):
+            container = self._find_result_container(yu, root)
+            if isinstance(container, Tag) and id(container) not in seen:
+                seen.add(id(container))
+                containers.append(container)
+        return containers
+
+    def _find_result_container(self, title_link: Tag, root: Tag) -> Tag | None:
+        """Walk up from a yuRUbf div to find the closest ancestor with a description."""
+        ancestor = title_link.parent
+        for _ in range(5):
+            if not isinstance(ancestor, Tag) or ancestor is root:
+                break
+            if ancestor.find("div", class_="VwiC3b") or ancestor.find("span", class_="st"):
+                return ancestor
+            ancestor = ancestor.parent
+        # No description ancestor found; use direct parent of yuRUbf
+        return title_link.parent if isinstance(title_link.parent, Tag) else None
+
+    def _parse_organic_result(self, item: Tag, position: int) -> SearchResult | None:
+        """Parse a single organic result div."""
+        # Find the title link
+        link_container = item.find("div", class_="yuRUbf")
+        if not isinstance(link_container, Tag):
+            # Try finding a direct link with h3
+            link = item.find("a")
+            h3 = item.find("h3")
+            if not isinstance(link, Tag) or not isinstance(h3, Tag):
+                return None
+            url = str(link.get("href", ""))
+            title = clean_text(h3.get_text())
+        else:
+            link = link_container.find("a")
+            if not isinstance(link, Tag):
+                return None
+            url = str(link.get("href", ""))
+            h3 = link.find("h3")
+            title = clean_text(h3.get_text()) if isinstance(h3, Tag) else ""
+
+        if not url or not title:
+            return None
+
+        # Find description
+        desc_div = item.find("div", class_="VwiC3b")
+        if not isinstance(desc_div, Tag):
+            desc_div = item.find("span", class_="st")
+        description = clean_text(desc_div.get_text()) if isinstance(desc_div, Tag) else None
+
+        return SearchResult(
+            title=title,
+            url=url,
+            description=description,
+            position=position,
+            result_type="organic",
+        )
+
+    def _extract_sponsored_results(self, soup: BeautifulSoup) -> list[SearchResult]:
+        """Extract sponsored (ad) results from Google search pages."""
+        results: list[SearchResult] = []
+        for block in soup.find_all("div", class_="vbIt3d"):
+            if not isinstance(block, Tag):
+                continue
+            for ad in block.find_all("div", class_="uEierd"):
+                result = self._parse_sponsored_ad(ad)
+                if result:
+                    results.append(result)
+        return results
+
+    def _parse_sponsored_ad(self, ad: Tag) -> SearchResult | None:
+        """Parse a single Google sponsored ad container."""
+        link = ad.find("a", class_="sVXRqc")
+        if not isinstance(link, Tag):
+            return None
+
+        url = str(link.get("href", ""))
+        if not url or not url.startswith("http"):
+            return None
+
+        heading = link.find(attrs={"role": "heading"})
+        title = clean_text(heading.get_text()) if isinstance(heading, Tag) else ""
+        if not title:
+            return None
+
+        # Find description outside the main ad link
+        description = None
+        for desc_div in ad.find_all("div", class_="Va3FIb"):
+            if not isinstance(desc_div, Tag):
+                continue
+            if desc_div.find_parent("a", class_="sVXRqc"):
+                continue
+            text = clean_text(desc_div.get_text())
+            if text and text != title:
+                description = text
+                break
+
+        return SearchResult(
+            title=title,
+            url=url,
+            description=description,
+            position=0,
+            result_type="sponsored",
+        )
+
+    def _extract_featured_snippet(self, soup: BeautifulSoup) -> SearchResult | None:
+        """Extract featured snippet if present."""
+        snippet_container = soup.find("div", class_="xpdopen")
+        if not isinstance(snippet_container, Tag):
+            snippet_container = soup.find("block-component")
+        if not isinstance(snippet_container, Tag):
+            return None
+
+        h3 = snippet_container.find("h3")
+        link = snippet_container.find("a")
+        if not isinstance(h3, Tag) or not isinstance(link, Tag):
+            return None
+
+        title = clean_text(h3.get_text())
+        url = str(link.get("href", ""))
+
+        # Get snippet text
+        desc_span = snippet_container.find("span", class_="hgKElc")
+        if not isinstance(desc_span, Tag):
+            desc_span = snippet_container.find("div", class_="LGOjhe")
+        description = clean_text(desc_span.get_text()) if isinstance(desc_span, Tag) else None
+
+        if not title or not url:
+            return None
+
+        return SearchResult(
+            title=title,
+            url=url,
+            description=description,
+            position=0,
+            result_type="featured_snippet",
+            metadata={"snippet_type": "paragraph"},
+        )

search_engine_parser/utils.py

@@ -0,0 +1,35 @@
+"""Utility functions for search engine parser."""
+
+from __future__ import annotations
+
+import logging
+
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+
+
+def make_soup(html: str) -> BeautifulSoup:
+    """Parse HTML string into a BeautifulSoup object.
+
+    Args:
+        html: Raw HTML string to parse.
+
+    Returns:
+        Parsed BeautifulSoup object.
+    """
+    return BeautifulSoup(html, "lxml")
+
+
+def clean_text(text: str | None) -> str:
+    """Clean and normalize text extracted from HTML.
+
+    Args:
+        text: Raw text that may contain extra whitespace.
+
+    Returns:
+        Cleaned text with normalized whitespace.
+    """
+    if not text:
+        return ""
+    return " ".join(text.split()).strip()