search-parser 0.0.1__py3-none-any.whl

search_engine_parser/__init__.py

@@ -0,0 +1,12 @@
+"""Search Engine Parser - Parse search engine HTML results into structured data."""
+
+from search_engine_parser.__version__ import __version__
+from search_engine_parser.core.models import SearchResult, SearchResults
+from search_engine_parser.core.parser import SearchParser
+
+__all__ = [
+    "__version__",
+    "SearchParser",
+    "SearchResult",
+    "SearchResults",
+]

search_engine_parser/__version__.py

@@ -0,0 +1,3 @@
+"""Version information."""
+
+__version__ = "0.1.0"

search_engine_parser/cli.py

@@ -0,0 +1,79 @@
+"""Command-line interface for search engine parser."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+try:
+    import click
+    from rich.console import Console
+    from rich.syntax import Syntax
+except ImportError:
+    print(
+        'CLI dependencies not installed. Install with: pip install "search-engine-parser[cli]"',
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+from search_engine_parser.core.parser import SearchParser
+from search_engine_parser.exceptions import SearchEngineParserError
+
+
+@click.command()
+@click.argument("input_file", type=click.Path(exists=True, path_type=Path))
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["json", "markdown", "dict"]),
+    default="json",
+    help="Output format (default: json).",
+)
+@click.option(
+    "--engine",
+    type=click.Choice(["google", "bing", "duckduckgo"]),
+    default=None,
+    help="Manually specify the search engine.",
+)
+@click.option(
+    "--pretty/--no-pretty",
+    default=True,
+    help="Pretty-print JSON output (default: true).",
+)
+def main(
+    input_file: Path,
+    output_format: str,
+    engine: str | None,
+    pretty: bool,
+) -> None:
+    """Parse search engine HTML results into structured data.
+
+    INPUT_FILE is the path to an HTML file containing search results.
+    """
+    console = Console()
+    html = input_file.read_text(encoding="utf-8")
+
+    parser = SearchParser()
+    try:
+        # dict format is not useful for CLI, convert to json for display
+        fmt = output_format if output_format != "dict" else "json"
+        result = parser.parse(html, engine=engine, output_format=fmt)  # type: ignore[arg-type]
+    except SearchEngineParserError as e:
+        console.print(f"[red]Error:[/red] {e}")
+        sys.exit(1)
+
+    if isinstance(result, str):
+        if output_format == "json" and pretty:
+            import json
+
+            formatted = json.dumps(json.loads(result), indent=2)
+            syntax = Syntax(formatted, "json", theme="monokai")
+            console.print(syntax)
+        elif output_format == "markdown":
+            from rich.markdown import Markdown
+
+            console.print(Markdown(result))
+        else:
+            console.print(result)
+    else:
+        console.print(result)

search_engine_parser/core/__init__.py

@@ -0,0 +1 @@
+"""Core module for search engine parser."""

search_engine_parser/core/detector.py

@@ -0,0 +1,130 @@
+"""Search engine auto-detection from HTML content."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+
+from bs4 import BeautifulSoup, Tag
+
+from search_engine_parser.utils import make_soup
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DetectionResult:
+    """Result of search engine detection."""
+
+    engine: str
+    confidence: float
+
+
+class SearchEngineDetector:
+    """Detects which search engine produced the given HTML.
+
+    Detection is performed in priority order:
+    1. HTML meta tags
+    2. DOM structure patterns
+    3. URL patterns in the HTML
+    """
+
+    def detect(self, html: str) -> DetectionResult | None:
+        """Detect the search engine from HTML content.
+
+        Args:
+            html: Raw HTML string to analyze.
+
+        Returns:
+            DetectionResult with engine name and confidence, or None if
+            confidence is below threshold.
+        """
+        soup = make_soup(html)
+        candidates: list[DetectionResult] = []
+
+        for check in [
+            self._check_meta_tags,
+            self._check_dom_structure,
+            self._check_url_patterns,
+        ]:
+            result = check(soup)
+            if result:
+                candidates.append(result)
+
+        if not candidates:
+            return None
+
+        best = max(candidates, key=lambda r: r.confidence)
+        if best.confidence < 0.3:
+            return None
+        return best
+
+    def _check_meta_tags(self, soup: BeautifulSoup) -> DetectionResult | None:
+        """Check HTML meta tags for search engine identification."""
+        # Google: <meta property="og:site_name" content="Google">
+        og_site = soup.find("meta", attrs={"property": "og:site_name"})
+        if isinstance(og_site, Tag) and str(og_site.get("content", "")).lower() == "google":
+            return DetectionResult(engine="google", confidence=0.95)
+
+        # Bing: <meta name="ms.application" content="Bing">
+        ms_app = soup.find("meta", attrs={"name": "ms.application"})
+        if isinstance(ms_app, Tag) and "bing" in str(ms_app.get("content", "")).lower():
+            return DetectionResult(engine="bing", confidence=0.95)
+
+        # Check for Bing in any meta content
+        for meta in soup.find_all("meta"):
+            content = str(meta.get("content", "")).lower()
+            name = str(meta.get("name", "")).lower()
+            if "bing" in content or "bing" in name:
+                return DetectionResult(engine="bing", confidence=0.85)
+
+        # DuckDuckGo: Check for DDG-specific meta tags
+        for meta in soup.find_all("meta"):
+            content = str(meta.get("content", "")).lower()
+            if "duckduckgo" in content:
+                return DetectionResult(engine="duckduckgo", confidence=0.95)
+
+        return None
+
+    def _check_dom_structure(self, soup: BeautifulSoup) -> DetectionResult | None:
+        """Check DOM structure patterns for search engine identification."""
+        # Google: div#search or div.g
+        if soup.find("div", id="search") or soup.find_all("div", class_="g"):
+            return DetectionResult(engine="google", confidence=0.8)
+
+        # Bing: div.b_algo
+        if soup.find_all("li", class_="b_algo"):
+            return DetectionResult(engine="bing", confidence=0.85)
+
+        # DuckDuckGo: various DDG-specific classes
+        if soup.find_all("article", attrs={"data-testid": "result"}):
+            return DetectionResult(engine="duckduckgo", confidence=0.85)
+        if soup.find_all("div", class_="result"):
+            return DetectionResult(engine="duckduckgo", confidence=0.6)
+
+        return None
+
+    def _check_url_patterns(self, soup: BeautifulSoup) -> DetectionResult | None:
+        """Check URL patterns in HTML for search engine identification."""
+        # Check canonical URL
+        canonical = soup.find("link", attrs={"rel": "canonical"})
+        if isinstance(canonical, Tag):
+            href = str(canonical.get("href", "")).lower()
+            if "google.com" in href:
+                return DetectionResult(engine="google", confidence=0.9)
+            if "bing.com" in href:
+                return DetectionResult(engine="bing", confidence=0.9)
+            if "duckduckgo.com" in href:
+                return DetectionResult(engine="duckduckgo", confidence=0.9)
+
+        # Check for search engine URLs in links
+        for link in soup.find_all("link"):
+            href = str(link.get("href", "")).lower()
+            if "google.com" in href or "gstatic.com" in href:
+                return DetectionResult(engine="google", confidence=0.6)
+            if "bing.com" in href:
+                return DetectionResult(engine="bing", confidence=0.6)
+            if "duckduckgo.com" in href:
+                return DetectionResult(engine="duckduckgo", confidence=0.6)
+
+        return None

search_engine_parser/core/models.py

@@ -0,0 +1,33 @@
+"""Pydantic data models for search results."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Literal, Optional
+
+from pydantic import BaseModel, Field
+
+
+class SearchResult(BaseModel):
+    """Single search result item."""
+
+    title: str
+    url: str
+    description: Optional[str] = None  # noqa: UP045 - Pydantic evaluates at runtime; breaks on 3.9
+    position: int
+    result_type: Literal[
+        "organic", "featured_snippet", "knowledge_panel", "news", "image", "sponsored"
+    ] = "organic"
+    metadata: dict[str, object] = Field(default_factory=dict)
+
+
+class SearchResults(BaseModel):
+    """Collection of search results from a page."""
+
+    search_engine: str
+    query: Optional[str] = None  # noqa: UP045
+    total_results: Optional[int] = None  # noqa: UP045
+    results: list[SearchResult]
+    detection_confidence: float = Field(ge=0.0, le=1.0)
+    parsed_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    metadata: dict[str, object] = Field(default_factory=dict)

search_engine_parser/core/parser.py

@@ -0,0 +1,124 @@
+"""Main SearchParser class - the primary public API."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Literal
+
+from search_engine_parser.core.detector import SearchEngineDetector
+from search_engine_parser.core.models import SearchResults
+from search_engine_parser.exceptions import (
+    ParseError,
+    ParserNotFoundError,
+    SearchEngineDetectionError,
+)
+from search_engine_parser.formatters.json_formatter import JSONFormatter
+from search_engine_parser.formatters.markdown_formatter import MarkdownFormatter
+from search_engine_parser.parsers import PARSER_REGISTRY
+from search_engine_parser.parsers.base import BaseParser
+
+logger = logging.getLogger(__name__)
+
+
+class SearchParser:
+    """Main entry point for parsing search engine HTML.
+
+    Automatically detects the search engine and parses results
+    into structured data in the requested format.
+
+    Example:
+        >>> parser = SearchParser()
+        >>> json_output = parser.parse(html_string)
+        >>> markdown_output = parser.parse(html_string, output_format="markdown")
+        >>> dict_output = parser.parse(html_string, output_format="dict")
+    """
+
+    def __init__(self) -> None:
+        self._detector = SearchEngineDetector()
+        self._json_formatter = JSONFormatter()
+        self._markdown_formatter = MarkdownFormatter()
+        self._parsers: dict[str, BaseParser] = {
+            name: cls() for name, cls in PARSER_REGISTRY.items()
+        }
+
+    def parse(
+        self,
+        html: str,
+        engine: str | None = None,
+        output_format: Literal["json", "markdown", "dict"] = "json",
+    ) -> str | dict[str, Any]:
+        """Parse search engine HTML and return results.
+
+        Args:
+            html: HTML string to parse.
+            engine: Optionally specify engine ("google", "bing", "duckduckgo").
+                If None, will auto-detect.
+            output_format: Output format - "json" (default), "markdown", or "dict".
+                - "json": Returns JSON string
+                - "markdown": Returns Markdown string
+                - "dict": Returns Python dictionary
+
+        Returns:
+            JSON string, Markdown string, or Python dictionary depending
+            on output_format.
+
+        Raises:
+            SearchEngineDetectionError: If engine cannot be detected and not specified.
+            ParserNotFoundError: If no parser available for the detected/specified engine.
+            ParseError: If parsing fails.
+        """
+        # Determine which engine to use
+        engine_name = self._resolve_engine(html, engine)
+
+        # Get the parser
+        parser = self._parsers.get(engine_name)
+        if not parser:
+            raise ParserNotFoundError(
+                f"No parser available for engine: {engine_name!r}. "
+                f"Available parsers: {list(self._parsers.keys())}"
+            )
+
+        # Parse the HTML
+        try:
+            results = parser.parse(html)
+        except Exception as e:
+            raise ParseError(f"Failed to parse HTML: {e}") from e
+
+        # Format the output
+        return self._format_output(results, output_format)
+
+    def _resolve_engine(self, html: str, engine: str | None) -> str:
+        """Resolve the search engine name, either from user input or auto-detection."""
+        if engine:
+            return engine.lower()
+
+        detection = self._detector.detect(html)
+        if not detection:
+            raise SearchEngineDetectionError(
+                "Could not detect the search engine. "
+                "Please specify the engine manually using the 'engine' parameter."
+            )
+
+        logger.info(
+            "Detected engine: %s (confidence: %.2f)",
+            detection.engine,
+            detection.confidence,
+        )
+        return detection.engine
+
+    def _format_output(
+        self,
+        results: SearchResults,
+        output_format: Literal["json", "markdown", "dict"],
+    ) -> str | dict[str, Any]:
+        """Format results into the requested output format."""
+        if output_format == "json":
+            return self._json_formatter.format(results)
+        elif output_format == "markdown":
+            return self._markdown_formatter.format(results)
+        elif output_format == "dict":
+            result: dict[str, Any] = json.loads(results.model_dump_json())
+            return result
+        else:
+            raise ValueError(f"Unknown output format: {output_format!r}")

search_engine_parser/exceptions.py

@@ -0,0 +1,21 @@
+"""Custom exceptions for search engine parser."""
+
+
+class SearchEngineParserError(Exception):
+    """Base exception for search engine parser."""
+
+
+class SearchEngineDetectionError(SearchEngineParserError):
+    """Cannot determine which search engine the HTML came from."""
+
+
+class ParserNotFoundError(SearchEngineParserError):
+    """No parser available for the detected or specified search engine."""
+
+
+class ParseError(SearchEngineParserError):
+    """Failed to parse the HTML content."""
+
+
+class InvalidHTMLError(SearchEngineParserError):
+    """The HTML structure is invalid or cannot be processed."""

search_engine_parser/formatters/__init__.py

@@ -0,0 +1,11 @@
+"""Output formatters for search results."""
+
+from search_engine_parser.formatters.base import BaseFormatter
+from search_engine_parser.formatters.json_formatter import JSONFormatter
+from search_engine_parser.formatters.markdown_formatter import MarkdownFormatter
+
+__all__ = [
+    "BaseFormatter",
+    "JSONFormatter",
+    "MarkdownFormatter",
+]

search_engine_parser/formatters/base.py

@@ -0,0 +1,27 @@
+"""Abstract base class for output formatters."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from search_engine_parser.core.models import SearchResults
+
+
+class BaseFormatter(ABC):
+    """Abstract base class for output formatters.
+
+    Subclass this to create custom output formats.
+    """
+
+    @abstractmethod
+    def format(self, results: SearchResults) -> Any:
+        """Format search results into the desired output.
+
+        Args:
+            results: Parsed search results to format.
+
+        Returns:
+            Formatted output (type depends on implementation).
+        """
+        ...

search_engine_parser/formatters/json_formatter.py

@@ -0,0 +1,21 @@
+"""JSON output formatter."""
+
+from __future__ import annotations
+
+from search_engine_parser.core.models import SearchResults
+from search_engine_parser.formatters.base import BaseFormatter
+
+
+class JSONFormatter(BaseFormatter):
+    """Formats search results as a JSON string using Pydantic serialization."""
+
+    def format(self, results: SearchResults) -> str:
+        """Format search results as a JSON string.
+
+        Args:
+            results: Parsed search results.
+
+        Returns:
+            JSON string representation.
+        """
+        return results.model_dump_json(indent=2)

search_engine_parser/formatters/markdown_formatter.py

@@ -0,0 +1,119 @@
+"""Markdown output formatter for LLM-friendly consumption."""
+
+from __future__ import annotations
+
+from search_engine_parser.__version__ import __version__
+from search_engine_parser.core.models import SearchResult, SearchResults
+from search_engine_parser.formatters.base import BaseFormatter
+
+
+class MarkdownFormatter(BaseFormatter):
+    """Formats search results as Markdown text.
+
+    Produces a clean, hierarchical Markdown document suitable for
+    both human reading and LLM consumption.
+    """
+
+    def format(self, results: SearchResults) -> str:
+        """Format search results as Markdown.
+
+        Args:
+            results: Parsed search results.
+
+        Returns:
+            Markdown string.
+        """
+        lines: list[str] = []
+
+        # Header
+        title = f"Search Results: {results.query}" if results.query else "Search Results"
+        lines.append(f"# {title}")
+        lines.append("")
+        lines.append(f"**Search Engine:** {results.search_engine.title()}")
+
+        if results.total_results is not None:
+            lines.append(f"**Total Results:** ~{results.total_results:,}")
+
+        parsed_time = results.parsed_at.strftime("%Y-%m-%d %H:%M:%S UTC")
+        lines.append(f"**Parsed:** {parsed_time}")
+        lines.append("")
+        lines.append("---")
+        lines.append("")
+
+        # Group results by type
+        featured = [r for r in results.results if r.result_type == "featured_snippet"]
+        organic = [r for r in results.results if r.result_type == "organic"]
+        knowledge = [r for r in results.results if r.result_type == "knowledge_panel"]
+        news = [r for r in results.results if r.result_type == "news"]
+
+        if featured:
+            lines.append("## Featured Snippet")
+            lines.append("")
+            for result in featured:
+                lines.extend(self._format_featured(result))
+            lines.append("---")
+            lines.append("")
+
+        if knowledge:
+            lines.append("## Knowledge Panel")
+            lines.append("")
+            for result in knowledge:
+                lines.extend(self._format_result(result))
+            lines.append("---")
+            lines.append("")
+
+        if organic:
+            lines.append("## Organic Results")
+            lines.append("")
+            for result in organic:
+                lines.extend(self._format_organic(result))
+
+        if news:
+            lines.append("## News Results")
+            lines.append("")
+            for result in news:
+                lines.extend(self._format_result(result))
+
+        lines.append("---")
+        lines.append("")
+        lines.append(f"*Parsed with search-engine-parser v{__version__}*")
+        lines.append("")
+
+        return "\n".join(lines)
+
+    def _format_featured(self, result: SearchResult) -> list[str]:
+        """Format a featured snippet result."""
+        lines: list[str] = []
+        lines.append(f"### {result.title}")
+        lines.append("")
+        if result.description:
+            lines.append(result.description)
+            lines.append("")
+        # Extract domain from URL for source
+        lines.append(f"**Source:** [{result.url}]({result.url})")
+        lines.append("")
+        return lines
+
+    def _format_organic(self, result: SearchResult) -> list[str]:
+        """Format an organic search result."""
+        lines: list[str] = []
+        lines.append(f"### {result.position}. {result.title}")
+        lines.append("")
+        if result.description:
+            lines.append(result.description)
+            lines.append("")
+        lines.append(f"**URL:** {result.url}")
+        lines.append("")
+        return lines
+
+    def _format_result(self, result: SearchResult) -> list[str]:
+        """Format a generic result."""
+        lines: list[str] = []
+        lines.append(f"### {result.title}")
+        lines.append("")
+        if result.description:
+            lines.append(result.description)
+            lines.append("")
+        lines.append(f"**URL:** {result.url}")
+        lines.append("")
+        return lines

search_engine_parser/parsers/__init__.py

@@ -0,0 +1,21 @@
+"""Search engine parser implementations."""
+
+from search_engine_parser.parsers.base import BaseParser
+from search_engine_parser.parsers.bing import BingParser
+from search_engine_parser.parsers.duckduckgo import DuckDuckGoParser
+from search_engine_parser.parsers.google import GoogleParser
+
+# Registry of all available parsers
+PARSER_REGISTRY: dict[str, type[BaseParser]] = {
+    "google": GoogleParser,
+    "bing": BingParser,
+    "duckduckgo": DuckDuckGoParser,
+}
+
+__all__ = [
+    "BaseParser",
+    "BingParser",
+    "DuckDuckGoParser",
+    "GoogleParser",
+    "PARSER_REGISTRY",
+]