perplexity-webui-scraper 0.3.4__py3-none-any.whl → 0.3.5__py3-none-any.whl

perplexity_webui_scraper/logging.py

@@ -0,0 +1,256 @@
+"""Logging configuration using loguru.
+
+Provides detailed, structured logging for all library operations.
+Logging is disabled by default and can be enabled via ClientConfig.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+import sys
+from typing import TYPE_CHECKING, Any
+
+from loguru import logger
+
+from .enums import LogLevel
+
+
+if TYPE_CHECKING:
+    from os import PathLike
+
+# Remove default handler to start with a clean slate
+logger.remove()
+
+# Flag to track if logging is configured
+_logging_configured: bool = False
+
+
+def configure_logging(
+    level: LogLevel | str = LogLevel.DISABLED,
+    log_file: str | PathLike[str] | None = None,
+) -> None:
+    """Configure logging for the library.
+
+    Args:
+        level: Logging level (LogLevel enum or string). Default is DISABLED.
+        log_file: Optional file path to write logs. If set, logs go to file only.
+                  If None, logs go to console. Logs are appended, never deleted.
+
+    Note:
+        - If log_file is set: logs go to file only (no console output)
+        - If log_file is None: logs go to console only
+        - Log format includes timestamp, level, module, function, and message
+    """
+
+    global _logging_configured  # noqa: PLW0603
+
+    # Remove any existing handlers
+    logger.remove()
+
+    # Normalize level to string
+    level_str = level.value if isinstance(level, LogLevel) else str(level).upper()
+
+    if level_str == "DISABLED":
+        # Logging disabled, add a null handler to suppress all output
+        logger.disable("perplexity_webui_scraper")
+        _logging_configured = False
+        return
+
+    # Enable the logger
+    logger.enable("perplexity_webui_scraper")
+
+    # Console format - concise but informative
+    console_format = (
+        "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | "
+        "<level>{level: <8}</level> | "
+        "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> | "
+        "<level>{message}</level>"
+    )
+
+    # File format - detailed with extra context
+    file_format = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} | {message} | {extra}"
+
+    if log_file is not None:
+        # Log to file only (no console output)
+        log_path = Path(log_file)
+        logger.add(
+            log_path,
+            format=file_format,
+            level=level_str,
+            rotation=None,  # Never rotate
+            retention=None,  # Never delete
+            compression=None,  # No compression
+            mode="a",  # Append mode
+            encoding="utf-8",
+            filter="perplexity_webui_scraper",
+            enqueue=True,  # Thread-safe
+        )
+    else:
+        # Log to console only (no file)
+        logger.add(
+            sys.stderr,
+            format=console_format,
+            level=level_str,
+            colorize=True,
+            filter="perplexity_webui_scraper",
+        )
+
+    _logging_configured = True
+
+
+def get_logger(name: str) -> Any:
+    """Get a logger instance bound to the given module name.
+
+    Args:
+        name: Module name (typically __name__).
+
+    Returns:
+        A loguru logger instance bound to the module.
+    """
+
+    return logger.bind(module=name)
+
+
+# Convenience shortcuts for common log operations
+def log_request(
+    method: str,
+    url: str,
+    *,
+    params: dict[str, Any] | None = None,
+    headers: dict[str, str] | None = None,
+    body_size: int | None = None,
+) -> None:
+    """Log an outgoing HTTP request with full details."""
+
+    logger.debug(
+        "HTTP request initiated | method={method} url={url} params={params} "
+        "headers_count={headers_count} body_size={body_size}",
+        method=method,
+        url=url,
+        params=params,
+        headers_count=len(headers) if headers else 0,
+        body_size=body_size,
+    )
+
+
+def log_response(
+    method: str,
+    url: str,
+    status_code: int,
+    *,
+    elapsed_ms: float | None = None,
+    content_length: int | None = None,
+    headers: dict[str, str] | None = None,
+) -> None:
+    """Log an HTTP response with full details."""
+
+    level = "DEBUG" if status_code < 400 else "WARNING"
+    logger.log(
+        level,
+        "HTTP response received | method={method} url={url} status={status_code} "
+        "elapsed_ms={elapsed_ms:.2f} content_length={content_length}",
+        method=method,
+        url=url,
+        status_code=status_code,
+        elapsed_ms=elapsed_ms or 0,
+        content_length=content_length,
+    )
+
+
+def log_retry(
+    attempt: int,
+    max_attempts: int,
+    exception: Exception,
+    wait_seconds: float,
+) -> None:
+    """Log a retry attempt."""
+
+    logger.warning(
+        "Retry attempt | attempt={attempt}/{max_attempts} exception={exception_type}: {exception_msg} "
+        "wait_seconds={wait_seconds:.2f}",
+        attempt=attempt,
+        max_attempts=max_attempts,
+        exception_type=type(exception).__name__,
+        exception_msg=str(exception),
+        wait_seconds=wait_seconds,
+    )
+
+
+def log_cloudflare_detected(status_code: int, markers_found: list[str]) -> None:
+    """Log Cloudflare challenge detection."""
+
+    logger.warning(
+        "Cloudflare challenge detected | status_code={status_code} markers={markers}",
+        status_code=status_code,
+        markers=markers_found,
+    )
+
+
+def log_fingerprint_rotation(old_profile: str, new_profile: str) -> None:
+    """Log browser fingerprint rotation."""
+
+    logger.info(
+        "Browser fingerprint rotated | old_profile={old} new_profile={new}",
+        old=old_profile,
+        new=new_profile,
+    )
+
+
+def log_rate_limit(wait_seconds: float) -> None:
+    """Log rate limiting wait."""
+
+    logger.debug(
+        "Rate limiter throttling | wait_seconds={wait_seconds:.3f}",
+        wait_seconds=wait_seconds,
+    )
+
+
+def log_session_created(impersonate: str, timeout: int) -> None:
+    """Log HTTP session creation."""
+
+    logger.info(
+        "HTTP session created | browser_profile={profile} timeout={timeout}s",
+        profile=impersonate,
+        timeout=timeout,
+    )
+
+
+def log_conversation_created(config_summary: str) -> None:
+    """Log conversation creation."""
+
+    logger.info(
+        "Conversation created | config={config}",
+        config=config_summary,
+    )
+
+
+def log_query_sent(query: str, model: str, has_files: bool) -> None:
+    """Log a query being sent."""
+
+    logger.info(
+        "Query sent | model={model} has_files={has_files} query_preview={query_preview}",
+        model=model,
+        has_files=has_files,
+        query_preview=query[:100] + "..." if len(query) > 100 else query,
+    )
+
+
+def log_stream_chunk(chunk_size: int, is_final: bool) -> None:
+    """Log a streaming chunk received."""
+
+    logger.debug(
+        "Stream chunk received | size={size} is_final={is_final}",
+        size=chunk_size,
+        is_final=is_final,
+    )
+
+
+def log_error(error: Exception, context: str = "") -> None:
+    """Log an error with full traceback."""
+
+    logger.exception(
+        "Error occurred | context={context} error_type={error_type} message={message}",
+        context=context,
+        error_type=type(error).__name__,
+        message=str(error),
+    )

perplexity_webui_scraper/mcp/__init__.py

@@ -0,0 +1,18 @@
+"""
+MCP (Model Context Protocol) server for Perplexity WebUI Scraper.
+
+This module provides an MCP server that exposes Perplexity AI search capabilities to AI assistants.
+"""
+
+from __future__ import annotations
+
+
+__all__: list[str] = ["run_server"]
+
+
+def run_server() -> None:
+    """Run the MCP server."""
+
+    from .server import main  # noqa: PLC0415
+
+    main()

perplexity_webui_scraper/mcp/__main__.py

@@ -0,0 +1,9 @@
+"""CLI entry point for MCP server."""
+
+from __future__ import annotations
+
+from . import run_server
+
+
+if __name__ == "__main__":
+    run_server()

perplexity_webui_scraper/mcp/server.py

@@ -0,0 +1,181 @@
+"""MCP server implementation using FastMCP."""
+
+from __future__ import annotations
+
+from os import environ
+from typing import Literal
+
+from fastmcp import FastMCP
+
+from perplexity_webui_scraper.config import ClientConfig, ConversationConfig
+from perplexity_webui_scraper.core import Perplexity
+from perplexity_webui_scraper.enums import CitationMode, SearchFocus, SourceFocus
+from perplexity_webui_scraper.models import Models
+
+
+# Create FastMCP server
+mcp = FastMCP(
+    "perplexity-webui-scraper-mcp",
+    instructions=(
+        "Search the web with Perplexity AI using the full range of premium models. "
+        "Unlike the official Perplexity API, this tool provides access to GPT-5.2, Claude 4.5, "
+        "Gemini 3, Grok 4.1, and other cutting-edge models with reasoning capabilities. "
+        "Use for real-time web research, academic searches, financial data, and current events. "
+        "Supports multiple source types: web, academic papers, social media, and SEC filings."
+    ),
+)
+
+# Model name mapping to Model objects
+MODEL_MAP = {
+    "best": Models.BEST,
+    "research": Models.RESEARCH,
+    "labs": Models.LABS,
+    "sonar": Models.SONAR,
+    "gpt52": Models.GPT_52,
+    "gpt52_thinking": Models.GPT_52_THINKING,
+    "claude_opus": Models.CLAUDE_45_OPUS,
+    "claude_opus_thinking": Models.CLAUDE_45_OPUS_THINKING,
+    "claude_sonnet": Models.CLAUDE_45_SONNET,
+    "claude_sonnet_thinking": Models.CLAUDE_45_SONNET_THINKING,
+    "gemini_pro": Models.GEMINI_3_PRO,
+    "gemini_flash": Models.GEMINI_3_FLASH,
+    "gemini_flash_thinking": Models.GEMINI_3_FLASH_THINKING,
+    "grok": Models.GROK_41,
+    "grok_thinking": Models.GROK_41_THINKING,
+    "kimi_thinking": Models.KIMI_K2_THINKING,
+}
+
+# Available model names for type hints
+ModelName = Literal[
+    "best",
+    "research",
+    "labs",
+    "sonar",
+    "gpt52",
+    "gpt52_thinking",
+    "claude_opus",
+    "claude_opus_thinking",
+    "claude_sonnet",
+    "claude_sonnet_thinking",
+    "gemini_pro",
+    "gemini_flash",
+    "gemini_flash_thinking",
+    "grok",
+    "grok_thinking",
+    "kimi_thinking",
+]
+
+# Source focus mapping
+SOURCE_FOCUS_MAP = {
+    "web": [SourceFocus.WEB],
+    "academic": [SourceFocus.ACADEMIC],
+    "social": [SourceFocus.SOCIAL],
+    "finance": [SourceFocus.FINANCE],
+    "all": [SourceFocus.WEB, SourceFocus.ACADEMIC, SourceFocus.SOCIAL],
+}
+
+SourceFocusName = Literal["web", "academic", "social", "finance", "all"]
+
+# Client singleton
+_client: Perplexity | None = None
+
+
+def _get_client() -> Perplexity:
+    """Get or create Perplexity client."""
+
+    global _client  # noqa: PLW0603
+    if _client is None:
+        token = environ.get("PERPLEXITY_SESSION_TOKEN", "")
+
+        if not token:
+            raise ValueError(
+                "PERPLEXITY_SESSION_TOKEN environment variable is required. "
+                "Set it with: export PERPLEXITY_SESSION_TOKEN='your_token_here'"
+            )
+        _client = Perplexity(token, config=ClientConfig())
+
+    return _client
+
+
+@mcp.tool
+def perplexity_ask(
+    query: str,
+    model: ModelName = "best",
+    source_focus: SourceFocusName = "web",
+) -> str:
+    """
+    Ask a question and get AI-generated answers with real-time data from the internet.
+
+    Returns up-to-date information from web sources. Use for factual queries, research,
+    current events, news, library versions, documentation, or any question requiring
+
+    Args:
+        query: The search query or question to ask Perplexity AI.
+        model: AI model to use. Options:
+            - "best": Automatically selects optimal model (default)
+            - "research": Fast and thorough for routine research
+            - "labs": Multi-step tasks with advanced troubleshooting
+            - "sonar": Perplexity's fast built-in model
+            - "gpt52": OpenAI's GPT-5.2
+            - "gpt52_thinking": GPT-5.2 with reasoning
+            - "claude_opus": Anthropic's Claude Opus 4.5
+            - "claude_opus_thinking": Claude Opus with reasoning
+            - "claude_sonnet": Anthropic's Claude Sonnet 4.5
+            - "claude_sonnet_thinking": Claude Sonnet with reasoning
+            - "gemini_pro": Google's Gemini 3 Pro
+            - "gemini_flash": Google's Gemini 3 Flash
+            - "gemini_flash_thinking": Gemini Flash with reasoning
+            - "grok": xAI's Grok 4.1
+            - "grok_thinking": Grok with reasoning
+            - "kimi_thinking": Moonshot's Kimi K2 with reasoning
+        source_focus: Type of sources to prioritize:
+            - "web": General web search (default)
+            - "academic": Scholarly articles and papers
+            - "social": Social media (Reddit, Twitter)
+            - "finance": SEC EDGAR financial filings
+            - "all": Combine web, academic, and social sources
+
+    Returns:
+        AI-generated answer with inline citations [1][2] and a Citations section.
+    """
+
+    client = _get_client()
+    selected_model = MODEL_MAP.get(model, Models.BEST)
+    sources = SOURCE_FOCUS_MAP.get(source_focus, [SourceFocus.WEB])
+
+    try:
+        conversation = client.create_conversation(
+            ConversationConfig(
+                model=selected_model,
+                citation_mode=CitationMode.DEFAULT,
+                search_focus=SearchFocus.WEB,
+                source_focus=sources,
+            )
+        )
+
+        conversation.ask(query)
+        answer = conversation.answer or "No answer received"
+
+        # Build response with Perplexity-style citations
+        response_parts = [answer]
+
+        if conversation.search_results:
+            response_parts.append("\n\nCitations:")
+
+            for i, result in enumerate(conversation.search_results, 1):
+                url = result.url or ""
+                response_parts.append(f"\n[{i}]: {url}")
+
+        return "".join(response_parts)
+    except Exception as error:
+        return f"Error searching Perplexity: {error!s}"
+
+
+def main() -> None:
+    """Run the MCP server."""
+
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

perplexity_webui_scraper/resilience.py

@@ -0,0 +1,179 @@
+"""
+Resilience utilities for HTTP requests.
+
+Provides retry mechanisms, rate limiting, and Cloudflare bypass utilities
+using the tenacity library for robust retry handling.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+import random
+from threading import Lock
+import time
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+T = TypeVar("T")
+
+
+# Browser profiles supported by curl_cffi for fingerprint rotation
+BROWSER_PROFILES: tuple[str, ...] = (
+    "chrome",
+    "chrome110",
+    "chrome116",
+    "chrome119",
+    "chrome120",
+    "chrome123",
+    "chrome124",
+    "chrome131",
+    "edge99",
+    "edge101",
+    "safari15_3",
+    "safari15_5",
+    "safari17_0",
+    "safari17_2_ios",
+)
+
+# Cloudflare challenge detection markers
+CLOUDFLARE_MARKERS: tuple[str, ...] = (
+    "cf-ray",
+    "cf-mitigated",
+    "__cf_chl_",
+    "Checking your browser",
+    "Just a moment...",
+    "cloudflare",
+    "Enable JavaScript and cookies to continue",
+    "challenge-platform",
+)
+
+
+@dataclass(slots=True)
+class RetryConfig:
+    """Configuration for retry behavior.
+
+    Attributes:
+        max_retries: Maximum number of retry attempts.
+        base_delay: Initial delay in seconds before first retry.
+        max_delay: Maximum delay between retries.
+        jitter: Random jitter factor to add to delays (0-1).
+    """
+
+    max_retries: int = 3
+    base_delay: float = 1.0
+    max_delay: float = 60.0
+    jitter: float = 0.5
+
+
+@dataclass
+class RateLimiter:
+    """Token bucket rate limiter for throttling requests.
+
+    Attributes:
+        requests_per_second: Maximum requests allowed per second.
+    """
+
+    requests_per_second: float = 0.5
+    _last_request: float = field(default=0.0, init=False)
+    _lock: Lock = field(default_factory=Lock, init=False)
+
+    def acquire(self) -> None:
+        """Wait until a request can be made within rate limits."""
+
+        with self._lock:
+            now = time.monotonic()
+            min_interval = 1.0 / self.requests_per_second
+
+            if self._last_request > 0:
+                elapsed = now - self._last_request
+                wait_time = min_interval - elapsed
+
+                if wait_time > 0:
+                    time.sleep(wait_time)
+
+            self._last_request = time.monotonic()
+
+
+def get_random_browser_profile() -> str:
+    """Get a random browser profile for fingerprint rotation.
+
+    Returns:
+        A browser profile identifier compatible with curl_cffi.
+    """
+
+    return random.choice(BROWSER_PROFILES)
+
+
+def is_cloudflare_challenge(response_text: str, headers: dict[str, Any] | None = None) -> bool:
+    """Detect if a response is a Cloudflare challenge page.
+
+    Args:
+        response_text: The response body text.
+        headers: Optional response headers.
+
+    Returns:
+        True if Cloudflare challenge markers are detected.
+    """
+
+    text_lower = response_text.lower()
+
+    for marker in CLOUDFLARE_MARKERS:
+        if marker.lower() in text_lower:
+            return True
+
+    if headers:
+        for key in headers:
+            key_lower = key.lower()
+
+            if "cf-" in key_lower or "cloudflare" in key_lower:
+                return True
+
+    return False
+
+
+def is_cloudflare_status(status_code: int) -> bool:
+    """Check if status code indicates a potential Cloudflare block.
+
+    Args:
+        status_code: HTTP status code.
+
+    Returns:
+        True if status code is commonly used by Cloudflare challenges.
+    """
+
+    return status_code in (403, 503, 520, 521, 522, 523, 524, 525, 526)
+
+
+def create_retry_decorator(
+    config: RetryConfig,
+    retryable_exceptions: tuple[type[Exception], ...],
+    on_retry: Callable[[RetryCallState], None] | None = None,
+) -> Callable[[Callable[..., T]], Callable[..., T]]:
+    """Create a tenacity retry decorator with the given configuration.
+
+    Args:
+        config: Retry configuration.
+        retryable_exceptions: Tuple of exception types to retry on.
+        on_retry: Optional callback to execute on each retry.
+
+    Returns:
+        A retry decorator configured with the given settings.
+    """
+
+    return retry(
+        stop=stop_after_attempt(config.max_retries + 1),
+        wait=wait_exponential_jitter(
+            initial=config.base_delay,
+            max=config.max_delay,
+            jitter=config.max_delay * config.jitter,
+        ),
+        retry=retry_if_exception_type(retryable_exceptions),
+        before_sleep=on_retry,
+        reraise=True,
+    )