perplexity-webui-scraper 0.3.4__py3-none-any.whl → 0.3.6__py3-none-any.whl

perplexity_webui_scraper/logging.py

@@ -0,0 +1,278 @@
+"""Logging configuration using loguru.
+
+Provides detailed, structured logging for all library operations.
+Logging is disabled by default and can be enabled via ClientConfig.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+import sys
+from typing import TYPE_CHECKING, Any
+
+from loguru import logger
+
+from .enums import LogLevel
+
+
+if TYPE_CHECKING:
+    from os import PathLike
+
+# Remove default handler to start with a clean slate
+logger.remove()
+
+# Flag to track if logging is configured
+_logging_configured: bool = False
+
+
+def configure_logging(
+    level: LogLevel | str = LogLevel.DISABLED,
+    log_file: str | PathLike[str] | None = None,
+) -> None:
+    """Configure logging for the library.
+
+    Args:
+        level: Logging level (LogLevel enum or string). Default is DISABLED.
+        log_file: Optional file path to write logs. If set, logs go to file only.
+                  If None, logs go to console. Logs are appended, never deleted.
+
+    Note:
+        - If log_file is set: logs go to file only (no console output)
+        - If log_file is None: logs go to console only
+        - Log format includes timestamp, level, module, function, and message
+    """
+
+    global _logging_configured  # noqa: PLW0603
+
+    # Remove any existing handlers
+    logger.remove()
+
+    # Normalize level to string
+    level_str = level.value if isinstance(level, LogLevel) else str(level).upper()
+
+    if level_str == "DISABLED":
+        # Logging disabled, add a null handler to suppress all output
+        logger.disable("perplexity_webui_scraper")
+        _logging_configured = False
+        return
+
+    # Enable the logger
+    logger.enable("perplexity_webui_scraper")
+
+    # Console format - concise but informative
+    console_format = (
+        "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | "
+        "<level>{level: <8}</level> | "
+        "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> | "
+        "<level>{message}</level>"
+    )
+
+    # File format - detailed with extra context
+    file_format = "{time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} | {message} | {extra}"
+
+    if log_file is not None:
+        # Log to file only (no console output)
+        log_path = Path(log_file)
+        logger.add(
+            log_path,
+            format=file_format,
+            level=level_str,
+            rotation=None,  # Never rotate
+            retention=None,  # Never delete
+            compression=None,  # No compression
+            mode="a",  # Append mode
+            encoding="utf-8",
+            filter="perplexity_webui_scraper",
+            enqueue=True,  # Thread-safe
+        )
+    else:
+        # Log to console only (no file)
+        logger.add(
+            sys.stderr,
+            format=console_format,
+            level=level_str,
+            colorize=True,
+            filter="perplexity_webui_scraper",
+        )
+
+    _logging_configured = True
+
+
+def get_logger(name: str) -> Any:
+    """Get a logger instance bound to the given module name.
+
+    Args:
+        name: Module name (typically __name__).
+
+    Returns:
+        A loguru logger instance bound to the module.
+    """
+
+    return logger.bind(module=name)
+
+
+# Convenience shortcuts for common log operations
+def log_request(
+    method: str,
+    url: str,
+    *,
+    params: dict[str, Any] | None = None,
+    headers: dict[str, str] | None = None,
+    body_size: int | None = None,
+) -> None:
+    """
+    Log an outgoing HTTP request with full details.
+    """
+
+    logger.debug(
+        "HTTP request initiated | method={method} url={url} params={params} "
+        "headers_count={headers_count} body_size={body_size}",
+        method=method,
+        url=url,
+        params=params,
+        headers_count=len(headers) if headers else 0,
+        body_size=body_size,
+    )
+
+
+def log_response(
+    method: str,
+    url: str,
+    status_code: int,
+    *,
+    elapsed_ms: float | None = None,
+    content_length: int | None = None,
+    headers: dict[str, str] | None = None,
+) -> None:
+    """
+    Log an HTTP response with full details.
+    """
+
+    level = "DEBUG" if status_code < 400 else "WARNING"
+    logger.log(
+        level,
+        "HTTP response received | method={method} url={url} status={status_code} "
+        "elapsed_ms={elapsed_ms:.2f} content_length={content_length}",
+        method=method,
+        url=url,
+        status_code=status_code,
+        elapsed_ms=elapsed_ms or 0,
+        content_length=content_length,
+    )
+
+
+def log_retry(
+    attempt: int,
+    max_attempts: int,
+    exception: BaseException | None,
+    wait_seconds: float,
+) -> None:
+    """
+    Log a retry attempt.
+    """
+
+    logger.warning(
+        "Retry attempt | attempt={attempt}/{max_attempts} exception={exception_type}: {exception_msg} "
+        "wait_seconds={wait_seconds:.2f}",
+        attempt=attempt,
+        max_attempts=max_attempts,
+        exception_type=type(exception).__name__ if exception else "None",
+        exception_msg=str(exception) if exception else "None",
+        wait_seconds=wait_seconds,
+    )
+
+
+def log_cloudflare_detected(status_code: int, markers_found: list[str]) -> None:
+    """
+    Log Cloudflare challenge detection.
+    """
+
+    logger.warning(
+        "Cloudflare challenge detected | status_code={status_code} markers={markers}",
+        status_code=status_code,
+        markers=markers_found,
+    )
+
+
+def log_fingerprint_rotation(old_profile: str, new_profile: str) -> None:
+    """
+    Log browser fingerprint rotation.
+    """
+
+    logger.info(
+        "Browser fingerprint rotated | old_profile={old} new_profile={new}",
+        old=old_profile,
+        new=new_profile,
+    )
+
+
+def log_rate_limit(wait_seconds: float) -> None:
+    """
+    Log rate limiting wait.
+    """
+
+    logger.debug(
+        "Rate limiter throttling | wait_seconds={wait_seconds:.3f}",
+        wait_seconds=wait_seconds,
+    )
+
+
+def log_session_created(impersonate: str, timeout: int) -> None:
+    """
+    Log HTTP session creation.
+    """
+
+    logger.info(
+        "HTTP session created | browser_profile={profile} timeout={timeout}s",
+        profile=impersonate,
+        timeout=timeout,
+    )
+
+
+def log_conversation_created(config_summary: str) -> None:
+    """
+    Log conversation creation.
+    """
+
+    logger.info(
+        "Conversation created | config={config}",
+        config=config_summary,
+    )
+
+
+def log_query_sent(query: str, model: str, has_files: bool) -> None:
+    """
+    Log a query being sent.
+    """
+
+    logger.info(
+        "Query sent | model={model} has_files={has_files} query_preview={query_preview}",
+        model=model,
+        has_files=has_files,
+        query_preview=query[:100] + "..." if len(query) > 100 else query,
+    )
+
+
+def log_stream_chunk(chunk_size: int, is_final: bool) -> None:
+    """
+    Log a streaming chunk received.
+    """
+
+    logger.debug(
+        "Stream chunk received | size={size} is_final={is_final}",
+        size=chunk_size,
+        is_final=is_final,
+    )
+
+
+def log_error(error: Exception, context: str = "") -> None:
+    """
+    Log an error with full traceback.
+    """
+
+    logger.exception(
+        "Error occurred | context={context} error_type={error_type} message={message}",
+        context=context,
+        error_type=type(error).__name__,
+        message=str(error),
+    )

perplexity_webui_scraper/mcp/__init__.py

@@ -0,0 +1,20 @@
+"""
+MCP (Model Context Protocol) server for Perplexity WebUI Scraper.
+
+This module provides an MCP server that exposes Perplexity AI search capabilities to AI assistants.
+"""
+
+from __future__ import annotations
+
+
+__all__: list[str] = ["run_server"]
+
+
+def run_server() -> None:
+    """
+    Run the MCP server.
+    """
+
+    from .server import main  # noqa: PLC0415
+
+    main()

perplexity_webui_scraper/mcp/__main__.py

@@ -0,0 +1,11 @@
+"""
+CLI entry point for MCP server.
+"""
+
+from __future__ import annotations
+
+from . import run_server
+
+
+if __name__ == "__main__":
+    run_server()

perplexity_webui_scraper/mcp/server.py

@@ -0,0 +1,166 @@
+"""
+MCP server implementation using FastMCP.
+"""
+
+from __future__ import annotations
+
+from os import environ
+from typing import Literal
+
+from fastmcp import FastMCP
+
+from perplexity_webui_scraper.config import ClientConfig, ConversationConfig
+from perplexity_webui_scraper.core import Perplexity
+from perplexity_webui_scraper.enums import CitationMode, SearchFocus, SourceFocus
+from perplexity_webui_scraper.models import Models
+
+
+# Create FastMCP server
+mcp = FastMCP(
+    "perplexity-webui-scraper-mcp",
+    instructions=(
+        "Search the web with Perplexity AI using the full range of premium models. "
+        "Unlike the official Perplexity API, this tool provides access to GPT-5.2, Claude 4.5, "
+        "Gemini 3, Grok 4.1, and other cutting-edge models with reasoning capabilities. "
+        "Use for real-time web research, academic searches, financial data, and current events. "
+        "Supports multiple source types: web, academic papers, social media, and SEC filings."
+    ),
+)
+
+# Model name mapping to Model objects
+MODEL_MAP = {
+    "best": Models.BEST,
+    "research": Models.RESEARCH,
+    "labs": Models.LABS,
+    "sonar": Models.SONAR,
+    "gpt52": Models.GPT_52,
+    "gpt52_thinking": Models.GPT_52_THINKING,
+    "claude_opus": Models.CLAUDE_45_OPUS,
+    "claude_opus_thinking": Models.CLAUDE_45_OPUS_THINKING,
+    "claude_sonnet": Models.CLAUDE_45_SONNET,
+    "claude_sonnet_thinking": Models.CLAUDE_45_SONNET_THINKING,
+    "gemini_pro": Models.GEMINI_3_PRO,
+    "gemini_flash": Models.GEMINI_3_FLASH,
+    "gemini_flash_thinking": Models.GEMINI_3_FLASH_THINKING,
+    "grok": Models.GROK_41,
+    "grok_thinking": Models.GROK_41_THINKING,
+    "kimi_thinking": Models.KIMI_K2_THINKING,
+}
+
+ModelName = Literal[
+    "best",
+    "research",
+    "labs",
+    "sonar",
+    "gpt52",
+    "gpt52_thinking",
+    "claude_opus",
+    "claude_opus_thinking",
+    "claude_sonnet",
+    "claude_sonnet_thinking",
+    "gemini_pro",
+    "gemini_flash",
+    "gemini_flash_thinking",
+    "grok",
+    "grok_thinking",
+    "kimi_thinking",
+]
+
+# Source focus mapping
+SOURCE_FOCUS_MAP = {
+    "web": [SourceFocus.WEB],
+    "academic": [SourceFocus.ACADEMIC],
+    "social": [SourceFocus.SOCIAL],
+    "finance": [SourceFocus.FINANCE],
+    "all": [SourceFocus.WEB, SourceFocus.ACADEMIC, SourceFocus.SOCIAL],
+}
+
+SourceFocusName = Literal["web", "academic", "social", "finance", "all"]
+
+# Client singleton
+_client: Perplexity | None = None
+
+
+def _get_client() -> Perplexity:
+    """
+    Get or create Perplexity client.
+    """
+
+    global _client  # noqa: PLW0603
+    if _client is None:
+        token = environ.get("PERPLEXITY_SESSION_TOKEN", "")
+
+        if not token:
+            raise ValueError(
+                "PERPLEXITY_SESSION_TOKEN environment variable is required. "
+                "Set it with: export PERPLEXITY_SESSION_TOKEN='your_token_here'"
+            )
+        _client = Perplexity(token, config=ClientConfig())
+
+    return _client
+
+
+@mcp.tool
+def perplexity_ask(
+    query: str,
+    model: ModelName = "best",
+    source_focus: SourceFocusName = "web",
+) -> str:
+    """
+    Ask a question and get AI-generated answers with real-time data from the internet.
+
+    Returns up-to-date information from web sources. Use for factual queries, research,
+    current events, news, library versions, documentation, or any question requiring
+    the latest information.
+
+    Args:
+        query: The question to ask.
+        model: AI model to use.
+        source_focus: Type of sources to prioritize (web, academic, social, finance, all).
+
+    Returns:
+        AI-generated answer with inline citations and a Citations section.
+    """
+
+    client = _get_client()
+    selected_model = MODEL_MAP.get(model, Models.BEST)
+    sources = SOURCE_FOCUS_MAP.get(source_focus, [SourceFocus.WEB])
+
+    try:
+        conversation = client.create_conversation(
+            ConversationConfig(
+                model=selected_model,
+                citation_mode=CitationMode.DEFAULT,
+                search_focus=SearchFocus.WEB,
+                source_focus=sources,
+            )
+        )
+
+        conversation.ask(query)
+        answer = conversation.answer or "No answer received"
+
+        # Build response with Perplexity-style citations
+        response_parts = [answer]
+
+        if conversation.search_results:
+            response_parts.append("\n\nCitations:")
+
+            for i, result in enumerate(conversation.search_results, 1):
+                url = result.url or ""
+                response_parts.append(f"\n[{i}]: {url}")
+
+        return "".join(response_parts)
+    except Exception as error:
+        return f"Error: {error!s}"
+
+
+def main() -> None:
+    """
+    Run the MCP server.
+    """
+
+    mcp.run()
+
+
+if __name__ == "__main__":
+    main()

perplexity_webui_scraper/models.py

@@ -1,4 +1,6 @@
-"""AI model definitions for Perplexity WebUI Scraper."""
+"""
+AI model definitions for Perplexity WebUI Scraper.
+"""
 
 from __future__ import annotations
 
@@ -7,7 +9,8 @@ from dataclasses import dataclass
 
 @dataclass(frozen=True, slots=True)
 class Model:
-    """AI model configuration.
+    """
+    AI model configuration.
 
     Attributes:
         identifier: Model identifier used by the API.
@@ -19,55 +22,88 @@ class Model:
 
 
 class Models:
-    """Available AI models with their configurations.
+    """
+    Available AI models with their configurations.
 
     All models use the "copilot" mode which enables web search.
     """
 
     RESEARCH = Model(identifier="pplx_alpha")
-    """Research - Fast and thorough for routine research"""
+    """
+    Research - Fast and thorough for routine research.
+    """
 
     LABS = Model(identifier="pplx_beta")
-    """Labs - Multi-step tasks with advanced troubleshooting"""
+    """
+    Labs - Multi-step tasks with advanced troubleshooting.
+    """
 
     BEST = Model(identifier="pplx_pro_upgraded")
-    """Best - Automatically selects the most responsive model based on the query"""
+    """
+    Best - Automatically selects the most responsive model based on the query.
+    """
 
     SONAR = Model(identifier="experimental")
-    """Sonar - Perplexity's fast model"""
+    """
+    Sonar - Perplexity's fast model.
+    """
 
     GPT_52 = Model(identifier="gpt52")
-    """GPT-5.2 - OpenAI's latest model"""
+    """
+    GPT-5.2 - OpenAI's latest model.
+    """
 
     GPT_52_THINKING = Model(identifier="gpt52_thinking")
-    """GPT-5.2 Thinking - OpenAI's latest model with thinking"""
+    """
+    GPT-5.2 Thinking - OpenAI's latest model with thinking.
+    """
 
     CLAUDE_45_OPUS = Model(identifier="claude45opus")
-    """Claude Opus 4.5 - Anthropic's Opus reasoning model"""
+    """
+    Claude Opus 4.5 - Anthropic's Opus reasoning model.
+    """
 
     CLAUDE_45_OPUS_THINKING = Model(identifier="claude45opusthinking")
-    """Claude Opus 4.5 Thinking - Anthropic's Opus reasoning model with thinking"""
+    """
+    Claude Opus 4.5 Thinking - Anthropic's Opus reasoning model with thinking.
+    """
 
     GEMINI_3_PRO = Model(identifier="gemini30pro")
-    """Gemini 3 Pro - Google's newest reasoning model"""
+    """
+    Gemini 3 Pro - Google's newest reasoning model.
+    """
 
     GEMINI_3_FLASH = Model(identifier="gemini30flash")
-    """Gemini 3 Flash - Google's fast reasoning model"""
+    """
+    Gemini 3 Flash - Google's fast reasoning model.
+    """
 
     GEMINI_3_FLASH_THINKING = Model(identifier="gemini30flash_high")
-    """Gemini 3 Flash Thinking - Google's fast reasoning model with enhanced thinking"""
+    """
+    Gemini 3 Flash Thinking - Google's fast reasoning model with enhanced thinking.
+    """
 
     GROK_41 = Model(identifier="grok41nonreasoning")
-    """Grok 4.1 - xAI's latest advanced model"""
+    """
+    Grok 4.1 - xAI's latest advanced model.
+    """
 
     GROK_41_THINKING = Model(identifier="grok41reasoning")
-    """Grok 4.1 Thinking - xAI's latest reasoning model"""
+    """
+    Grok 4.1 Thinking - xAI's latest reasoning model.
+    """
 
     KIMI_K2_THINKING = Model(identifier="kimik2thinking")
-    """Kimi K2 Thinking - Moonshot AI's latest reasoning model"""
+    """
+    Kimi K2 Thinking - Moonshot AI's latest reasoning model.
+    """
 
     CLAUDE_45_SONNET = Model(identifier="claude45sonnet")
-    """Claude Sonnet 4.5 - Anthropic's newest advanced model"""
+    """
+    Claude Sonnet 4.5 - Anthropic's newest advanced model.
+    """
 
     CLAUDE_45_SONNET_THINKING = Model(identifier="claude45sonnetthinking")
-    """Claude Sonnet 4.5 Thinking - Anthropic's newest reasoning model"""
+    """
+    Claude Sonnet 4.5 Thinking - Anthropic's newest reasoning model.
+    """