perplexity-webui-scraper 0.3.4__py3-none-any.whl → 0.3.6__py3-none-any.whl

perplexity_webui_scraper/resilience.py

@@ -0,0 +1,181 @@
+"""
+Resilience utilities for HTTP requests.
+
+Provides retry mechanisms, rate limiting, and Cloudflare bypass utilities
+using the tenacity library for robust retry handling.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+import random
+from threading import Lock
+import time
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+T = TypeVar("T")
+
+
+# Browser profiles supported by curl_cffi for fingerprint rotation
+BROWSER_PROFILES: tuple[str, ...] = (
+    "chrome",
+    "chrome110",
+    "chrome116",
+    "chrome119",
+    "chrome120",
+    "chrome123",
+    "chrome124",
+    "chrome131",
+    "edge99",
+    "edge101",
+    "safari15_3",
+    "safari15_5",
+    "safari17_0",
+    "safari17_2_ios",
+)
+
+# Cloudflare challenge detection markers
+CLOUDFLARE_MARKERS: tuple[str, ...] = (
+    "cf-ray",
+    "cf-mitigated",
+    "__cf_chl_",
+    "Checking your browser",
+    "Just a moment...",
+    "cloudflare",
+    "Enable JavaScript and cookies to continue",
+    "challenge-platform",
+)
+
+
+@dataclass(slots=True)
+class RetryConfig:
+    """Configuration for retry behavior.
+
+    Attributes:
+        max_retries: Maximum number of retry attempts.
+        base_delay: Initial delay in seconds before first retry.
+        max_delay: Maximum delay between retries.
+        jitter: Random jitter factor to add to delays (0-1).
+    """
+
+    max_retries: int = 3
+    base_delay: float = 1.0
+    max_delay: float = 60.0
+    jitter: float = 0.5
+
+
+@dataclass
+class RateLimiter:
+    """Token bucket rate limiter for throttling requests.
+
+    Attributes:
+        requests_per_second: Maximum requests allowed per second.
+    """
+
+    requests_per_second: float = 0.5
+    _last_request: float = field(default=0.0, init=False)
+    _lock: Lock = field(default_factory=Lock, init=False)
+
+    def acquire(self) -> None:
+        """
+        Wait until a request can be made within rate limits.
+        """
+
+        with self._lock:
+            now = time.monotonic()
+            min_interval = 1.0 / self.requests_per_second
+
+            if self._last_request > 0:
+                elapsed = now - self._last_request
+                wait_time = min_interval - elapsed
+
+                if wait_time > 0:
+                    time.sleep(wait_time)
+
+            self._last_request = time.monotonic()
+
+
+def get_random_browser_profile() -> str:
+    """Get a random browser profile for fingerprint rotation.
+
+    Returns:
+        A browser profile identifier compatible with curl_cffi.
+    """
+
+    return random.choice(BROWSER_PROFILES)
+
+
+def is_cloudflare_challenge(response_text: str, headers: dict[str, Any] | None = None) -> bool:
+    """Detect if a response is a Cloudflare challenge page.
+
+    Args:
+        response_text: The response body text.
+        headers: Optional response headers.
+
+    Returns:
+        True if Cloudflare challenge markers are detected.
+    """
+
+    text_lower = response_text.lower()
+
+    for marker in CLOUDFLARE_MARKERS:
+        if marker.lower() in text_lower:
+            return True
+
+    if headers:
+        for key in headers:
+            key_lower = key.lower()
+
+            if "cf-" in key_lower or "cloudflare" in key_lower:
+                return True
+
+    return False
+
+
+def is_cloudflare_status(status_code: int) -> bool:
+    """Check if status code indicates a potential Cloudflare block.
+
+    Args:
+        status_code: HTTP status code.
+
+    Returns:
+        True if status code is commonly used by Cloudflare challenges.
+    """
+
+    return status_code in (403, 503, 520, 521, 522, 523, 524, 525, 526)
+
+
+def create_retry_decorator(
+    config: RetryConfig,
+    retryable_exceptions: tuple[type[Exception], ...],
+    on_retry: Callable[[RetryCallState], None] | None = None,
+) -> Callable[[Callable[..., T]], Callable[..., T]]:
+    """Create a tenacity retry decorator with the given configuration.
+
+    Args:
+        config: Retry configuration.
+        retryable_exceptions: Tuple of exception types to retry on.
+        on_retry: Optional callback to execute on each retry.
+
+    Returns:
+        A retry decorator configured with the given settings.
+    """
+
+    return retry(
+        stop=stop_after_attempt(config.max_retries + 1),
+        wait=wait_exponential_jitter(
+            initial=config.base_delay,
+            max=config.max_delay,
+            jitter=config.max_delay * config.jitter,
+        ),
+        retry=retry_if_exception_type(retryable_exceptions),
+        before_sleep=on_retry,
+        reraise=True,
+    )

perplexity_webui_scraper/types.py

@@ -1,4 +1,6 @@
-"""Response types and data models."""
+"""
+Response types and data models.
+"""
 
 from __future__ import annotations
 
@@ -8,7 +10,9 @@ from typing import Any
 
 @dataclass(frozen=True, slots=True)
 class Coordinates:
-    """Geographic coordinates (lat/lng)."""
+    """
+    Geographic coordinates (lat/lng).
+    """
 
     latitude: float
     longitude: float
@@ -16,7 +20,9 @@ class Coordinates:
 
 @dataclass(frozen=True, slots=True)
 class SearchResultItem:
-    """A single search result."""
+    """
+    A single search result.
+    """
 
     title: str | None = None
     snippet: str | None = None
@@ -25,7 +31,9 @@ class SearchResultItem:
 
 @dataclass(slots=True)
 class Response:
-    """Response from Perplexity AI."""
+    """
+    Response from Perplexity AI.
+    """
 
     title: str | None = None
     answer: str | None = None
@@ -38,7 +46,9 @@ class Response:
 
 @dataclass(frozen=True, slots=True)
 class _FileInfo:
-    """Internal file info for uploads."""
+    """
+    Internal file info for uploads.
+    """
 
     path: str
     size: int

{perplexity_webui_scraper-0.3.4.dist-info → perplexity_webui_scraper-0.3.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: perplexity-webui-scraper
-Version: 0.3.4
+Version: 0.3.6
 Summary: Python scraper to extract AI responses from Perplexity's web interface.
 Keywords: perplexity,ai,scraper,webui,api,client
 Author: henrique-coder
@@ -20,14 +20,18 @@ Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
 Requires-Dist: curl-cffi>=0.14.0
+Requires-Dist: loguru>=0.7.3
 Requires-Dist: orjson>=3.11.5
 Requires-Dist: pydantic>=2.12.5
+Requires-Dist: tenacity>=9.1.2
+Requires-Dist: fastmcp>=2.14.2 ; extra == 'mcp'
 Requires-Python: >=3.10
 Project-URL: Changelog, https://github.com/henrique-coder/perplexity-webui-scraper/releases
 Project-URL: Documentation, https://github.com/henrique-coder/perplexity-webui-scraper#readme
 Project-URL: Homepage, https://github.com/henrique-coder/perplexity-webui-scraper
 Project-URL: Issues, https://github.com/henrique-coder/perplexity-webui-scraper/issues
 Project-URL: Repository, https://github.com/henrique-coder/perplexity-webui-scraper.git
+Provides-Extra: mcp
 Description-Content-Type: text/markdown
 
 <div align="center">
@@ -47,7 +51,8 @@ Python scraper to extract AI responses from [Perplexity's](https://www.perplexit
 ## Installation
 
 ```bash
-uv pip install perplexity-webui-scraper
+uv pip install perplexity-webui-scraper  # from PyPI (stable)
+uv pip install git+https://github.com/henrique-coder/perplexity-webui-scraper.git@dev  # from GitHub (development)
 ```
 
 ## Requirements
@@ -197,18 +202,103 @@ conversation.ask("Latest AI research", files=["paper.pdf"])
 | `timezone`        | `None`        | Timezone           |
 | `coordinates`     | `None`        | Location (lat/lng) |
 
-## CLI Tools
+## Exceptions
 
-### Session Token Generator
+The library provides specific exception types for better error handling:
+
+| Exception                          | Description                                                  |
+| ---------------------------------- | ------------------------------------------------------------ |
+| `PerplexityError`                  | Base exception for all library errors                        |
+| `AuthenticationError`              | Session token is invalid or expired (HTTP 403)               |
+| `RateLimitError`                   | Rate limit exceeded (HTTP 429)                               |
+| `FileUploadError`                  | File upload failed                                           |
+| `FileValidationError`              | File validation failed (size, type, etc.)                    |
+| `ResearchClarifyingQuestionsError` | Research mode is asking clarifying questions (not supported) |
+| `ResponseParsingError`             | API response could not be parsed                             |
+| `StreamingError`                   | Error during streaming response                              |
+
+### Handling Research Mode Clarifying Questions
+
+When using Research mode (`Models.RESEARCH`), the API may ask clarifying questions before providing an answer. Since programmatic interaction is not supported, the library raises a `ResearchClarifyingQuestionsError` with the questions:
+
+```python
+from perplexity_webui_scraper import (
+    Perplexity,
+    ResearchClarifyingQuestionsError,
+)
+
+try:
+    conversation.ask("Research this topic", model=Models.RESEARCH)
+except ResearchClarifyingQuestionsError as error:
+    print("The AI needs clarification:")
+    for question in error.questions:
+        print(f"  - {question}")
+    # Consider rephrasing your query to be more specific
+```
+
+## MCP Server (Model Context Protocol)
+
+The library includes an MCP server that allows AI assistants (like Claude) to search using Perplexity AI directly.
+
+### Installation
 
 ```bash
-get-perplexity-session-token
+uv pip install perplexity-webui-scraper[mcp]
+```
+
+### Running the Server
+
+```bash
+# Set your session token
+export PERPLEXITY_SESSION_TOKEN="your_token_here"  # For Linux/Mac
+set PERPLEXITY_SESSION_TOKEN="your_token_here"  # For Windows
+
+# Run with FastMCP
+uv run fastmcp run src/perplexity_webui_scraper/mcp/server.py
+
+# Or test with the dev inspector
+uv run fastmcp dev src/perplexity_webui_scraper/mcp/server.py
+```
+
+### Claude Desktop Configuration
+
+Add to `~/.config/claude/claude_desktop_config.json`:
+
+```json
+{
+  "mcpServers": {
+    "perplexity": {
+      "command": "uv",
+      "args": [
+        "run",
+        "fastmcp",
+        "run",
+        "path/to/perplexity_webui_scraper/mcp/server.py"
+      ],
+      "env": {
+        "PERPLEXITY_SESSION_TOKEN": "your_token_here"
+      }
+    }
+  }
+}
 ```
 
-Interactive tool to automatically obtain your Perplexity session token via email authentication. The token can be automatically saved to your `.env` file for immediate use.
+### Available Tool
+
+| Tool             | Description                                                                 |
+| ---------------- | --------------------------------------------------------------------------- |
+| `perplexity_ask` | Ask questions and get AI-generated answers with real-time data from the web |
+
+**Parameters:**
+
+| Parameter      | Type  | Default  | Description                                                   |
+| -------------- | ----- | -------- | ------------------------------------------------------------- |
+| `query`        | `str` | -        | Question to ask (required)                                    |
+| `model`        | `str` | `"best"` | AI model (`best`, `research`, `gpt52`, `claude_sonnet`, etc.) |
+| `source_focus` | `str` | `"web"`  | Source type (`web`, `academic`, `social`, `finance`, `all`)   |
 
 ## Disclaimer
 
-This is an **unofficial** library. It uses internal APIs that may change without notice. Use at your own risk. Not for production use.
+This is an **unofficial** library. It uses internal APIs that may change without notice. Use at your own risk.
 
 By using this library, you agree to Perplexity AI's Terms of Service.

perplexity_webui_scraper-0.3.6.dist-info/RECORD

@@ -0,0 +1,21 @@
+perplexity_webui_scraper/__init__.py,sha256=DQdyF667plbSLdULrEll1ovrOPB7HwVNrD7fttLNkuQ,715
+perplexity_webui_scraper/cli/get_perplexity_session_token.py,sha256=PxHZbrTbekUQX2MIMDZC0iXBTmnTSBFZMb85mLgezR8,7009
+perplexity_webui_scraper/config.py,sha256=nQYNsyCvMQW1V51R-6UV5hA0DI02qBLJ-ufX6r3NZyU,2067
+perplexity_webui_scraper/constants.py,sha256=j32_i67mZGJ4B3KPV1ICE-6NR5g3KxDtDY5OxQXIthw,1887
+perplexity_webui_scraper/core.py,sha256=X8aQgIBCFzz000Cy58ofWCByT94HYuqyuQQmlgEQBVk,22460
+perplexity_webui_scraper/enums.py,sha256=QHrDfdawedGuuThcAR0XC0xPnc7VwSaK4BFQ9jqzQag,2953
+perplexity_webui_scraper/exceptions.py,sha256=vdr5fm6OSgruII8zLwcYT3NsrU58GAXFX-WyS6nuW0M,3987
+perplexity_webui_scraper/http.py,sha256=blRITnFrJsqcsZXUDub8_ZFjSNtLz7mK2ZBRV6tRc1I,19757
+perplexity_webui_scraper/limits.py,sha256=4-eO9xbfKtObN4kMDIVglXmfC4dZvYtvjyUKUP_LKSU,474
+perplexity_webui_scraper/logging.py,sha256=jxgho9jjrZvr3tZ5m0z3caQQsqdtFT9sM-HDVFWL67M,7300
+perplexity_webui_scraper/mcp/__init__.py,sha256=Ur7fmsPDrxewBzq9UdvMoFHp85a8pGsW32WBHG5pfrc,376
+perplexity_webui_scraper/mcp/__main__.py,sha256=cmGev_HXYQK2hoNSQAziEo7bVqHdc0lXaTquzMItSiU,148
+perplexity_webui_scraper/mcp/server.py,sha256=3RJeOHOjv048ZleUURG3sFdMaDHsSPmdn7GY47yAm2s,4758
+perplexity_webui_scraper/models.py,sha256=I5PhVD7B5XeeWfjWvXBtnl3CHMv4P59DO_hnYk9O0b0,2636
+perplexity_webui_scraper/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+perplexity_webui_scraper/resilience.py,sha256=sVsLW1kU6dMit4L6-dyPhVoT00EXGfokbEUnzvBNb0k,4680
+perplexity_webui_scraper/types.py,sha256=qWsIABBxn1vcQQ2KUbC_hyiQHeaLsVb63epmRcpcmLk,1070
+perplexity_webui_scraper-0.3.6.dist-info/WHEEL,sha256=RRVLqVugUmFOqBedBFAmA4bsgFcROUBiSUKlERi0Hcg,79
+perplexity_webui_scraper-0.3.6.dist-info/entry_points.txt,sha256=ODpXpDTkmoQ_o3Y3lsy22PLs-8ndapvMKYwxcz6A9gs,189
+perplexity_webui_scraper-0.3.6.dist-info/METADATA,sha256=OkwWS6PQwj1-hTkMPXvqA8peuBL-ei5AUp_kgX9KpAo,12168
+perplexity_webui_scraper-0.3.6.dist-info/RECORD,,

{perplexity_webui_scraper-0.3.4.dist-info → perplexity_webui_scraper-0.3.6.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: uv 0.9.18
+Generator: uv 0.9.21
 Root-Is-Purelib: true
 Tag: py3-none-any

{perplexity_webui_scraper-0.3.4.dist-info → perplexity_webui_scraper-0.3.6.dist-info}/entry_points.txt

@@ -1,3 +1,4 @@
 [console_scripts]
 get-perplexity-session-token = perplexity_webui_scraper.cli.get_perplexity_session_token:get_token
+perplexity-webui-scraper-mcp = perplexity_webui_scraper.mcp:run_server
 

perplexity_webui_scraper-0.3.4.dist-info/RECORD

@@ -1,16 +0,0 @@
-perplexity_webui_scraper/__init__.py,sha256=aUs8lsx11WeqDi-ZT4Y2RiEhQVd6K66EZZ6x0BDuU90,949
-perplexity_webui_scraper/cli/get_perplexity_session_token.py,sha256=67Ck4S2MJ0701LnRHq73qY5oRLCsFOyu_8SMgsbTNFc,6937
-perplexity_webui_scraper/config.py,sha256=tjTwTFO39ONI4yOodqAov6sAXti18DiOOwFndpct68o,944
-perplexity_webui_scraper/constants.py,sha256=Kq-4i6yyTZ5VhUvbiZmbUmHrjMQm-p7H82Emm7b10-c,1867
-perplexity_webui_scraper/core.py,sha256=QGDjdd_h8fV7yE4ukV6iL8xuidxlxDzu5HvUx0W-0tA,14824
-perplexity_webui_scraper/enums.py,sha256=Xo7RmtWFxhSQU2Zma5sFmMyitOqlqjAb4XwRC0KJON0,2124
-perplexity_webui_scraper/exceptions.py,sha256=0oOWe_A0B0wBsFeogt323BGJY3oBzaFK9PItXs77J70,1629
-perplexity_webui_scraper/http.py,sha256=4x0LSCpKFtIR_izFYaGWXvNcewnXGnICFoPJNP892W8,5615
-perplexity_webui_scraper/limits.py,sha256=GwcwC8CnSNhlcLWGLpuDYA37gn8OXSfsXLIOc-QbxNs,465
-perplexity_webui_scraper/models.py,sha256=QVeZI-WQzpyi9JnE15QIMJ7nsG0YjIjOsZEA6YfX0tw,2448
-perplexity_webui_scraper/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-perplexity_webui_scraper/types.py,sha256=VlnzvNilIHrDXM2YOGjJa1y2VY0tfR-F0zaPjQHoPKs,1028
-perplexity_webui_scraper-0.3.4.dist-info/WHEEL,sha256=ZyFSCYkV2BrxH6-HRVRg3R9Fo7MALzer9KiPYqNxSbo,79
-perplexity_webui_scraper-0.3.4.dist-info/entry_points.txt,sha256=x98Wqg3iD6aXxtm27KmB-BXPFo1ccDDGRaCp6fWn9m4,118
-perplexity_webui_scraper-0.3.4.dist-info/METADATA,sha256=Mm1LlknQT4ipS2bT-mtrcnNvCSjgjrW79tyQU0mxu9s,8557
-perplexity_webui_scraper-0.3.4.dist-info/RECORD,,