litescrape-sdk 0.1.0__tar.gz

litescrape_sdk-0.1.0/.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

litescrape_sdk-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,21 @@
+name: ci
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: uv sync --frozen
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run pytest -q

litescrape_sdk-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,34 @@
+name: publish
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - name: Check that the tag matches the package version
+        run: |
+          version=$(python -c "import re, pathlib; print(re.search(r'\"([^\"]+)\"', pathlib.Path('src/litescrape_sdk/_version.py').read_text()).group(1))")
+          test "v$version" = "$GITHUB_REF_NAME" || { echo "tag $GITHUB_REF_NAME does not match version $version"; exit 1; }
+      - run: uv build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

litescrape_sdk-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+.venv/
+__pycache__/
+*.py[cod]
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.env

litescrape_sdk-0.1.0/.python-version

@@ -0,0 +1 @@
+3.11

litescrape_sdk-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Litescrape
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

litescrape_sdk-0.1.0/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.5
+Name: litescrape-sdk
+Version: 0.1.0
+Summary: Python SDK for the Litescrape API: validated, batched, retrying calls to search, maps, and reviews endpoints.
+Project-URL: Homepage, https://litescrape.com
+Project-URL: Source, https://github.com/litescrape/litescrape-sdk
+Author-email: Litescraper <support@litescrape.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: api,google,litescrape,maps,reviews,scraping,serp
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: httpx<1,>=0.27
+Requires-Dist: pydantic<3,>=2.7
+Requires-Dist: tqdm<5,>=4.66
+Description-Content-Type: text/markdown
+
+# litescrape-sdk
+
+Python SDK for the [Litescrape API](https://litescrape.com): validated, concurrent, retrying calls to the
+Google, Bing, DuckDuckGo, Yelp, Tripadvisor, and Apple Maps endpoints, with results returned in input order.
+
+```
+pip install litescrape-sdk
+```
+
+```python
+import os
+from litescrape_sdk import GoogleMaps, scrape
+
+os.environ["LITESCRAPE_API_KEY"] = "ls_live_..."
+
+results = scrape(
+    [
+        {"endpoint": "google_search", "q": "coffee grinders", "gl": "us"},
+        GoogleMaps(q="coffee", type="search", ll="@40.745,-73.988,14z"),
+    ]
+)
+for result in results:
+    print(result.ok, result.data or result.error)
+```
+
+`help(litescrape_sdk.scrape)` documents every argument, the retry and concurrency rules, and the error
+types. `ascrape` is the same function for asyncio code, and `litescrape_sdk.REQUEST_TYPES` maps each
+`endpoint` slug to its request class.
+
+Development: `uv sync`, then `uv run pytest`, `uv run ruff check .`, and `uv run ruff format .`.

litescrape_sdk-0.1.0/README.md

@@ -0,0 +1,30 @@
+# litescrape-sdk
+
+Python SDK for the [Litescrape API](https://litescrape.com): validated, concurrent, retrying calls to the
+Google, Bing, DuckDuckGo, Yelp, Tripadvisor, and Apple Maps endpoints, with results returned in input order.
+
+```
+pip install litescrape-sdk
+```
+
+```python
+import os
+from litescrape_sdk import GoogleMaps, scrape
+
+os.environ["LITESCRAPE_API_KEY"] = "ls_live_..."
+
+results = scrape(
+    [
+        {"endpoint": "google_search", "q": "coffee grinders", "gl": "us"},
+        GoogleMaps(q="coffee", type="search", ll="@40.745,-73.988,14z"),
+    ]
+)
+for result in results:
+    print(result.ok, result.data or result.error)
+```
+
+`help(litescrape_sdk.scrape)` documents every argument, the retry and concurrency rules, and the error
+types. `ascrape` is the same function for asyncio code, and `litescrape_sdk.REQUEST_TYPES` maps each
+`endpoint` slug to its request class.
+
+Development: `uv sync`, then `uv run pytest`, `uv run ruff check .`, and `uv run ruff format .`.

litescrape_sdk-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "litescrape-sdk"
+dynamic = ["version"]
+description = "Python SDK for the Litescrape API: validated, batched, retrying calls to search, maps, and reviews endpoints."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.10"
+authors = [{ name = "Litescraper", email = "support@litescrape.com" }]
+keywords = ["litescrape", "serp", "google", "maps", "reviews", "scraping", "api"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "License :: OSI Approved :: MIT License",
+  "Typing :: Typed",
+]
+dependencies = [
+  "httpx>=0.27,<1",
+  "pydantic>=2.7,<3",
+  "tqdm>=4.66,<5",
+]
+
+[project.urls]
+Homepage = "https://litescrape.com"
+Source = "https://github.com/litescrape/litescrape-sdk"
+
+[dependency-groups]
+dev = [
+  "pytest>=8,<9",
+  "ruff>=0.12,<1",
+]
+
+[tool.hatch.version]
+path = "src/litescrape_sdk/_version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/litescrape_sdk"]
+
+[tool.ruff]
+line-length = 110
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q"

litescrape_sdk-0.1.0/src/litescrape_sdk/__init__.py

@@ -0,0 +1,91 @@
+"""Python SDK for the Litescrape API. Start with ``help(litescrape_sdk.scrape)``."""
+
+from ._version import __version__
+from .client import Result, akey_status, ascrape, key_status, scrape
+from .errors import (
+    APIError,
+    AuthenticationError,
+    LitescrapeError,
+    NotFoundError,
+    PaymentRequiredError,
+    RateLimitError,
+    TransportError,
+    ValidationError,
+)
+from .models import (
+    REQUEST_TYPES,
+    AnyRequest,
+    AppleMapsPlaces,
+    AppleMapsReviews,
+    BingMaps,
+    BingSearch,
+    DuckDuckGoMaps,
+    DuckDuckGoSearch,
+    GoogleAds,
+    GoogleAiMode,
+    GoogleAiOverview,
+    GoogleContributorReviews,
+    GoogleLocal,
+    GoogleMaps,
+    GoogleMapsLiveFootTraffic,
+    GoogleMapsPhoto,
+    GoogleMapsPosts,
+    GoogleMapsWebResults,
+    GoogleReviews,
+    GoogleSearch,
+    GoogleShopping,
+    GoogleShoppingProduct,
+    KeyStatus,
+    ScrapeRequest,
+    TripadvisorPlace,
+    TripadvisorReviews,
+    TripadvisorSearch,
+    YelpReviews,
+    YelpSearch,
+)
+
+__all__ = [
+    "__version__",
+    "scrape",
+    "ascrape",
+    "key_status",
+    "akey_status",
+    "Result",
+    "KeyStatus",
+    "ScrapeRequest",
+    "AnyRequest",
+    "REQUEST_TYPES",
+    "LitescrapeError",
+    "ValidationError",
+    "TransportError",
+    "APIError",
+    "AuthenticationError",
+    "PaymentRequiredError",
+    "NotFoundError",
+    "RateLimitError",
+    "GoogleSearch",
+    "GoogleAiOverview",
+    "GoogleAiMode",
+    "GoogleAds",
+    "GoogleShopping",
+    "GoogleShoppingProduct",
+    "GoogleLocal",
+    "GoogleMaps",
+    "GoogleMapsLiveFootTraffic",
+    "GoogleMapsPosts",
+    "GoogleMapsPhoto",
+    "GoogleMapsWebResults",
+    "GoogleReviews",
+    "GoogleContributorReviews",
+    "BingSearch",
+    "BingMaps",
+    "DuckDuckGoSearch",
+    "DuckDuckGoMaps",
+    "YelpSearch",
+    "YelpReviews",
+    "TripadvisorSearch",
+    "TripadvisorPlace",
+    "TripadvisorReviews",
+    "AppleMapsPlaces",
+    "AppleMapsReviews",
+]

litescrape_sdk-0.1.0/src/litescrape_sdk/_runtime.py

@@ -0,0 +1,201 @@
+"""Event loop, HTTP client, concurrency gate, and retry loop behind the public functions."""
+
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import contextlib
+import random
+import sys
+import threading
+from collections.abc import Callable, Coroutine
+from dataclasses import dataclass
+from typing import Any, TypeVar
+
+import httpx
+
+from ._version import __version__
+from .errors import APIError, LitescrapeError, TransportError, api_error_from_response
+
+DEFAULT_BASE_URL = "https://api.litescrape.com"
+DEFAULT_CONCURRENCY = 25
+SELECTOR_LOOP_CAP = 500
+BACKOFF_CAP = 30.0
+CONNECT_TIMEOUT = 10.0
+
+_RETRY_CODES = frozenset(
+    {"proxy_capacity_unavailable", "upstream_session_unavailable", "service_unavailable"}
+)
+_RETRY_TRANSPORT = (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError)
+
+T = TypeVar("T")
+_LoopEntry = tuple[asyncio.AbstractEventLoop, dict[Any, Any]]
+
+_clients: dict[int, _LoopEntry] = {}
+_semaphores: dict[int, _LoopEntry] = {}
+_sleep = asyncio.sleep
+
+
+@dataclass(slots=True)
+class Outcome:
+    data: dict[str, Any] | None = None
+    error: LitescrapeError | None = None
+    status_code: int | None = None
+    request_id: str = ""
+    attempts: int = 0
+
+
+def _per_loop(registry: dict[int, _LoopEntry]) -> dict[Any, Any]:
+    loop = asyncio.get_running_loop()
+    for key, (known, _) in list(registry.items()):
+        if known.is_closed():
+            del registry[key]
+    return registry.setdefault(id(loop), (loop, {}))[1]
+
+
+def _make_client(base_url: str) -> httpx.AsyncClient:
+    return httpx.AsyncClient(
+        base_url=base_url,
+        headers={"Accept": "application/json", "User-Agent": f"litescrape-sdk/{__version__}"},
+        limits=httpx.Limits(max_connections=None, max_keepalive_connections=None),
+        timeout=httpx.Timeout(120.0, connect=CONNECT_TIMEOUT),
+    )
+
+
+def client_for(base_url: str) -> httpx.AsyncClient:
+    clients = _per_loop(_clients)
+    client = clients.get(base_url)
+    if client is None:
+        client = clients[base_url] = _make_client(base_url)
+    return client
+
+
+def semaphore_for(api_key: str, base_url: str, limit: int) -> asyncio.Semaphore:
+    semaphores = _per_loop(_semaphores)
+    entry = semaphores.get((api_key, base_url))
+    if entry is None or entry[0] != limit:
+        entry = semaphores[(api_key, base_url)] = (limit, asyncio.Semaphore(limit))
+    return entry[1]
+
+
+def is_selector_limited(loop: asyncio.AbstractEventLoop) -> bool:
+    return sys.platform == "win32" and isinstance(loop, asyncio.SelectorEventLoop)
+
+
+def effective_limit(status_limit: int | None, override: int | None, *, selector_limited: bool) -> int:
+    limit = status_limit or DEFAULT_CONCURRENCY
+    if override is not None:
+        limit = min(limit, override)
+    if selector_limited:
+        limit = min(limit, SELECTOR_LOOP_CAP)
+    return max(1, limit)
+
+
+def _should_retry(error: LitescrapeError) -> bool:
+    if isinstance(error, TransportError):
+        return error.retryable
+    if isinstance(error, APIError):
+        status = error.status_code or 0
+        return error.retryable or status == 429 or status >= 500 or error.error_code in _RETRY_CODES
+    return False
+
+
+def _delay(attempt: int, retry_after: float | None) -> float:
+    if retry_after is not None:
+        return min(max(retry_after, 0.0), BACKOFF_CAP)
+    ceiling = min(BACKOFF_CAP, float(2 ** (attempt - 1)))
+    return random.uniform(ceiling / 2, ceiling)
+
+
+async def _attempt(
+    client: httpx.AsyncClient,
+    path: str,
+    params: dict[str, str],
+    headers: dict[str, str],
+    timeout: float,
+    outcome: Outcome,
+) -> LitescrapeError | None:
+    outcome.status_code, outcome.request_id = None, ""
+    try:
+        response = await client.get(
+            path,
+            params=params,
+            headers=headers,
+            timeout=httpx.Timeout(timeout, connect=min(CONNECT_TIMEOUT, timeout)),
+        )
+    except _RETRY_TRANSPORT as exc:
+        return TransportError(f"{type(exc).__name__}: {exc}", retryable=True, cause=exc)
+    except Exception as exc:
+        return TransportError(f"{type(exc).__name__}: {exc}", retryable=False, cause=exc)
+    outcome.status_code = response.status_code
+    outcome.request_id = response.headers.get("x-request-id", "")
+    if response.status_code != 200:
+        error = api_error_from_response(response)
+        outcome.request_id = error.request_id or outcome.request_id
+        return error
+    try:
+        data = response.json()
+    except ValueError as exc:
+        return TransportError("Response body was not JSON", retryable=True, cause=exc)
+    if not isinstance(data, dict):
+        return TransportError("Response body was not a JSON object", retryable=True)
+    outcome.data = data
+    return None
+
+
+async def request_with_retries(
+    client: httpx.AsyncClient,
+    path: str,
+    params: dict[str, str],
+    *,
+    headers: dict[str, str],
+    attempts: int,
+    timeout: float,
+    semaphore: asyncio.Semaphore | None = None,
+    gate: Callable[[], LitescrapeError | None] | None = None,
+) -> Outcome:
+    outcome = Outcome()
+    slot = semaphore if semaphore is not None else contextlib.nullcontext()
+    for attempt in range(1, attempts + 1):
+        async with slot:
+            blocked = gate() if gate is not None else None
+            if blocked is not None:
+                outcome.error = blocked
+                outcome.status_code = getattr(blocked, "status_code", None)
+                return outcome
+            outcome.attempts = attempt
+            error = await _attempt(client, path, params, headers, timeout, outcome)
+        outcome.error = error
+        if error is None:
+            return outcome
+        if attempt == attempts or not _should_retry(error):
+            return outcome
+        await _sleep(_delay(attempt, getattr(error, "retry_after", None)))
+    return outcome
+
+
+_loop_lock = threading.Lock()
+_loop: asyncio.AbstractEventLoop | None = None
+
+
+def background_loop() -> asyncio.AbstractEventLoop:
+    global _loop
+    with _loop_lock:
+        if _loop is None or _loop.is_closed():
+            loop = asyncio.ProactorEventLoop() if sys.platform == "win32" else asyncio.new_event_loop()
+            threading.Thread(target=loop.run_forever, name="litescrape-sdk", daemon=True).start()
+            _loop = loop
+        return _loop
+
+
+def run_sync(coro: Coroutine[Any, Any, T]) -> T:
+    future = asyncio.run_coroutine_threadsafe(coro, background_loop())
+    try:
+        while True:
+            try:
+                return future.result(timeout=0.25)
+            except concurrent.futures.TimeoutError:
+                continue
+    except BaseException:
+        future.cancel()
+        raise

litescrape_sdk-0.1.0/src/litescrape_sdk/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"