scrapecrate 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapecrate-1.0.0/.gitignore +32 -0
- scrapecrate-1.0.0/LICENSE +21 -0
- scrapecrate-1.0.0/PKG-INFO +95 -0
- scrapecrate-1.0.0/README.md +72 -0
- scrapecrate-1.0.0/pyproject.toml +46 -0
- scrapecrate-1.0.0/src/scrapecrate/__init__.py +32 -0
- scrapecrate-1.0.0/src/scrapecrate/_base.py +238 -0
- scrapecrate-1.0.0/src/scrapecrate/_client.py +100 -0
- scrapecrate-1.0.0/src/scrapecrate/_generated.py +473 -0
- scrapecrate-1.0.0/src/scrapecrate/errors.py +81 -0
- scrapecrate-1.0.0/src/scrapecrate/py.typed +0 -0
- scrapecrate-1.0.0/uv.lock +401 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Virtual environment
|
|
2
|
+
.venv/
|
|
3
|
+
venv/
|
|
4
|
+
env/
|
|
5
|
+
|
|
6
|
+
# Environment variables (contains API key)
|
|
7
|
+
.env
|
|
8
|
+
|
|
9
|
+
# Python cache
|
|
10
|
+
__pycache__/
|
|
11
|
+
*.py[cod]
|
|
12
|
+
*$py.class
|
|
13
|
+
*.so
|
|
14
|
+
|
|
15
|
+
# Distribution
|
|
16
|
+
dist/
|
|
17
|
+
build/
|
|
18
|
+
*.egg-info/
|
|
19
|
+
|
|
20
|
+
# IDE
|
|
21
|
+
.idea/
|
|
22
|
+
.vscode/
|
|
23
|
+
*.swp
|
|
24
|
+
*.swo
|
|
25
|
+
|
|
26
|
+
# Test artifacts
|
|
27
|
+
.pytest_cache/
|
|
28
|
+
.coverage
|
|
29
|
+
htmlcov/
|
|
30
|
+
|
|
31
|
+
# Output directory (test results)
|
|
32
|
+
output/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ScrapeCrate
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: scrapecrate
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Official Python SDK for the ScrapeCrate API: YouTube, Google Maps, Trustpilot and Meta Ad Library data with one API key
|
|
5
|
+
Project-URL: Homepage, https://scrapecrate.com
|
|
6
|
+
Project-URL: Documentation, https://scrapecrate.com/docs/sdks
|
|
7
|
+
Author: ScrapeCrate
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: api,google maps,meta ad library,scraping,sdk,trustpilot,web data,youtube
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Typing :: Typed
|
|
16
|
+
Requires-Python: >=3.9
|
|
17
|
+
Requires-Dist: httpx>=0.27.0
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
|
|
20
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
21
|
+
Requires-Dist: ruff>=0.6.0; extra == 'dev'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# ScrapeCrate Python SDK
|
|
25
|
+
|
|
26
|
+
One API key for YouTube, Google Maps, Trustpilot and Meta Ad Library data.
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install scrapecrate
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from scrapecrate import ScrapeCrate
|
|
34
|
+
|
|
35
|
+
client = ScrapeCrate(api_key="YOUR_API_KEY") # or set SCRAPECRATE_API_KEY
|
|
36
|
+
|
|
37
|
+
t = client.youtube.transcript(video_id="dQw4w9WgXcQ", lang="en")
|
|
38
|
+
print(t.data["text"], t.credits_used, t.credits_remaining)
|
|
39
|
+
|
|
40
|
+
places = client.maps.places_search(q="coffee in Berlin", limit=20)
|
|
41
|
+
for review in client.maps.iterate("place_reviews", place_id=places.data[0]["id"], sort="newest"):
|
|
42
|
+
print(review["rating"], review["text"])
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
- One namespace per API (`youtube`, `maps`, `trustpilot`, `ads`), one method per endpoint, same
|
|
46
|
+
parameter names as the HTTP API. `iterate()` follows cursors.
|
|
47
|
+
- `AsyncScrapeCrate` has the same methods (awaitable, `async for` with `iterate`).
|
|
48
|
+
- Typed errors: `ValidationError`, `AuthenticationError`, `InsufficientCreditsError`,
|
|
49
|
+
`PermissionDeniedError`, `NotFoundError`, `RateLimitError`, `ServiceUnavailableError` (all `ApiError`), see below.
|
|
50
|
+
- Retries 429/5xx with backoff (failed calls are never billed).
|
|
51
|
+
|
|
52
|
+
## Errors
|
|
53
|
+
|
|
54
|
+
Every API error arrives as `{"error": {"code", "message", "request_id", "details"?}}` and is never billed.
|
|
55
|
+
The SDK raises it as a typed exception with the same fields:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from scrapecrate import ScrapeCrate, ApiError, NotFoundError, InsufficientCreditsError
|
|
59
|
+
|
|
60
|
+
client = ScrapeCrate()
|
|
61
|
+
try:
|
|
62
|
+
company = client.trustpilot.company(domain="example.com").data
|
|
63
|
+
except NotFoundError:
|
|
64
|
+
company = None # 404 not_found (or a specific *_not_found code)
|
|
65
|
+
except InsufficientCreditsError as e:
|
|
66
|
+
print(e.details) # {"required": 3, "available": 0}
|
|
67
|
+
except ApiError as e:
|
|
68
|
+
print(e.status, e.code, e.message, e.request_id, e.details, e.retry_after)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
| Exception | Status | Codes |
|
|
72
|
+
|---|---|---|
|
|
73
|
+
| `ValidationError` | 400 | `invalid_parameter` (`details`: `[{"param", "message"}]`), `invalid_cursor` |
|
|
74
|
+
| `AuthenticationError` | 401 | `missing_api_key`, `invalid_api_key` |
|
|
75
|
+
| `InsufficientCreditsError` | 402 | `insufficient_credits` (`details`: `{"required", "available"}`) |
|
|
76
|
+
| `PermissionDeniedError` | 403 | `account_disabled` |
|
|
77
|
+
| `NotFoundError` | 404 | `not_found`, `*_not_found` |
|
|
78
|
+
| `RateLimitError` | 429 | `rate_limited`, `too_many_concurrent_requests`, `insufficient_credits_retry_later` (`retry_after` set) |
|
|
79
|
+
| `ServiceUnavailableError` | 5xx | `source_unavailable`, `source_busy`, `source_timeout`, `source_error`, `internal_error`, `not_available` |
|
|
80
|
+
| `ApiError` | other | e.g. 410 `cursor_expired`, 405 `method_not_allowed` |
|
|
81
|
+
| `ConnectionError` | — | the API could not be reached |
|
|
82
|
+
|
|
83
|
+
429 and 5xx answers are retried up to `max_retries` (default 3) before raising. Branch on `e.code`, not only on the
|
|
84
|
+
class: `not_available` (503) means the API is not launched yet and will not succeed on retry, while
|
|
85
|
+
`source_unavailable` is temporary. Full list: https://scrapecrate.com/docs/errors
|
|
86
|
+
|
|
87
|
+
Docs: https://scrapecrate.com/docs/sdks · API reference: https://scrapecrate.com/docs/api
|
|
88
|
+
|
|
89
|
+
## Development
|
|
90
|
+
|
|
91
|
+
The endpoint methods are generated from the catalog: `python3 sdks/generate.py` (repo root).
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
uv venv && uv pip install -e ".[dev]" && pytest
|
|
95
|
+
```
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# ScrapeCrate Python SDK
|
|
2
|
+
|
|
3
|
+
One API key for YouTube, Google Maps, Trustpilot and Meta Ad Library data.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install scrapecrate
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
from scrapecrate import ScrapeCrate
|
|
11
|
+
|
|
12
|
+
client = ScrapeCrate(api_key="YOUR_API_KEY") # or set SCRAPECRATE_API_KEY
|
|
13
|
+
|
|
14
|
+
t = client.youtube.transcript(video_id="dQw4w9WgXcQ", lang="en")
|
|
15
|
+
print(t.data["text"], t.credits_used, t.credits_remaining)
|
|
16
|
+
|
|
17
|
+
places = client.maps.places_search(q="coffee in Berlin", limit=20)
|
|
18
|
+
for review in client.maps.iterate("place_reviews", place_id=places.data[0]["id"], sort="newest"):
|
|
19
|
+
print(review["rating"], review["text"])
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
- One namespace per API (`youtube`, `maps`, `trustpilot`, `ads`), one method per endpoint, same
|
|
23
|
+
parameter names as the HTTP API. `iterate()` follows cursors.
|
|
24
|
+
- `AsyncScrapeCrate` has the same methods (awaitable, `async for` with `iterate`).
|
|
25
|
+
- Typed errors: `ValidationError`, `AuthenticationError`, `InsufficientCreditsError`,
|
|
26
|
+
`PermissionDeniedError`, `NotFoundError`, `RateLimitError`, `ServiceUnavailableError` (all `ApiError`), see below.
|
|
27
|
+
- Retries 429/5xx with backoff (failed calls are never billed).
|
|
28
|
+
|
|
29
|
+
## Errors
|
|
30
|
+
|
|
31
|
+
Every API error arrives as `{"error": {"code", "message", "request_id", "details"?}}` and is never billed.
|
|
32
|
+
The SDK raises it as a typed exception with the same fields:
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from scrapecrate import ScrapeCrate, ApiError, NotFoundError, InsufficientCreditsError
|
|
36
|
+
|
|
37
|
+
client = ScrapeCrate()
|
|
38
|
+
try:
|
|
39
|
+
company = client.trustpilot.company(domain="example.com").data
|
|
40
|
+
except NotFoundError:
|
|
41
|
+
company = None # 404 not_found (or a specific *_not_found code)
|
|
42
|
+
except InsufficientCreditsError as e:
|
|
43
|
+
print(e.details) # {"required": 3, "available": 0}
|
|
44
|
+
except ApiError as e:
|
|
45
|
+
print(e.status, e.code, e.message, e.request_id, e.details, e.retry_after)
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
| Exception | Status | Codes |
|
|
49
|
+
|---|---|---|
|
|
50
|
+
| `ValidationError` | 400 | `invalid_parameter` (`details`: `[{"param", "message"}]`), `invalid_cursor` |
|
|
51
|
+
| `AuthenticationError` | 401 | `missing_api_key`, `invalid_api_key` |
|
|
52
|
+
| `InsufficientCreditsError` | 402 | `insufficient_credits` (`details`: `{"required", "available"}`) |
|
|
53
|
+
| `PermissionDeniedError` | 403 | `account_disabled` |
|
|
54
|
+
| `NotFoundError` | 404 | `not_found`, `*_not_found` |
|
|
55
|
+
| `RateLimitError` | 429 | `rate_limited`, `too_many_concurrent_requests`, `insufficient_credits_retry_later` (`retry_after` set) |
|
|
56
|
+
| `ServiceUnavailableError` | 5xx | `source_unavailable`, `source_busy`, `source_timeout`, `source_error`, `internal_error`, `not_available` |
|
|
57
|
+
| `ApiError` | other | e.g. 410 `cursor_expired`, 405 `method_not_allowed` |
|
|
58
|
+
| `ConnectionError` | — | the API could not be reached |
|
|
59
|
+
|
|
60
|
+
429 and 5xx answers are retried up to `max_retries` (default 3) before raising. Branch on `e.code`, not only on the
|
|
61
|
+
class: `not_available` (503) means the API is not launched yet and will not succeed on retry, while
|
|
62
|
+
`source_unavailable` is temporary. Full list: https://scrapecrate.com/docs/errors
|
|
63
|
+
|
|
64
|
+
Docs: https://scrapecrate.com/docs/sdks · API reference: https://scrapecrate.com/docs/api
|
|
65
|
+
|
|
66
|
+
## Development
|
|
67
|
+
|
|
68
|
+
The endpoint methods are generated from the catalog: `python3 sdks/generate.py` (repo root).
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
uv venv && uv pip install -e ".[dev]" && pytest
|
|
72
|
+
```
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scrapecrate"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Official Python SDK for the ScrapeCrate API: YouTube, Google Maps, Trustpilot and Meta Ad Library data with one API key"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [{ name = "ScrapeCrate" }]
|
|
13
|
+
keywords = ["api", "sdk", "scraping", "youtube", "google maps", "trustpilot", "meta ad library", "web data"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 5 - Production/Stable",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Typing :: Typed",
|
|
20
|
+
]
|
|
21
|
+
dependencies = ["httpx>=0.27.0"]
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
dev = ["pytest>=8.0.0", "pytest-asyncio>=0.23.0", "ruff>=0.6.0"]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://scrapecrate.com"
|
|
28
|
+
Documentation = "https://scrapecrate.com/docs/sdks"
|
|
29
|
+
|
|
30
|
+
[tool.hatch.build.targets.wheel]
|
|
31
|
+
packages = ["src/scrapecrate"]
|
|
32
|
+
|
|
33
|
+
[tool.pytest.ini_options]
|
|
34
|
+
asyncio_mode = "auto"
|
|
35
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
36
|
+
|
|
37
|
+
[tool.ruff]
|
|
38
|
+
line-length = 120
|
|
39
|
+
extend-exclude = ["src/scrapecrate/_generated.py"]
|
|
40
|
+
target-version = "py39"
|
|
41
|
+
|
|
42
|
+
[tool.ruff.lint]
|
|
43
|
+
select = ["E", "W", "F", "I", "B", "UP"]
|
|
44
|
+
|
|
45
|
+
[tool.hatch.build.targets.sdist]
|
|
46
|
+
exclude = ["tests"]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Official Python SDK of the ScrapeCrate API: one key for YouTube, Google Maps, Trustpilot
|
|
2
|
+
and Meta Ad Library data."""
|
|
3
|
+
|
|
4
|
+
from ._base import Response, __version__
|
|
5
|
+
from ._client import AsyncScrapeCrate, ScrapeCrate
|
|
6
|
+
from .errors import (
|
|
7
|
+
ApiError,
|
|
8
|
+
AuthenticationError,
|
|
9
|
+
ConnectionError,
|
|
10
|
+
InsufficientCreditsError,
|
|
11
|
+
NotFoundError,
|
|
12
|
+
PermissionDeniedError,
|
|
13
|
+
RateLimitError,
|
|
14
|
+
ServiceUnavailableError,
|
|
15
|
+
ValidationError,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"ApiError",
|
|
20
|
+
"AsyncScrapeCrate",
|
|
21
|
+
"AuthenticationError",
|
|
22
|
+
"ConnectionError",
|
|
23
|
+
"InsufficientCreditsError",
|
|
24
|
+
"NotFoundError",
|
|
25
|
+
"PermissionDeniedError",
|
|
26
|
+
"RateLimitError",
|
|
27
|
+
"Response",
|
|
28
|
+
"ScrapeCrate",
|
|
29
|
+
"ServiceUnavailableError",
|
|
30
|
+
"ValidationError",
|
|
31
|
+
"__version__",
|
|
32
|
+
]
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"""HTTP transport (sync and async), response object and API namespaces."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import os
|
|
7
|
+
import random
|
|
8
|
+
import time
|
|
9
|
+
from collections.abc import AsyncIterator, Iterator
|
|
10
|
+
from dataclasses import dataclass, field
|
|
11
|
+
from typing import TYPE_CHECKING, Any
|
|
12
|
+
from urllib.parse import quote
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
|
|
16
|
+
from .errors import ApiError, ConnectionError, error_for
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from ._client import AsyncScrapeCrate, ScrapeCrate
|
|
20
|
+
|
|
21
|
+
__version__ = "1.0.0"
|
|
22
|
+
DEFAULT_BASE_URL = "https://api.scrapecrate.com"
|
|
23
|
+
ENV_API_KEY = "SCRAPECRATE_API_KEY"
|
|
24
|
+
ENV_BASE_URL = "SCRAPECRATE_BASE_URL"
|
|
25
|
+
RETRY_STATUSES = {429, 500, 502, 503, 504}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class Response:
|
|
30
|
+
"""A successful API response."""
|
|
31
|
+
|
|
32
|
+
data: Any
|
|
33
|
+
meta: dict = field(default_factory=dict)
|
|
34
|
+
next_cursor: str | None = None
|
|
35
|
+
has_more: bool = False
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def credits_used(self) -> int:
|
|
39
|
+
return int(self.meta.get("credits_used", 0))
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def credits_remaining(self) -> int:
|
|
43
|
+
return int(self.meta.get("credits_remaining", 0))
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def request_id(self) -> str | None:
|
|
47
|
+
return self.meta.get("request_id")
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def from_body(cls, body: dict) -> Response:
|
|
51
|
+
pag = body.get("pagination") or {}
|
|
52
|
+
return cls(
|
|
53
|
+
data=body.get("data"),
|
|
54
|
+
meta=body.get("meta") or {},
|
|
55
|
+
next_cursor=pag.get("next_cursor"),
|
|
56
|
+
has_more=bool(pag.get("has_more")),
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _clean(params: dict[str, Any]) -> dict[str, str]:
|
|
61
|
+
out: dict[str, str] = {}
|
|
62
|
+
for k, v in params.items():
|
|
63
|
+
if v is None:
|
|
64
|
+
continue
|
|
65
|
+
out[k] = ("true" if v else "false") if isinstance(v, bool) else str(v)
|
|
66
|
+
return out
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def build_request(path: str, path_params: tuple[str, ...], params: dict[str, Any]) -> tuple[str, dict]:
|
|
70
|
+
query = dict(params)
|
|
71
|
+
for name in path_params:
|
|
72
|
+
value = query.pop(name, None)
|
|
73
|
+
if value is None:
|
|
74
|
+
raise ValueError(f"missing required parameter: {name}")
|
|
75
|
+
path = path.replace("{" + name + "}", quote(str(value), safe=""))
|
|
76
|
+
return path, _clean(query)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _retry_after(resp: httpx.Response) -> int | None:
|
|
80
|
+
try:
|
|
81
|
+
return int(resp.headers.get("Retry-After", ""))
|
|
82
|
+
except ValueError:
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _backoff(attempt: int, retry_after: int | None) -> float:
|
|
87
|
+
if retry_after is not None:
|
|
88
|
+
return float(min(retry_after, 60))
|
|
89
|
+
return min(0.5 * 2**attempt, 8.0) + random.random() * 0.25
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
NO_RETRY_CODES = {"not_available"} # the API is not launched: retrying never helps
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _retryable(resp: httpx.Response) -> bool:
|
|
96
|
+
if resp.status_code not in RETRY_STATUSES:
|
|
97
|
+
return False
|
|
98
|
+
body = _json(resp)
|
|
99
|
+
code = body.get("error", {}).get("code") if isinstance(body, dict) else None
|
|
100
|
+
return code not in NO_RETRY_CODES
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class _Config:
|
|
104
|
+
def __init__(
|
|
105
|
+
self,
|
|
106
|
+
api_key: str | None,
|
|
107
|
+
base_url: str | None,
|
|
108
|
+
timeout: float,
|
|
109
|
+
max_retries: int,
|
|
110
|
+
) -> None:
|
|
111
|
+
self.api_key = api_key or os.environ.get(ENV_API_KEY)
|
|
112
|
+
if not self.api_key:
|
|
113
|
+
raise ValueError(f"No API key: pass api_key=... or set {ENV_API_KEY}")
|
|
114
|
+
self.base_url = (base_url or os.environ.get(ENV_BASE_URL) or DEFAULT_BASE_URL).rstrip("/")
|
|
115
|
+
self.timeout = timeout
|
|
116
|
+
self.max_retries = max_retries
|
|
117
|
+
|
|
118
|
+
@property
|
|
119
|
+
def headers(self) -> dict[str, str]:
|
|
120
|
+
return {"X-API-Key": self.api_key or "", "User-Agent": f"scrapecrate-python/{__version__}"}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class SyncTransport:
|
|
124
|
+
def __init__(self, config: _Config, http_client: httpx.Client | None = None) -> None:
|
|
125
|
+
self.config = config
|
|
126
|
+
self._http = http_client or httpx.Client(timeout=config.timeout)
|
|
127
|
+
|
|
128
|
+
def get(self, path: str, query: dict[str, str]) -> Response:
|
|
129
|
+
attempt = 0
|
|
130
|
+
while True:
|
|
131
|
+
try:
|
|
132
|
+
resp = self._http.get(self.config.base_url + path, params=query, headers=self.config.headers)
|
|
133
|
+
except httpx.HTTPError as e:
|
|
134
|
+
if attempt < self.config.max_retries:
|
|
135
|
+
time.sleep(_backoff(attempt, None))
|
|
136
|
+
attempt += 1
|
|
137
|
+
continue
|
|
138
|
+
raise ConnectionError(0, "connection_error", str(e)) from e
|
|
139
|
+
if resp.status_code == 200:
|
|
140
|
+
return Response.from_body(resp.json())
|
|
141
|
+
ra = _retry_after(resp)
|
|
142
|
+
if _retryable(resp) and attempt < self.config.max_retries:
|
|
143
|
+
time.sleep(_backoff(attempt, ra))
|
|
144
|
+
attempt += 1
|
|
145
|
+
continue
|
|
146
|
+
raise error_for(resp.status_code, _json(resp), ra)
|
|
147
|
+
|
|
148
|
+
def close(self) -> None:
|
|
149
|
+
self._http.close()
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class AsyncTransport:
|
|
153
|
+
def __init__(self, config: _Config, http_client: httpx.AsyncClient | None = None) -> None:
|
|
154
|
+
self.config = config
|
|
155
|
+
self._http = http_client or httpx.AsyncClient(timeout=config.timeout)
|
|
156
|
+
|
|
157
|
+
async def get(self, path: str, query: dict[str, str]) -> Response:
|
|
158
|
+
attempt = 0
|
|
159
|
+
while True:
|
|
160
|
+
try:
|
|
161
|
+
resp = await self._http.get(self.config.base_url + path, params=query, headers=self.config.headers)
|
|
162
|
+
except httpx.HTTPError as e:
|
|
163
|
+
if attempt < self.config.max_retries:
|
|
164
|
+
await asyncio.sleep(_backoff(attempt, None))
|
|
165
|
+
attempt += 1
|
|
166
|
+
continue
|
|
167
|
+
raise ConnectionError(0, "connection_error", str(e)) from e
|
|
168
|
+
if resp.status_code == 200:
|
|
169
|
+
return Response.from_body(resp.json())
|
|
170
|
+
ra = _retry_after(resp)
|
|
171
|
+
if _retryable(resp) and attempt < self.config.max_retries:
|
|
172
|
+
await asyncio.sleep(_backoff(attempt, ra))
|
|
173
|
+
attempt += 1
|
|
174
|
+
continue
|
|
175
|
+
raise error_for(resp.status_code, _json(resp), ra)
|
|
176
|
+
|
|
177
|
+
async def close(self) -> None:
|
|
178
|
+
await self._http.aclose()
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _json(resp: httpx.Response) -> Any:
|
|
182
|
+
try:
|
|
183
|
+
return resp.json()
|
|
184
|
+
except ValueError:
|
|
185
|
+
return resp.text
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
class Namespace:
|
|
189
|
+
_api = ""
|
|
190
|
+
|
|
191
|
+
def __init__(self, client: ScrapeCrate) -> None:
|
|
192
|
+
self._client = client
|
|
193
|
+
|
|
194
|
+
def _call(self, key: str, params: dict[str, Any], extra: dict[str, Any]) -> Response:
|
|
195
|
+
from ._generated import ENDPOINTS
|
|
196
|
+
|
|
197
|
+
path, path_params, _ = ENDPOINTS[key]
|
|
198
|
+
p, q = build_request(path, path_params, {**params, **extra})
|
|
199
|
+
return self._client._transport.get(p, q)
|
|
200
|
+
|
|
201
|
+
def iterate(self, method: str, **params: Any) -> Iterator[Any]:
|
|
202
|
+
"""Yield every item of a list endpoint, following the cursor across pages."""
|
|
203
|
+
fn = getattr(self, method)
|
|
204
|
+
cursor = params.pop("cursor", None)
|
|
205
|
+
while True:
|
|
206
|
+
page = fn(**params, cursor=cursor) if cursor else fn(**params)
|
|
207
|
+
yield from page.data or []
|
|
208
|
+
if not page.has_more or not page.next_cursor:
|
|
209
|
+
return
|
|
210
|
+
cursor = page.next_cursor
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
class AsyncNamespace:
|
|
214
|
+
_api = ""
|
|
215
|
+
|
|
216
|
+
def __init__(self, client: AsyncScrapeCrate) -> None:
|
|
217
|
+
self._client = client
|
|
218
|
+
|
|
219
|
+
async def _call(self, key: str, params: dict[str, Any], extra: dict[str, Any]) -> Response:
|
|
220
|
+
from ._generated import ENDPOINTS
|
|
221
|
+
|
|
222
|
+
path, path_params, _ = ENDPOINTS[key]
|
|
223
|
+
p, q = build_request(path, path_params, {**params, **extra})
|
|
224
|
+
return await self._client._transport.get(p, q)
|
|
225
|
+
|
|
226
|
+
async def iterate(self, method: str, **params: Any) -> AsyncIterator[Any]:
|
|
227
|
+
fn = getattr(self, method)
|
|
228
|
+
cursor = params.pop("cursor", None)
|
|
229
|
+
while True:
|
|
230
|
+
page = await (fn(**params, cursor=cursor) if cursor else fn(**params))
|
|
231
|
+
for item in page.data or []:
|
|
232
|
+
yield item
|
|
233
|
+
if not page.has_more or not page.next_cursor:
|
|
234
|
+
return
|
|
235
|
+
cursor = page.next_cursor
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
__all__ = ["ApiError", "Response"]
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Clients: one namespace per API (client.youtube, client.maps, client.trustpilot, client.ads)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from ._base import AsyncTransport, Response, SyncTransport, _Config
|
|
10
|
+
from ._generated import (
|
|
11
|
+
AdsApi,
|
|
12
|
+
AsyncAdsApi,
|
|
13
|
+
AsyncMapsApi,
|
|
14
|
+
AsyncTrustpilotApi,
|
|
15
|
+
AsyncYoutubeApi,
|
|
16
|
+
MapsApi,
|
|
17
|
+
TrustpilotApi,
|
|
18
|
+
YoutubeApi,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ScrapeCrate:
|
|
23
|
+
"""Synchronous client.
|
|
24
|
+
|
|
25
|
+
>>> client = ScrapeCrate(api_key="sc_live_...") # or set SCRAPECRATE_API_KEY
|
|
26
|
+
>>> client.youtube.transcript(video_id="dQw4w9WgXcQ").data["text"]
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
youtube: YoutubeApi
|
|
30
|
+
maps: MapsApi
|
|
31
|
+
trustpilot: TrustpilotApi
|
|
32
|
+
ads: AdsApi
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
api_key: str | None = None,
|
|
37
|
+
*,
|
|
38
|
+
base_url: str | None = None,
|
|
39
|
+
timeout: float = 90.0,
|
|
40
|
+
max_retries: int = 3,
|
|
41
|
+
http_client: httpx.Client | None = None,
|
|
42
|
+
) -> None:
|
|
43
|
+
self._transport = SyncTransport(_Config(api_key, base_url, timeout, max_retries), http_client)
|
|
44
|
+
self.youtube = YoutubeApi(self)
|
|
45
|
+
self.maps = MapsApi(self)
|
|
46
|
+
self.trustpilot = TrustpilotApi(self)
|
|
47
|
+
self.ads = AdsApi(self)
|
|
48
|
+
|
|
49
|
+
def account(self) -> Response:
|
|
50
|
+
"""Credit balance of the account (free of charge)."""
|
|
51
|
+
return self._transport.get("/v1/account", {})
|
|
52
|
+
|
|
53
|
+
def request(self, path: str, **params: Any) -> Response:
|
|
54
|
+
"""Call any endpoint by path, e.g. client.request("/v1/maps/places/search", q="pizza")."""
|
|
55
|
+
return self._transport.get(path, {k: str(v) for k, v in params.items() if v is not None})
|
|
56
|
+
|
|
57
|
+
def close(self) -> None:
|
|
58
|
+
self._transport.close()
|
|
59
|
+
|
|
60
|
+
def __enter__(self) -> ScrapeCrate:
|
|
61
|
+
return self
|
|
62
|
+
|
|
63
|
+
def __exit__(self, *exc: object) -> None:
|
|
64
|
+
self.close()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class AsyncScrapeCrate:
|
|
68
|
+
"""Asynchronous client with the same namespaces and methods (awaitable)."""
|
|
69
|
+
|
|
70
|
+
youtube: AsyncYoutubeApi
|
|
71
|
+
maps: AsyncMapsApi
|
|
72
|
+
trustpilot: AsyncTrustpilotApi
|
|
73
|
+
ads: AsyncAdsApi
|
|
74
|
+
|
|
75
|
+
def __init__(
|
|
76
|
+
self,
|
|
77
|
+
api_key: str | None = None,
|
|
78
|
+
*,
|
|
79
|
+
base_url: str | None = None,
|
|
80
|
+
timeout: float = 90.0,
|
|
81
|
+
max_retries: int = 3,
|
|
82
|
+
http_client: httpx.AsyncClient | None = None,
|
|
83
|
+
) -> None:
|
|
84
|
+
self._transport = AsyncTransport(_Config(api_key, base_url, timeout, max_retries), http_client)
|
|
85
|
+
self.youtube = AsyncYoutubeApi(self)
|
|
86
|
+
self.maps = AsyncMapsApi(self)
|
|
87
|
+
self.trustpilot = AsyncTrustpilotApi(self)
|
|
88
|
+
self.ads = AsyncAdsApi(self)
|
|
89
|
+
|
|
90
|
+
async def account(self) -> Response:
|
|
91
|
+
return await self._transport.get("/v1/account", {})
|
|
92
|
+
|
|
93
|
+
async def close(self) -> None:
|
|
94
|
+
await self._transport.close()
|
|
95
|
+
|
|
96
|
+
async def __aenter__(self) -> AsyncScrapeCrate:
|
|
97
|
+
return self
|
|
98
|
+
|
|
99
|
+
async def __aexit__(self, *exc: object) -> None:
|
|
100
|
+
await self.close()
|