fastsaver 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastsaver/__init__.py +115 -0
- fastsaver/__main__.py +10 -0
- fastsaver/_base.py +578 -0
- fastsaver/_constants.py +141 -0
- fastsaver/_endpoints.py +204 -0
- fastsaver/_transport.py +358 -0
- fastsaver/_version.py +1 -0
- fastsaver/async_client.py +301 -0
- fastsaver/cli.py +196 -0
- fastsaver/client.py +292 -0
- fastsaver/exceptions.py +255 -0
- fastsaver/models.py +614 -0
- fastsaver/py.typed +0 -0
- fastsaver/utils.py +100 -0
- fastsaver-0.1.0.dist-info/METADATA +829 -0
- fastsaver-0.1.0.dist-info/RECORD +19 -0
- fastsaver-0.1.0.dist-info/WHEEL +4 -0
- fastsaver-0.1.0.dist-info/entry_points.txt +2 -0
- fastsaver-0.1.0.dist-info/licenses/LICENSE +21 -0
fastsaver/__init__.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""fastsaver — Python SDK for FastSaverAPI (https://api.fastsaver.io).
|
|
2
|
+
|
|
3
|
+
Download media from Instagram, TikTok, YouTube (up to 4K + MP3), Pinterest,
|
|
4
|
+
Facebook, X/Twitter, RuTube and Likee (8 platforms, one JSON shape), get Telegram
|
|
5
|
+
``file_id`` audio for bots and identify songs with Shazam. Sync
|
|
6
|
+
(:class:`FastSaver`) and async (:class:`AsyncFastSaver`) clients with identical
|
|
7
|
+
method names.
|
|
8
|
+
|
|
9
|
+
Quick start::
|
|
10
|
+
|
|
11
|
+
from fastsaver import FastSaver
|
|
12
|
+
|
|
13
|
+
client = FastSaver(api_key="fs_sk_...") # or set FASTSAVER_API_KEY
|
|
14
|
+
media = client.fetch("https://www.instagram.com/reel/DRsmm9UjKfH/")
|
|
15
|
+
print(media.type, media.download_url)
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from ._base import FileInput, TimeoutArg
|
|
21
|
+
from ._constants import (
|
|
22
|
+
CREDIT_COSTS,
|
|
23
|
+
DEFAULT_BASE_URL,
|
|
24
|
+
DEFAULT_TIMEOUT,
|
|
25
|
+
PLATFORM_HOSTS,
|
|
26
|
+
YOUTUBE_FORMATS,
|
|
27
|
+
YouTubeFormat,
|
|
28
|
+
)
|
|
29
|
+
from ._version import __version__
|
|
30
|
+
from .async_client import AsyncFastSaver, AsyncShazam, AsyncYouTube
|
|
31
|
+
from .client import FastSaver, Shazam, YouTube
|
|
32
|
+
from .exceptions import (
|
|
33
|
+
APIError,
|
|
34
|
+
AuthenticationError,
|
|
35
|
+
ConfigurationError,
|
|
36
|
+
DownloadError,
|
|
37
|
+
FastSaverError,
|
|
38
|
+
FetchError,
|
|
39
|
+
InsufficientCreditsError,
|
|
40
|
+
InvalidArgumentError,
|
|
41
|
+
InvalidRequestError,
|
|
42
|
+
NotFoundError,
|
|
43
|
+
RateLimitError,
|
|
44
|
+
RequestTimeoutError,
|
|
45
|
+
ServerError,
|
|
46
|
+
TransportError,
|
|
47
|
+
ValidationError,
|
|
48
|
+
)
|
|
49
|
+
from .models import (
|
|
50
|
+
Balance,
|
|
51
|
+
MediaItem,
|
|
52
|
+
MediaResult,
|
|
53
|
+
Music,
|
|
54
|
+
ShazamMatch,
|
|
55
|
+
ShazamTop,
|
|
56
|
+
TelegramAudio,
|
|
57
|
+
Thumbnails,
|
|
58
|
+
Track,
|
|
59
|
+
YouTubeDownload,
|
|
60
|
+
YouTubeFormatInfo,
|
|
61
|
+
YouTubeInfo,
|
|
62
|
+
YouTubeSearch,
|
|
63
|
+
parse_duration_seconds,
|
|
64
|
+
)
|
|
65
|
+
from .utils import detect_platform, extract_youtube_id, normalize_bot_username
|
|
66
|
+
|
|
67
|
+
__all__ = [
|
|
68
|
+
"CREDIT_COSTS",
|
|
69
|
+
"DEFAULT_BASE_URL",
|
|
70
|
+
"DEFAULT_TIMEOUT",
|
|
71
|
+
"PLATFORM_HOSTS",
|
|
72
|
+
"YOUTUBE_FORMATS",
|
|
73
|
+
"APIError",
|
|
74
|
+
"AsyncFastSaver",
|
|
75
|
+
"AsyncShazam",
|
|
76
|
+
"AsyncYouTube",
|
|
77
|
+
"AuthenticationError",
|
|
78
|
+
"Balance",
|
|
79
|
+
"ConfigurationError",
|
|
80
|
+
"DownloadError",
|
|
81
|
+
"FastSaver",
|
|
82
|
+
"FastSaverError",
|
|
83
|
+
"FetchError",
|
|
84
|
+
"FileInput",
|
|
85
|
+
"InsufficientCreditsError",
|
|
86
|
+
"InvalidArgumentError",
|
|
87
|
+
"InvalidRequestError",
|
|
88
|
+
"MediaItem",
|
|
89
|
+
"MediaResult",
|
|
90
|
+
"Music",
|
|
91
|
+
"NotFoundError",
|
|
92
|
+
"RateLimitError",
|
|
93
|
+
"RequestTimeoutError",
|
|
94
|
+
"ServerError",
|
|
95
|
+
"Shazam",
|
|
96
|
+
"ShazamMatch",
|
|
97
|
+
"ShazamTop",
|
|
98
|
+
"TelegramAudio",
|
|
99
|
+
"Thumbnails",
|
|
100
|
+
"TimeoutArg",
|
|
101
|
+
"Track",
|
|
102
|
+
"TransportError",
|
|
103
|
+
"ValidationError",
|
|
104
|
+
"YouTube",
|
|
105
|
+
"YouTubeDownload",
|
|
106
|
+
"YouTubeFormat",
|
|
107
|
+
"YouTubeFormatInfo",
|
|
108
|
+
"YouTubeInfo",
|
|
109
|
+
"YouTubeSearch",
|
|
110
|
+
"__version__",
|
|
111
|
+
"detect_platform",
|
|
112
|
+
"extract_youtube_id",
|
|
113
|
+
"normalize_bot_username",
|
|
114
|
+
"parse_duration_seconds",
|
|
115
|
+
]
|
fastsaver/__main__.py
ADDED
fastsaver/_base.py
ADDED
|
@@ -0,0 +1,578 @@
|
|
|
1
|
+
"""Plumbing shared by :class:`~fastsaver.FastSaver` and :class:`~fastsaver.AsyncFastSaver`.
|
|
2
|
+
|
|
3
|
+
Handles API-key/base-URL resolution, header construction, timeout
|
|
4
|
+
normalisation, upload-file coercion and download filename derivation. Nothing
|
|
5
|
+
here performs network I/O.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import contextlib
|
|
11
|
+
import logging
|
|
12
|
+
import mimetypes
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import tempfile
|
|
16
|
+
import time
|
|
17
|
+
from pathlib import Path, PureWindowsPath
|
|
18
|
+
from typing import IO, Any, Optional, Union
|
|
19
|
+
from urllib.parse import unquote, urlparse
|
|
20
|
+
|
|
21
|
+
import httpx
|
|
22
|
+
|
|
23
|
+
from ._constants import (
|
|
24
|
+
DEFAULT_BASE_URL,
|
|
25
|
+
DEFAULT_CONNECT_TIMEOUT,
|
|
26
|
+
DEFAULT_MAX_RETRIES,
|
|
27
|
+
DEFAULT_TIMEOUT,
|
|
28
|
+
ENV_API_KEY,
|
|
29
|
+
ENV_BASE_URL,
|
|
30
|
+
MAX_IDENTIFY_FILE_SIZE,
|
|
31
|
+
SIGNUP_URL,
|
|
32
|
+
build_user_agent,
|
|
33
|
+
)
|
|
34
|
+
from ._endpoints import RequestSpec, UploadTuple
|
|
35
|
+
from ._transport import backoff_delay, parse_json, parse_retry_after, should_retry
|
|
36
|
+
from .exceptions import ConfigurationError, DownloadError, InvalidArgumentError
|
|
37
|
+
|
|
38
|
+
logger = logging.getLogger("fastsaver")
|
|
39
|
+
|
|
40
|
+
#: Accepted ``timeout`` arguments: seconds, an ``httpx.Timeout`` or ``None`` (no timeout).
|
|
41
|
+
TimeoutArg = Union[float, httpx.Timeout, None]
|
|
42
|
+
|
|
43
|
+
#: Accepted ``file`` arguments for ``shazam.identify``.
|
|
44
|
+
FileInput = Union[
|
|
45
|
+
str,
|
|
46
|
+
"os.PathLike[str]",
|
|
47
|
+
bytes,
|
|
48
|
+
bytearray,
|
|
49
|
+
memoryview,
|
|
50
|
+
IO[bytes],
|
|
51
|
+
tuple[str, Any],
|
|
52
|
+
tuple[str, Any, Optional[str]],
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
_AUDIO_MIME_TYPES: dict[str, str] = {
|
|
56
|
+
".mp3": "audio/mpeg",
|
|
57
|
+
".m4a": "audio/mp4",
|
|
58
|
+
".aac": "audio/aac",
|
|
59
|
+
".ogg": "audio/ogg",
|
|
60
|
+
".oga": "audio/ogg",
|
|
61
|
+
".opus": "audio/ogg",
|
|
62
|
+
".wav": "audio/wav",
|
|
63
|
+
".flac": "audio/flac",
|
|
64
|
+
".weba": "audio/webm",
|
|
65
|
+
".webm": "video/webm",
|
|
66
|
+
".mp4": "video/mp4",
|
|
67
|
+
".m4v": "video/mp4",
|
|
68
|
+
}
|
|
69
|
+
_DEFAULT_CONTENT_TYPE = "application/octet-stream"
|
|
70
|
+
_DEFAULT_UPLOAD_NAME = "audio"
|
|
71
|
+
|
|
72
|
+
_CD_FILENAME_STAR_RE = re.compile(r"filename\*\s*=\s*([^;]+)", re.IGNORECASE)
|
|
73
|
+
_CD_FILENAME_RE = re.compile(r'filename\s*=\s*(?:"([^"]*)"|([^;]+))', re.IGNORECASE)
|
|
74
|
+
#: Control characters plus the characters Windows forbids in file names.
|
|
75
|
+
_UNSAFE_FILENAME_CHARS_RE = re.compile(r'[\x00-\x1f\x7f<>:"|?*]')
|
|
76
|
+
_WINDOWS_RESERVED_NAMES = frozenset(
|
|
77
|
+
{"con", "prn", "aux", "nul"}
|
|
78
|
+
| {f"com{i}" for i in range(1, 10)}
|
|
79
|
+
| {f"lpt{i}" for i in range(1, 10)}
|
|
80
|
+
)
|
|
81
|
+
#: Byte budget for a derived file name (well under every common filesystem limit).
|
|
82
|
+
_MAX_FILENAME_BYTES = 200
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# --- Configuration resolution ------------------------------------------------
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def resolve_api_key(api_key: Optional[str]) -> str:
|
|
89
|
+
"""Return the API key from the argument or ``FASTSAVER_API_KEY``; raise when missing."""
|
|
90
|
+
key = api_key if api_key is not None else os.environ.get(ENV_API_KEY)
|
|
91
|
+
key = (key or "").strip()
|
|
92
|
+
if not key:
|
|
93
|
+
raise ConfigurationError(
|
|
94
|
+
"No API key provided. Pass api_key='fs_sk_...' to the client or set the "
|
|
95
|
+
f"{ENV_API_KEY} environment variable. A free key with 1,000 credits "
|
|
96
|
+
f"(no credit card required) is available at {SIGNUP_URL}"
|
|
97
|
+
)
|
|
98
|
+
return key
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def resolve_base_url(base_url: Optional[str]) -> str:
|
|
102
|
+
"""Return the base URL (argument > ``FASTSAVER_BASE_URL`` > default) without trailing slash."""
|
|
103
|
+
url = base_url if base_url is not None else os.environ.get(ENV_BASE_URL)
|
|
104
|
+
url = (url or "").strip() or DEFAULT_BASE_URL
|
|
105
|
+
return url.rstrip("/")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def normalize_timeout(timeout: TimeoutArg) -> httpx.Timeout:
|
|
109
|
+
"""``float`` → ``httpx.Timeout(t, connect=10.0)``; ``None`` → no timeout.
|
|
110
|
+
|
|
111
|
+
An ``httpx.Timeout`` instance is passed through unchanged.
|
|
112
|
+
"""
|
|
113
|
+
if timeout is None:
|
|
114
|
+
return httpx.Timeout(None)
|
|
115
|
+
if isinstance(timeout, httpx.Timeout):
|
|
116
|
+
return timeout
|
|
117
|
+
if isinstance(timeout, bool) or not isinstance(timeout, (int, float)):
|
|
118
|
+
raise TypeError("timeout must be a number of seconds, an httpx.Timeout or None")
|
|
119
|
+
seconds = float(timeout)
|
|
120
|
+
if seconds <= 0:
|
|
121
|
+
raise InvalidArgumentError("timeout must be positive (or None to disable)")
|
|
122
|
+
return httpx.Timeout(seconds, connect=DEFAULT_CONNECT_TIMEOUT)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def mask_api_key(key: str) -> str:
|
|
126
|
+
"""``fs_sk_abcdef...yz`` → ``fs_sk_ab…yz`` (never reveals the full key)."""
|
|
127
|
+
if not key:
|
|
128
|
+
return ""
|
|
129
|
+
if key.startswith("fs_sk_") and len(key) > 10:
|
|
130
|
+
return f"{key[:8]}…{key[-2:]}"
|
|
131
|
+
if len(key) > 6:
|
|
132
|
+
return f"{key[:2]}…{key[-2:]}"
|
|
133
|
+
return "…"
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# --- Upload coercion ---------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def guess_content_type(filename: str) -> Optional[str]:
|
|
140
|
+
ext = os.path.splitext(filename)[1].lower()
|
|
141
|
+
if ext in _AUDIO_MIME_TYPES:
|
|
142
|
+
return _AUDIO_MIME_TYPES[ext]
|
|
143
|
+
guessed, _ = mimetypes.guess_type(filename)
|
|
144
|
+
return guessed
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _too_large(size: int) -> InvalidArgumentError:
|
|
148
|
+
return InvalidArgumentError(
|
|
149
|
+
f"File is too large for /shazam/identify: {size} bytes "
|
|
150
|
+
f"(max {MAX_IDENTIFY_FILE_SIZE} bytes = 50 MiB). Trim the clip — 5-15 s of audio is enough."
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _read_content(obj: Any) -> tuple[bytes, Optional[str]]:
|
|
155
|
+
"""Materialise ``obj`` into bytes and return ``(data, filename_hint)``.
|
|
156
|
+
|
|
157
|
+
Reads are capped at the 50 MiB limit plus one byte, so an oversized stream is
|
|
158
|
+
rejected without being held in memory in full.
|
|
159
|
+
"""
|
|
160
|
+
if isinstance(obj, (bytes, bytearray, memoryview)):
|
|
161
|
+
return bytes(obj), None
|
|
162
|
+
if isinstance(obj, (str, os.PathLike)):
|
|
163
|
+
raw = os.fspath(obj)
|
|
164
|
+
if not raw:
|
|
165
|
+
raise InvalidArgumentError("file path must not be empty")
|
|
166
|
+
path = Path(raw)
|
|
167
|
+
if not path.is_file():
|
|
168
|
+
raise InvalidArgumentError(f"{path} is not a regular file")
|
|
169
|
+
size = path.stat().st_size
|
|
170
|
+
if size > MAX_IDENTIFY_FILE_SIZE:
|
|
171
|
+
raise _too_large(size)
|
|
172
|
+
with open(path, "rb") as fh:
|
|
173
|
+
return fh.read(MAX_IDENTIFY_FILE_SIZE + 1), path.name
|
|
174
|
+
if hasattr(obj, "read"):
|
|
175
|
+
data = obj.read(MAX_IDENTIFY_FILE_SIZE + 1)
|
|
176
|
+
if isinstance(data, str):
|
|
177
|
+
raise TypeError("file objects must be opened in binary mode ('rb')")
|
|
178
|
+
name = getattr(obj, "name", None)
|
|
179
|
+
hint = os.path.basename(name) if isinstance(name, str) and name else None
|
|
180
|
+
return bytes(data), hint
|
|
181
|
+
raise TypeError(
|
|
182
|
+
"file must be a path, bytes, a binary file object or a "
|
|
183
|
+
"(filename, content[, content_type]) tuple; got " + type(obj).__name__
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def coerce_upload_file(
|
|
188
|
+
file: FileInput,
|
|
189
|
+
filename: Optional[str] = None,
|
|
190
|
+
content_type: Optional[str] = None,
|
|
191
|
+
) -> UploadTuple:
|
|
192
|
+
"""Turn any supported ``file`` argument into ``(filename, bytes, content_type)``.
|
|
193
|
+
|
|
194
|
+
Raises :class:`ValueError` when the content exceeds 50 MiB or is empty and
|
|
195
|
+
:class:`TypeError` for unsupported inputs.
|
|
196
|
+
"""
|
|
197
|
+
name = filename
|
|
198
|
+
ctype = content_type
|
|
199
|
+
if isinstance(file, tuple):
|
|
200
|
+
if len(file) not in (2, 3):
|
|
201
|
+
raise InvalidArgumentError(
|
|
202
|
+
"file tuple must be (filename, content) or (filename, content, type)"
|
|
203
|
+
)
|
|
204
|
+
tuple_name = file[0]
|
|
205
|
+
if name is None and isinstance(tuple_name, str) and tuple_name:
|
|
206
|
+
name = tuple_name
|
|
207
|
+
if len(file) == 3 and ctype is None and isinstance(file[2], str) and file[2]:
|
|
208
|
+
ctype = file[2]
|
|
209
|
+
data, hint = _read_content(file[1])
|
|
210
|
+
else:
|
|
211
|
+
data, hint = _read_content(file)
|
|
212
|
+
if name is None:
|
|
213
|
+
name = hint
|
|
214
|
+
size = len(data)
|
|
215
|
+
if size > MAX_IDENTIFY_FILE_SIZE:
|
|
216
|
+
raise _too_large(size)
|
|
217
|
+
if size == 0:
|
|
218
|
+
raise InvalidArgumentError("file is empty — nothing to identify")
|
|
219
|
+
if not name:
|
|
220
|
+
ext = mimetypes.guess_extension(ctype) if ctype else None
|
|
221
|
+
name = _DEFAULT_UPLOAD_NAME + (ext or "")
|
|
222
|
+
name = os.path.basename(name) or _DEFAULT_UPLOAD_NAME
|
|
223
|
+
if not ctype:
|
|
224
|
+
ctype = guess_content_type(name) or _DEFAULT_CONTENT_TYPE
|
|
225
|
+
return (name, data, ctype)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
# --- Download filename derivation -------------------------------------------
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _sanitize_filename(name: str) -> Optional[str]:
|
|
232
|
+
"""Reduce a server-supplied name to a single safe path component (or ``None``)."""
|
|
233
|
+
name = name.replace("\\", "/").rsplit("/", 1)[-1]
|
|
234
|
+
name = _UNSAFE_FILENAME_CHARS_RE.sub("", name).strip().strip(".").strip()
|
|
235
|
+
if not name or name in (".", ".."):
|
|
236
|
+
return None
|
|
237
|
+
if PureWindowsPath(name).anchor or PureWindowsPath(name).name != name:
|
|
238
|
+
return None
|
|
239
|
+
if name.split(".", 1)[0].lower() in _WINDOWS_RESERVED_NAMES:
|
|
240
|
+
return None
|
|
241
|
+
encoded = name.encode("utf-8", errors="ignore")
|
|
242
|
+
if len(encoded) > _MAX_FILENAME_BYTES:
|
|
243
|
+
stem, dot, ext = name.rpartition(".")
|
|
244
|
+
ext = ("." + ext) if dot and len(ext) <= 16 and stem else ""
|
|
245
|
+
base = name[: len(name) - len(ext)]
|
|
246
|
+
budget = _MAX_FILENAME_BYTES - len(ext.encode("utf-8"))
|
|
247
|
+
name = base.encode("utf-8", errors="ignore")[:budget].decode("utf-8", errors="ignore")
|
|
248
|
+
name = name.rstrip(" .") + ext
|
|
249
|
+
if not name.strip(" ."):
|
|
250
|
+
return None
|
|
251
|
+
return name
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def filename_from_content_disposition(value: Optional[str]) -> Optional[str]:
|
|
255
|
+
"""Extract a filename from a ``Content-Disposition`` header (``filename*=`` preferred)."""
|
|
256
|
+
if not value:
|
|
257
|
+
return None
|
|
258
|
+
star = _CD_FILENAME_STAR_RE.search(value)
|
|
259
|
+
if star:
|
|
260
|
+
token = star.group(1).strip().strip('"')
|
|
261
|
+
charset, sep, encoded = token.partition("''")
|
|
262
|
+
if sep:
|
|
263
|
+
try:
|
|
264
|
+
decoded = unquote(encoded, encoding=charset or "utf-8", errors="replace")
|
|
265
|
+
except LookupError:
|
|
266
|
+
decoded = unquote(encoded, errors="replace")
|
|
267
|
+
else:
|
|
268
|
+
decoded = unquote(token, errors="replace")
|
|
269
|
+
name = _sanitize_filename(decoded)
|
|
270
|
+
if name:
|
|
271
|
+
return name
|
|
272
|
+
plain = _CD_FILENAME_RE.search(value)
|
|
273
|
+
if plain:
|
|
274
|
+
candidate = plain.group(1) if plain.group(1) is not None else plain.group(2)
|
|
275
|
+
name = _sanitize_filename(candidate.strip())
|
|
276
|
+
if name:
|
|
277
|
+
return name
|
|
278
|
+
return None
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def filename_from_url(url: Optional[str]) -> Optional[str]:
|
|
282
|
+
if not url:
|
|
283
|
+
return None
|
|
284
|
+
path = urlparse(url).path
|
|
285
|
+
if not path:
|
|
286
|
+
return None
|
|
287
|
+
return _sanitize_filename(unquote(path.rsplit("/", 1)[-1]))
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def derive_filename(headers: Optional[httpx.Headers], final_url: Optional[str]) -> str:
|
|
291
|
+
"""``Content-Disposition`` → URL path basename → ``"download"``."""
|
|
292
|
+
cd = headers.get("Content-Disposition") if headers is not None else None
|
|
293
|
+
return filename_from_content_disposition(cd) or filename_from_url(final_url) or "download"
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def dest_is_directory(dest: Union[str, os.PathLike[str]]) -> bool:
|
|
297
|
+
"""``True`` when ``dest`` is an existing directory or is written with a trailing separator.
|
|
298
|
+
|
|
299
|
+
``"downloads/"`` therefore means "put the file inside *downloads*" even before that
|
|
300
|
+
directory exists (it is created on demand), while ``"downloads"`` is a file name.
|
|
301
|
+
"""
|
|
302
|
+
raw = os.fspath(dest)
|
|
303
|
+
seps = tuple(sep for sep in (os.sep, os.altsep, "/") if sep)
|
|
304
|
+
return raw.endswith(seps) or Path(raw).is_dir()
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def resolve_dest_path(
|
|
308
|
+
dest: Union[str, os.PathLike[str]],
|
|
309
|
+
headers: Optional[httpx.Headers],
|
|
310
|
+
final_url: Optional[str],
|
|
311
|
+
) -> Path:
|
|
312
|
+
"""Return ``dest`` itself, or ``dest/<derived filename>`` when ``dest`` is a directory.
|
|
313
|
+
|
|
314
|
+
A directory destination (existing, or given with a trailing ``/``) is created
|
|
315
|
+
when missing. A resolved target that is itself a directory raises
|
|
316
|
+
:class:`IsADirectoryError` before anything is written.
|
|
317
|
+
"""
|
|
318
|
+
path = Path(dest)
|
|
319
|
+
if dest_is_directory(dest):
|
|
320
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
321
|
+
target = path / derive_filename(headers, final_url)
|
|
322
|
+
else:
|
|
323
|
+
target = path
|
|
324
|
+
if target.is_dir():
|
|
325
|
+
raise IsADirectoryError(f"{target} is a directory; pass a file path or a directory dest")
|
|
326
|
+
return target
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def warn_if_not_media(response: httpx.Response) -> None:
|
|
330
|
+
"""Log a warning when a download link answered with an HTML page instead of a file."""
|
|
331
|
+
content_type = (response.headers.get("Content-Type") or "").lower()
|
|
332
|
+
if content_type.startswith("text/html"):
|
|
333
|
+
logger.warning(
|
|
334
|
+
"download_url answered with an HTML page (%s), not media: %s",
|
|
335
|
+
content_type.split(";")[0],
|
|
336
|
+
response.url,
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def download_error(response: httpx.Response) -> DownloadError:
|
|
341
|
+
"""Build the :class:`DownloadError` for a non-2xx final download response."""
|
|
342
|
+
body = parse_json(response)
|
|
343
|
+
detail = body.get("detail") if isinstance(body, dict) else None
|
|
344
|
+
message = f"Download failed with HTTP {response.status_code}"
|
|
345
|
+
if isinstance(detail, str) and detail:
|
|
346
|
+
message += f" ({detail})"
|
|
347
|
+
if detail == "tunnel.not_found":
|
|
348
|
+
message += " — the download link expired (links are valid for 10 minutes)"
|
|
349
|
+
message += f": {response.url}"
|
|
350
|
+
return DownloadError(
|
|
351
|
+
message,
|
|
352
|
+
status_code=response.status_code,
|
|
353
|
+
url=str(response.url),
|
|
354
|
+
body=body if body is not None else response.text[:500],
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
class PartFile:
|
|
359
|
+
"""A uniquely named ``.part`` temp file next to ``target``, atomically renamed on success.
|
|
360
|
+
|
|
361
|
+
``tempfile.mkstemp`` creates the file exclusively (no pre-existing symlink is
|
|
362
|
+
followed) and with a random suffix, so concurrent saves to the same
|
|
363
|
+
destination cannot corrupt each other. Use as a context manager: the temp
|
|
364
|
+
file is removed unless :meth:`commit` was called.
|
|
365
|
+
"""
|
|
366
|
+
|
|
367
|
+
def __init__(self, target: Path) -> None:
|
|
368
|
+
self.target = target
|
|
369
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
370
|
+
fd, name = tempfile.mkstemp(
|
|
371
|
+
dir=str(target.parent), prefix="." + target.name[:40] + ".", suffix=".part"
|
|
372
|
+
)
|
|
373
|
+
self.path = Path(name)
|
|
374
|
+
self.file = os.fdopen(fd, "wb")
|
|
375
|
+
self._committed = False
|
|
376
|
+
|
|
377
|
+
def write(self, chunk: bytes) -> None:
|
|
378
|
+
self.file.write(chunk)
|
|
379
|
+
|
|
380
|
+
def commit(self) -> None:
|
|
381
|
+
self.file.close()
|
|
382
|
+
os.replace(self.path, self.target)
|
|
383
|
+
self._committed = True
|
|
384
|
+
|
|
385
|
+
def __enter__(self) -> PartFile:
|
|
386
|
+
return self
|
|
387
|
+
|
|
388
|
+
def __exit__(self, *exc_info: object) -> None:
|
|
389
|
+
if not self.file.closed:
|
|
390
|
+
self.file.close()
|
|
391
|
+
if not self._committed:
|
|
392
|
+
with contextlib.suppress(OSError):
|
|
393
|
+
self.path.unlink()
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def validate_save_args(url: object, chunk_size: object) -> None:
|
|
397
|
+
"""Argument checks shared by the sync and async ``save()``."""
|
|
398
|
+
if not isinstance(url, str) or not url.strip():
|
|
399
|
+
raise InvalidArgumentError("url must be a non-empty string")
|
|
400
|
+
if isinstance(chunk_size, bool) or not isinstance(chunk_size, int) or chunk_size <= 0:
|
|
401
|
+
raise InvalidArgumentError("chunk_size must be a positive integer")
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def check_overwrite(dest: Union[str, os.PathLike[str]], overwrite: bool) -> None:
|
|
405
|
+
"""Raise :class:`FileExistsError` early for a file destination that already exists."""
|
|
406
|
+
if not overwrite and not dest_is_directory(dest) and Path(dest).exists():
|
|
407
|
+
raise FileExistsError(f"{os.fspath(dest)} already exists (overwrite=False)")
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
# --- Base client --------------------------------------------------------------
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
class BaseClient:
|
|
414
|
+
"""Configuration and request preparation shared by the sync and async clients."""
|
|
415
|
+
|
|
416
|
+
def __init__(
|
|
417
|
+
self,
|
|
418
|
+
api_key: Optional[str] = None,
|
|
419
|
+
*,
|
|
420
|
+
base_url: Optional[str] = None,
|
|
421
|
+
timeout: TimeoutArg = DEFAULT_TIMEOUT,
|
|
422
|
+
max_retries: int = DEFAULT_MAX_RETRIES,
|
|
423
|
+
user_agent: Optional[str] = None,
|
|
424
|
+
) -> None:
|
|
425
|
+
self._api_key = resolve_api_key(api_key)
|
|
426
|
+
self._base_url = resolve_base_url(base_url)
|
|
427
|
+
self._timeout = normalize_timeout(timeout)
|
|
428
|
+
if isinstance(max_retries, bool) or not isinstance(max_retries, int) or max_retries < 0:
|
|
429
|
+
raise InvalidArgumentError("max_retries must be a non-negative integer")
|
|
430
|
+
self._max_retries = max_retries
|
|
431
|
+
self._user_agent = build_user_agent(user_agent)
|
|
432
|
+
self._closed = False
|
|
433
|
+
|
|
434
|
+
# -- public read-only configuration --------------------------------------
|
|
435
|
+
|
|
436
|
+
@property
|
|
437
|
+
def api_key(self) -> str:
|
|
438
|
+
"""The API key in use (masked in ``repr``)."""
|
|
439
|
+
return self._api_key
|
|
440
|
+
|
|
441
|
+
@property
|
|
442
|
+
def base_url(self) -> str:
|
|
443
|
+
return self._base_url
|
|
444
|
+
|
|
445
|
+
@property
|
|
446
|
+
def timeout(self) -> httpx.Timeout:
|
|
447
|
+
return self._timeout
|
|
448
|
+
|
|
449
|
+
@property
|
|
450
|
+
def max_retries(self) -> int:
|
|
451
|
+
return self._max_retries
|
|
452
|
+
|
|
453
|
+
@property
|
|
454
|
+
def user_agent(self) -> str:
|
|
455
|
+
return self._user_agent
|
|
456
|
+
|
|
457
|
+
@property
|
|
458
|
+
def is_closed(self) -> bool:
|
|
459
|
+
"""``True`` once :meth:`close`/:meth:`aclose` has been called on this client."""
|
|
460
|
+
return self._closed
|
|
461
|
+
|
|
462
|
+
def __repr__(self) -> str:
|
|
463
|
+
return (
|
|
464
|
+
f"{type(self).__name__}(api_key={mask_api_key(self._api_key)!r}, "
|
|
465
|
+
f"base_url={self._base_url!r}, max_retries={self._max_retries})"
|
|
466
|
+
)
|
|
467
|
+
|
|
468
|
+
def _ensure_open(self) -> None:
|
|
469
|
+
if self._closed:
|
|
470
|
+
raise ConfigurationError(
|
|
471
|
+
f"{type(self).__name__} is closed — create a new client "
|
|
472
|
+
"(or use it as a context manager)"
|
|
473
|
+
)
|
|
474
|
+
|
|
475
|
+
# -- request preparation --------------------------------------------------
|
|
476
|
+
|
|
477
|
+
def _url(self, path: str) -> str:
|
|
478
|
+
return self._base_url + path
|
|
479
|
+
|
|
480
|
+
def _headers(self) -> dict[str, str]:
|
|
481
|
+
return {
|
|
482
|
+
"X-Api-Key": self._api_key,
|
|
483
|
+
"Accept": "application/json",
|
|
484
|
+
"User-Agent": self._user_agent,
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
def _download_headers(self) -> dict[str, str]:
|
|
488
|
+
"""Headers for ``save()`` — deliberately **without** ``X-Api-Key``."""
|
|
489
|
+
return {"Accept": "*/*", "User-Agent": self._user_agent}
|
|
490
|
+
|
|
491
|
+
def _resolve_timeout(self, timeout: TimeoutArg, default_read: Optional[float]) -> httpx.Timeout:
|
|
492
|
+
"""Per-call ``timeout`` > per-endpoint read default > client default."""
|
|
493
|
+
if timeout is not None:
|
|
494
|
+
return normalize_timeout(timeout)
|
|
495
|
+
if default_read is not None:
|
|
496
|
+
return httpx.Timeout(
|
|
497
|
+
connect=self._timeout.connect,
|
|
498
|
+
read=default_read,
|
|
499
|
+
write=self._timeout.write,
|
|
500
|
+
pool=self._timeout.pool,
|
|
501
|
+
)
|
|
502
|
+
return self._timeout
|
|
503
|
+
|
|
504
|
+
def _request_kwargs(self, spec: RequestSpec[Any], timeout: TimeoutArg) -> dict[str, Any]:
|
|
505
|
+
headers = self._headers()
|
|
506
|
+
if spec.headers:
|
|
507
|
+
headers.update(spec.headers)
|
|
508
|
+
kwargs: dict[str, Any] = {
|
|
509
|
+
"method": spec.method,
|
|
510
|
+
"url": self._url(spec.path),
|
|
511
|
+
"headers": headers,
|
|
512
|
+
"timeout": self._resolve_timeout(timeout, spec.timeout_default),
|
|
513
|
+
# API calls never follow redirects (also when the user's own httpx client
|
|
514
|
+
# is configured to), so the key is never replayed to another origin.
|
|
515
|
+
"follow_redirects": False,
|
|
516
|
+
}
|
|
517
|
+
if spec.params is not None:
|
|
518
|
+
kwargs["params"] = spec.params
|
|
519
|
+
if spec.json is not None:
|
|
520
|
+
kwargs["json"] = spec.json
|
|
521
|
+
if spec.files is not None:
|
|
522
|
+
kwargs["files"] = spec.files
|
|
523
|
+
return kwargs
|
|
524
|
+
|
|
525
|
+
# -- retry decisions ------------------------------------------------------
|
|
526
|
+
|
|
527
|
+
def _retry_delay_for_exception(self, exc: BaseException, attempt: int) -> Optional[float]:
|
|
528
|
+
if not should_retry(exc, attempt, self._max_retries):
|
|
529
|
+
return None
|
|
530
|
+
return backoff_delay(attempt)
|
|
531
|
+
|
|
532
|
+
def _retry_delay_for_response(self, response: httpx.Response, attempt: int) -> Optional[float]:
|
|
533
|
+
if not should_retry(response.status_code, attempt, self._max_retries):
|
|
534
|
+
return None
|
|
535
|
+
rate_limited = response.status_code == 429
|
|
536
|
+
retry_after = None
|
|
537
|
+
if rate_limited:
|
|
538
|
+
retry_after = parse_retry_after(response.headers.get("Retry-After"))
|
|
539
|
+
return backoff_delay(attempt, retry_after, rate_limited=rate_limited)
|
|
540
|
+
|
|
541
|
+
# -- logging --------------------------------------------------------------
|
|
542
|
+
|
|
543
|
+
@staticmethod
|
|
544
|
+
def _elapsed_ms(started: float) -> int:
|
|
545
|
+
return int((time.perf_counter() - started) * 1000)
|
|
546
|
+
|
|
547
|
+
def _log_response(self, spec: RequestSpec[Any], status_code: int, started: float) -> None:
|
|
548
|
+
if logger.isEnabledFor(logging.DEBUG):
|
|
549
|
+
logger.debug(
|
|
550
|
+
"%s %s -> %s in %dms",
|
|
551
|
+
spec.method,
|
|
552
|
+
spec.path,
|
|
553
|
+
status_code,
|
|
554
|
+
self._elapsed_ms(started),
|
|
555
|
+
)
|
|
556
|
+
|
|
557
|
+
def _log_retry(self, spec: RequestSpec[Any], reason: str, attempt: int, delay: float) -> None:
|
|
558
|
+
logger.debug(
|
|
559
|
+
"%s %s failed (%s); retry %d/%d in %.2fs",
|
|
560
|
+
spec.method,
|
|
561
|
+
spec.path,
|
|
562
|
+
reason,
|
|
563
|
+
attempt + 1,
|
|
564
|
+
self._max_retries,
|
|
565
|
+
delay,
|
|
566
|
+
)
|
|
567
|
+
|
|
568
|
+
def _log_transport_error(
|
|
569
|
+
self, spec: RequestSpec[Any], exc: BaseException, started: float
|
|
570
|
+
) -> None:
|
|
571
|
+
if logger.isEnabledFor(logging.DEBUG):
|
|
572
|
+
logger.debug(
|
|
573
|
+
"%s %s -> %s after %dms",
|
|
574
|
+
spec.method,
|
|
575
|
+
spec.path,
|
|
576
|
+
type(exc).__name__,
|
|
577
|
+
self._elapsed_ms(started),
|
|
578
|
+
)
|