flashdata 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flashdata/__init__.py +34 -0
- flashdata/_async_client.py +213 -0
- flashdata/_client.py +210 -0
- flashdata/_common.py +265 -0
- flashdata/_resources.py +1945 -0
- flashdata/_schemas.json +1096 -0
- flashdata/errors.py +69 -0
- flashdata/py.typed +0 -0
- flashdata/types.py +1016 -0
- flashdata-1.0.0.dist-info/METADATA +315 -0
- flashdata-1.0.0.dist-info/RECORD +14 -0
- flashdata-1.0.0.dist-info/WHEEL +5 -0
- flashdata-1.0.0.dist-info/licenses/LICENSE +21 -0
- flashdata-1.0.0.dist-info/top_level.txt +1 -0
flashdata/_common.py
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import copy
|
|
4
|
+
import gzip
|
|
5
|
+
import io
|
|
6
|
+
import ipaddress
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import re
|
|
10
|
+
import zlib
|
|
11
|
+
from importlib.resources import files
|
|
12
|
+
from typing import Any
|
|
13
|
+
from urllib.parse import urlsplit
|
|
14
|
+
|
|
15
|
+
from jsonschema import Draft202012Validator
|
|
16
|
+
|
|
17
|
+
from .errors import APIError, ProtocolError, ValidationError
|
|
18
|
+
|
|
19
|
+
SCHEMAS = json.loads(files("flashdata").joinpath("_schemas.json").read_text(encoding="utf-8"))
|
|
20
|
+
SOURCES = tuple(SCHEMAS)
|
|
21
|
+
BULK_SOURCES = (
|
|
22
|
+
"youtube_metadata",
|
|
23
|
+
"youtube_transcript",
|
|
24
|
+
"youtube_captions",
|
|
25
|
+
"youtube_trainability",
|
|
26
|
+
"youtube_download",
|
|
27
|
+
)
|
|
28
|
+
SECRET_KEYS = {"api_key", "access_key", "secret_key", "private_key", "sas_token", "authorization", "x-api-key"}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def positive(value: float, name: str) -> float:
|
|
32
|
+
if isinstance(value, bool) or not isinstance(value, (float, int)) or not math.isfinite(value) or value <= 0:
|
|
33
|
+
raise ValidationError(f"{name} must be a finite positive number.")
|
|
34
|
+
return value
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def integer(value: int, name: str, minimum: int, maximum: int) -> int:
|
|
38
|
+
if type(value) is not int or not minimum <= value <= maximum:
|
|
39
|
+
raise ValidationError(f"{name} must be an integer from {minimum} to {maximum}.")
|
|
40
|
+
return value
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def origin(value: str) -> str:
|
|
44
|
+
if not isinstance(value, str):
|
|
45
|
+
raise ValidationError("API URLs must be HTTPS origins, or loopback HTTP origins.")
|
|
46
|
+
try:
|
|
47
|
+
p = urlsplit(value)
|
|
48
|
+
except ValueError:
|
|
49
|
+
raise ValidationError("Invalid API URL.") from None
|
|
50
|
+
if (
|
|
51
|
+
p.scheme not in ("https", "http")
|
|
52
|
+
or not p.hostname
|
|
53
|
+
or p.username
|
|
54
|
+
or p.password
|
|
55
|
+
or p.query
|
|
56
|
+
or p.fragment
|
|
57
|
+
or p.path not in ("", "/")
|
|
58
|
+
):
|
|
59
|
+
raise ValidationError("API URLs must be origins without credentials, paths, queries or fragments.")
|
|
60
|
+
if p.scheme == "http" and p.hostname not in ("localhost", "127.0.0.1", "::1"):
|
|
61
|
+
raise ValidationError("HTTP is supported only for loopback development.")
|
|
62
|
+
try:
|
|
63
|
+
port = p.port
|
|
64
|
+
except ValueError:
|
|
65
|
+
raise ValidationError("Invalid API port.") from None
|
|
66
|
+
host = f"[{p.hostname}]" if ":" in p.hostname else p.hostname
|
|
67
|
+
port_text = f":{port}" if port and port != (443 if p.scheme == "https" else 80) else ""
|
|
68
|
+
return f"{p.scheme}://{host}{port_text}"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def signed_url(url: str, expected: str | None) -> str:
|
|
72
|
+
if not isinstance(url, str) or not isinstance(expected, str):
|
|
73
|
+
raise ProtocolError("The API has not advertised a signed-storage origin.")
|
|
74
|
+
try:
|
|
75
|
+
p = urlsplit(url)
|
|
76
|
+
actual = origin(f"{p.scheme}://{p.netloc}")
|
|
77
|
+
advertised = origin(expected)
|
|
78
|
+
except (ValidationError, ValueError):
|
|
79
|
+
raise ProtocolError("Invalid signed-storage origin.") from None
|
|
80
|
+
if p.username or p.password or p.fragment or actual != advertised:
|
|
81
|
+
raise ProtocolError("Signed URL does not match the authenticated storage origin.")
|
|
82
|
+
return url
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def identifier(value: str, bulk: bool = False) -> str:
|
|
86
|
+
pattern = r"bulk_[A-Za-z0-9_-]{8,}" if bulk else r"job_[A-Za-z0-9_-]+"
|
|
87
|
+
if not isinstance(value, str) or not re.fullmatch(pattern, value):
|
|
88
|
+
raise ValidationError("Invalid bulk ID." if bulk else "Invalid job ID.")
|
|
89
|
+
return value
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def redact(value: Any, secrets: tuple[str, ...] = ()) -> Any:
|
|
93
|
+
if isinstance(value, dict):
|
|
94
|
+
return {k: "[REDACTED]" if k.lower() in SECRET_KEYS else redact(v, secrets) for k, v in value.items()}
|
|
95
|
+
if isinstance(value, list):
|
|
96
|
+
return [redact(v, secrets) for v in value]
|
|
97
|
+
if isinstance(value, str):
|
|
98
|
+
for secret in secrets:
|
|
99
|
+
if secret:
|
|
100
|
+
value = value.replace(secret, "[REDACTED]")
|
|
101
|
+
return value
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def secret_values(value: Any) -> tuple[str, ...]:
|
|
105
|
+
found = []
|
|
106
|
+
if isinstance(value, dict):
|
|
107
|
+
for k, v in value.items():
|
|
108
|
+
if k.lower() in SECRET_KEYS and isinstance(v, str):
|
|
109
|
+
found.append(v)
|
|
110
|
+
else:
|
|
111
|
+
found.extend(secret_values(v))
|
|
112
|
+
elif isinstance(value, list):
|
|
113
|
+
for item in value:
|
|
114
|
+
found.extend(secret_values(item))
|
|
115
|
+
return tuple(found)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def api_error(response: Any, body: bytes, secrets: tuple[str, ...]) -> APIError:
|
|
119
|
+
try:
|
|
120
|
+
data = redact(json.loads(body), secrets)
|
|
121
|
+
except (ValueError, UnicodeDecodeError):
|
|
122
|
+
data = None
|
|
123
|
+
message = (data.get("detail") or data.get("message") or data.get("title")) if isinstance(data, dict) else None
|
|
124
|
+
return APIError(
|
|
125
|
+
response.status_code,
|
|
126
|
+
message if isinstance(message, str) else f"HTTP {response.status_code}.",
|
|
127
|
+
code=data.get("code") if isinstance(data, dict) else None,
|
|
128
|
+
request_id=response.headers.get("x-request-id") or (data.get("requestId") if isinstance(data, dict) else None),
|
|
129
|
+
retry_after=response.headers.get("retry-after")
|
|
130
|
+
or (str(data["retry_after"]) if isinstance(data, dict) and data.get("retry_after") is not None else None),
|
|
131
|
+
body=data,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def validate_callback(callback: Any) -> None:
|
|
136
|
+
if callback is None or callback == "":
|
|
137
|
+
return
|
|
138
|
+
message = "callback_url must be a public HTTPS URL on port 443."
|
|
139
|
+
try:
|
|
140
|
+
if not isinstance(callback, str) or len(callback) > 2048:
|
|
141
|
+
raise ValueError
|
|
142
|
+
p = urlsplit(callback)
|
|
143
|
+
if (
|
|
144
|
+
p.scheme != "https"
|
|
145
|
+
or not p.hostname
|
|
146
|
+
or "." not in p.hostname
|
|
147
|
+
or p.username
|
|
148
|
+
or p.password
|
|
149
|
+
or p.fragment
|
|
150
|
+
or p.port not in (None, 443)
|
|
151
|
+
):
|
|
152
|
+
raise ValueError
|
|
153
|
+
try:
|
|
154
|
+
address = ipaddress.ip_address(p.hostname)
|
|
155
|
+
except ValueError:
|
|
156
|
+
address = None
|
|
157
|
+
if address is not None and not address.is_global:
|
|
158
|
+
raise ValueError
|
|
159
|
+
except ValueError:
|
|
160
|
+
raise ValidationError(message) from None
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def validate_query(value: Any, *, async_mode: bool = False) -> dict[str, Any]:
|
|
164
|
+
if not isinstance(value, dict) or not isinstance(value.get("source"), str) or value["source"] not in SCHEMAS:
|
|
165
|
+
raise ValidationError("Choose a supported source and provide a query object.")
|
|
166
|
+
data = copy.deepcopy(value)
|
|
167
|
+
source = data["source"]
|
|
168
|
+
if not async_mode and source == "youtube_download":
|
|
169
|
+
raise ValidationError("youtube_download supports background submission and batch only.")
|
|
170
|
+
schema = copy.deepcopy(SCHEMAS[source])
|
|
171
|
+
schema["additionalProperties"] = False
|
|
172
|
+
if async_mode:
|
|
173
|
+
schema["properties"]["callback_url"] = {"type": ["string", "null"], "maxLength": 2048}
|
|
174
|
+
error = next(Draft202012Validator(schema).iter_errors(data), None)
|
|
175
|
+
if error:
|
|
176
|
+
path = ".".join(str(part) for part in error.absolute_path) or "query"
|
|
177
|
+
# ValidationError.message may contain credentials or whole input objects.
|
|
178
|
+
constraint = error.validator
|
|
179
|
+
expected = error.validator_value
|
|
180
|
+
hint = {
|
|
181
|
+
"minimum": f"must be at least {expected}",
|
|
182
|
+
"maximum": f"must be at most {expected}",
|
|
183
|
+
"type": f"must have type {expected}",
|
|
184
|
+
"minLength": f"must contain at least {expected} characters",
|
|
185
|
+
"maxLength": f"must contain at most {expected} characters",
|
|
186
|
+
"required": "a required field is missing",
|
|
187
|
+
"additionalProperties": "contains unsupported fields",
|
|
188
|
+
"enum": f"must be one of {expected}",
|
|
189
|
+
"pattern": "has an invalid format",
|
|
190
|
+
}.get(constraint, f"violates {constraint}")
|
|
191
|
+
raise ValidationError(f"Invalid {path}: {hint}.")
|
|
192
|
+
validate_callback(data.get("callback_url"))
|
|
193
|
+
for field in ("query", "handle", "video_id"):
|
|
194
|
+
if field in data and not data[field].strip():
|
|
195
|
+
raise ValidationError(f"{field} must not be blank.")
|
|
196
|
+
return data
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def bulk_input(value: Any, *, partial: bool = False) -> dict[str, Any]:
|
|
200
|
+
if (
|
|
201
|
+
not isinstance(value, dict)
|
|
202
|
+
or not value
|
|
203
|
+
or set(value) - ({"options", "callback_url"} if partial else {"source", "options", "callback_url"})
|
|
204
|
+
):
|
|
205
|
+
raise ValidationError("Invalid bulk request fields.")
|
|
206
|
+
data = copy.deepcopy(value)
|
|
207
|
+
if not partial and data.get("source") not in BULK_SOURCES:
|
|
208
|
+
raise ValidationError("Bulk supports metadata, transcript, captions, license checks and downloads.")
|
|
209
|
+
if "options" in data and not isinstance(data["options"], dict):
|
|
210
|
+
raise ValidationError("Bulk options must be an object.")
|
|
211
|
+
try:
|
|
212
|
+
json.dumps(data, allow_nan=False)
|
|
213
|
+
except (TypeError, ValueError):
|
|
214
|
+
raise ValidationError("Bulk request values must be valid JSON.") from None
|
|
215
|
+
validate_callback(data.get("callback_url"))
|
|
216
|
+
if not partial:
|
|
217
|
+
options = copy.deepcopy(data.get("options", {}))
|
|
218
|
+
result_storage = options.pop("result_storage", None)
|
|
219
|
+
if result_storage is not None:
|
|
220
|
+
if data["source"] == "youtube_download" or not isinstance(result_storage, dict):
|
|
221
|
+
raise ValidationError("result_storage applies to bulk data sources and must be an object.")
|
|
222
|
+
storage_schema = SCHEMAS["youtube_download"]["properties"]["params"]["properties"]["storage"]
|
|
223
|
+
if not Draft202012Validator(storage_schema).is_valid(result_storage):
|
|
224
|
+
raise ValidationError("Invalid options.result_storage configuration.")
|
|
225
|
+
validate_query({"source": data["source"], "video_id": "dQw4w9WgXcQ", "params": options}, async_mode=True)
|
|
226
|
+
if not partial and data["source"] == "youtube_download" and not data.get("options", {}).get("storage"):
|
|
227
|
+
raise ValidationError("Bulk downloads require options.storage.")
|
|
228
|
+
return data
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def decode_json(body: bytes) -> dict[str, Any]:
|
|
232
|
+
try:
|
|
233
|
+
value = json.loads(body)
|
|
234
|
+
except (ValueError, UnicodeDecodeError):
|
|
235
|
+
raise ProtocolError("The response was not valid JSON.") from None
|
|
236
|
+
if not isinstance(value, dict):
|
|
237
|
+
raise ProtocolError("Expected a JSON object response.")
|
|
238
|
+
return value
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def result_rows(raw: bytes, max_bytes: int, max_line_bytes: int):
|
|
242
|
+
"""Read a bounded NDJSON part, allowing HTTP stacks that already decoded gzip."""
|
|
243
|
+
stream = gzip.GzipFile(fileobj=io.BytesIO(raw)) if raw.startswith(b"\x1f\x8b") else io.BytesIO(raw)
|
|
244
|
+
try:
|
|
245
|
+
with stream:
|
|
246
|
+
total = 0
|
|
247
|
+
while True:
|
|
248
|
+
line = stream.readline(max_line_bytes + 1)
|
|
249
|
+
if not line:
|
|
250
|
+
break
|
|
251
|
+
total += len(line)
|
|
252
|
+
if len(line) > max_line_bytes or total > max_bytes:
|
|
253
|
+
from .errors import ResponseTooLargeError
|
|
254
|
+
|
|
255
|
+
raise ResponseTooLargeError("Decompressed result part exceeds configured limits.")
|
|
256
|
+
if line.strip():
|
|
257
|
+
yield decode_json(line)
|
|
258
|
+
except (OSError, EOFError, zlib.error):
|
|
259
|
+
raise ProtocolError("Invalid or truncated gzip result part.") from None
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def job_record(value: Any) -> Any:
|
|
263
|
+
if not isinstance(value, dict) or not isinstance(value.get("job"), dict):
|
|
264
|
+
raise ProtocolError("Expected a job response containing a job object.")
|
|
265
|
+
return value["job"]
|