flashdata 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
flashdata/_common.py ADDED
@@ -0,0 +1,265 @@
1
+ from __future__ import annotations
2
+
3
+ import copy
4
+ import gzip
5
+ import io
6
+ import ipaddress
7
+ import json
8
+ import math
9
+ import re
10
+ import zlib
11
+ from importlib.resources import files
12
+ from typing import Any
13
+ from urllib.parse import urlsplit
14
+
15
+ from jsonschema import Draft202012Validator
16
+
17
+ from .errors import APIError, ProtocolError, ValidationError
18
+
19
+ SCHEMAS = json.loads(files("flashdata").joinpath("_schemas.json").read_text(encoding="utf-8"))
20
+ SOURCES = tuple(SCHEMAS)
21
+ BULK_SOURCES = (
22
+ "youtube_metadata",
23
+ "youtube_transcript",
24
+ "youtube_captions",
25
+ "youtube_trainability",
26
+ "youtube_download",
27
+ )
28
+ SECRET_KEYS = {"api_key", "access_key", "secret_key", "private_key", "sas_token", "authorization", "x-api-key"}
29
+
30
+
31
+ def positive(value: float, name: str) -> float:
32
+ if isinstance(value, bool) or not isinstance(value, (float, int)) or not math.isfinite(value) or value <= 0:
33
+ raise ValidationError(f"{name} must be a finite positive number.")
34
+ return value
35
+
36
+
37
+ def integer(value: int, name: str, minimum: int, maximum: int) -> int:
38
+ if type(value) is not int or not minimum <= value <= maximum:
39
+ raise ValidationError(f"{name} must be an integer from {minimum} to {maximum}.")
40
+ return value
41
+
42
+
43
+ def origin(value: str) -> str:
44
+ if not isinstance(value, str):
45
+ raise ValidationError("API URLs must be HTTPS origins, or loopback HTTP origins.")
46
+ try:
47
+ p = urlsplit(value)
48
+ except ValueError:
49
+ raise ValidationError("Invalid API URL.") from None
50
+ if (
51
+ p.scheme not in ("https", "http")
52
+ or not p.hostname
53
+ or p.username
54
+ or p.password
55
+ or p.query
56
+ or p.fragment
57
+ or p.path not in ("", "/")
58
+ ):
59
+ raise ValidationError("API URLs must be origins without credentials, paths, queries or fragments.")
60
+ if p.scheme == "http" and p.hostname not in ("localhost", "127.0.0.1", "::1"):
61
+ raise ValidationError("HTTP is supported only for loopback development.")
62
+ try:
63
+ port = p.port
64
+ except ValueError:
65
+ raise ValidationError("Invalid API port.") from None
66
+ host = f"[{p.hostname}]" if ":" in p.hostname else p.hostname
67
+ port_text = f":{port}" if port and port != (443 if p.scheme == "https" else 80) else ""
68
+ return f"{p.scheme}://{host}{port_text}"
69
+
70
+
71
+ def signed_url(url: str, expected: str | None) -> str:
72
+ if not isinstance(url, str) or not isinstance(expected, str):
73
+ raise ProtocolError("The API has not advertised a signed-storage origin.")
74
+ try:
75
+ p = urlsplit(url)
76
+ actual = origin(f"{p.scheme}://{p.netloc}")
77
+ advertised = origin(expected)
78
+ except (ValidationError, ValueError):
79
+ raise ProtocolError("Invalid signed-storage origin.") from None
80
+ if p.username or p.password or p.fragment or actual != advertised:
81
+ raise ProtocolError("Signed URL does not match the authenticated storage origin.")
82
+ return url
83
+
84
+
85
+ def identifier(value: str, bulk: bool = False) -> str:
86
+ pattern = r"bulk_[A-Za-z0-9_-]{8,}" if bulk else r"job_[A-Za-z0-9_-]+"
87
+ if not isinstance(value, str) or not re.fullmatch(pattern, value):
88
+ raise ValidationError("Invalid bulk ID." if bulk else "Invalid job ID.")
89
+ return value
90
+
91
+
92
+ def redact(value: Any, secrets: tuple[str, ...] = ()) -> Any:
93
+ if isinstance(value, dict):
94
+ return {k: "[REDACTED]" if k.lower() in SECRET_KEYS else redact(v, secrets) for k, v in value.items()}
95
+ if isinstance(value, list):
96
+ return [redact(v, secrets) for v in value]
97
+ if isinstance(value, str):
98
+ for secret in secrets:
99
+ if secret:
100
+ value = value.replace(secret, "[REDACTED]")
101
+ return value
102
+
103
+
104
+ def secret_values(value: Any) -> tuple[str, ...]:
105
+ found = []
106
+ if isinstance(value, dict):
107
+ for k, v in value.items():
108
+ if k.lower() in SECRET_KEYS and isinstance(v, str):
109
+ found.append(v)
110
+ else:
111
+ found.extend(secret_values(v))
112
+ elif isinstance(value, list):
113
+ for item in value:
114
+ found.extend(secret_values(item))
115
+ return tuple(found)
116
+
117
+
118
+ def api_error(response: Any, body: bytes, secrets: tuple[str, ...]) -> APIError:
119
+ try:
120
+ data = redact(json.loads(body), secrets)
121
+ except (ValueError, UnicodeDecodeError):
122
+ data = None
123
+ message = (data.get("detail") or data.get("message") or data.get("title")) if isinstance(data, dict) else None
124
+ return APIError(
125
+ response.status_code,
126
+ message if isinstance(message, str) else f"HTTP {response.status_code}.",
127
+ code=data.get("code") if isinstance(data, dict) else None,
128
+ request_id=response.headers.get("x-request-id") or (data.get("requestId") if isinstance(data, dict) else None),
129
+ retry_after=response.headers.get("retry-after")
130
+ or (str(data["retry_after"]) if isinstance(data, dict) and data.get("retry_after") is not None else None),
131
+ body=data,
132
+ )
133
+
134
+
135
+ def validate_callback(callback: Any) -> None:
136
+ if callback is None or callback == "":
137
+ return
138
+ message = "callback_url must be a public HTTPS URL on port 443."
139
+ try:
140
+ if not isinstance(callback, str) or len(callback) > 2048:
141
+ raise ValueError
142
+ p = urlsplit(callback)
143
+ if (
144
+ p.scheme != "https"
145
+ or not p.hostname
146
+ or "." not in p.hostname
147
+ or p.username
148
+ or p.password
149
+ or p.fragment
150
+ or p.port not in (None, 443)
151
+ ):
152
+ raise ValueError
153
+ try:
154
+ address = ipaddress.ip_address(p.hostname)
155
+ except ValueError:
156
+ address = None
157
+ if address is not None and not address.is_global:
158
+ raise ValueError
159
+ except ValueError:
160
+ raise ValidationError(message) from None
161
+
162
+
163
+ def validate_query(value: Any, *, async_mode: bool = False) -> dict[str, Any]:
164
+ if not isinstance(value, dict) or not isinstance(value.get("source"), str) or value["source"] not in SCHEMAS:
165
+ raise ValidationError("Choose a supported source and provide a query object.")
166
+ data = copy.deepcopy(value)
167
+ source = data["source"]
168
+ if not async_mode and source == "youtube_download":
169
+ raise ValidationError("youtube_download supports background submission and batch only.")
170
+ schema = copy.deepcopy(SCHEMAS[source])
171
+ schema["additionalProperties"] = False
172
+ if async_mode:
173
+ schema["properties"]["callback_url"] = {"type": ["string", "null"], "maxLength": 2048}
174
+ error = next(Draft202012Validator(schema).iter_errors(data), None)
175
+ if error:
176
+ path = ".".join(str(part) for part in error.absolute_path) or "query"
177
+ # ValidationError.message may contain credentials or whole input objects.
178
+ constraint = error.validator
179
+ expected = error.validator_value
180
+ hint = {
181
+ "minimum": f"must be at least {expected}",
182
+ "maximum": f"must be at most {expected}",
183
+ "type": f"must have type {expected}",
184
+ "minLength": f"must contain at least {expected} characters",
185
+ "maxLength": f"must contain at most {expected} characters",
186
+ "required": "a required field is missing",
187
+ "additionalProperties": "contains unsupported fields",
188
+ "enum": f"must be one of {expected}",
189
+ "pattern": "has an invalid format",
190
+ }.get(constraint, f"violates {constraint}")
191
+ raise ValidationError(f"Invalid {path}: {hint}.")
192
+ validate_callback(data.get("callback_url"))
193
+ for field in ("query", "handle", "video_id"):
194
+ if field in data and not data[field].strip():
195
+ raise ValidationError(f"{field} must not be blank.")
196
+ return data
197
+
198
+
199
+ def bulk_input(value: Any, *, partial: bool = False) -> dict[str, Any]:
200
+ if (
201
+ not isinstance(value, dict)
202
+ or not value
203
+ or set(value) - ({"options", "callback_url"} if partial else {"source", "options", "callback_url"})
204
+ ):
205
+ raise ValidationError("Invalid bulk request fields.")
206
+ data = copy.deepcopy(value)
207
+ if not partial and data.get("source") not in BULK_SOURCES:
208
+ raise ValidationError("Bulk supports metadata, transcript, captions, license checks and downloads.")
209
+ if "options" in data and not isinstance(data["options"], dict):
210
+ raise ValidationError("Bulk options must be an object.")
211
+ try:
212
+ json.dumps(data, allow_nan=False)
213
+ except (TypeError, ValueError):
214
+ raise ValidationError("Bulk request values must be valid JSON.") from None
215
+ validate_callback(data.get("callback_url"))
216
+ if not partial:
217
+ options = copy.deepcopy(data.get("options", {}))
218
+ result_storage = options.pop("result_storage", None)
219
+ if result_storage is not None:
220
+ if data["source"] == "youtube_download" or not isinstance(result_storage, dict):
221
+ raise ValidationError("result_storage applies to bulk data sources and must be an object.")
222
+ storage_schema = SCHEMAS["youtube_download"]["properties"]["params"]["properties"]["storage"]
223
+ if not Draft202012Validator(storage_schema).is_valid(result_storage):
224
+ raise ValidationError("Invalid options.result_storage configuration.")
225
+ validate_query({"source": data["source"], "video_id": "dQw4w9WgXcQ", "params": options}, async_mode=True)
226
+ if not partial and data["source"] == "youtube_download" and not data.get("options", {}).get("storage"):
227
+ raise ValidationError("Bulk downloads require options.storage.")
228
+ return data
229
+
230
+
231
+ def decode_json(body: bytes) -> dict[str, Any]:
232
+ try:
233
+ value = json.loads(body)
234
+ except (ValueError, UnicodeDecodeError):
235
+ raise ProtocolError("The response was not valid JSON.") from None
236
+ if not isinstance(value, dict):
237
+ raise ProtocolError("Expected a JSON object response.")
238
+ return value
239
+
240
+
241
+ def result_rows(raw: bytes, max_bytes: int, max_line_bytes: int):
242
+ """Read a bounded NDJSON part, allowing HTTP stacks that already decoded gzip."""
243
+ stream = gzip.GzipFile(fileobj=io.BytesIO(raw)) if raw.startswith(b"\x1f\x8b") else io.BytesIO(raw)
244
+ try:
245
+ with stream:
246
+ total = 0
247
+ while True:
248
+ line = stream.readline(max_line_bytes + 1)
249
+ if not line:
250
+ break
251
+ total += len(line)
252
+ if len(line) > max_line_bytes or total > max_bytes:
253
+ from .errors import ResponseTooLargeError
254
+
255
+ raise ResponseTooLargeError("Decompressed result part exceeds configured limits.")
256
+ if line.strip():
257
+ yield decode_json(line)
258
+ except (OSError, EOFError, zlib.error):
259
+ raise ProtocolError("Invalid or truncated gzip result part.") from None
260
+
261
+
262
+ def job_record(value: Any) -> Any:
263
+ if not isinstance(value, dict) or not isinstance(value.get("job"), dict):
264
+ raise ProtocolError("Expected a job response containing a job object.")
265
+ return value["job"]