flashdata 1.1.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flashdata-1.1.0 → flashdata-1.2.0}/CHANGELOG.md +4 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/PKG-INFO +5 -1
- {flashdata-1.1.0 → flashdata-1.2.0}/README.md +4 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/README.zh-CN.md +4 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/bulk.md +167 -160
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/getting-started.md +6 -3
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/index.md +1 -0
- flashdata-1.2.0/docs/release-notes-1.2.0.md +18 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/release.md +6 -6
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/youtube_captions.md +69 -40
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/youtube_transcript.md +69 -40
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/youtube_captions.py +36 -30
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/youtube_transcript.py +36 -30
- {flashdata-1.1.0 → flashdata-1.2.0}/openapi.json +5175 -5111
- {flashdata-1.1.0 → flashdata-1.2.0}/pyproject.toml +1 -1
- {flashdata-1.1.0 → flashdata-1.2.0}/source-contract.json +1726 -1694
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata/__init__.py +1 -1
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata/_async_client.py +1 -1
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata/_client.py +1 -1
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata/_schemas.json +1128 -1096
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata/types.py +1036 -1034
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata.egg-info/PKG-INFO +5 -1
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata.egg-info/SOURCES.txt +1 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/test_sdk.py +57 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/.readthedocs.yaml +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/CONTRIBUTING.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/LICENSE +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/MANIFEST.in +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/SECURITY.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/_static/custom.css +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/account.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/api-reference.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/architecture.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/client.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/conf.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/errors.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/examples.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/jobs.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/pagination.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/recipes.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/release-notes-1.1.0.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/response-formats.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sdk-parity.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/google_images.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/google_news.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/google_places.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/google_search.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/google_shopping.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/google_videos.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/index.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/youtube_channel.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/youtube_download.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/youtube_metadata.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/youtube_search.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/youtube_search_extended.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/youtube_suggestions.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/sources/youtube_trainability.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/storage.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/docs/validation.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/README.md +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/_bulk.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/_support.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/account.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/async_client.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/async_query.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/batch.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/bulk.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/bulk_download.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/discovery.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/google_images.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/google_news.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/google_places.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/google_search.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/google_shopping.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/google_videos.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/offline.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/pagination.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/response_formats.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/search_to_csv.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/video-ids.example.txt +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/youtube_channel.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/youtube_download.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/youtube_metadata.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/youtube_search.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/youtube_search_extended.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/youtube_suggestions.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/examples/youtube_trainability.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/scripts/update_contract.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/scripts/update_response_fixtures.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/scripts/verify_package.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/setup.cfg +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata/_common.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata/_resources.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata/errors.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata/py.typed +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata.egg-info/dependency_links.txt +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata.egg-info/requires.txt +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/src/flashdata.egg-info/top_level.txt +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/conftest.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/fixtures/response-formats.json +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/fixtures/source-examples.json +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/fixtures/stored-result-formats.json +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/response_fixtures.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/test_examples_and_docs.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/test_format_typing.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/test_http_integration.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/test_live.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/test_regressions.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/test_response_formats.py +0 -0
- {flashdata-1.1.0 → flashdata-1.2.0}/tests/test_workflow_parity.py +0 -0
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.2.0 — 2026-11-05
|
|
4
|
+
|
|
5
|
+
- Support single or multiple transcript/caption languages, omitted-language fallback and per-language results. Other source APIs are unchanged.
|
|
6
|
+
|
|
3
7
|
[FlashData](https://flashdata.dev) · [API documentation](https://flashdata.dev/docs)
|
|
4
8
|
|
|
5
9
|
Final local release check: verified Python 3.10/3.12, rejected API-key control characters
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flashdata
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: Official Python SDK for FlashData Google Search, YouTube data and cloud media delivery APIs
|
|
5
5
|
Author-email: FlashData <support@flashdata.dev>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -34,6 +34,8 @@ Dynamic: license-file
|
|
|
34
34
|
|
|
35
35
|
# FlashData Python SDK
|
|
36
36
|
|
|
37
|
+
Multiple transcript/caption languages require version 1.2.0 or newer.
|
|
38
|
+
|
|
37
39
|
[English](https://github.com/flashdatadev/flashdata-python/blob/main/README.md) | [简体中文](https://github.com/flashdatadev/flashdata-python/blob/main/README.zh-CN.md)
|
|
38
40
|
|
|
39
41
|
[FlashData](https://flashdata.dev) · [API documentation](https://flashdata.dev/docs)
|
|
@@ -240,6 +242,8 @@ Captions default to an uploaded track (`auto=False`); transcripts default to an
|
|
|
240
242
|
track (`auto=True`). Choose a language and track that exist on the video. Missing tracks
|
|
241
243
|
return an API error; the service does not generate or translate them.
|
|
242
244
|
|
|
245
|
+
Transcript and captions accept one language or a list of 1–10 codes. Omit the language to prefer `en-orig`, then `en`. Multi-language results include available tracks and per-language errors. See [subtitle languages](docs/sources/youtube_transcript.md).
|
|
246
|
+
|
|
243
247
|
## Use asyncio
|
|
244
248
|
|
|
245
249
|
```python
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# FlashData Python SDK
|
|
2
2
|
|
|
3
|
+
Multiple transcript/caption languages require version 1.2.0 or newer.
|
|
4
|
+
|
|
3
5
|
[English](https://github.com/flashdatadev/flashdata-python/blob/main/README.md) | [简体中文](https://github.com/flashdatadev/flashdata-python/blob/main/README.zh-CN.md)
|
|
4
6
|
|
|
5
7
|
[FlashData](https://flashdata.dev) · [API documentation](https://flashdata.dev/docs)
|
|
@@ -206,6 +208,8 @@ Captions default to an uploaded track (`auto=False`); transcripts default to an
|
|
|
206
208
|
track (`auto=True`). Choose a language and track that exist on the video. Missing tracks
|
|
207
209
|
return an API error; the service does not generate or translate them.
|
|
208
210
|
|
|
211
|
+
Transcript and captions accept one language or a list of 1–10 codes. Omit the language to prefer `en-orig`, then `en`. Multi-language results include available tracks and per-language errors. See [subtitle languages](docs/sources/youtube_transcript.md).
|
|
212
|
+
|
|
209
213
|
## Use asyncio
|
|
210
214
|
|
|
211
215
|
```python
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# FlashData Python SDK
|
|
2
2
|
|
|
3
|
+
转录与字幕的多语言请求需要 1.2.0 或更新版本。
|
|
4
|
+
|
|
3
5
|
[English](https://github.com/flashdatadev/flashdata-python/blob/main/README.md) | [简体中文](https://github.com/flashdatadev/flashdata-python/blob/main/README.zh-CN.md)
|
|
4
6
|
|
|
5
7
|
[FlashData](https://flashdata.dev) · [API 文档](https://flashdata.dev/docs)
|
|
@@ -194,6 +196,8 @@ with FlashData(api_key=os.getenv("FLASHDATA_API_KEY")) as client:
|
|
|
194
196
|
`captions` 默认选择上传的字幕轨道(`auto=False`),`transcript` 默认选择自动生成的轨道(`auto=True`)。
|
|
195
197
|
请指定视频实际拥有的语言和轨道。轨道不存在时,API 会返回错误;服务不会生成或翻译字幕。
|
|
196
198
|
|
|
199
|
+
转录和字幕支持单个语言代码或 1–10 个代码的列表。省略语言时优先使用 `en-orig`,其次使用 `en`,返回一条字幕。多语言结果保留可用字幕及逐语言错误。参见[字幕语言说明](docs/sources/youtube_transcript.md)。
|
|
200
|
+
|
|
197
201
|
## 使用 asyncio
|
|
198
202
|
|
|
199
203
|
```python
|
|
@@ -1,160 +1,167 @@
|
|
|
1
|
-
# Build a YouTube dataset with bulk jobs
|
|
2
|
-
|
|
3
|
-
[FlashData](https://flashdata.dev) · [API documentation](https://flashdata.dev/docs)
|
|
4
|
-
|
|
5
|
-
Bulk jobs process a UTF-8 file containing one video ID per line. Supported sources:
|
|
6
|
-
`youtube_metadata`, `youtube_transcript`, `youtube_captions`, `youtube_trainability`,
|
|
7
|
-
and `youtube_download`. File uploads support 1 byte to 2 GiB. A bulk job uses one source
|
|
8
|
-
and one set of options for all IDs; use a [mixed batch](jobs.md) for different queries.
|
|
9
|
-
|
|
10
|
-
## Create, upload, start, export
|
|
11
|
-
|
|
12
|
-
Create a local `video-ids.txt` file containing, for example, `dQw4w9WgXcQ` on one line.
|
|
13
|
-
Then run:
|
|
14
|
-
|
|
15
|
-
```python
|
|
16
|
-
import json
|
|
17
|
-
import os
|
|
18
|
-
from pathlib import Path
|
|
19
|
-
from flashdata import FlashData
|
|
20
|
-
|
|
21
|
-
with (
|
|
22
|
-
FlashData(api_key=os.getenv("FLASHDATA_API_KEY")) as client,
|
|
23
|
-
Path("videos.ndjson").open("x", encoding="utf-8") as output,
|
|
24
|
-
Path("videos.ndjson.job.json").open("x", encoding="utf-8") as record,
|
|
25
|
-
):
|
|
26
|
-
receipt = client.bulk.create({
|
|
27
|
-
"source": "youtube_metadata",
|
|
28
|
-
"options": {"include_formats": False},
|
|
29
|
-
})
|
|
30
|
-
bulk_id = receipt["job"]["id"]
|
|
31
|
-
print("Save this bulk ID:", bulk_id, flush=True)
|
|
32
|
-
json.dump({"id": bulk_id}, record)
|
|
33
|
-
record.flush()
|
|
34
|
-
os.fsync(record.fileno())
|
|
35
|
-
client.bulk.upload_file(receipt, Path("video-ids.txt"))
|
|
36
|
-
client.bulk.start(bulk_id)
|
|
37
|
-
completed = client.bulk.wait(bulk_id, timeout=3600, poll_interval=5)
|
|
38
|
-
|
|
39
|
-
for row in client.bulk.rows(bulk_id):
|
|
40
|
-
output.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
41
|
-
print("Saved videos.ndjson; counts:", completed["counts"])
|
|
42
|
-
if completed["counts"].get("failed") or completed["counts"].get("invalid"):
|
|
43
|
-
raise RuntimeError("Some items failed; inspect the saved rows before retrying")
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
Creation returns a job plus a short-lived signed PUT upload descriptor (15 minutes).
|
|
47
|
-
`upload_file` streams the input file with the required content type and length; it does
|
|
48
|
-
not start the job. If upload fails, keep the ID and inspect its state. Retry the same
|
|
49
|
-
upload only while its descriptor is valid. An expired input upload needs a new draft;
|
|
50
|
-
cancel the unused draft before creating its replacement. Re-creation is not automatic.
|
|
51
|
-
Do not log the upload descriptor or signed URLs.
|
|
52
|
-
|
|
53
|
-
Bulk options use `options`, rather than individual query `params`. Metadata accepts
|
|
54
|
-
`include_formats` and `include_chapters`; transcript/captions accept `language` and `auto`;
|
|
55
|
-
license checks have no data options. Data sources may also set `result_storage` for
|
|
56
|
-
cloud delivery. Downloads require `options.storage`. See [storage](storage.md).
|
|
57
|
-
|
|
58
|
-
## Download media into your storage
|
|
59
|
-
|
|
60
|
-
Use `source="youtube_download"` and put the private destination in `options.storage`.
|
|
61
|
-
Prepare `video-ids.txt` and a private destination JSON file as described in
|
|
62
|
-
[storage configuration](storage.md), then run the complete task:
|
|
63
|
-
|
|
64
|
-
```python
|
|
65
|
-
import json
|
|
66
|
-
import os
|
|
67
|
-
from pathlib import Path
|
|
68
|
-
from flashdata import FlashData
|
|
69
|
-
|
|
70
|
-
storage = json.loads(Path("private-storage.json").read_text(encoding="utf-8"))
|
|
71
|
-
|
|
72
|
-
with (
|
|
73
|
-
FlashData(api_key=os.getenv("FLASHDATA_API_KEY")) as client,
|
|
74
|
-
Path("downloads.ndjson").open("x", encoding="utf-8") as output,
|
|
75
|
-
Path("downloads.ndjson.job.json").open("x", encoding="utf-8") as record,
|
|
76
|
-
):
|
|
77
|
-
receipt = client.bulk.create({
|
|
78
|
-
"source": "youtube_download",
|
|
79
|
-
"options": {
|
|
80
|
-
"download_type": "audio_video",
|
|
81
|
-
"video_quality": "<=720",
|
|
82
|
-
"output_format": "mp4",
|
|
83
|
-
"storage": storage,
|
|
84
|
-
},
|
|
85
|
-
})
|
|
86
|
-
bulk_id = receipt["job"]["id"]
|
|
87
|
-
print("Save this bulk ID:", bulk_id, flush=True)
|
|
88
|
-
json.dump({"id": bulk_id}, record)
|
|
89
|
-
record.flush()
|
|
90
|
-
os.fsync(record.fileno())
|
|
91
|
-
client.bulk.upload_file(receipt, Path("video-ids.txt"))
|
|
92
|
-
client.bulk.start(bulk_id)
|
|
93
|
-
|
|
94
|
-
completed = client.bulk.wait(bulk_id, timeout=3600, poll_interval=5)
|
|
95
|
-
print("Counts:", completed["counts"])
|
|
96
|
-
for row in client.bulk.rows(bulk_id):
|
|
97
|
-
output.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
98
|
-
print(row["video_id"], row["status"])
|
|
99
|
-
if completed["counts"].get("failed") or completed["counts"].get("invalid"):
|
|
100
|
-
raise RuntimeError("Some items failed; inspect downloads.ndjson before retrying")
|
|
101
|
-
```
|
|
102
|
-
|
|
103
|
-
For a runnable export:
|
|
104
|
-
|
|
105
|
-
```sh
|
|
106
|
-
python examples/bulk_download.py video-ids.txt --storage-file private-storage.json --output artifacts/downloads.ndjson
|
|
107
|
-
python examples/bulk_download.py --bulk-id bulk_YOUR_SAVED_ID --output artifacts/downloads-recovered.ndjson
|
|
108
|
-
```
|
|
109
|
-
|
|
110
|
-
Both bulk CLIs save only the job record to `OUTPUT.job.json` before upload and refresh
|
|
111
|
-
it on completion. They never save the signed upload descriptor. Result rows are exported
|
|
112
|
-
before an item-failure exit status, preserving successful and failed records for recovery.
|
|
113
|
-
The media goes to your bucket; the NDJSON describes delivery, rather than containing media.
|
|
114
|
-
|
|
115
|
-
A resume command waits for a task that has already started. An `awaiting_upload` draft
|
|
116
|
-
still needs its input uploaded and `start` called. A paused task requires resolving the
|
|
117
|
-
blocking condition and calling `resume` before waiting again.
|
|
118
|
-
|
|
119
|
-
## Progress and controls
|
|
120
|
-
|
|
121
|
-
`get(id)` returns a record with counts, status, pause reason and delivery status.
|
|
122
|
-
Typical states are `awaiting_upload`, `running`, `paused`, `finalizing`, `completed`,
|
|
123
|
-
`failed`, and `cancelled`. Counts include accepted, completed, failed, invalid, and results.
|
|
124
|
-
A completed bulk can contain failed rows; inspect counts and each row's status.
|
|
125
|
-
|
|
126
|
-
- `pause(id)` requests a pause for running work; already active items may finish.
|
|
127
|
-
- `resume(id)` resumes a paused job once its blocking condition is resolved.
|
|
128
|
-
- `update(id, {"options": ...})` updates settings allowed in its current state.
|
|
129
|
-
Storage credentials can be patched without reconstructing the whole destination.
|
|
130
|
-
- `cancel(id)` cancels remaining work; completed work may already be charged.
|
|
131
|
-
|
|
132
|
-
`wait` stops with `JobPausedError` on a pause and `JobFailedError` on failure/cancellation.
|
|
133
|
-
It does not resume or cancel work automatically. Download billing uses actual delivered
|
|
134
|
-
bytes; data bulk jobs are billed per admitted item under your plan's current pricing.
|
|
135
|
-
|
|
136
|
-
## Read results and list jobs
|
|
137
|
-
|
|
138
|
-
`manifest(id)` retrieves counts, expiry, and ordered part descriptors. `part(id, number)`
|
|
139
|
-
returns bounded bytes; `iter_rows(id, number)` decodes a gzip/NDJSON part. `rows(id)` walks
|
|
140
|
-
manifest parts in order and yields their JSON objects. It uses constructed API routes,
|
|
141
|
-
not arbitrary URLs supplied in a manifest. Parts are zero based. Result files expire
|
|
142
|
-
24 hours after completion; export them promptly.
|
|
143
|
-
|
|
144
|
-
`list(cursor=..., limit=20)` returns `data` and `pagination`. `iter_jobs(max_pages=..., limit=...)`
|
|
145
|
-
walks a bounded number of cursor pages. Both have async equivalents; use `async for`
|
|
146
|
-
for async job and row iterators.
|
|
147
|
-
|
|
148
|
-
The runnable [bulk example](examples.md) supports submitting and resuming an
|
|
149
|
-
existing ID, so a process restart does not require re-uploading or re-submitting work.
|
|
150
|
-
|
|
151
|
-
## Safe local exports
|
|
152
|
-
|
|
153
|
-
The bulk scripts reserve new output and `OUTPUT.job.json` files before any API call,
|
|
154
|
-
refuse existing files, and sync accepted IDs before upload. Completion records are
|
|
155
|
-
replaced atomically. A lost start response leaves its accepted ID available; recovery
|
|
156
|
-
uses that ID and a fresh output path without another creation or upload. Empty reserved
|
|
157
|
-
records after an uncertain create require account/server reconciliation.
|
|
158
|
-
|
|
159
|
-
Use `bulk_download.py` with `--audio` for MP3; the default is MP4.
|
|
160
|
-
Compare [all SDKs and the CLI](sdk-parity.md) for coverage and persistence differences.
|
|
1
|
+
# Build a YouTube dataset with bulk jobs
|
|
2
|
+
|
|
3
|
+
[FlashData](https://flashdata.dev) · [API documentation](https://flashdata.dev/docs)
|
|
4
|
+
|
|
5
|
+
Bulk jobs process a UTF-8 file containing one video ID per line. Supported sources:
|
|
6
|
+
`youtube_metadata`, `youtube_transcript`, `youtube_captions`, `youtube_trainability`,
|
|
7
|
+
and `youtube_download`. File uploads support 1 byte to 2 GiB. A bulk job uses one source
|
|
8
|
+
and one set of options for all IDs; use a [mixed batch](jobs.md) for different queries.
|
|
9
|
+
|
|
10
|
+
## Create, upload, start, export
|
|
11
|
+
|
|
12
|
+
Create a local `video-ids.txt` file containing, for example, `dQw4w9WgXcQ` on one line.
|
|
13
|
+
Then run:
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from flashdata import FlashData
|
|
20
|
+
|
|
21
|
+
with (
|
|
22
|
+
FlashData(api_key=os.getenv("FLASHDATA_API_KEY")) as client,
|
|
23
|
+
Path("videos.ndjson").open("x", encoding="utf-8") as output,
|
|
24
|
+
Path("videos.ndjson.job.json").open("x", encoding="utf-8") as record,
|
|
25
|
+
):
|
|
26
|
+
receipt = client.bulk.create({
|
|
27
|
+
"source": "youtube_metadata",
|
|
28
|
+
"options": {"include_formats": False},
|
|
29
|
+
})
|
|
30
|
+
bulk_id = receipt["job"]["id"]
|
|
31
|
+
print("Save this bulk ID:", bulk_id, flush=True)
|
|
32
|
+
json.dump({"id": bulk_id}, record)
|
|
33
|
+
record.flush()
|
|
34
|
+
os.fsync(record.fileno())
|
|
35
|
+
client.bulk.upload_file(receipt, Path("video-ids.txt"))
|
|
36
|
+
client.bulk.start(bulk_id)
|
|
37
|
+
completed = client.bulk.wait(bulk_id, timeout=3600, poll_interval=5)
|
|
38
|
+
|
|
39
|
+
for row in client.bulk.rows(bulk_id):
|
|
40
|
+
output.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
41
|
+
print("Saved videos.ndjson; counts:", completed["counts"])
|
|
42
|
+
if completed["counts"].get("failed") or completed["counts"].get("invalid"):
|
|
43
|
+
raise RuntimeError("Some items failed; inspect the saved rows before retrying")
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Creation returns a job plus a short-lived signed PUT upload descriptor (15 minutes).
|
|
47
|
+
`upload_file` streams the input file with the required content type and length; it does
|
|
48
|
+
not start the job. If upload fails, keep the ID and inspect its state. Retry the same
|
|
49
|
+
upload only while its descriptor is valid. An expired input upload needs a new draft;
|
|
50
|
+
cancel the unused draft before creating its replacement. Re-creation is not automatic.
|
|
51
|
+
Do not log the upload descriptor or signed URLs.
|
|
52
|
+
|
|
53
|
+
Bulk options use `options`, rather than individual query `params`. Metadata accepts
|
|
54
|
+
`include_formats` and `include_chapters`; transcript/captions accept `language` and `auto`;
|
|
55
|
+
license checks have no data options. Data sources may also set `result_storage` for
|
|
56
|
+
cloud delivery. Downloads require `options.storage`. See [storage](storage.md).
|
|
57
|
+
|
|
58
|
+
## Download media into your storage
|
|
59
|
+
|
|
60
|
+
Use `source="youtube_download"` and put the private destination in `options.storage`.
|
|
61
|
+
Prepare `video-ids.txt` and a private destination JSON file as described in
|
|
62
|
+
[storage configuration](storage.md), then run the complete task:
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
import json
|
|
66
|
+
import os
|
|
67
|
+
from pathlib import Path
|
|
68
|
+
from flashdata import FlashData
|
|
69
|
+
|
|
70
|
+
storage = json.loads(Path("private-storage.json").read_text(encoding="utf-8"))
|
|
71
|
+
|
|
72
|
+
with (
|
|
73
|
+
FlashData(api_key=os.getenv("FLASHDATA_API_KEY")) as client,
|
|
74
|
+
Path("downloads.ndjson").open("x", encoding="utf-8") as output,
|
|
75
|
+
Path("downloads.ndjson.job.json").open("x", encoding="utf-8") as record,
|
|
76
|
+
):
|
|
77
|
+
receipt = client.bulk.create({
|
|
78
|
+
"source": "youtube_download",
|
|
79
|
+
"options": {
|
|
80
|
+
"download_type": "audio_video",
|
|
81
|
+
"video_quality": "<=720",
|
|
82
|
+
"output_format": "mp4",
|
|
83
|
+
"storage": storage,
|
|
84
|
+
},
|
|
85
|
+
})
|
|
86
|
+
bulk_id = receipt["job"]["id"]
|
|
87
|
+
print("Save this bulk ID:", bulk_id, flush=True)
|
|
88
|
+
json.dump({"id": bulk_id}, record)
|
|
89
|
+
record.flush()
|
|
90
|
+
os.fsync(record.fileno())
|
|
91
|
+
client.bulk.upload_file(receipt, Path("video-ids.txt"))
|
|
92
|
+
client.bulk.start(bulk_id)
|
|
93
|
+
|
|
94
|
+
completed = client.bulk.wait(bulk_id, timeout=3600, poll_interval=5)
|
|
95
|
+
print("Counts:", completed["counts"])
|
|
96
|
+
for row in client.bulk.rows(bulk_id):
|
|
97
|
+
output.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
98
|
+
print(row["video_id"], row["status"])
|
|
99
|
+
if completed["counts"].get("failed") or completed["counts"].get("invalid"):
|
|
100
|
+
raise RuntimeError("Some items failed; inspect downloads.ndjson before retrying")
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
For a runnable export:
|
|
104
|
+
|
|
105
|
+
```sh
|
|
106
|
+
python examples/bulk_download.py video-ids.txt --storage-file private-storage.json --output artifacts/downloads.ndjson
|
|
107
|
+
python examples/bulk_download.py --bulk-id bulk_YOUR_SAVED_ID --output artifacts/downloads-recovered.ndjson
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Both bulk CLIs save only the job record to `OUTPUT.job.json` before upload and refresh
|
|
111
|
+
it on completion. They never save the signed upload descriptor. Result rows are exported
|
|
112
|
+
before an item-failure exit status, preserving successful and failed records for recovery.
|
|
113
|
+
The media goes to your bucket; the NDJSON describes delivery, rather than containing media.
|
|
114
|
+
|
|
115
|
+
A resume command waits for a task that has already started. An `awaiting_upload` draft
|
|
116
|
+
still needs its input uploaded and `start` called. A paused task requires resolving the
|
|
117
|
+
blocking condition and calling `resume` before waiting again.
|
|
118
|
+
|
|
119
|
+
## Progress and controls
|
|
120
|
+
|
|
121
|
+
`get(id)` returns a record with counts, status, pause reason and delivery status.
|
|
122
|
+
Typical states are `awaiting_upload`, `running`, `paused`, `finalizing`, `completed`,
|
|
123
|
+
`failed`, and `cancelled`. Counts include accepted, completed, failed, invalid, and results.
|
|
124
|
+
A completed bulk can contain failed rows; inspect counts and each row's status.
|
|
125
|
+
|
|
126
|
+
- `pause(id)` requests a pause for running work; already active items may finish.
|
|
127
|
+
- `resume(id)` resumes a paused job once its blocking condition is resolved.
|
|
128
|
+
- `update(id, {"options": ...})` updates settings allowed in its current state.
|
|
129
|
+
Storage credentials can be patched without reconstructing the whole destination.
|
|
130
|
+
- `cancel(id)` cancels remaining work; completed work may already be charged.
|
|
131
|
+
|
|
132
|
+
`wait` stops with `JobPausedError` on a pause and `JobFailedError` on failure/cancellation.
|
|
133
|
+
It does not resume or cancel work automatically. Download billing uses actual delivered
|
|
134
|
+
bytes; data bulk jobs are billed per admitted item under your plan's current pricing.
|
|
135
|
+
|
|
136
|
+
## Read results and list jobs
|
|
137
|
+
|
|
138
|
+
`manifest(id)` retrieves counts, expiry, and ordered part descriptors. `part(id, number)`
|
|
139
|
+
returns bounded bytes; `iter_rows(id, number)` decodes a gzip/NDJSON part. `rows(id)` walks
|
|
140
|
+
manifest parts in order and yields their JSON objects. It uses constructed API routes,
|
|
141
|
+
not arbitrary URLs supplied in a manifest. Parts are zero based. Result files expire
|
|
142
|
+
24 hours after completion; export them promptly.
|
|
143
|
+
|
|
144
|
+
`list(cursor=..., limit=20)` returns `data` and `pagination`. `iter_jobs(max_pages=..., limit=...)`
|
|
145
|
+
walks a bounded number of cursor pages. Both have async equivalents; use `async for`
|
|
146
|
+
for async job and row iterators.
|
|
147
|
+
|
|
148
|
+
The runnable [bulk example](examples.md) supports submitting and resuming an
|
|
149
|
+
existing ID, so a process restart does not require re-uploading or re-submitting work.
|
|
150
|
+
|
|
151
|
+
## Safe local exports
|
|
152
|
+
|
|
153
|
+
The bulk scripts reserve new output and `OUTPUT.job.json` files before any API call,
|
|
154
|
+
refuse existing files, and sync accepted IDs before upload. Completion records are
|
|
155
|
+
replaced atomically. A lost start response leaves its accepted ID available; recovery
|
|
156
|
+
uses that ID and a fresh output path without another creation or upload. Empty reserved
|
|
157
|
+
records after an uncertain create require account/server reconciliation.
|
|
158
|
+
|
|
159
|
+
Use `bulk_download.py` with `--audio` for MP3; the default is MP4.
|
|
160
|
+
Compare [all SDKs and the CLI](sdk-parity.md) for coverage and persistence differences.
|
|
161
|
+
|
|
162
|
+
## Subtitle languages
|
|
163
|
+
|
|
164
|
+
For `youtube_transcript` and `youtube_captions`, use shared options such as
|
|
165
|
+
`{"language":["en","es"],"auto":true}`. Omit `language` to prefer `en-orig`, then `en`.
|
|
166
|
+
Bulk exports remain JSONL with one record per video; inspect `tracks` and `errors`
|
|
167
|
+
inside each result for available languages and partial failures.
|
|
@@ -7,11 +7,14 @@
|
|
|
7
7
|
Use Python 3.10 or newer:
|
|
8
8
|
|
|
9
9
|
```sh
|
|
10
|
-
python -m pip install --upgrade
|
|
10
|
+
python -m pip install --upgrade https://github.com/flashdatadev/flashdata-python/releases/download/v1.2.0/flashdata-1.2.0-py3-none-any.whl
|
|
11
11
|
```
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
The command installs the GitHub release wheel. Once 1.2.0 is available on PyPI,
|
|
14
|
+
you can also use `python -m pip install --upgrade "flashdata>=1.2.0"`.
|
|
15
|
+
|
|
16
|
+
Version 1.2.0 adds multiple subtitle languages and per-language results. See the
|
|
17
|
+
[release notes](release-notes-1.2.0.md).
|
|
15
18
|
|
|
16
19
|
## Install from source
|
|
17
20
|
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# FlashData Python SDK 1.2.0
|
|
2
|
+
|
|
3
|
+
[FlashData](https://flashdata.dev) · [API documentation](https://flashdata.dev/docs)
|
|
4
|
+
|
|
5
|
+
Transcript and captions now accept one language code or a list of 1–10 distinct
|
|
6
|
+
codes. Omit language to prefer `en-orig`, then `en`, returning one track.
|
|
7
|
+
|
|
8
|
+
- Explicit single-language requests keep their existing response shape.
|
|
9
|
+
- Lists, including one-element lists, return successful `tracks` and per-language
|
|
10
|
+
`errors`. If no languages succeed, `tracks` is empty and the errors remain available.
|
|
11
|
+
- The selected automatic or uploader caption source applies to every language.
|
|
12
|
+
- Realtime, asynchronous, batch and bulk requests support the new language options.
|
|
13
|
+
Bulk exports remain JSONL. Other API contracts and response-format defaults are unchanged.
|
|
14
|
+
|
|
15
|
+
Use `language=["en", "es"]` with the sync or async client.
|
|
16
|
+
|
|
17
|
+
See the [subtitle examples](sources/youtube_transcript.md) and
|
|
18
|
+
[API documentation](https://flashdata.dev/docs/sources/youtube_transcript).
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
[FlashData](https://flashdata.dev) · [API documentation](https://flashdata.dev/docs)
|
|
4
4
|
|
|
5
|
-
The official package is `flashdata` for Python 3.10+.
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
JSON response defaults.
|
|
5
|
+
The official package is `flashdata` for Python 3.10+.
|
|
6
|
+
Version 1.2.0 adds multiple transcript/caption languages and per-language results.
|
|
7
|
+
Read the [release notes](release-notes-1.2.0.md). Existing single-language calls and
|
|
8
|
+
JSON response defaults remain compatible.
|
|
9
9
|
|
|
10
|
-
The [GitHub release](https://github.com/flashdatadev/flashdata-python/releases/tag/v1.
|
|
11
|
-
provides packages from the tagged source. Published registry versions are listed on
|
|
10
|
+
The [GitHub release](https://github.com/flashdatadev/flashdata-python/releases/tag/v1.2.0)
|
|
11
|
+
provides packages from the tagged source. PyPI upload of 1.2.0 is handled manually. Published registry versions are listed on
|
|
12
12
|
[PyPI](https://pypi.org/project/flashdata/). GitHub wheels can also be installed directly with pip.
|
|
13
13
|
|
|
14
14
|
## Maintainer checks
|
|
@@ -1,40 +1,69 @@
|
|
|
1
|
-
# YouTube captions
|
|
2
|
-
|
|
3
|
-
[FlashData](https://flashdata.dev) · [API documentation](https://flashdata.dev/docs/sources/youtube_captions)
|
|
4
|
-
|
|
5
|
-
Read an existing uploaded caption track by default. Check `languageCode` and `isAutoGenerated` before consuming `segments` or `fullText`. Select a track that exists on the video.
|
|
6
|
-
|
|
7
|
-
## Example
|
|
8
|
-
|
|
9
|
-
```python
|
|
10
|
-
import os
|
|
11
|
-
from flashdata import FlashData
|
|
12
|
-
|
|
13
|
-
with FlashData(api_key=os.getenv("FLASHDATA_API_KEY")) as client:
|
|
14
|
-
response = client.youtube.captions("dQw4w9WgXcQ", language="en", auto=False)
|
|
15
|
-
for track in response["results"]:
|
|
16
|
-
print(track["languageCode"], track["isAutoGenerated"])
|
|
17
|
-
for segment in track.get("segments", [])[:3]:
|
|
18
|
-
print(segment["start"], segment["end"], segment["text"])
|
|
19
|
-
```
|
|
20
|
-
|
|
21
|
-
Run `python examples/youtube_captions.py dQw4w9WgXcQ --output artifacts/youtube_captions.json` from the checkout. Use `--help` for options. The command consumes credits.
|
|
22
|
-
|
|
23
|
-
## Parameters
|
|
24
|
-
|
|
25
|
-
`video_id` is required. Source options are keyword arguments, or entries in `params` when using `queries`. Data helpers default to realtime; pass `mode="async"` for background work. `timeout` controls the HTTP request.
|
|
26
|
-
|
|
27
|
-
JSON is the default. Use `accept="text/plain"` for text, or `respond_with="markdown"`
|
|
28
|
-
for Markdown content in realtime results. For background work, choose Markdown when
|
|
29
|
-
reading the completed result. See [response formats](../response-formats.md).
|
|
30
|
-
|
|
31
|
-
| Option | Type / constraints | Default |
|
|
32
|
-
| --- | --- | --- |
|
|
33
|
-
| `language` | string
|
|
34
|
-
| `auto` | boolean | false |
|
|
35
|
-
|
|
36
|
-
`
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
1
|
+
# YouTube captions
|
|
2
|
+
|
|
3
|
+
[FlashData](https://flashdata.dev) · [API documentation](https://flashdata.dev/docs/sources/youtube_captions)
|
|
4
|
+
|
|
5
|
+
Read an existing uploaded caption track by default. Check `languageCode` and `isAutoGenerated` before consuming `segments` or `fullText`. Select a track that exists on the video.
|
|
6
|
+
|
|
7
|
+
## Example
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
import os
|
|
11
|
+
from flashdata import FlashData
|
|
12
|
+
|
|
13
|
+
with FlashData(api_key=os.getenv("FLASHDATA_API_KEY")) as client:
|
|
14
|
+
response = client.youtube.captions("dQw4w9WgXcQ", language="en", auto=False)
|
|
15
|
+
for track in response["results"]:
|
|
16
|
+
print(track["languageCode"], track["isAutoGenerated"])
|
|
17
|
+
for segment in track.get("segments", [])[:3]:
|
|
18
|
+
print(segment["start"], segment["end"], segment["text"])
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Run `python examples/youtube_captions.py dQw4w9WgXcQ --output artifacts/youtube_captions.json` from the checkout. Use `--help` for options. The command consumes credits.
|
|
22
|
+
|
|
23
|
+
## Parameters
|
|
24
|
+
|
|
25
|
+
`video_id` is required. Source options are keyword arguments, or entries in `params` when using `queries`. Data helpers default to realtime; pass `mode="async"` for background work. `timeout` controls the HTTP request.
|
|
26
|
+
|
|
27
|
+
JSON is the default. Use `accept="text/plain"` for text, or `respond_with="markdown"`
|
|
28
|
+
for Markdown content in realtime results. For background work, choose Markdown when
|
|
29
|
+
reading the completed result. See [response formats](../response-formats.md).
|
|
30
|
+
|
|
31
|
+
| Option | Type / constraints | Default |
|
|
32
|
+
| --- | --- | --- |
|
|
33
|
+
| `language` | string or array of 1–10 distinct codes | `en-orig`, then `en` |
|
|
34
|
+
| `auto` | boolean | false |
|
|
35
|
+
|
|
36
|
+
Omit `language` to prefer `en-orig`, falling back to `en` and returning one track.
|
|
37
|
+
A string such as `"en"` selects that exact language and keeps the single-track response.
|
|
38
|
+
An array of 1–10 distinct codes, including a one-element array, returns `tracks` and
|
|
39
|
+
`errors` inside each result. Each code must match `[A-Za-z0-9-]{1,35}`.
|
|
40
|
+
|
|
41
|
+
Read successful tracks even when another language fails. Each error includes
|
|
42
|
+
`languageCode`, `code` (`language_unavailable` or `retrieval_failed`) and `message`.
|
|
43
|
+
Both lists preserve request order; if all requested languages fail, `tracks` is empty.
|
|
44
|
+
Video-level extraction failures still fail the request. Explicit languages never fall
|
|
45
|
+
back to other languages. `auto` selects one caption source for the whole request:
|
|
46
|
+
`true` for automatic captions, `false` for uploaded captions. It defaults to `true`
|
|
47
|
+
for transcripts and `false` for captions, with no switch between the two.
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
## Multiple languages
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
import os
|
|
54
|
+
from flashdata import FlashData
|
|
55
|
+
|
|
56
|
+
with FlashData(api_key=os.getenv("FLASHDATA_API_KEY")) as client:
|
|
57
|
+
response = client.youtube.captions("aircAruvnKk", language=["en", "es"])
|
|
58
|
+
for result in response["results"]:
|
|
59
|
+
for track in result.get("tracks", []):
|
|
60
|
+
print(track["languageCode"], track["fullText"])
|
|
61
|
+
for error in result.get("errors", []):
|
|
62
|
+
print(error["languageCode"], error["code"], error["message"])
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The same language array works with realtime, async, batch and bulk requests.
|
|
66
|
+
|
|
67
|
+
## Use at scale
|
|
68
|
+
|
|
69
|
+
Use [bulk jobs](../bulk.md) for a file of video IDs with the same options. All sources support [background and batch queries](../jobs.md). Read [errors and recovery](../errors.md) before adding application retries.
|