webscout 8.3.6__py3-none-any.whl → 2025.10.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of webscout might be problematic. Click here for more details.
- webscout/AIauto.py +250 -250
- webscout/AIbase.py +379 -379
- webscout/AIutel.py +60 -58
- webscout/Bard.py +1012 -1012
- webscout/Bing_search.py +417 -417
- webscout/DWEBS.py +529 -529
- webscout/Extra/Act.md +309 -309
- webscout/Extra/GitToolkit/__init__.py +10 -10
- webscout/Extra/GitToolkit/gitapi/README.md +110 -110
- webscout/Extra/GitToolkit/gitapi/__init__.py +11 -11
- webscout/Extra/GitToolkit/gitapi/repository.py +195 -195
- webscout/Extra/GitToolkit/gitapi/user.py +96 -96
- webscout/Extra/GitToolkit/gitapi/utils.py +61 -61
- webscout/Extra/YTToolkit/README.md +375 -375
- webscout/Extra/YTToolkit/YTdownloader.py +956 -956
- webscout/Extra/YTToolkit/__init__.py +2 -2
- webscout/Extra/YTToolkit/transcriber.py +475 -475
- webscout/Extra/YTToolkit/ytapi/README.md +44 -44
- webscout/Extra/YTToolkit/ytapi/__init__.py +6 -6
- webscout/Extra/YTToolkit/ytapi/channel.py +307 -307
- webscout/Extra/YTToolkit/ytapi/errors.py +13 -13
- webscout/Extra/YTToolkit/ytapi/extras.py +118 -118
- webscout/Extra/YTToolkit/ytapi/https.py +88 -88
- webscout/Extra/YTToolkit/ytapi/patterns.py +61 -61
- webscout/Extra/YTToolkit/ytapi/playlist.py +58 -58
- webscout/Extra/YTToolkit/ytapi/pool.py +7 -7
- webscout/Extra/YTToolkit/ytapi/query.py +39 -39
- webscout/Extra/YTToolkit/ytapi/stream.py +62 -62
- webscout/Extra/YTToolkit/ytapi/utils.py +62 -62
- webscout/Extra/YTToolkit/ytapi/video.py +232 -232
- webscout/Extra/autocoder/__init__.py +9 -9
- webscout/Extra/autocoder/autocoder.py +1105 -1105
- webscout/Extra/autocoder/autocoder_utiles.py +332 -332
- webscout/Extra/gguf.md +429 -429
- webscout/Extra/gguf.py +1213 -1213
- webscout/Extra/tempmail/README.md +487 -487
- webscout/Extra/tempmail/__init__.py +27 -27
- webscout/Extra/tempmail/async_utils.py +140 -140
- webscout/Extra/tempmail/base.py +160 -160
- webscout/Extra/tempmail/cli.py +186 -186
- webscout/Extra/tempmail/emailnator.py +84 -84
- webscout/Extra/tempmail/mail_tm.py +360 -360
- webscout/Extra/tempmail/temp_mail_io.py +291 -291
- webscout/Extra/weather.md +281 -281
- webscout/Extra/weather.py +193 -193
- webscout/Litlogger/README.md +10 -10
- webscout/Litlogger/__init__.py +15 -15
- webscout/Litlogger/formats.py +13 -13
- webscout/Litlogger/handlers.py +121 -121
- webscout/Litlogger/levels.py +13 -13
- webscout/Litlogger/logger.py +134 -134
- webscout/Provider/AISEARCH/Perplexity.py +332 -332
- webscout/Provider/AISEARCH/README.md +279 -279
- webscout/Provider/AISEARCH/__init__.py +33 -11
- webscout/Provider/AISEARCH/felo_search.py +206 -206
- webscout/Provider/AISEARCH/genspark_search.py +323 -323
- webscout/Provider/AISEARCH/hika_search.py +185 -185
- webscout/Provider/AISEARCH/iask_search.py +410 -410
- webscout/Provider/AISEARCH/monica_search.py +219 -219
- webscout/Provider/AISEARCH/scira_search.py +316 -314
- webscout/Provider/AISEARCH/stellar_search.py +177 -177
- webscout/Provider/AISEARCH/webpilotai_search.py +255 -255
- webscout/Provider/Aitopia.py +314 -315
- webscout/Provider/Andi.py +3 -3
- webscout/Provider/Apriel.py +306 -0
- webscout/Provider/ChatGPTClone.py +236 -236
- webscout/Provider/ChatSandbox.py +343 -342
- webscout/Provider/Cloudflare.py +324 -324
- webscout/Provider/Cohere.py +208 -207
- webscout/Provider/Deepinfra.py +370 -369
- webscout/Provider/ExaAI.py +260 -260
- webscout/Provider/ExaChat.py +308 -387
- webscout/Provider/Flowith.py +221 -221
- webscout/Provider/GMI.py +293 -0
- webscout/Provider/Gemini.py +164 -162
- webscout/Provider/GeminiProxy.py +167 -166
- webscout/Provider/GithubChat.py +371 -370
- webscout/Provider/Groq.py +800 -800
- webscout/Provider/HeckAI.py +383 -379
- webscout/Provider/Jadve.py +282 -297
- webscout/Provider/K2Think.py +308 -0
- webscout/Provider/Koboldai.py +206 -384
- webscout/Provider/LambdaChat.py +423 -425
- webscout/Provider/Nemotron.py +244 -245
- webscout/Provider/Netwrck.py +248 -247
- webscout/Provider/OLLAMA.py +395 -394
- webscout/Provider/OPENAI/Cloudflare.py +394 -395
- webscout/Provider/OPENAI/FalconH1.py +452 -457
- webscout/Provider/OPENAI/FreeGemini.py +297 -299
- webscout/Provider/OPENAI/{monochat.py → K2Think.py} +432 -329
- webscout/Provider/OPENAI/NEMOTRON.py +241 -244
- webscout/Provider/OPENAI/PI.py +428 -427
- webscout/Provider/OPENAI/README.md +959 -959
- webscout/Provider/OPENAI/TogetherAI.py +345 -345
- webscout/Provider/OPENAI/TwoAI.py +466 -467
- webscout/Provider/OPENAI/__init__.py +33 -59
- webscout/Provider/OPENAI/ai4chat.py +313 -303
- webscout/Provider/OPENAI/base.py +249 -269
- webscout/Provider/OPENAI/chatglm.py +528 -0
- webscout/Provider/OPENAI/chatgpt.py +593 -588
- webscout/Provider/OPENAI/chatgptclone.py +521 -524
- webscout/Provider/OPENAI/chatsandbox.py +202 -177
- webscout/Provider/OPENAI/deepinfra.py +319 -315
- webscout/Provider/OPENAI/e2b.py +1665 -1665
- webscout/Provider/OPENAI/exaai.py +420 -420
- webscout/Provider/OPENAI/exachat.py +452 -452
- webscout/Provider/OPENAI/friendli.py +232 -232
- webscout/Provider/OPENAI/{refact.py → gmi.py} +324 -274
- webscout/Provider/OPENAI/groq.py +364 -364
- webscout/Provider/OPENAI/heckai.py +314 -311
- webscout/Provider/OPENAI/llmchatco.py +337 -337
- webscout/Provider/OPENAI/netwrck.py +355 -354
- webscout/Provider/OPENAI/oivscode.py +290 -290
- webscout/Provider/OPENAI/opkfc.py +518 -518
- webscout/Provider/OPENAI/pydantic_imports.py +1 -1
- webscout/Provider/OPENAI/scirachat.py +535 -529
- webscout/Provider/OPENAI/sonus.py +308 -308
- webscout/Provider/OPENAI/standardinput.py +442 -442
- webscout/Provider/OPENAI/textpollinations.py +340 -348
- webscout/Provider/OPENAI/toolbaz.py +419 -413
- webscout/Provider/OPENAI/typefully.py +362 -362
- webscout/Provider/OPENAI/utils.py +295 -295
- webscout/Provider/OPENAI/venice.py +436 -436
- webscout/Provider/OPENAI/wisecat.py +387 -387
- webscout/Provider/OPENAI/writecream.py +166 -166
- webscout/Provider/OPENAI/x0gpt.py +378 -378
- webscout/Provider/OPENAI/yep.py +389 -389
- webscout/Provider/OpenGPT.py +230 -230
- webscout/Provider/Openai.py +244 -496
- webscout/Provider/PI.py +405 -404
- webscout/Provider/Perplexitylabs.py +430 -431
- webscout/Provider/QwenLM.py +272 -254
- webscout/Provider/STT/__init__.py +32 -2
- webscout/Provider/{Llama3.py → Sambanova.py} +257 -258
- webscout/Provider/StandardInput.py +309 -309
- webscout/Provider/TTI/README.md +82 -82
- webscout/Provider/TTI/__init__.py +33 -12
- webscout/Provider/TTI/aiarta.py +413 -413
- webscout/Provider/TTI/base.py +136 -136
- webscout/Provider/TTI/bing.py +243 -243
- webscout/Provider/TTI/gpt1image.py +149 -149
- webscout/Provider/TTI/imagen.py +196 -196
- webscout/Provider/TTI/infip.py +211 -211
- webscout/Provider/TTI/magicstudio.py +232 -232
- webscout/Provider/TTI/monochat.py +219 -219
- webscout/Provider/TTI/piclumen.py +214 -214
- webscout/Provider/TTI/pixelmuse.py +232 -232
- webscout/Provider/TTI/pollinations.py +232 -232
- webscout/Provider/TTI/together.py +288 -288
- webscout/Provider/TTI/utils.py +12 -12
- webscout/Provider/TTI/venice.py +367 -367
- webscout/Provider/TTS/README.md +192 -192
- webscout/Provider/TTS/__init__.py +33 -10
- webscout/Provider/TTS/parler.py +110 -110
- webscout/Provider/TTS/streamElements.py +333 -333
- webscout/Provider/TTS/utils.py +280 -280
- webscout/Provider/TeachAnything.py +237 -236
- webscout/Provider/TextPollinationsAI.py +311 -318
- webscout/Provider/TogetherAI.py +356 -357
- webscout/Provider/TwoAI.py +313 -569
- webscout/Provider/TypliAI.py +312 -311
- webscout/Provider/UNFINISHED/ChatHub.py +208 -208
- webscout/Provider/UNFINISHED/ChutesAI.py +313 -313
- webscout/Provider/{GizAI.py → UNFINISHED/GizAI.py} +294 -294
- webscout/Provider/{Marcus.py → UNFINISHED/Marcus.py} +198 -198
- webscout/Provider/{Qodo.py → UNFINISHED/Qodo.py} +477 -477
- webscout/Provider/UNFINISHED/VercelAIGateway.py +338 -338
- webscout/Provider/{XenAI.py → UNFINISHED/XenAI.py} +324 -324
- webscout/Provider/UNFINISHED/Youchat.py +330 -330
- webscout/Provider/UNFINISHED/liner.py +334 -0
- webscout/Provider/UNFINISHED/liner_api_request.py +262 -262
- webscout/Provider/UNFINISHED/puterjs.py +634 -634
- webscout/Provider/UNFINISHED/samurai.py +223 -223
- webscout/Provider/UNFINISHED/test_lmarena.py +119 -119
- webscout/Provider/Venice.py +251 -250
- webscout/Provider/VercelAI.py +256 -255
- webscout/Provider/WiseCat.py +232 -231
- webscout/Provider/WrDoChat.py +367 -366
- webscout/Provider/__init__.py +33 -86
- webscout/Provider/ai4chat.py +174 -174
- webscout/Provider/akashgpt.py +331 -334
- webscout/Provider/cerebras.py +446 -340
- webscout/Provider/chatglm.py +394 -214
- webscout/Provider/cleeai.py +211 -212
- webscout/Provider/deepseek_assistant.py +1 -1
- webscout/Provider/elmo.py +282 -282
- webscout/Provider/geminiapi.py +208 -208
- webscout/Provider/granite.py +261 -261
- webscout/Provider/hermes.py +263 -265
- webscout/Provider/julius.py +223 -222
- webscout/Provider/learnfastai.py +309 -309
- webscout/Provider/llama3mitril.py +214 -214
- webscout/Provider/llmchat.py +243 -243
- webscout/Provider/llmchatco.py +290 -290
- webscout/Provider/meta.py +801 -801
- webscout/Provider/oivscode.py +309 -309
- webscout/Provider/scira_chat.py +384 -457
- webscout/Provider/searchchat.py +292 -291
- webscout/Provider/sonus.py +258 -258
- webscout/Provider/toolbaz.py +370 -364
- webscout/Provider/turboseek.py +274 -265
- webscout/Provider/typefully.py +208 -207
- webscout/Provider/x0gpt.py +1 -0
- webscout/Provider/yep.py +372 -371
- webscout/__init__.py +30 -31
- webscout/__main__.py +5 -5
- webscout/auth/api_key_manager.py +189 -189
- webscout/auth/config.py +175 -175
- webscout/auth/models.py +185 -185
- webscout/auth/routes.py +664 -664
- webscout/auth/simple_logger.py +236 -236
- webscout/cli.py +523 -523
- webscout/conversation.py +438 -438
- webscout/exceptions.py +361 -361
- webscout/litagent/Readme.md +298 -298
- webscout/litagent/__init__.py +28 -28
- webscout/litagent/agent.py +581 -581
- webscout/litagent/constants.py +59 -59
- webscout/litprinter/__init__.py +58 -58
- webscout/models.py +181 -181
- webscout/optimizers.py +419 -419
- webscout/prompt_manager.py +288 -288
- webscout/sanitize.py +1078 -1078
- webscout/scout/README.md +401 -401
- webscout/scout/__init__.py +8 -8
- webscout/scout/core/__init__.py +6 -6
- webscout/scout/core/crawler.py +297 -297
- webscout/scout/core/scout.py +706 -706
- webscout/scout/core/search_result.py +95 -95
- webscout/scout/core/text_analyzer.py +62 -62
- webscout/scout/core/text_utils.py +277 -277
- webscout/scout/core/web_analyzer.py +51 -51
- webscout/scout/element.py +599 -599
- webscout/scout/parsers/__init__.py +69 -69
- webscout/scout/parsers/html5lib_parser.py +172 -172
- webscout/scout/parsers/html_parser.py +236 -236
- webscout/scout/parsers/lxml_parser.py +178 -178
- webscout/scout/utils.py +37 -37
- webscout/swiftcli/Readme.md +323 -323
- webscout/swiftcli/__init__.py +95 -95
- webscout/swiftcli/core/__init__.py +7 -7
- webscout/swiftcli/core/cli.py +308 -308
- webscout/swiftcli/core/context.py +104 -104
- webscout/swiftcli/core/group.py +241 -241
- webscout/swiftcli/decorators/__init__.py +28 -28
- webscout/swiftcli/decorators/command.py +221 -221
- webscout/swiftcli/decorators/options.py +220 -220
- webscout/swiftcli/decorators/output.py +302 -302
- webscout/swiftcli/exceptions.py +21 -21
- webscout/swiftcli/plugins/__init__.py +9 -9
- webscout/swiftcli/plugins/base.py +135 -135
- webscout/swiftcli/plugins/manager.py +269 -269
- webscout/swiftcli/utils/__init__.py +59 -59
- webscout/swiftcli/utils/formatting.py +252 -252
- webscout/swiftcli/utils/parsing.py +267 -267
- webscout/update_checker.py +117 -117
- webscout/version.py +1 -1
- webscout/webscout_search.py +1183 -1183
- webscout/webscout_search_async.py +649 -649
- webscout/yep_search.py +346 -346
- webscout/zeroart/README.md +89 -89
- webscout/zeroart/__init__.py +134 -134
- webscout/zeroart/base.py +66 -66
- webscout/zeroart/effects.py +100 -100
- webscout/zeroart/fonts.py +1238 -1238
- {webscout-8.3.6.dist-info → webscout-2025.10.11.dist-info}/METADATA +937 -936
- webscout-2025.10.11.dist-info/RECORD +300 -0
- webscout/Provider/AISEARCH/DeepFind.py +0 -254
- webscout/Provider/AllenAI.py +0 -440
- webscout/Provider/Blackboxai.py +0 -793
- webscout/Provider/FreeGemini.py +0 -250
- webscout/Provider/GptOss.py +0 -207
- webscout/Provider/Hunyuan.py +0 -283
- webscout/Provider/Kimi.py +0 -445
- webscout/Provider/MCPCore.py +0 -322
- webscout/Provider/MiniMax.py +0 -207
- webscout/Provider/OPENAI/BLACKBOXAI.py +0 -1045
- webscout/Provider/OPENAI/MiniMax.py +0 -298
- webscout/Provider/OPENAI/Qwen3.py +0 -304
- webscout/Provider/OPENAI/autoproxy.py +0 -1067
- webscout/Provider/OPENAI/copilot.py +0 -321
- webscout/Provider/OPENAI/gptoss.py +0 -288
- webscout/Provider/OPENAI/kimi.py +0 -469
- webscout/Provider/OPENAI/mcpcore.py +0 -431
- webscout/Provider/OPENAI/multichat.py +0 -378
- webscout/Provider/OPENAI/qodo.py +0 -630
- webscout/Provider/OPENAI/xenai.py +0 -514
- webscout/Provider/Reka.py +0 -214
- webscout/Provider/UNFINISHED/fetch_together_models.py +0 -90
- webscout/Provider/asksteve.py +0 -220
- webscout/Provider/copilot.py +0 -441
- webscout/Provider/freeaichat.py +0 -294
- webscout/Provider/koala.py +0 -182
- webscout/Provider/lmarena.py +0 -198
- webscout/Provider/monochat.py +0 -275
- webscout/Provider/multichat.py +0 -375
- webscout/Provider/scnet.py +0 -244
- webscout/Provider/talkai.py +0 -194
- webscout/tempid.py +0 -128
- webscout-8.3.6.dist-info/RECORD +0 -327
- {webscout-8.3.6.dist-info → webscout-2025.10.11.dist-info}/WHEEL +0 -0
- {webscout-8.3.6.dist-info → webscout-2025.10.11.dist-info}/entry_points.txt +0 -0
- {webscout-8.3.6.dist-info → webscout-2025.10.11.dist-info}/licenses/LICENSE.md +0 -0
- {webscout-8.3.6.dist-info → webscout-2025.10.11.dist-info}/top_level.txt +0 -0
|
@@ -1,476 +1,476 @@
|
|
|
1
|
-
"""
|
|
2
|
-
>>> from webscout import YTTranscriber
|
|
3
|
-
>>> transcript = YTTranscriber.get_transcript('https://www.youtube.com/watch?v=dQw4w9WgXcQ')
|
|
4
|
-
>>> print(transcript)
|
|
5
|
-
{'text': 'Never gonna give you up', 'start': 0.0, 'duration': 4.5}
|
|
6
|
-
|
|
7
|
-
"""
|
|
8
|
-
|
|
9
|
-
import requests
|
|
10
|
-
import http.cookiejar as cookiejar
|
|
11
|
-
import json
|
|
12
|
-
from xml.etree import ElementTree
|
|
13
|
-
import re
|
|
14
|
-
import html
|
|
15
|
-
from typing import List, Dict, Union, Optional
|
|
16
|
-
from functools import lru_cache #
|
|
17
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
18
|
-
from webscout.exceptions import *
|
|
19
|
-
|
|
20
|
-
WATCH_URL = 'https://www.youtube.com/watch?v={video_id}'
|
|
21
|
-
MAX_WORKERS = 4
|
|
22
|
-
|
|
23
|
-
class YTTranscriber:
|
|
24
|
-
"""Transcribe YouTube videos with style! 🎤
|
|
25
|
-
|
|
26
|
-
>>> transcript = YTTranscriber.get_transcript('https://youtu.be/dQw4w9WgXcQ')
|
|
27
|
-
>>> print(transcript[0]['text'])
|
|
28
|
-
'Never gonna give you up'
|
|
29
|
-
"""
|
|
30
|
-
|
|
31
|
-
_session = None
|
|
32
|
-
_executor = ThreadPoolExecutor(max_workers=MAX_WORKERS)
|
|
33
|
-
|
|
34
|
-
@classmethod
|
|
35
|
-
def _get_session(cls):
|
|
36
|
-
if cls._session is None:
|
|
37
|
-
cls._session = requests.Session()
|
|
38
|
-
cls._session.headers.update({
|
|
39
|
-
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
|
40
|
-
})
|
|
41
|
-
return cls._session
|
|
42
|
-
|
|
43
|
-
@classmethod
|
|
44
|
-
@lru_cache(maxsize=100)
|
|
45
|
-
def get_transcript(cls, video_url: str, languages: Optional[str] = 'en',
|
|
46
|
-
proxies: Dict[str, str] = None,
|
|
47
|
-
cookies: str = None,
|
|
48
|
-
preserve_formatting: bool = False) -> List[Dict[str, Union[str, float]]]:
|
|
49
|
-
"""
|
|
50
|
-
Retrieves the transcript for a given YouTube video URL.
|
|
51
|
-
|
|
52
|
-
Args:
|
|
53
|
-
video_url (str): YouTube video URL (supports various formats).
|
|
54
|
-
languages (str, optional): Language code for the transcript.
|
|
55
|
-
If None, fetches the auto-generated transcript.
|
|
56
|
-
Defaults to 'en'.
|
|
57
|
-
proxies (Dict[str, str], optional): Proxies to use for the request. Defaults to None.
|
|
58
|
-
cookies (str, optional): Path to the cookie file. Defaults to None.
|
|
59
|
-
preserve_formatting (bool, optional): Whether to preserve formatting tags. Defaults to False.
|
|
60
|
-
|
|
61
|
-
Returns:
|
|
62
|
-
List[Dict[str, Union[str, float]]]: A list of dictionaries, each containing:
|
|
63
|
-
- 'text': The transcribed text.
|
|
64
|
-
- 'start': The start time of the text segment (in seconds).
|
|
65
|
-
- 'duration': The duration of the text segment (in seconds).
|
|
66
|
-
|
|
67
|
-
Raises:
|
|
68
|
-
TranscriptRetrievalError: If there's an error retrieving the transcript.
|
|
69
|
-
"""
|
|
70
|
-
video_id = cls._extract_video_id(video_url)
|
|
71
|
-
http_client = cls._get_session()
|
|
72
|
-
|
|
73
|
-
if proxies:
|
|
74
|
-
http_client.proxies.update(proxies)
|
|
75
|
-
|
|
76
|
-
if cookies:
|
|
77
|
-
cls._load_cookies(cookies, video_id)
|
|
78
|
-
|
|
79
|
-
transcript_list = TranscriptListFetcher(http_client).fetch(video_id)
|
|
80
|
-
language_codes = [languages] if languages else None
|
|
81
|
-
transcript = transcript_list.find_transcript(language_codes)
|
|
82
|
-
|
|
83
|
-
return transcript.fetch(preserve_formatting)
|
|
84
|
-
|
|
85
|
-
@staticmethod
|
|
86
|
-
def _extract_video_id(video_url: str) -> str:
|
|
87
|
-
"""Extracts the video ID from different YouTube URL formats."""
|
|
88
|
-
patterns = [
|
|
89
|
-
r'(?:v=|\/)([0-9A-Za-z_-]{11}).*',
|
|
90
|
-
r'youtu\.be\/([0-9A-Za-z_-]{11})',
|
|
91
|
-
r'youtube\.com\/embed\/([0-9A-Za-z_-]{11})'
|
|
92
|
-
]
|
|
93
|
-
|
|
94
|
-
for pattern in patterns:
|
|
95
|
-
match = re.search(pattern, video_url)
|
|
96
|
-
if match:
|
|
97
|
-
return match.group(1)
|
|
98
|
-
|
|
99
|
-
if re.match(r'^[0-9A-Za-z_-]{11}$', video_url):
|
|
100
|
-
return video_url
|
|
101
|
-
|
|
102
|
-
raise InvalidVideoIdError(video_url)
|
|
103
|
-
|
|
104
|
-
@staticmethod
|
|
105
|
-
def _load_cookies(cookies: str, video_id: str) -> None:
|
|
106
|
-
"""Loads cookies from a file."""
|
|
107
|
-
try:
|
|
108
|
-
cj = cookiejar.MozillaCookieJar(cookies)
|
|
109
|
-
cj.load()
|
|
110
|
-
return cj
|
|
111
|
-
except (cookiejar.LoadError, FileNotFoundError):
|
|
112
|
-
raise CookiePathInvalidError(video_id)
|
|
113
|
-
|
|
114
|
-
class TranscriptListFetcher:
|
|
115
|
-
"""Fetches the list of transcripts for a YouTube video."""
|
|
116
|
-
|
|
117
|
-
def __init__(self, http_client: requests.Session):
|
|
118
|
-
"""Initializes TranscriptListFetcher."""
|
|
119
|
-
self._http_client = http_client
|
|
120
|
-
|
|
121
|
-
def fetch(self, video_id: str):
|
|
122
|
-
"""Fetches and returns a TranscriptList."""
|
|
123
|
-
return TranscriptList.build(
|
|
124
|
-
self._http_client,
|
|
125
|
-
video_id,
|
|
126
|
-
self._extract_captions_json(self._fetch_video_html(video_id), video_id),
|
|
127
|
-
)
|
|
128
|
-
|
|
129
|
-
def _extract_captions_json(self, html: str, video_id: str) -> dict:
|
|
130
|
-
"""Extracts the captions JSON data from the video's HTML."""
|
|
131
|
-
splitted_html = html.split('"captions":')
|
|
132
|
-
|
|
133
|
-
if len(splitted_html) <= 1:
|
|
134
|
-
if video_id.startswith('http://') or video_id.startswith('https://'):
|
|
135
|
-
raise InvalidVideoIdError(video_id)
|
|
136
|
-
if 'class="g-recaptcha"' in html:
|
|
137
|
-
raise TooManyRequestsError(video_id)
|
|
138
|
-
if '"playabilityStatus":' not in html:
|
|
139
|
-
raise VideoUnavailableError(video_id)
|
|
140
|
-
|
|
141
|
-
raise TranscriptsDisabledError(video_id)
|
|
142
|
-
|
|
143
|
-
captions_json = json.loads(
|
|
144
|
-
splitted_html[1].split(',"videoDetails')[0].replace('\n', '')
|
|
145
|
-
).get('playerCaptionsTracklistRenderer')
|
|
146
|
-
if captions_json is None:
|
|
147
|
-
raise TranscriptsDisabledError(video_id)
|
|
148
|
-
|
|
149
|
-
if 'captionTracks' not in captions_json:
|
|
150
|
-
raise TranscriptsDisabledError(video_id)
|
|
151
|
-
|
|
152
|
-
return captions_json
|
|
153
|
-
|
|
154
|
-
def _create_consent_cookie(self, html, video_id):
|
|
155
|
-
match = re.search('name="v" value="(.*?)"', html)
|
|
156
|
-
if match is None:
|
|
157
|
-
raise FailedToCreateConsentCookieError(video_id)
|
|
158
|
-
self._http_client.cookies.set('CONSENT', 'YES+' + match.group(1), domain='.youtube.com')
|
|
159
|
-
|
|
160
|
-
def _fetch_video_html(self, video_id):
|
|
161
|
-
html = self._fetch_html(video_id)
|
|
162
|
-
if 'action="https://consent.youtube.com/s"' in html:
|
|
163
|
-
self._create_consent_cookie(html, video_id)
|
|
164
|
-
html = self._fetch_html(video_id)
|
|
165
|
-
if 'action="https://consent.youtube.com/s"' in html:
|
|
166
|
-
raise FailedToCreateConsentCookieError(video_id)
|
|
167
|
-
return html
|
|
168
|
-
|
|
169
|
-
def _fetch_html(self, video_id):
|
|
170
|
-
response = self._http_client.get(WATCH_URL.format(video_id=video_id), headers={'Accept-Language': 'en-US'})
|
|
171
|
-
return html.unescape(_raise_http_errors(response, video_id).text)
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
class TranscriptList:
|
|
175
|
-
"""
|
|
176
|
-
>>> transcript_list = TranscriptList.build(http_client, video_id, captions_json)
|
|
177
|
-
>>> transcript = transcript_list.find_transcript(['en'])
|
|
178
|
-
>>> print(transcript)
|
|
179
|
-
en ("English")[TRANSLATABLE]
|
|
180
|
-
"""
|
|
181
|
-
|
|
182
|
-
def __init__(self, video_id, manually_created_transcripts, generated_transcripts, translation_languages):
|
|
183
|
-
"""Init that transcript list with all the good stuff! 💯"""
|
|
184
|
-
self.video_id = video_id
|
|
185
|
-
self._manually_created_transcripts = manually_created_transcripts
|
|
186
|
-
self._generated_transcripts = generated_transcripts
|
|
187
|
-
self._translation_languages = translation_languages
|
|
188
|
-
|
|
189
|
-
@staticmethod
|
|
190
|
-
def build(http_client, video_id, captions_json):
|
|
191
|
-
"""
|
|
192
|
-
Factory method for TranscriptList.
|
|
193
|
-
|
|
194
|
-
:param http_client: http client which is used to make the transcript retrieving http calls
|
|
195
|
-
:type http_client: requests.Session
|
|
196
|
-
:param video_id: the id of the video this TranscriptList is for
|
|
197
|
-
:type video_id: str
|
|
198
|
-
:param captions_json: the JSON parsed from the YouTube pages static HTML
|
|
199
|
-
:type captions_json: dict
|
|
200
|
-
:return: the created TranscriptList
|
|
201
|
-
:rtype TranscriptList:
|
|
202
|
-
"""
|
|
203
|
-
translation_languages = [
|
|
204
|
-
{
|
|
205
|
-
'language': translation_language['languageName']['simpleText'],
|
|
206
|
-
'language_code': translation_language['languageCode'],
|
|
207
|
-
} for translation_language in captions_json.get('translationLanguages', [])
|
|
208
|
-
]
|
|
209
|
-
|
|
210
|
-
manually_created_transcripts = {}
|
|
211
|
-
generated_transcripts = {}
|
|
212
|
-
|
|
213
|
-
for caption in captions_json['captionTracks']:
|
|
214
|
-
if caption.get('kind', '') == 'asr':
|
|
215
|
-
transcript_dict = generated_transcripts
|
|
216
|
-
else:
|
|
217
|
-
transcript_dict = manually_created_transcripts
|
|
218
|
-
|
|
219
|
-
transcript_dict[caption['languageCode']] = Transcript(
|
|
220
|
-
http_client,
|
|
221
|
-
video_id,
|
|
222
|
-
caption['baseUrl'],
|
|
223
|
-
caption['name']['simpleText'],
|
|
224
|
-
caption['languageCode'],
|
|
225
|
-
caption.get('kind', '') == 'asr',
|
|
226
|
-
translation_languages if caption.get('isTranslatable', False) else [],
|
|
227
|
-
)
|
|
228
|
-
|
|
229
|
-
return TranscriptList(
|
|
230
|
-
video_id,
|
|
231
|
-
manually_created_transcripts,
|
|
232
|
-
generated_transcripts,
|
|
233
|
-
translation_languages,
|
|
234
|
-
)
|
|
235
|
-
|
|
236
|
-
def __iter__(self):
|
|
237
|
-
return iter(list(self._manually_created_transcripts.values()) + list(self._generated_transcripts.values()))
|
|
238
|
-
|
|
239
|
-
def find_transcript(self, language_codes):
|
|
240
|
-
"""
|
|
241
|
-
Finds a transcript for a given language code. If no language is provided, it will
|
|
242
|
-
return the auto-generated transcript.
|
|
243
|
-
|
|
244
|
-
:param language_codes: A list of language codes in a descending priority.
|
|
245
|
-
:type languages: list[str]
|
|
246
|
-
:return: the found Transcript
|
|
247
|
-
:rtype Transcript:
|
|
248
|
-
:raises: NoTranscriptFound
|
|
249
|
-
"""
|
|
250
|
-
if 'any' in language_codes:
|
|
251
|
-
for transcript in self:
|
|
252
|
-
return transcript
|
|
253
|
-
return self._find_transcript(language_codes, [self._manually_created_transcripts, self._generated_transcripts])
|
|
254
|
-
|
|
255
|
-
def find_generated_transcript(self, language_codes):
|
|
256
|
-
"""
|
|
257
|
-
Finds an automatically generated transcript for a given language code.
|
|
258
|
-
|
|
259
|
-
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
|
|
260
|
-
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
|
|
261
|
-
it fails to do so.
|
|
262
|
-
:type languages: list[str]
|
|
263
|
-
:return: the found Transcript
|
|
264
|
-
:rtype Transcript:
|
|
265
|
-
:raises: NoTranscriptFound
|
|
266
|
-
"""
|
|
267
|
-
if 'any' in language_codes:
|
|
268
|
-
for transcript in self:
|
|
269
|
-
if transcript.is_generated:
|
|
270
|
-
return transcript
|
|
271
|
-
return self._find_transcript(language_codes, [self._generated_transcripts])
|
|
272
|
-
|
|
273
|
-
def find_manually_created_transcript(self, language_codes):
|
|
274
|
-
"""
|
|
275
|
-
Finds a manually created transcript for a given language code.
|
|
276
|
-
|
|
277
|
-
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
|
|
278
|
-
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
|
|
279
|
-
it fails to do so.
|
|
280
|
-
:type languages: list[str]
|
|
281
|
-
:return: the found Transcript
|
|
282
|
-
:rtype Transcript:
|
|
283
|
-
:raises: NoTranscriptFound
|
|
284
|
-
"""
|
|
285
|
-
return self._find_transcript(language_codes, [self._manually_created_transcripts])
|
|
286
|
-
|
|
287
|
-
def _find_transcript(self, language_codes, transcript_dicts):
|
|
288
|
-
for language_code in language_codes:
|
|
289
|
-
for transcript_dict in transcript_dicts:
|
|
290
|
-
if language_code in transcript_dict:
|
|
291
|
-
return transcript_dict[language_code]
|
|
292
|
-
|
|
293
|
-
raise NoTranscriptFoundError(
|
|
294
|
-
self.video_id,
|
|
295
|
-
language_codes,
|
|
296
|
-
self
|
|
297
|
-
)
|
|
298
|
-
|
|
299
|
-
def __str__(self):
|
|
300
|
-
return (
|
|
301
|
-
'For this video ({video_id}) transcripts are available in the following languages:\n\n'
|
|
302
|
-
'(MANUALLY CREATED)\n'
|
|
303
|
-
'{available_manually_created_transcript_languages}\n\n'
|
|
304
|
-
'(GENERATED)\n'
|
|
305
|
-
'{available_generated_transcripts}\n\n'
|
|
306
|
-
'(TRANSLATION LANGUAGES)\n'
|
|
307
|
-
'{available_translation_languages}'
|
|
308
|
-
).format(
|
|
309
|
-
video_id=self.video_id,
|
|
310
|
-
available_manually_created_transcript_languages=self._get_language_description(
|
|
311
|
-
str(transcript) for transcript in self._manually_created_transcripts.values()
|
|
312
|
-
),
|
|
313
|
-
available_generated_transcripts=self._get_language_description(
|
|
314
|
-
str(transcript) for transcript in self._generated_transcripts.values()
|
|
315
|
-
),
|
|
316
|
-
available_translation_languages=self._get_language_description(
|
|
317
|
-
'{language_code} ("{language}")'.format(
|
|
318
|
-
language=translation_language['language'],
|
|
319
|
-
language_code=translation_language['language_code'],
|
|
320
|
-
) for translation_language in self._translation_languages
|
|
321
|
-
)
|
|
322
|
-
)
|
|
323
|
-
|
|
324
|
-
def _get_language_description(self, transcript_strings):
|
|
325
|
-
description = '\n'.join(' - {transcript}'.format(transcript=transcript) for transcript in transcript_strings)
|
|
326
|
-
return description if description else 'None'
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
class Transcript:
|
|
330
|
-
"""Your personal transcript handler! 🎭
|
|
331
|
-
|
|
332
|
-
>>> transcript = transcript_list.find_transcript(['en'])
|
|
333
|
-
>>> print(transcript.language)
|
|
334
|
-
'English'
|
|
335
|
-
>>> if transcript.is_translatable:
|
|
336
|
-
... es_transcript = transcript.translate('es')
|
|
337
|
-
... print(es_transcript.language)
|
|
338
|
-
'Spanish'
|
|
339
|
-
"""
|
|
340
|
-
|
|
341
|
-
def __init__(self, http_client, video_id, url, language, language_code, is_generated, translation_languages):
|
|
342
|
-
"""Initialize with all the goodies! 🎁"""
|
|
343
|
-
self._http_client = http_client
|
|
344
|
-
self.video_id = video_id
|
|
345
|
-
self._url = url
|
|
346
|
-
self.language = language
|
|
347
|
-
self.language_code = language_code
|
|
348
|
-
self.is_generated = is_generated
|
|
349
|
-
self.translation_languages = translation_languages
|
|
350
|
-
self._translation_languages_dict = {
|
|
351
|
-
translation_language['language_code']: translation_language['language']
|
|
352
|
-
for translation_language in translation_languages
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
def fetch(self, preserve_formatting=False):
|
|
356
|
-
"""Get that transcript data! 🎯
|
|
357
|
-
|
|
358
|
-
Args:
|
|
359
|
-
preserve_formatting (bool): Keep HTML formatting? Default is nah fam.
|
|
360
|
-
|
|
361
|
-
Returns:
|
|
362
|
-
list: That sweet transcript data with text, start time, and duration! 📝
|
|
363
|
-
"""
|
|
364
|
-
response = self._http_client.get(self._url, headers={'Accept-Language': 'en-US'})
|
|
365
|
-
return TranscriptParser(preserve_formatting=preserve_formatting).parse(
|
|
366
|
-
_raise_http_errors(response, self.video_id).text,
|
|
367
|
-
)
|
|
368
|
-
|
|
369
|
-
def __str__(self):
|
|
370
|
-
"""String representation looking clean! 💅"""
|
|
371
|
-
return '{language_code} ("{language}"){translation_description}'.format(
|
|
372
|
-
language=self.language,
|
|
373
|
-
language_code=self.language_code,
|
|
374
|
-
translation_description='[TRANSLATABLE]' if self.is_translatable else ''
|
|
375
|
-
)
|
|
376
|
-
|
|
377
|
-
@property
|
|
378
|
-
def is_translatable(self):
|
|
379
|
-
"""Can we translate this? 🌍"""
|
|
380
|
-
return len(self.translation_languages) > 0
|
|
381
|
-
|
|
382
|
-
def translate(self, language_code):
|
|
383
|
-
"""Translate to another language! 🌎
|
|
384
|
-
|
|
385
|
-
Args:
|
|
386
|
-
language_code (str): Which language you want fam?
|
|
387
|
-
|
|
388
|
-
Returns:
|
|
389
|
-
Transcript: A fresh transcript in your requested language! 🔄
|
|
390
|
-
|
|
391
|
-
Raises:
|
|
392
|
-
NotTranslatableError: If we can't translate this one 😢
|
|
393
|
-
TranslationLanguageNotAvailableError: If that language isn't available 🚫
|
|
394
|
-
"""
|
|
395
|
-
if not self.is_translatable:
|
|
396
|
-
raise NotTranslatableError(self.video_id)
|
|
397
|
-
|
|
398
|
-
if language_code not in self._translation_languages_dict:
|
|
399
|
-
raise TranslationLanguageNotAvailableError(self.video_id)
|
|
400
|
-
|
|
401
|
-
return Transcript(
|
|
402
|
-
self._http_client,
|
|
403
|
-
self.video_id,
|
|
404
|
-
'{url}&tlang={language_code}'.format(url=self._url, language_code=language_code),
|
|
405
|
-
self._translation_languages_dict[language_code],
|
|
406
|
-
language_code,
|
|
407
|
-
True,
|
|
408
|
-
[],
|
|
409
|
-
)
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
class TranscriptParser:
|
|
413
|
-
"""Parsing those transcripts like a pro! 🎯
|
|
414
|
-
|
|
415
|
-
>>> parser = TranscriptParser(preserve_formatting=True)
|
|
416
|
-
>>> data = parser.parse(xml_data)
|
|
417
|
-
>>> print(data[0])
|
|
418
|
-
{'text': 'Never gonna give you up', 'start': 0.0, 'duration': 4.5}
|
|
419
|
-
"""
|
|
420
|
-
|
|
421
|
-
_FORMATTING_TAGS = [
|
|
422
|
-
'strong', # For that extra emphasis 💪
|
|
423
|
-
'em', # When you need that italic swag 🎨
|
|
424
|
-
'b', # Bold and beautiful 💯
|
|
425
|
-
'i', # More italic vibes ✨
|
|
426
|
-
'mark', # Highlight that text 🌟
|
|
427
|
-
'small', # Keep it lowkey 🤫
|
|
428
|
-
'del', # Strike it out ⚡
|
|
429
|
-
'ins', # Insert new stuff 🆕
|
|
430
|
-
'sub', # Subscript gang 📉
|
|
431
|
-
'sup', # Superscript squad 📈
|
|
432
|
-
]
|
|
433
|
-
|
|
434
|
-
def __init__(self, preserve_formatting=False):
|
|
435
|
-
"""Get ready to parse with style! 🎨"""
|
|
436
|
-
self._html_regex = self._get_html_regex(preserve_formatting)
|
|
437
|
-
|
|
438
|
-
def _get_html_regex(self, preserve_formatting):
|
|
439
|
-
"""Get that regex pattern ready! 🎯"""
|
|
440
|
-
if preserve_formatting:
|
|
441
|
-
formats_regex = '|'.join(self._FORMATTING_TAGS)
|
|
442
|
-
formats_regex = r'<\/?(?!\/?(' + formats_regex + r')\b).*?\b>'
|
|
443
|
-
html_regex = re.compile(formats_regex, re.IGNORECASE)
|
|
444
|
-
else:
|
|
445
|
-
html_regex = re.compile(r'<[^>]*>', re.IGNORECASE)
|
|
446
|
-
return html_regex
|
|
447
|
-
|
|
448
|
-
def parse(self, plain_data):
|
|
449
|
-
"""Parse that XML data into something beautiful! ✨"""
|
|
450
|
-
return [
|
|
451
|
-
{
|
|
452
|
-
'text': re.sub(self._html_regex, '', html.unescape(xml_element.text)),
|
|
453
|
-
'start': float(xml_element.attrib['start']),
|
|
454
|
-
'duration': float(xml_element.attrib.get('dur', '0.0')),
|
|
455
|
-
}
|
|
456
|
-
for xml_element in ElementTree.fromstring(plain_data)
|
|
457
|
-
if xml_element.text is not None
|
|
458
|
-
]
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
def _raise_http_errors(response, video_id):
|
|
462
|
-
"""Handle those HTTP errors with style! 🛠️"""
|
|
463
|
-
try:
|
|
464
|
-
response.raise_for_status()
|
|
465
|
-
return response
|
|
466
|
-
except requests.exceptions.HTTPError as error:
|
|
467
|
-
raise YouTubeRequestFailedError(video_id, error)
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
if __name__ == "__main__":
|
|
471
|
-
# Let's get this party started! 🎉
|
|
472
|
-
from rich import print
|
|
473
|
-
video_url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
|
|
474
|
-
transcript = YTTranscriber.get_transcript(video_url, languages=None)
|
|
475
|
-
print("Here's what we got! 🔥")
|
|
1
|
+
"""
|
|
2
|
+
>>> from webscout import YTTranscriber
|
|
3
|
+
>>> transcript = YTTranscriber.get_transcript('https://www.youtube.com/watch?v=dQw4w9WgXcQ')
|
|
4
|
+
>>> print(transcript)
|
|
5
|
+
{'text': 'Never gonna give you up', 'start': 0.0, 'duration': 4.5}
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import requests
|
|
10
|
+
import http.cookiejar as cookiejar
|
|
11
|
+
import json
|
|
12
|
+
from xml.etree import ElementTree
|
|
13
|
+
import re
|
|
14
|
+
import html
|
|
15
|
+
from typing import List, Dict, Union, Optional
|
|
16
|
+
from functools import lru_cache #
|
|
17
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
18
|
+
from webscout.exceptions import *
|
|
19
|
+
|
|
20
|
+
WATCH_URL = 'https://www.youtube.com/watch?v={video_id}'
|
|
21
|
+
MAX_WORKERS = 4
|
|
22
|
+
|
|
23
|
+
class YTTranscriber:
|
|
24
|
+
"""Transcribe YouTube videos with style! 🎤
|
|
25
|
+
|
|
26
|
+
>>> transcript = YTTranscriber.get_transcript('https://youtu.be/dQw4w9WgXcQ')
|
|
27
|
+
>>> print(transcript[0]['text'])
|
|
28
|
+
'Never gonna give you up'
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
_session = None
|
|
32
|
+
_executor = ThreadPoolExecutor(max_workers=MAX_WORKERS)
|
|
33
|
+
|
|
34
|
+
@classmethod
|
|
35
|
+
def _get_session(cls):
|
|
36
|
+
if cls._session is None:
|
|
37
|
+
cls._session = requests.Session()
|
|
38
|
+
cls._session.headers.update({
|
|
39
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
|
40
|
+
})
|
|
41
|
+
return cls._session
|
|
42
|
+
|
|
43
|
+
@classmethod
|
|
44
|
+
@lru_cache(maxsize=100)
|
|
45
|
+
def get_transcript(cls, video_url: str, languages: Optional[str] = 'en',
|
|
46
|
+
proxies: Dict[str, str] = None,
|
|
47
|
+
cookies: str = None,
|
|
48
|
+
preserve_formatting: bool = False) -> List[Dict[str, Union[str, float]]]:
|
|
49
|
+
"""
|
|
50
|
+
Retrieves the transcript for a given YouTube video URL.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
video_url (str): YouTube video URL (supports various formats).
|
|
54
|
+
languages (str, optional): Language code for the transcript.
|
|
55
|
+
If None, fetches the auto-generated transcript.
|
|
56
|
+
Defaults to 'en'.
|
|
57
|
+
proxies (Dict[str, str], optional): Proxies to use for the request. Defaults to None.
|
|
58
|
+
cookies (str, optional): Path to the cookie file. Defaults to None.
|
|
59
|
+
preserve_formatting (bool, optional): Whether to preserve formatting tags. Defaults to False.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
List[Dict[str, Union[str, float]]]: A list of dictionaries, each containing:
|
|
63
|
+
- 'text': The transcribed text.
|
|
64
|
+
- 'start': The start time of the text segment (in seconds).
|
|
65
|
+
- 'duration': The duration of the text segment (in seconds).
|
|
66
|
+
|
|
67
|
+
Raises:
|
|
68
|
+
TranscriptRetrievalError: If there's an error retrieving the transcript.
|
|
69
|
+
"""
|
|
70
|
+
video_id = cls._extract_video_id(video_url)
|
|
71
|
+
http_client = cls._get_session()
|
|
72
|
+
|
|
73
|
+
if proxies:
|
|
74
|
+
http_client.proxies.update(proxies)
|
|
75
|
+
|
|
76
|
+
if cookies:
|
|
77
|
+
cls._load_cookies(cookies, video_id)
|
|
78
|
+
|
|
79
|
+
transcript_list = TranscriptListFetcher(http_client).fetch(video_id)
|
|
80
|
+
language_codes = [languages] if languages else None
|
|
81
|
+
transcript = transcript_list.find_transcript(language_codes)
|
|
82
|
+
|
|
83
|
+
return transcript.fetch(preserve_formatting)
|
|
84
|
+
|
|
85
|
+
@staticmethod
|
|
86
|
+
def _extract_video_id(video_url: str) -> str:
|
|
87
|
+
"""Extracts the video ID from different YouTube URL formats."""
|
|
88
|
+
patterns = [
|
|
89
|
+
r'(?:v=|\/)([0-9A-Za-z_-]{11}).*',
|
|
90
|
+
r'youtu\.be\/([0-9A-Za-z_-]{11})',
|
|
91
|
+
r'youtube\.com\/embed\/([0-9A-Za-z_-]{11})'
|
|
92
|
+
]
|
|
93
|
+
|
|
94
|
+
for pattern in patterns:
|
|
95
|
+
match = re.search(pattern, video_url)
|
|
96
|
+
if match:
|
|
97
|
+
return match.group(1)
|
|
98
|
+
|
|
99
|
+
if re.match(r'^[0-9A-Za-z_-]{11}$', video_url):
|
|
100
|
+
return video_url
|
|
101
|
+
|
|
102
|
+
raise InvalidVideoIdError(video_url)
|
|
103
|
+
|
|
104
|
+
@staticmethod
|
|
105
|
+
def _load_cookies(cookies: str, video_id: str) -> None:
|
|
106
|
+
"""Loads cookies from a file."""
|
|
107
|
+
try:
|
|
108
|
+
cj = cookiejar.MozillaCookieJar(cookies)
|
|
109
|
+
cj.load()
|
|
110
|
+
return cj
|
|
111
|
+
except (cookiejar.LoadError, FileNotFoundError):
|
|
112
|
+
raise CookiePathInvalidError(video_id)
|
|
113
|
+
|
|
114
|
+
class TranscriptListFetcher:
|
|
115
|
+
"""Fetches the list of transcripts for a YouTube video."""
|
|
116
|
+
|
|
117
|
+
def __init__(self, http_client: requests.Session):
|
|
118
|
+
"""Initializes TranscriptListFetcher."""
|
|
119
|
+
self._http_client = http_client
|
|
120
|
+
|
|
121
|
+
def fetch(self, video_id: str):
|
|
122
|
+
"""Fetches and returns a TranscriptList."""
|
|
123
|
+
return TranscriptList.build(
|
|
124
|
+
self._http_client,
|
|
125
|
+
video_id,
|
|
126
|
+
self._extract_captions_json(self._fetch_video_html(video_id), video_id),
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
def _extract_captions_json(self, html: str, video_id: str) -> dict:
|
|
130
|
+
"""Extracts the captions JSON data from the video's HTML."""
|
|
131
|
+
splitted_html = html.split('"captions":')
|
|
132
|
+
|
|
133
|
+
if len(splitted_html) <= 1:
|
|
134
|
+
if video_id.startswith('http://') or video_id.startswith('https://'):
|
|
135
|
+
raise InvalidVideoIdError(video_id)
|
|
136
|
+
if 'class="g-recaptcha"' in html:
|
|
137
|
+
raise TooManyRequestsError(video_id)
|
|
138
|
+
if '"playabilityStatus":' not in html:
|
|
139
|
+
raise VideoUnavailableError(video_id)
|
|
140
|
+
|
|
141
|
+
raise TranscriptsDisabledError(video_id)
|
|
142
|
+
|
|
143
|
+
captions_json = json.loads(
|
|
144
|
+
splitted_html[1].split(',"videoDetails')[0].replace('\n', '')
|
|
145
|
+
).get('playerCaptionsTracklistRenderer')
|
|
146
|
+
if captions_json is None:
|
|
147
|
+
raise TranscriptsDisabledError(video_id)
|
|
148
|
+
|
|
149
|
+
if 'captionTracks' not in captions_json:
|
|
150
|
+
raise TranscriptsDisabledError(video_id)
|
|
151
|
+
|
|
152
|
+
return captions_json
|
|
153
|
+
|
|
154
|
+
def _create_consent_cookie(self, html, video_id):
|
|
155
|
+
match = re.search('name="v" value="(.*?)"', html)
|
|
156
|
+
if match is None:
|
|
157
|
+
raise FailedToCreateConsentCookieError(video_id)
|
|
158
|
+
self._http_client.cookies.set('CONSENT', 'YES+' + match.group(1), domain='.youtube.com')
|
|
159
|
+
|
|
160
|
+
def _fetch_video_html(self, video_id):
|
|
161
|
+
html = self._fetch_html(video_id)
|
|
162
|
+
if 'action="https://consent.youtube.com/s"' in html:
|
|
163
|
+
self._create_consent_cookie(html, video_id)
|
|
164
|
+
html = self._fetch_html(video_id)
|
|
165
|
+
if 'action="https://consent.youtube.com/s"' in html:
|
|
166
|
+
raise FailedToCreateConsentCookieError(video_id)
|
|
167
|
+
return html
|
|
168
|
+
|
|
169
|
+
def _fetch_html(self, video_id):
|
|
170
|
+
response = self._http_client.get(WATCH_URL.format(video_id=video_id), headers={'Accept-Language': 'en-US'})
|
|
171
|
+
return html.unescape(_raise_http_errors(response, video_id).text)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
class TranscriptList:
|
|
175
|
+
"""
|
|
176
|
+
>>> transcript_list = TranscriptList.build(http_client, video_id, captions_json)
|
|
177
|
+
>>> transcript = transcript_list.find_transcript(['en'])
|
|
178
|
+
>>> print(transcript)
|
|
179
|
+
en ("English")[TRANSLATABLE]
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
def __init__(self, video_id, manually_created_transcripts, generated_transcripts, translation_languages):
|
|
183
|
+
"""Init that transcript list with all the good stuff! 💯"""
|
|
184
|
+
self.video_id = video_id
|
|
185
|
+
self._manually_created_transcripts = manually_created_transcripts
|
|
186
|
+
self._generated_transcripts = generated_transcripts
|
|
187
|
+
self._translation_languages = translation_languages
|
|
188
|
+
|
|
189
|
+
@staticmethod
|
|
190
|
+
def build(http_client, video_id, captions_json):
|
|
191
|
+
"""
|
|
192
|
+
Factory method for TranscriptList.
|
|
193
|
+
|
|
194
|
+
:param http_client: http client which is used to make the transcript retrieving http calls
|
|
195
|
+
:type http_client: requests.Session
|
|
196
|
+
:param video_id: the id of the video this TranscriptList is for
|
|
197
|
+
:type video_id: str
|
|
198
|
+
:param captions_json: the JSON parsed from the YouTube pages static HTML
|
|
199
|
+
:type captions_json: dict
|
|
200
|
+
:return: the created TranscriptList
|
|
201
|
+
:rtype TranscriptList:
|
|
202
|
+
"""
|
|
203
|
+
translation_languages = [
|
|
204
|
+
{
|
|
205
|
+
'language': translation_language['languageName']['simpleText'],
|
|
206
|
+
'language_code': translation_language['languageCode'],
|
|
207
|
+
} for translation_language in captions_json.get('translationLanguages', [])
|
|
208
|
+
]
|
|
209
|
+
|
|
210
|
+
manually_created_transcripts = {}
|
|
211
|
+
generated_transcripts = {}
|
|
212
|
+
|
|
213
|
+
for caption in captions_json['captionTracks']:
|
|
214
|
+
if caption.get('kind', '') == 'asr':
|
|
215
|
+
transcript_dict = generated_transcripts
|
|
216
|
+
else:
|
|
217
|
+
transcript_dict = manually_created_transcripts
|
|
218
|
+
|
|
219
|
+
transcript_dict[caption['languageCode']] = Transcript(
|
|
220
|
+
http_client,
|
|
221
|
+
video_id,
|
|
222
|
+
caption['baseUrl'],
|
|
223
|
+
caption['name']['simpleText'],
|
|
224
|
+
caption['languageCode'],
|
|
225
|
+
caption.get('kind', '') == 'asr',
|
|
226
|
+
translation_languages if caption.get('isTranslatable', False) else [],
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
return TranscriptList(
|
|
230
|
+
video_id,
|
|
231
|
+
manually_created_transcripts,
|
|
232
|
+
generated_transcripts,
|
|
233
|
+
translation_languages,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
def __iter__(self):
|
|
237
|
+
return iter(list(self._manually_created_transcripts.values()) + list(self._generated_transcripts.values()))
|
|
238
|
+
|
|
239
|
+
def find_transcript(self, language_codes):
|
|
240
|
+
"""
|
|
241
|
+
Finds a transcript for a given language code. If no language is provided, it will
|
|
242
|
+
return the auto-generated transcript.
|
|
243
|
+
|
|
244
|
+
:param language_codes: A list of language codes in a descending priority.
|
|
245
|
+
:type languages: list[str]
|
|
246
|
+
:return: the found Transcript
|
|
247
|
+
:rtype Transcript:
|
|
248
|
+
:raises: NoTranscriptFound
|
|
249
|
+
"""
|
|
250
|
+
if 'any' in language_codes:
|
|
251
|
+
for transcript in self:
|
|
252
|
+
return transcript
|
|
253
|
+
return self._find_transcript(language_codes, [self._manually_created_transcripts, self._generated_transcripts])
|
|
254
|
+
|
|
255
|
+
def find_generated_transcript(self, language_codes):
|
|
256
|
+
"""
|
|
257
|
+
Finds an automatically generated transcript for a given language code.
|
|
258
|
+
|
|
259
|
+
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
|
|
260
|
+
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
|
|
261
|
+
it fails to do so.
|
|
262
|
+
:type languages: list[str]
|
|
263
|
+
:return: the found Transcript
|
|
264
|
+
:rtype Transcript:
|
|
265
|
+
:raises: NoTranscriptFound
|
|
266
|
+
"""
|
|
267
|
+
if 'any' in language_codes:
|
|
268
|
+
for transcript in self:
|
|
269
|
+
if transcript.is_generated:
|
|
270
|
+
return transcript
|
|
271
|
+
return self._find_transcript(language_codes, [self._generated_transcripts])
|
|
272
|
+
|
|
273
|
+
def find_manually_created_transcript(self, language_codes):
|
|
274
|
+
"""
|
|
275
|
+
Finds a manually created transcript for a given language code.
|
|
276
|
+
|
|
277
|
+
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
|
|
278
|
+
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
|
|
279
|
+
it fails to do so.
|
|
280
|
+
:type languages: list[str]
|
|
281
|
+
:return: the found Transcript
|
|
282
|
+
:rtype Transcript:
|
|
283
|
+
:raises: NoTranscriptFound
|
|
284
|
+
"""
|
|
285
|
+
return self._find_transcript(language_codes, [self._manually_created_transcripts])
|
|
286
|
+
|
|
287
|
+
def _find_transcript(self, language_codes, transcript_dicts):
|
|
288
|
+
for language_code in language_codes:
|
|
289
|
+
for transcript_dict in transcript_dicts:
|
|
290
|
+
if language_code in transcript_dict:
|
|
291
|
+
return transcript_dict[language_code]
|
|
292
|
+
|
|
293
|
+
raise NoTranscriptFoundError(
|
|
294
|
+
self.video_id,
|
|
295
|
+
language_codes,
|
|
296
|
+
self
|
|
297
|
+
)
|
|
298
|
+
|
|
299
|
+
def __str__(self):
|
|
300
|
+
return (
|
|
301
|
+
'For this video ({video_id}) transcripts are available in the following languages:\n\n'
|
|
302
|
+
'(MANUALLY CREATED)\n'
|
|
303
|
+
'{available_manually_created_transcript_languages}\n\n'
|
|
304
|
+
'(GENERATED)\n'
|
|
305
|
+
'{available_generated_transcripts}\n\n'
|
|
306
|
+
'(TRANSLATION LANGUAGES)\n'
|
|
307
|
+
'{available_translation_languages}'
|
|
308
|
+
).format(
|
|
309
|
+
video_id=self.video_id,
|
|
310
|
+
available_manually_created_transcript_languages=self._get_language_description(
|
|
311
|
+
str(transcript) for transcript in self._manually_created_transcripts.values()
|
|
312
|
+
),
|
|
313
|
+
available_generated_transcripts=self._get_language_description(
|
|
314
|
+
str(transcript) for transcript in self._generated_transcripts.values()
|
|
315
|
+
),
|
|
316
|
+
available_translation_languages=self._get_language_description(
|
|
317
|
+
'{language_code} ("{language}")'.format(
|
|
318
|
+
language=translation_language['language'],
|
|
319
|
+
language_code=translation_language['language_code'],
|
|
320
|
+
) for translation_language in self._translation_languages
|
|
321
|
+
)
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
def _get_language_description(self, transcript_strings):
|
|
325
|
+
description = '\n'.join(' - {transcript}'.format(transcript=transcript) for transcript in transcript_strings)
|
|
326
|
+
return description if description else 'None'
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
class Transcript:
|
|
330
|
+
"""Your personal transcript handler! 🎭
|
|
331
|
+
|
|
332
|
+
>>> transcript = transcript_list.find_transcript(['en'])
|
|
333
|
+
>>> print(transcript.language)
|
|
334
|
+
'English'
|
|
335
|
+
>>> if transcript.is_translatable:
|
|
336
|
+
... es_transcript = transcript.translate('es')
|
|
337
|
+
... print(es_transcript.language)
|
|
338
|
+
'Spanish'
|
|
339
|
+
"""
|
|
340
|
+
|
|
341
|
+
def __init__(self, http_client, video_id, url, language, language_code, is_generated, translation_languages):
|
|
342
|
+
"""Initialize with all the goodies! 🎁"""
|
|
343
|
+
self._http_client = http_client
|
|
344
|
+
self.video_id = video_id
|
|
345
|
+
self._url = url
|
|
346
|
+
self.language = language
|
|
347
|
+
self.language_code = language_code
|
|
348
|
+
self.is_generated = is_generated
|
|
349
|
+
self.translation_languages = translation_languages
|
|
350
|
+
self._translation_languages_dict = {
|
|
351
|
+
translation_language['language_code']: translation_language['language']
|
|
352
|
+
for translation_language in translation_languages
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
def fetch(self, preserve_formatting=False):
|
|
356
|
+
"""Get that transcript data! 🎯
|
|
357
|
+
|
|
358
|
+
Args:
|
|
359
|
+
preserve_formatting (bool): Keep HTML formatting? Default is nah fam.
|
|
360
|
+
|
|
361
|
+
Returns:
|
|
362
|
+
list: That sweet transcript data with text, start time, and duration! 📝
|
|
363
|
+
"""
|
|
364
|
+
response = self._http_client.get(self._url, headers={'Accept-Language': 'en-US'})
|
|
365
|
+
return TranscriptParser(preserve_formatting=preserve_formatting).parse(
|
|
366
|
+
_raise_http_errors(response, self.video_id).text,
|
|
367
|
+
)
|
|
368
|
+
|
|
369
|
+
def __str__(self):
|
|
370
|
+
"""String representation looking clean! 💅"""
|
|
371
|
+
return '{language_code} ("{language}"){translation_description}'.format(
|
|
372
|
+
language=self.language,
|
|
373
|
+
language_code=self.language_code,
|
|
374
|
+
translation_description='[TRANSLATABLE]' if self.is_translatable else ''
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
@property
|
|
378
|
+
def is_translatable(self):
|
|
379
|
+
"""Can we translate this? 🌍"""
|
|
380
|
+
return len(self.translation_languages) > 0
|
|
381
|
+
|
|
382
|
+
def translate(self, language_code):
|
|
383
|
+
"""Translate to another language! 🌎
|
|
384
|
+
|
|
385
|
+
Args:
|
|
386
|
+
language_code (str): Which language you want fam?
|
|
387
|
+
|
|
388
|
+
Returns:
|
|
389
|
+
Transcript: A fresh transcript in your requested language! 🔄
|
|
390
|
+
|
|
391
|
+
Raises:
|
|
392
|
+
NotTranslatableError: If we can't translate this one 😢
|
|
393
|
+
TranslationLanguageNotAvailableError: If that language isn't available 🚫
|
|
394
|
+
"""
|
|
395
|
+
if not self.is_translatable:
|
|
396
|
+
raise NotTranslatableError(self.video_id)
|
|
397
|
+
|
|
398
|
+
if language_code not in self._translation_languages_dict:
|
|
399
|
+
raise TranslationLanguageNotAvailableError(self.video_id)
|
|
400
|
+
|
|
401
|
+
return Transcript(
|
|
402
|
+
self._http_client,
|
|
403
|
+
self.video_id,
|
|
404
|
+
'{url}&tlang={language_code}'.format(url=self._url, language_code=language_code),
|
|
405
|
+
self._translation_languages_dict[language_code],
|
|
406
|
+
language_code,
|
|
407
|
+
True,
|
|
408
|
+
[],
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
class TranscriptParser:
|
|
413
|
+
"""Parsing those transcripts like a pro! 🎯
|
|
414
|
+
|
|
415
|
+
>>> parser = TranscriptParser(preserve_formatting=True)
|
|
416
|
+
>>> data = parser.parse(xml_data)
|
|
417
|
+
>>> print(data[0])
|
|
418
|
+
{'text': 'Never gonna give you up', 'start': 0.0, 'duration': 4.5}
|
|
419
|
+
"""
|
|
420
|
+
|
|
421
|
+
_FORMATTING_TAGS = [
|
|
422
|
+
'strong', # For that extra emphasis 💪
|
|
423
|
+
'em', # When you need that italic swag 🎨
|
|
424
|
+
'b', # Bold and beautiful 💯
|
|
425
|
+
'i', # More italic vibes ✨
|
|
426
|
+
'mark', # Highlight that text 🌟
|
|
427
|
+
'small', # Keep it lowkey 🤫
|
|
428
|
+
'del', # Strike it out ⚡
|
|
429
|
+
'ins', # Insert new stuff 🆕
|
|
430
|
+
'sub', # Subscript gang 📉
|
|
431
|
+
'sup', # Superscript squad 📈
|
|
432
|
+
]
|
|
433
|
+
|
|
434
|
+
def __init__(self, preserve_formatting=False):
|
|
435
|
+
"""Get ready to parse with style! 🎨"""
|
|
436
|
+
self._html_regex = self._get_html_regex(preserve_formatting)
|
|
437
|
+
|
|
438
|
+
def _get_html_regex(self, preserve_formatting):
|
|
439
|
+
"""Get that regex pattern ready! 🎯"""
|
|
440
|
+
if preserve_formatting:
|
|
441
|
+
formats_regex = '|'.join(self._FORMATTING_TAGS)
|
|
442
|
+
formats_regex = r'<\/?(?!\/?(' + formats_regex + r')\b).*?\b>'
|
|
443
|
+
html_regex = re.compile(formats_regex, re.IGNORECASE)
|
|
444
|
+
else:
|
|
445
|
+
html_regex = re.compile(r'<[^>]*>', re.IGNORECASE)
|
|
446
|
+
return html_regex
|
|
447
|
+
|
|
448
|
+
def parse(self, plain_data):
|
|
449
|
+
"""Parse that XML data into something beautiful! ✨"""
|
|
450
|
+
return [
|
|
451
|
+
{
|
|
452
|
+
'text': re.sub(self._html_regex, '', html.unescape(xml_element.text)),
|
|
453
|
+
'start': float(xml_element.attrib['start']),
|
|
454
|
+
'duration': float(xml_element.attrib.get('dur', '0.0')),
|
|
455
|
+
}
|
|
456
|
+
for xml_element in ElementTree.fromstring(plain_data)
|
|
457
|
+
if xml_element.text is not None
|
|
458
|
+
]
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _raise_http_errors(response, video_id):
|
|
462
|
+
"""Handle those HTTP errors with style! 🛠️"""
|
|
463
|
+
try:
|
|
464
|
+
response.raise_for_status()
|
|
465
|
+
return response
|
|
466
|
+
except requests.exceptions.HTTPError as error:
|
|
467
|
+
raise YouTubeRequestFailedError(video_id, error)
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
if __name__ == "__main__":
|
|
471
|
+
# Let's get this party started! 🎉
|
|
472
|
+
from rich import print
|
|
473
|
+
video_url = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
|
|
474
|
+
transcript = YTTranscriber.get_transcript(video_url, languages=None)
|
|
475
|
+
print("Here's what we got! 🔥")
|
|
476
476
|
print(transcript)
|