webscout 1.2.8__py3-none-any.whl → 1.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of webscout might be problematic. Click here for more details.
- webscout/__init__.py +1 -1
- webscout/transcriber.py +496 -496
- webscout/version.py +1 -1
- webscout/voice.py +27 -0
- {webscout-1.2.8.dist-info → webscout-1.3.0.dist-info}/METADATA +29 -11
- {webscout-1.2.8.dist-info → webscout-1.3.0.dist-info}/RECORD +10 -9
- {webscout-1.2.8.dist-info → webscout-1.3.0.dist-info}/LICENSE.md +0 -0
- {webscout-1.2.8.dist-info → webscout-1.3.0.dist-info}/WHEEL +0 -0
- {webscout-1.2.8.dist-info → webscout-1.3.0.dist-info}/entry_points.txt +0 -0
- {webscout-1.2.8.dist-info → webscout-1.3.0.dist-info}/top_level.txt +0 -0
webscout/__init__.py
CHANGED
|
@@ -9,8 +9,8 @@ from .webscout_search import WEBS
|
|
|
9
9
|
from .webscout_search_async import AsyncWEBS
|
|
10
10
|
from .version import __version__
|
|
11
11
|
from .DWEBS import DeepWEBS
|
|
12
|
-
from .AIutel import appdir
|
|
13
12
|
from .transcriber import transcriber
|
|
13
|
+
from .voice import play_audio
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
__all__ = ["WEBS", "AsyncWEBS", "__version__", "cli"]
|
webscout/transcriber.py
CHANGED
|
@@ -1,497 +1,497 @@
|
|
|
1
|
-
import requests
|
|
2
|
-
import http.cookiejar as cookiejar
|
|
3
|
-
import sys
|
|
4
|
-
import json
|
|
5
|
-
from xml.etree import ElementTree
|
|
6
|
-
import re
|
|
7
|
-
from requests import HTTPError
|
|
8
|
-
import html.parser
|
|
9
|
-
|
|
10
|
-
html_parser = html.parser.HTMLParser()
|
|
11
|
-
import html
|
|
12
|
-
|
|
13
|
-
def unescape(string):
|
|
14
|
-
return html.unescape(string)
|
|
15
|
-
WATCH_URL = 'https://www.youtube.com/watch?v={video_id}'
|
|
16
|
-
|
|
17
|
-
class TranscriptRetrievalError(Exception):
|
|
18
|
-
"""
|
|
19
|
-
Base class for exceptions raised when a transcript cannot be retrieved.
|
|
20
|
-
"""
|
|
21
|
-
ERROR_MESSAGE = '\nCould not retrieve a transcript for the video {video_url}!'
|
|
22
|
-
CAUSE_MESSAGE_INTRO = ' This is most likely caused by:\n\n{cause}'
|
|
23
|
-
CAUSE_MESSAGE = ''
|
|
24
|
-
GITHUB_REFERRAL = (
|
|
25
|
-
'\n\nIf you are sure that the described cause is not responsible for this error '
|
|
26
|
-
'and that a transcript should be retrievable, please create an issue at '
|
|
27
|
-
'https://github.com/OE-LUCIFER/Webscout/issues. '
|
|
28
|
-
'Please add which version of
|
|
29
|
-
'and provide the information needed to replicate the error. '
|
|
30
|
-
)
|
|
31
|
-
|
|
32
|
-
def __init__(self, video_id):
|
|
33
|
-
self.video_id = video_id
|
|
34
|
-
super(TranscriptRetrievalError, self).__init__(self._build_error_message())
|
|
35
|
-
|
|
36
|
-
def _build_error_message(self):
|
|
37
|
-
cause = self.cause
|
|
38
|
-
error_message = self.ERROR_MESSAGE.format(video_url=WATCH_URL.format(video_id=self.video_id))
|
|
39
|
-
|
|
40
|
-
if cause:
|
|
41
|
-
error_message += self.CAUSE_MESSAGE_INTRO.format(cause=cause) + self.GITHUB_REFERRAL
|
|
42
|
-
|
|
43
|
-
return error_message
|
|
44
|
-
|
|
45
|
-
@property
|
|
46
|
-
def cause(self):
|
|
47
|
-
return self.CAUSE_MESSAGE
|
|
48
|
-
|
|
49
|
-
class YouTubeRequestFailedError(TranscriptRetrievalError):
|
|
50
|
-
CAUSE_MESSAGE = 'Request to YouTube failed: {reason}'
|
|
51
|
-
|
|
52
|
-
def __init__(self, video_id, http_error):
|
|
53
|
-
self.reason = str(http_error)
|
|
54
|
-
super(YouTubeRequestFailedError, self).__init__(video_id)
|
|
55
|
-
|
|
56
|
-
@property
|
|
57
|
-
def cause(self):
|
|
58
|
-
return self.CAUSE_MESSAGE.format(reason=self.reason)
|
|
59
|
-
|
|
60
|
-
class VideoUnavailableError(TranscriptRetrievalError):
|
|
61
|
-
CAUSE_MESSAGE = 'The video is no longer available'
|
|
62
|
-
|
|
63
|
-
class InvalidVideoIdError(TranscriptRetrievalError):
|
|
64
|
-
CAUSE_MESSAGE = (
|
|
65
|
-
'You provided an invalid video id. Make sure you are using the video id and NOT the url!\n\n'
|
|
66
|
-
'Do NOT run: `YouTubeTranscriptApi.get_transcript("https://www.youtube.com/watch?v=1234")`\n'
|
|
67
|
-
'Instead run: `YouTubeTranscriptApi.get_transcript("1234")`'
|
|
68
|
-
)
|
|
69
|
-
|
|
70
|
-
class TooManyRequestsError(TranscriptRetrievalError):
|
|
71
|
-
CAUSE_MESSAGE = (
|
|
72
|
-
'YouTube is receiving too many requests from this IP and now requires solving a captcha to continue. '
|
|
73
|
-
'One of the following things can be done to work around this:\n\
|
|
74
|
-
- Manually solve the captcha in a browser and export the cookie. '
|
|
75
|
-
'Read here how to use that cookie with '
|
|
76
|
-
'youtube-transcript-api: https://github.com/jdepoix/youtube-transcript-api#cookies\n\
|
|
77
|
-
- Use a different IP address\n\
|
|
78
|
-
- Wait until the ban on your IP has been lifted'
|
|
79
|
-
)
|
|
80
|
-
|
|
81
|
-
class TranscriptsDisabledError(TranscriptRetrievalError):
|
|
82
|
-
CAUSE_MESSAGE = 'Subtitles are disabled for this video'
|
|
83
|
-
|
|
84
|
-
class NoTranscriptAvailableError(TranscriptRetrievalError):
|
|
85
|
-
CAUSE_MESSAGE = 'No transcripts are available for this video'
|
|
86
|
-
|
|
87
|
-
class NotTranslatableError(TranscriptRetrievalError):
|
|
88
|
-
CAUSE_MESSAGE = 'The requested language is not translatable'
|
|
89
|
-
|
|
90
|
-
class TranslationLanguageNotAvailableError(TranscriptRetrievalError):
|
|
91
|
-
CAUSE_MESSAGE = 'The requested translation language is not available'
|
|
92
|
-
|
|
93
|
-
class CookiePathInvalidError(TranscriptRetrievalError):
|
|
94
|
-
CAUSE_MESSAGE = 'The provided cookie file was unable to be loaded'
|
|
95
|
-
|
|
96
|
-
class CookiesInvalidError(TranscriptRetrievalError):
|
|
97
|
-
CAUSE_MESSAGE = 'The cookies provided are not valid (may have expired)'
|
|
98
|
-
|
|
99
|
-
class FailedToCreateConsentCookieError(TranscriptRetrievalError):
|
|
100
|
-
CAUSE_MESSAGE = 'Failed to automatically give consent to saving cookies'
|
|
101
|
-
|
|
102
|
-
class NoTranscriptFoundError(TranscriptRetrievalError):
|
|
103
|
-
CAUSE_MESSAGE = (
|
|
104
|
-
'No transcripts were found for any of the requested language codes: {requested_language_codes}\n\n'
|
|
105
|
-
'{transcript_data}'
|
|
106
|
-
)
|
|
107
|
-
|
|
108
|
-
def __init__(self, video_id, requested_language_codes, transcript_data):
|
|
109
|
-
self._requested_language_codes = requested_language_codes
|
|
110
|
-
self._transcript_data = transcript_data
|
|
111
|
-
super(NoTranscriptFoundError, self).__init__(video_id)
|
|
112
|
-
|
|
113
|
-
@property
|
|
114
|
-
def cause(self):
|
|
115
|
-
return self.CAUSE_MESSAGE.format(
|
|
116
|
-
requested_language_codes=self._requested_language_codes,
|
|
117
|
-
transcript_data=str(self._transcript_data),
|
|
118
|
-
)
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
def _raise_http_errors(response, video_id):
|
|
123
|
-
try:
|
|
124
|
-
response.raise_for_status()
|
|
125
|
-
return response
|
|
126
|
-
except HTTPError as error:
|
|
127
|
-
raise YouTubeRequestFailedError(error, video_id)
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
class TranscriptListFetcher(object):
|
|
131
|
-
def __init__(self, http_client):
|
|
132
|
-
self._http_client = http_client
|
|
133
|
-
|
|
134
|
-
def fetch(self, video_id):
|
|
135
|
-
return TranscriptList.build(
|
|
136
|
-
self._http_client,
|
|
137
|
-
video_id,
|
|
138
|
-
self._extract_captions_json(self._fetch_video_html(video_id), video_id),
|
|
139
|
-
)
|
|
140
|
-
|
|
141
|
-
def _extract_captions_json(self, html, video_id):
|
|
142
|
-
splitted_html = html.split('"captions":')
|
|
143
|
-
|
|
144
|
-
if len(splitted_html) <= 1:
|
|
145
|
-
if video_id.startswith('http://') or video_id.startswith('https://'):
|
|
146
|
-
raise InvalidVideoIdError(video_id)
|
|
147
|
-
if 'class="g-recaptcha"' in html:
|
|
148
|
-
raise TooManyRequestsError(video_id)
|
|
149
|
-
if '"playabilityStatus":' not in html:
|
|
150
|
-
raise VideoUnavailableError(video_id)
|
|
151
|
-
|
|
152
|
-
raise TranscriptsDisabledError(video_id)
|
|
153
|
-
|
|
154
|
-
captions_json = json.loads(
|
|
155
|
-
splitted_html[1].split(',"videoDetails')[0].replace('\n', '')
|
|
156
|
-
).get('playerCaptionsTracklistRenderer')
|
|
157
|
-
if captions_json is None:
|
|
158
|
-
raise TranscriptsDisabledError(video_id)
|
|
159
|
-
|
|
160
|
-
if 'captionTracks' not in captions_json:
|
|
161
|
-
raise TranscriptsDisabledError(video_id)
|
|
162
|
-
|
|
163
|
-
return captions_json
|
|
164
|
-
|
|
165
|
-
def _create_consent_cookie(self, html, video_id):
|
|
166
|
-
match = re.search('name="v" value="(.*?)"', html)
|
|
167
|
-
if match is None:
|
|
168
|
-
raise FailedToCreateConsentCookieError(video_id)
|
|
169
|
-
self._http_client.cookies.set('CONSENT', 'YES+' + match.group(1), domain='.youtube.com')
|
|
170
|
-
|
|
171
|
-
def _fetch_video_html(self, video_id):
|
|
172
|
-
html = self._fetch_html(video_id)
|
|
173
|
-
if 'action="https://consent.youtube.com/s"' in html:
|
|
174
|
-
self._create_consent_cookie(html, video_id)
|
|
175
|
-
html = self._fetch_html(video_id)
|
|
176
|
-
if 'action="https://consent.youtube.com/s"' in html:
|
|
177
|
-
raise FailedToCreateConsentCookieError(video_id)
|
|
178
|
-
return html
|
|
179
|
-
|
|
180
|
-
def _fetch_html(self, video_id):
|
|
181
|
-
response = self._http_client.get(WATCH_URL.format(video_id=video_id), headers={'Accept-Language': 'en-US'})
|
|
182
|
-
return unescape(_raise_http_errors(response, video_id).text)
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
class TranscriptList(object):
|
|
186
|
-
"""
|
|
187
|
-
This object represents a list of transcripts. It can be iterated over to list all transcripts which are available
|
|
188
|
-
for a given YouTube video. Also it provides functionality to search for a transcript in a given language.
|
|
189
|
-
"""
|
|
190
|
-
|
|
191
|
-
def __init__(self, video_id, manually_created_transcripts, generated_transcripts, translation_languages):
|
|
192
|
-
"""
|
|
193
|
-
The constructor is only for internal use. Use the static build method instead.
|
|
194
|
-
|
|
195
|
-
:param video_id: the id of the video this TranscriptList is for
|
|
196
|
-
:type video_id: str
|
|
197
|
-
:param manually_created_transcripts: dict mapping language codes to the manually created transcripts
|
|
198
|
-
:type manually_created_transcripts: dict[str, Transcript]
|
|
199
|
-
:param generated_transcripts: dict mapping language codes to the generated transcripts
|
|
200
|
-
:type generated_transcripts: dict[str, Transcript]
|
|
201
|
-
:param translation_languages: list of languages which can be used for translatable languages
|
|
202
|
-
:type translation_languages: list[dict[str, str]]
|
|
203
|
-
"""
|
|
204
|
-
self.video_id = video_id
|
|
205
|
-
self._manually_created_transcripts = manually_created_transcripts
|
|
206
|
-
self._generated_transcripts = generated_transcripts
|
|
207
|
-
self._translation_languages = translation_languages
|
|
208
|
-
|
|
209
|
-
@staticmethod
|
|
210
|
-
def build(http_client, video_id, captions_json):
|
|
211
|
-
"""
|
|
212
|
-
Factory method for TranscriptList.
|
|
213
|
-
|
|
214
|
-
:param http_client: http client which is used to make the transcript retrieving http calls
|
|
215
|
-
:type http_client: requests.Session
|
|
216
|
-
:param video_id: the id of the video this TranscriptList is for
|
|
217
|
-
:type video_id: str
|
|
218
|
-
:param captions_json: the JSON parsed from the YouTube pages static HTML
|
|
219
|
-
:type captions_json: dict
|
|
220
|
-
:return: the created TranscriptList
|
|
221
|
-
:rtype TranscriptList:
|
|
222
|
-
"""
|
|
223
|
-
translation_languages = [
|
|
224
|
-
{
|
|
225
|
-
'language': translation_language['languageName']['simpleText'],
|
|
226
|
-
'language_code': translation_language['languageCode'],
|
|
227
|
-
} for translation_language in captions_json.get('translationLanguages', [])
|
|
228
|
-
]
|
|
229
|
-
|
|
230
|
-
manually_created_transcripts = {}
|
|
231
|
-
generated_transcripts = {}
|
|
232
|
-
|
|
233
|
-
for caption in captions_json['captionTracks']:
|
|
234
|
-
if caption.get('kind', '') == 'asr':
|
|
235
|
-
transcript_dict = generated_transcripts
|
|
236
|
-
else:
|
|
237
|
-
transcript_dict = manually_created_transcripts
|
|
238
|
-
|
|
239
|
-
transcript_dict[caption['languageCode']] = Transcript(
|
|
240
|
-
http_client,
|
|
241
|
-
video_id,
|
|
242
|
-
caption['baseUrl'],
|
|
243
|
-
caption['name']['simpleText'],
|
|
244
|
-
caption['languageCode'],
|
|
245
|
-
caption.get('kind', '') == 'asr',
|
|
246
|
-
translation_languages if caption.get('isTranslatable', False) else [],
|
|
247
|
-
)
|
|
248
|
-
|
|
249
|
-
return TranscriptList(
|
|
250
|
-
video_id,
|
|
251
|
-
manually_created_transcripts,
|
|
252
|
-
generated_transcripts,
|
|
253
|
-
translation_languages,
|
|
254
|
-
)
|
|
255
|
-
|
|
256
|
-
def __iter__(self):
|
|
257
|
-
return iter(list(self._manually_created_transcripts.values()) + list(self._generated_transcripts.values()))
|
|
258
|
-
|
|
259
|
-
def find_transcript(self, language_codes):
|
|
260
|
-
"""
|
|
261
|
-
Finds a transcript for a given language code. Manually created transcripts are returned first and only if none
|
|
262
|
-
are found, generated transcripts are used. If you only want generated transcripts use
|
|
263
|
-
`find_manually_created_transcript` instead.
|
|
264
|
-
|
|
265
|
-
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
|
|
266
|
-
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
|
|
267
|
-
it fails to do so.
|
|
268
|
-
:type languages: list[str]
|
|
269
|
-
:return: the found Transcript
|
|
270
|
-
:rtype Transcript:
|
|
271
|
-
:raises: NoTranscriptFound
|
|
272
|
-
"""
|
|
273
|
-
return self._find_transcript(language_codes, [self._manually_created_transcripts, self._generated_transcripts])
|
|
274
|
-
|
|
275
|
-
def find_generated_transcript(self, language_codes):
|
|
276
|
-
"""
|
|
277
|
-
Finds an automatically generated transcript for a given language code.
|
|
278
|
-
|
|
279
|
-
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
|
|
280
|
-
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
|
|
281
|
-
it fails to do so.
|
|
282
|
-
:type languages: list[str]
|
|
283
|
-
:return: the found Transcript
|
|
284
|
-
:rtype Transcript:
|
|
285
|
-
:raises: NoTranscriptFound
|
|
286
|
-
"""
|
|
287
|
-
return self._find_transcript(language_codes, [self._generated_transcripts])
|
|
288
|
-
|
|
289
|
-
def find_manually_created_transcript(self, language_codes):
|
|
290
|
-
"""
|
|
291
|
-
Finds a manually created transcript for a given language code.
|
|
292
|
-
|
|
293
|
-
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
|
|
294
|
-
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
|
|
295
|
-
it fails to do so.
|
|
296
|
-
:type languages: list[str]
|
|
297
|
-
:return: the found Transcript
|
|
298
|
-
:rtype Transcript:
|
|
299
|
-
:raises: NoTranscriptFound
|
|
300
|
-
"""
|
|
301
|
-
return self._find_transcript(language_codes, [self._manually_created_transcripts])
|
|
302
|
-
|
|
303
|
-
def _find_transcript(self, language_codes, transcript_dicts):
|
|
304
|
-
for language_code in language_codes:
|
|
305
|
-
for transcript_dict in transcript_dicts:
|
|
306
|
-
if language_code in transcript_dict:
|
|
307
|
-
return transcript_dict[language_code]
|
|
308
|
-
|
|
309
|
-
raise NoTranscriptFoundError(
|
|
310
|
-
self.video_id,
|
|
311
|
-
language_codes,
|
|
312
|
-
self
|
|
313
|
-
)
|
|
314
|
-
|
|
315
|
-
def __str__(self):
|
|
316
|
-
return (
|
|
317
|
-
'For this video ({video_id}) transcripts are available in the following languages:\n\n'
|
|
318
|
-
'(MANUALLY CREATED)\n'
|
|
319
|
-
'{available_manually_created_transcript_languages}\n\n'
|
|
320
|
-
'(GENERATED)\n'
|
|
321
|
-
'{available_generated_transcripts}\n\n'
|
|
322
|
-
'(TRANSLATION LANGUAGES)\n'
|
|
323
|
-
'{available_translation_languages}'
|
|
324
|
-
).format(
|
|
325
|
-
video_id=self.video_id,
|
|
326
|
-
available_manually_created_transcript_languages=self._get_language_description(
|
|
327
|
-
str(transcript) for transcript in self._manually_created_transcripts.values()
|
|
328
|
-
),
|
|
329
|
-
available_generated_transcripts=self._get_language_description(
|
|
330
|
-
str(transcript) for transcript in self._generated_transcripts.values()
|
|
331
|
-
),
|
|
332
|
-
available_translation_languages=self._get_language_description(
|
|
333
|
-
'{language_code} ("{language}")'.format(
|
|
334
|
-
language=translation_language['language'],
|
|
335
|
-
language_code=translation_language['language_code'],
|
|
336
|
-
) for translation_language in self._translation_languages
|
|
337
|
-
)
|
|
338
|
-
)
|
|
339
|
-
|
|
340
|
-
def _get_language_description(self, transcript_strings):
|
|
341
|
-
description = '\n'.join(' - {transcript}'.format(transcript=transcript) for transcript in transcript_strings)
|
|
342
|
-
return description if description else 'None'
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
class Transcript(object):
|
|
346
|
-
def __init__(self, http_client, video_id, url, language, language_code, is_generated, translation_languages):
|
|
347
|
-
"""
|
|
348
|
-
You probably don't want to initialize this directly. Usually you'll access Transcript objects using a
|
|
349
|
-
TranscriptList.
|
|
350
|
-
|
|
351
|
-
:param http_client: http client which is used to make the transcript retrieving http calls
|
|
352
|
-
:type http_client: requests.Session
|
|
353
|
-
:param video_id: the id of the video this TranscriptList is for
|
|
354
|
-
:type video_id: str
|
|
355
|
-
:param url: the url which needs to be called to fetch the transcript
|
|
356
|
-
:param language: the name of the language this transcript uses
|
|
357
|
-
:param language_code:
|
|
358
|
-
:param is_generated:
|
|
359
|
-
:param translation_languages:
|
|
360
|
-
"""
|
|
361
|
-
self._http_client = http_client
|
|
362
|
-
self.video_id = video_id
|
|
363
|
-
self._url = url
|
|
364
|
-
self.language = language
|
|
365
|
-
self.language_code = language_code
|
|
366
|
-
self.is_generated = is_generated
|
|
367
|
-
self.translation_languages = translation_languages
|
|
368
|
-
self._translation_languages_dict = {
|
|
369
|
-
translation_language['language_code']: translation_language['language']
|
|
370
|
-
for translation_language in translation_languages
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
def fetch(self, preserve_formatting=False):
|
|
374
|
-
"""
|
|
375
|
-
Loads the actual transcript data.
|
|
376
|
-
:param preserve_formatting: whether to keep select HTML text formatting
|
|
377
|
-
:type preserve_formatting: bool
|
|
378
|
-
:return: a list of dictionaries containing the 'text', 'start' and 'duration' keys
|
|
379
|
-
:rtype [{'text': str, 'start': float, 'end': float}]:
|
|
380
|
-
"""
|
|
381
|
-
response = self._http_client.get(self._url, headers={'Accept-Language': 'en-US'})
|
|
382
|
-
return _TranscriptParser(preserve_formatting=preserve_formatting).parse(
|
|
383
|
-
_raise_http_errors(response, self.video_id).text,
|
|
384
|
-
)
|
|
385
|
-
|
|
386
|
-
def __str__(self):
|
|
387
|
-
return '{language_code} ("{language}"){translation_description}'.format(
|
|
388
|
-
language=self.language,
|
|
389
|
-
language_code=self.language_code,
|
|
390
|
-
translation_description='[TRANSLATABLE]' if self.is_translatable else ''
|
|
391
|
-
)
|
|
392
|
-
|
|
393
|
-
@property
|
|
394
|
-
def is_translatable(self):
|
|
395
|
-
return len(self.translation_languages) > 0
|
|
396
|
-
|
|
397
|
-
def translate(self, language_code):
|
|
398
|
-
if not self.is_translatable:
|
|
399
|
-
raise NotTranslatableError(self.video_id)
|
|
400
|
-
|
|
401
|
-
if language_code not in self._translation_languages_dict:
|
|
402
|
-
raise TranslationLanguageNotAvailableError(self.video_id)
|
|
403
|
-
|
|
404
|
-
return Transcript(
|
|
405
|
-
self._http_client,
|
|
406
|
-
self.video_id,
|
|
407
|
-
'{url}&tlang={language_code}'.format(url=self._url, language_code=language_code),
|
|
408
|
-
self._translation_languages_dict[language_code],
|
|
409
|
-
language_code,
|
|
410
|
-
True,
|
|
411
|
-
[],
|
|
412
|
-
)
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
class _TranscriptParser(object):
|
|
416
|
-
_FORMATTING_TAGS = [
|
|
417
|
-
'strong', # important
|
|
418
|
-
'em', # emphasized
|
|
419
|
-
'b', # bold
|
|
420
|
-
'i', # italic
|
|
421
|
-
'mark', # marked
|
|
422
|
-
'small', # smaller
|
|
423
|
-
'del', # deleted
|
|
424
|
-
'ins', # inserted
|
|
425
|
-
'sub', # subscript
|
|
426
|
-
'sup', # superscript
|
|
427
|
-
]
|
|
428
|
-
|
|
429
|
-
def __init__(self, preserve_formatting=False):
|
|
430
|
-
self._html_regex = self._get_html_regex(preserve_formatting)
|
|
431
|
-
|
|
432
|
-
def _get_html_regex(self, preserve_formatting):
|
|
433
|
-
if preserve_formatting:
|
|
434
|
-
formats_regex = '|'.join(self._FORMATTING_TAGS)
|
|
435
|
-
formats_regex = r'<\/?(?!\/?(' + formats_regex + r')\b).*?\b>'
|
|
436
|
-
html_regex = re.compile(formats_regex, re.IGNORECASE)
|
|
437
|
-
else:
|
|
438
|
-
html_regex = re.compile(r'<[^>]*>', re.IGNORECASE)
|
|
439
|
-
return html_regex
|
|
440
|
-
|
|
441
|
-
def parse(self, plain_data):
|
|
442
|
-
return [
|
|
443
|
-
{
|
|
444
|
-
'text': re.sub(self._html_regex, '', unescape(xml_element.text)),
|
|
445
|
-
'start': float(xml_element.attrib['start']),
|
|
446
|
-
'duration': float(xml_element.attrib.get('dur', '0.0')),
|
|
447
|
-
}
|
|
448
|
-
for xml_element in ElementTree.fromstring(plain_data)
|
|
449
|
-
if xml_element.text is not None
|
|
450
|
-
]
|
|
451
|
-
|
|
452
|
-
WATCH_URL = 'https://www.youtube.com/watch?v={video_id}'
|
|
453
|
-
|
|
454
|
-
class transcriber(object):
|
|
455
|
-
@classmethod
|
|
456
|
-
def list_transcripts(cls, video_id, proxies=None, cookies=None):
|
|
457
|
-
with requests.Session() as http_client:
|
|
458
|
-
if cookies:
|
|
459
|
-
http_client.cookies = cls._load_cookies(cookies, video_id)
|
|
460
|
-
http_client.proxies = proxies if proxies else {}
|
|
461
|
-
return TranscriptListFetcher(http_client).fetch(video_id)
|
|
462
|
-
|
|
463
|
-
@classmethod
|
|
464
|
-
def get_transcripts(cls, video_ids, languages=('en',), continue_after_error=False, proxies=None,
|
|
465
|
-
cookies=None, preserve_formatting=False):
|
|
466
|
-
|
|
467
|
-
assert isinstance(video_ids, list), "`video_ids` must be a list of strings"
|
|
468
|
-
|
|
469
|
-
data = {}
|
|
470
|
-
unretrievable_videos = []
|
|
471
|
-
|
|
472
|
-
for video_id in video_ids:
|
|
473
|
-
try:
|
|
474
|
-
data[video_id] = cls.get_transcript(video_id, languages, proxies, cookies, preserve_formatting)
|
|
475
|
-
except Exception as exception:
|
|
476
|
-
if not continue_after_error:
|
|
477
|
-
raise exception
|
|
478
|
-
|
|
479
|
-
unretrievable_videos.append(video_id)
|
|
480
|
-
|
|
481
|
-
return data, unretrievable_videos
|
|
482
|
-
|
|
483
|
-
@classmethod
|
|
484
|
-
def get_transcript(cls, video_id, languages=('en',), proxies=None, cookies=None, preserve_formatting=False):
|
|
485
|
-
assert isinstance(video_id, str), "`video_id` must be a string"
|
|
486
|
-
return cls.list_transcripts(video_id, proxies, cookies).find_transcript(languages).fetch(preserve_formatting=preserve_formatting)
|
|
487
|
-
|
|
488
|
-
@classmethod
|
|
489
|
-
def _load_cookies(cls, cookies, video_id):
|
|
490
|
-
try:
|
|
491
|
-
cookie_jar = cookiejar.MozillaCookieJar()
|
|
492
|
-
cookie_jar.load(cookies)
|
|
493
|
-
if not cookie_jar:
|
|
494
|
-
raise CookiesInvalidError(video_id)
|
|
495
|
-
return cookie_jar
|
|
496
|
-
except:
|
|
1
|
+
import requests
|
|
2
|
+
import http.cookiejar as cookiejar
|
|
3
|
+
import sys
|
|
4
|
+
import json
|
|
5
|
+
from xml.etree import ElementTree
|
|
6
|
+
import re
|
|
7
|
+
from requests import HTTPError
|
|
8
|
+
import html.parser
|
|
9
|
+
|
|
10
|
+
html_parser = html.parser.HTMLParser()
|
|
11
|
+
import html
|
|
12
|
+
|
|
13
|
+
def unescape(string):
|
|
14
|
+
return html.unescape(string)
|
|
15
|
+
WATCH_URL = 'https://www.youtube.com/watch?v={video_id}'
|
|
16
|
+
|
|
17
|
+
class TranscriptRetrievalError(Exception):
|
|
18
|
+
"""
|
|
19
|
+
Base class for exceptions raised when a transcript cannot be retrieved.
|
|
20
|
+
"""
|
|
21
|
+
ERROR_MESSAGE = '\nCould not retrieve a transcript for the video {video_url}!'
|
|
22
|
+
CAUSE_MESSAGE_INTRO = ' This is most likely caused by:\n\n{cause}'
|
|
23
|
+
CAUSE_MESSAGE = ''
|
|
24
|
+
GITHUB_REFERRAL = (
|
|
25
|
+
'\n\nIf you are sure that the described cause is not responsible for this error '
|
|
26
|
+
'and that a transcript should be retrievable, please create an issue at '
|
|
27
|
+
'https://github.com/OE-LUCIFER/Webscout/issues. '
|
|
28
|
+
'Please add which version of webscout you are using '
|
|
29
|
+
'and provide the information needed to replicate the error. '
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
def __init__(self, video_id):
|
|
33
|
+
self.video_id = video_id
|
|
34
|
+
super(TranscriptRetrievalError, self).__init__(self._build_error_message())
|
|
35
|
+
|
|
36
|
+
def _build_error_message(self):
|
|
37
|
+
cause = self.cause
|
|
38
|
+
error_message = self.ERROR_MESSAGE.format(video_url=WATCH_URL.format(video_id=self.video_id))
|
|
39
|
+
|
|
40
|
+
if cause:
|
|
41
|
+
error_message += self.CAUSE_MESSAGE_INTRO.format(cause=cause) + self.GITHUB_REFERRAL
|
|
42
|
+
|
|
43
|
+
return error_message
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def cause(self):
|
|
47
|
+
return self.CAUSE_MESSAGE
|
|
48
|
+
|
|
49
|
+
class YouTubeRequestFailedError(TranscriptRetrievalError):
|
|
50
|
+
CAUSE_MESSAGE = 'Request to YouTube failed: {reason}'
|
|
51
|
+
|
|
52
|
+
def __init__(self, video_id, http_error):
|
|
53
|
+
self.reason = str(http_error)
|
|
54
|
+
super(YouTubeRequestFailedError, self).__init__(video_id)
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def cause(self):
|
|
58
|
+
return self.CAUSE_MESSAGE.format(reason=self.reason)
|
|
59
|
+
|
|
60
|
+
class VideoUnavailableError(TranscriptRetrievalError):
|
|
61
|
+
CAUSE_MESSAGE = 'The video is no longer available'
|
|
62
|
+
|
|
63
|
+
class InvalidVideoIdError(TranscriptRetrievalError):
|
|
64
|
+
CAUSE_MESSAGE = (
|
|
65
|
+
'You provided an invalid video id. Make sure you are using the video id and NOT the url!\n\n'
|
|
66
|
+
'Do NOT run: `YouTubeTranscriptApi.get_transcript("https://www.youtube.com/watch?v=1234")`\n'
|
|
67
|
+
'Instead run: `YouTubeTranscriptApi.get_transcript("1234")`'
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
class TooManyRequestsError(TranscriptRetrievalError):
|
|
71
|
+
CAUSE_MESSAGE = (
|
|
72
|
+
'YouTube is receiving too many requests from this IP and now requires solving a captcha to continue. '
|
|
73
|
+
'One of the following things can be done to work around this:\n\
|
|
74
|
+
- Manually solve the captcha in a browser and export the cookie. '
|
|
75
|
+
'Read here how to use that cookie with '
|
|
76
|
+
'youtube-transcript-api: https://github.com/jdepoix/youtube-transcript-api#cookies\n\
|
|
77
|
+
- Use a different IP address\n\
|
|
78
|
+
- Wait until the ban on your IP has been lifted'
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
class TranscriptsDisabledError(TranscriptRetrievalError):
|
|
82
|
+
CAUSE_MESSAGE = 'Subtitles are disabled for this video'
|
|
83
|
+
|
|
84
|
+
class NoTranscriptAvailableError(TranscriptRetrievalError):
|
|
85
|
+
CAUSE_MESSAGE = 'No transcripts are available for this video'
|
|
86
|
+
|
|
87
|
+
class NotTranslatableError(TranscriptRetrievalError):
|
|
88
|
+
CAUSE_MESSAGE = 'The requested language is not translatable'
|
|
89
|
+
|
|
90
|
+
class TranslationLanguageNotAvailableError(TranscriptRetrievalError):
|
|
91
|
+
CAUSE_MESSAGE = 'The requested translation language is not available'
|
|
92
|
+
|
|
93
|
+
class CookiePathInvalidError(TranscriptRetrievalError):
|
|
94
|
+
CAUSE_MESSAGE = 'The provided cookie file was unable to be loaded'
|
|
95
|
+
|
|
96
|
+
class CookiesInvalidError(TranscriptRetrievalError):
|
|
97
|
+
CAUSE_MESSAGE = 'The cookies provided are not valid (may have expired)'
|
|
98
|
+
|
|
99
|
+
class FailedToCreateConsentCookieError(TranscriptRetrievalError):
|
|
100
|
+
CAUSE_MESSAGE = 'Failed to automatically give consent to saving cookies'
|
|
101
|
+
|
|
102
|
+
class NoTranscriptFoundError(TranscriptRetrievalError):
|
|
103
|
+
CAUSE_MESSAGE = (
|
|
104
|
+
'No transcripts were found for any of the requested language codes: {requested_language_codes}\n\n'
|
|
105
|
+
'{transcript_data}'
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
def __init__(self, video_id, requested_language_codes, transcript_data):
|
|
109
|
+
self._requested_language_codes = requested_language_codes
|
|
110
|
+
self._transcript_data = transcript_data
|
|
111
|
+
super(NoTranscriptFoundError, self).__init__(video_id)
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def cause(self):
|
|
115
|
+
return self.CAUSE_MESSAGE.format(
|
|
116
|
+
requested_language_codes=self._requested_language_codes,
|
|
117
|
+
transcript_data=str(self._transcript_data),
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _raise_http_errors(response, video_id):
|
|
123
|
+
try:
|
|
124
|
+
response.raise_for_status()
|
|
125
|
+
return response
|
|
126
|
+
except HTTPError as error:
|
|
127
|
+
raise YouTubeRequestFailedError(error, video_id)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class TranscriptListFetcher(object):
|
|
131
|
+
def __init__(self, http_client):
|
|
132
|
+
self._http_client = http_client
|
|
133
|
+
|
|
134
|
+
def fetch(self, video_id):
|
|
135
|
+
return TranscriptList.build(
|
|
136
|
+
self._http_client,
|
|
137
|
+
video_id,
|
|
138
|
+
self._extract_captions_json(self._fetch_video_html(video_id), video_id),
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
def _extract_captions_json(self, html, video_id):
|
|
142
|
+
splitted_html = html.split('"captions":')
|
|
143
|
+
|
|
144
|
+
if len(splitted_html) <= 1:
|
|
145
|
+
if video_id.startswith('http://') or video_id.startswith('https://'):
|
|
146
|
+
raise InvalidVideoIdError(video_id)
|
|
147
|
+
if 'class="g-recaptcha"' in html:
|
|
148
|
+
raise TooManyRequestsError(video_id)
|
|
149
|
+
if '"playabilityStatus":' not in html:
|
|
150
|
+
raise VideoUnavailableError(video_id)
|
|
151
|
+
|
|
152
|
+
raise TranscriptsDisabledError(video_id)
|
|
153
|
+
|
|
154
|
+
captions_json = json.loads(
|
|
155
|
+
splitted_html[1].split(',"videoDetails')[0].replace('\n', '')
|
|
156
|
+
).get('playerCaptionsTracklistRenderer')
|
|
157
|
+
if captions_json is None:
|
|
158
|
+
raise TranscriptsDisabledError(video_id)
|
|
159
|
+
|
|
160
|
+
if 'captionTracks' not in captions_json:
|
|
161
|
+
raise TranscriptsDisabledError(video_id)
|
|
162
|
+
|
|
163
|
+
return captions_json
|
|
164
|
+
|
|
165
|
+
def _create_consent_cookie(self, html, video_id):
|
|
166
|
+
match = re.search('name="v" value="(.*?)"', html)
|
|
167
|
+
if match is None:
|
|
168
|
+
raise FailedToCreateConsentCookieError(video_id)
|
|
169
|
+
self._http_client.cookies.set('CONSENT', 'YES+' + match.group(1), domain='.youtube.com')
|
|
170
|
+
|
|
171
|
+
def _fetch_video_html(self, video_id):
|
|
172
|
+
html = self._fetch_html(video_id)
|
|
173
|
+
if 'action="https://consent.youtube.com/s"' in html:
|
|
174
|
+
self._create_consent_cookie(html, video_id)
|
|
175
|
+
html = self._fetch_html(video_id)
|
|
176
|
+
if 'action="https://consent.youtube.com/s"' in html:
|
|
177
|
+
raise FailedToCreateConsentCookieError(video_id)
|
|
178
|
+
return html
|
|
179
|
+
|
|
180
|
+
def _fetch_html(self, video_id):
|
|
181
|
+
response = self._http_client.get(WATCH_URL.format(video_id=video_id), headers={'Accept-Language': 'en-US'})
|
|
182
|
+
return unescape(_raise_http_errors(response, video_id).text)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
class TranscriptList(object):
|
|
186
|
+
"""
|
|
187
|
+
This object represents a list of transcripts. It can be iterated over to list all transcripts which are available
|
|
188
|
+
for a given YouTube video. Also it provides functionality to search for a transcript in a given language.
|
|
189
|
+
"""
|
|
190
|
+
|
|
191
|
+
def __init__(self, video_id, manually_created_transcripts, generated_transcripts, translation_languages):
|
|
192
|
+
"""
|
|
193
|
+
The constructor is only for internal use. Use the static build method instead.
|
|
194
|
+
|
|
195
|
+
:param video_id: the id of the video this TranscriptList is for
|
|
196
|
+
:type video_id: str
|
|
197
|
+
:param manually_created_transcripts: dict mapping language codes to the manually created transcripts
|
|
198
|
+
:type manually_created_transcripts: dict[str, Transcript]
|
|
199
|
+
:param generated_transcripts: dict mapping language codes to the generated transcripts
|
|
200
|
+
:type generated_transcripts: dict[str, Transcript]
|
|
201
|
+
:param translation_languages: list of languages which can be used for translatable languages
|
|
202
|
+
:type translation_languages: list[dict[str, str]]
|
|
203
|
+
"""
|
|
204
|
+
self.video_id = video_id
|
|
205
|
+
self._manually_created_transcripts = manually_created_transcripts
|
|
206
|
+
self._generated_transcripts = generated_transcripts
|
|
207
|
+
self._translation_languages = translation_languages
|
|
208
|
+
|
|
209
|
+
@staticmethod
|
|
210
|
+
def build(http_client, video_id, captions_json):
|
|
211
|
+
"""
|
|
212
|
+
Factory method for TranscriptList.
|
|
213
|
+
|
|
214
|
+
:param http_client: http client which is used to make the transcript retrieving http calls
|
|
215
|
+
:type http_client: requests.Session
|
|
216
|
+
:param video_id: the id of the video this TranscriptList is for
|
|
217
|
+
:type video_id: str
|
|
218
|
+
:param captions_json: the JSON parsed from the YouTube pages static HTML
|
|
219
|
+
:type captions_json: dict
|
|
220
|
+
:return: the created TranscriptList
|
|
221
|
+
:rtype TranscriptList:
|
|
222
|
+
"""
|
|
223
|
+
translation_languages = [
|
|
224
|
+
{
|
|
225
|
+
'language': translation_language['languageName']['simpleText'],
|
|
226
|
+
'language_code': translation_language['languageCode'],
|
|
227
|
+
} for translation_language in captions_json.get('translationLanguages', [])
|
|
228
|
+
]
|
|
229
|
+
|
|
230
|
+
manually_created_transcripts = {}
|
|
231
|
+
generated_transcripts = {}
|
|
232
|
+
|
|
233
|
+
for caption in captions_json['captionTracks']:
|
|
234
|
+
if caption.get('kind', '') == 'asr':
|
|
235
|
+
transcript_dict = generated_transcripts
|
|
236
|
+
else:
|
|
237
|
+
transcript_dict = manually_created_transcripts
|
|
238
|
+
|
|
239
|
+
transcript_dict[caption['languageCode']] = Transcript(
|
|
240
|
+
http_client,
|
|
241
|
+
video_id,
|
|
242
|
+
caption['baseUrl'],
|
|
243
|
+
caption['name']['simpleText'],
|
|
244
|
+
caption['languageCode'],
|
|
245
|
+
caption.get('kind', '') == 'asr',
|
|
246
|
+
translation_languages if caption.get('isTranslatable', False) else [],
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
return TranscriptList(
|
|
250
|
+
video_id,
|
|
251
|
+
manually_created_transcripts,
|
|
252
|
+
generated_transcripts,
|
|
253
|
+
translation_languages,
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
def __iter__(self):
|
|
257
|
+
return iter(list(self._manually_created_transcripts.values()) + list(self._generated_transcripts.values()))
|
|
258
|
+
|
|
259
|
+
def find_transcript(self, language_codes):
|
|
260
|
+
"""
|
|
261
|
+
Finds a transcript for a given language code. Manually created transcripts are returned first and only if none
|
|
262
|
+
are found, generated transcripts are used. If you only want generated transcripts use
|
|
263
|
+
`find_manually_created_transcript` instead.
|
|
264
|
+
|
|
265
|
+
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
|
|
266
|
+
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
|
|
267
|
+
it fails to do so.
|
|
268
|
+
:type languages: list[str]
|
|
269
|
+
:return: the found Transcript
|
|
270
|
+
:rtype Transcript:
|
|
271
|
+
:raises: NoTranscriptFound
|
|
272
|
+
"""
|
|
273
|
+
return self._find_transcript(language_codes, [self._manually_created_transcripts, self._generated_transcripts])
|
|
274
|
+
|
|
275
|
+
def find_generated_transcript(self, language_codes):
|
|
276
|
+
"""
|
|
277
|
+
Finds an automatically generated transcript for a given language code.
|
|
278
|
+
|
|
279
|
+
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
|
|
280
|
+
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
|
|
281
|
+
it fails to do so.
|
|
282
|
+
:type languages: list[str]
|
|
283
|
+
:return: the found Transcript
|
|
284
|
+
:rtype Transcript:
|
|
285
|
+
:raises: NoTranscriptFound
|
|
286
|
+
"""
|
|
287
|
+
return self._find_transcript(language_codes, [self._generated_transcripts])
|
|
288
|
+
|
|
289
|
+
def find_manually_created_transcript(self, language_codes):
|
|
290
|
+
"""
|
|
291
|
+
Finds a manually created transcript for a given language code.
|
|
292
|
+
|
|
293
|
+
:param language_codes: A list of language codes in a descending priority. For example, if this is set to
|
|
294
|
+
['de', 'en'] it will first try to fetch the german transcript (de) and then fetch the english transcript (en) if
|
|
295
|
+
it fails to do so.
|
|
296
|
+
:type languages: list[str]
|
|
297
|
+
:return: the found Transcript
|
|
298
|
+
:rtype Transcript:
|
|
299
|
+
:raises: NoTranscriptFound
|
|
300
|
+
"""
|
|
301
|
+
return self._find_transcript(language_codes, [self._manually_created_transcripts])
|
|
302
|
+
|
|
303
|
+
def _find_transcript(self, language_codes, transcript_dicts):
|
|
304
|
+
for language_code in language_codes:
|
|
305
|
+
for transcript_dict in transcript_dicts:
|
|
306
|
+
if language_code in transcript_dict:
|
|
307
|
+
return transcript_dict[language_code]
|
|
308
|
+
|
|
309
|
+
raise NoTranscriptFoundError(
|
|
310
|
+
self.video_id,
|
|
311
|
+
language_codes,
|
|
312
|
+
self
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
def __str__(self):
|
|
316
|
+
return (
|
|
317
|
+
'For this video ({video_id}) transcripts are available in the following languages:\n\n'
|
|
318
|
+
'(MANUALLY CREATED)\n'
|
|
319
|
+
'{available_manually_created_transcript_languages}\n\n'
|
|
320
|
+
'(GENERATED)\n'
|
|
321
|
+
'{available_generated_transcripts}\n\n'
|
|
322
|
+
'(TRANSLATION LANGUAGES)\n'
|
|
323
|
+
'{available_translation_languages}'
|
|
324
|
+
).format(
|
|
325
|
+
video_id=self.video_id,
|
|
326
|
+
available_manually_created_transcript_languages=self._get_language_description(
|
|
327
|
+
str(transcript) for transcript in self._manually_created_transcripts.values()
|
|
328
|
+
),
|
|
329
|
+
available_generated_transcripts=self._get_language_description(
|
|
330
|
+
str(transcript) for transcript in self._generated_transcripts.values()
|
|
331
|
+
),
|
|
332
|
+
available_translation_languages=self._get_language_description(
|
|
333
|
+
'{language_code} ("{language}")'.format(
|
|
334
|
+
language=translation_language['language'],
|
|
335
|
+
language_code=translation_language['language_code'],
|
|
336
|
+
) for translation_language in self._translation_languages
|
|
337
|
+
)
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
def _get_language_description(self, transcript_strings):
|
|
341
|
+
description = '\n'.join(' - {transcript}'.format(transcript=transcript) for transcript in transcript_strings)
|
|
342
|
+
return description if description else 'None'
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
class Transcript(object):
|
|
346
|
+
def __init__(self, http_client, video_id, url, language, language_code, is_generated, translation_languages):
|
|
347
|
+
"""
|
|
348
|
+
You probably don't want to initialize this directly. Usually you'll access Transcript objects using a
|
|
349
|
+
TranscriptList.
|
|
350
|
+
|
|
351
|
+
:param http_client: http client which is used to make the transcript retrieving http calls
|
|
352
|
+
:type http_client: requests.Session
|
|
353
|
+
:param video_id: the id of the video this TranscriptList is for
|
|
354
|
+
:type video_id: str
|
|
355
|
+
:param url: the url which needs to be called to fetch the transcript
|
|
356
|
+
:param language: the name of the language this transcript uses
|
|
357
|
+
:param language_code:
|
|
358
|
+
:param is_generated:
|
|
359
|
+
:param translation_languages:
|
|
360
|
+
"""
|
|
361
|
+
self._http_client = http_client
|
|
362
|
+
self.video_id = video_id
|
|
363
|
+
self._url = url
|
|
364
|
+
self.language = language
|
|
365
|
+
self.language_code = language_code
|
|
366
|
+
self.is_generated = is_generated
|
|
367
|
+
self.translation_languages = translation_languages
|
|
368
|
+
self._translation_languages_dict = {
|
|
369
|
+
translation_language['language_code']: translation_language['language']
|
|
370
|
+
for translation_language in translation_languages
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
def fetch(self, preserve_formatting=False):
|
|
374
|
+
"""
|
|
375
|
+
Loads the actual transcript data.
|
|
376
|
+
:param preserve_formatting: whether to keep select HTML text formatting
|
|
377
|
+
:type preserve_formatting: bool
|
|
378
|
+
:return: a list of dictionaries containing the 'text', 'start' and 'duration' keys
|
|
379
|
+
:rtype [{'text': str, 'start': float, 'end': float}]:
|
|
380
|
+
"""
|
|
381
|
+
response = self._http_client.get(self._url, headers={'Accept-Language': 'en-US'})
|
|
382
|
+
return _TranscriptParser(preserve_formatting=preserve_formatting).parse(
|
|
383
|
+
_raise_http_errors(response, self.video_id).text,
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
def __str__(self):
|
|
387
|
+
return '{language_code} ("{language}"){translation_description}'.format(
|
|
388
|
+
language=self.language,
|
|
389
|
+
language_code=self.language_code,
|
|
390
|
+
translation_description='[TRANSLATABLE]' if self.is_translatable else ''
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
@property
|
|
394
|
+
def is_translatable(self):
|
|
395
|
+
return len(self.translation_languages) > 0
|
|
396
|
+
|
|
397
|
+
def translate(self, language_code):
|
|
398
|
+
if not self.is_translatable:
|
|
399
|
+
raise NotTranslatableError(self.video_id)
|
|
400
|
+
|
|
401
|
+
if language_code not in self._translation_languages_dict:
|
|
402
|
+
raise TranslationLanguageNotAvailableError(self.video_id)
|
|
403
|
+
|
|
404
|
+
return Transcript(
|
|
405
|
+
self._http_client,
|
|
406
|
+
self.video_id,
|
|
407
|
+
'{url}&tlang={language_code}'.format(url=self._url, language_code=language_code),
|
|
408
|
+
self._translation_languages_dict[language_code],
|
|
409
|
+
language_code,
|
|
410
|
+
True,
|
|
411
|
+
[],
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
class _TranscriptParser(object):
|
|
416
|
+
_FORMATTING_TAGS = [
|
|
417
|
+
'strong', # important
|
|
418
|
+
'em', # emphasized
|
|
419
|
+
'b', # bold
|
|
420
|
+
'i', # italic
|
|
421
|
+
'mark', # marked
|
|
422
|
+
'small', # smaller
|
|
423
|
+
'del', # deleted
|
|
424
|
+
'ins', # inserted
|
|
425
|
+
'sub', # subscript
|
|
426
|
+
'sup', # superscript
|
|
427
|
+
]
|
|
428
|
+
|
|
429
|
+
def __init__(self, preserve_formatting=False):
|
|
430
|
+
self._html_regex = self._get_html_regex(preserve_formatting)
|
|
431
|
+
|
|
432
|
+
def _get_html_regex(self, preserve_formatting):
|
|
433
|
+
if preserve_formatting:
|
|
434
|
+
formats_regex = '|'.join(self._FORMATTING_TAGS)
|
|
435
|
+
formats_regex = r'<\/?(?!\/?(' + formats_regex + r')\b).*?\b>'
|
|
436
|
+
html_regex = re.compile(formats_regex, re.IGNORECASE)
|
|
437
|
+
else:
|
|
438
|
+
html_regex = re.compile(r'<[^>]*>', re.IGNORECASE)
|
|
439
|
+
return html_regex
|
|
440
|
+
|
|
441
|
+
def parse(self, plain_data):
|
|
442
|
+
return [
|
|
443
|
+
{
|
|
444
|
+
'text': re.sub(self._html_regex, '', unescape(xml_element.text)),
|
|
445
|
+
'start': float(xml_element.attrib['start']),
|
|
446
|
+
'duration': float(xml_element.attrib.get('dur', '0.0')),
|
|
447
|
+
}
|
|
448
|
+
for xml_element in ElementTree.fromstring(plain_data)
|
|
449
|
+
if xml_element.text is not None
|
|
450
|
+
]
|
|
451
|
+
|
|
452
|
+
WATCH_URL = 'https://www.youtube.com/watch?v={video_id}'
|
|
453
|
+
|
|
454
|
+
class transcriber(object):
|
|
455
|
+
@classmethod
|
|
456
|
+
def list_transcripts(cls, video_id, proxies=None, cookies=None):
|
|
457
|
+
with requests.Session() as http_client:
|
|
458
|
+
if cookies:
|
|
459
|
+
http_client.cookies = cls._load_cookies(cookies, video_id)
|
|
460
|
+
http_client.proxies = proxies if proxies else {}
|
|
461
|
+
return TranscriptListFetcher(http_client).fetch(video_id)
|
|
462
|
+
|
|
463
|
+
@classmethod
|
|
464
|
+
def get_transcripts(cls, video_ids, languages=('en',), continue_after_error=False, proxies=None,
|
|
465
|
+
cookies=None, preserve_formatting=False):
|
|
466
|
+
|
|
467
|
+
assert isinstance(video_ids, list), "`video_ids` must be a list of strings"
|
|
468
|
+
|
|
469
|
+
data = {}
|
|
470
|
+
unretrievable_videos = []
|
|
471
|
+
|
|
472
|
+
for video_id in video_ids:
|
|
473
|
+
try:
|
|
474
|
+
data[video_id] = cls.get_transcript(video_id, languages, proxies, cookies, preserve_formatting)
|
|
475
|
+
except Exception as exception:
|
|
476
|
+
if not continue_after_error:
|
|
477
|
+
raise exception
|
|
478
|
+
|
|
479
|
+
unretrievable_videos.append(video_id)
|
|
480
|
+
|
|
481
|
+
return data, unretrievable_videos
|
|
482
|
+
|
|
483
|
+
@classmethod
|
|
484
|
+
def get_transcript(cls, video_id, languages=('en',), proxies=None, cookies=None, preserve_formatting=False):
|
|
485
|
+
assert isinstance(video_id, str), "`video_id` must be a string"
|
|
486
|
+
return cls.list_transcripts(video_id, proxies, cookies).find_transcript(languages).fetch(preserve_formatting=preserve_formatting)
|
|
487
|
+
|
|
488
|
+
@classmethod
|
|
489
|
+
def _load_cookies(cls, cookies, video_id):
|
|
490
|
+
try:
|
|
491
|
+
cookie_jar = cookiejar.MozillaCookieJar()
|
|
492
|
+
cookie_jar.load(cookies)
|
|
493
|
+
if not cookie_jar:
|
|
494
|
+
raise CookiesInvalidError(video_id)
|
|
495
|
+
return cookie_jar
|
|
496
|
+
except:
|
|
497
497
|
raise CookiePathInvalidError(video_id)
|
webscout/version.py
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
__version__ = "1.
|
|
1
|
+
__version__ = "1.3.0"
|
|
2
2
|
|
webscout/voice.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
import typing
|
|
3
|
+
|
|
4
|
+
def play_audio(message: str, voice: str = "Brian") -> typing.Union[str, typing.NoReturn]:
|
|
5
|
+
"""
|
|
6
|
+
Text to speech using StreamElements API
|
|
7
|
+
|
|
8
|
+
Parameters:
|
|
9
|
+
message (str): The text to convert to speech
|
|
10
|
+
voice (str): The voice to use for speech synthesis. Default is "Brian".
|
|
11
|
+
|
|
12
|
+
Returns:
|
|
13
|
+
result (Union[str, None]): Temporary file path or None in failure
|
|
14
|
+
"""
|
|
15
|
+
# Base URL for provider API
|
|
16
|
+
url: str = f"https://api.streamelements.com/kappa/v2/speech?voice={voice}&text={{{message}}}"
|
|
17
|
+
|
|
18
|
+
# Request headers
|
|
19
|
+
headers: typing.Dict[str, str] = {
|
|
20
|
+
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'
|
|
21
|
+
}
|
|
22
|
+
# Try to send request or return None on failure
|
|
23
|
+
try:
|
|
24
|
+
result = requests.get(url=url, headers=headers)
|
|
25
|
+
return result.content
|
|
26
|
+
except:
|
|
27
|
+
return None
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: webscout
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Search for words, documents, images, videos, news, maps and text translation using the Google, DuckDuckGo.com, yep.com, phind.com, you.com, etc Also containes AI models and now can transcribe yt videos
|
|
5
5
|
Author: OEvortex
|
|
6
6
|
Author-email: helpingai5@gmail.com
|
|
@@ -72,6 +72,9 @@ Search for words, documents, images, videos, news, maps and text translation usi
|
|
|
72
72
|
- [Activating DeepWEBS](#activating-deepwebs)
|
|
73
73
|
- [Point to remember before using `DeepWEBS`](#point-to-remember-before-using-deepwebs)
|
|
74
74
|
- [Usage Example](#usage-example)
|
|
75
|
+
- [Text-to-Speech:](#text-to-speech)
|
|
76
|
+
- [Available TTS Voices:](#available-tts-voices)
|
|
77
|
+
- [ALL voices:](#all-voices)
|
|
75
78
|
- [WEBS and AsyncWEBS classes](#webs-and-asyncwebs-classes)
|
|
76
79
|
- [Exceptions](#exceptions)
|
|
77
80
|
- [usage of webscout](#usage-of-webscout)
|
|
@@ -337,6 +340,23 @@ if __name__ == "__main__":
|
|
|
337
340
|
main()
|
|
338
341
|
|
|
339
342
|
```
|
|
343
|
+
## Text-to-Speech:
|
|
344
|
+
```python
|
|
345
|
+
from webscout import play_audio
|
|
346
|
+
|
|
347
|
+
message = "This is an example of text-to-speech."
|
|
348
|
+
audio_content = play_audio(message, voice="Brian")
|
|
349
|
+
|
|
350
|
+
# Save the audio to a file
|
|
351
|
+
with open("output.mp3", "wb") as f:
|
|
352
|
+
f.write(audio_content)
|
|
353
|
+
```
|
|
354
|
+
### Available TTS Voices:
|
|
355
|
+
You can choose from a wide range of voices, including:
|
|
356
|
+
- Filiz, Astrid, Tatyana, Maxim, Carmen, Ines, Cristiano, Vitoria, Ricardo, Maja, Jan, Jacek, Ewa, Ruben, Lotte, Liv, Seoyeon, Takumi, Mizuki, Giorgio, Carla, Bianca, Karl, Dora, Mathieu, Celine, Chantal, Penelope, Miguel, Mia, Enrique, Conchita, Geraint, Salli, Matthew, Kimberly, Kendra, Justin, Joey, Joanna, Ivy, Raveena, Aditi, Emma, Brian, Amy, Russell, Nicole, Vicki, Marlene, Hans, Naja, Mads, Gwyneth, Zhiyu
|
|
357
|
+
- Standard and WaveNet voices for various languages (e.g., en-US, es-ES, ja-JP, etc.)
|
|
358
|
+
### ALL voices:
|
|
359
|
+
[Filiz, Astrid, Tatyana, Maxim, Carmen, Ines, Cristiano, Vitoria, Ricardo, Maja, Jan, Jacek, Ewa, Ruben, Lotte, Liv, Seoyeon, Takumi, Mizuki, Giorgio, Carla, Bianca, Karl, Dora, Mathieu, Celine, Chantal, Penelope, Miguel, Mia, Enrique, Conchita, Geraint, Salli, Matthew, Kimberly, Kendra, Justin, Joey, Joanna, Ivy, Raveena, Aditi, Emma, Brian, Amy, Russell, Nicole, Vicki, Marlene, Hans, Naja, Mads, Gwyneth, Zhiyu, es-ES-Standard-A, it-IT-Standard-A, it-IT-Wavenet-A, ja-JP-Standard-A, ja-JP-Wavenet-A, ko-KR-Standard-A, ko-KR-Wavenet-A, pt-BR-Standard-A, tr-TR-Standard-A, sv-SE-Standard-A, nl-NL-Standard-A, nl-NL-Wavenet-A, en-US-Wavenet-A, en-US-Wavenet-B, en-US-Wavenet-C, en-US-Wavenet-D, en-US-Wavenet-E, en-US-Wavenet-F, en-GB-Standard-A, en-GB-Standard-B, en-GB-Standard-C, en-GB-Standard-D, en-GB-Wavenet-A, en-GB-Wavenet-B, en-GB-Wavenet-C, en-GB-Wavenet-D, en-US-Standard-B, en-US-Standard-C, en-US-Standard-D, en-US-Standard-E, de-DE-Standard-A, de-DE-Standard-B, de-DE-Wavenet-A, de-DE-Wavenet-B, de-DE-Wavenet-C, de-DE-Wavenet-D, en-AU-Standard-A, en-AU-Standard-B, en-AU-Wavenet-A, en-AU-Wavenet-B, en-AU-Wavenet-C, en-AU-Wavenet-D, en-AU-Standard-C, en-AU-Standard-D, fr-CA-Standard-A, fr-CA-Standard-B, fr-CA-Standard-C, fr-CA-Standard-D, fr-FR-Standard-C, fr-FR-Standard-D, fr-FR-Wavenet-A, fr-FR-Wavenet-B, fr-FR-Wavenet-C, fr-FR-Wavenet-D, da-DK-Wavenet-A, pl-PL-Wavenet-A, pl-PL-Wavenet-B, pl-PL-Wavenet-C, pl-PL-Wavenet-D, pt-PT-Wavenet-A, pt-PT-Wavenet-B, pt-PT-Wavenet-C, pt-PT-Wavenet-D, ru-RU-Wavenet-A, ru-RU-Wavenet-B, ru-RU-Wavenet-C, ru-RU-Wavenet-D, sk-SK-Wavenet-A, tr-TR-Wavenet-A, tr-TR-Wavenet-B, tr-TR-Wavenet-C, tr-TR-Wavenet-D, tr-TR-Wavenet-E, uk-UA-Wavenet-A, ar-XA-Wavenet-A, ar-XA-Wavenet-B, ar-XA-Wavenet-C, cs-CZ-Wavenet-A, nl-NL-Wavenet-B, nl-NL-Wavenet-C, nl-NL-Wavenet-D, nl-NL-Wavenet-E, en-IN-Wavenet-A, en-IN-Wavenet-B, en-IN-Wavenet-C, fil-PH-Wavenet-A, fi-FI-Wavenet-A, el-GR-Wavenet-A, hi-IN-Wavenet-A, hi-IN-Wavenet-B, hi-IN-Wavenet-C, hu-HU-Wavenet-A, id-ID-Wavenet-A, id-ID-Wavenet-B, id-ID-Wavenet-C, it-IT-Wavenet-B, it-IT-Wavenet-C, it-IT-Wavenet-D, ja-JP-Wavenet-B, ja-JP-Wavenet-C, ja-JP-Wavenet-D, cmn-CN-Wavenet-A, cmn-CN-Wavenet-B, cmn-CN-Wavenet-C, cmn-CN-Wavenet-D, nb-no-Wavenet-E, nb-no-Wavenet-A, nb-no-Wavenet-B, nb-no-Wavenet-C, nb-no-Wavenet-D, vi-VN-Wavenet-A, vi-VN-Wavenet-B, vi-VN-Wavenet-C, vi-VN-Wavenet-D, sr-rs-Standard-A, lv-lv-Standard-A, is-is-Standard-A, bg-bg-Standard-A, af-ZA-Standard-A, Tracy, Danny, Huihui, Yaoyao, Kangkang, HanHan, Zhiwei, Asaf, An, Stefanos, Filip, Ivan, Heidi, Herena, Kalpana, Hemant, Matej, Andika, Rizwan, Lado, Valluvar, Linda, Heather, Sean, Michael, Karsten, Guillaume, Pattara, Jakub, Szabolcs, Hoda, Naayf]
|
|
340
360
|
## WEBS and AsyncWEBS classes
|
|
341
361
|
|
|
342
362
|
The WEBS and AsyncWEBS classes are used to retrieve search results from DuckDuckGo.com and yep.com periodically.
|
|
@@ -357,11 +377,12 @@ import logging
|
|
|
357
377
|
import sys
|
|
358
378
|
from itertools import chain
|
|
359
379
|
from random import shuffle
|
|
360
|
-
|
|
361
380
|
import requests
|
|
362
381
|
from webscout import AsyncWEBS
|
|
363
382
|
|
|
364
|
-
#
|
|
383
|
+
# If you have proxies, define them here
|
|
384
|
+
proxies = None
|
|
385
|
+
|
|
365
386
|
if sys.platform.lower().startswith("win"):
|
|
366
387
|
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
|
367
388
|
|
|
@@ -373,24 +394,21 @@ def get_words():
|
|
|
373
394
|
|
|
374
395
|
async def aget_results(word):
|
|
375
396
|
async with AsyncWEBS(proxies=proxies) as WEBS:
|
|
376
|
-
results =
|
|
397
|
+
results = await WEBS.text(word, max_results=None)
|
|
377
398
|
return results
|
|
378
399
|
|
|
379
400
|
async def main():
|
|
380
401
|
words = get_words()
|
|
381
402
|
shuffle(words)
|
|
382
|
-
tasks = []
|
|
383
|
-
for word in words[:10]:
|
|
384
|
-
tasks.append(aget_results(word))
|
|
403
|
+
tasks = [aget_results(word) for word in words[:10]]
|
|
385
404
|
results = await asyncio.gather(*tasks)
|
|
386
405
|
print(f"Done")
|
|
387
406
|
for r in chain.from_iterable(results):
|
|
388
407
|
print(r)
|
|
389
|
-
|
|
390
408
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
409
|
+
logging.basicConfig(level=logging.DEBUG)
|
|
410
|
+
|
|
411
|
+
await main()
|
|
394
412
|
```
|
|
395
413
|
It is important to note that the WEBS and AsyncWEBS classes should always be used as a context manager (with statement).
|
|
396
414
|
This ensures proper resource management and cleanup, as the context manager will automatically handle opening and closing the HTTP client connection.
|
|
@@ -16,19 +16,20 @@ webscout/AIutel.py,sha256=fNN4mmjXcxjJGq2CVJP1MU2oQ78p8OyExQBjVif6e-k,24123
|
|
|
16
16
|
webscout/DWEBS.py,sha256=QT-7-dUgWhQ_H7EVZD53AVyXxyskoPMKCkFIpzkN56Q,7332
|
|
17
17
|
webscout/HelpingAI.py,sha256=YeZw0zYVHMcBFFPNdd3_Ghpm9ebt_EScQjHO_IIs4lg,8103
|
|
18
18
|
webscout/LLM.py,sha256=XByJPiATLA_57FBWKw18Xx_PGRCPOj-GJE96aQH1k2Y,3309
|
|
19
|
-
webscout/__init__.py,sha256=
|
|
19
|
+
webscout/__init__.py,sha256=1DBQX84kGCCzGNFV2MtNqASb_2WlM7BuM0enCqnOx8I,550
|
|
20
20
|
webscout/__main__.py,sha256=ZtTRgsRjUi2JOvYFLF1ZCh55Sdoz94I-BS-TlJC7WDU,126
|
|
21
21
|
webscout/cli.py,sha256=F888fdrFUQgczMBN4yMOSf6Nh-IbvkqpPhDsbnA2FtQ,17059
|
|
22
22
|
webscout/exceptions.py,sha256=4AOO5wexeL96nvUS-badcckcwrPS7UpZyAgB9vknHZE,276
|
|
23
23
|
webscout/models.py,sha256=5iQIdtedT18YuTZ3npoG7kLMwcrKwhQ7928dl_7qZW0,692
|
|
24
|
-
webscout/transcriber.py,sha256=
|
|
24
|
+
webscout/transcriber.py,sha256=EddvTSq7dPJ42V3pQVnGuEiYQ7WjJ9uyeR9kMSxN7uY,20622
|
|
25
25
|
webscout/utils.py,sha256=c_98M4oqpb54pUun3fpGGlCerFD6ZHUbghyp5b7Mwgo,2605
|
|
26
|
-
webscout/version.py,sha256=
|
|
26
|
+
webscout/version.py,sha256=R-_i2z0hXcrgq3LLNeJVLbwdS54wc3y_4KzSPxzw8j0,25
|
|
27
|
+
webscout/voice.py,sha256=1Ids_2ToPBMX0cH_UyPMkY_6eSE9H4Gazrl0ujPmFag,941
|
|
27
28
|
webscout/webscout_search.py,sha256=3_lli-hDb8_kCGwscK29xuUcOS833ROgpNhDzrxh0dk,3085
|
|
28
29
|
webscout/webscout_search_async.py,sha256=Y5frH0k3hLqBCR-8dn7a_b7EvxdYxn6wHiKl3jWosE0,40670
|
|
29
|
-
webscout-1.
|
|
30
|
-
webscout-1.
|
|
31
|
-
webscout-1.
|
|
32
|
-
webscout-1.
|
|
33
|
-
webscout-1.
|
|
34
|
-
webscout-1.
|
|
30
|
+
webscout-1.3.0.dist-info/LICENSE.md,sha256=mRVwJuT4SXC5O93BFdsfWBjlXjGn2Np90Zm5SocUzM0,3150
|
|
31
|
+
webscout-1.3.0.dist-info/METADATA,sha256=E9A3XzQIkmBA6C6E0PXYyFxdqqPCH2PHYSVSKUJczcc,28258
|
|
32
|
+
webscout-1.3.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
33
|
+
webscout-1.3.0.dist-info/entry_points.txt,sha256=8-93eRslYrzTHs5E-6yFRJrve00C9q-SkXJD113jzRY,197
|
|
34
|
+
webscout-1.3.0.dist-info/top_level.txt,sha256=OD5YKy6Y3hldL7SmuxsiEDxAG4LgdSSWwzYk22MF9fk,18
|
|
35
|
+
webscout-1.3.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|