warp-beacon 2.0.5__py3-none-any.whl → 2.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etc/warp_beacon/warp_beacon.conf +2 -1
- warp_beacon/__version__.py +1 -1
- warp_beacon/jobs/abstract.py +4 -0
- warp_beacon/mediainfo/silencer.py +1 -1
- warp_beacon/scraper/__init__.py +32 -4
- warp_beacon/scraper/abstract.py +3 -0
- warp_beacon/scraper/exceptions.py +6 -0
- warp_beacon/scraper/instagram.py +1 -0
- warp_beacon/scraper/youtube/abstract.py +78 -3
- warp_beacon/scraper/youtube/music.py +30 -24
- warp_beacon/scraper/youtube/shorts.py +23 -19
- warp_beacon/scraper/youtube/youtube.py +37 -6
- warp_beacon/telegram/bot.py +29 -3
- warp_beacon/telegram/handlers.py +11 -2
- warp_beacon/uploader/__init__.py +7 -1
- {warp_beacon-2.0.5.dist-info → warp_beacon-2.0.7.dist-info}/METADATA +1 -1
- {warp_beacon-2.0.5.dist-info → warp_beacon-2.0.7.dist-info}/RECORD +21 -21
- {warp_beacon-2.0.5.dist-info → warp_beacon-2.0.7.dist-info}/LICENSE +0 -0
- {warp_beacon-2.0.5.dist-info → warp_beacon-2.0.7.dist-info}/WHEEL +0 -0
- {warp_beacon-2.0.5.dist-info → warp_beacon-2.0.7.dist-info}/entry_points.txt +0 -0
- {warp_beacon-2.0.5.dist-info → warp_beacon-2.0.7.dist-info}/top_level.txt +0 -0
etc/warp_beacon/warp_beacon.conf
CHANGED
@@ -2,6 +2,7 @@ TG_TOKEN=""
|
|
2
2
|
TG_API_ID=""
|
3
3
|
TG_API_HASH=""
|
4
4
|
TG_BOT_NAME=""
|
5
|
+
TG_BOT_ADMIN_USERNAME=""
|
5
6
|
INSTAGRAM_LOGIN=""
|
6
7
|
INSTAGRAM_PASSWORD=""
|
7
8
|
INSTAGRAM_VERIFICATION_CODE=""
|
@@ -10,4 +11,4 @@ MONGODB_PORT="27017"
|
|
10
11
|
MONGODB_USER="root"
|
11
12
|
MONGODB_PASSWORD="changeme"
|
12
13
|
VIDEO_STORAGE_DIR="/var/warp_beacon/videos"
|
13
|
-
ENABLE_DONATES=true
|
14
|
+
ENABLE_DONATES=true
|
warp_beacon/__version__.py
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
__version__ = "2.0.
|
1
|
+
__version__ = "2.0.7"
|
2
2
|
|
warp_beacon/jobs/abstract.py
CHANGED
@@ -29,6 +29,8 @@ class JobSettings(TypedDict):
|
|
29
29
|
media_collection: list
|
30
30
|
job_origin: Origin
|
31
31
|
canonical_name: str
|
32
|
+
is_message_to_admin: bool
|
33
|
+
message_text: str
|
32
34
|
|
33
35
|
class AbstractJob(ABC):
|
34
36
|
job_id: uuid.UUID = None
|
@@ -52,6 +54,8 @@ class AbstractJob(ABC):
|
|
52
54
|
media_collection: list = []
|
53
55
|
job_origin: Origin = Origin.UNKNOWN
|
54
56
|
canonical_name: str = ""
|
57
|
+
is_message_to_admin: bool = False
|
58
|
+
message_text: str = ""
|
55
59
|
|
56
60
|
def __init__(self, **kwargs: Unpack[JobSettings]) -> None:
|
57
61
|
if kwargs:
|
warp_beacon/scraper/__init__.py
CHANGED
@@ -1,11 +1,12 @@
|
|
1
1
|
import os
|
2
2
|
import time
|
3
|
+
import asyncio
|
3
4
|
|
4
5
|
from typing import Optional
|
5
6
|
import multiprocessing
|
6
7
|
from queue import Empty
|
7
8
|
|
8
|
-
from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig
|
9
|
+
from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YotubeLiveError, YotubeAgeRestrictedError
|
9
10
|
from warp_beacon.mediainfo.video import VideoInfo
|
10
11
|
from warp_beacon.mediainfo.audio import AudioInfo
|
11
12
|
from warp_beacon.mediainfo.silencer import Silencer
|
@@ -13,6 +14,7 @@ from warp_beacon.compress.video import VideoCompress
|
|
13
14
|
from warp_beacon.uploader import AsyncUploader
|
14
15
|
from warp_beacon.jobs import Origin
|
15
16
|
from warp_beacon.jobs.download_job import DownloadJob
|
17
|
+
from warp_beacon.jobs.upload_job import UploadJob
|
16
18
|
from warp_beacon.jobs.types import JobType
|
17
19
|
|
18
20
|
import logging
|
@@ -24,6 +26,7 @@ class AsyncDownloader(object):
|
|
24
26
|
job_queue = multiprocessing.Queue()
|
25
27
|
uploader = None
|
26
28
|
workers_count = 0
|
29
|
+
auth_event = multiprocessing.Event()
|
27
30
|
|
28
31
|
def __init__(self, uploader: AsyncUploader, workers_count: int) -> None:
|
29
32
|
self.allow_loop = multiprocessing.Value('i', 1)
|
@@ -86,6 +89,9 @@ class AsyncDownloader(object):
|
|
86
89
|
elif job.job_origin is Origin.YOUTUBE:
|
87
90
|
from warp_beacon.scraper.youtube.youtube import YoutubeScraper
|
88
91
|
actor = YoutubeScraper()
|
92
|
+
#self.auth_event = multiprocessing.Event()
|
93
|
+
actor.send_message_to_admin_func = self.send_message_to_admin
|
94
|
+
actor.auth_event = self.auth_event
|
89
95
|
while True:
|
90
96
|
try:
|
91
97
|
logging.info("Downloading URL '%s'", job.url)
|
@@ -115,6 +121,22 @@ class AsyncDownloader(object):
|
|
115
121
|
job_failed_msg="Unfortunately this file has exceeded the Telegram limits. A file cannot be larger than 2 gigabytes.")
|
116
122
|
)
|
117
123
|
break
|
124
|
+
except YotubeLiveError as e:
|
125
|
+
logging.warning("Youtube Live videos are not supported. Skipping.")
|
126
|
+
logging.exception(e)
|
127
|
+
self.uploader.queue_task(job.to_upload_job(
|
128
|
+
job_failed=True,
|
129
|
+
job_failed_msg="Youtube Live videos are not supported. Please wait until the live broadcast ends.")
|
130
|
+
)
|
131
|
+
break
|
132
|
+
except YotubeAgeRestrictedError as e:
|
133
|
+
logging.error("Youtube Age Restricted error")
|
134
|
+
logging.exception(e)
|
135
|
+
self.uploader.queue_task(job.to_upload_job(
|
136
|
+
job_failed=True,
|
137
|
+
job_failed_msg="Youtube Age Restricted error. Check your bot Youtube account settings.")
|
138
|
+
)
|
139
|
+
break
|
118
140
|
except (UnknownError, Exception) as e:
|
119
141
|
logging.warning("UnknownError occurred!")
|
120
142
|
logging.exception(e)
|
@@ -131,7 +153,7 @@ class AsyncDownloader(object):
|
|
131
153
|
break
|
132
154
|
self.uploader.queue_task(job.to_upload_job(
|
133
155
|
job_failed=True,
|
134
|
-
job_failed_msg="WOW, unknown error occured! Please
|
156
|
+
job_failed_msg="WOW, unknown error occured! Please [create issue](https://github.com/sb0y/warp_beacon/issues) with service logs.")
|
135
157
|
)
|
136
158
|
break
|
137
159
|
|
@@ -174,7 +196,7 @@ class AsyncDownloader(object):
|
|
174
196
|
if not v["media_info"]["has_sound"]:
|
175
197
|
silencer = Silencer(v["local_media_path"])
|
176
198
|
silent_video_path = silencer.add_silent_audio()
|
177
|
-
os.unlink(
|
199
|
+
os.unlink(v["local_media_path"])
|
178
200
|
v["local_media_path"] = silent_video_path
|
179
201
|
v["media_info"].update(silencer.get_finfo())
|
180
202
|
v["media_info"]["has_sound"] = True
|
@@ -234,4 +256,10 @@ class AsyncDownloader(object):
|
|
234
256
|
return str(job.job_id)
|
235
257
|
|
236
258
|
def notify_task_failed(self, job: DownloadJob) -> None:
|
237
|
-
self.uploader.queue_task(job.to_upload_job(job_failed=True))
|
259
|
+
self.uploader.queue_task(job.to_upload_job(job_failed=True))
|
260
|
+
|
261
|
+
def send_message_to_admin(self, text: str) -> None:
|
262
|
+
self.uploader.queue_task(UploadJob.build(
|
263
|
+
is_message_to_admin=True,
|
264
|
+
message_text=text
|
265
|
+
))
|
warp_beacon/scraper/abstract.py
CHANGED
warp_beacon/scraper/instagram.py
CHANGED
@@ -145,6 +145,7 @@ class InstagramScraper(ScraperAbstract):
|
|
145
145
|
effective_url = "https://www.instagram.com/stories/%s/%s/" % (story_info.user.username, effective_story_id)
|
146
146
|
if story_info.media_type == 1: # photo
|
147
147
|
path = str(self._download_hndlr(self.cl.story_download_by_url, url=story_info.thumbnail_url, folder='/tmp'))
|
148
|
+
path_lowered = path.lower()
|
148
149
|
if ".webp" in path_lowered:
|
149
150
|
path = InstagramScraper.convert_webp_to_png(path)
|
150
151
|
if ".heic" in path_lowered:
|
@@ -7,6 +7,8 @@ import ssl
|
|
7
7
|
from abc import abstractmethod
|
8
8
|
from typing import Callable, Union
|
9
9
|
|
10
|
+
import json
|
11
|
+
|
10
12
|
import requests
|
11
13
|
import urllib
|
12
14
|
import http.client
|
@@ -17,12 +19,66 @@ from warp_beacon.scraper.abstract import ScraperAbstract
|
|
17
19
|
from warp_beacon.mediainfo.abstract import MediaInfoAbstract
|
18
20
|
from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, extract_exception_message
|
19
21
|
|
22
|
+
from pytubefix import YouTube
|
23
|
+
from pytubefix.innertube import _default_clients
|
24
|
+
from pytubefix.streams import Stream
|
25
|
+
from pytubefix.innertube import InnerTube, _client_id, _client_secret
|
20
26
|
from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
|
27
|
+
from pytubefix import request
|
21
28
|
|
22
29
|
import logging
|
23
30
|
|
31
|
+
def patched_fetch_bearer_token(self) -> None:
|
32
|
+
"""Fetch an OAuth token."""
|
33
|
+
# Subtracting 30 seconds is arbitrary to avoid potential time discrepencies
|
34
|
+
start_time = int(time.time() - 30)
|
35
|
+
data = {
|
36
|
+
'client_id': _client_id,
|
37
|
+
'scope': 'https://www.googleapis.com/auth/youtube'
|
38
|
+
}
|
39
|
+
response = request._execute_request(
|
40
|
+
'https://oauth2.googleapis.com/device/code',
|
41
|
+
'POST',
|
42
|
+
headers={
|
43
|
+
'Content-Type': 'application/json'
|
44
|
+
},
|
45
|
+
data=data
|
46
|
+
)
|
47
|
+
response_data = json.loads(response.read())
|
48
|
+
verification_url = response_data['verification_url']
|
49
|
+
user_code = response_data['user_code']
|
50
|
+
|
51
|
+
logging.warning("Please open %s and input code '%s'", verification_url, user_code)
|
52
|
+
self.send_message_to_admin_func(
|
53
|
+
f"Please open {verification_url} and input code `{user_code}`.\n\n"
|
54
|
+
"Please select a Google account with verified age.\n"
|
55
|
+
"This will allow you to avoid error the **AgeRestrictedError** when accessing some content.")
|
56
|
+
self.auth_event.wait()
|
57
|
+
|
58
|
+
data = {
|
59
|
+
'client_id': _client_id,
|
60
|
+
'client_secret': _client_secret,
|
61
|
+
'device_code': response_data['device_code'],
|
62
|
+
'grant_type': 'urn:ietf:params:oauth:grant-type:device_code'
|
63
|
+
}
|
64
|
+
response = request._execute_request(
|
65
|
+
'https://oauth2.googleapis.com/token',
|
66
|
+
'POST',
|
67
|
+
headers={
|
68
|
+
'Content-Type': 'application/json'
|
69
|
+
},
|
70
|
+
data=data
|
71
|
+
)
|
72
|
+
response_data = json.loads(response.read())
|
73
|
+
|
74
|
+
self.access_token = response_data['access_token']
|
75
|
+
self.refresh_token = response_data['refresh_token']
|
76
|
+
self.expires = start_time + response_data['expires_in']
|
77
|
+
self.cache_tokens()
|
78
|
+
|
24
79
|
class YoutubeAbstract(ScraperAbstract):
|
25
80
|
DOWNLOAD_DIR = "/tmp"
|
81
|
+
YT_SESSION_FILE = '/var/warp_beacon/yt_session.json'
|
26
82
|
|
27
83
|
def __init__(self) -> None:
|
28
84
|
pass
|
@@ -49,9 +105,9 @@ class YoutubeAbstract(ScraperAbstract):
|
|
49
105
|
if "yt_download_" in i:
|
50
106
|
os.unlink("%s/%s" % (self.DOWNLOAD_DIR, i))
|
51
107
|
|
52
|
-
def download_thumbnail(self, url: str) -> Union[io.BytesIO, None]:
|
108
|
+
def download_thumbnail(self, url: str, timeout: int) -> Union[io.BytesIO, None]:
|
53
109
|
try:
|
54
|
-
reply = requests.get(url,
|
110
|
+
reply = requests.get(url, timeout=(timeout, timeout))
|
55
111
|
if reply.ok and reply.status_code == 200:
|
56
112
|
image = Image.open(io.BytesIO(reply.content))
|
57
113
|
image = MediaInfoAbstract.shrink_image_to_fit(image)
|
@@ -65,7 +121,7 @@ class YoutubeAbstract(ScraperAbstract):
|
|
65
121
|
|
66
122
|
return None
|
67
123
|
|
68
|
-
def _download_hndlr(self, func: Callable, *args: tuple[str], **kwargs: dict[str]) -> Union[str, dict]:
|
124
|
+
def _download_hndlr(self, func: Callable, *args: tuple[str], **kwargs: dict[str]) -> Union[str, dict, io.BytesIO]:
|
69
125
|
ret_val = ''
|
70
126
|
max_retries = int(os.environ.get("YT_MAX_RETRIES", default=self.YT_MAX_RETRIES_DEFAULT))
|
71
127
|
pause_secs = int(os.environ.get("YT_PAUSE_BEFORE_RETRY", default=self.YT_PAUSE_BEFORE_RETRY_DEFAULT))
|
@@ -103,3 +159,22 @@ class YoutubeAbstract(ScraperAbstract):
|
|
103
159
|
raise Unavailable(extract_exception_message(e))
|
104
160
|
|
105
161
|
return ret_val
|
162
|
+
|
163
|
+
def yt_on_progress(self, stream: Stream, chunk: bytes, bytes_remaining: int) -> None:
|
164
|
+
pass
|
165
|
+
#logging.info("bytes: %d, bytes remaining: %d", chunk, bytes_remaining)
|
166
|
+
|
167
|
+
def build_yt(self, url: str) -> YouTube:
|
168
|
+
InnerTube.send_message_to_admin_func = self.send_message_to_admin_func
|
169
|
+
InnerTube.auth_event = self.auth_event
|
170
|
+
InnerTube.fetch_bearer_token = patched_fetch_bearer_token
|
171
|
+
_default_clients["ANDROID"]["innertube_context"]["context"]["client"]["clientVersion"] = "19.08.35"
|
172
|
+
_default_clients["ANDROID_MUSIC"] = _default_clients["ANDROID"]
|
173
|
+
return YouTube(
|
174
|
+
url=url,
|
175
|
+
#client="WEB",
|
176
|
+
use_oauth=True,
|
177
|
+
allow_oauth_cache=True,
|
178
|
+
token_file=self.YT_SESSION_FILE,
|
179
|
+
on_progress_callback = self.yt_on_progress
|
180
|
+
)
|
@@ -1,5 +1,6 @@
|
|
1
1
|
from warp_beacon.jobs.types import JobType
|
2
2
|
from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
|
3
|
+
from warp_beacon.scraper.exceptions import NotFound
|
3
4
|
|
4
5
|
from pytubefix import YouTube
|
5
6
|
|
@@ -14,32 +15,37 @@ class YoutubeMusicScraper(YoutubeAbstract):
|
|
14
15
|
def _download(self, url: str, timeout: int = 0) -> list:
|
15
16
|
res = []
|
16
17
|
thumbnail = None
|
17
|
-
yt =
|
18
|
+
yt = self.build_yt(url)
|
19
|
+
|
18
20
|
if yt and yt.thumbnail_url:
|
19
|
-
thumbnail = self.download_thumbnail
|
21
|
+
thumbnail = self._download_hndlr(self.download_thumbnail, yt.thumbnail_url)
|
22
|
+
|
20
23
|
stream = yt.streams.get_audio_only()
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
24
|
+
|
25
|
+
if not stream:
|
26
|
+
raise NotFound("No suitable audio stream found")
|
27
|
+
|
28
|
+
logging.info("Announced audio file size: '%d'", stream.filesize)
|
29
|
+
if stream.filesize > 2e+9:
|
30
|
+
logging.warning("Downloading size reported by YouTube is over than 2 GB!")
|
31
|
+
raise FileTooBig("YouTube file is larger than 2 GB")
|
32
|
+
logging.info("Operation timeout is '%d'", timeout)
|
33
|
+
local_file = stream.download(
|
34
|
+
output_path=self.DOWNLOAD_DIR,
|
35
|
+
max_retries=0,
|
36
|
+
timeout=timeout,
|
37
|
+
skip_existing=False,
|
38
|
+
filename_prefix='yt_download_',
|
39
|
+
mp3=True
|
40
|
+
)
|
41
|
+
logging.debug("Temp filename: '%s'", local_file)
|
42
|
+
res.append({
|
43
|
+
"local_media_path": self.rename_local_file(local_file),
|
44
|
+
"performer": yt.author,
|
45
|
+
"thumb": thumbnail,
|
46
|
+
"canonical_name": stream.title,
|
47
|
+
"media_type": JobType.AUDIO
|
48
|
+
})
|
43
49
|
|
44
50
|
return res
|
45
51
|
|
@@ -1,5 +1,6 @@
|
|
1
1
|
from warp_beacon.jobs.types import JobType
|
2
2
|
from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
|
3
|
+
from warp_beacon.scraper.exceptions import NotFound
|
3
4
|
|
4
5
|
from pytubefix import YouTube
|
5
6
|
|
@@ -14,27 +15,30 @@ class YoutubeShortsScraper(YoutubeAbstract):
|
|
14
15
|
def _download(self, url: str, timeout: int = 0) -> list:
|
15
16
|
res = []
|
16
17
|
thumbnail = None
|
17
|
-
yt =
|
18
|
+
yt = self.build_yt(url)
|
18
19
|
stream = yt.streams.get_highest_resolution()
|
20
|
+
|
21
|
+
if not stream:
|
22
|
+
raise NotFound("No suitable video stream found")
|
23
|
+
|
19
24
|
if yt and yt.thumbnail_url:
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
})
|
25
|
+
thumbnail = self._download_hndlr(self.download_thumbnail, yt.thumbnail_url)
|
26
|
+
|
27
|
+
local_file = stream.download(
|
28
|
+
output_path=self.DOWNLOAD_DIR,
|
29
|
+
max_retries=0,
|
30
|
+
timeout=timeout,
|
31
|
+
skip_existing=False,
|
32
|
+
filename_prefix="yt_download_"
|
33
|
+
)
|
34
|
+
logging.debug("Temp filename: '%s'", local_file)
|
35
|
+
res.append({
|
36
|
+
"local_media_path": self.rename_local_file(local_file),
|
37
|
+
"performer": yt.author,
|
38
|
+
"thumb": thumbnail,
|
39
|
+
"canonical_name": stream.title,
|
40
|
+
"media_type": JobType.VIDEO
|
41
|
+
})
|
38
42
|
|
39
43
|
return res
|
40
44
|
|
@@ -1,7 +1,9 @@
|
|
1
1
|
from warp_beacon.jobs.types import JobType
|
2
2
|
from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
|
3
|
+
from warp_beacon.scraper.exceptions import YotubeLiveError, NotFound, YotubeAgeRestrictedError
|
3
4
|
|
4
5
|
from pytubefix import YouTube
|
6
|
+
from pytubefix.exceptions import AgeRestrictedError
|
5
7
|
|
6
8
|
import logging
|
7
9
|
|
@@ -11,14 +13,41 @@ class YoutubeScraper(YoutubeAbstract):
|
|
11
13
|
YT_TIMEOUT_DEFAULT = 2
|
12
14
|
YT_TIMEOUT_INCREMENT_DEFAULT = 60
|
13
15
|
|
16
|
+
def is_live(self, data: dict) -> bool:
|
17
|
+
'''
|
18
|
+
x.contents.twoColumnWatchNextResults.results.results.contents[0].videoPrimaryInfoRenderer.viewCount.videoViewCountRenderer.isLive
|
19
|
+
'''
|
20
|
+
try:
|
21
|
+
contents = data.get("contents", {}).get("twoColumnWatchNextResults", {}).get("results", {}).get("results", {}).get("contents", [])
|
22
|
+
for i in contents:
|
23
|
+
video_view_count_renderer = i.get("videoPrimaryInfoRenderer", {}).get("viewCount", {}).get("videoViewCountRenderer", {})
|
24
|
+
if video_view_count_renderer:
|
25
|
+
return video_view_count_renderer.get("isLive", False)
|
26
|
+
except Exception as e:
|
27
|
+
logging.warning("Failed to check if stream is live!")
|
28
|
+
logging.exception(e)
|
29
|
+
|
30
|
+
return False
|
31
|
+
|
14
32
|
def _download(self, url: str, timeout: int = 0) -> list:
|
15
33
|
res = []
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
34
|
+
try:
|
35
|
+
thumbnail = None
|
36
|
+
yt = self.build_yt(url)
|
37
|
+
|
38
|
+
if self.is_live(yt.initial_data):
|
39
|
+
raise YotubeLiveError("Youtube Live is not supported")
|
40
|
+
|
41
|
+
if yt and yt.thumbnail_url:
|
42
|
+
thumbnail = self._download_hndlr(self.download_thumbnail, yt.thumbnail_url)
|
43
|
+
|
44
|
+
stream = yt.streams.get_highest_resolution()
|
45
|
+
|
46
|
+
if not stream:
|
47
|
+
raise NotFound("No suitable video stream found")
|
48
|
+
|
49
|
+
logging.info("Starting download ...")
|
50
|
+
|
22
51
|
local_file = stream.download(
|
23
52
|
output_path=self.DOWNLOAD_DIR,
|
24
53
|
max_retries=0,
|
@@ -34,6 +63,8 @@ class YoutubeScraper(YoutubeAbstract):
|
|
34
63
|
"canonical_name": stream.title,
|
35
64
|
"media_type": JobType.VIDEO
|
36
65
|
})
|
66
|
+
except AgeRestrictedError as e:
|
67
|
+
raise YotubeAgeRestrictedError("Youtube Age Restricted error")
|
37
68
|
|
38
69
|
return res
|
39
70
|
|
warp_beacon/telegram/bot.py
CHANGED
@@ -6,7 +6,7 @@ import asyncio
|
|
6
6
|
|
7
7
|
from pyrogram import Client, filters
|
8
8
|
from pyrogram.enums import ParseMode
|
9
|
-
from pyrogram.handlers import MessageHandler
|
9
|
+
from pyrogram.handlers import MessageHandler, CallbackQueryHandler
|
10
10
|
from pyrogram.types import Message, InputMedia, InputMediaAudio, InputMediaPhoto, InputMediaVideo, InputMediaAnimation, InputMediaDocument, InlineKeyboardButton, InlineKeyboardMarkup
|
11
11
|
from pyrogram.errors import RPCError, FloodWait, NetworkMigrate, BadRequest, MultiMediaTooLong, MessageIdInvalid
|
12
12
|
|
@@ -64,6 +64,7 @@ class Bot(object):
|
|
64
64
|
|
65
65
|
self.uploader = AsyncUploader(
|
66
66
|
storage=self.storage,
|
67
|
+
admin_message_callback=self.send_text_to_admin,
|
67
68
|
pool_size=int(os.environ.get("UPLOAD_POOL_SIZE", default=workers_amount)),
|
68
69
|
loop=self.client.loop
|
69
70
|
)
|
@@ -81,6 +82,7 @@ class Bot(object):
|
|
81
82
|
self.client.add_handler(MessageHandler(self.handlers.help, filters.command("help")))
|
82
83
|
self.client.add_handler(MessageHandler(self.handlers.random, filters.command("random")))
|
83
84
|
self.client.add_handler(MessageHandler(self.handlers.handler))
|
85
|
+
self.client.add_handler(CallbackQueryHandler(self.handlers.simple_button_handler))
|
84
86
|
|
85
87
|
self.placeholder = PlaceholderMessage(self)
|
86
88
|
|
@@ -114,6 +116,30 @@ class Bot(object):
|
|
114
116
|
|
115
117
|
return 0
|
116
118
|
|
119
|
+
async def send_text_to_admin(self, text: str) -> int:
|
120
|
+
try:
|
121
|
+
admin = os.environ.get("TG_BOT_ADMIN_USERNAME", None)
|
122
|
+
if not admin:
|
123
|
+
raise ValueError("Configuration value `TG_BOT_ADMIN_USERNAME` is empty!")
|
124
|
+
message_reply = await self.client.send_message(
|
125
|
+
chat_id=admin,
|
126
|
+
text=text,
|
127
|
+
parse_mode=ParseMode.MARKDOWN,
|
128
|
+
reply_markup=InlineKeyboardMarkup(
|
129
|
+
[
|
130
|
+
[
|
131
|
+
InlineKeyboardButton("✅ Done", callback_data="auth_process_done")
|
132
|
+
]
|
133
|
+
]
|
134
|
+
)
|
135
|
+
)
|
136
|
+
return message_reply.id
|
137
|
+
except Exception as e:
|
138
|
+
logging.error("Failed to send text message to admin!")
|
139
|
+
logging.exception(e)
|
140
|
+
|
141
|
+
return 0
|
142
|
+
|
117
143
|
def build_tg_args(self, job: UploadJob) -> dict:
|
118
144
|
args = {}
|
119
145
|
if job.media_type == JobType.VIDEO:
|
@@ -342,7 +368,7 @@ class Bot(object):
|
|
342
368
|
except Exception as e:
|
343
369
|
logging.error("Error occurred!")
|
344
370
|
logging.exception(e)
|
345
|
-
finally:
|
346
|
-
job.remove_files()
|
371
|
+
#finally:
|
372
|
+
#job.remove_files()
|
347
373
|
|
348
374
|
return tg_file_ids
|
warp_beacon/telegram/handlers.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
from pyrogram import Client
|
2
|
-
from pyrogram.types import Message
|
2
|
+
from pyrogram.types import Message, CallbackQuery
|
3
3
|
from pyrogram.enums import ChatType, ParseMode
|
4
4
|
from pyrogram.types import Chat, BotCommand
|
5
5
|
|
@@ -165,4 +165,13 @@ class Handlers(object):
|
|
165
165
|
logging.exception(e)
|
166
166
|
|
167
167
|
if chat.type not in (ChatType.GROUP, ChatType.SUPERGROUP) and not urls:
|
168
|
-
await self.bot.send_text(rext=reply_text, reply_id=effective_message_id)
|
168
|
+
await self.bot.send_text(rext=reply_text, reply_id=effective_message_id)
|
169
|
+
|
170
|
+
async def simple_button_handler(self, client: Client, query: CallbackQuery) -> None:
|
171
|
+
await client.answer_callback_query(
|
172
|
+
callback_query_id=query.id,
|
173
|
+
text="Please wait, bot will try to download media with obtained credentials.\nIf authorization is not successful, the operation will be repeated.",
|
174
|
+
show_alert=True
|
175
|
+
)
|
176
|
+
self.bot.downloader.auth_event.set()
|
177
|
+
self.bot.downloader.auth_event.clear()
|
warp_beacon/uploader/__init__.py
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
import os
|
1
2
|
import threading
|
2
3
|
import multiprocessing
|
3
4
|
from warp_beacon.jobs.upload_job import UploadJob
|
@@ -20,12 +21,14 @@ class AsyncUploader(object):
|
|
20
21
|
storage = None
|
21
22
|
in_process = set()
|
22
23
|
loop = None
|
24
|
+
admin_message_callback = None
|
23
25
|
pool_size = 1
|
24
26
|
|
25
|
-
def __init__(self, loop: asyncio.AbstractEventLoop, storage: Storage, pool_size: int=
|
27
|
+
def __init__(self, loop: asyncio.AbstractEventLoop, storage: Storage, admin_message_callback: Callable, pool_size: int=min(32, os.cpu_count() + 4)) -> None:
|
26
28
|
self.storage = storage
|
27
29
|
self.loop = loop
|
28
30
|
self.job_queue = multiprocessing.Queue()
|
31
|
+
self.admin_message_callback = admin_message_callback
|
29
32
|
self.pool_size = pool_size
|
30
33
|
|
31
34
|
def __del__(self) -> None:
|
@@ -80,6 +83,9 @@ class AsyncUploader(object):
|
|
80
83
|
job = self.job_queue.get()
|
81
84
|
if job is self.__JOE_BIDEN_WAKEUP:
|
82
85
|
continue
|
86
|
+
if job.is_message_to_admin and job.message_text and self.admin_message_callback:
|
87
|
+
asyncio.ensure_future(self.admin_message_callback(job.message_text), loop=self.loop)
|
88
|
+
continue
|
83
89
|
path = ""
|
84
90
|
if job.media_type == JobType.COLLECTION:
|
85
91
|
for i in job.media_collection:
|
@@ -1,40 +1,40 @@
|
|
1
|
-
etc/warp_beacon/warp_beacon.conf,sha256
|
1
|
+
etc/warp_beacon/warp_beacon.conf,sha256=-QyHfsyo-1Ukxlsz3KSE2vKvLCRm5NtA36_jmfqSAsI,307
|
2
2
|
lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuwwPfa2mDLUA,372
|
3
3
|
var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
|
4
4
|
warp_beacon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
|
-
warp_beacon/__version__.py,sha256=
|
5
|
+
warp_beacon/__version__.py,sha256=DztBFRsZn6dmSiL6gaH4Vp0DP0-eSBMtFnCE4Xras4g,23
|
6
6
|
warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
|
7
7
|
warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
8
|
warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
|
9
9
|
warp_beacon/jobs/__init__.py,sha256=ED8_tPle4iL4kqNW0apAVkgNQtRRTnYfAJwBjO1g0JY,180
|
10
|
-
warp_beacon/jobs/abstract.py,sha256=
|
10
|
+
warp_beacon/jobs/abstract.py,sha256=NCwsogk3-m8F6a-SeiY4CEdTgqJn2V_MU4H7_iYuF7k,2463
|
11
11
|
warp_beacon/jobs/download_job.py,sha256=5HiPcnJppFMhO14___3eSkoMygM3y-vhpGkMAuNhK7s,854
|
12
12
|
warp_beacon/jobs/types.py,sha256=Ae8zINgbs7cOcYkYoOCOACA7duyhnIGMQAJ_SJB1QRQ,176
|
13
13
|
warp_beacon/jobs/upload_job.py,sha256=_ul4psPej1jLEs-BMcMR80GbXDSmm38jE9yoZtecclY,741
|
14
14
|
warp_beacon/mediainfo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
15
15
|
warp_beacon/mediainfo/abstract.py,sha256=ZR2JMuRpoh7nDNov9a8YkAfr6BI2HXnXzQtVrLgDxjs,1185
|
16
16
|
warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-ZM,651
|
17
|
-
warp_beacon/mediainfo/silencer.py,sha256=
|
17
|
+
warp_beacon/mediainfo/silencer.py,sha256=MgUc9Ibbhjhg9GbJMNfJqrdDkMsQShZkQ1sCwvW_-qI,1647
|
18
18
|
warp_beacon/mediainfo/video.py,sha256=AIRy_op_BvehsjarM1rvT5Qo0QWwf-Q6xVVd_aCnbJ4,2505
|
19
|
-
warp_beacon/scraper/__init__.py,sha256=
|
20
|
-
warp_beacon/scraper/abstract.py,sha256
|
21
|
-
warp_beacon/scraper/exceptions.py,sha256=
|
22
|
-
warp_beacon/scraper/instagram.py,sha256=
|
19
|
+
warp_beacon/scraper/__init__.py,sha256=3pwcnYp4fdGG_zdnKGryfbtKJWHOZXuUOTSiCB3mvdQ,11049
|
20
|
+
warp_beacon/scraper/abstract.py,sha256=-yMCEW2JUQWUVs4TbiUi6PEw8RyRuTvrJRIUeEpASP0,1723
|
21
|
+
warp_beacon/scraper/exceptions.py,sha256=j3sUi3LT3l-mqW8X6sNSpmx5JIioFODWlOZW8I9RJHA,1191
|
22
|
+
warp_beacon/scraper/instagram.py,sha256=SHyyNvy5dIPhKllIBEvUhM6yq2zeOw4V6r1cvP69NwU,8580
|
23
23
|
warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
24
|
-
warp_beacon/scraper/youtube/abstract.py,sha256=
|
25
|
-
warp_beacon/scraper/youtube/music.py,sha256=
|
26
|
-
warp_beacon/scraper/youtube/shorts.py,sha256=
|
27
|
-
warp_beacon/scraper/youtube/youtube.py,sha256=
|
24
|
+
warp_beacon/scraper/youtube/abstract.py,sha256=ZbCaYAsGowGsrbJDY39Yk_JAhETncw9aE9GYA9sPXTA,5940
|
25
|
+
warp_beacon/scraper/youtube/music.py,sha256=Cf7QNgers2zo-cQRxi__NPz7V8ZoWCC-g9b8yZNfJ4E,1529
|
26
|
+
warp_beacon/scraper/youtube/shorts.py,sha256=vMEshUBtFwchwp_W2uUoPKXwgll2wFgKdIfvSFKXxLE,1234
|
27
|
+
warp_beacon/scraper/youtube/youtube.py,sha256=TmKMdGbhORTBcBYB6V4ItvWKnxWtUfMbXhomonsmAso,2280
|
28
28
|
warp_beacon/storage/__init__.py,sha256=8XsJXq9X7GDlTaWREF4W1PDX9PH5utwhjf5c5M8Bb7o,3378
|
29
29
|
warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
30
|
-
warp_beacon/telegram/bot.py,sha256=
|
31
|
-
warp_beacon/telegram/handlers.py,sha256=
|
30
|
+
warp_beacon/telegram/bot.py,sha256=U2UZzzty13BdzDFi6K7kJr2FiyuUpwC5w0PuPG0TMU0,12858
|
31
|
+
warp_beacon/telegram/handlers.py,sha256=4GWts4GkPN4JNI26GCCdsVSVzuagVf_zh_b5bdILPU8,6333
|
32
32
|
warp_beacon/telegram/placeholder_message.py,sha256=u5kVfTjGmVYkwA5opniRltHXGpsdSxI41WEde8J5os0,6418
|
33
33
|
warp_beacon/telegram/utils.py,sha256=1tm_DH1F2snDxSqwZnKD4ijvTrobv_kscgt3w-bWa6g,2027
|
34
|
-
warp_beacon/uploader/__init__.py,sha256=
|
35
|
-
warp_beacon-2.0.
|
36
|
-
warp_beacon-2.0.
|
37
|
-
warp_beacon-2.0.
|
38
|
-
warp_beacon-2.0.
|
39
|
-
warp_beacon-2.0.
|
40
|
-
warp_beacon-2.0.
|
34
|
+
warp_beacon/uploader/__init__.py,sha256=qODBuIvWtypQQyKl3Y0QiBXC5SipVFL4b64D91EmTyw,4764
|
35
|
+
warp_beacon-2.0.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
36
|
+
warp_beacon-2.0.7.dist-info/METADATA,sha256=mi4ONCAXsCRhF6jG42ipqQf_Bw3MEqssfO4muB4wXc0,20987
|
37
|
+
warp_beacon-2.0.7.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
|
38
|
+
warp_beacon-2.0.7.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
|
39
|
+
warp_beacon-2.0.7.dist-info/top_level.txt,sha256=pu6xG8OO_nCGllnOfAZ6QpVfivtmHVxPlYK8SZzUDqA,840
|
40
|
+
warp_beacon-2.0.7.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|