warp-beacon 1.2.6__py3-none-any.whl → 2.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etc/warp_beacon/warp_beacon.conf +4 -2
- warp_beacon/__version__.py +1 -1
- warp_beacon/jobs/__init__.py +2 -0
- warp_beacon/jobs/abstract.py +21 -4
- warp_beacon/jobs/download_job.py +6 -3
- warp_beacon/jobs/types.py +9 -0
- warp_beacon/jobs/upload_job.py +1 -0
- warp_beacon/mediainfo/abstract.py +11 -1
- warp_beacon/mediainfo/silencer.py +46 -0
- warp_beacon/mediainfo/video.py +13 -1
- warp_beacon/scraper/__init__.py +38 -23
- warp_beacon/scraper/abstract.py +26 -0
- warp_beacon/scraper/instagram.py +35 -24
- warp_beacon/scraper/youtube/abstract.py +105 -0
- warp_beacon/scraper/youtube/music.py +12 -108
- warp_beacon/scraper/youtube/shorts.py +20 -73
- warp_beacon/scraper/youtube/youtube.py +41 -0
- warp_beacon/storage/__init__.py +27 -6
- warp_beacon/telegram/__init__.py +0 -0
- warp_beacon/telegram/bot.py +348 -0
- warp_beacon/telegram/handlers.py +163 -0
- warp_beacon/telegram/placeholder_message.py +191 -0
- warp_beacon/telegram/utils.py +73 -0
- warp_beacon/uploader/__init__.py +9 -9
- warp_beacon/warp_beacon.py +8 -594
- {warp_beacon-1.2.6.dist-info → warp_beacon-2.0.1.dist-info}/METADATA +4 -2
- warp_beacon-2.0.1.dist-info/RECORD +40 -0
- {warp_beacon-1.2.6.dist-info → warp_beacon-2.0.1.dist-info}/WHEEL +1 -1
- {warp_beacon-1.2.6.dist-info → warp_beacon-2.0.1.dist-info}/top_level.txt +9 -0
- warp_beacon-1.2.6.dist-info/RECORD +0 -31
- {warp_beacon-1.2.6.dist-info → warp_beacon-2.0.1.dist-info}/LICENSE +0 -0
- {warp_beacon-1.2.6.dist-info → warp_beacon-2.0.1.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,348 @@
|
|
1
|
+
import os, io
|
2
|
+
import signal
|
3
|
+
|
4
|
+
import uvloop
|
5
|
+
import asyncio
|
6
|
+
|
7
|
+
from pyrogram import Client, filters
|
8
|
+
from pyrogram.enums import ParseMode
|
9
|
+
from pyrogram.handlers import MessageHandler
|
10
|
+
from pyrogram.types import Message, InputMedia, InputMediaAudio, InputMediaPhoto, InputMediaVideo, InputMediaAnimation, InputMediaDocument, InlineKeyboardButton, InlineKeyboardMarkup
|
11
|
+
from pyrogram.errors import RPCError, FloodWait, NetworkMigrate, BadRequest, MultiMediaTooLong, MessageIdInvalid
|
12
|
+
|
13
|
+
from warp_beacon.__version__ import __version__
|
14
|
+
from warp_beacon.telegram.handlers import Handlers
|
15
|
+
import warp_beacon.scraper
|
16
|
+
from warp_beacon.telegram.placeholder_message import PlaceholderMessage
|
17
|
+
from warp_beacon.storage import Storage
|
18
|
+
from warp_beacon.uploader import AsyncUploader
|
19
|
+
from warp_beacon.jobs.download_job import DownloadJob
|
20
|
+
from warp_beacon.jobs.upload_job import UploadJob
|
21
|
+
from warp_beacon.jobs import Origin
|
22
|
+
from warp_beacon.jobs.types import JobType
|
23
|
+
from warp_beacon.telegram.utils import Utils
|
24
|
+
|
25
|
+
import logging
|
26
|
+
|
27
|
+
class Bot(object):
|
28
|
+
storage = Storage()
|
29
|
+
uploader = None
|
30
|
+
downloader = None
|
31
|
+
allow_loop = True
|
32
|
+
client = None
|
33
|
+
handlers = None
|
34
|
+
placeholder = None
|
35
|
+
|
36
|
+
def __init__(self, tg_bot_name: str, tg_token: str, tg_api_id: str, tg_api_hash: str) -> None:
|
37
|
+
# Enable logging
|
38
|
+
logging.basicConfig(
|
39
|
+
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO
|
40
|
+
)
|
41
|
+
|
42
|
+
logging.info(f"Starting Warp Beacon version '{__version__}' ...")
|
43
|
+
|
44
|
+
workers_amount = min(32, os.cpu_count() + 4)
|
45
|
+
|
46
|
+
uvloop.install()
|
47
|
+
self.client = Client(
|
48
|
+
name=tg_bot_name,
|
49
|
+
app_version=__version__,
|
50
|
+
bot_token=tg_token,
|
51
|
+
api_id=tg_api_id,
|
52
|
+
api_hash=tg_api_hash,
|
53
|
+
workdir='/',
|
54
|
+
workers=int(os.environ.get("TG_WORKERS_POOL_SIZE", default=workers_amount))
|
55
|
+
)
|
56
|
+
|
57
|
+
this = self
|
58
|
+
def __terminator() -> None:
|
59
|
+
this.stop()
|
60
|
+
|
61
|
+
stop_signals = (signal.SIGINT, signal.SIGTERM, signal.SIGABRT)
|
62
|
+
for sig in stop_signals:
|
63
|
+
self.client.loop.add_signal_handler(sig, __terminator)
|
64
|
+
|
65
|
+
self.uploader = AsyncUploader(
|
66
|
+
storage=self.storage,
|
67
|
+
pool_size=int(os.environ.get("UPLOAD_POOL_SIZE", default=workers_amount)),
|
68
|
+
loop=self.client.loop
|
69
|
+
)
|
70
|
+
self.downloader = warp_beacon.scraper.AsyncDownloader(
|
71
|
+
workers_count=int(os.environ.get("WORKERS_POOL_SIZE", default=workers_amount)),
|
72
|
+
uploader=self.uploader
|
73
|
+
)
|
74
|
+
|
75
|
+
self.downloader.start()
|
76
|
+
self.uploader.start()
|
77
|
+
|
78
|
+
self.handlers = Handlers(self)
|
79
|
+
|
80
|
+
self.client.add_handler(MessageHandler(self.handlers.start, filters.command("start")))
|
81
|
+
self.client.add_handler(MessageHandler(self.handlers.help, filters.command("help")))
|
82
|
+
self.client.add_handler(MessageHandler(self.handlers.random, filters.command("random")))
|
83
|
+
self.client.add_handler(MessageHandler(self.handlers.handler))
|
84
|
+
|
85
|
+
self.placeholder = PlaceholderMessage(self)
|
86
|
+
|
87
|
+
self.client.run()
|
88
|
+
|
89
|
+
def __del__(self) -> None:
|
90
|
+
self.stop()
|
91
|
+
logging.info("Warp Beacon terminated.")
|
92
|
+
|
93
|
+
def start(self) -> None:
|
94
|
+
self.client.run()
|
95
|
+
|
96
|
+
def stop(self) -> None:
|
97
|
+
logging.info("Warp Beacon terminating. This may take a while ...")
|
98
|
+
self.downloader.stop_all()
|
99
|
+
self.uploader.stop_all()
|
100
|
+
#self.client.stop()
|
101
|
+
|
102
|
+
async def send_text(self, chat_id: int, text: str, reply_id: int = None) -> int:
|
103
|
+
try:
|
104
|
+
message_reply = await self.client.send_message(
|
105
|
+
chat_id=chat_id,
|
106
|
+
text=text,
|
107
|
+
parse_mode=ParseMode.MARKDOWN,
|
108
|
+
reply_to_message_id=reply_id
|
109
|
+
)
|
110
|
+
return message_reply.id
|
111
|
+
except Exception as e:
|
112
|
+
logging.error("Failed to send text message!")
|
113
|
+
logging.exception(e)
|
114
|
+
|
115
|
+
return 0
|
116
|
+
|
117
|
+
def build_tg_args(self, job: UploadJob) -> dict:
|
118
|
+
args = {}
|
119
|
+
if job.media_type == JobType.VIDEO:
|
120
|
+
if job.tg_file_id:
|
121
|
+
if job.placeholder_message_id:
|
122
|
+
args["media"] = InputMediaVideo(media=job.tg_file_id.replace(":video", ''), supports_streaming=True)
|
123
|
+
else:
|
124
|
+
args["video"] = job.tg_file_id.replace(":video", '')
|
125
|
+
else:
|
126
|
+
if job.placeholder_message_id:
|
127
|
+
args["media"] = InputMediaVideo(
|
128
|
+
media=job.local_media_path,
|
129
|
+
supports_streaming=True,
|
130
|
+
width=job.media_info["width"],
|
131
|
+
height=job.media_info["height"],
|
132
|
+
duration=job.media_info["duration"],
|
133
|
+
thumb=job.media_info["thumb"]
|
134
|
+
)
|
135
|
+
else:
|
136
|
+
args["video"] = job.local_media_path
|
137
|
+
args["supports_streaming"] = True
|
138
|
+
args["width"] = job.media_info["width"]
|
139
|
+
args["height"] = job.media_info["height"]
|
140
|
+
args["duration"] = job.media_info["duration"]
|
141
|
+
args["thumb"] = job.media_info["thumb"]
|
142
|
+
|
143
|
+
args["file_name"] = "downloaded_via_warp_beacon_bot%s" % (os.path.splitext(job.local_media_path)[-1])
|
144
|
+
elif job.media_type == JobType.IMAGE:
|
145
|
+
if job.tg_file_id:
|
146
|
+
if job.placeholder_message_id:
|
147
|
+
args["media"] = InputMediaPhoto(media=job.tg_file_id.replace(":image", ''))
|
148
|
+
else:
|
149
|
+
args["photo"] = job.tg_file_id.replace(":image", '')
|
150
|
+
else:
|
151
|
+
if job.placeholder_message_id:
|
152
|
+
args["media"] = InputMediaPhoto(
|
153
|
+
media=job.local_media_path
|
154
|
+
)
|
155
|
+
else:
|
156
|
+
args["photo"] = job.local_media_path
|
157
|
+
elif job.media_type == JobType.AUDIO:
|
158
|
+
if job.tg_file_id:
|
159
|
+
if job.placeholder_message_id:
|
160
|
+
args["media"] = InputMediaAudio(
|
161
|
+
media=job.tg_file_id.replace(":audio", '')
|
162
|
+
)
|
163
|
+
else:
|
164
|
+
args["audio"] = job.tg_file_id.replace(":audio", '')
|
165
|
+
else:
|
166
|
+
if job.placeholder_message_id:
|
167
|
+
args["media"] = InputMediaAudio(
|
168
|
+
media=job.local_media_path,
|
169
|
+
performer=job.media_info["performer"],
|
170
|
+
thumb=job.media_info["thumb"],
|
171
|
+
duration=job.media_info["duration"],
|
172
|
+
title=job.canonical_name,
|
173
|
+
)
|
174
|
+
else:
|
175
|
+
args["audio"] = job.local_media_path
|
176
|
+
args["performer"] = job.media_info["performer"]
|
177
|
+
args["thumb"] = job.media_info["thumb"]
|
178
|
+
args["duration"] = job.media_info["duration"]
|
179
|
+
args["title"] = job.canonical_name
|
180
|
+
#args["file_name"] = "%s%s" % (job.canonical_name, os.path.splitext(job.local_media_path)[-1]),
|
181
|
+
elif job.media_type == JobType.ANIMATION:
|
182
|
+
if job.tg_file_id:
|
183
|
+
if job.placeholder_message_id:
|
184
|
+
args["media"] = InputMediaAnimation(
|
185
|
+
media=job.tg_file_id.replace(":animation", '')
|
186
|
+
)
|
187
|
+
else:
|
188
|
+
args["animation"] = job.tg_file_id.replace(":animation", '')
|
189
|
+
else:
|
190
|
+
if job.placeholder_message_id:
|
191
|
+
args["media"] = InputMediaAnimation(
|
192
|
+
media=job.local_media_path,
|
193
|
+
thumb=job.media_info["thumb"],
|
194
|
+
duration=job.media_info["duration"],
|
195
|
+
width=job.media_info["width"],
|
196
|
+
height=job.media_info["height"]
|
197
|
+
)
|
198
|
+
else:
|
199
|
+
args["animation"] = job.local_media_path
|
200
|
+
args["width"] = job.media_info["width"]
|
201
|
+
args["height"] = job.media_info["height"]
|
202
|
+
args["duration"] = job.media_info["duration"]
|
203
|
+
args["thumb"] = job.media_info["thumb"]
|
204
|
+
elif job.media_type == JobType.COLLECTION:
|
205
|
+
if job.tg_file_id:
|
206
|
+
args["media"] = []
|
207
|
+
for chunk in Utils.chunker(job.tg_file_id.split(','), 10):
|
208
|
+
tg_chunk = []
|
209
|
+
for i in chunk:
|
210
|
+
tg_id, mtype = i.split(':')
|
211
|
+
ctype = JobType[mtype.upper()]
|
212
|
+
ptr = None
|
213
|
+
if ctype == JobType.VIDEO:
|
214
|
+
ptr = InputMediaVideo(media=tg_id)
|
215
|
+
elif ctype == JobType.IMAGE:
|
216
|
+
ptr = InputMediaPhoto(media=tg_id)
|
217
|
+
elif ctype == JobType.ANIMATION:
|
218
|
+
ptr = InputMediaAnimation(media=tg_id)
|
219
|
+
tg_chunk.append(ptr)
|
220
|
+
|
221
|
+
args["media"].append(tg_chunk)
|
222
|
+
else:
|
223
|
+
mediafs = []
|
224
|
+
for chunk in job.media_collection:
|
225
|
+
tg_chunk = []
|
226
|
+
for j in chunk:
|
227
|
+
if j.media_type == JobType.VIDEO:
|
228
|
+
vid = InputMediaVideo(
|
229
|
+
media=j.local_media_path,
|
230
|
+
supports_streaming=True,
|
231
|
+
width=j.media_info["width"],
|
232
|
+
height=j.media_info["height"],
|
233
|
+
duration=int(j.media_info["duration"]),
|
234
|
+
thumb=j.media_info["thumb"],
|
235
|
+
)
|
236
|
+
tg_chunk.append(vid)
|
237
|
+
elif j.media_type == JobType.IMAGE:
|
238
|
+
photo = InputMediaPhoto(
|
239
|
+
media=j.local_media_path
|
240
|
+
)
|
241
|
+
tg_chunk.append(photo)
|
242
|
+
mediafs.append(tg_chunk)
|
243
|
+
args["media"] = mediafs
|
244
|
+
|
245
|
+
args["chat_id"] = job.chat_id
|
246
|
+
|
247
|
+
# common args
|
248
|
+
if job.placeholder_message_id and job.media_type is not JobType.COLLECTION:
|
249
|
+
args["message_id"] = job.placeholder_message_id
|
250
|
+
else:
|
251
|
+
args["disable_notification"] = True
|
252
|
+
args["reply_to_message_id"] = job.message_id
|
253
|
+
|
254
|
+
if os.environ.get("ENABLE_DONATES", None) == "true" and job.media_type is not JobType.COLLECTION:
|
255
|
+
args["reply_markup"] = InlineKeyboardMarkup([[InlineKeyboardButton("❤ Donate", url=os.environ.get("DONATE_LINK", "https://pay.cryptocloud.plus/pos/W5BMtNQt5bJFoW2E"))]])
|
256
|
+
|
257
|
+
return args
|
258
|
+
|
259
|
+
async def upload_job(self, job: UploadJob) -> list[str]:
|
260
|
+
tg_file_ids = []
|
261
|
+
try:
|
262
|
+
retry_amount = 0
|
263
|
+
max_retries = int(os.environ.get("TG_MAX_RETRIES", default=5))
|
264
|
+
while not retry_amount >= max_retries:
|
265
|
+
try:
|
266
|
+
reply_message = None
|
267
|
+
if job.media_type in (JobType.VIDEO, JobType.IMAGE, JobType.AUDIO):
|
268
|
+
if job.placeholder_message_id:
|
269
|
+
try:
|
270
|
+
reply_message = await self.client.edit_message_media(**self.build_tg_args(job))
|
271
|
+
except MessageIdInvalid:
|
272
|
+
logging.warning("Placeholder message not found. Looks like placeholder message was deleted by administrator.")
|
273
|
+
job.placeholder_message_id = None
|
274
|
+
continue
|
275
|
+
else:
|
276
|
+
send_funcs = {
|
277
|
+
JobType.VIDEO: self.client.send_video,
|
278
|
+
JobType.IMAGE: self.client.send_photo,
|
279
|
+
JobType.AUDIO: self.client.send_audio,
|
280
|
+
JobType.ANIMATION: self.client.send_animation
|
281
|
+
}
|
282
|
+
try:
|
283
|
+
reply_message = await send_funcs[job.media_type](**self.build_tg_args(job))
|
284
|
+
except ValueError as e:
|
285
|
+
err_text = str(e)
|
286
|
+
if "Expected" in err_text:
|
287
|
+
logging.warning("Expectations exceeded reality.")
|
288
|
+
logging.warning(err_text)
|
289
|
+
expectation, reality = Utils.parse_expected_patronum_error(err_text)
|
290
|
+
job_args = self.build_tg_args(job)
|
291
|
+
job_args[reality.value.lower()] = job_args.pop(expectation.value.lower())
|
292
|
+
reply_message = await send_funcs[reality](**job_args)
|
293
|
+
|
294
|
+
tg_file_id = Utils.extract_file_id(reply_message)
|
295
|
+
tg_file_ids.append(tg_file_id)
|
296
|
+
job.tg_file_id = tg_file_id
|
297
|
+
logging.info("Uploaded media file with type '%s' tg_file_id is '%s'", job.media_type.value, job.tg_file_id)
|
298
|
+
elif job.media_type == JobType.COLLECTION:
|
299
|
+
col_job_args = self.build_tg_args(job)
|
300
|
+
sent_messages = []
|
301
|
+
for i, media_chunk in enumerate(col_job_args["media"]):
|
302
|
+
messages = await self.client.send_media_group(
|
303
|
+
chat_id=job.chat_id,
|
304
|
+
reply_to_message_id=job.message_id,
|
305
|
+
media=media_chunk,
|
306
|
+
)
|
307
|
+
sent_messages += messages
|
308
|
+
if job.media_collection:
|
309
|
+
for j, chunk in enumerate(media_chunk):
|
310
|
+
tg_file_id = Utils.eƒƒxtract_file_id(messages[j])
|
311
|
+
if tg_file_id:
|
312
|
+
job.media_collection[i][j].tg_file_id = tg_file_id
|
313
|
+
if i == 0 and job.placeholder_message_id:
|
314
|
+
await self.placeholder.remove(job.chat_id, job.placeholder_message_id)
|
315
|
+
for msg in sent_messages:
|
316
|
+
if msg.video:
|
317
|
+
tg_file_ids.append(msg.video.file_id + ':video')
|
318
|
+
elif msg.photo:
|
319
|
+
tg_file_ids.append(msg.photo.file_id + ':image')
|
320
|
+
logging.info("Uploaded to Telegram")
|
321
|
+
break
|
322
|
+
except MultiMediaTooLong as e:
|
323
|
+
logging.error("Failed to upload due telegram limitations :(")
|
324
|
+
logging.exception(e)
|
325
|
+
await self.placeholder.remove(job.chat_id, job.placeholder_message_id)
|
326
|
+
await self.send_text(job.chat_id, e.MESSAGE, job.message_id)
|
327
|
+
break
|
328
|
+
except (NetworkMigrate, BadRequest) as e:
|
329
|
+
logging.error("Network error. Check you Internet connection.")
|
330
|
+
logging.exception(e)
|
331
|
+
|
332
|
+
if retry_amount+1 >= max_retries:
|
333
|
+
msg = ""
|
334
|
+
if e.MESSAGE:
|
335
|
+
msg = "Telegram error: %s" % str(e.MESSAGE)
|
336
|
+
else:
|
337
|
+
msg = "Unfortunately, Telegram limits were exceeded. Your media size is %.2f MB." % job.media_info["filesize"]
|
338
|
+
await self.placeholder.remove(job.chat_id, job.placeholder_message_id)
|
339
|
+
await self.send_text(job.chat_id, msg, job.message_id)
|
340
|
+
break
|
341
|
+
retry_amount += 1
|
342
|
+
except Exception as e:
|
343
|
+
logging.error("Error occurred!")
|
344
|
+
logging.exception(e)
|
345
|
+
finally:
|
346
|
+
job.remove_files()
|
347
|
+
|
348
|
+
return tg_file_ids
|
@@ -0,0 +1,163 @@
|
|
1
|
+
from pyrogram import Client
|
2
|
+
from pyrogram.types import Message
|
3
|
+
from pyrogram.enums import ChatType, ParseMode
|
4
|
+
from pyrogram.types import Chat, BotCommand
|
5
|
+
|
6
|
+
from urlextract import URLExtract
|
7
|
+
|
8
|
+
from warp_beacon.storage import Storage
|
9
|
+
from warp_beacon.telegram.utils import Utils
|
10
|
+
from warp_beacon.jobs.download_job import DownloadJob
|
11
|
+
from warp_beacon.jobs.upload_job import UploadJob
|
12
|
+
from warp_beacon.jobs import Origin
|
13
|
+
from warp_beacon.jobs.types import JobType
|
14
|
+
|
15
|
+
import logging
|
16
|
+
|
17
|
+
class Handlers(object):
|
18
|
+
storage = None
|
19
|
+
bot = None
|
20
|
+
|
21
|
+
def __init__(self, bot: "Bot") -> None:
|
22
|
+
self.bot = bot
|
23
|
+
self.storage = bot.storage
|
24
|
+
|
25
|
+
async def help(self, client: Client, message: Message) -> None:
|
26
|
+
"""Send a message when the command /help is issued."""
|
27
|
+
await self.bot.send_text(text="Send me a link to remote media", reply_id=message.id, chat_id=message.chat.id)
|
28
|
+
|
29
|
+
async def random(self, client: Client, message: Message) -> None:
|
30
|
+
d = self.storage.get_random()
|
31
|
+
if not d:
|
32
|
+
await message.reply_text("No random content yet. Try to send link first.")
|
33
|
+
return
|
34
|
+
await self.bot.upload_job(
|
35
|
+
UploadJob(
|
36
|
+
tg_file_id=d["tg_file_id"],
|
37
|
+
chat_id=message.chat.id,
|
38
|
+
media_type=JobType[d["media_type"].upper()],
|
39
|
+
message_id=message.id
|
40
|
+
)
|
41
|
+
)
|
42
|
+
|
43
|
+
async def start(self, client: Client, message: Message) -> None:
|
44
|
+
bot_name = await self.bot.client.get_me()
|
45
|
+
await self.bot.client.set_bot_commands([
|
46
|
+
BotCommand("start", "Start bot"),
|
47
|
+
BotCommand("help", "Show help message"),
|
48
|
+
BotCommand("random", "Get random media")
|
49
|
+
])
|
50
|
+
await message.reply_text(
|
51
|
+
parse_mode=ParseMode.MARKDOWN,
|
52
|
+
text=f"Welcome to @{bot_name.username}!\n"
|
53
|
+
"Send link to external social network with content and I'll reply to it.\n"
|
54
|
+
"Currently supported: Instagram, YouTube Shorts and YouTube Music."
|
55
|
+
)
|
56
|
+
|
57
|
+
async def handler(self, client: Client, message: Message) -> None:
|
58
|
+
chat = message.chat
|
59
|
+
effective_message_id = message.id
|
60
|
+
extractor = URLExtract()
|
61
|
+
urls = extractor.find_urls(message.text)
|
62
|
+
|
63
|
+
reply_text = "Wut?"
|
64
|
+
if not urls:
|
65
|
+
reply_text = "Your message should contains URLs"
|
66
|
+
else:
|
67
|
+
for url in urls:
|
68
|
+
origin = Utils.extract_origin(url)
|
69
|
+
if origin is Origin.YOUTU_BE:
|
70
|
+
url = Utils.extract_youtu_be_link(url)
|
71
|
+
if not url:
|
72
|
+
raise ValueError("Failed to extract youtu.be link")
|
73
|
+
origin = Origin.YOUTUBE
|
74
|
+
if origin is Origin.UNKNOWN:
|
75
|
+
logging.info("Only Instagram, YouTube Shorts and YouTube Music are now supported. Skipping.")
|
76
|
+
continue
|
77
|
+
entities, tg_file_ids = [], []
|
78
|
+
uniq_id = Storage.compute_uniq(url)
|
79
|
+
try:
|
80
|
+
entities = self.storage.db_lookup_id(uniq_id)
|
81
|
+
except Exception as e:
|
82
|
+
logging.error("Failed to search link in DB!")
|
83
|
+
logging.exception(e)
|
84
|
+
if entities:
|
85
|
+
tg_file_ids = [i["tg_file_id"] for i in entities]
|
86
|
+
logging.info("URL '%s' is found in DB. Sending with tg_file_ids = '%s'", url, str(tg_file_ids))
|
87
|
+
ent_len = len(entities)
|
88
|
+
if ent_len > 1:
|
89
|
+
await self.bot.upload_job(
|
90
|
+
UploadJob(
|
91
|
+
tg_file_id=",".join(tg_file_ids),
|
92
|
+
message_id=effective_message_id,
|
93
|
+
media_type=JobType.COLLECTION,
|
94
|
+
chat_id=chat.id
|
95
|
+
)
|
96
|
+
)
|
97
|
+
elif ent_len:
|
98
|
+
logging.info(entities[0])
|
99
|
+
media_type = JobType[entities[0]["media_type"].upper()]
|
100
|
+
await self.bot.upload_job(
|
101
|
+
UploadJob(
|
102
|
+
tg_file_id=tg_file_ids.pop(),
|
103
|
+
message_id=effective_message_id,
|
104
|
+
media_type=media_type,
|
105
|
+
chat_id=chat.id
|
106
|
+
)
|
107
|
+
)
|
108
|
+
else:
|
109
|
+
async def upload_wrapper(job: UploadJob) -> None:
|
110
|
+
try:
|
111
|
+
if job.job_failed and job.job_failed_msg:
|
112
|
+
if job.placeholder_message_id:
|
113
|
+
await self.bot.placeholder.remove(chat.id, job.placeholder_message_id)
|
114
|
+
return await self.bot.send_text(chat_id=chat.id, text=job.job_failed_msg, reply_id=job.message_id)
|
115
|
+
if job.job_warning and job.job_warning_msg:
|
116
|
+
return await self.bot.placeholder.update_text(chat.id, job.placeholder_message_id, job.job_warning_msg)
|
117
|
+
tg_file_ids = await self.bot.upload_job(job)
|
118
|
+
if tg_file_ids:
|
119
|
+
if job.media_type == JobType.COLLECTION and job.save_items:
|
120
|
+
for chunk in job.media_collection:
|
121
|
+
for i in chunk:
|
122
|
+
self.storage.add_media(tg_file_ids=[i.tg_file_id], media_url=i.effective_url, media_type=i.media_type.value, origin=origin.value)
|
123
|
+
else:
|
124
|
+
self.storage.add_media(tg_file_ids=[','.join(tg_file_ids)], media_url=job.url, media_type=job.media_type.value, origin=origin.value)
|
125
|
+
except Exception as e:
|
126
|
+
logging.error("Exception occurred while performing upload callback!")
|
127
|
+
logging.exception(e)
|
128
|
+
|
129
|
+
try:
|
130
|
+
# create placeholder message for long download
|
131
|
+
placeholder_message_id = await self.bot.placeholder.create(
|
132
|
+
chat_id=chat.id,
|
133
|
+
reply_id=effective_message_id
|
134
|
+
)
|
135
|
+
|
136
|
+
if not placeholder_message_id:
|
137
|
+
await self.bot.send_text(
|
138
|
+
chat_id=chat.id,
|
139
|
+
reply_id=effective_message_id,
|
140
|
+
text="Failed to create message placeholder. Please check your bot Internet connection.")
|
141
|
+
return
|
142
|
+
|
143
|
+
self.bot.uploader.add_callback(
|
144
|
+
placeholder_message_id,
|
145
|
+
upload_wrapper
|
146
|
+
)
|
147
|
+
|
148
|
+
self.bot.downloader.queue_task(DownloadJob.build(
|
149
|
+
url=url,
|
150
|
+
placeholder_message_id=placeholder_message_id,
|
151
|
+
message_id=effective_message_id,
|
152
|
+
chat_id=chat.id,
|
153
|
+
in_process=self.bot.uploader.is_inprocess(uniq_id),
|
154
|
+
uniq_id=uniq_id,
|
155
|
+
job_origin=origin
|
156
|
+
))
|
157
|
+
self.bot.uploader.set_inprocess(uniq_id)
|
158
|
+
except Exception as e:
|
159
|
+
logging.error("Failed to schedule download task!")
|
160
|
+
logging.exception(e)
|
161
|
+
|
162
|
+
if chat.type not in (ChatType.GROUP, ChatType.SUPERGROUP) and not urls:
|
163
|
+
await self.bot.send_text(rext=reply_text, reply_id=effective_message_id)
|
@@ -0,0 +1,191 @@
|
|
1
|
+
import os, io
|
2
|
+
import time
|
3
|
+
from enum import Enum
|
4
|
+
from typing import Optional
|
5
|
+
|
6
|
+
from pyrogram.types import Message
|
7
|
+
from pyrogram.errors import RPCError, FloodWait
|
8
|
+
from pyrogram.enums import ParseMode
|
9
|
+
|
10
|
+
import warp_beacon
|
11
|
+
from warp_beacon.telegram.utils import Utils
|
12
|
+
from warp_beacon.mediainfo.video import VideoInfo
|
13
|
+
|
14
|
+
import logging
|
15
|
+
|
16
|
+
class PlaceholderType(Enum):
|
17
|
+
UNKNOWN = 0
|
18
|
+
ANIMATION = 1
|
19
|
+
DOCUMENT = 2
|
20
|
+
PHOTO = 3
|
21
|
+
|
22
|
+
class PlaceHolder(object):
|
23
|
+
pl_type: PlaceholderType = PlaceholderType.ANIMATION
|
24
|
+
tg_file_id: str = None
|
25
|
+
|
26
|
+
def __init__(self, pl_type: PlaceholderType, tg_file_id: str) -> None:
|
27
|
+
self.pl_type = pl_type
|
28
|
+
self.tg_file_id = tg_file_id
|
29
|
+
|
30
|
+
class PlaceholderMessage(object):
|
31
|
+
bot = None
|
32
|
+
placeholder = PlaceHolder(PlaceholderType.ANIMATION, None)
|
33
|
+
|
34
|
+
def __init__(self, bot: "Bot") -> None:
|
35
|
+
self.bot = bot
|
36
|
+
|
37
|
+
def __del__(self) -> None:
|
38
|
+
pass
|
39
|
+
|
40
|
+
async def reuse_ph_animation(self, chat_id: int, reply_id: int, text: str) -> Optional["types.Message"]:
|
41
|
+
reply = await self.bot.client.send_animation(
|
42
|
+
chat_id=chat_id,
|
43
|
+
animation=self.placeholder.tg_file_id,
|
44
|
+
caption=text,
|
45
|
+
parse_mode=ParseMode.MARKDOWN,
|
46
|
+
reply_to_message_id=reply_id
|
47
|
+
)
|
48
|
+
|
49
|
+
return reply
|
50
|
+
|
51
|
+
async def reuse_ph_photo(self, chat_id: int, reply_id: int, text: str) -> Optional["types.Message"]:
|
52
|
+
reply = await self.bot.client.send_photo(
|
53
|
+
chat_id=chat_id,
|
54
|
+
photo=self.placeholder.tg_file_id,
|
55
|
+
caption=text,
|
56
|
+
parse_mode=ParseMode.MARKDOWN,
|
57
|
+
reply_to_message_id=reply_id
|
58
|
+
)
|
59
|
+
|
60
|
+
return reply
|
61
|
+
|
62
|
+
async def reuse_ph_document(self, chat_id: int, reply_id: int, text: str) -> Optional["types.Message"]:
|
63
|
+
reply = await self.bot.client.send_document(
|
64
|
+
chat_id=chat_id,
|
65
|
+
document=self.placeholder.tg_file_id,
|
66
|
+
caption=text,
|
67
|
+
parse_mode=ParseMode.MARKDOWN,
|
68
|
+
reply_to_message_id=reply_id
|
69
|
+
)
|
70
|
+
|
71
|
+
return reply
|
72
|
+
|
73
|
+
async def create(self, chat_id: int, reply_id: int) -> int:
|
74
|
+
retry_amount = 0
|
75
|
+
max_retries = int(os.environ.get("TG_MAX_RETRIES", default=5))
|
76
|
+
while not retry_amount >= max_retries:
|
77
|
+
try:
|
78
|
+
text = "**Loading, this may take a moment ...** ⏱️ "
|
79
|
+
reply = None
|
80
|
+
if self.placeholder.tg_file_id is None:
|
81
|
+
ph_found = False
|
82
|
+
for ph in ('/var/warp_beacon/placeholder.gif', "%s/../var/warp_beacon/placeholder.gif" % os.path.dirname(os.path.abspath(warp_beacon.__file__))):
|
83
|
+
if not os.path.exists(ph):
|
84
|
+
continue
|
85
|
+
try:
|
86
|
+
#pl_info = VideoInfo(ph)
|
87
|
+
#pl_resolution = pl_info.get_demensions()
|
88
|
+
reply = await self.bot.client.send_document(
|
89
|
+
chat_id=chat_id,
|
90
|
+
document=ph,
|
91
|
+
force_document=False,
|
92
|
+
caption=text,
|
93
|
+
parse_mode=ParseMode.MARKDOWN,
|
94
|
+
reply_to_message_id=reply_id,
|
95
|
+
file_name=os.path.basename(ph),
|
96
|
+
#width=pl_resolution["width"],
|
97
|
+
#height=pl_resolution["height"],
|
98
|
+
#duration=round(pl_info.get_duration()),
|
99
|
+
#thumb=pl_info.generate_thumbnail()
|
100
|
+
)
|
101
|
+
self.placeholder = PlaceHolder(PlaceholderType.ANIMATION, Utils.extract_file_id(reply))
|
102
|
+
ph_found = True
|
103
|
+
break
|
104
|
+
except Exception as e:
|
105
|
+
logging.warning("Failed to send placeholder message!")
|
106
|
+
logging.exception(e)
|
107
|
+
if not ph_found:
|
108
|
+
try:
|
109
|
+
reply = await self.bot.client.send_animation(
|
110
|
+
chat_id=chat_id,
|
111
|
+
animation="https://bagrintsev.me/warp_beacon/placeholder_that_we_deserve.mp4",
|
112
|
+
caption=text,
|
113
|
+
parse_mode=ParseMode.MARKDOWN,
|
114
|
+
reply_to_message_id=reply_id
|
115
|
+
)
|
116
|
+
self.placeholder = PlaceHolder(PlaceholderType.ANIMATION, Utils.extract_file_id(reply))
|
117
|
+
except Exception as e:
|
118
|
+
logging.error("Failed to download secret placeholder!")
|
119
|
+
logging.exception(e)
|
120
|
+
img = self.create_default_placeholder_img("Loading, this may take a moment ...")
|
121
|
+
reply = await self.bot.client.send_photo(
|
122
|
+
chat_id=chat.id,
|
123
|
+
parse_mode=ParseMode.MARKDOWN,
|
124
|
+
reply_to_message_id=reply_id,
|
125
|
+
photo=img
|
126
|
+
)
|
127
|
+
self.placeholder = PlaceHolder(PlaceholderType.PHOTO, Utils.extract_file_id(reply))
|
128
|
+
else:
|
129
|
+
if self.placeholder.pl_type == PlaceholderType.ANIMATION:
|
130
|
+
try:
|
131
|
+
reply = await self.reuse_ph_animation(chat_id, reply_id, text)
|
132
|
+
except ValueError as e:
|
133
|
+
logging.warning("Failed to reuse tg_file_id!")
|
134
|
+
logging.exception(e)
|
135
|
+
reply = await self.reuse_ph_document(chat_id, reply_id, text)
|
136
|
+
self.placeholder.pl_type = PlaceholderType.DOCUMENT
|
137
|
+
elif self.placeholder.pl_type == PlaceholderType.DOCUMENT:
|
138
|
+
try:
|
139
|
+
reply = await self.reuse_ph_document(chat_id, reply_id, text)
|
140
|
+
except ValueError as e:
|
141
|
+
logging.warning("Failed to reuse tg_file_id!")
|
142
|
+
logging.exception(e)
|
143
|
+
reply = await self.reuse_ph_animation(chat_id, reply_id, text)
|
144
|
+
self.placeholder.pl_type = PlaceholderType.ANIMATION
|
145
|
+
else:
|
146
|
+
reply = await self.reuse_ph_photo(chat_id, reply_id, text)
|
147
|
+
return reply.id
|
148
|
+
except FloodWait as e:
|
149
|
+
logging.warning("FloodWait exception!")
|
150
|
+
logging.exception(e)
|
151
|
+
await self.bot.send_text(None, "Telegram error: %s" % e.MESSAGE)
|
152
|
+
time.sleep(e.value)
|
153
|
+
except Exception as e:
|
154
|
+
logging.error("Failed to create placeholder message!")
|
155
|
+
logging.exception(e)
|
156
|
+
retry_amount += 1
|
157
|
+
time.sleep(2)
|
158
|
+
|
159
|
+
return 0
|
160
|
+
|
161
|
+
def create_default_placeholder_img(self, text: str, width: int = 800, height: int = 1280) -> io.BytesIO:
|
162
|
+
from PIL import Image, ImageDraw, ImageFont
|
163
|
+
bio = io.BytesIO()
|
164
|
+
bio.name = 'placeholder.png'
|
165
|
+
img = Image.new("RGB", (width, height), (255, 255, 255))
|
166
|
+
draw = ImageDraw.Draw(img)
|
167
|
+
font = ImageFont.load_default(size=48)
|
168
|
+
_, _, w, h = draw.textbbox((0, 0), text, font=font)
|
169
|
+
draw.text(((width-w)/2, (height-h)/2), text, font=font, fill="#000")
|
170
|
+
img.save(bio, 'PNG')
|
171
|
+
bio.seek(0)
|
172
|
+
|
173
|
+
return bio
|
174
|
+
|
175
|
+
async def update_text(self, chat_id: int, placeholder_message_id: int, placeholder_text: str) -> None:
|
176
|
+
try:
|
177
|
+
await self.bot.client.edit_message_caption(
|
178
|
+
chat_id=chat_id,
|
179
|
+
message_id=placeholder_message_id,
|
180
|
+
caption=" ⚠️ *%s*" % placeholder_text
|
181
|
+
)
|
182
|
+
except Exception as e:
|
183
|
+
logging.error("Failed to update placeholder message!")
|
184
|
+
logging.exception(e)
|
185
|
+
|
186
|
+
async def remove(self, chat_id: int, placeholder_message_id: int) -> None:
|
187
|
+
try:
|
188
|
+
await self.bot.client.delete_messages(chat_id, (placeholder_message_id,))
|
189
|
+
except Exception as e:
|
190
|
+
logging.error("Failed to remove placeholder message!")
|
191
|
+
logging.exception(e)
|