warp-beacon 1.2.5__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etc/warp_beacon/warp_beacon.conf +4 -2
- warp_beacon/__version__.py +1 -1
- warp_beacon/jobs/__init__.py +2 -0
- warp_beacon/jobs/abstract.py +21 -4
- warp_beacon/jobs/download_job.py +6 -3
- warp_beacon/jobs/types.py +9 -0
- warp_beacon/mediainfo/abstract.py +11 -1
- warp_beacon/mediainfo/silencer.py +46 -0
- warp_beacon/mediainfo/video.py +13 -1
- warp_beacon/scraper/__init__.py +38 -23
- warp_beacon/scraper/abstract.py +26 -0
- warp_beacon/scraper/instagram.py +35 -24
- warp_beacon/scraper/youtube/abstract.py +105 -0
- warp_beacon/scraper/youtube/music.py +12 -108
- warp_beacon/scraper/youtube/shorts.py +20 -73
- warp_beacon/scraper/youtube/youtube.py +41 -0
- warp_beacon/storage/__init__.py +27 -6
- warp_beacon/telegram/__init__.py +0 -0
- warp_beacon/telegram/bot.py +318 -0
- warp_beacon/telegram/handlers.py +156 -0
- warp_beacon/telegram/placeholder_message.py +191 -0
- warp_beacon/telegram/utils.py +73 -0
- warp_beacon/uploader/__init__.py +9 -9
- warp_beacon/warp_beacon.py +8 -592
- {warp_beacon-1.2.5.dist-info → warp_beacon-2.0.0.dist-info}/METADATA +4 -2
- warp_beacon-2.0.0.dist-info/RECORD +40 -0
- {warp_beacon-1.2.5.dist-info → warp_beacon-2.0.0.dist-info}/WHEEL +1 -1
- {warp_beacon-1.2.5.dist-info → warp_beacon-2.0.0.dist-info}/top_level.txt +9 -0
- warp_beacon-1.2.5.dist-info/RECORD +0 -31
- {warp_beacon-1.2.5.dist-info → warp_beacon-2.0.0.dist-info}/LICENSE +0 -0
- {warp_beacon-1.2.5.dist-info → warp_beacon-2.0.0.dist-info}/entry_points.txt +0 -0
warp_beacon/warp_beacon.py
CHANGED
@@ -1,598 +1,14 @@
|
|
1
|
-
#!/usr/bin/env python3
|
2
|
-
# -*- coding: utf-8 -*-
|
3
|
-
|
4
1
|
import os
|
5
|
-
import signal
|
6
|
-
import asyncio
|
7
|
-
import time
|
8
|
-
from io import BytesIO
|
9
|
-
|
10
|
-
from urlextract import URLExtract
|
11
|
-
|
12
|
-
import telegram
|
13
|
-
from telegram import Bot, ForceReply, Update, Chat, error, InputMediaVideo, InputMediaPhoto, InputMediaAudio, MessageEntity, InlineKeyboardMarkup, InlineKeyboardButton
|
14
|
-
from telegram.ext import Application, CommandHandler, ContextTypes, MessageHandler, filters
|
15
|
-
from telegram.constants import ParseMode
|
16
2
|
|
17
|
-
|
18
|
-
from warp_beacon.storage import Storage
|
19
|
-
from warp_beacon.uploader import AsyncUploader
|
20
|
-
from warp_beacon.jobs.download_job import DownloadJob
|
21
|
-
from warp_beacon.jobs.upload_job import UploadJob
|
22
|
-
from warp_beacon.jobs import Origin
|
3
|
+
from warp_beacon.telegram.bot import Bot
|
23
4
|
|
24
5
|
import logging
|
25
6
|
|
26
|
-
# Enable logging
|
27
|
-
logging.basicConfig(
|
28
|
-
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO
|
29
|
-
)
|
30
|
-
# set higher logging level for httpx to avoid all GET and POST requests being logged
|
31
|
-
logging.getLogger("httpx").setLevel(logging.WARNING)
|
32
|
-
|
33
|
-
logger = logging.getLogger(__name__)
|
34
|
-
|
35
|
-
storage = Storage()
|
36
|
-
uploader = None
|
37
|
-
downloader = None
|
38
|
-
placeholder = ("animation", None)
|
39
|
-
|
40
|
-
# Define a few command handlers. These usually take the two arguments update and
|
41
|
-
# context.
|
42
|
-
async def start(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
|
43
|
-
"""Send a message when the command /start is issued."""
|
44
|
-
user = update.effective_user
|
45
|
-
await update.message.reply_html(
|
46
|
-
rf"Hi {user.mention_html()}!",
|
47
|
-
reply_markup=ForceReply(selective=True),
|
48
|
-
)
|
49
|
-
|
50
|
-
async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
|
51
|
-
"""Send a message when the command /help is issued."""
|
52
|
-
await update.message.reply_text("Send me a link to remote media")
|
53
|
-
|
54
|
-
async def random(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
|
55
|
-
d = storage.get_random()
|
56
|
-
if not d:
|
57
|
-
await update.message.reply_text("No random content yet.")
|
58
|
-
return
|
59
|
-
await upload_job(update, context, UploadJob(tg_file_id=d["tg_file_id"], media_type=d["media_type"], message_id=update.message.message_id))
|
60
|
-
|
61
|
-
async def remove_placeholder(update: Update, context: ContextTypes.DEFAULT_TYPE, placeholder_message_id: int) -> None:
|
62
|
-
try:
|
63
|
-
timeout = int(os.environ.get("TG_WRITE_TIMEOUT", default=120))
|
64
|
-
deleted_message = await context.bot.delete_message(
|
65
|
-
chat_id=update.message.chat_id,
|
66
|
-
message_id=placeholder_message_id,
|
67
|
-
write_timeout=timeout,
|
68
|
-
read_timeout=timeout,
|
69
|
-
connect_timeout=timeout
|
70
|
-
)
|
71
|
-
except Exception as e:
|
72
|
-
logging.error("Failed to remove placeholder message!")
|
73
|
-
logging.exception(e)
|
74
|
-
|
75
|
-
async def update_placeholder_text(update: Update, context: ContextTypes.DEFAULT_TYPE, placeholder_message_id: int, placeholder_text: str) -> None:
|
76
|
-
try:
|
77
|
-
timeout = int(os.environ.get("TG_WRITE_TIMEOUT", default=120))
|
78
|
-
await context.bot.edit_message_caption(
|
79
|
-
chat_id=update.message.chat_id,
|
80
|
-
message_id=placeholder_message_id,
|
81
|
-
parse_mode="MarkdownV2",
|
82
|
-
caption=" ⚠️ *%s*" % placeholder_text,
|
83
|
-
show_caption_above_media=True,
|
84
|
-
write_timeout=timeout,
|
85
|
-
read_timeout=timeout,
|
86
|
-
connect_timeout=timeout
|
87
|
-
)
|
88
|
-
except Exception as e:
|
89
|
-
logging.error("Failed to update placeholder message!")
|
90
|
-
logging.exception(e)
|
91
|
-
|
92
|
-
async def send_text(update: Update, context: ContextTypes.DEFAULT_TYPE, reply_id: int, text: str) -> int:
|
93
|
-
try:
|
94
|
-
reply = await update.message.reply_text(
|
95
|
-
text,
|
96
|
-
reply_to_message_id=reply_id
|
97
|
-
)
|
98
|
-
|
99
|
-
return reply.message_id
|
100
|
-
except Exception as e:
|
101
|
-
logging.error("Failed to send text message!")
|
102
|
-
logging.exception(e)
|
103
|
-
|
104
|
-
return 0
|
105
|
-
|
106
|
-
def create_default_placeholder_img(text: str, width: int = 800, height: int = 1280) -> BytesIO:
|
107
|
-
from PIL import Image, ImageDraw, ImageFont
|
108
|
-
bio = BytesIO()
|
109
|
-
bio.name = 'placeholder.png'
|
110
|
-
img = Image.new("RGB", (width, height), (255, 255, 255))
|
111
|
-
draw = ImageDraw.Draw(img)
|
112
|
-
font = ImageFont.load_default(size=48)
|
113
|
-
_, _, w, h = draw.textbbox((0, 0), text, font=font)
|
114
|
-
draw.text(((width-w)/2, (height-h)/2), text, font=font, fill="#000")
|
115
|
-
img.save(bio, 'PNG')
|
116
|
-
bio.seek(0)
|
117
|
-
return bio
|
118
|
-
|
119
|
-
def extract_file_id(message: "Message") -> str:
|
120
|
-
if message.animation:
|
121
|
-
return message.animation.file_id
|
122
|
-
if message.photo:
|
123
|
-
return message.photo[-1].file_id
|
124
|
-
if message.document:
|
125
|
-
return message.document.file_id
|
126
|
-
if message.video:
|
127
|
-
return message.video.file_id
|
128
|
-
|
129
|
-
return None
|
130
|
-
|
131
|
-
async def create_placeholder_message(update: Update, context: ContextTypes.DEFAULT_TYPE, reply_id: int) -> int:
|
132
|
-
global placeholder
|
133
|
-
retry_amount = 0
|
134
|
-
max_retries = int(os.environ.get("TG_MAX_RETRIES", default=5))
|
135
|
-
while not retry_amount >= max_retries:
|
136
|
-
try:
|
137
|
-
text = "*Loading, this may take a moment \.\.\.* ⏱️ "
|
138
|
-
reply = None
|
139
|
-
if placeholder[1] is None:
|
140
|
-
ph_found = False
|
141
|
-
for ph in ('/var/warp_beacon/placeholder.gif', "%s/placeholder.gif" % os.path.dirname(os.path.abspath(warp_beacon.__file__))):
|
142
|
-
if not os.path.exists(ph):
|
143
|
-
continue
|
144
|
-
reply = await update.message.reply_animation(
|
145
|
-
caption=text,
|
146
|
-
parse_mode="MarkdownV2",
|
147
|
-
show_caption_above_media=True,
|
148
|
-
animation=open(ph, 'rb'),
|
149
|
-
reply_to_message_id=reply_id
|
150
|
-
)
|
151
|
-
placeholder = ("animation", extract_file_id(reply))
|
152
|
-
ph_found = True
|
153
|
-
if not ph_found:
|
154
|
-
try:
|
155
|
-
reply = await update.message.reply_animation(
|
156
|
-
caption=text,
|
157
|
-
parse_mode="MarkdownV2",
|
158
|
-
show_caption_above_media=True,
|
159
|
-
animation="https://bagrintsev.me/warp_beacon/placeholder_that_we_deserve.mp4",
|
160
|
-
reply_to_message_id=reply_id
|
161
|
-
)
|
162
|
-
placeholder = ("animation", extract_file_id(reply))
|
163
|
-
except Exception as e:
|
164
|
-
logging.error("Failed to download secret placeholder!")
|
165
|
-
logging.exception(e)
|
166
|
-
img = create_default_placeholder_img("Loading, this may take a moment ...")
|
167
|
-
reply = await update.message.reply_photo(
|
168
|
-
caption=text,
|
169
|
-
show_caption_above_media=True,
|
170
|
-
parse_mode="MarkdownV2",
|
171
|
-
filename="placeholder.png",
|
172
|
-
photo=img,
|
173
|
-
reply_to_message_id=reply_id
|
174
|
-
)
|
175
|
-
placeholder = ("photo", extract_file_id(reply))
|
176
|
-
else:
|
177
|
-
if placeholder[0] == "animation":
|
178
|
-
reply = await update.message.reply_animation(
|
179
|
-
caption=text,
|
180
|
-
parse_mode="MarkdownV2",
|
181
|
-
show_caption_above_media=True,
|
182
|
-
animation=placeholder[1],
|
183
|
-
reply_to_message_id=reply_id
|
184
|
-
)
|
185
|
-
else:
|
186
|
-
reply = await update.message.reply_photo(
|
187
|
-
caption=text,
|
188
|
-
parse_mode="MarkdownV2",
|
189
|
-
show_caption_above_media=True,
|
190
|
-
photo=placeholder[1],
|
191
|
-
reply_to_message_id=reply_id
|
192
|
-
)
|
193
|
-
return reply.message_id
|
194
|
-
except telegram.error.RetryAfter as e:
|
195
|
-
logging.warning("RetryAfter exception!")
|
196
|
-
logging.exception(e)
|
197
|
-
await send_text(update, context, None, "Telegram error: %s" % e.message)
|
198
|
-
time.sleep(e.retry_after)
|
199
|
-
except Exception as e:
|
200
|
-
logging.error("Failed to create placeholder message!")
|
201
|
-
logging.exception(e)
|
202
|
-
retry_amount += 1
|
203
|
-
time.sleep(2)
|
204
|
-
|
205
|
-
return 0
|
206
|
-
|
207
|
-
def build_tg_args(update: Update, context: ContextTypes.DEFAULT_TYPE, job: UploadJob) -> dict:
|
208
|
-
args = {}
|
209
|
-
timeout = int(os.environ.get("TG_WRITE_TIMEOUT", default=120))
|
210
|
-
if job.media_type == "video":
|
211
|
-
if job.tg_file_id:
|
212
|
-
if job.placeholder_message_id:
|
213
|
-
args["media"] = InputMediaVideo(media=job.tg_file_id.replace(":video", ''), supports_streaming=True)
|
214
|
-
else:
|
215
|
-
args["video"] = job.tg_file_id.replace(":video", '')
|
216
|
-
else:
|
217
|
-
args["media"] = InputMediaVideo(
|
218
|
-
media=open(job.local_media_path, 'rb'),
|
219
|
-
supports_streaming=True,
|
220
|
-
width=job.media_info["width"],
|
221
|
-
height=job.media_info["height"],
|
222
|
-
duration=int(job.media_info["duration"]),
|
223
|
-
thumbnail=job.media_info["thumb"],
|
224
|
-
filename="downloaded_via_warp_beacon_bot%s" % (os.path.splitext(job.local_media_path)[-1])
|
225
|
-
)
|
226
|
-
elif job.media_type == "image":
|
227
|
-
if job.tg_file_id:
|
228
|
-
if job.placeholder_message_id:
|
229
|
-
args["media"] = InputMediaPhoto(media=job.tg_file_id.replace(":image", ''))
|
230
|
-
else:
|
231
|
-
args["photo"] = job.tg_file_id.replace(":image", '')
|
232
|
-
else:
|
233
|
-
args["media"] = InputMediaPhoto(
|
234
|
-
media=open(job.local_media_path, 'rb'),
|
235
|
-
filename="downloaded_via_warp_beacon_bot%s" % (os.path.splitext(job.local_media_path)[-1])
|
236
|
-
)
|
237
|
-
elif job.media_type == "audio":
|
238
|
-
if job.tg_file_id:
|
239
|
-
if job.placeholder_message_id:
|
240
|
-
args["media"] = InputMediaAudio(
|
241
|
-
media=job.tg_file_id.replace(":audio", '')
|
242
|
-
)
|
243
|
-
else:
|
244
|
-
args["audio"] = job.tg_file_id.replace(":audio", '')
|
245
|
-
else:
|
246
|
-
args["media"] = InputMediaAudio(
|
247
|
-
media=open(job.local_media_path, 'rb'),
|
248
|
-
filename="%s%s" % (job.canonical_name, os.path.splitext(job.local_media_path)[-1]),
|
249
|
-
performer=job.media_info["performer"],
|
250
|
-
thumbnail=job.media_info["thumb"]
|
251
|
-
)
|
252
|
-
elif job.media_type == "collection":
|
253
|
-
if job.tg_file_id:
|
254
|
-
args["media"] = []
|
255
|
-
for i in job.tg_file_id.split(','):
|
256
|
-
tg_id, mtype = i.split(':')
|
257
|
-
ptr = None
|
258
|
-
if mtype == "video":
|
259
|
-
ptr = InputMediaVideo(media=tg_id)
|
260
|
-
elif mtype == "image":
|
261
|
-
ptr = InputMediaPhoto(media=tg_id)
|
262
|
-
args["media"].append(ptr)
|
263
|
-
else:
|
264
|
-
mediafs = []
|
265
|
-
for j in job.media_collection:
|
266
|
-
if j.media_type == "video":
|
267
|
-
vid = InputMediaVideo(
|
268
|
-
media=open(j.local_media_path, 'rb'),
|
269
|
-
supports_streaming=True,
|
270
|
-
width=j.media_info["width"],
|
271
|
-
height=j.media_info["height"],
|
272
|
-
duration=int(j.media_info["duration"]),
|
273
|
-
thumbnail=j.media_info["thumb"],
|
274
|
-
filename="downloaded_via_warp_beacon_bot%s" % (os.path.splitext(j.local_media_path)[-1])
|
275
|
-
)
|
276
|
-
mediafs.append(vid)
|
277
|
-
elif j.media_type == "image":
|
278
|
-
photo = InputMediaPhoto(
|
279
|
-
media=open(j.local_media_path, 'rb'),
|
280
|
-
filename="downloaded_via_warp_beacon_bot%s" % (os.path.splitext(job.local_media_path)[-1])
|
281
|
-
)
|
282
|
-
mediafs.append(photo)
|
283
|
-
args["media"] = mediafs
|
284
|
-
|
285
|
-
# common args
|
286
|
-
if job.placeholder_message_id and job.media_type != "collection":
|
287
|
-
args["message_id"] = job.placeholder_message_id
|
288
|
-
args["chat_id"] = update.message.chat_id
|
289
|
-
else:
|
290
|
-
args["disable_notification"] = True
|
291
|
-
args["reply_to_message_id"] = job.message_id
|
292
|
-
args["write_timeout"] = timeout
|
293
|
-
args["read_timeout"] = timeout
|
294
|
-
args["connect_timeout"] = timeout
|
295
|
-
if os.environ.get("ENABLE_DONATES", None) == "true" and job.media_type != "collection":
|
296
|
-
args["reply_markup"] = InlineKeyboardMarkup([[InlineKeyboardButton("❤ Donate", url=os.environ.get("DONATE_LINK", "https://pay.cryptocloud.plus/pos/W5BMtNQt5bJFoW2E"))]])
|
297
|
-
return args
|
298
|
-
|
299
|
-
async def upload_job(update: Update, context: ContextTypes.DEFAULT_TYPE, job: UploadJob) -> list[str]:
|
300
|
-
timeout = int(os.environ.get("TG_WRITE_TIMEOUT", default=120))
|
301
|
-
tg_file_ids = []
|
302
|
-
try:
|
303
|
-
retry_amount = 0
|
304
|
-
max_retries = int(os.environ.get("TG_MAX_RETRIES", default=5))
|
305
|
-
while not retry_amount >= max_retries:
|
306
|
-
try:
|
307
|
-
message = None
|
308
|
-
if job.media_type == "video":
|
309
|
-
if job.placeholder_message_id:
|
310
|
-
message = await context.bot.edit_message_media(**build_tg_args(update, context, job))
|
311
|
-
else:
|
312
|
-
message = await update.message.reply_video(**build_tg_args(update, context, job))
|
313
|
-
tg_file_ids.append(message.video.file_id)
|
314
|
-
job.tg_file_id = message.video.file_id
|
315
|
-
logging.info("Uploaded video file tg_file_id is '%s'", job.tg_file_id)
|
316
|
-
elif job.media_type == "image":
|
317
|
-
if job.placeholder_message_id:
|
318
|
-
message = await context.bot.edit_message_media(**build_tg_args(update, context, job))
|
319
|
-
else:
|
320
|
-
message = await update.message.reply_photo(**build_tg_args(update, context, job))
|
321
|
-
if message.photo:
|
322
|
-
tg_file_ids.append(message.photo[-1].file_id)
|
323
|
-
job.tg_file_id = message.photo[-1].file_id
|
324
|
-
elif job.media_type == "audio":
|
325
|
-
if job.placeholder_message_id:
|
326
|
-
message = await context.bot.edit_message_media(**build_tg_args(update, context, job))
|
327
|
-
else:
|
328
|
-
message = await update.message.reply_audio(**build_tg_args(update, context, job))
|
329
|
-
if message.audio:
|
330
|
-
tg_file_ids.append(message.audio.file_id)
|
331
|
-
job.tg_file_id = message.audio.file_id
|
332
|
-
elif job.media_type == "collection":
|
333
|
-
sent_messages = await update.message.reply_media_group(**build_tg_args(update, context, job))
|
334
|
-
if job.placeholder_message_id:
|
335
|
-
await remove_placeholder(update, context, job.placeholder_message_id)
|
336
|
-
for i, msg in enumerate(sent_messages):
|
337
|
-
if msg.video:
|
338
|
-
tg_file_ids.append(msg.video.file_id + ':video')
|
339
|
-
if job.media_collection:
|
340
|
-
job.media_collection[i].tg_file_id = msg.video.file_id + ':video'
|
341
|
-
elif msg.photo:
|
342
|
-
tg_file_ids.append(msg.photo[-1].file_id + ':image')
|
343
|
-
if job.media_collection:
|
344
|
-
job.media_collection[i].tg_file_id = msg.photo[-1].file_id + ':image'
|
345
|
-
logging.info("Uploaded to Telegram")
|
346
|
-
break
|
347
|
-
except (error.NetworkError, error.TimedOut) as e:
|
348
|
-
logging.error("Failed to upload due telegram limitations :(")
|
349
|
-
logging.exception(e)
|
350
|
-
if not "Request Entity Too Large" in e.message:
|
351
|
-
logging.info("TG upload will be retried. Configuration `TG_MAX_RETRIES` values is %d.", max_retries)
|
352
|
-
|
353
|
-
if "Message to reply not found" in e.message:
|
354
|
-
logging.warning("No message to reply found. Looks like original message was deleted by author.")
|
355
|
-
job.message_id = None
|
356
|
-
continue
|
357
|
-
|
358
|
-
if retry_amount+1 >= max_retries or "Request Entity Too Large" in e.message:
|
359
|
-
msg = ""
|
360
|
-
if e.message:
|
361
|
-
msg = "Telegram error: %s" % str(e.message)
|
362
|
-
else:
|
363
|
-
msg = "Unfortunately, Telegram limits were exceeded. Your video size is %.2f MB." % job.media_info["filesize"]
|
364
|
-
await remove_placeholder(update, context, job.placeholder_message_id)
|
365
|
-
await send_text(
|
366
|
-
update,
|
367
|
-
context,
|
368
|
-
job.message_id,
|
369
|
-
msg
|
370
|
-
)
|
371
|
-
break
|
372
|
-
retry_amount += 1
|
373
|
-
except Exception as e:
|
374
|
-
logging.error("Error occurred!")
|
375
|
-
logging.exception(e)
|
376
|
-
finally:
|
377
|
-
if job.media_type == "collection":
|
378
|
-
for j in job.media_collection:
|
379
|
-
if os.path.exists(j.local_media_path):
|
380
|
-
os.unlink(j.local_media_path)
|
381
|
-
else:
|
382
|
-
if os.path.exists(job.local_media_path):
|
383
|
-
os.unlink(job.local_media_path)
|
384
|
-
if job.local_compressed_media_path:
|
385
|
-
if os.path.exists(job.local_compressed_media_path):
|
386
|
-
os.unlink(job.local_compressed_media_path)
|
387
|
-
|
388
|
-
return tg_file_ids
|
389
|
-
|
390
|
-
def extract_origin(url: str) -> Origin:
|
391
|
-
if "instagram.com/" in url:
|
392
|
-
return Origin.INSTAGRAM
|
393
|
-
|
394
|
-
if "youtube.com/" in url and "shorts/" in url:
|
395
|
-
return Origin.YT_SHORTS
|
396
|
-
|
397
|
-
if "youtube.com/" in url and "music." in url:
|
398
|
-
return Origin.YT_MUSIC
|
399
|
-
|
400
|
-
return Origin.UNKNOWN
|
401
|
-
|
402
|
-
async def handler(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
|
403
|
-
if update.message is None:
|
404
|
-
return
|
405
|
-
chat = update.effective_chat
|
406
|
-
effective_message_id = update.message.message_id
|
407
|
-
extractor = URLExtract()
|
408
|
-
urls = extractor.find_urls(update.message.text_html)
|
409
|
-
|
410
|
-
reply_text = "Wut?"
|
411
|
-
if not urls:
|
412
|
-
reply_text = "Your message should contains URLs"
|
413
|
-
else:
|
414
|
-
for url in urls:
|
415
|
-
origin = extract_origin(url)
|
416
|
-
if origin is Origin.UNKNOWN:
|
417
|
-
logging.info("Only Instagram, YouTube Shorts and YouTube Music are now supported. Skipping.")
|
418
|
-
continue
|
419
|
-
entities, tg_file_ids = [], []
|
420
|
-
uniq_id = Storage.compute_uniq(url)
|
421
|
-
try:
|
422
|
-
entities = storage.db_lookup_id(uniq_id)
|
423
|
-
except Exception as e:
|
424
|
-
logging.error("Failed to search link in DB!")
|
425
|
-
logging.exception(e)
|
426
|
-
if entities:
|
427
|
-
tg_file_ids = [i["tg_file_id"] for i in entities]
|
428
|
-
logging.info("URL '%s' is found in DB. Sending with tg_file_ids = '%s'", url, str(tg_file_ids))
|
429
|
-
ent_len = len(entities)
|
430
|
-
if ent_len > 1:
|
431
|
-
await upload_job(
|
432
|
-
update,
|
433
|
-
context,
|
434
|
-
UploadJob(
|
435
|
-
tg_file_id=",".join(tg_file_ids),
|
436
|
-
message_id=effective_message_id,
|
437
|
-
media_type="collection"
|
438
|
-
)
|
439
|
-
)
|
440
|
-
elif ent_len:
|
441
|
-
media_type = entities.pop()["media_type"]
|
442
|
-
await upload_job(
|
443
|
-
update,
|
444
|
-
context,
|
445
|
-
UploadJob(
|
446
|
-
tg_file_id=tg_file_ids.pop(),
|
447
|
-
message_id=effective_message_id,
|
448
|
-
media_type=media_type
|
449
|
-
)
|
450
|
-
)
|
451
|
-
else:
|
452
|
-
async def upload_wrapper(job: UploadJob) -> None:
|
453
|
-
try:
|
454
|
-
if job.job_failed and job.job_failed_msg:
|
455
|
-
if job.placeholder_message_id:
|
456
|
-
await remove_placeholder(update, context, job.placeholder_message_id)
|
457
|
-
return await send_text(update, context, reply_id=job.message_id, text=job.job_failed_msg)
|
458
|
-
if job.job_warning and job.job_warning_msg:
|
459
|
-
return await update_placeholder_text(update, context, job.placeholder_message_id, job.job_warning_msg)
|
460
|
-
tg_file_ids = await upload_job(update, context, job)
|
461
|
-
if tg_file_ids:
|
462
|
-
if job.media_type == "collection" and job.save_items:
|
463
|
-
for i in job.media_collection:
|
464
|
-
storage.add_media(tg_file_ids=[i.tg_file_id], media_url=i.effective_url, media_type=i.media_type, origin=origin.value)
|
465
|
-
else:
|
466
|
-
storage.add_media(tg_file_ids=[','.join(tg_file_ids)], media_url=job.url, media_type=job.media_type, origin=origin.value)
|
467
|
-
except Exception as e:
|
468
|
-
logging.error("Exception occurred while performing upload callback!")
|
469
|
-
logging.exception(e)
|
470
|
-
|
471
|
-
try:
|
472
|
-
# create placeholder message for long download
|
473
|
-
placeholder_message_id = await create_placeholder_message(
|
474
|
-
update,
|
475
|
-
context,
|
476
|
-
reply_id=effective_message_id
|
477
|
-
)
|
478
|
-
|
479
|
-
if not placeholder_message_id:
|
480
|
-
await send_text(
|
481
|
-
update=update,
|
482
|
-
context=context,
|
483
|
-
reply_id=effective_message_id,
|
484
|
-
text="Failed to create message placeholder. Please check your bot Internet connection.")
|
485
|
-
return
|
486
|
-
|
487
|
-
uploader.add_callback(
|
488
|
-
placeholder_message_id,
|
489
|
-
upload_wrapper,
|
490
|
-
update,
|
491
|
-
context
|
492
|
-
)
|
493
|
-
|
494
|
-
downloader.queue_task(DownloadJob.build(
|
495
|
-
url=url,
|
496
|
-
placeholder_message_id=placeholder_message_id,
|
497
|
-
message_id=effective_message_id,
|
498
|
-
in_process=uploader.is_inprocess(uniq_id),
|
499
|
-
uniq_id=uniq_id,
|
500
|
-
job_origin = origin
|
501
|
-
))
|
502
|
-
uploader.set_inprocess(uniq_id)
|
503
|
-
except Exception as e:
|
504
|
-
logging.error("Failed to schedule download task!")
|
505
|
-
logging.exception(e)
|
506
|
-
|
507
|
-
if chat.type not in (Chat.GROUP, Chat.SUPERGROUP) and not urls:
|
508
|
-
await update.message.reply_text(reply_text, reply_to_message_id=effective_message_id)
|
509
|
-
|
510
|
-
@staticmethod
|
511
|
-
def _raise_system_exit() -> None:
|
512
|
-
raise SystemExit
|
513
|
-
|
514
7
|
def main() -> None:
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
# on different commands - answer in Telegram
|
524
|
-
application.add_handler(CommandHandler("start", start))
|
525
|
-
application.add_handler(CommandHandler("random", random))
|
526
|
-
application.add_handler(CommandHandler("help", help_command))
|
527
|
-
|
528
|
-
# on non command i.e message - echo the message on Telegram
|
529
|
-
application.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, handler))
|
530
|
-
|
531
|
-
allow_loop = True
|
532
|
-
loop = None
|
533
|
-
while allow_loop:
|
534
|
-
try:
|
535
|
-
loop = asyncio.get_event_loop()
|
536
|
-
|
537
|
-
stop_signals = (signal.SIGINT, signal.SIGTERM, signal.SIGABRT)
|
538
|
-
for sig in stop_signals or []:
|
539
|
-
loop.add_signal_handler(sig, _raise_system_exit)
|
540
|
-
loop.add_signal_handler(sig, _raise_system_exit)
|
541
|
-
|
542
|
-
uploader = AsyncUploader(
|
543
|
-
storage=storage,
|
544
|
-
pool_size=int(os.environ.get("UPLOAD_POOL_SIZE", default=warp_beacon.scraper.CONST_CPU_COUNT)),
|
545
|
-
loop=loop
|
546
|
-
)
|
547
|
-
downloader = warp_beacon.scraper.AsyncDownloader(
|
548
|
-
workers_count=int(os.environ.get("WORKERS_POOL_SIZE", default=warp_beacon.scraper.CONST_CPU_COUNT)),
|
549
|
-
uploader=uploader
|
550
|
-
)
|
551
|
-
loop.run_until_complete(application.initialize())
|
552
|
-
if application.post_init:
|
553
|
-
loop.run_until_complete(application.post_init(application))
|
554
|
-
loop.run_until_complete(application.updater.start_polling())
|
555
|
-
loop.run_until_complete(application.start())
|
556
|
-
while allow_loop:
|
557
|
-
try:
|
558
|
-
downloader.start()
|
559
|
-
uploader.start()
|
560
|
-
loop.run_forever()
|
561
|
-
except (KeyboardInterrupt, SystemExit) as e:
|
562
|
-
allow_loop = False
|
563
|
-
raise
|
564
|
-
except Exception as e:
|
565
|
-
logging.error("Main loop Telegram error!")
|
566
|
-
logging.exception(e)
|
567
|
-
except (KeyboardInterrupt, SystemExit):
|
568
|
-
logging.debug("Application received stop signal. Shutting down.")
|
569
|
-
except telegram.error.TimedOut as e:
|
570
|
-
logging.error("Telegram connection timeout!")
|
571
|
-
logging.exception(e)
|
572
|
-
time.sleep(2)
|
573
|
-
continue
|
574
|
-
except Exception as e:
|
575
|
-
logging.error("Failed to start application!")
|
576
|
-
logging.exception(e)
|
577
|
-
finally:
|
578
|
-
try:
|
579
|
-
if application.updater.running: # type: ignore[union-attr]
|
580
|
-
loop.run_until_complete(application.updater.stop()) # type: ignore[union-attr]
|
581
|
-
if application.running:
|
582
|
-
loop.run_until_complete(application.stop())
|
583
|
-
if application.post_stop:
|
584
|
-
loop.run_until_complete(application.post_stop(application))
|
585
|
-
loop.run_until_complete(application.shutdown())
|
586
|
-
if application.post_shutdown:
|
587
|
-
loop.run_until_complete(application.post_shutdown(application))
|
588
|
-
finally:
|
589
|
-
loop.close()
|
590
|
-
downloader.stop_all()
|
591
|
-
uploader.stop_all()
|
592
|
-
except Exception as e:
|
593
|
-
logging.exception(e)
|
594
|
-
|
595
|
-
logging.info("Warp Beacon terminated.")
|
596
|
-
|
597
|
-
if __name__ == "__main__":
|
598
|
-
main()
|
8
|
+
bot = Bot(
|
9
|
+
tg_bot_name=os.environ.get("TG_BOT_NAME", default=None),
|
10
|
+
tg_token=os.environ.get("TG_TOKEN", default=None),
|
11
|
+
tg_api_id=os.environ.get("TG_API_ID", default=None),
|
12
|
+
tg_api_hash=os.environ.get("TG_API_HASH", default=None)
|
13
|
+
)
|
14
|
+
bot.start()
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: warp_beacon
|
3
|
-
Version:
|
3
|
+
Version: 2.0.0
|
4
4
|
Summary: Telegram bot for expanding external media links
|
5
5
|
Home-page: https://github.com/sb0y/warp_beacon
|
6
6
|
Author: Andrey Bagrintsev
|
@@ -225,7 +225,9 @@ Requires-Python: >=3.10
|
|
225
225
|
Description-Content-Type: text/markdown
|
226
226
|
License-File: LICENSE
|
227
227
|
Requires-Dist: ffmpeg-python
|
228
|
-
Requires-Dist:
|
228
|
+
Requires-Dist: uvloop
|
229
|
+
Requires-Dist: tgcrypto
|
230
|
+
Requires-Dist: pyrogram
|
229
231
|
Requires-Dist: pytubefix
|
230
232
|
Requires-Dist: av
|
231
233
|
Requires-Dist: urlextract
|
@@ -0,0 +1,40 @@
|
|
1
|
+
etc/warp_beacon/warp_beacon.conf,sha256=xihOuNBqVXGNdmMO14wwYNrqSQ4Z1wlAjp-xJk5SIvA,283
|
2
|
+
lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuwwPfa2mDLUA,372
|
3
|
+
var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
|
4
|
+
warp_beacon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
|
+
warp_beacon/__version__.py,sha256=qOioaGGl5NerFveysRg4FBs-w1ZS2KuRyW5CW5zOmU8,23
|
6
|
+
warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
|
7
|
+
warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
8
|
+
warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
|
9
|
+
warp_beacon/jobs/__init__.py,sha256=ED8_tPle4iL4kqNW0apAVkgNQtRRTnYfAJwBjO1g0JY,180
|
10
|
+
warp_beacon/jobs/abstract.py,sha256=kvPTxnzf8bYtggJgVZRYV3DVMCpfSia2pFrHLKhn7lY,2358
|
11
|
+
warp_beacon/jobs/download_job.py,sha256=5HiPcnJppFMhO14___3eSkoMygM3y-vhpGkMAuNhK7s,854
|
12
|
+
warp_beacon/jobs/types.py,sha256=Ae8zINgbs7cOcYkYoOCOACA7duyhnIGMQAJ_SJB1QRQ,176
|
13
|
+
warp_beacon/jobs/upload_job.py,sha256=Vaogc4vbpAfyaT4VkIHEPLFRELmM44TDqkmnPYh3Ymc,740
|
14
|
+
warp_beacon/mediainfo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
15
|
+
warp_beacon/mediainfo/abstract.py,sha256=ZR2JMuRpoh7nDNov9a8YkAfr6BI2HXnXzQtVrLgDxjs,1185
|
16
|
+
warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-ZM,651
|
17
|
+
warp_beacon/mediainfo/silencer.py,sha256=yn9w1LVHG8JQy_D45_RzrlEWID1zbj3ecx7nYGUP4ZE,1647
|
18
|
+
warp_beacon/mediainfo/video.py,sha256=AIRy_op_BvehsjarM1rvT5Qo0QWwf-Q6xVVd_aCnbJ4,2505
|
19
|
+
warp_beacon/scraper/__init__.py,sha256=Gv5XxSuvPeJgMTrJrnYAtf3xviXyK6QAX0n6Ml6dIvI,9863
|
20
|
+
warp_beacon/scraper/abstract.py,sha256=cxQxt5eQkieCZsgQwz425iF69pgCrWZ69ItlikPsWHc,1036
|
21
|
+
warp_beacon/scraper/exceptions.py,sha256=9VJBK5ufokyUpT_q7XVFBYC8lK_BSBfo-S1pd6re9VY,1094
|
22
|
+
warp_beacon/scraper/instagram.py,sha256=YDFdrSB1ghw9f43004lFyk2S5QTopFvpyoUYN-qOVdw,8335
|
23
|
+
warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
24
|
+
warp_beacon/scraper/youtube/abstract.py,sha256=XnVhF1DVSYDMuZ34A6T9hyRlUhjJ7-_JPRF9TOaZ_eo,3358
|
25
|
+
warp_beacon/scraper/youtube/music.py,sha256=Gg1J6QnvpplTOcT_KSiNFZAqZyMyy3Gjgo62e9VcRLM,1404
|
26
|
+
warp_beacon/scraper/youtube/shorts.py,sha256=co4lpNTnncIEScVB9htAT2Hy4fvx56z5xgDo6scu6n4,1162
|
27
|
+
warp_beacon/scraper/youtube/youtube.py,sha256=xIrGe2ElPo3889_w5128xCnkdgwG1u6lA-QoachMUfc,1104
|
28
|
+
warp_beacon/storage/__init__.py,sha256=8XsJXq9X7GDlTaWREF4W1PDX9PH5utwhjf5c5M8Bb7o,3378
|
29
|
+
warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
30
|
+
warp_beacon/telegram/bot.py,sha256=W2RuhHTTs7OW1uuDsplhyeRrFauPba0dBTTcrjanshk,10899
|
31
|
+
warp_beacon/telegram/handlers.py,sha256=4C0Z-u6Kw-9tYmgXlK92PSV2HzZpl0GMpu5DblZZzM8,5708
|
32
|
+
warp_beacon/telegram/placeholder_message.py,sha256=wLMoiDdQGMxRF4R9vbGDjP9awyODheLEasSVKsP4Mvg,6381
|
33
|
+
warp_beacon/telegram/utils.py,sha256=3MNlRoYvdoiYU7uBWu4x10QynSW_rE6Ei4GDyZxwVFs,1793
|
34
|
+
warp_beacon/uploader/__init__.py,sha256=chX9oOrwO05O7DFmUfskTAnoKse66r3sY2s4NFF7bmM,4442
|
35
|
+
warp_beacon-2.0.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
36
|
+
warp_beacon-2.0.0.dist-info/METADATA,sha256=GgqPqUtUgZlblRv-shLeO2nTxYa_Cz5V4lX86bGuz0M,18279
|
37
|
+
warp_beacon-2.0.0.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
|
38
|
+
warp_beacon-2.0.0.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
|
39
|
+
warp_beacon-2.0.0.dist-info/top_level.txt,sha256=pu6xG8OO_nCGllnOfAZ6QpVfivtmHVxPlYK8SZzUDqA,840
|
40
|
+
warp_beacon-2.0.0.dist-info/RECORD,,
|