warp-beacon 1.2.6__py3-none-any.whl → 2.0.0__py3-none-any.whl

warp_beacon/scraper/youtube/music.py

@@ -1,111 +1,15 @@
-import os
-import io
-import pathlib
-import time
-
-import socket
-import ssl
-
-from typing import Callable, Union
-
-import requests
-import urllib
-import http.client
-
-from PIL import Image
+from warp_beacon.jobs.types import JobType
+from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
 
 from pytubefix import YouTube
-from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
-
-from warp_beacon.mediainfo.abstract import MediaInfoAbstract
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, extract_exception_message
-from warp_beacon.scraper.abstract import ScraperAbstract
 
 import logging
 
-DOWNLOAD_DIR = "/tmp"
-
-class YoutubeMusicScraper(ScraperAbstract):
-
-	def __init__(self) -> None:
-		pass
-
-	def __del__(self) -> None:
-		pass
-
-	def remove_tmp_files(self) -> None:
-		for i in os.listdir(DOWNLOAD_DIR):
-			if "yt_download_" in i:
-				os.unlink("%s/%s" % (DOWNLOAD_DIR, i))
-
-	def _download_hndlr(self, func: Callable, *args: tuple[str], **kwargs: dict[str]) -> Union[str, dict]:
-		ret_val = ''
-		max_retries = int(os.environ.get("YT_MUSIC_MAX_RETRIES", default=6))
-		pause_secs = int(os.environ.get("YT_MUSIC_PAUSE_BEFORE_RETRY", default=3))
-		timeout = int(os.environ.get("YT_MUSIC_TIMEOUT", default=60))
-		timeout_increment = int(os.environ.get("YT_MUSIC_TIMEOUT_INCREMENT", default=60))
-		retries = 0
-		while max_retries >= retries:
-			try:
-				kwargs["timeout"] = timeout
-				ret_val = func(*args, **kwargs)
-				break
-			except MaxRetriesExceeded:
-				# do noting, not interested
-				pass
-			#except http.client.IncompleteRead as e:
-			except (socket.timeout, 
-					ssl.SSLError, 
-					http.client.IncompleteRead, 
-					http.client.HTTPException, 
-					requests.RequestException, 
-					urllib.error.URLError, 
-					urllib.error.HTTPError) as e:
-				if hasattr(e, "code") and int(e.code) == 403:
-					raise Unavailable(extract_exception_message(e))
-				logging.warning("Youtube read timeout! Retrying in %d seconds ...", pause_secs)
-				logging.info("Your `YT_MUSIC_MAX_RETRIES` values is %d", max_retries)
-				logging.exception(extract_exception_message(e))
-				if max_retries <= retries:
-					self.remove_tmp_files()
-					raise TimeOut(extract_exception_message(e))
-				retries += 1
-				timeout += timeout_increment
-				time.sleep(pause_secs)
-			except (VideoUnavailable, VideoPrivate) as e:
-				raise Unavailable(extract_exception_message(e))
-
-		return ret_val
-
-	def rename_local_file(self, filename: str) -> str:
-		if not os.path.exists(filename):
-			raise NameError("No file provided")
-		path_info = pathlib.Path(filename)
-		ext = path_info.suffix
-		old_filename = path_info.stem
-		time_name = str(time.time()).replace('.', '_')
-		new_filename = "%s%s" % (time_name, ext)
-		new_filepath = "%s/%s" % (os.path.dirname(filename), new_filename)
-
-		os.rename(filename, new_filepath)
-
-		return new_filepath
-
-	def download_thumbnail(self, url: str) -> Union[io.BytesIO, None]:
-		try:
-			reply = requests.get(url, stream=True)
-			if reply.ok and reply.status_code == 200:
-				image = Image.open(io.BytesIO(reply.content))
-				image = MediaInfoAbstract.shrink_image_to_fit(image)
-				io_buf = io.BytesIO()
-				image.save(io_buf, format='JPEG')
-				io_buf.seek(0)
-				return io_buf
-		except Exception as e:
-			logging.error("Failed to download download thumbnail!")
-			logging.exception(e)
-
-		return None
+class YoutubeMusicScraper(YoutubeAbstract):
+	YT_MAX_RETRIES_DEFAULT = 6
+	YT_PAUSE_BEFORE_RETRY_DEFAULT = 3
+	YT_TIMEOUT_DEFAULT = 2
+	YT_TIMEOUT_INCREMENT_DEFAULT = 60
 
 	def _download(self, url: str, timeout: int = 0) -> list:
 		res = []
@@ -116,12 +20,12 @@ class YoutubeMusicScraper(ScraperAbstract):
 		stream = yt.streams.get_audio_only()
 		if stream:
 			logging.info("Announced audio file size: '%d'", stream.filesize)
-			if stream.filesize > 5e+7:
-				logging.warning("Downloading size reported by YouTube is over than 50 mb!")
-				raise FileTooBig("YouTube file is larger than 50 mb")
+			if stream.filesize > 2e+9:
+				logging.warning("Downloading size reported by YouTube is over than 2 GB!")
+				raise FileTooBig("YouTube file is larger than 2 GB")
 			logging.info("Operation timeout is '%d'", timeout)
 			local_file = stream.download(
-				output_path=DOWNLOAD_DIR,
+				output_path=self.DOWNLOAD_DIR,
 				max_retries=0,
 				timeout=timeout,
 				skip_existing=False,
@@ -134,7 +38,7 @@ class YoutubeMusicScraper(ScraperAbstract):
 				"performer": yt.author,
 				"thumb": thumbnail,
 				"canonical_name": stream.title,
-				"media_type": "audio"
+				"media_type": JobType.AUDIO
 			})
 
 		return res

warp_beacon/scraper/youtube/shorts.py

@@ -1,93 +1,40 @@
-import os
-import pathlib
-import time
-
-import socket
-import ssl
-
-from typing import Callable, Union
-
-import requests
-import urllib
-import http.client
+from warp_beacon.jobs.types import JobType
+from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
 
 from pytubefix import YouTube
-from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
-
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, extract_exception_message
-from warp_beacon.scraper.abstract import ScraperAbstract
 
 import logging
 
-class YoutubeShortsScraper(ScraperAbstract):
-	def __init__(self) -> None:
-		pass
-
-	def __del__(self) -> None:
-		pass
-
-	def remove_tmp_files(self) -> None:
-		for i in os.listdir(DOWNLOAD_DIR):
-			if "yt_download_" in i:
-				os.unlink("%s/%s" % (DOWNLOAD_DIR, i))
-
-	def _download_hndlr(self, func: Callable, *args: tuple[str], **kwargs: dict[str]) -> Union[str, dict]:
-		ret_val = ''
-		max_retries = int(os.environ.get("YT_MAX_RETRIES", default=8))
-		pause_secs = int(os.environ.get("YT_PAUSE_BEFORE_RETRY", default=3))
-		retries = 0
-		while max_retries >= retries:
-			try:
-				ret_val = func(*args, **kwargs)
-				break
-			except MaxRetriesExceeded:
-				# do noting, not interested
-				pass
-			except (socket.timeout, ssl.SSLError, http.client.HTTPException, requests.RequestException, urllib.error.URLError) as e:
-				if hasattr(e, "code") and int(e.code) == 403:
-					raise Unavailable(extract_exception_message(e))
-				logging.warning("Youtube read timeout! Retrying in %d seconds ...", pause_secs)
-				logging.info("Your `YT_MAX_RETRIES` values is %d", max_retries)
-				logging.exception(extract_exception_message(e))
-				if max_retries <= retries:
-					self.remove_tmp_files()
-					raise TimeOut(extract_exception_message(e))
-				retries += 1
-				time.sleep(pause_secs)
-			except (VideoUnavailable, VideoPrivate) as e:
-				raise Unavailable(extract_exception_message(e))
-
-		return ret_val
-
-	def rename_local_file(self, filename: str) -> str:
-		if not os.path.exists(filename):
-			raise NameError("No file provided")
-		path_info = pathlib.Path(filename)
-		ext = path_info.suffix
-		old_filename = path_info.stem
-		time_name = str(time.time()).replace('.', '_')
-		new_filename = "%s%s" % (time_name, ext)
-		new_filepath = "%s/%s" % (os.path.dirname(filename), new_filename)
-
-		os.rename(filename, new_filepath)
-
-		return new_filepath
+class YoutubeShortsScraper(YoutubeAbstract):
+	YT_MAX_RETRIES_DEFAULT = 8
+	YT_PAUSE_BEFORE_RETRY_DEFAULT = 3
+	YT_TIMEOUT_DEFAULT = 2
+	YT_TIMEOUT_INCREMENT_DEFAULT = 60
 
-	def _download(self, url: str) -> list:
+	def _download(self, url: str, timeout: int = 0) -> list:
 		res = []
-		timeout = int(os.environ.get("YT_TIMEOUT", default=2))
+		thumbnail = None
 		yt = YouTube(url)
 		stream = yt.streams.get_highest_resolution()
+		if yt and yt.thumbnail_url:
+			logging.debug("Generation thumb for Shorts ...")
+			thumbnail = self.download_thumbnail(yt.thumbnail_url)
 		if stream:
 			local_file = stream.download(
-				output_path="/tmp",
+				output_path=self.DOWNLOAD_DIR,
 				max_retries=0,
 				timeout=timeout,
 				skip_existing=False,
 				filename_prefix="yt_download_"
 			)
 			logging.debug("Temp filename: '%s'", local_file)
-			res.append({"local_media_path": self.rename_local_file(local_file), "media_type": "video"})
+			res.append({
+				"local_media_path": self.rename_local_file(local_file),
+				"performer": yt.author,
+				"thumb": thumbnail,
+				"canonical_name": stream.title,
+				"media_type": JobType.VIDEO
+			})
 
 		return res
 

warp_beacon/scraper/youtube/youtube.py

@@ -0,0 +1,41 @@
+from warp_beacon.jobs.types import JobType
+from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
+
+from pytubefix import YouTube
+
+import logging
+
+class YoutubeScraper(YoutubeAbstract):
+	YT_MAX_RETRIES_DEFAULT = 8
+	YT_PAUSE_BEFORE_RETRY_DEFAULT = 3
+	YT_TIMEOUT_DEFAULT = 2
+	YT_TIMEOUT_INCREMENT_DEFAULT = 60
+
+	def _download(self, url: str, timeout: int = 0) -> list:
+		res = []
+		thumbnail = None
+		yt = YouTube(url)
+		if yt and yt.thumbnail_url:
+			thumbnail = self.download_thumbnail(yt.thumbnail_url)
+		stream = yt.streams.get_highest_resolution()
+		if stream:
+			local_file = stream.download(
+				output_path=self.DOWNLOAD_DIR,
+				max_retries=0,
+				timeout=timeout,
+				skip_existing=False,
+				filename_prefix="yt_download_"
+			)
+			logging.debug("Temp filename: '%s'", local_file)
+			res.append({
+				"local_media_path": self.rename_local_file(local_file),
+				"performer": yt.author,
+				"thumb": thumbnail,
+				"canonical_name": stream.title,
+				"media_type": JobType.VIDEO
+			})
+
+		return res
+
+	def download(self, url: str) -> list:
+		return self._download_hndlr(self._download, url)

warp_beacon/storage/__init__.py

@@ -1,11 +1,19 @@
 import os
 #from typing import Optional
-import logging
+from enum import Enum
 
 from urllib.parse import urlparse, parse_qs
 
 from pymongo import MongoClient
 
+import logging
+
+class UrlParseMode(Enum):
+	OTHER = 0
+	YT_MUSIC = 1
+	YT_SHORTS = 2
+	YOUTUBE = 3
+
 VIDEO_STORAGE_DIR = os.environ.get("VIDEO_STORAGE_DIR", default="/var/warp_beacon/videos")
 
 class Storage(object):
@@ -28,12 +36,22 @@ class Storage(object):
 
 	@staticmethod
 	def compute_uniq(url: str) -> str:
-		if "music.youtube.com" in url:
-			qs = parse_qs(urlparse(url).query)
-			yt_vid_id = qs.get('v', None)
+		parse_mode = UrlParseMode.OTHER
+		if "music.youtube.com/" in url:
+			parse_mode = UrlParseMode.YT_MUSIC
+		elif "youtube.com/shorts/" in url:
+			parse_mode = UrlParseMode.YT_SHORTS
+		elif "youtube.com/" in url:
+			parse_mode = UrlParseMode.YOUTUBE
+
+		if parse_mode is not UrlParseMode.OTHER and parse_mode is not UrlParseMode.YT_SHORTS:
+			purl = urlparse(url)
+			qs = parse_qs(purl.query)
+			yt_vid_id_list = qs.get('v', None)
+			yt_vid_id = yt_vid_id_list.pop() if yt_vid_id_list else ""
 			if yt_vid_id:
-				path = urlparse(url).path.strip('/').replace("watch", "yt_music")
-				return "%s/%s" % (path, yt_vid_id)
+				path = urlparse(url).path.strip('/').replace("watch", ("yt_music" if parse_mode is UrlParseMode.YT_MUSIC else "youtube"))
+				return ("%s/%s" % (path, yt_vid_id)).strip('/')
 			else:
 				raise ValueError("Failed to generate uniq_id for url '%s'", url)
 			
@@ -65,6 +83,9 @@ class Storage(object):
 		uniq_id = self.compute_uniq(media_url)
 		media_ids = []
 		for tg_file_id in tg_file_ids:
+			if not tg_file_id:
+				logging.warning("Passed empty `tg_file_id`! Skipping.")
+				continue
 			if self.db_lookup_id(uniq_id):
 				logging.info("Detected existing uniq_id, skipping storage write operation")
 				continue

warp_beacon/telegram/bot.py

@@ -0,0 +1,318 @@
+import os, io
+import signal
+
+import uvloop
+import asyncio
+
+from pyrogram import Client, filters
+from pyrogram.enums import ParseMode
+from pyrogram.handlers import MessageHandler
+from pyrogram.types import Message, InputMedia, InputMediaAudio, InputMediaPhoto, InputMediaVideo, InputMediaAnimation, InputMediaDocument, InlineKeyboardButton, InlineKeyboardMarkup
+from pyrogram.errors import RPCError, FloodWait, NetworkMigrate, BadRequest, MultiMediaTooLong
+
+from warp_beacon.__version__ import __version__
+from warp_beacon.telegram.handlers import Handlers
+import warp_beacon.scraper
+from warp_beacon.telegram.placeholder_message import PlaceholderMessage
+from warp_beacon.storage import Storage
+from warp_beacon.uploader import AsyncUploader
+from warp_beacon.jobs.download_job import DownloadJob
+from warp_beacon.jobs.upload_job import UploadJob
+from warp_beacon.jobs import Origin
+from warp_beacon.jobs.types import JobType
+from warp_beacon.telegram.utils import Utils
+
+import logging
+
+class Bot(object):
+	storage = Storage()
+	uploader = None
+	downloader = None
+	allow_loop = True
+	client = None
+	handlers = None
+	placeholder = None
+
+	def __init__(self, tg_bot_name: str, tg_token: str, tg_api_id: str, tg_api_hash: str) -> None:
+		# Enable logging
+		logging.basicConfig(
+			format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO
+		)
+
+		logging.info(f"Starting Warp Beacon version '{__version__}' ...")
+
+		workers_amount = min(32, os.cpu_count() + 4)
+
+		uvloop.install()
+		self.client = Client(
+			name=tg_bot_name,
+			app_version=__version__,
+			bot_token=tg_token,
+			api_id=tg_api_id,
+			api_hash=tg_api_hash,
+			workdir='/',
+			workers=int(os.environ.get("TG_WORKERS_POOL_SIZE", default=workers_amount))
+		)
+
+		this = self
+		def __terminator() -> None:
+			this.stop()
+
+		stop_signals = (signal.SIGINT, signal.SIGTERM, signal.SIGABRT)
+		for sig in stop_signals:
+			self.client.loop.add_signal_handler(sig, __terminator)
+
+		self.uploader = AsyncUploader(
+			storage=self.storage,
+			pool_size=int(os.environ.get("UPLOAD_POOL_SIZE", default=workers_amount)),
+			loop=self.client.loop
+		)
+		self.downloader = warp_beacon.scraper.AsyncDownloader(
+			workers_count=int(os.environ.get("WORKERS_POOL_SIZE", default=workers_amount)),
+			uploader=self.uploader
+		)
+
+		self.downloader.start()
+		self.uploader.start()
+
+		self.handlers = Handlers(self)
+
+		self.client.add_handler(MessageHandler(self.handlers.start, filters.command("start")))
+		self.client.add_handler(MessageHandler(self.handlers.help, filters.command("help")))
+		self.client.add_handler(MessageHandler(self.handlers.random, filters.command("random")))
+		self.client.add_handler(MessageHandler(self.handlers.handler))
+
+		self.placeholder = PlaceholderMessage(self)
+
+		self.client.run()
+
+	def __del__(self) -> None:
+		self.stop()
+		logging.info("Warp Beacon terminated.")
+
+	def start(self) -> None:
+		self.client.run()
+
+	def stop(self) -> None:
+		logging.info("Warp Beacon terminating. This may take a while ...")
+		self.downloader.stop_all()
+		self.uploader.stop_all()
+		#self.client.stop()
+
+	async def send_text(self, chat_id: int, text: str, reply_id: int = None) -> int:
+		try:
+			message_reply = await self.client.send_message(
+				chat_id=chat_id,
+				text=text,
+				parse_mode=ParseMode.MARKDOWN,
+				reply_to_message_id=reply_id
+			)
+			return message_reply.id
+		except Exception as e:
+			logging.error("Failed to send text message!")
+			logging.exception(e)
+
+		return 0
+
+	def build_tg_args(self, job: UploadJob) -> dict:
+		args = {}
+		if job.media_type == JobType.VIDEO:
+			if job.tg_file_id:
+				if job.placeholder_message_id:
+					args["media"] = InputMediaVideo(media=job.tg_file_id.replace(":video", ''), supports_streaming=True)
+				else:
+					args["video"] = job.tg_file_id.replace(":video", '')
+			else:
+				args["media"] = InputMediaVideo(
+					media=job.local_media_path,
+					supports_streaming=True,
+					width=job.media_info["width"],
+					height=job.media_info["height"],
+					duration=int(job.media_info["duration"]),
+					thumb=job.media_info["thumb"]
+				)
+				args["file_name"] = "downloaded_via_warp_beacon_bot%s" % (os.path.splitext(job.local_media_path)[-1])
+		elif job.media_type == JobType.IMAGE:
+			if job.tg_file_id:
+				if job.placeholder_message_id:
+					args["media"] = InputMediaPhoto(media=job.tg_file_id.replace(":image", ''))
+				else:
+					args["photo"] = job.tg_file_id.replace(":image", '')
+			else:
+				args["media"] = InputMediaPhoto(
+					media=job.local_media_path
+				)
+		elif job.media_type == JobType.AUDIO:
+			if job.tg_file_id:
+				if job.placeholder_message_id:
+					args["media"] = InputMediaAudio(
+						media=job.tg_file_id.replace(":audio", '')
+					)
+				else:
+					args["audio"] = job.tg_file_id.replace(":audio", '')
+			else:
+				args["media"] = InputMediaAudio(
+					media=job.local_media_path,
+					performer=job.media_info["performer"],
+					thumb=job.media_info["thumb"],
+					duration=job.media_info["duration"],
+					title=job.canonical_name,
+				)
+				#args["file_name"] = "%s%s" % (job.canonical_name, os.path.splitext(job.local_media_path)[-1]),
+		elif job.media_type == JobType.ANIMATION:
+			if job.tg_file_id:
+				if job.placeholder_message_id:
+					args["media"] = InputMediaAnimation(
+						media=job.tg_file_id.replace(":animation", '')
+					)
+				else:
+					args["animation"] = job.tg_file_id.replace(":animation", '')
+			else:
+				args["media"] = InputMediaAudio(
+					media=job.local_media_path,
+					performer=job.media_info["performer"],
+					thumb=job.media_info["thumb"],
+					duration=job.media_info["duration"],
+					title=job.canonical_name,
+				)
+		elif job.media_type == JobType.COLLECTION:
+			if job.tg_file_id:
+				args["media"] = []
+				for chunk in Utils.chunker(job.tg_file_id.split(','), 10):
+					tg_chunk = []
+					for i in chunk:
+						tg_id, mtype = i.split(':')
+						ctype = JobType[mtype.upper()]
+						ptr = None
+						if ctype == JobType.VIDEO:
+							ptr = InputMediaVideo(media=tg_id)
+						elif ctype == JobType.IMAGE:
+							ptr = InputMediaPhoto(media=tg_id)
+						elif ctype == JobType.ANIMATION:
+							ptr = InputMediaAnimation(media=tg_id)
+						tg_chunk.append(ptr)
+
+					args["media"].append(tg_chunk)
+			else:
+				mediafs = []
+				for chunk in job.media_collection:
+					tg_chunk = []
+					for j in chunk:
+						if j.media_type == JobType.VIDEO:
+							vid = InputMediaVideo(
+								media=j.local_media_path,
+								supports_streaming=True,
+								width=j.media_info["width"],
+								height=j.media_info["height"],
+								duration=int(j.media_info["duration"]),
+								thumb=j.media_info["thumb"],
+							)
+							tg_chunk.append(vid)
+						elif j.media_type == JobType.IMAGE:
+							photo = InputMediaPhoto(
+								media=j.local_media_path
+							)
+							tg_chunk.append(photo)
+					mediafs.append(tg_chunk)
+				args["media"] = mediafs
+
+		args["chat_id"] = job.chat_id
+
+		# common args
+		if job.placeholder_message_id and job.media_type != JobType.COLLECTION:
+			args["message_id"] = job.placeholder_message_id
+		else:
+			args["disable_notification"] = True
+			args["reply_to_message_id"] = job.message_id
+
+		if os.environ.get("ENABLE_DONATES", None) == "true" and job.media_type != JobType.COLLECTION:
+			args["reply_markup"] = InlineKeyboardMarkup([[InlineKeyboardButton("❤ Donate", url=os.environ.get("DONATE_LINK", "https://pay.cryptocloud.plus/pos/W5BMtNQt5bJFoW2E"))]])
+
+		return args
+
+	async def upload_job(self, job: UploadJob) -> list[str]:
+		timeout = int(os.environ.get("TG_WRITE_TIMEOUT", default=120))
+		tg_file_ids = []
+		try:
+			retry_amount = 0
+			max_retries = int(os.environ.get("TG_MAX_RETRIES", default=5))
+			while not retry_amount >= max_retries:
+				try:
+					reply_message = None
+					if job.media_type in (JobType.VIDEO, JobType.IMAGE, JobType.AUDIO):
+						if job.placeholder_message_id:
+							reply_message = await self.client.edit_message_media(**self.build_tg_args(job))
+						else:
+							send_funcs = {
+								JobType.VIDEO: self.client.send_video,
+								JobType.IMAGE: self.client.send_photo,
+								JobType.AUDIO: self.client.send_audio,
+								JobType.ANIMATION: self.client.send_animation
+							}
+							try:
+								reply_message = await send_funcs[job.media_type](**self.build_tg_args(job))
+							except ValueError as e:
+								err_text = str(e)
+								if "Expected" in err_text:
+									logging.warning("Expectations exceeded reality.")
+									logging.warning(err_text)
+									expectation, reality = Utils.parse_expected_patronum_error(err_text)
+									job_args = self.build_tg_args(job)
+									job_args[reality.value.lower()] = job_args.pop(expectation.value.lower())
+									reply_message = await send_funcs[reality](**job_args)
+
+						tg_file_id = Utils.extract_file_id(reply_message)
+						tg_file_ids.append(tg_file_id)
+						job.tg_file_id = tg_file_id
+						logging.info("Uploaded media file with type '%s' tg_file_id is '%s'", job.media_type.value, job.tg_file_id)
+					elif job.media_type == JobType.COLLECTION:
+						col_job_args = self.build_tg_args(job)
+						sent_messages = []
+						for i, media_chunk in enumerate(col_job_args["media"]):
+							messages = await self.client.send_media_group(
+								chat_id=job.chat_id,
+								reply_to_message_id=job.message_id,
+								media=media_chunk,
+							)
+							sent_messages += messages
+							if job.media_collection:
+								for j, chunk in enumerate(media_chunk):
+									tg_file_id = Utils.extract_file_id(messages[j])
+									if tg_file_id:
+										job.media_collection[i][j].tg_file_id = tg_file_id
+							if i == 0 and job.placeholder_message_id:
+								await self.placeholder.remove(job.chat_id, job.placeholder_message_id)
+						for msg in sent_messages:
+							if msg.video:
+								tg_file_ids.append(msg.video.file_id + ':video')
+							elif msg.photo:
+								tg_file_ids.append(msg.photo.file_id + ':image')
+					logging.info("Uploaded to Telegram")
+					break
+				except MultiMediaTooLong as e:
+					logging.error("Failed to upload due telegram limitations :(")
+					logging.exception(e)
+					await self.placeholder.remove(job.chat_id, job.placeholder_message_id)
+					await self.send_text(job.chat_id, e.MESSAGE, job.message_id)
+					break
+				except (NetworkMigrate, BadRequest) as e:
+					logging.error("Network error. Check you Internet connection.")
+					logging.exception(e)
+
+					if retry_amount+1 >= max_retries:
+						msg = ""
+						if e.MESSAGE:
+							msg = "Telegram error: %s" % str(e.MESSAGE)
+						else:
+							msg = "Unfortunately, Telegram limits were exceeded. Your media size is %.2f MB." % job.media_info["filesize"]
+						await self.placeholder.remove(job.chat_id, job.placeholder_message_id)
+						await self.send_text(job.chat_id, msg, job.message_id)
+						break
+				retry_amount += 1
+		except Exception as e:
+			logging.error("Error occurred!")
+			logging.exception(e)
+		finally:
+			job.remove_files()
+
+		return tg_file_ids