warp-beacon 1.0.9__py3-none-any.whl → 1.1.2__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "1.0.9"
+__version__ = "1.1.2"
 

warp_beacon/jobs/__init__.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+class Origin(Enum):
+	INSTAGRAM = "instagram"
+	YT_SHORTS = "yt_shorts"
+	UNKNOWN = "unknown"

warp_beacon/jobs/abstract.py

@@ -3,6 +3,8 @@ from typing import TypedDict
 from typing_extensions import Unpack
 import uuid
 
+from warp_beacon.jobs import Origin
+
 class JobSettings(TypedDict):
 	job_id: uuid.UUID
 	message_id: int
@@ -22,6 +24,7 @@ class JobSettings(TypedDict):
 	effective_url: str
 	save_items: bool
 	media_collection: list
+	job_origin: Origin
 
 class AbstractJob(ABC):
 	job_id: uuid.UUID = None
@@ -42,6 +45,7 @@ class AbstractJob(ABC):
 	effective_url: str = ""
 	save_items: bool = False
 	media_collection: list = []
+	job_origin: Origin = Origin.UNKNOWN
 
 	def __init__(self, **kwargs: Unpack[JobSettings]) -> None:
 		if kwargs:

warp_beacon/mediainfo/abstract.py

@@ -0,0 +1,28 @@
+import os
+from abc import ABC, abstractmethod
+
+import av
+
+class MediaInfoAbstract(ABC):
+	filename = ""
+	container = None
+	duration = 0.0
+
+	def __init__(self, filename: str) -> None:
+		self.filename = filename
+		self.container = av.open(file=self.filename, mode='r')
+
+	def __del__(self) -> None:
+		if self.container:
+			self.container.close()
+
+	def get_duration(self) -> float:
+		return self.duration
+
+	@staticmethod
+	def get_filesize(filename: str) -> float:
+		return os.stat(filename).st_size
+
+	@abstractmethod
+	def get_finfo(cls, except_info: tuple=()) -> dict:
+		raise NotImplementedError

warp_beacon/mediainfo/audio.py

@@ -0,0 +1,19 @@
+from warp_beacon.mediainfo.abstract import MediaInfoAbstract
+
+class AudioInfo(MediaInfoAbstract):
+	def __init__(self, filename: str) -> None:
+		super(AudioInfo, self).__init__(filename)
+		if self.container:
+			stream_list = self.container.streams.get(audio=0)
+			if stream_list:
+				stream = stream_list[0]
+				time_base = stream.time_base
+				self.duration = float(stream.duration * time_base)
+
+	def get_finfo(self, except_info: tuple=()) -> dict:
+		res = {}
+		if "duration" not in except_info:
+			res["duration"] = round(self.get_duration())
+		if "filesize" not in except_info:
+			res["filesize"] = AudioInfo.get_filesize(self.filename)
+		return res

warp_beacon/mediainfo/video.py

@@ -10,24 +10,31 @@ class VideoInfo(object):
 	width = 0
 	height = 0
 	duration = 0.0
-	ffmpeg = None
 	filename = ""
+	container = None
 
 	def __init__(self, filename: str) -> None:
 		self.filename = filename
-		with av.open(file=self.filename, mode='r') as container:
-			stream = container.streams.video[0]
+		self.container = av.open(file=self.filename, mode='r')
+		
+		if self.container:
+			stream = self.container.streams.video[0]
 			time_base = stream.time_base
 			self.duration = float(stream.duration * time_base)
 			framerate = stream.average_rate
 			frame_container_pts = round((1 / framerate) / time_base)
-			container.seek(frame_container_pts, backward=True, stream=stream)
-			frame = next(container.decode(video=0))
+			# !
+			self.container.seek(frame_container_pts, backward=True, stream=stream)
+			#
+			frame = next(self.container.decode(stream))
 			self.width = frame.width
 			self.height = frame.height
+			# restore original position after previous frame search
+			self.container.seek(0, backward=False, stream=stream)
 		
 	def __del__(self) -> None:
-		pass
+		if self.container:
+			self.container.close()
 
 	def get_demensions(self) -> dict:
 		return {"width": self.width, "height": self.height}
@@ -59,17 +66,17 @@ class VideoInfo(object):
 	def generate_thumbnail(self) -> Union[io.BytesIO, None]:
 		try:
 			image = None
-			with av.open(file=self.filename, mode='r') as container:
+			if self.container:
 				# Signal that we only want to look at keyframes.
-				stream = container.streams.video[0]
+				stream = self.container.streams.video[0]
 				stream.codec_context.skip_frame = "NONKEY"
 				frame_num = 30
-				time_base = container.streams.video[0].time_base
-				framerate = container.streams.video[0].average_rate
+				time_base = stream.time_base
+				framerate = stream.average_rate
 				frame_container_pts = round((frame_num / framerate) / time_base)
 					
-				container.seek(frame_container_pts, backward=True, stream=container.streams.video[0])
-				frame = next(container.decode(stream))
+				self.container.seek(frame_container_pts, backward=True, stream=stream)
+				frame = next(self.container.decode(stream))
 				
 				image = frame.to_image()
 				#image.save(

warp_beacon/{scrapler → scraper}/__init__.py

@@ -3,12 +3,13 @@ import time
 
 from typing import Optional
 import multiprocessing
-from requests.exceptions import ConnectTimeout, HTTPError
-from instagrapi.exceptions import MediaNotFound, UnknownError, ClientNotFoundError, UserNotFound
+from queue import Empty
 
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable
 from warp_beacon.mediainfo.video import VideoInfo
 from warp_beacon.compress.video import VideoCompress
 from warp_beacon.uploader import AsyncUploader
+from warp_beacon.jobs import Origin
 from warp_beacon.jobs.download_job import DownloadJob
 
 import logging
@@ -16,6 +17,7 @@ import logging
 CONST_CPU_COUNT = multiprocessing.cpu_count()
 
 class AsyncDownloader(object):
+	__JOE_BIDEN_WAKEUP = None
 	workers = []
 	allow_loop = None
 	job_queue = multiprocessing.Queue()
@@ -57,30 +59,41 @@ class AsyncDownloader(object):
 				job = None
 				try:
 					job = self.job_queue.get()
+					if job is self.__JOE_BIDEN_WAKEUP:
+						continue
 					actor = None
 					try:
 						items = []
-						if "instagram.com/" in job.url:
+						if job.job_origin is not Origin.UNKNOWN:
 							if not job.in_process:
-								from warp_beacon.scrapler.instagram import InstagramScrapler
-								actor = InstagramScrapler()
+								actor = None
+								if job.job_origin is Origin.INSTAGRAM:
+									from warp_beacon.scraper.instagram import InstagramScraper
+									actor = InstagramScraper()
+								elif job.job_origin is Origin.YT_SHORTS:
+									from warp_beacon.scraper.youtube.shorts import YoutubeShortsScraper
+									actor = YoutubeShortsScraper()
 								while True:
 									try:
 										logging.info("Downloading URL '%s'", job.url)
 										items = actor.download(job.url)
 										break
-									except ConnectTimeout as e:
-										logging.error("ConnectTimeout download error!")
-										logging.exception(e)
-										time.sleep(2)
-									except (MediaNotFound, ClientNotFoundError, UserNotFound) as e:
-										logging.warning("Not found error occurred!")
+									except (NotFound, Unavailable) as e:
+										logging.warning("Not found or unavailable error occurred!")
 										logging.exception(e)
 										self.uploader.queue_task(job.to_upload_job(
 											job_failed=True,
 											job_failed_msg="Unable to access to media under this URL. Seems like the media is private.")
 										)
 										break
+									except TimeOut as e:
+										logging.warning("Timeout error occurred!")
+										logging.exception(e)
+										self.uploader.queue_task(job.to_upload_job(
+											job_failed=True,
+											job_failed_msg="Failed to download content. Please check you Internet connection or retry amount bot configuration settings.")
+										)
+										break
 									except (UnknownError, Exception) as e:
 										logging.warning("UnknownError occurred!")
 										logging.exception(e)
@@ -105,8 +118,7 @@ class AsyncDownloader(object):
 									for item in items:
 										media_info = {"filesize": 0}
 										if item["media_type"] == "video":
-											logging.info("Instagram media_info: %s", item["media_info"])
-											media_info = self.get_media_info(item["local_media_path"], item["media_info"])
+											media_info = self.get_media_info(item["local_media_path"], item.get("media_info", {}))
 											logging.info("Final media info: %s", media_info)
 											if media_info["filesize"] > 52428800:
 												logging.info("Filesize is '%d' MiB", round(media_info["filesize"] / 1024 / 1024))
@@ -155,27 +167,27 @@ class AsyncDownloader(object):
 							else:
 								logging.info("Job already in work in parallel worker. Redirecting job to upload worker.")
 								self.uploader.queue_task(job.to_upload_job())
-					except HTTPError as e:
-						logging.error("HTTP error inside download worker!")
-						logging.exception(e)
 					except Exception as e:
 						logging.error("Error inside download worker!")
 						logging.exception(e)
 						self.notify_task_failed(job)
 						#self.queue_task(url=item["url"], message_id=item["message_id"], item_in_process=item["in_process"], uniq_id=item["uniq_id"])
-				except multiprocessing.Queue.empty:
+				except Empty:
 					pass
 			except Exception as e:
 				logging.error("Exception occurred inside worker!")
 				logging.exception(e)
 
+		logging.info("Process done")
+
 	def stop_all(self) -> None:
 		self.allow_loop.value = 0
 		for proc in self.workers:
 			if proc.is_alive():
 				logging.info("stopping process #%d", proc.pid)
-				proc.terminate()
-				#proc.join()
+				self.job_queue.put_nowait(self.__JOE_BIDEN_WAKEUP)
+				proc.join()
+				#proc.terminate()
 				logging.info("process #%d stopped", proc.pid)
 		self.workers.clear()
 

warp_beacon/{scrapler → scraper}/abstract.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
+from typing import Callable, Union
 
-class ScraplerAbstract(ABC):
+class ScraperAbstract(ABC):
 	def __init__(self) -> None:
 		pass
 
@@ -8,9 +9,10 @@ class ScraplerAbstract(ABC):
 		pass
 
 	@abstractmethod
-	def scrap(self, url: str) -> str:
+	def download(self, url: str) -> bool:
 		raise NotImplementedError
 
 	@abstractmethod
-	def download(self, url: str) -> bool:
+	def _download_hndlr(self, func: Callable, *args: tuple[str], **kwargs: dict[str]) -> Union[str, dict]:
 		raise NotImplementedError
+

warp_beacon/scraper/exceptions.py

@@ -0,0 +1,42 @@
+from typing import Union
+
+class ScraperError(Exception):
+	def __init__(self, *args, **kwargs):
+		args = list(args)
+		if len(args) > 0:
+			self.message = str(args.pop(0))
+		for key in list(kwargs.keys()):
+			setattr(self, key, kwargs.pop(key))
+		if not self.message:
+			self.message = "{title} ({body})".format(
+				title=getattr(self, "reason", "Unknown"),
+				body=getattr(self, "error_type", vars(self)),
+			)
+		super().__init__(self.message, *args, **kwargs)
+		if hasattr(self, "response") and self.response: 
+			self.code = self.response.status_code
+
+class TimeOut(ScraperError):
+	pass
+
+class NotFound(ScraperError):
+	pass
+
+class Unavailable(ScraperError):
+	pass
+
+class UnknownError(ScraperError):
+	pass
+
+def extract_exception_message(e: Exception) -> str:
+	msg = ""
+	if hasattr(e, "error_string"):
+		msg = e.error_string
+	elif hasattr(e, "message"):
+		msg = e.message
+	elif hasattr(e, "reason"):
+		msg = e.reason
+	elif hasattr(e, "args"):
+		msg = str(e.args)
+		
+	return msg

warp_beacon/{scrapler → scraper}/instagram.py

@@ -1,10 +1,11 @@
 import os
-from pathlib import Path
 import time
-import json
 from typing import Callable, Optional, Union
+from pathlib import Path
+import json
 
 import requests
+from requests.exceptions import ConnectTimeout, HTTPError
 import urllib3
 from urllib.parse import urljoin, urlparse
 import logging
@@ -12,13 +13,14 @@ import logging
 from instagrapi.mixins.story import Story
 from instagrapi.types import Media
 from instagrapi import Client
-from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes
+from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotFound, ClientNotFoundError, UserNotFound, UnknownError as IGUnknownError
 
-from warp_beacon.scrapler.abstract import ScraplerAbstract
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, extract_exception_message
+from warp_beacon.scraper.abstract import ScraperAbstract
 
 INST_SESSION_FILE = "/var/warp_beacon/inst_session.json"
 
-class InstagramScrapler(ScraplerAbstract):
+class InstagramScraper(ScraperAbstract):
 	cl = None
 
 	def __init__(self) -> None:
@@ -85,7 +87,7 @@ class InstagramScrapler(ScraplerAbstract):
 		logging.info("media_id is '%s'", media_id)
 		return media_id
 	
-	def __download_hndlr(self, func: Callable, *args: tuple[str], **kwargs: dict[str]) -> Union[Path, Media]:
+	def _download_hndlr(self, func: Callable, *args: tuple[str], **kwargs: dict[str]) -> Union[str, dict]:
 		ret_val = {}
 		max_retries = int(os.environ.get("IG_MAX_RETRIES", default=5))
 		retries = 0
@@ -96,12 +98,14 @@ class InstagramScrapler(ScraplerAbstract):
 			except (requests.exceptions.ConnectionError,
 					requests.exceptions.ReadTimeout,
 					urllib3.exceptions.ReadTimeoutError,
-					urllib3.exceptions.ConnectionError) as e:
+					urllib3.exceptions.ConnectionError,
+					ConnectTimeout,
+					HTTPError) as e:
 				logging.warning("Instagram read timeout! Retrying in 2 seconds ...")
 				logging.info("Your `IG_MAX_RETRIES` values is %d", max_retries)
 				logging.exception(e)
 				if max_retries == retries:
-					raise e
+					raise TimeOut(extract_exception_message(e))
 				retries += 1
 				time.sleep(2)
 
@@ -109,11 +113,11 @@ class InstagramScrapler(ScraplerAbstract):
 
 	
 	def download_video(self, url: str, media_info: dict) -> dict:
-		path = self.__download_hndlr(self.cl.video_download_by_url, url, folder='/tmp')
+		path = self._download_hndlr(self.cl.video_download_by_url, url, folder='/tmp')
 		return {"local_media_path": str(path), "media_type": "video", "media_info": {"duration": round(media_info.video_duration)}}
 
 	def download_photo(self, url: str) -> dict:
-		path = self.__download_hndlr(self.cl.photo_download_by_url, url, folder='/tmp')
+		path = self._download_hndlr(self.cl.photo_download_by_url, url, folder='/tmp')
 		return {"local_media_path": str(path), "media_type": "image"}
 	
 	def download_story(self, story_info: Story) -> dict:
@@ -127,10 +131,10 @@ class InstagramScrapler(ScraplerAbstract):
 		logging.info("Effective story id is '%s'", effective_story_id)
 		effective_url = "https://www.instagram.com/stories/%s/%s/" % (story_info.user.username, effective_story_id)
 		if story_info.media_type == 1: # photo
-			path = self.__download_hndlr(self.cl.story_download_by_url, url=story_info.thumbnail_url, folder='/tmp')
+			path = self._download_hndlr(self.cl.story_download_by_url, url=story_info.thumbnail_url, folder='/tmp')
 			media_type = "image"
 		elif story_info.media_type == 2: # video
-			path = self.__download_hndlr(self.cl.story_download_by_url, url=story_info.video_url, folder='/tmp')
+			path = self._download_hndlr(self.cl.story_download_by_url, url=story_info.video_url, folder='/tmp')
 			media_type = "video"
 			media_info["duration"] = story_info.video_duration
 
@@ -160,7 +164,7 @@ class InstagramScrapler(ScraplerAbstract):
 			try:
 				scrap_type, media_id = self.scrap(url)
 				if scrap_type == "media":
-					media_info = self.__download_hndlr(self.cl.media_info, media_id)
+					media_info = self._download_hndlr(self.cl.media_info, media_id)
 					logging.info("media_type is '%d', product_type is '%s'", media_info.media_type, media_info.product_type)
 					if media_info.media_type == 2 and media_info.product_type == "clips": # Reels
 						res.append(self.download_video(url=media_info.video_url, media_info=media_info))
@@ -192,4 +196,8 @@ class InstagramScrapler(ScraplerAbstract):
 								os.unlink(i["local_media_path"])
 				os.unlink(INST_SESSION_FILE)
 				time.sleep(wait_timeout)
+			except (MediaNotFound, ClientNotFoundError, UserNotFound) as e:
+				raise NotFound(extract_exception_message(e))
+			except IGUnknownError as e:
+				raise UnknownError(extract_exception_message(e))
 		return res

warp_beacon/scraper/youtube/music.py

@@ -0,0 +1,110 @@
+import os
+import pathlib
+import time
+
+import socket
+import ssl
+
+from typing import Callable, Union
+
+import requests
+import urllib
+import http.client
+
+from pytubefix import YouTube
+from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
+
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, extract_exception_message
+from warp_beacon.scraper.abstract import ScraperAbstract
+
+import logging
+
+DOWNLOAD_DIR = "/tmp"
+
+class YoutubeMusicScraper(ScraperAbstract):
+
+	def __init__(self) -> None:
+		pass
+
+	def __del__(self) -> None:
+		pass
+
+	def remove_tmp_files(self) -> None:
+		for i in os.listdir(DOWNLOAD_DIR):
+			if "yt_download_" in i:
+				os.unlink("%s/%s" % (DOWNLOAD_DIR, i))
+
+	def _download_hndlr(self, func: Callable, *args: tuple[str], **kwargs: dict[str]) -> Union[str, dict]:
+		ret_val = ''
+		max_retries = int(os.environ.get("YT_MUSIC_MAX_RETRIES", default=6))
+		pause_secs = int(os.environ.get("YT_MUSIC_PAUSE_BEFORE_RETRY", default=3))
+		timeout = int(os.environ.get("YT_MUSIC_TIMEOUT", default=60))
+		timeout_increment = int(os.environ.get("YT_MUSIC_TIMEOUT_INCREMENT", default=60))
+		retries = 0
+		while max_retries >= retries:
+			try:
+				kwargs["timeout"] = timeout
+				ret_val = func(*args, **kwargs)
+				break
+			except MaxRetriesExceeded:
+				# do noting, not interested
+				pass
+			#except http.client.IncompleteRead as e:
+			except (socket.timeout, 
+					ssl.SSLError, 
+					http.client.IncompleteRead, 
+					http.client.HTTPException, 
+					requests.RequestException, 
+					urllib.error.URLError, 
+					urllib.error.HTTPError) as e:
+				if hasattr(e, "code") and int(e.code) == 403:
+					raise Unavailable(extract_exception_message(e))
+				logging.warning("Youtube read timeout! Retrying in %d seconds ...", pause_secs)
+				logging.info("Your `YT_MUSIC_MAX_RETRIES` values is %d", max_retries)
+				logging.exception(extract_exception_message(e))
+				if max_retries <= retries:
+					self.remove_tmp_files()
+					raise TimeOut(extract_exception_message(e))
+				retries += 1
+				timeout += timeout_increment
+				time.sleep(pause_secs)
+			except (VideoUnavailable, VideoPrivate) as e:
+				raise Unavailable(extract_exception_message(e))
+
+		return ret_val
+
+	def rename_local_file(self, filename: str) -> str:
+		if not os.path.exists(filename):
+			raise NameError("No file provided")
+		path_info = pathlib.Path(filename)
+		ext = path_info.suffix
+		old_filename = path_info.stem
+		time_name = str(time.time()).replace('.', '_')
+		new_filename = "%s%s" % (time_name, ext)
+		new_filepath = "%s/%s" % (os.path.dirname(filename), new_filename)
+
+		os.rename(filename, new_filepath)
+
+		return new_filepath
+
+	def _download(self, url: str, timeout: int = 0) -> list:
+		res = []
+		yt = YouTube(url)
+		stream = yt.streams.get_audio_only()
+		if stream:
+			logging.info("Operation timeout is '%d'", timeout)
+			local_file = stream.download(
+				output_path=DOWNLOAD_DIR,
+				max_retries=0,
+				timeout=timeout,
+				skip_existing=False,
+				filename_prefix='yt_download_',
+				mp3=True
+			)
+			logging.info("Temp filename: '%s'", local_file)
+			res.append({"local_media_path": self.rename_local_file(local_file), "canonical_name": stream.title, "media_type": "audio"})
+
+		return res
+
+	def download(self, url: str) -> list:
+		return self._download_hndlr(self._download, url)

warp_beacon/scraper/youtube/shorts.py

@@ -0,0 +1,84 @@
+import os
+import pathlib
+import time
+
+from typing import Callable, Union
+
+from socket import timeout
+from ssl import SSLError
+from requests.exceptions import RequestException
+from urllib.error import URLError
+from http.client import HTTPException
+
+from pytubefix import YouTube
+from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
+
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, extract_exception_message
+from warp_beacon.scraper.abstract import ScraperAbstract
+
+import logging
+
+class YoutubeShortsScraper(ScraperAbstract):
+	def __init__(self) -> None:
+		pass
+
+	def __del__(self) -> None:
+		pass
+
+	def _download_hndlr(self, func: Callable, *args: tuple[str], **kwargs: dict[str]) -> Union[str, dict]:
+		ret_val = ''
+		max_retries = int(os.environ.get("YT_MAX_RETRIES", default=8))
+		pause_secs = int(os.environ.get("YT_PAUSE_BEFORE_RETRY", default=3))
+		retries = 0
+		while max_retries >= retries:
+			try:
+				ret_val = func(*args, **kwargs)
+				break
+			except MaxRetriesExceeded:
+				# do noting, not interested
+				pass
+			except (timeout, SSLError, HTTPException, RequestException, URLError) as e:
+				logging.warning("Youtube read timeout! Retrying in %d seconds ...", pause_secs)
+				logging.info("Your `YT_MAX_RETRIES` values is %d", max_retries)
+				logging.exception(extract_exception_message(e))
+				if max_retries >= retries:
+					raise TimeOut(extract_exception_message(e))
+				retries += 1
+				time.sleep(pause_secs)
+			except (VideoUnavailable, VideoPrivate) as e:
+				raise Unavailable(extract_exception_message(e))
+
+		return ret_val
+
+	def rename_local_file(self, filename: str) -> str:
+		if not os.path.exists(filename):
+			raise NameError("No file provided")
+		path_info = pathlib.Path(filename)
+		ext = path_info.suffix
+		old_filename = path_info.stem
+		time_name = str(time.time()).replace('.', '_')
+		new_filename = "%s%s" % (time_name, ext)
+		new_filepath = "%s/%s" % (os.path.dirname(filename), new_filename)
+
+		os.rename(filename, new_filepath)
+
+		return new_filepath
+
+	def _download(self, url: str) -> list:
+		res = []
+		timeout = int(os.environ.get("YT_TIMEOUT", default=2))
+		yt = YouTube(url)
+		stream = yt.streams.get_highest_resolution()
+		if stream:
+			local_file = stream.download(
+				output_path="/tmp",
+				max_retries=0,
+				timeout=timeout,
+				skip_existing=False
+			)
+			res.append({"local_media_path": self.rename_local_file(local_file), "media_type": "video"})
+
+		return res
+
+	def download(self, url: str) -> list:
+		return self._download_hndlr(self._download, url)

warp_beacon/uploader/__init__.py

@@ -13,6 +13,7 @@ from typing import Optional, Callable, Coroutine
 from warp_beacon.storage import Storage
 
 class AsyncUploader(object):
+	__JOE_BIDEN_WAKEUP = None
 	threads = []
 	allow_loop = True
 	job_queue = None
@@ -34,8 +35,8 @@ class AsyncUploader(object):
 	def start(self) -> None:
 		for _ in range(self.pool_size):
 			thread = threading.Thread(target=self.do_work)
-			self.threads.append(thread)
 			thread.start()
+			self.threads.append(thread)
 
 	def add_callback(self, message_id: int, callback: Callable, update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
 		def callback_wrap(*args, **kwargs) -> None:
@@ -55,6 +56,7 @@ class AsyncUploader(object):
 			for i in self.threads:
 				t_id = i.native_id
 				logging.info("Stopping thread #'%s'", t_id)
+				self.job_queue.put(self.__JOE_BIDEN_WAKEUP)
 				i.join()
 				logging.info("Thread #'%s' stopped", t_id)
 		self.threads.clear()
@@ -77,6 +79,8 @@ class AsyncUploader(object):
 			try:
 				try:
 					job = self.job_queue.get()
+					if job is self.__JOE_BIDEN_WAKEUP:
+						continue
 					path = ""
 					if job.media_type == "collection":
 						for i in job.media_collection:
@@ -130,4 +134,5 @@ class AsyncUploader(object):
 					pass
 			except Exception as e:
 				logging.error("Exception occurred inside upload worker!")
-				logging.exception(e)
+				logging.exception(e)
+		logging.info("Thread done")

warp_beacon/warp_beacon.py

@@ -6,7 +6,6 @@ import signal
 import asyncio
 import time
 from io import BytesIO
-import logging
 
 from urlextract import URLExtract
 
@@ -15,10 +14,14 @@ from telegram import Bot, ForceReply, Update, Chat, error, InputMediaVideo, Inpu
 from telegram.ext import Application, CommandHandler, ContextTypes, MessageHandler, filters
 from telegram.constants import ParseMode
 
-import warp_beacon.scrapler
+import warp_beacon.scraper
 from warp_beacon.storage import Storage
 from warp_beacon.uploader import AsyncUploader
-from warp_beacon.jobs.download_job import DownloadJob, UploadJob
+from warp_beacon.jobs.download_job import DownloadJob
+from warp_beacon.jobs.upload_job import UploadJob
+from warp_beacon.jobs import Origin
+
+import logging
 
 # Enable logging
 logging.basicConfig(
@@ -361,6 +364,15 @@ async def upload_job(update: Update, context: ContextTypes.DEFAULT_TYPE, job: Up
 
 	return tg_file_ids
 
+def extract_origin(url: str) -> Origin:
+	if "instagram.com/" in url:
+		return Origin.INSTAGRAM
+
+	if "youtube.com/" in url and "shorts/" in url:
+		return Origin.YT_SHORTS
+
+	return Origin.UNKNOWN
+
 async def handler(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
 	if update.message is None:
 		return
@@ -374,8 +386,9 @@ async def handler(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
 		reply_text = "Your message should contains URLs"
 	else:
 		for url in urls:
-			if "instagram.com" not in url:
-				logging.info("Only instagram.com is now supported. Skipping.")
+			origin = extract_origin(url)
+			if origin is Origin.UNKNOWN:
+				logging.info("Only Instagram and YouTube Shorts are now supported. Skipping.")
 				continue
 			entities, tg_file_ids = [], []
 			uniq_id = Storage.compute_uniq(url)
@@ -422,9 +435,9 @@ async def handler(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
 						if tg_file_ids:
 							if job.media_type == "collection" and job.save_items:
 								for i in job.media_collection:
-									storage.add_media(tg_file_ids=[i.tg_file_id], media_url=i.effective_url, media_type=i.media_type, origin="instagram")
+									storage.add_media(tg_file_ids=[i.tg_file_id], media_url=i.effective_url, media_type=i.media_type, origin=origin.value)
 							else:
-								storage.add_media(tg_file_ids=[','.join(tg_file_ids)], media_url=job.url, media_type=job.media_type, origin="instagram")
+								storage.add_media(tg_file_ids=[','.join(tg_file_ids)], media_url=job.url, media_type=job.media_type, origin=origin.value)
 					except Exception as e:
 						logging.error("Exception occurred while performing upload callback!")
 						logging.exception(e)
@@ -457,7 +470,8 @@ async def handler(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
 						placeholder_message_id=placeholder_message_id,
 						message_id=effective_message_id,
 						in_process=uploader.is_inprocess(uniq_id),
-						uniq_id=uniq_id
+						uniq_id=uniq_id,
+						job_origin = origin
 					))
 					uploader.set_inprocess(uniq_id)
 				except Exception as e:
@@ -476,25 +490,6 @@ def main() -> None:
 	try:
 		global uploader, downloader
 
-		loop = asyncio.get_event_loop()
-
-		uploader = AsyncUploader(
-			storage=storage,
-			pool_size=int(os.environ.get("UPLOAD_POOL_SIZE", default=warp_beacon.scrapler.CONST_CPU_COUNT)),
-			loop=loop
-		)
-		downloader = warp_beacon.scrapler.AsyncDownloader(
-			workers_count=int(os.environ.get("WORKERS_POOL_SIZE", default=warp_beacon.scrapler.CONST_CPU_COUNT)),
-			uploader=uploader
-		)
-		downloader.start()
-		uploader.start()
-
-		stop_signals = (signal.SIGINT, signal.SIGTERM, signal.SIGABRT)
-		for sig in stop_signals or []:
-			loop.add_signal_handler(sig, _raise_system_exit)
-		loop.add_signal_handler(sig, _raise_system_exit)
-
 		# Create the Application and pass it your bot's token.
 		tg_token = os.environ.get("TG_TOKEN", default=None)
 		application = Application.builder().token(tg_token).concurrent_updates(True).build()
@@ -508,8 +503,25 @@ def main() -> None:
 		application.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, handler))
 
 		allow_loop = True
+		loop = None
 		while allow_loop:
 			try:
+				loop = asyncio.get_event_loop()
+
+				stop_signals = (signal.SIGINT, signal.SIGTERM, signal.SIGABRT)
+				for sig in stop_signals or []:
+					loop.add_signal_handler(sig, _raise_system_exit)
+				loop.add_signal_handler(sig, _raise_system_exit)
+
+				uploader = AsyncUploader(
+					storage=storage,
+					pool_size=int(os.environ.get("UPLOAD_POOL_SIZE", default=warp_beacon.scraper.CONST_CPU_COUNT)),
+					loop=loop
+				)
+				downloader = warp_beacon.scraper.AsyncDownloader(
+					workers_count=int(os.environ.get("WORKERS_POOL_SIZE", default=warp_beacon.scraper.CONST_CPU_COUNT)),
+					uploader=uploader
+				)
 				loop.run_until_complete(application.initialize())
 				if application.post_init:
 					loop.run_until_complete(application.post_init(application))
@@ -517,10 +529,12 @@ def main() -> None:
 				loop.run_until_complete(application.start())
 				while allow_loop:
 					try:
+						downloader.start()
+						uploader.start()
 						loop.run_forever()
 					except (KeyboardInterrupt, SystemExit) as e:
 						allow_loop = False
-						raise e
+						raise
 					except Exception as e:
 						logging.error("Main loop Telegram error!")
 						logging.exception(e)
@@ -546,11 +560,13 @@ def main() -> None:
 					if application.post_shutdown:
 						loop.run_until_complete(application.post_shutdown(application))
 				finally:
+					loop.close()
 					downloader.stop_all()
 					uploader.stop_all()
-					loop.close()
 	except Exception as e:
 		logging.exception(e)
 
+	logging.info("Warp Beacon terminated.")
+
 if __name__ == "__main__":
 	main()

{warp_beacon-1.0.9.dist-info → warp_beacon-1.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 1.0.9
+Version: 1.1.2
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -226,6 +226,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: ffmpeg-python
 Requires-Dist: python-telegram-bot
+Requires-Dist: pytubefix
 Requires-Dist: av
 Requires-Dist: urlextract
 Requires-Dist: pillow

warp_beacon-1.1.2.dist-info/RECORD

@@ -0,0 +1,31 @@
+etc/warp_beacon/warp_beacon.conf,sha256=1gGvh36cnFr0rU4mVomfy66hQz9EvugaNzeH6_tmBM0,266
+lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuwwPfa2mDLUA,372
+var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
+warp_beacon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+warp_beacon/__version__.py,sha256=y-YwjKcQPThc0FMcn_TMBl8-1vHN4zwnWOJXVrYbWTI,23
+warp_beacon/warp_beacon.py,sha256=Z3zY7MTMZp3zKRderhNOVe6XvM-93Djo0DIsiaoucmo,20012
+warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
+warp_beacon/jobs/__init__.py,sha256=KsSaS0KlCNyffNnWKuvqmdfgyfKyn3niXYyZ38-exQ8,113
+warp_beacon/jobs/abstract.py,sha256=PCr8RXzocKi-sTsi2Y1_spiv6D95G1NlzZ2wD2WJXRc,1760
+warp_beacon/jobs/download_job.py,sha256=wfZrKUerfYIjWkRxPzfl5gwIlcotIMH7OpTUM9ae8NY,736
+warp_beacon/jobs/upload_job.py,sha256=Vaogc4vbpAfyaT4VkIHEPLFRELmM44TDqkmnPYh3Ymc,740
+warp_beacon/mediainfo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+warp_beacon/mediainfo/abstract.py,sha256=O9ZROcW1cDlLMqBrUW2KI06tHCQ_iflDJknZA3iFaHE,591
+warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-ZM,651
+warp_beacon/mediainfo/video.py,sha256=8h7p4k0w45Vm0lPQNlaQaUjaDTBybX3RcKgL1QQbioA,2638
+warp_beacon/scraper/__init__.py,sha256=c7NySK5Krm-zlWQckFs-uN4fD3J19A0pTS4CByXjmMs,7918
+warp_beacon/scraper/abstract.py,sha256=um4wUthO_7IsoXjKiUTWyBBbKlf-N01aZJK9N2UQI9I,408
+warp_beacon/scraper/exceptions.py,sha256=qra_Jx53RVCnrCKvw2OxvEHl4cXJCrPDa_yef3cvGXM,978
+warp_beacon/scraper/instagram.py,sha256=s7slumqdqVVWQYpnVphx-dOLIAWvgA_UdqkTQVp6GsI,7758
+warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+warp_beacon/scraper/youtube/music.py,sha256=htxLJW-RaRhgxqJ9276ibaLnuStNtw26-pKJwfaQpgY,3335
+warp_beacon/scraper/youtube/shorts.py,sha256=xAOeHNT3_Ayjjglid00UqGqCgSMfbJua26PNrbbDYUo,2565
+warp_beacon/storage/__init__.py,sha256=NhD3V7UNRiZNf61yQEAjXOfi-tfA2LaJa7a7kvbkmtE,2402
+warp_beacon/uploader/__init__.py,sha256=auD1arKpJdN1eFUbTFoa9Gmv-ZYZNesMoT193__pDz8,4507
+warp_beacon-1.1.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-1.1.2.dist-info/METADATA,sha256=eRC_mlblWo5zz0fAw2pFXJFhEQAny4V8ieMKh2kAMmE,18244
+warp_beacon-1.1.2.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
+warp_beacon-1.1.2.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-1.1.2.dist-info/top_level.txt,sha256=QuN6MynevEblMhPPAVeMrNAkcyqYUpYDholtIRq8-ew,473
+warp_beacon-1.1.2.dist-info/RECORD,,

{warp_beacon-1.0.9.dist-info → warp_beacon-1.1.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (71.0.4)
+Generator: setuptools (71.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{warp_beacon-1.0.9.dist-info → warp_beacon-1.1.2.dist-info}/top_level.txt

@@ -8,9 +8,12 @@ warp_beacon/jobs/download_job
 warp_beacon/jobs/upload_job
 warp_beacon/mediainfo
 warp_beacon/mediainfo/video
-warp_beacon/scrapler
-warp_beacon/scrapler/abstract
-warp_beacon/scrapler/instagram
+warp_beacon/scraper
+warp_beacon/scraper/abstract
+warp_beacon/scraper/exceptions
+warp_beacon/scraper/instagram
+warp_beacon/scraper/youtube
+warp_beacon/scraper/youtube/shorts
 warp_beacon/storage
 warp_beacon/uploader
 warp_beacon/warp_beacon

warp_beacon-1.0.9.dist-info/RECORD

@@ -1,25 +0,0 @@
-etc/warp_beacon/warp_beacon.conf,sha256=1gGvh36cnFr0rU4mVomfy66hQz9EvugaNzeH6_tmBM0,266
-lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuwwPfa2mDLUA,372
-var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
-warp_beacon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/__version__.py,sha256=EgXzK34Fe1bfzUqfSu8mFPvk3Cjne5jkEdf5koaCcsQ,23
-warp_beacon/warp_beacon.py,sha256=agUu6_Qt1UnGTSqYqXQZtHeeWqRkhlIHt_Repph1G3k,19581
-warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
-warp_beacon/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/jobs/abstract.py,sha256=-eXvrRgrUBed1z9iYZd0RxZ8fi0Okq3r0i7PDN31FkY,1666
-warp_beacon/jobs/download_job.py,sha256=wfZrKUerfYIjWkRxPzfl5gwIlcotIMH7OpTUM9ae8NY,736
-warp_beacon/jobs/upload_job.py,sha256=Vaogc4vbpAfyaT4VkIHEPLFRELmM44TDqkmnPYh3Ymc,740
-warp_beacon/mediainfo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/mediainfo/video.py,sha256=ucBx1wUBfUZdwtgprg6RbCL7chTiPUyz0ylymbnAC8E,2514
-warp_beacon/scrapler/__init__.py,sha256=ypzGd8xsdh7KYcbpuof-TZUIPkc2Gr1MDUQ8aimnams,7513
-warp_beacon/scrapler/abstract.py,sha256=MJxpEovCWDYq2SwbbMsRDfp77WTwvbXXKiQxKWoj0ZQ,304
-warp_beacon/scrapler/instagram.py,sha256=LqvO9-Taf52tFrgraA-DZzDUQ8sP4lhfelUYWtEHGT0,7255
-warp_beacon/storage/__init__.py,sha256=NhD3V7UNRiZNf61yQEAjXOfi-tfA2LaJa7a7kvbkmtE,2402
-warp_beacon/uploader/__init__.py,sha256=9qQAuYisXiVIjQghxcxpF4WAdW7lm7HmpkOXQjGNJXk,4346
-warp_beacon-1.0.9.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-1.0.9.dist-info/METADATA,sha256=eXSrgOuHdsNia9BiVNo2yAGA0kqelTVs5VP37DvGQXg,18219
-warp_beacon-1.0.9.dist-info/WHEEL,sha256=rWxmBtp7hEUqVLOnTaDOPpR-cZpCDkzhhcBce-Zyd5k,91
-warp_beacon-1.0.9.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-1.0.9.dist-info/top_level.txt,sha256=510sqsM4LLO-DC4HbUkwdVKmYY_26lbnvJwSq_RLT00,382
-warp_beacon-1.0.9.dist-info/RECORD,,