warp-beacon 2.7.11__tar.gz → 2.7.13__tar.gz

{warp_beacon-2.7.11/warp_beacon.egg-info → warp_beacon-2.7.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.7.11
+Version: 2.7.13
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.7.13/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.7.13"
+

{warp_beacon-2.7.11 → warp_beacon-2.7.13}/warp_beacon/scraper/instagram/instagram.py

@@ -475,7 +475,7 @@ class InstagramScraper(ScraperAbstract):
 				"report_type": ReportType.PROGRESS
 			}
 			self.status_pipe.send(msg)
-			self._download_progress_threshold += 10
+			self._download_progress_threshold += 20
 
 	def report_seen(self, items: dict) -> None:
 		try:

{warp_beacon-2.7.11 → warp_beacon-2.7.13}/warp_beacon/scraper/youtube/abstract.py

@@ -13,6 +13,7 @@ from typing import Callable, Optional, Union
 from urllib.parse import parse_qs, urlparse
 
 import numpy as np
+import av
 import pytubefix
 import pytubefix.exceptions
 import requests
@@ -245,7 +246,7 @@ class YoutubeAbstract(ScraperAbstract):
 			self.status_pipe.send(msg)
 			logging.debug("[Download worker] Downloaded %d%%", percentage_of_completion)
 			if total_size > 0:
-				self._download_progress_threshold += 10
+				self._download_progress_threshold += 20
 
 	def build_proxies(self, proxy_dsn: str) -> dict:
 		if not proxy_dsn:
@@ -310,7 +311,7 @@ class YoutubeAbstract(ScraperAbstract):
 				self.status_pipe.send(msg)
 				logging.debug("[Download worker][yt_dlp] Downloaded %d%%", percentage_of_completion)
 				if total_size > 0:
-					self._download_progress_threshold += 10
+					self._download_progress_threshold += 20
 
 	def build_yt_dlp(self, timeout: int = 60) -> yt_dlp.YoutubeDL:
 		auth_data = {}
@@ -387,3 +388,30 @@ class YoutubeAbstract(ScraperAbstract):
 			raise Unavailable("Сontent unvailable")
 
 		return ret
+
+	def mux_raw_copy(self, video_path: str, audio_path: str, output_path: str) -> str:
+		try:
+			with av.open(video_path) as input_video, av.open(audio_path) as input_audio, av.open(output_path, mode='w') as output:
+				in_video_stream = input_video.streams.video[0]
+				in_audio_stream = input_audio.streams.audio[0]
+
+				video_stream_map = output.add_stream(template=in_video_stream)
+				audio_stream_map = output.add_stream(template=in_audio_stream)
+
+				for packet in input_video.demux(in_video_stream):
+					if packet.dts is None:
+						continue
+					packet.stream = video_stream_map
+					output.mux(packet)
+
+				for packet in input_audio.demux(in_audio_stream):
+					if packet.dts is None:
+						continue
+					packet.stream = audio_stream_map
+					output.mux(packet)
+		except Exception as e:
+			logging.error("Failed to mux audio and video!")
+			logging.exception(e)
+			return ''
+
+		return output_path

warp_beacon-2.7.13/warp_beacon/scraper/youtube/shorts.py

@@ -0,0 +1,124 @@
+import os
+import io
+from typing import Optional
+import time
+
+import logging
+
+from pytubefix.exceptions import AgeRestrictedError
+
+from warp_beacon.jobs.types import JobType
+from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
+
+from warp_beacon.scraper.exceptions import NotFound, YotubeAgeRestrictedError
+
+class YoutubeShortsScraper(YoutubeAbstract):
+	YT_MAX_RETRIES_DEFAULT = 8
+	YT_PAUSE_BEFORE_RETRY_DEFAULT = 3
+	YT_TIMEOUT_DEFAULT = 2
+	YT_TIMEOUT_INCREMENT_DEFAULT = 60
+
+	def _download(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
+		res = self._download_pytubefix_max_res(url=url, session=session, thumbnail=thumbnail, timeout=timeout)
+		if not res:
+			res = self._download_pytube_dash(url=url, session=session, thumbnail=thumbnail, timeout=timeout)
+
+		return res
+
+	def _download_pytube_dash(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 0) -> list:
+		res = []
+		yt = self.build_yt(url, session=session)
+		stream = yt.streams.get_highest_resolution()
+
+		if not stream:
+			raise NotFound("No suitable video stream found")
+
+		local_file = stream.download(
+			output_path=self.DOWNLOAD_DIR,
+			max_retries=0,
+			timeout=timeout,
+			skip_existing=False,
+			filename_prefix="yt_download_"
+		)
+
+		local_file = self.rename_local_file(local_file)
+
+		logging.debug("Temp filename: '%s'", local_file)
+		res.append({
+			"local_media_path": local_file,
+			"performer": yt.author,
+			"thumb": thumbnail,
+			"canonical_name": stream.title,
+			"media_type": JobType.VIDEO
+		})
+
+		return res
+
+	def _download_yt_dlp(self, url: str, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
+		res = []
+		with self.build_yt_dlp(timeout) as ydl:
+			info = ydl.extract_info(url, download=True)
+			local_file = ydl.prepare_filename(info)
+			logging.debug("Temp filename: '%s'", local_file)
+			res.append({
+				"local_media_path": local_file,
+				"performer": info.get("uploader", "Unknown"),
+				"thumb": thumbnail,
+				"canonical_name": info.get("title", ''),
+				"media_type": JobType.VIDEO
+			})
+
+		return res
+	
+	def _download_pytubefix_max_res(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
+		res = []
+		local_video_file, local_audio_file = '', ''
+		try:
+			yt = self.build_yt(url, session=session)
+			
+			video_stream = yt.streams.filter(adaptive=True, file_extension='mp4', only_video=True).order_by('resolution').desc().first()
+			audio_stream = yt.streams.filter(adaptive=True, file_extension='mp4', only_audio=True).order_by('abr').desc().first()
+
+			local_video_file = video_stream.download(
+				output_path=self.DOWNLOAD_DIR,
+				max_retries=3,
+				timeout=timeout,
+				skip_existing=False,
+				filename_prefix="yt_download_video_"
+			)
+			local_video_file = self.rename_local_file(local_video_file)
+			logging.debug("Temp video filename: '%s'", local_video_file)
+			local_audio_file = audio_stream.download(
+				output_path=self.DOWNLOAD_DIR,
+				max_retries=3,
+				timeout=timeout,
+				skip_existing=False,
+				filename_prefix="yt_download_audio_"
+			)
+			local_audio_file = self.rename_local_file(local_audio_file)
+			logging.debug("Temp audio filename: '%s'", local_audio_file)
+
+			muxed_video = self.mux_raw_copy(
+				video_path=local_video_file,
+				audio_path=local_audio_file,
+				output_path=f"{self.DOWNLOAD_DIR}/yt_muxed_video_{int(time.time())}.mp4")
+			if muxed_video:
+				muxed_video = self.rename_local_file(muxed_video)
+				logging.debug("Temp muxed filename: '%s'", muxed_video)
+
+				res.append({
+					"local_media_path": muxed_video,
+					"performer": yt.author,
+					"thumb": thumbnail,
+					"canonical_name": yt.title,
+					"media_type": JobType.VIDEO
+				})
+		except AgeRestrictedError as e:
+			raise YotubeAgeRestrictedError("Youtube Age Restricted error")
+		finally:
+			if os.path.exists(local_video_file):
+				os.unlink(local_video_file)
+			if os.path.exists(local_audio_file):
+				os.unlink(local_audio_file)
+		
+		return res

{warp_beacon-2.7.11 → warp_beacon-2.7.13}/warp_beacon/scraper/youtube/youtube.py

@@ -4,7 +4,6 @@ import io
 from typing import Optional
 import logging
 
-import av
 from pytubefix.exceptions import AgeRestrictedError
 
 from warp_beacon.jobs.types import JobType
@@ -39,33 +38,6 @@ class YoutubeScraper(YoutubeAbstract):
 			res = self._download_pytube_dash(url=url, session=session, thumbnail=thumbnail, timeout=timeout)
 
 		return res
-	
-	def mux_raw_copy(self, video_path: str, audio_path: str, output_path: str) -> str:
-		try:
-			with av.open(video_path) as input_video, av.open(audio_path) as input_audio, av.open(output_path, mode='w') as output:
-				in_video_stream = input_video.streams.video[0]
-				in_audio_stream = input_audio.streams.audio[0]
-
-				video_stream_map = output.add_stream(template=in_video_stream)
-				audio_stream_map = output.add_stream(template=in_audio_stream)
-
-				for packet in input_video.demux(in_video_stream):
-					if packet.dts is None:
-						continue
-					packet.stream = video_stream_map
-					output.mux(packet)
-
-				for packet in input_audio.demux(in_audio_stream):
-					if packet.dts is None:
-						continue
-					packet.stream = audio_stream_map
-					output.mux(packet)
-		except Exception as e:
-			logging.error("Failed to mux audio and video!")
-			logging.exception(e)
-			return ''
-
-		return output_path
 
 	def _download_pytubefix_max_res(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
 		res = []

{warp_beacon-2.7.11 → warp_beacon-2.7.13}/warp_beacon/telegram/progress_bar.py

@@ -125,7 +125,7 @@ class ProgressBar(object):
 				logging.warning("An error occurred while setup task to update progress bar")
 				logging.exception(e)
 			if total > 0 and percent != 0:
-				self._next_threshold += 15
+				self._next_threshold += 20
 
 	@staticmethod
 	def make_hash(chat_id: str | int, message_id: int, algorithm: str = 'sha256') -> str:

{warp_beacon-2.7.11 → warp_beacon-2.7.13/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.7.11
+Version: 2.7.13
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.7.11/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.7.11"
-

warp_beacon-2.7.11/warp_beacon/scraper/youtube/shorts.py

@@ -1,61 +0,0 @@
-import io
-from typing import Optional
-
-import logging
-
-from warp_beacon.jobs.types import JobType
-from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
-from warp_beacon.scraper.exceptions import NotFound
-
-from warp_beacon.mediainfo.video import VideoInfo
-
-class YoutubeShortsScraper(YoutubeAbstract):
-	YT_MAX_RETRIES_DEFAULT = 8
-	YT_PAUSE_BEFORE_RETRY_DEFAULT = 3
-	YT_TIMEOUT_DEFAULT = 2
-	YT_TIMEOUT_INCREMENT_DEFAULT = 60
-
-	def _download(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 0) -> list:
-		res = []
-		yt = self.build_yt(url, session=session)
-		stream = yt.streams.get_highest_resolution()
-
-		if not stream:
-			raise NotFound("No suitable video stream found")
-
-		local_file = stream.download(
-			output_path=self.DOWNLOAD_DIR,
-			max_retries=0,
-			timeout=timeout,
-			skip_existing=False,
-			filename_prefix="yt_download_"
-		)
-
-		local_file = self.rename_local_file(local_file)
-
-		logging.debug("Temp filename: '%s'", local_file)
-		res.append({
-			"local_media_path": local_file,
-			"performer": yt.author,
-			"thumb": thumbnail,
-			"canonical_name": stream.title,
-			"media_type": JobType.VIDEO
-		})
-
-		return res
-
-	def _download_yt_dlp(self, url: str, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
-		res = []
-		with self.build_yt_dlp(timeout) as ydl:
-			info = ydl.extract_info(url, download=True)
-			local_file = ydl.prepare_filename(info)
-			logging.debug("Temp filename: '%s'", local_file)
-			res.append({
-				"local_media_path": local_file,
-				"performer": info.get("uploader", "Unknown"),
-				"thumb": thumbnail,
-				"canonical_name": info.get("title", ''),
-				"media_type": JobType.VIDEO
-			})
-
-		return res