warp-beacon 2.7.11__py3-none-any.whl → 2.7.13__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.7.11"
+__version__ = "2.7.13"
 

warp_beacon/scraper/instagram/instagram.py

@@ -475,7 +475,7 @@ class InstagramScraper(ScraperAbstract):
 				"report_type": ReportType.PROGRESS
 			}
 			self.status_pipe.send(msg)
-			self._download_progress_threshold += 10
+			self._download_progress_threshold += 20
 
 	def report_seen(self, items: dict) -> None:
 		try:

warp_beacon/scraper/youtube/abstract.py

@@ -13,6 +13,7 @@ from typing import Callable, Optional, Union
 from urllib.parse import parse_qs, urlparse
 
 import numpy as np
+import av
 import pytubefix
 import pytubefix.exceptions
 import requests
@@ -245,7 +246,7 @@ class YoutubeAbstract(ScraperAbstract):
 			self.status_pipe.send(msg)
 			logging.debug("[Download worker] Downloaded %d%%", percentage_of_completion)
 			if total_size > 0:
-				self._download_progress_threshold += 10
+				self._download_progress_threshold += 20
 
 	def build_proxies(self, proxy_dsn: str) -> dict:
 		if not proxy_dsn:
@@ -310,7 +311,7 @@ class YoutubeAbstract(ScraperAbstract):
 				self.status_pipe.send(msg)
 				logging.debug("[Download worker][yt_dlp] Downloaded %d%%", percentage_of_completion)
 				if total_size > 0:
-					self._download_progress_threshold += 10
+					self._download_progress_threshold += 20
 
 	def build_yt_dlp(self, timeout: int = 60) -> yt_dlp.YoutubeDL:
 		auth_data = {}
@@ -387,3 +388,30 @@ class YoutubeAbstract(ScraperAbstract):
 			raise Unavailable("Сontent unvailable")
 
 		return ret
+
+	def mux_raw_copy(self, video_path: str, audio_path: str, output_path: str) -> str:
+		try:
+			with av.open(video_path) as input_video, av.open(audio_path) as input_audio, av.open(output_path, mode='w') as output:
+				in_video_stream = input_video.streams.video[0]
+				in_audio_stream = input_audio.streams.audio[0]
+
+				video_stream_map = output.add_stream(template=in_video_stream)
+				audio_stream_map = output.add_stream(template=in_audio_stream)
+
+				for packet in input_video.demux(in_video_stream):
+					if packet.dts is None:
+						continue
+					packet.stream = video_stream_map
+					output.mux(packet)
+
+				for packet in input_audio.demux(in_audio_stream):
+					if packet.dts is None:
+						continue
+					packet.stream = audio_stream_map
+					output.mux(packet)
+		except Exception as e:
+			logging.error("Failed to mux audio and video!")
+			logging.exception(e)
+			return ''
+
+		return output_path

warp_beacon/scraper/youtube/shorts.py

@@ -1,13 +1,16 @@
+import os
 import io
 from typing import Optional
+import time
 
 import logging
 
+from pytubefix.exceptions import AgeRestrictedError
+
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
-from warp_beacon.scraper.exceptions import NotFound
 
-from warp_beacon.mediainfo.video import VideoInfo
+from warp_beacon.scraper.exceptions import NotFound, YotubeAgeRestrictedError
 
 class YoutubeShortsScraper(YoutubeAbstract):
 	YT_MAX_RETRIES_DEFAULT = 8
@@ -15,7 +18,14 @@ class YoutubeShortsScraper(YoutubeAbstract):
 	YT_TIMEOUT_DEFAULT = 2
 	YT_TIMEOUT_INCREMENT_DEFAULT = 60
 
-	def _download(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 0) -> list:
+	def _download(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
+		res = self._download_pytubefix_max_res(url=url, session=session, thumbnail=thumbnail, timeout=timeout)
+		if not res:
+			res = self._download_pytube_dash(url=url, session=session, thumbnail=thumbnail, timeout=timeout)
+
+		return res
+
+	def _download_pytube_dash(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 0) -> list:
 		res = []
 		yt = self.build_yt(url, session=session)
 		stream = yt.streams.get_highest_resolution()
@@ -58,4 +68,57 @@ class YoutubeShortsScraper(YoutubeAbstract):
 				"media_type": JobType.VIDEO
 			})
 
+		return res
+	
+	def _download_pytubefix_max_res(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
+		res = []
+		local_video_file, local_audio_file = '', ''
+		try:
+			yt = self.build_yt(url, session=session)
+			
+			video_stream = yt.streams.filter(adaptive=True, file_extension='mp4', only_video=True).order_by('resolution').desc().first()
+			audio_stream = yt.streams.filter(adaptive=True, file_extension='mp4', only_audio=True).order_by('abr').desc().first()
+
+			local_video_file = video_stream.download(
+				output_path=self.DOWNLOAD_DIR,
+				max_retries=3,
+				timeout=timeout,
+				skip_existing=False,
+				filename_prefix="yt_download_video_"
+			)
+			local_video_file = self.rename_local_file(local_video_file)
+			logging.debug("Temp video filename: '%s'", local_video_file)
+			local_audio_file = audio_stream.download(
+				output_path=self.DOWNLOAD_DIR,
+				max_retries=3,
+				timeout=timeout,
+				skip_existing=False,
+				filename_prefix="yt_download_audio_"
+			)
+			local_audio_file = self.rename_local_file(local_audio_file)
+			logging.debug("Temp audio filename: '%s'", local_audio_file)
+
+			muxed_video = self.mux_raw_copy(
+				video_path=local_video_file,
+				audio_path=local_audio_file,
+				output_path=f"{self.DOWNLOAD_DIR}/yt_muxed_video_{int(time.time())}.mp4")
+			if muxed_video:
+				muxed_video = self.rename_local_file(muxed_video)
+				logging.debug("Temp muxed filename: '%s'", muxed_video)
+
+				res.append({
+					"local_media_path": muxed_video,
+					"performer": yt.author,
+					"thumb": thumbnail,
+					"canonical_name": yt.title,
+					"media_type": JobType.VIDEO
+				})
+		except AgeRestrictedError as e:
+			raise YotubeAgeRestrictedError("Youtube Age Restricted error")
+		finally:
+			if os.path.exists(local_video_file):
+				os.unlink(local_video_file)
+			if os.path.exists(local_audio_file):
+				os.unlink(local_audio_file)
+		
 		return res

warp_beacon/scraper/youtube/youtube.py

@@ -4,7 +4,6 @@ import io
 from typing import Optional
 import logging
 
-import av
 from pytubefix.exceptions import AgeRestrictedError
 
 from warp_beacon.jobs.types import JobType
@@ -39,33 +38,6 @@ class YoutubeScraper(YoutubeAbstract):
 			res = self._download_pytube_dash(url=url, session=session, thumbnail=thumbnail, timeout=timeout)
 
 		return res
-	
-	def mux_raw_copy(self, video_path: str, audio_path: str, output_path: str) -> str:
-		try:
-			with av.open(video_path) as input_video, av.open(audio_path) as input_audio, av.open(output_path, mode='w') as output:
-				in_video_stream = input_video.streams.video[0]
-				in_audio_stream = input_audio.streams.audio[0]
-
-				video_stream_map = output.add_stream(template=in_video_stream)
-				audio_stream_map = output.add_stream(template=in_audio_stream)
-
-				for packet in input_video.demux(in_video_stream):
-					if packet.dts is None:
-						continue
-					packet.stream = video_stream_map
-					output.mux(packet)
-
-				for packet in input_audio.demux(in_audio_stream):
-					if packet.dts is None:
-						continue
-					packet.stream = audio_stream_map
-					output.mux(packet)
-		except Exception as e:
-			logging.error("Failed to mux audio and video!")
-			logging.exception(e)
-			return ''
-
-		return output_path
 
 	def _download_pytubefix_max_res(self, url: str, session: bool = True, thumbnail: Optional[io.BytesIO] = None, timeout: int = 60) -> list:
 		res = []

warp_beacon/telegram/progress_bar.py

@@ -125,7 +125,7 @@ class ProgressBar(object):
 				logging.warning("An error occurred while setup task to update progress bar")
 				logging.exception(e)
 			if total > 0 and percent != 0:
-				self._next_threshold += 15
+				self._next_threshold += 20
 
 	@staticmethod
 	def make_hash(chat_id: str | int, message_id: int, algorithm: str = 'sha256') -> str:

{warp_beacon-2.7.11.dist-info → warp_beacon-2.7.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.7.11
+Version: 2.7.13
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.7.11.dist-info → warp_beacon-2.7.13.dist-info}/RECORD

@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=lqKHRYdVpzdXmWoJV13ehvNCGpYmFAl2Y41YmvWzQXo,24
+warp_beacon/__version__.py,sha256=x219y8Xqh6qOh4lcwrZPGXW00waIfTBdHsot4GpeNiI,24
 warp_beacon/warp_beacon.py,sha256=ADCR30uGXIsDrt9WoiI9Ghu2QtWs0qZIK6x3pQKM_B4,1109
 warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -31,13 +31,13 @@ warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5
 warp_beacon/scraper/utils.py,sha256=yL-nrdCOAXNZgmIFPurIYFHfPsxHDl8pqif6BYKrzJI,1218
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/captcha.py,sha256=9UYziuqB3Tsat_ET6ex-cnZDbi6yCnsXHSpmE8MuUHk,4651
-warp_beacon/scraper/instagram/instagram.py,sha256=qypIbRLPiOn98n9DOFlkU7X2Ux-cOSu7skJAO2JudIQ,18629
+warp_beacon/scraper/instagram/instagram.py,sha256=oVXP6VweJCeffhyDU-GZLa66ESW06GvfuGcW2V6YdQ4,18629
 warp_beacon/scraper/instagram/wb_instagrapi.py,sha256=M5NCtLwdUvByjmDBZMWljgB275R0LSBFblsGpapluD0,5968
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scraper/youtube/abstract.py,sha256=yQCJzi5YCEJFr365uPnMzKB8Rmi_zKVIlxw9-DI3sKE,15278
+warp_beacon/scraper/youtube/abstract.py,sha256=H53RjCGB5W-uvKEgFrPn0kSBxc8EQgpme-9DJF9I7rU,16193
 warp_beacon/scraper/youtube/music.py,sha256=5AeSBQyUgVCJT2hoBCV2WvlyuV9US09SYJhmBG_P9F8,2755
-warp_beacon/scraper/youtube/shorts.py,sha256=1GtoYUlxAwcgSQcn80u5ehNJytH5AN5dPOicmX-XD8E,1705
-warp_beacon/scraper/youtube/youtube.py,sha256=x9v9p1coA9TvBhxjNAofGu4UBkAEdYPE2ePRnU-5tK0,7233
+warp_beacon/scraper/youtube/shorts.py,sha256=y0591kpWU35rt5OoWamkcHIstNZ98SXUlUKvYmUsyEY,4030
+warp_beacon/scraper/youtube/youtube.py,sha256=i-3Z5l3S-EltIFbxkbvDjXMgH8fGOrB5-ua1p6iH_Bc,6316
 warp_beacon/storage/__init__.py,sha256=0Vajd0oITKJfu2vmNx5uQSt3-L6vwIvUYWJo8HZCjco,3398
 warp_beacon/storage/mongo.py,sha256=qC4ZiO8XXvPnP0rJwz4CJx42pqFsyAjCiW10W5QdT6E,527
 warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -47,14 +47,14 @@ warp_beacon/telegram/download_status.py,sha256=N-Qg13LVcPskyQNKG1lw50K1lhFtXu003
 warp_beacon/telegram/edit_message.py,sha256=6DX8eJV70MG_b79WbuJG55qLBs66boElzYdCr8zpMeI,5380
 warp_beacon/telegram/handlers.py,sha256=PaO90370v-S90nwr4PYMzrlHRFBU0VBO6YkqDqTbnjw,9634
 warp_beacon/telegram/placeholder_message.py,sha256=wN9-BRiyrtHG-EvXtZkGJHt2CX71munQ57ITttjt0mw,6400
-warp_beacon/telegram/progress_bar.py,sha256=T0YR-I159mB-X1IwSN3Kppz3rKDDUZLPz9pe_voSFR4,5014
+warp_beacon/telegram/progress_bar.py,sha256=IP4xtvLtdJtqdr2C-0YaU428iQGrKurbP4Npr31iW74,5014
 warp_beacon/telegram/progress_file_reader.py,sha256=e3equyNKlKs764AD-iE9QRsh3YDHTzP78Mx5tdvPPWs,969
 warp_beacon/telegram/types.py,sha256=Kvdng6uCF1HRoqQgGW1ZYYPJoVuYkFb-LDvMBbW5Hjk,89
 warp_beacon/telegram/utils.py,sha256=S1N_JbHM_ExiM5tS0CeYKWIlvaJMlyq85TQNbK_GVds,5029
 warp_beacon/uploader/__init__.py,sha256=ewvR60k9osJxw_kb5U-TlZkCGR_574biq3w_aR1tjIU,5689
-warp_beacon-2.7.11.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.7.11.dist-info/METADATA,sha256=C8Y4rTMfgyxlL2A13F6qEFy4UZr_KNLeTOYNZhAfkGg,23215
-warp_beacon-2.7.11.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-warp_beacon-2.7.11.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.7.11.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
-warp_beacon-2.7.11.dist-info/RECORD,,
+warp_beacon-2.7.13.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.7.13.dist-info/METADATA,sha256=wRZPk7SRIYDZ9xSsAVfRdW_TK206mInTQIiZYLRD_sY,23215
+warp_beacon-2.7.13.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+warp_beacon-2.7.13.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.7.13.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
+warp_beacon-2.7.13.dist-info/RECORD,,