warp-beacon 2.3.39__py3-none-any.whl → 2.3.41__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.3.39"
+__version__ = "2.3.41"
 

warp_beacon/scraper/exceptions.py

@@ -54,6 +54,8 @@ def extract_exception_message(e: Exception) -> str:
 		msg = e.message
 	elif hasattr(e, "reason"):
 		msg = e.reason
+	elif hasattr(e, "msg"):
+		msg = e.msg
 	elif hasattr(e, "args"):
 		msg = str(e.args)
 		

warp_beacon/scraper/youtube/abstract.py

@@ -7,26 +7,29 @@ import socket
 import ssl
 #from abc import abstractmethod
 from typing import Callable, Union, Optional
+import logging
 import json
 import urllib
 import http.client
+import pytubefix.exceptions
 import requests
 from PIL import Image
 import numpy as np
+from urllib.parse import urlparse, parse_qs
 
-from warp_beacon.jobs.download_job import DownloadJob
-from warp_beacon.scraper.abstract import ScraperAbstract
-#from warp_beacon.mediainfo.abstract import MediaInfoAbstract
-from warp_beacon.scraper.exceptions import TimeOut, Unavailable, extract_exception_message
-
+import pytubefix
 from pytubefix import YouTube
 from pytubefix.innertube import _default_clients
 from pytubefix.streams import Stream
 from pytubefix.innertube import InnerTube, _client_id, _client_secret
-from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
+#from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
 from pytubefix import request
+import yt_dlp
 
-import logging
+from warp_beacon.jobs.download_job import DownloadJob
+from warp_beacon.scraper.abstract import ScraperAbstract
+#from warp_beacon.mediainfo.abstract import MediaInfoAbstract
+from warp_beacon.scraper.exceptions import TimeOut, Unavailable, extract_exception_message
 
 def patched_fetch_bearer_token(self) -> None:
 	"""Fetch an OAuth token."""
@@ -101,13 +104,24 @@ class YoutubeAbstract(ScraperAbstract):
 		os.rename(filename, new_filepath)
 
 		return new_filepath
+	
+	def get_video_id(self, url: str) -> Optional[str]:
+		parsed_url = urlparse(url)
+		query = parse_qs(parsed_url.query)
+		return query.get('v', [None])[0]
 
 	def remove_tmp_files(self) -> None:
 		for i in os.listdir(self.DOWNLOAD_DIR):
 			if "yt_download_" in i:
 				os.unlink("%s/%s" % (self.DOWNLOAD_DIR, i))
 
-	def calculate_size_with_padding(self, image: Image, aspect_ratio_width: int, aspect_ratio_height: int, target_size: tuple=(320, 320), background_color: tuple=(0, 0, 0)) -> Image:
+	def calculate_size_with_padding(self,
+		image: Image,
+		aspect_ratio_width: int,
+		aspect_ratio_height: int,
+		target_size: tuple=(320, 320),
+		background_color: tuple=(0, 0, 0)
+	) -> Image:
 		aspect_ratio = aspect_ratio_width / aspect_ratio_height
 		target_width, target_height = target_size
 
@@ -194,7 +208,7 @@ class YoutubeAbstract(ScraperAbstract):
 				kwargs["timeout"] = timeout
 				ret_val = func(*args, **kwargs)
 				break
-			except MaxRetriesExceeded:
+			except pytubefix.exceptions.MaxRetriesExceeded:
 				# do noting, not interested
 				pass
 			#except http.client.IncompleteRead as e:
@@ -216,7 +230,21 @@ class YoutubeAbstract(ScraperAbstract):
 				retries += 1
 				timeout += timeout_increment
 				time.sleep(pause_secs)
-			except (VideoUnavailable, VideoPrivate) as e:
+			except (pytubefix.exceptions.VideoUnavailable, pytubefix.exceptions.VideoPrivate) as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.DownloadError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.GeoRestrictedError:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.PostProcessingError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.ExtractorError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.MaxDownloadsReached as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.UnavailableVideoError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.ThrottledDownload as e:
 				raise Unavailable(extract_exception_message(e))
 
 		return ret_val
@@ -241,8 +269,8 @@ class YoutubeAbstract(ScraperAbstract):
 			yt_opts["token_file"] = self.YT_SESSION_FILE % self.account_index
 		if self.proxy:
 			proxy_dsn = self.proxy.get("dsn", "")
+			logging.info("Using proxy DSN '%s'", proxy_dsn)
 			if proxy_dsn:
-				logging.info("Using proxy DSN '%s'", proxy_dsn)
 				if "https://" in proxy_dsn:
 					yt_opts["proxies"] = {"https": proxy_dsn}
 				elif "http://" in proxy_dsn:
@@ -251,14 +279,44 @@ class YoutubeAbstract(ScraperAbstract):
 					logging.warning("Proxy DSN malformed!")
 		return YouTube(**yt_opts)
 	
-	def _download(self, _: str) -> list:
+	def build_yt_dlp(self, timeout: int = 0) -> yt_dlp.YoutubeDL:
+		auth_data = {}
+		with open(self.YT_SESSION_FILE % self.account_index, 'r', encoding="utf-8") as f:
+			auth_data = json.loads(f.read())
+		time_name = str(time.time()).replace('.', '_')
+		ydl_opts = {
+			'outtmpl': f'{self.DOWNLOAD_DIR}/{time_name}.%(ext)s',
+			'format': 'bestvideo+bestaudio/best',
+			'merge_output_format': 'mp4',
+			'noplaylist': True,
+			'tv_auth': auth_data
+		}
+
+		if self.proxy:
+			proxy_dsn = self.proxy.get("dsn", "")
+			logging.info("Using proxy DSN '%s'", proxy_dsn)
+			if proxy_dsn:
+				ydl_opts["proxy"] = proxy_dsn
+
+		return yt_dlp.YoutubeDL(ydl_opts)
+	
+	def _download(self, _: str, __: int = 60) -> list:
 		raise NotImplementedError("You should to implement _download method")
+	
+	def _download_yt_dlp(self, _: str, __: int = 60) -> list:
+		raise NotImplementedError("You should to implement _download_yt_dlp method")
 
 	def download(self, job: DownloadJob) -> list:
+		ret = []
 		try:
-			return self.download_hndlr(self._download, job.url, session=False)
-		except Unavailable:
-			logging.warning("Failed sessionless download attempt. Trying with session.")
-			time.sleep(2)
+			ret = self.download_hndlr(self._download, job.url, session=True)
+		except (Unavailable, TimeOut):
+			logging.warning("Download failed, trying to download with yt_dlp")
 		
-		return self.download_hndlr(self._download, job.url, session=True)
+		try:
+			ret = self.download_hndlr(self._download_yt_dlp, job.url)
+		except NotImplementedError:
+			logging.info("yt_dlp is not supported for this submodule yet")
+			raise Unavailable("Video unvailable")
+
+		return ret

warp_beacon/scraper/youtube/youtube.py

@@ -1,10 +1,10 @@
-from warp_beacon.jobs.types import JobType
-from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
-from warp_beacon.scraper.exceptions import YoutubeLiveError, NotFound, YotubeAgeRestrictedError
+import logging
 
 from pytubefix.exceptions import AgeRestrictedError
 
-import logging
+from warp_beacon.jobs.types import JobType
+from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
+from warp_beacon.scraper.exceptions import YoutubeLiveError, NotFound, YotubeAgeRestrictedError
 
 class YoutubeScraper(YoutubeAbstract):
 	YT_MAX_RETRIES_DEFAULT = 8
@@ -28,18 +28,18 @@ class YoutubeScraper(YoutubeAbstract):
 
 		return False
 
-	def _download(self, url: str, session: bool = True, timeout: int = 0) -> list:
+	def _download(self, url: str, session: bool = True, timeout: int = 60) -> list:
 		res = []
 		try:
 			thumbnail = None
+			video_id = self.get_video_id(url)
+			if video_id:
+				thumbnail = self.download_hndlr(self.download_thumbnail, video_id)
 			yt = self.build_yt(url, session=session)
 
 			if self.is_live(yt.initial_data):
 				raise YoutubeLiveError("Youtube Live is not supported")
 
-			if yt:
-				thumbnail = self.download_hndlr(self.download_thumbnail, yt.video_id)
-
 			stream = yt.streams.get_highest_resolution()
 
 			if not stream:
@@ -65,4 +65,25 @@ class YoutubeScraper(YoutubeAbstract):
 		except AgeRestrictedError as e:
 			raise YotubeAgeRestrictedError("Youtube Age Restricted error")
 
+		return res
+	
+	def _download_yt_dlp(self, url: str, timeout: int = 60) -> list:
+		res = []
+		thumbnail = None
+		video_id = self.get_video_id(url)
+		if video_id:
+			thumbnail = self.download_hndlr(self.download_thumbnail, video_id)
+		with self.build_yt_dlp() as ydl:
+			info = ydl.extract_info(url, download=True)
+			local_file = ydl.prepare_filename(info)
+			logging.debug("Temp filename: '%s'", local_file)
+			res.append({
+				"socket_timeout": timeout,
+				"local_media_path": local_file,
+				"performer": info.get("uploader", "Unknown"),
+				"thumb": thumbnail,
+				"canonical_name": info.get("title", ''),
+				"media_type": JobType.VIDEO
+			})
+
 		return res

{warp_beacon-2.3.39.dist-info → warp_beacon-2.3.41.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.3.39
+Version: 2.3.41
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -237,6 +237,7 @@ Requires-Dist: pillow
 Requires-Dist: pymongo
 Requires-Dist: instagrapi==2.0.0
 Requires-Dist: bs4
+Requires-Dist: yt_dlp
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{warp_beacon-2.3.39.dist-info → warp_beacon-2.3.41.dist-info}/RECORD

@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=TfAWHvchJxNby1o5FoKtWae8mrExNMeQhHNn0j9tiww,24
+warp_beacon/__version__.py,sha256=eJo4w0wy-rkbDxnK_FYM-eSEodG5CN8jr8NCff2SMpU,24
 warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
@@ -24,16 +24,16 @@ warp_beacon/scheduler/scheduler.py,sha256=ddPhExSgj_RGR1eSDlIm0hVgTVM_76UEMEl9i-
 warp_beacon/scraper/__init__.py,sha256=V_C5SmAmRtjFfVBtTotOHCW3ILMQ8m_ulrBF6ry59_A,16944
 warp_beacon/scraper/abstract.py,sha256=28a0aBKZpi8IKptLWdB6RuVbOkrUbrhT7LSZX7QRQtg,2725
 warp_beacon/scraper/account_selector.py,sha256=oqwtyT2_xBYkx_7ysoz0VLa-x8EpiBifmF12MUNLCW8,5316
-warp_beacon/scraper/exceptions.py,sha256=Qkz76yo-X5kucEZIP9tWaK-oYO-kvsPEl8Y0W63oDhU,1300
+warp_beacon/scraper/exceptions.py,sha256=QBHLA8hK1sL8uJu5JhidbXoBXPZtMMtV26nYfHvO4NU,1339
 warp_beacon/scraper/fail_handler.py,sha256=_blvckfTZ4xWVancQKVRXH5ClKGwfrBxMwvXIFZh1qA,975
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/instagram.py,sha256=TpIBS9S3zcLbeGax4CENVo6WP75EE6oIttc-MjWYVjs,13815
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scraper/youtube/abstract.py,sha256=HnjSTGumo4uzxEZydH18zhsqMrus38Vx3SRGhuuXl0c,9707
+warp_beacon/scraper/youtube/abstract.py,sha256=rJrcJ9g-BETkDGqbfE7gq5TxpoF2HPxOC_XdpeTJEpA,11675
 warp_beacon/scraper/youtube/music.py,sha256=6FIU1tqKvW7Osh3jg8fqli2W30Lgp_GM33S9lfOnk0s,1418
 warp_beacon/scraper/youtube/shorts.py,sha256=k3yGF9b8HTzNE6h4VE0zsMGB6kWP-V2xiLhIY3QZXvs,1269
-warp_beacon/scraper/youtube/youtube.py,sha256=TH5od6UPh1lAzDU_UC9Y5uQ-O8scLfk6OWWKWLF6LDo,2173
+warp_beacon/scraper/youtube/youtube.py,sha256=JX3hHHuWiGK81A8POcc0F1GYr1FLjQyIzrp0Mzy9_dU,2863
 warp_beacon/storage/__init__.py,sha256=0Vajd0oITKJfu2vmNx5uQSt3-L6vwIvUYWJo8HZCjco,3398
 warp_beacon/storage/mongo.py,sha256=qC4ZiO8XXvPnP0rJwz4CJx42pqFsyAjCiW10W5QdT6E,527
 warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -43,9 +43,9 @@ warp_beacon/telegram/handlers.py,sha256=XXIfdV_RCj7tyZMPXchuKmGoDdweOaR08ADDaBPW
 warp_beacon/telegram/placeholder_message.py,sha256=wN9-BRiyrtHG-EvXtZkGJHt2CX71munQ57ITttjt0mw,6400
 warp_beacon/telegram/utils.py,sha256=9uebX53G16mV7ER7WgfdWBLFHHw14S8HBt9URrIskg0,4440
 warp_beacon/uploader/__init__.py,sha256=E9rlZIf7xlQz6MutMOwJ8S5Vm2uheR5nv23Kv8duRQg,5427
-warp_beacon-2.3.39.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.3.39.dist-info/METADATA,sha256=J2Hp81NnTvPQT4CtXBcTOwQ4gp7lf0Kn_3J2V4I3VcE,22604
-warp_beacon-2.3.39.dist-info/WHEEL,sha256=tTnHoFhvKQHCh4jz3yCn0WPTYIy7wXx3CJtJ7SJGV7c,91
-warp_beacon-2.3.39.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.3.39.dist-info/top_level.txt,sha256=qGjHVVfyf6lTmbdSA-fQq0rHS1YVS4HoJT3rag5xgPE,1141
-warp_beacon-2.3.39.dist-info/RECORD,,
+warp_beacon-2.3.41.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.3.41.dist-info/METADATA,sha256=S18xumQDMp35RZ49TerDM62XACK9XjEQ-d4N3mS1A_I,22626
+warp_beacon-2.3.41.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+warp_beacon-2.3.41.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.3.41.dist-info/top_level.txt,sha256=qGjHVVfyf6lTmbdSA-fQq0rHS1YVS4HoJT3rag5xgPE,1141
+warp_beacon-2.3.41.dist-info/RECORD,,

{warp_beacon-2.3.39.dist-info → warp_beacon-2.3.41.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (77.0.1)
+Generator: setuptools (77.0.3)
 Root-Is-Purelib: true
 Tag: py3-none-any