warp-beacon 2.3.39__tar.gz → 2.3.40__tar.gz

{warp_beacon-2.3.39/warp_beacon.egg-info → warp_beacon-2.3.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.3.39
+Version: 2.3.40
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -237,6 +237,7 @@ Requires-Dist: pillow
 Requires-Dist: pymongo
 Requires-Dist: instagrapi==2.0.0
 Requires-Dist: bs4
+Requires-Dist: yt_dlp
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{warp_beacon-2.3.39 → warp_beacon-2.3.40}/pyproject.toml

@@ -25,7 +25,8 @@ dependencies = [
 	"pillow",
 	"pymongo",
 	"instagrapi == 2.0.0",
-	"bs4"
+	"bs4",
+	"yt_dlp"
 ]
 keywords = ["bot", "telegram", "instagram", "export"]
 classifiers = [

warp_beacon-2.3.40/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.3.40"
+

{warp_beacon-2.3.39 → warp_beacon-2.3.40}/warp_beacon/scraper/exceptions.py

@@ -54,6 +54,8 @@ def extract_exception_message(e: Exception) -> str:
 		msg = e.message
 	elif hasattr(e, "reason"):
 		msg = e.reason
+	elif hasattr(e, "msg"):
+		msg = e.msg
 	elif hasattr(e, "args"):
 		msg = str(e.args)
 		

{warp_beacon-2.3.39 → warp_beacon-2.3.40}/warp_beacon/scraper/youtube/abstract.py

@@ -7,26 +7,29 @@ import socket
 import ssl
 #from abc import abstractmethod
 from typing import Callable, Union, Optional
+import logging
 import json
 import urllib
 import http.client
+import pytubefix.exceptions
 import requests
 from PIL import Image
 import numpy as np
+from urllib.parse import urlparse, parse_qs
 
-from warp_beacon.jobs.download_job import DownloadJob
-from warp_beacon.scraper.abstract import ScraperAbstract
-#from warp_beacon.mediainfo.abstract import MediaInfoAbstract
-from warp_beacon.scraper.exceptions import TimeOut, Unavailable, extract_exception_message
-
+import pytubefix
 from pytubefix import YouTube
 from pytubefix.innertube import _default_clients
 from pytubefix.streams import Stream
 from pytubefix.innertube import InnerTube, _client_id, _client_secret
-from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
+#from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
 from pytubefix import request
+import yt_dlp
 
-import logging
+from warp_beacon.jobs.download_job import DownloadJob
+from warp_beacon.scraper.abstract import ScraperAbstract
+#from warp_beacon.mediainfo.abstract import MediaInfoAbstract
+from warp_beacon.scraper.exceptions import TimeOut, Unavailable, extract_exception_message
 
 def patched_fetch_bearer_token(self) -> None:
 	"""Fetch an OAuth token."""
@@ -101,13 +104,24 @@ class YoutubeAbstract(ScraperAbstract):
 		os.rename(filename, new_filepath)
 
 		return new_filepath
+	
+	def get_video_id(self, url: str) -> Optional[str]:
+		parsed_url = urlparse(url)
+		query = parse_qs(parsed_url.query)
+		return query.get('v', [None])[0]
 
 	def remove_tmp_files(self) -> None:
 		for i in os.listdir(self.DOWNLOAD_DIR):
 			if "yt_download_" in i:
 				os.unlink("%s/%s" % (self.DOWNLOAD_DIR, i))
 
-	def calculate_size_with_padding(self, image: Image, aspect_ratio_width: int, aspect_ratio_height: int, target_size: tuple=(320, 320), background_color: tuple=(0, 0, 0)) -> Image:
+	def calculate_size_with_padding(self,
+		image: Image,
+		aspect_ratio_width: int,
+		aspect_ratio_height: int,
+		target_size: tuple=(320, 320),
+		background_color: tuple=(0, 0, 0)
+	) -> Image:
 		aspect_ratio = aspect_ratio_width / aspect_ratio_height
 		target_width, target_height = target_size
 
@@ -194,7 +208,7 @@ class YoutubeAbstract(ScraperAbstract):
 				kwargs["timeout"] = timeout
 				ret_val = func(*args, **kwargs)
 				break
-			except MaxRetriesExceeded:
+			except pytubefix.exceptions.MaxRetriesExceeded:
 				# do noting, not interested
 				pass
 			#except http.client.IncompleteRead as e:
@@ -216,7 +230,21 @@ class YoutubeAbstract(ScraperAbstract):
 				retries += 1
 				timeout += timeout_increment
 				time.sleep(pause_secs)
-			except (VideoUnavailable, VideoPrivate) as e:
+			except (pytubefix.exceptions.VideoUnavailable, pytubefix.exceptions.VideoPrivate) as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.DownloadError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.GeoRestrictedError:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.PostProcessingError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.ExtractorError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.MaxDownloadsReached as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.UnavailableVideoError as e:
+				raise Unavailable(extract_exception_message(e))
+			except yt_dlp.utils.ThrottledDownload as e:
 				raise Unavailable(extract_exception_message(e))
 
 		return ret_val
@@ -241,8 +269,8 @@ class YoutubeAbstract(ScraperAbstract):
 			yt_opts["token_file"] = self.YT_SESSION_FILE % self.account_index
 		if self.proxy:
 			proxy_dsn = self.proxy.get("dsn", "")
+			logging.info("Using proxy DSN '%s'", proxy_dsn)
 			if proxy_dsn:
-				logging.info("Using proxy DSN '%s'", proxy_dsn)
 				if "https://" in proxy_dsn:
 					yt_opts["proxies"] = {"https": proxy_dsn}
 				elif "http://" in proxy_dsn:
@@ -251,14 +279,44 @@ class YoutubeAbstract(ScraperAbstract):
 					logging.warning("Proxy DSN malformed!")
 		return YouTube(**yt_opts)
 	
+	def build_yt_dlp(self) -> yt_dlp.YoutubeDL:
+		auth_data = {}
+		with open(self.YT_SESSION_FILE % self.account_index, 'r', encoding="utf-8") as f:
+			auth_data = json.loads(f.read())
+		time_name = str(time.time()).replace('.', '_')
+		ydl_opts = {
+			'outtmpl': f'{self.DOWNLOAD_DIR}/{time_name}.%(ext)s',
+			'format': 'bestvideo+bestaudio/best',
+			'merge_output_format': 'mp4',
+			'noplaylist': True,
+			'tv_auth': auth_data
+		}
+
+		if self.proxy:
+			proxy_dsn = self.proxy.get("dsn", "")
+			logging.info("Using proxy DSN '%s'", proxy_dsn)
+			if proxy_dsn:
+				ydl_opts["proxy"] = proxy_dsn
+
+		return yt_dlp.YoutubeDL(ydl_opts)
+	
 	def _download(self, _: str) -> list:
 		raise NotImplementedError("You should to implement _download method")
+	
+	def _download_yt_dlp(self, _: str) -> list:
+		raise NotImplementedError("You should to implement _download_yt_dlp method")
 
 	def download(self, job: DownloadJob) -> list:
+		ret = []
 		try:
-			return self.download_hndlr(self._download, job.url, session=False)
-		except Unavailable:
-			logging.warning("Failed sessionless download attempt. Trying with session.")
-			time.sleep(2)
+			ret = self.download_hndlr(self._download, job.url, session=True)
+		except (Unavailable, TimeOut):
+			logging.warning("Download failed, trying to download with yt_dlp")
 		
-		return self.download_hndlr(self._download, job.url, session=True)
+		try:
+			ret = self.download_hndlr(self._download_yt_dlp, job.url)
+		except NotImplementedError:
+			logging.info("yt_dlp is not supported for this submodule yet")
+			raise Unavailable("Video unvailable")
+
+		return ret

{warp_beacon-2.3.39 → warp_beacon-2.3.40}/warp_beacon/scraper/youtube/youtube.py

@@ -1,10 +1,10 @@
-from warp_beacon.jobs.types import JobType
-from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
-from warp_beacon.scraper.exceptions import YoutubeLiveError, NotFound, YotubeAgeRestrictedError
+import logging
 
 from pytubefix.exceptions import AgeRestrictedError
 
-import logging
+from warp_beacon.jobs.types import JobType
+from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
+from warp_beacon.scraper.exceptions import YoutubeLiveError, NotFound, YotubeAgeRestrictedError
 
 class YoutubeScraper(YoutubeAbstract):
 	YT_MAX_RETRIES_DEFAULT = 8
@@ -32,14 +32,14 @@ class YoutubeScraper(YoutubeAbstract):
 		res = []
 		try:
 			thumbnail = None
+			video_id = self.get_video_id(url)
+			if video_id:
+				thumbnail = self.download_hndlr(self.download_thumbnail, video_id)
 			yt = self.build_yt(url, session=session)
 
 			if self.is_live(yt.initial_data):
 				raise YoutubeLiveError("Youtube Live is not supported")
 
-			if yt:
-				thumbnail = self.download_hndlr(self.download_thumbnail, yt.video_id)
-
 			stream = yt.streams.get_highest_resolution()
 
 			if not stream:
@@ -65,4 +65,24 @@ class YoutubeScraper(YoutubeAbstract):
 		except AgeRestrictedError as e:
 			raise YotubeAgeRestrictedError("Youtube Age Restricted error")
 
+		return res
+	
+	def _download_yt_dlp(self, url: str) -> list:
+		res = []
+		thumbnail = None
+		video_id = self.get_video_id(url)
+		if video_id:
+			thumbnail = self.download_hndlr(self.download_thumbnail, video_id)
+		with self.build_yt_dlp() as ydl:
+			info = ydl.extract_info(url, download=True)
+			local_file = ydl.prepare_filename(info)
+			logging.debug("Temp filename: '%s'", local_file)
+			res.append({
+				"local_media_path": local_file,
+				"performer": info.get("uploader", "Unknown"),
+				"thumb": thumbnail,
+				"canonical_name": info.get("title", ''),
+				"media_type": JobType.VIDEO
+			})
+
 		return res

{warp_beacon-2.3.39 → warp_beacon-2.3.40/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.3.39
+Version: 2.3.40
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -237,6 +237,7 @@ Requires-Dist: pillow
 Requires-Dist: pymongo
 Requires-Dist: instagrapi==2.0.0
 Requires-Dist: bs4
+Requires-Dist: yt_dlp
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file

{warp_beacon-2.3.39 → warp_beacon-2.3.40}/warp_beacon.egg-info/requires.txt

@@ -11,3 +11,4 @@ pillow
 pymongo
 instagrapi==2.0.0
 bs4
+yt_dlp

warp_beacon-2.3.39/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.3.39"
-