warp-beacon 1.1.2__tar.gz → 1.2.4__tar.gz

{warp_beacon-1.1.2/warp_beacon.egg-info → warp_beacon-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 1.1.2
+Version: 1.2.4
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/setup.py

@@ -60,12 +60,15 @@ setup(
 		"warp_beacon/jobs/abstract",
 		"warp_beacon/jobs/download_job",
 		"warp_beacon/jobs/upload_job",
+		"warp_beacon/mediainfo/abstract",
 		"warp_beacon/mediainfo/video",
+		"warp_beacon/mediainfo/audio",
 		"warp_beacon/compress/video",
 		"warp_beacon/scraper/abstract",
 		"warp_beacon/scraper/exceptions",
 		"warp_beacon/scraper/instagram",
-		"warp_beacon/scraper/youtube/shorts"
+		"warp_beacon/scraper/youtube/shorts",
+		"warp_beacon/scraper/youtube/music"
 	],
 	#scripts=['scripts/wait_dc_update.py'],
 	data_files=[

warp_beacon-1.2.4/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "1.2.4"
+

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/warp_beacon/jobs/__init__.py

@@ -3,4 +3,5 @@ from enum import Enum
 class Origin(Enum):
 	INSTAGRAM = "instagram"
 	YT_SHORTS = "yt_shorts"
+	YT_MUSIC = "yt_music"
 	UNKNOWN = "unknown"

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/warp_beacon/jobs/abstract.py

@@ -25,6 +25,7 @@ class JobSettings(TypedDict):
 	save_items: bool
 	media_collection: list
 	job_origin: Origin
+	canonical_name: str
 
 class AbstractJob(ABC):
 	job_id: uuid.UUID = None
@@ -46,6 +47,7 @@ class AbstractJob(ABC):
 	save_items: bool = False
 	media_collection: list = []
 	job_origin: Origin = Origin.UNKNOWN
+	canonical_name: str = ""
 
 	def __init__(self, **kwargs: Unpack[JobSettings]) -> None:
 		if kwargs:

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/warp_beacon/mediainfo/video.py

@@ -2,20 +2,17 @@ import io, os
 
 from typing import Union
 from PIL import Image
-import av
+
+from warp_beacon.mediainfo.abstract import MediaInfoAbstract
 
 import logging
 
-class VideoInfo(object):
+class VideoInfo(MediaInfoAbstract):
 	width = 0
 	height = 0
-	duration = 0.0
-	filename = ""
-	container = None
 
 	def __init__(self, filename: str) -> None:
-		self.filename = filename
-		self.container = av.open(file=self.filename, mode='r')
+		super(VideoInfo, self).__init__(filename)
 		
 		if self.container:
 			stream = self.container.streams.video[0]
@@ -31,21 +28,10 @@ class VideoInfo(object):
 			self.height = frame.height
 			# restore original position after previous frame search
 			self.container.seek(0, backward=False, stream=stream)
-		
-	def __del__(self) -> None:
-		if self.container:
-			self.container.close()
 
 	def get_demensions(self) -> dict:
 		return {"width": self.width, "height": self.height}
 
-	def get_duration(self) -> float:
-		return self.duration
-
-	@staticmethod
-	def get_filesize(filename: str) -> float:
-		return os.stat(filename).st_size
-	
 	def get_finfo(self, except_info: tuple=()) -> dict:
 		res = {}
 		res.update(self.get_demensions())

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/warp_beacon/scraper/__init__.py

@@ -5,8 +5,9 @@ from typing import Optional
 import multiprocessing
 from queue import Empty
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig
 from warp_beacon.mediainfo.video import VideoInfo
+from warp_beacon.mediainfo.audio import AudioInfo
 from warp_beacon.compress.video import VideoCompress
 from warp_beacon.uploader import AsyncUploader
 from warp_beacon.jobs import Origin
@@ -38,14 +39,18 @@ class AsyncDownloader(object):
 			self.workers.append(proc)
 			proc.start()
 
-	def get_media_info(self, path: str, fr_media_info: dict={}) -> Optional[dict]:
+	def get_media_info(self, path: str, fr_media_info: dict={}, media_type: str = "video") -> Optional[dict]:
 		media_info = None
 		try:
 			if path:
-				video_info = VideoInfo(path)
-				media_info = video_info.get_finfo(tuple(fr_media_info.keys()))
-				media_info.update(fr_media_info)
-				media_info["thumb"] = video_info.generate_thumbnail()
+				if media_type == "video":
+					video_info = VideoInfo(path)
+					media_info = video_info.get_finfo(tuple(fr_media_info.keys()))
+					media_info.update(fr_media_info)
+					media_info["thumb"] = video_info.generate_thumbnail()
+				elif media_type == "audio":
+					audio_info = AudioInfo(path)
+					media_info = audio_info.get_finfo(tuple(fr_media_info.keys()))
 		except Exception as e:
 			logging.error("Failed to process media info!")
 			logging.exception(e)
@@ -73,6 +78,9 @@ class AsyncDownloader(object):
 								elif job.job_origin is Origin.YT_SHORTS:
 									from warp_beacon.scraper.youtube.shorts import YoutubeShortsScraper
 									actor = YoutubeShortsScraper()
+								elif job.job_origin is Origin.YT_MUSIC:
+									from warp_beacon.scraper.youtube.music import YoutubeMusicScraper
+									actor = YoutubeMusicScraper()
 								while True:
 									try:
 										logging.info("Downloading URL '%s'", job.url)
@@ -94,6 +102,14 @@ class AsyncDownloader(object):
 											job_failed_msg="Failed to download content. Please check you Internet connection or retry amount bot configuration settings.")
 										)
 										break
+									except FileTooBig as e:
+										logging.warning("Telegram limits exceeded :(")
+										logging.exception(e)
+										self.uploader.queue_task(job.to_upload_job(
+											job_failed=True,
+											job_failed_msg="Unfortunately, this file exceeds the telegram limit of 50 megabytes.")
+										)
+										break
 									except (UnknownError, Exception) as e:
 										logging.warning("UnknownError occurred!")
 										logging.exception(e)
@@ -136,6 +152,9 @@ class AsyncDownloader(object):
 													item["local_compressed_media_path"] = new_filepath
 													media_info["filesize"] = VideoInfo.get_filesize(new_filepath)
 													logging.info("New file size of compressed file is '%.3f'", media_info["filesize"])
+										elif item["media_type"] == "audio":
+											media_info = self.get_media_info(item["local_media_path"], item.get("media_info", {}), "audio")
+											logging.info("Final media info: %s", media_info)
 										elif item["media_type"] == "collection":
 											for v in item["items"]:
 												if v["media_type"] == "video":
@@ -153,6 +172,8 @@ class AsyncDownloader(object):
 											if item.get("local_compressed_media_path", None):
 												job_args["local_media_path"] = item.get("local_compressed_media_path", None)
 
+										job_args["canonical_name"] = item.get("canonical_name", "")
+
 										logging.debug("local_media_path: '%s'", job_args.get("local_media_path", ""))
 										logging.debug("media_collection: '%s'", str(job_args.get("media_collection", {})))
 										upload_job = job.to_upload_job(**job_args)

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/warp_beacon/scraper/exceptions.py

@@ -19,6 +19,9 @@ class ScraperError(Exception):
 class TimeOut(ScraperError):
 	pass
 
+class FileTooBig(ScraperError):
+	pass
+
 class NotFound(ScraperError):
 	pass
 
@@ -30,7 +33,9 @@ class UnknownError(ScraperError):
 
 def extract_exception_message(e: Exception) -> str:
 	msg = ""
-	if hasattr(e, "error_string"):
+	if hasattr(e, "expected"):
+		msg = "Expected bytes: %d" % int(e.expected)
+	elif hasattr(e, "error_string"):
 		msg = e.error_string
 	elif hasattr(e, "message"):
 		msg = e.message

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/warp_beacon/scraper/instagram.py

@@ -1,14 +1,17 @@
 import os
 import time
+
+import socket
+import ssl
+
 from typing import Callable, Optional, Union
+
 from pathlib import Path
 import json
 
 import requests
-from requests.exceptions import ConnectTimeout, HTTPError
 import urllib3
 from urllib.parse import urljoin, urlparse
-import logging
 
 from instagrapi.mixins.story import Story
 from instagrapi.types import Media
@@ -18,6 +21,8 @@ from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotF
 from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 
+import logging
+
 INST_SESSION_FILE = "/var/warp_beacon/inst_session.json"
 
 class InstagramScraper(ScraperAbstract):
@@ -95,16 +100,18 @@ class InstagramScraper(ScraperAbstract):
 			try:
 				ret_val = func(*args, **kwargs)
 				break
-			except (requests.exceptions.ConnectionError,
+			except (socket.timeout,
+					ssl.SSLError,
+					requests.exceptions.ConnectionError,
 					requests.exceptions.ReadTimeout,
+					requests.exceptions.ConnectTimeout,
+					requests.exceptions.HTTPError,
 					urllib3.exceptions.ReadTimeoutError,
-					urllib3.exceptions.ConnectionError,
-					ConnectTimeout,
-					HTTPError) as e:
+					urllib3.exceptions.ConnectionError) as e:
 				logging.warning("Instagram read timeout! Retrying in 2 seconds ...")
 				logging.info("Your `IG_MAX_RETRIES` values is %d", max_retries)
 				logging.exception(e)
-				if max_retries == retries:
+				if max_retries <= retries:
 					raise TimeOut(extract_exception_message(e))
 				retries += 1
 				time.sleep(2)

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/warp_beacon/scraper/youtube/music.py

@@ -14,7 +14,7 @@ import http.client
 from pytubefix import YouTube
 from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, extract_exception_message
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 
 import logging
@@ -92,6 +92,10 @@ class YoutubeMusicScraper(ScraperAbstract):
 		yt = YouTube(url)
 		stream = yt.streams.get_audio_only()
 		if stream:
+			logging.info("Announced audio file size: '%d'", stream.filesize)
+			if stream.filesize > 5e+7:
+				logging.warning("Downloading size reported by YouTube is over than 50 mb!")
+				raise FileTooBig("YouTube file is larger than 50 mb")
 			logging.info("Operation timeout is '%d'", timeout)
 			local_file = stream.download(
 				output_path=DOWNLOAD_DIR,

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/warp_beacon/scraper/youtube/shorts.py

@@ -2,18 +2,19 @@ import os
 import pathlib
 import time
 
+import socket
+import ssl
+
 from typing import Callable, Union
 
-from socket import timeout
-from ssl import SSLError
-from requests.exceptions import RequestException
-from urllib.error import URLError
-from http.client import HTTPException
+import requests
+import urllib
+import http.client
 
 from pytubefix import YouTube
 from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, extract_exception_message
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 
 import logging
@@ -25,6 +26,11 @@ class YoutubeShortsScraper(ScraperAbstract):
 	def __del__(self) -> None:
 		pass
 
+	def remove_tmp_files(self) -> None:
+		for i in os.listdir(DOWNLOAD_DIR):
+			if "yt_download_" in i:
+				os.unlink("%s/%s" % (DOWNLOAD_DIR, i))
+
 	def _download_hndlr(self, func: Callable, *args: tuple[str], **kwargs: dict[str]) -> Union[str, dict]:
 		ret_val = ''
 		max_retries = int(os.environ.get("YT_MAX_RETRIES", default=8))
@@ -37,11 +43,14 @@ class YoutubeShortsScraper(ScraperAbstract):
 			except MaxRetriesExceeded:
 				# do noting, not interested
 				pass
-			except (timeout, SSLError, HTTPException, RequestException, URLError) as e:
+			except (socket.timeout, ssl.SSLError, http.client.HTTPException, requests.RequestException, urllib.error.URLError) as e:
+				if hasattr(e, "code") and int(e.code) == 403:
+					raise Unavailable(extract_exception_message(e))
 				logging.warning("Youtube read timeout! Retrying in %d seconds ...", pause_secs)
 				logging.info("Your `YT_MAX_RETRIES` values is %d", max_retries)
 				logging.exception(extract_exception_message(e))
-				if max_retries >= retries:
+				if max_retries <= retries:
+					self.remove_tmp_files()
 					raise TimeOut(extract_exception_message(e))
 				retries += 1
 				time.sleep(pause_secs)
@@ -74,8 +83,10 @@ class YoutubeShortsScraper(ScraperAbstract):
 				output_path="/tmp",
 				max_retries=0,
 				timeout=timeout,
-				skip_existing=False
+				skip_existing=False,
+				filename_prefix="yt_download_"
 			)
+			logging.debug("Temp filename: '%s'", local_file)
 			res.append({"local_media_path": self.rename_local_file(local_file), "media_type": "video"})
 
 		return res

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/warp_beacon/storage/__init__.py

@@ -2,7 +2,7 @@ import os
 #from typing import Optional
 import logging
 
-from urllib.parse import urlparse
+from urllib.parse import urlparse, parse_qs
 
 from pymongo import MongoClient
 
@@ -28,6 +28,15 @@ class Storage(object):
 
 	@staticmethod
 	def compute_uniq(url: str) -> str:
+		if "music.youtube.com" in url:
+			qs = parse_qs(urlparse(url).query)
+			yt_vid_id = qs.get('v', None)
+			if yt_vid_id:
+				path = urlparse(url).path.strip('/').replace("watch", "yt_music")
+				return "%s/%s" % (path, yt_vid_id)
+			else:
+				raise ValueError("Failed to generate uniq_id for url '%s'", url)
+			
 		path = urlparse(url).path.strip('/')
 		return path
 	

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/warp_beacon/warp_beacon.py

@@ -10,7 +10,7 @@ from io import BytesIO
 from urlextract import URLExtract
 
 import telegram
-from telegram import Bot, ForceReply, Update, Chat, error, InputMediaVideo, InputMediaPhoto, MessageEntity, InlineKeyboardMarkup, InlineKeyboardButton
+from telegram import Bot, ForceReply, Update, Chat, error, InputMediaVideo, InputMediaPhoto, InputMediaAudio, MessageEntity, InlineKeyboardMarkup, InlineKeyboardButton
 from telegram.ext import Application, CommandHandler, ContextTypes, MessageHandler, filters
 from telegram.constants import ParseMode
 
@@ -234,6 +234,17 @@ def build_tg_args(update: Update, context: ContextTypes.DEFAULT_TYPE, job: Uploa
 				media=open(job.local_media_path, 'rb'),
 				filename="downloaded_via_warp_beacon_bot%s" % (os.path.splitext(job.local_media_path)[-1])
 			)
+	elif job.media_type == "audio":
+		if job.tg_file_id:
+			if job.placeholder_message_id:
+				args["media"] = InputMediaAudio(media=job.tg_file_id.replace(":audio", ''))
+			else:
+				args["audio"] = job.tg_file_id.replace(":audio", '')
+		else:
+			args["media"] = InputMediaAudio(
+				media=open(job.local_media_path, 'rb'),
+				filename="%s%s" % (job.canonical_name, os.path.splitext(job.local_media_path)[-1])
+			)
 	elif job.media_type == "collection":
 		if job.tg_file_id:
 			args["media"] = []
@@ -306,6 +317,14 @@ async def upload_job(update: Update, context: ContextTypes.DEFAULT_TYPE, job: Up
 					if message.photo:
 						tg_file_ids.append(message.photo[-1].file_id)
 						job.tg_file_id = message.photo[-1].file_id
+				elif job.media_type == "audio":
+					if job.placeholder_message_id:
+						message = await context.bot.edit_message_media(**build_tg_args(update, context, job))
+					else:
+						message = await update.message.reply_audio(**build_tg_args(update, context, job))
+					if message.audio:
+						tg_file_ids.append(message.audio.file_id)
+						job.tg_file_id = message.audio.file_id
 				elif job.media_type == "collection":
 					sent_messages = await update.message.reply_media_group(**build_tg_args(update, context, job))
 					if job.placeholder_message_id:
@@ -371,6 +390,9 @@ def extract_origin(url: str) -> Origin:
 	if "youtube.com/" in url and "shorts/" in url:
 		return Origin.YT_SHORTS
 
+	if "youtube.com/" in url and "music." in url:
+		return Origin.YT_MUSIC
+
 	return Origin.UNKNOWN
 
 async def handler(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
@@ -388,7 +410,7 @@ async def handler(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
 		for url in urls:
 			origin = extract_origin(url)
 			if origin is Origin.UNKNOWN:
-				logging.info("Only Instagram and YouTube Shorts are now supported. Skipping.")
+				logging.info("Only Instagram, YouTube Shorts and YouTube Music are now supported. Skipping.")
 				continue
 			entities, tg_file_ids = [], []
 			uniq_id = Storage.compute_uniq(url)

{warp_beacon-1.1.2 → warp_beacon-1.2.4/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 1.1.2
+Version: 1.2.4
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-1.1.2 → warp_beacon-1.2.4}/warp_beacon.egg-info/top_level.txt

@@ -7,12 +7,15 @@ warp_beacon/jobs/abstract
 warp_beacon/jobs/download_job
 warp_beacon/jobs/upload_job
 warp_beacon/mediainfo
+warp_beacon/mediainfo/abstract
+warp_beacon/mediainfo/audio
 warp_beacon/mediainfo/video
 warp_beacon/scraper
 warp_beacon/scraper/abstract
 warp_beacon/scraper/exceptions
 warp_beacon/scraper/instagram
 warp_beacon/scraper/youtube
+warp_beacon/scraper/youtube/music
 warp_beacon/scraper/youtube/shorts
 warp_beacon/storage
 warp_beacon/uploader

warp_beacon-1.1.2/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "1.1.2"
-