warp-beacon 1.1.2__py3-none-any.whl → 1.2.3__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "1.1.2"
+__version__ = "1.2.3"
 

warp_beacon/jobs/__init__.py

@@ -3,4 +3,5 @@ from enum import Enum
 class Origin(Enum):
 	INSTAGRAM = "instagram"
 	YT_SHORTS = "yt_shorts"
+	YT_MUSIC = "yt_music"
 	UNKNOWN = "unknown"

warp_beacon/jobs/abstract.py

@@ -25,6 +25,7 @@ class JobSettings(TypedDict):
 	save_items: bool
 	media_collection: list
 	job_origin: Origin
+	canonical_name: str
 
 class AbstractJob(ABC):
 	job_id: uuid.UUID = None
@@ -46,6 +47,7 @@ class AbstractJob(ABC):
 	save_items: bool = False
 	media_collection: list = []
 	job_origin: Origin = Origin.UNKNOWN
+	canonical_name: str = ""
 
 	def __init__(self, **kwargs: Unpack[JobSettings]) -> None:
 		if kwargs:

warp_beacon/mediainfo/video.py

@@ -2,20 +2,17 @@ import io, os
 
 from typing import Union
 from PIL import Image
-import av
+
+from warp_beacon.mediainfo.abstract import MediaInfoAbstract
 
 import logging
 
-class VideoInfo(object):
+class VideoInfo(MediaInfoAbstract):
 	width = 0
 	height = 0
-	duration = 0.0
-	filename = ""
-	container = None
 
 	def __init__(self, filename: str) -> None:
-		self.filename = filename
-		self.container = av.open(file=self.filename, mode='r')
+		super(VideoInfo, self).__init__(filename)
 		
 		if self.container:
 			stream = self.container.streams.video[0]
@@ -31,21 +28,10 @@ class VideoInfo(object):
 			self.height = frame.height
 			# restore original position after previous frame search
 			self.container.seek(0, backward=False, stream=stream)
-		
-	def __del__(self) -> None:
-		if self.container:
-			self.container.close()
 
 	def get_demensions(self) -> dict:
 		return {"width": self.width, "height": self.height}
 
-	def get_duration(self) -> float:
-		return self.duration
-
-	@staticmethod
-	def get_filesize(filename: str) -> float:
-		return os.stat(filename).st_size
-	
 	def get_finfo(self, except_info: tuple=()) -> dict:
 		res = {}
 		res.update(self.get_demensions())

warp_beacon/scraper/__init__.py

@@ -7,6 +7,7 @@ from queue import Empty
 
 from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable
 from warp_beacon.mediainfo.video import VideoInfo
+from warp_beacon.mediainfo.audio import AudioInfo
 from warp_beacon.compress.video import VideoCompress
 from warp_beacon.uploader import AsyncUploader
 from warp_beacon.jobs import Origin
@@ -38,14 +39,18 @@ class AsyncDownloader(object):
 			self.workers.append(proc)
 			proc.start()
 
-	def get_media_info(self, path: str, fr_media_info: dict={}) -> Optional[dict]:
+	def get_media_info(self, path: str, fr_media_info: dict={}, media_type: str = "video") -> Optional[dict]:
 		media_info = None
 		try:
 			if path:
-				video_info = VideoInfo(path)
-				media_info = video_info.get_finfo(tuple(fr_media_info.keys()))
-				media_info.update(fr_media_info)
-				media_info["thumb"] = video_info.generate_thumbnail()
+				if media_type == "video":
+					video_info = VideoInfo(path)
+					media_info = video_info.get_finfo(tuple(fr_media_info.keys()))
+					media_info.update(fr_media_info)
+					media_info["thumb"] = video_info.generate_thumbnail()
+				elif media_type == "audio":
+					audio_info = AudioInfo(path)
+					media_info = audio_info.get_finfo(tuple(fr_media_info.keys()))
 		except Exception as e:
 			logging.error("Failed to process media info!")
 			logging.exception(e)
@@ -73,6 +78,9 @@ class AsyncDownloader(object):
 								elif job.job_origin is Origin.YT_SHORTS:
 									from warp_beacon.scraper.youtube.shorts import YoutubeShortsScraper
 									actor = YoutubeShortsScraper()
+								elif job.job_origin is Origin.YT_MUSIC:
+									from warp_beacon.scraper.youtube.music import YoutubeMusicScraper
+									actor = YoutubeMusicScraper()
 								while True:
 									try:
 										logging.info("Downloading URL '%s'", job.url)
@@ -136,6 +144,9 @@ class AsyncDownloader(object):
 													item["local_compressed_media_path"] = new_filepath
 													media_info["filesize"] = VideoInfo.get_filesize(new_filepath)
 													logging.info("New file size of compressed file is '%.3f'", media_info["filesize"])
+										elif item["media_type"] == "audio":
+											media_info = self.get_media_info(item["local_media_path"], item.get("media_info", {}), "audio")
+											logging.info("Final media info: %s", media_info)
 										elif item["media_type"] == "collection":
 											for v in item["items"]:
 												if v["media_type"] == "video":
@@ -153,6 +164,8 @@ class AsyncDownloader(object):
 											if item.get("local_compressed_media_path", None):
 												job_args["local_media_path"] = item.get("local_compressed_media_path", None)
 
+										job_args["canonical_name"] = item.get("canonical_name", "")
+
 										logging.debug("local_media_path: '%s'", job_args.get("local_media_path", ""))
 										logging.debug("media_collection: '%s'", str(job_args.get("media_collection", {})))
 										upload_job = job.to_upload_job(**job_args)

warp_beacon/scraper/exceptions.py

@@ -30,7 +30,9 @@ class UnknownError(ScraperError):
 
 def extract_exception_message(e: Exception) -> str:
 	msg = ""
-	if hasattr(e, "error_string"):
+	if hasattr(e, "expected"):
+		msg = "Expected bytes: %d" % int(e.expected)
+	elif hasattr(e, "error_string"):
 		msg = e.error_string
 	elif hasattr(e, "message"):
 		msg = e.message

warp_beacon/scraper/instagram.py

@@ -1,14 +1,17 @@
 import os
 import time
+
+import socket
+import ssl
+
 from typing import Callable, Optional, Union
+
 from pathlib import Path
 import json
 
 import requests
-from requests.exceptions import ConnectTimeout, HTTPError
 import urllib3
 from urllib.parse import urljoin, urlparse
-import logging
 
 from instagrapi.mixins.story import Story
 from instagrapi.types import Media
@@ -18,6 +21,8 @@ from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotF
 from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 
+import logging
+
 INST_SESSION_FILE = "/var/warp_beacon/inst_session.json"
 
 class InstagramScraper(ScraperAbstract):
@@ -95,16 +100,18 @@ class InstagramScraper(ScraperAbstract):
 			try:
 				ret_val = func(*args, **kwargs)
 				break
-			except (requests.exceptions.ConnectionError,
+			except (socket.timeout,
+					ssl.SSLError,
+					requests.exceptions.ConnectionError,
 					requests.exceptions.ReadTimeout,
+					requests.exceptions.ConnectTimeout,
+					requests.exceptions.HTTPError,
 					urllib3.exceptions.ReadTimeoutError,
-					urllib3.exceptions.ConnectionError,
-					ConnectTimeout,
-					HTTPError) as e:
+					urllib3.exceptions.ConnectionError) as e:
 				logging.warning("Instagram read timeout! Retrying in 2 seconds ...")
 				logging.info("Your `IG_MAX_RETRIES` values is %d", max_retries)
 				logging.exception(e)
-				if max_retries == retries:
+				if max_retries <= retries:
 					raise TimeOut(extract_exception_message(e))
 				retries += 1
 				time.sleep(2)

warp_beacon/scraper/youtube/shorts.py

@@ -2,18 +2,19 @@ import os
 import pathlib
 import time
 
+import socket
+import ssl
+
 from typing import Callable, Union
 
-from socket import timeout
-from ssl import SSLError
-from requests.exceptions import RequestException
-from urllib.error import URLError
-from http.client import HTTPException
+import requests
+import urllib
+import http.client
 
 from pytubefix import YouTube
 from pytubefix.exceptions import VideoUnavailable, VideoPrivate, MaxRetriesExceeded
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, extract_exception_message
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 
 import logging
@@ -25,6 +26,11 @@ class YoutubeShortsScraper(ScraperAbstract):
 	def __del__(self) -> None:
 		pass
 
+	def remove_tmp_files(self) -> None:
+		for i in os.listdir(DOWNLOAD_DIR):
+			if "yt_download_" in i:
+				os.unlink("%s/%s" % (DOWNLOAD_DIR, i))
+
 	def _download_hndlr(self, func: Callable, *args: tuple[str], **kwargs: dict[str]) -> Union[str, dict]:
 		ret_val = ''
 		max_retries = int(os.environ.get("YT_MAX_RETRIES", default=8))
@@ -37,11 +43,14 @@ class YoutubeShortsScraper(ScraperAbstract):
 			except MaxRetriesExceeded:
 				# do noting, not interested
 				pass
-			except (timeout, SSLError, HTTPException, RequestException, URLError) as e:
+			except (socket.timeout, ssl.SSLError, http.client.HTTPException, requests.RequestException, urllib.error.URLError) as e:
+				if hasattr(e, "code") and int(e.code) == 403:
+					raise Unavailable(extract_exception_message(e))
 				logging.warning("Youtube read timeout! Retrying in %d seconds ...", pause_secs)
 				logging.info("Your `YT_MAX_RETRIES` values is %d", max_retries)
 				logging.exception(extract_exception_message(e))
-				if max_retries >= retries:
+				if max_retries <= retries:
+					self.remove_tmp_files()
 					raise TimeOut(extract_exception_message(e))
 				retries += 1
 				time.sleep(pause_secs)
@@ -74,8 +83,10 @@ class YoutubeShortsScraper(ScraperAbstract):
 				output_path="/tmp",
 				max_retries=0,
 				timeout=timeout,
-				skip_existing=False
+				skip_existing=False,
+				filename_prefix="yt_download_"
 			)
+			logging.debug("Temp filename: '%s'", local_file)
 			res.append({"local_media_path": self.rename_local_file(local_file), "media_type": "video"})
 
 		return res

warp_beacon/storage/__init__.py

@@ -2,7 +2,7 @@ import os
 #from typing import Optional
 import logging
 
-from urllib.parse import urlparse
+from urllib.parse import urlparse, parse_qs
 
 from pymongo import MongoClient
 
@@ -28,6 +28,15 @@ class Storage(object):
 
 	@staticmethod
 	def compute_uniq(url: str) -> str:
+		if "music.youtube.com" in url:
+			qs = parse_qs(urlparse(url).query)
+			yt_vid_id = qs.get('v', None)
+			if yt_vid_id:
+				path = urlparse(url).path.strip('/').replace("watch", "yt_music")
+				return "%s/%s" % (path, yt_vid_id)
+			else:
+				raise ValueError("Failed to generate uniq_id for url '%s'", url)
+			
 		path = urlparse(url).path.strip('/')
 		return path
 	

warp_beacon/warp_beacon.py

@@ -10,7 +10,7 @@ from io import BytesIO
 from urlextract import URLExtract
 
 import telegram
-from telegram import Bot, ForceReply, Update, Chat, error, InputMediaVideo, InputMediaPhoto, MessageEntity, InlineKeyboardMarkup, InlineKeyboardButton
+from telegram import Bot, ForceReply, Update, Chat, error, InputMediaVideo, InputMediaPhoto, InputMediaAudio, MessageEntity, InlineKeyboardMarkup, InlineKeyboardButton
 from telegram.ext import Application, CommandHandler, ContextTypes, MessageHandler, filters
 from telegram.constants import ParseMode
 
@@ -234,6 +234,17 @@ def build_tg_args(update: Update, context: ContextTypes.DEFAULT_TYPE, job: Uploa
 				media=open(job.local_media_path, 'rb'),
 				filename="downloaded_via_warp_beacon_bot%s" % (os.path.splitext(job.local_media_path)[-1])
 			)
+	elif job.media_type == "audio":
+		if job.tg_file_id:
+			if job.placeholder_message_id:
+				args["media"] = InputMediaAudio(media=job.tg_file_id.replace(":audio", ''))
+			else:
+				args["audio"] = job.tg_file_id.replace(":audio", '')
+		else:
+			args["media"] = InputMediaAudio(
+				media=open(job.local_media_path, 'rb'),
+				filename="%s%s" % (job.canonical_name, os.path.splitext(job.local_media_path)[-1])
+			)
 	elif job.media_type == "collection":
 		if job.tg_file_id:
 			args["media"] = []
@@ -306,6 +317,14 @@ async def upload_job(update: Update, context: ContextTypes.DEFAULT_TYPE, job: Up
 					if message.photo:
 						tg_file_ids.append(message.photo[-1].file_id)
 						job.tg_file_id = message.photo[-1].file_id
+				elif job.media_type == "audio":
+					if job.placeholder_message_id:
+						message = await context.bot.edit_message_media(**build_tg_args(update, context, job))
+					else:
+						message = await update.message.reply_audio(**build_tg_args(update, context, job))
+					if message.audio:
+						tg_file_ids.append(message.audio.file_id)
+						job.tg_file_id = message.audio.file_id
 				elif job.media_type == "collection":
 					sent_messages = await update.message.reply_media_group(**build_tg_args(update, context, job))
 					if job.placeholder_message_id:
@@ -371,6 +390,9 @@ def extract_origin(url: str) -> Origin:
 	if "youtube.com/" in url and "shorts/" in url:
 		return Origin.YT_SHORTS
 
+	if "youtube.com/" in url and "music." in url:
+		return Origin.YT_MUSIC
+
 	return Origin.UNKNOWN
 
 async def handler(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
@@ -388,7 +410,7 @@ async def handler(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
 		for url in urls:
 			origin = extract_origin(url)
 			if origin is Origin.UNKNOWN:
-				logging.info("Only Instagram and YouTube Shorts are now supported. Skipping.")
+				logging.info("Only Instagram, YouTube Shorts and YouTube Music are now supported. Skipping.")
 				continue
 			entities, tg_file_ids = [], []
 			uniq_id = Storage.compute_uniq(url)

{warp_beacon-1.1.2.dist-info → warp_beacon-1.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 1.1.2
+Version: 1.2.3
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-1.1.2.dist-info → warp_beacon-1.2.3.dist-info}/RECORD

@@ -2,30 +2,30 @@ etc/warp_beacon/warp_beacon.conf,sha256=1gGvh36cnFr0rU4mVomfy66hQz9EvugaNzeH6_tm
 lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuwwPfa2mDLUA,372
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 warp_beacon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/__version__.py,sha256=y-YwjKcQPThc0FMcn_TMBl8-1vHN4zwnWOJXVrYbWTI,23
-warp_beacon/warp_beacon.py,sha256=Z3zY7MTMZp3zKRderhNOVe6XvM-93Djo0DIsiaoucmo,20012
+warp_beacon/__version__.py,sha256=FoCfbP1cOqI5xxyfhI-2bmJFC6d3DUpIK7513kJdTxI,23
+warp_beacon/warp_beacon.py,sha256=DeENFTvlwa8qWhQrLPvwReaOF9LcC3rzdR2_nouEiRs,20910
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
-warp_beacon/jobs/__init__.py,sha256=KsSaS0KlCNyffNnWKuvqmdfgyfKyn3niXYyZ38-exQ8,113
-warp_beacon/jobs/abstract.py,sha256=PCr8RXzocKi-sTsi2Y1_spiv6D95G1NlzZ2wD2WJXRc,1760
+warp_beacon/jobs/__init__.py,sha256=99x1MDo8GLfnKtqiNExNnrWCP6_rKa4WclQDQhzjSxY,136
+warp_beacon/jobs/abstract.py,sha256=zHkh31JT4YL-607hmqswAlI3kwhuXwUZafjYOBqpEqw,1807
 warp_beacon/jobs/download_job.py,sha256=wfZrKUerfYIjWkRxPzfl5gwIlcotIMH7OpTUM9ae8NY,736
 warp_beacon/jobs/upload_job.py,sha256=Vaogc4vbpAfyaT4VkIHEPLFRELmM44TDqkmnPYh3Ymc,740
 warp_beacon/mediainfo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/mediainfo/abstract.py,sha256=O9ZROcW1cDlLMqBrUW2KI06tHCQ_iflDJknZA3iFaHE,591
 warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-ZM,651
-warp_beacon/mediainfo/video.py,sha256=8h7p4k0w45Vm0lPQNlaQaUjaDTBybX3RcKgL1QQbioA,2638
-warp_beacon/scraper/__init__.py,sha256=c7NySK5Krm-zlWQckFs-uN4fD3J19A0pTS4CByXjmMs,7918
+warp_beacon/mediainfo/video.py,sha256=A0CZX3wdL9i1M4j_YW8hbg7betGA2UXwY1RKbiOL-FY,2381
+warp_beacon/scraper/__init__.py,sha256=lb6ehjcrEt_YF_urpNobnO44RD7nKH5jqcSpMkSkNsE,8609
 warp_beacon/scraper/abstract.py,sha256=um4wUthO_7IsoXjKiUTWyBBbKlf-N01aZJK9N2UQI9I,408
-warp_beacon/scraper/exceptions.py,sha256=qra_Jx53RVCnrCKvw2OxvEHl4cXJCrPDa_yef3cvGXM,978
-warp_beacon/scraper/instagram.py,sha256=s7slumqdqVVWQYpnVphx-dOLIAWvgA_UdqkTQVp6GsI,7758
+warp_beacon/scraper/exceptions.py,sha256=BdC9JRXRSuvZR8nlfYG62SITMeg3DGvRLsCsBr_ACy0,1055
+warp_beacon/scraper/instagram.py,sha256=atYkK--4ctvSfTqiPQnYHsJ4_q5fCYfNj9CwWr9tvRM,7809
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/youtube/music.py,sha256=htxLJW-RaRhgxqJ9276ibaLnuStNtw26-pKJwfaQpgY,3335
-warp_beacon/scraper/youtube/shorts.py,sha256=xAOeHNT3_Ayjjglid00UqGqCgSMfbJua26PNrbbDYUo,2565
-warp_beacon/storage/__init__.py,sha256=NhD3V7UNRiZNf61yQEAjXOfi-tfA2LaJa7a7kvbkmtE,2402
+warp_beacon/scraper/youtube/shorts.py,sha256=yihisHSVDDT-4MflfjYkHWpnp8PrlyNo21ICKSf-rYU,2890
+warp_beacon/storage/__init__.py,sha256=ljeEP_zKDxKVBXWXdhJL1c2hTKWhP8ubazOkojAkjZs,2724
 warp_beacon/uploader/__init__.py,sha256=auD1arKpJdN1eFUbTFoa9Gmv-ZYZNesMoT193__pDz8,4507
-warp_beacon-1.1.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-1.1.2.dist-info/METADATA,sha256=eRC_mlblWo5zz0fAw2pFXJFhEQAny4V8ieMKh2kAMmE,18244
-warp_beacon-1.1.2.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-warp_beacon-1.1.2.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-1.1.2.dist-info/top_level.txt,sha256=QuN6MynevEblMhPPAVeMrNAkcyqYUpYDholtIRq8-ew,473
-warp_beacon-1.1.2.dist-info/RECORD,,
+warp_beacon-1.2.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-1.2.3.dist-info/METADATA,sha256=kooNNS-QQyjIXZRjkZDJ7S7WyI3gt6MiN6GDESaZHXA,18244
+warp_beacon-1.2.3.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
+warp_beacon-1.2.3.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-1.2.3.dist-info/top_level.txt,sha256=-jYi-GhbnF8nFw16lQwExFZqTiFRW62R8HztQDNQ1po,566
+warp_beacon-1.2.3.dist-info/RECORD,,

{warp_beacon-1.1.2.dist-info → warp_beacon-1.2.3.dist-info}/top_level.txt

@@ -7,12 +7,15 @@ warp_beacon/jobs/abstract
 warp_beacon/jobs/download_job
 warp_beacon/jobs/upload_job
 warp_beacon/mediainfo
+warp_beacon/mediainfo/abstract
+warp_beacon/mediainfo/audio
 warp_beacon/mediainfo/video
 warp_beacon/scraper
 warp_beacon/scraper/abstract
 warp_beacon/scraper/exceptions
 warp_beacon/scraper/instagram
 warp_beacon/scraper/youtube
+warp_beacon/scraper/youtube/music
 warp_beacon/scraper/youtube/shorts
 warp_beacon/storage
 warp_beacon/uploader