warp-beacon 2.2.3__py3-none-any.whl → 2.2.5__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.2.3"
+__version__ = "2.2.5"
 

warp_beacon/scraper/__init__.py

@@ -35,14 +35,16 @@ class AsyncDownloader(object):
 		self.allow_loop = multiprocessing.Value('i', 1)
 		self.uploader = uploader
 		self.workers_count = workers_count
-		self.acc_selector = AccountSelector(ACC_FILE)
+		AccountSelector.register('AccoutSelector', AccountSelector)
 
 	def __del__(self) -> None:
 		self.stop_all()
 
 	def start(self) -> None:
+		self.acc_selector = AccountSelector(ACC_FILE)
+		self.acc_selector.start()
 		for _ in range(self.workers_count):
-			proc = multiprocessing.Process(target=self.do_work)
+			proc = multiprocessing.Process(target=self.do_work, args=(self.acc_selector,))
 			self.workers.append(proc)
 			proc.start()
 
@@ -53,7 +55,8 @@ class AsyncDownloader(object):
 				if media_type == JobType.VIDEO:
 					video_info = VideoInfo(path)
 					media_info = video_info.get_finfo(tuple(fr_media_info.keys()))
-					media_info.update(fr_media_info)
+					if fr_media_info:
+						media_info.update(fr_media_info)
 					if not media_info.get("thumb", None):
 						media_info["thumb"] = video_info.generate_thumbnail()
 					media_info["has_sound"] = video_info.has_sound()
@@ -66,18 +69,18 @@ class AsyncDownloader(object):
 
 		return media_info
 
-	def try_next_account(self, job: DownloadJob, report_error: str = None) -> None:
+	def try_next_account(self, selector: AccountSelector, job: DownloadJob, report_error: str = None) -> None:
 		logging.warning("Switching account!")
 		if job.account_switches > self.acc_selector.count_service_accounts(job.job_origin):
 			raise AllAccountsFailed("All config accounts failed!")
 		if report_error:
-			self.acc_selector.bump_acc_fail("rate_limits")
-		self.acc_selector.next()
-		cur_acc = self.acc_selector.get_current()
+			selector.bump_acc_fail("rate_limits")
+		selector.next()
+		cur_acc = selector.get_current()
 		logging.info("Current account: '%s'", str(cur_acc))
 		job.account_switches += 1
 
-	def do_work(self) -> None:
+	def do_work(self, selector: AccountSelector) -> None:
 		logging.info("download worker started")
 		while self.allow_loop.value == 1:
 			try:
@@ -97,16 +100,16 @@ class AsyncDownloader(object):
 							self.acc_selector.set_module(job.job_origin)
 							if job.job_origin is Origin.INSTAGRAM:
 								from warp_beacon.scraper.instagram.instagram import InstagramScraper
-								actor = InstagramScraper(self.acc_selector.get_current())
+								actor = InstagramScraper(selector.get_current())
 							elif job.job_origin is Origin.YT_SHORTS:
 								from warp_beacon.scraper.youtube.shorts import YoutubeShortsScraper
-								actor = YoutubeShortsScraper(self.acc_selector.get_current())
+								actor = YoutubeShortsScraper(selector.get_current())
 							elif job.job_origin is Origin.YT_MUSIC:
 								from warp_beacon.scraper.youtube.music import YoutubeMusicScraper
-								actor = YoutubeMusicScraper(self.acc_selector.get_current())
+								actor = YoutubeMusicScraper(selector.get_current())
 							elif job.job_origin is Origin.YOUTUBE:
 								from warp_beacon.scraper.youtube.youtube import YoutubeScraper
-								actor = YoutubeScraper(self.acc_selector.get_current())
+								actor = YoutubeScraper(selector.get_current())
 							actor.send_message_to_admin_func = self.send_message_to_admin
 							actor.auth_event = self.auth_event
 							while True:
@@ -141,7 +144,7 @@ class AsyncDownloader(object):
 										break
 									job.unvailable_error_count += 1
 									logging.info("Trying to switch account")
-									self.acc_selector.next()
+									selector.next()
 									self.job_queue.put(job)
 									break
 								except TimeOut as e:
@@ -169,13 +172,13 @@ class AsyncDownloader(object):
 								except IGRateLimitOccurred as e:
 									logging.warning("IG ratelimit occurred :(")
 									logging.exception(e)
-									self.try_next_account(job, report_error="rate_limits")
+									self.try_next_account(selector, job, report_error="rate_limits")
 									self.job_queue.put(job)
 									break
 								except CaptchaIssue as e:
 									logging.warning("Challange occurred!")
 									logging.exception(e)
-									self.try_next_account(job)
+									self.try_next_account(selector, job)
 									self.job_queue.put(job)
 									break
 								except YotubeLiveError as e:
@@ -334,6 +337,7 @@ class AsyncDownloader(object):
 				#proc.terminate()
 				logging.info("process #%d stopped", proc.pid)
 		self.workers.clear()
+		self.acc_selector.shutdown()
 
 	def queue_task(self, job: DownloadJob) -> str:
 		self.job_queue.put_nowait(job)

warp_beacon/scraper/account_selector.py

@@ -4,11 +4,13 @@ import re
 
 from itertools import cycle
 
+from multiprocessing.managers import BaseManager
+
 from warp_beacon.jobs import Origin
 
 import logging
 
-class AccountSelector(object):
+class AccountSelector(BaseManager):
 	accounts = []
 	acc_pool = None
 	current = None
@@ -26,6 +28,8 @@ class AccountSelector(object):
 				self.load_yt_sessions()
 		else:
 			raise ValueError("Accounts file not found")
+		
+		super().__init__()
 
 	def __del__(self) -> None:
 		pass

warp_beacon/scraper/youtube/abstract.py

@@ -3,16 +3,12 @@ import pathlib
 import time
 import socket
 import ssl
-
 #from abc import abstractmethod
-from typing import Callable, Union
-
+from typing import Callable, Union, Optional
 import json
-
-import requests
 import urllib
+import urllib.request
 import http.client
-import requests
 from PIL import Image
 
 from warp_beacon.scraper.abstract import ScraperAbstract
@@ -106,11 +102,11 @@ class YoutubeAbstract(ScraperAbstract):
 			if "yt_download_" in i:
 				os.unlink("%s/%s" % (self.DOWNLOAD_DIR, i))
 
-	def download_thumbnail(self, url: str, timeout: int) -> Union[io.BytesIO, None]:
+	def download_thumbnail(self, url: str, timeout: int) -> Optional[io.BytesIO]:
 		try:
-			reply = requests.get(url, timeout=(timeout, timeout))
-			if reply.ok and reply.status_code == 200:
-				image = Image.open(io.BytesIO(reply.content))
+			logging.info("Youtube thumbnail url '%s'", url)
+			with urllib.request.urlopen(url, timeout=timeout) as f:
+				image = Image.open(io.BytesIO(f.read()))
 				image = MediaInfoAbstract.shrink_image_to_fit(image)
 				io_buf = io.BytesIO()
 				image.save(io_buf, format='JPEG')

warp_beacon/telegram/handlers.py

@@ -95,6 +95,7 @@ class Handlers(object):
 					if ent_len > 1:
 						await self.bot.upload_job(
 							UploadJob(
+								url=url,
 								tg_file_id=",".join(tg_file_ids),
 								message_id=effective_message_id,
 								media_type=JobType.COLLECTION,
@@ -108,6 +109,7 @@ class Handlers(object):
 						media_type = JobType[entities[0]["media_type"].upper()]
 						await self.bot.upload_job(
 							UploadJob(
+								url=url,
 								tg_file_id=tg_file_ids.pop(),
 								message_id=effective_message_id,
 								media_type=media_type,

{warp_beacon-2.2.3.dist-info → warp_beacon-2.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 2.2.3
+Version: 2.2.5
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.2.3.dist-info → warp_beacon-2.2.5.dist-info}/RECORD

@@ -3,7 +3,7 @@ lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuww
 var/warp_beacon/accounts.json,sha256=rKFQM_b9eoDS4mJ1B_SZNolPLXx1SQdQMdY2F_ZcBt8,1523
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=nvqkil3SMedV5KXNVnynIXuCKj_S_SPGux-qeZaP5OQ,23
+warp_beacon/__version__.py,sha256=1N7zioGPswYd_sGrglKng-K01jHFk6oVItbCYK4wBSM,23
 warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
@@ -19,27 +19,27 @@ warp_beacon/mediainfo/silencer.py,sha256=MgUc9Ibbhjhg9GbJMNfJqrdDkMsQShZkQ1sCwvW
 warp_beacon/mediainfo/video.py,sha256=AIRy_op_BvehsjarM1rvT5Qo0QWwf-Q6xVVd_aCnbJ4,2505
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scheduler/scheduler.py,sha256=8gjW0CLuX3KyYpSDSkRE0i48AsJbi-fQ2qJZAJ-yt_s,2411
-warp_beacon/scraper/__init__.py,sha256=0ZxLxzXweg1LJ1qBRJ1s3EOKe7xpUrp9Nj26gjsLMOQ,14734
+warp_beacon/scraper/__init__.py,sha256=mY0DdzEy1ZV5wmirwuV9f6e57LbNVzH3ZVPMnbUPGB4,14908
 warp_beacon/scraper/abstract.py,sha256=aNZ9ypF9B8BjflcIwi-7wEzIqF-XPeF0xvfX9CP_iIw,2708
-warp_beacon/scraper/account_selector.py,sha256=RYBmLDhE-fqaLqQiQ6zVd13q-1pqQ3F3nYJu2Gx3gV4,2881
+warp_beacon/scraper/account_selector.py,sha256=x1TAqHVWtxvhzKCTBERpHoUJ0fOcnswStQjEcDxkrFU,2960
 warp_beacon/scraper/exceptions.py,sha256=lHsPrYy5iYnHsIdUHqRxZmzlqrkNj_TS4TkiTJfWhTs,1326
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/instagram.py,sha256=iT66IM7nsbVDWQzBW-CJUWDBjoBbRZH4BHx6TJw3oqo,13517
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scraper/youtube/abstract.py,sha256=UX6TWEM5x4C_FZpifB8IGW9WRgrxYcctA_HMqR_XEvo,6051
+warp_beacon/scraper/youtube/abstract.py,sha256=oL0ZtgPdyBMe_BGQS31T2kQHTceSM2vlWPhB57XQSrc,6048
 warp_beacon/scraper/youtube/music.py,sha256=8TguqXc1BC9Slp09BzCU8dtQ6BnyKwzZSLLFHpqDHtg,1498
 warp_beacon/scraper/youtube/shorts.py,sha256=pHfvEBau8Zp7Ar3LBuPmjqYq8fmjJUQvzeZXujQMpG4,1203
 warp_beacon/scraper/youtube/youtube.py,sha256=K98n2TSJaDZt-xT7mADZL1UEf2exIYm0Wnenn2GAYfI,2250
 warp_beacon/storage/__init__.py,sha256=8XsJXq9X7GDlTaWREF4W1PDX9PH5utwhjf5c5M8Bb7o,3378
 warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/telegram/bot.py,sha256=khk2NvVz_UvxEA_Cz43D5jFuV4z_mUNLOXLhwXKUxz8,15069
-warp_beacon/telegram/handlers.py,sha256=ccqUY7YbddDNbTxYVxpDPjLVJmcAL14yoaQ7xT9WBLU,6725
+warp_beacon/telegram/handlers.py,sha256=LWDdJBLuB-ImemPtzldgm9UY42T2mr0yJsHHQZDmQkI,6759
 warp_beacon/telegram/placeholder_message.py,sha256=u5kVfTjGmVYkwA5opniRltHXGpsdSxI41WEde8J5os0,6418
 warp_beacon/telegram/utils.py,sha256=ikfwmuDvd0XtOuwP29tZPjAz7kwiGSSu_QR-vpv6pMY,2352
 warp_beacon/uploader/__init__.py,sha256=rsUkzzmtaeQY-kNkcIiPsrayS9idFKY3hBC71mR19XE,4743
-warp_beacon-2.2.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.2.3.dist-info/METADATA,sha256=wSApMs3tRXNIqaqLPBZzWqPoyRtsqN3UjmSSrrIqlNQ,21250
-warp_beacon-2.2.3.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-warp_beacon-2.2.3.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.2.3.dist-info/top_level.txt,sha256=ALb_Ft_eG-OY4_m0TWUifNUOZsrx483L-zw7G7vqXoc,971
-warp_beacon-2.2.3.dist-info/RECORD,,
+warp_beacon-2.2.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.2.5.dist-info/METADATA,sha256=GJue-K8EOklIVmECLOVU4NFd6LxdaX1hj8WA6E_m7Ck,21250
+warp_beacon-2.2.5.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+warp_beacon-2.2.5.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.2.5.dist-info/top_level.txt,sha256=ALb_Ft_eG-OY4_m0TWUifNUOZsrx483L-zw7G7vqXoc,971
+warp_beacon-2.2.5.dist-info/RECORD,,