warp-beacon 2.3.50__py3-none-any.whl → 2.3.52__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.3.50"
+__version__ = "2.3.52"
 

warp_beacon/scraper/__init__.py

@@ -121,6 +121,11 @@ class AsyncDownloader(object):
 							if job.job_origin is Origin.INSTAGRAM:
 								from warp_beacon.scraper.instagram.instagram import InstagramScraper
 								actor = InstagramScraper(selector.get_current(), proxy)
+								selector.inc_ig_request_count()
+								if selector.get_ig_request_count() >= int(os.environ.get("IG_REQUESTS_PER_ACCOUNT", default="20")):
+									logging.info("The account request limit has been reached. Selecting the next account.")
+									selector.reset_ig_request_count()
+									selector.next()
 							elif job.job_origin is Origin.YT_SHORTS:
 								from warp_beacon.scraper.youtube.shorts import YoutubeShortsScraper
 								actor = YoutubeShortsScraper(selector.get_current(), proxy)

warp_beacon/scraper/account_selector.py

@@ -23,6 +23,7 @@ class AccountSelector(object):
 	manager = None
 	account_index = {}
 	current_proxy = None
+	ig_request_count = None
 
 	def __init__(self, manager: multiprocessing.managers.SyncManager, acc_file_path: str, proxy_file_path: str=None) -> None:
 		self.manager = manager
@@ -31,6 +32,7 @@ class AccountSelector(object):
 			with open(acc_file_path, 'r', encoding="utf-8") as f:
 				self.accounts = json.loads(f.read())
 			if self.accounts:
+				self.ig_request_count = self.manager.Value('i', 0)
 				self.__init_meta_data()
 				#self.load_yt_sessions()
 				for acc_type, _ in self.accounts.items():
@@ -170,4 +172,13 @@ class AccountSelector(object):
 		module_name = 'youtube' if next((s for s in ("yt", "youtube", "youtu_be") if s in mod_name.value), None) else 'instagram'
 		if module_name not in self.accounts:
 			return 0
-		return len(self.accounts[module_name])
+		return len(self.accounts[module_name])
+	
+	def inc_ig_request_count(self) -> None:
+		self.ig_request_count.value += 1
+
+	def reset_ig_request_count(self) -> None:
+		self.ig_request_count.value = 0
+
+	def get_ig_request_count(self) -> int:
+		return self.ig_request_count.value

warp_beacon/scraper/youtube/abstract.py

@@ -212,14 +212,15 @@ class YoutubeAbstract(ScraperAbstract):
 				# do noting, not interested
 				pass
 			#except http.client.IncompleteRead as e:
+			except KeyError:
+				raise Unavailable("Library failed")
 			except (socket.timeout,
 					ssl.SSLError,
 					http.client.IncompleteRead,
 					http.client.HTTPException,
 					requests.RequestException,
 					urllib.error.URLError,
-					urllib.error.HTTPError,
-					KeyError) as e:
+					urllib.error.HTTPError) as e:
 				if hasattr(e, "code") and (int(e.code) == 403 or int(e.code) == 400):
 					raise Unavailable(extract_exception_message(e))
 				logging.warning("Youtube read timeout! Retrying in '%d' seconds ...", pause_secs)

warp_beacon/scraper/youtube/shorts.py

@@ -42,4 +42,24 @@ class YoutubeShortsScraper(YoutubeAbstract):
 			"media_type": JobType.VIDEO
 		})
 
+		return res
+
+	def _download_yt_dlp(self, url: str, timeout: int = 60) -> list:
+		res = []
+		thumbnail = None
+		video_id = self.get_video_id(url)
+		if video_id:
+			thumbnail = self.download_hndlr(self.download_thumbnail, video_id)
+		with self.build_yt_dlp(timeout) as ydl:
+			info = ydl.extract_info(url, download=True)
+			local_file = ydl.prepare_filename(info)
+			logging.debug("Temp filename: '%s'", local_file)
+			res.append({
+				"local_media_path": local_file,
+				"performer": info.get("uploader", "Unknown"),
+				"thumb": thumbnail,
+				"canonical_name": info.get("title", ''),
+				"media_type": JobType.VIDEO
+			})
+
 		return res

{warp_beacon-2.3.50.dist-info → warp_beacon-2.3.52.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.3.50
+Version: 2.3.52
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.3.50.dist-info → warp_beacon-2.3.52.dist-info}/RECORD

@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=y8JRICAnmgUTv7b4LAieXIpjmtCqdTVaif1BXy2qHMU,24
+warp_beacon/__version__.py,sha256=N9lDdnosxoFEBTxW9ntgJbodH_TnSWwfzA2LqaATl-Y,24
 warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
@@ -21,18 +21,18 @@ warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tE
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scheduler/instagram_human.py,sha256=GRzryMWws4ibJZ_gPGn_Y2I2plcPIVgMWK3Yubqywa0,5020
 warp_beacon/scheduler/scheduler.py,sha256=FPSrR1Z6lWbF1aCAMU2rRYCy1w4zdZ8Q4e7YWQE-Ahk,4041
-warp_beacon/scraper/__init__.py,sha256=bNZI9YBKQB4UyxEjethenXjvp9g1RrWfYgOyepH4O_U,17356
+warp_beacon/scraper/__init__.py,sha256=iqWDK2vzKXkd9RPGYXxQ4O-IrydOBBFR89aX50sP2Yg,17669
 warp_beacon/scraper/abstract.py,sha256=28a0aBKZpi8IKptLWdB6RuVbOkrUbrhT7LSZX7QRQtg,2725
-warp_beacon/scraper/account_selector.py,sha256=HSfmi5H73aWYTjDa8GErCCQpTwWtDLBw92Q-I8TZilg,6282
+warp_beacon/scraper/account_selector.py,sha256=TzAh4fQigsRcS2AomAMvlms8sH5RKKbPT20LJemWGtU,6596
 warp_beacon/scraper/exceptions.py,sha256=L4UCZqDOLKu5-TPUcTErIbNDLrAQJghkMaPqzTfsm1g,1376
 warp_beacon/scraper/fail_handler.py,sha256=_blvckfTZ4xWVancQKVRXH5ClKGwfrBxMwvXIFZh1qA,975
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/instagram.py,sha256=27065cZ8YAHvUCmSWerKew7Agyw25YtPU3i9rOQxK_0,13902
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scraper/youtube/abstract.py,sha256=cO40GXICfHussKa-yySvYVwnONG9FYYa9n-ft-rh4y4,11798
+warp_beacon/scraper/youtube/abstract.py,sha256=98ZIM9ITi2VhCkKl8pPJnlHrhrjfNZ6bei3yMzBcJMM,11843
 warp_beacon/scraper/youtube/music.py,sha256=-R6Iys8IpglH5UL7F640UQ3ieYHitvGXA0A2GhSemVA,2927
-warp_beacon/scraper/youtube/shorts.py,sha256=TVQP-Dm4ZV3bHd7u25yOpDHRzoXL3jUAszDb6ZW8p0U,1269
+warp_beacon/scraper/youtube/shorts.py,sha256=sPrOjhcLlV0m5a-j4mSFsF6CkNapeUdGZqoWAYheiCM,1894
 warp_beacon/scraper/youtube/youtube.py,sha256=B1NBz6vv0-zIuf3MDmT2QB7eX2qO8k9orj7-4wU6MBk,2839
 warp_beacon/storage/__init__.py,sha256=0Vajd0oITKJfu2vmNx5uQSt3-L6vwIvUYWJo8HZCjco,3398
 warp_beacon/storage/mongo.py,sha256=qC4ZiO8XXvPnP0rJwz4CJx42pqFsyAjCiW10W5QdT6E,527
@@ -43,9 +43,9 @@ warp_beacon/telegram/handlers.py,sha256=XXIfdV_RCj7tyZMPXchuKmGoDdweOaR08ADDaBPW
 warp_beacon/telegram/placeholder_message.py,sha256=wN9-BRiyrtHG-EvXtZkGJHt2CX71munQ57ITttjt0mw,6400
 warp_beacon/telegram/utils.py,sha256=9uebX53G16mV7ER7WgfdWBLFHHw14S8HBt9URrIskg0,4440
 warp_beacon/uploader/__init__.py,sha256=E9rlZIf7xlQz6MutMOwJ8S5Vm2uheR5nv23Kv8duRQg,5427
-warp_beacon-2.3.50.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.3.50.dist-info/METADATA,sha256=-o9kJ9gadfV5LiLlOFfNsYD1udqkLRW2QWoVnVH5_h4,22626
-warp_beacon-2.3.50.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
-warp_beacon-2.3.50.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.3.50.dist-info/top_level.txt,sha256=qGjHVVfyf6lTmbdSA-fQq0rHS1YVS4HoJT3rag5xgPE,1141
-warp_beacon-2.3.50.dist-info/RECORD,,
+warp_beacon-2.3.52.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.3.52.dist-info/METADATA,sha256=l_5FB3tcZVHujkxeBY-ijGQ6S2vzJzqXmX78cXYcOH0,22626
+warp_beacon-2.3.52.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+warp_beacon-2.3.52.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.3.52.dist-info/top_level.txt,sha256=qGjHVVfyf6lTmbdSA-fQq0rHS1YVS4HoJT3rag5xgPE,1141
+warp_beacon-2.3.52.dist-info/RECORD,,

{warp_beacon-2.3.50.dist-info → warp_beacon-2.3.52.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.0.2)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any