warp-beacon 2.1.6__py3-none-any.whl → 2.1.7__py3-none-any.whl

warp_beacon/__init__.py

@@ -0,0 +1,7 @@
+from .scraper import *
+from .compress import *
+from .mediainfo import *
+from .jobs import *
+from .storage import *
+from .telegram import *
+from .uploader import *

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.1.6"
+__version__ = "2.1.7"
 

warp_beacon/jobs/abstract.py

@@ -36,6 +36,7 @@ class JobSettings(TypedDict):
 	geoblock_error_count: int
 	account_switches: int
 	yt_auth: bool
+	session_validation: bool
 
 class AbstractJob(ABC):
 	job_id: uuid.UUID = None
@@ -66,6 +67,7 @@ class AbstractJob(ABC):
 	geoblock_error_count: int = 0
 	account_switches: int = 0
 	yt_auth: bool = False
+	session_validation: bool = False
 
 	def __init__(self, **kwargs: Unpack[JobSettings]) -> None:
 		if kwargs:

warp_beacon/scheduler/scheduler.py

@@ -0,0 +1,62 @@
+import threading
+
+import warp_beacon
+
+import logging
+
+class IGScheduler(object):
+	downloader = None
+	running = True
+	thread = None
+	event = None
+
+	def __init__(self, downloader: "warp_beacon.scraper.AsyncDownloader") -> None:
+		self.downloader = downloader
+		self.event = threading.Event()
+
+	def __del__(self) -> None:
+		self.stop()
+
+	def start(self) -> None:
+		self.thread = threading.Thread(target=self.do_work)
+		self.thread.start()
+
+	def stop(self) -> None:
+		self.running = False
+		if self.thread:
+			t_id = self.thread.native_id
+			logging.info("Stopping scheduler thread #'%s'", t_id)
+			self.thread.join()
+			logging.info("Scheduler thread #'%s' stopped", t_id)
+			self.thread = None
+
+	def post_image(self, image) -> None:
+		pass
+
+	def post_story(self) -> None:
+		pass
+
+	def download_random_image(self) -> None:
+		pass
+
+	def validate_ig_session(self) -> bool:
+		try:
+			self.downloader.queue_task(warp_beacon.jobs.download_job.DownloadJob.build(
+				session_validation=True
+			))
+		except Exception as e:
+			logging.warning("An error occurred while validating instagram session!")
+			logging.exception(e)
+
+		return False
+
+	def do_work(self) -> None:
+		logging.info("Scheduler thread started ...")
+		while self.running:
+			try:
+				logging.info("Scheduler waking up")
+				self.validate_ig_session()
+				self.event.wait(timeout=3600)
+			except Exception as e:
+				logging.error("An error occurred in scheduler thread!")
+				logging.exception(e)

warp_beacon/scraper/__init__.py

@@ -4,7 +4,7 @@ from typing import Optional
 import multiprocessing
 from queue import Empty
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YotubeLiveError, YotubeAgeRestrictedError, IGRateLimitAccured, CaptchaIssue, AllAccountsFailed
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YotubeLiveError, YotubeAgeRestrictedError, IGRateLimitOccurred, CaptchaIssue, AllAccountsFailed
 from warp_beacon.mediainfo.video import VideoInfo
 from warp_beacon.mediainfo.audio import AudioInfo
 from warp_beacon.mediainfo.silencer import Silencer
@@ -15,10 +15,11 @@ from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.jobs.upload_job import UploadJob
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.account_selector import AccountSelector
+from warp_beacon.scheduler.scheduler import IGScheduler
 
 import logging
 
-ACC_FILE = os.environ.get("SERVICE_ACCOUNTS_FILE", default="/var/warp_beacon/instagram_accounts.json")
+ACC_FILE = os.environ.get("SERVICE_ACCOUNTS_FILE", default="/var/warp_beacon/accounts.json")
 
 class AsyncDownloader(object):
 	__JOE_BIDEN_WAKEUP = None
@@ -29,12 +30,14 @@ class AsyncDownloader(object):
 	workers_count = 0
 	auth_event = multiprocessing.Event()
 	acc_selector = None
+	scheduler = None
 
 	def __init__(self, uploader: AsyncUploader, workers_count: int) -> None:
 		self.allow_loop = multiprocessing.Value('i', 1)
 		self.uploader = uploader
 		self.workers_count = workers_count
 		self.acc_selector = AccountSelector(ACC_FILE)
+		self.scheduler = IGScheduler(self)
 
 	def __del__(self) -> None:
 		self.stop_all()
@@ -104,13 +107,17 @@ class AsyncDownloader(object):
 								elif job.job_origin is Origin.YOUTUBE:
 									from warp_beacon.scraper.youtube.youtube import YoutubeScraper
 									actor = YoutubeScraper(self.acc_selector.get_current())
-								#self.auth_event = multiprocessing.Event()
 								actor.send_message_to_admin_func = self.send_message_to_admin
 								actor.auth_event = self.auth_event
 								while True:
 									try:
-										logging.info("Downloading URL '%s'", job.url)
-										items = actor.download(job.url)
+										if job.session_validation:
+											logging.info("Validating '%s' session ...", job.origin.value)
+											actor.validate_session()
+											logging.info("done")
+										else:
+											logging.info("Downloading URL '%s'", job.url)
+											items = actor.download(job.url)
 										break
 									except NotFound as e:
 										logging.warning("Not found error occurred!")
@@ -159,14 +166,14 @@ class AsyncDownloader(object):
 											f"Task {job.job_id} failed. URL: '{job.url}'. Reason: 'FileTooBig'."
 										)
 										break
-									except IGRateLimitAccured as e:
-										logging.warning("IG ratelimit accured :(")
+									except IGRateLimitOccurred as e:
+										logging.warning("IG ratelimit occurred :(")
 										logging.exception(e)
 										self.try_next_account(job, report_error="rate_limits")
 										self.job_queue.put(job)
 										break
 									except CaptchaIssue as e:
-										logging.warning("Challange accured!")
+										logging.warning("Challange occurred!")
 										logging.exception(e)
 										self.try_next_account(job)
 										self.job_queue.put(job)
@@ -215,6 +222,9 @@ class AsyncDownloader(object):
 													job_failed=True,
 													job_failed_msg="This content does not accessible for all yout bot accounts. Seems like author blocked certain regions.")
 												)
+												self.send_message_to_admin(
+													f"Task {job.job_id} failed. URL: '{job.url}'. Reason: 'geoblock_required'."
+												)
 												break
 											job.geoblock_error_count += 1
 											logging.info("Trying to switch account")
@@ -225,6 +235,10 @@ class AsyncDownloader(object):
 											job_failed=True,
 											job_failed_msg="WOW, unknown error occured! Please [create issue](https://github.com/sb0y/warp_beacon/issues) with service logs.")
 										)
+										self.send_message_to_admin(
+											f"Task {job.job_id} failed. URL: '{job.url}'. Reason: 'UnknownError'."
+											f"Exception:\n```\n{exception_msg}\n```"
+										)
 										break
 
 								if items:

warp_beacon/scraper/exceptions.py

@@ -34,7 +34,7 @@ class YotubeLiveError(ScraperError):
 class YotubeAgeRestrictedError(ScraperError):
 	pass
 
-class IGRateLimitAccured(ScraperError):
+class IGRateLimitOccurred(ScraperError):
 	pass
 
 class CaptchaIssue(ScraperError):

warp_beacon/scraper/instagram/instagram.py

@@ -19,7 +19,7 @@ from instagrapi import Client
 from instagrapi.mixins.challenge import ChallengeChoice
 from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotFound, ClientNotFoundError, UserNotFound, ChallengeRequired, ChallengeSelfieCaptcha, UnknownError as IGUnknownError
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitAccured, CaptchaIssue, extract_exception_message
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitOccurred, CaptchaIssue, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 from warp_beacon.jobs.types import JobType
 from warp_beacon.telegram.utils import Utils
@@ -85,9 +85,13 @@ class InstagramScraper(ScraperAbstract):
 			self.cl.login(username=username, password=password, verification_code="")
 		self.safe_write_session()
 
-	def scrap(self, url: str) -> tuple[str]:
+	def validate_session(self) -> None:
 		self.load_session()
 		self._download_hndlr(self.cl.get_timeline_feed)
+		self._download_hndlr(self.cl.get_reels_tray_feed)
+
+	def scrap(self, url: str) -> tuple[str]:
+		self.load_session()
 		def _scrap() -> tuple[str]:
 			if "stories" in url:
 				# remove URL options
@@ -258,8 +262,8 @@ class InstagramScraper(ScraperAbstract):
 				logging.warning("Please wait a few minutes error. Trying to relogin ...")
 				logging.exception(e)
 				if please_wait_few_minutes_count >= ratelimit_threshold:
-					logging.warning("IG ratelimit accured")
-					raise IGRateLimitAccured()
+					logging.warning("IG ratelimit occurred")
+					raise IGRateLimitOccurred()
 				wait_timeout += timeout_increment
 				logging.info("Waiting %d seconds according configuration option `IG_WAIT_TIMEOUT` with `IG_TIMEOUT_INCREMENT`", wait_timeout)
 				if res:

warp_beacon/telegram/bot.py

@@ -2,7 +2,6 @@ import os, io
 import signal
 
 import uvloop
-import asyncio
 
 from pyrogram import Client, filters
 from pyrogram.enums import ParseMode
@@ -10,15 +9,13 @@ from pyrogram.handlers import MessageHandler, CallbackQueryHandler
 from pyrogram.types import Message, InputMedia, InputMediaAudio, InputMediaPhoto, InputMediaVideo, InputMediaAnimation, InputMediaDocument, InlineKeyboardButton, InlineKeyboardMarkup
 from pyrogram.errors import RPCError, FloodWait, NetworkMigrate, BadRequest, MultiMediaTooLong, MessageIdInvalid
 
+import warp_beacon
 from warp_beacon.__version__ import __version__
 from warp_beacon.telegram.handlers import Handlers
-import warp_beacon.scraper
 from warp_beacon.telegram.placeholder_message import PlaceholderMessage
 from warp_beacon.storage import Storage
 from warp_beacon.uploader import AsyncUploader
-from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.jobs.upload_job import UploadJob
-from warp_beacon.jobs import Origin
 from warp_beacon.jobs.types import JobType
 from warp_beacon.telegram.utils import Utils
 

{warp_beacon-2.1.6.dist-info → warp_beacon-2.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 2.1.6
+Version: 2.1.7
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.1.6.dist-info → warp_beacon-2.1.7.dist-info}/RECORD

@@ -2,13 +2,13 @@ etc/warp_beacon/warp_beacon.conf,sha256=LNq6YThFSadZKGrsAfMVFWX7liBwePAXeApdEz9Z
 lib/systemd/system/warp_beacon.service,sha256=lPmHqLqcI2eIV7nwHS0qcALQrznixqJuwwPfa2mDLUA,372
 var/warp_beacon/accounts.json,sha256=rKFQM_b9eoDS4mJ1B_SZNolPLXx1SQdQMdY2F_ZcBt8,1523
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
-warp_beacon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/__version__.py,sha256=-iLR2m1cWSKt_pazB1U-9oRhCt6a5WXh_q9QDD7gNoI,23
+warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
+warp_beacon/__version__.py,sha256=jeLIOt0QPdZSDoTjqmhbY0jCL-tG5StFqtGqMqHBw1g,23
 warp_beacon/warp_beacon.py,sha256=7KEtZDj-pdhtl6m-zFLsSojs1ZR4o7L0xbqtdmYPvfE,342
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
 warp_beacon/jobs/__init__.py,sha256=ED8_tPle4iL4kqNW0apAVkgNQtRRTnYfAJwBjO1g0JY,180
-warp_beacon/jobs/abstract.py,sha256=_fgaeNGmtGnzWbXF0hDqsH4okAlDHRdIE6RYuGM4sIA,2719
+warp_beacon/jobs/abstract.py,sha256=lXU9c5LOb1jXmew1IBoFO3k-MChHm0Y8SUX8OsESJGk,2779
 warp_beacon/jobs/download_job.py,sha256=5HiPcnJppFMhO14___3eSkoMygM3y-vhpGkMAuNhK7s,854
 warp_beacon/jobs/types.py,sha256=Ae8zINgbs7cOcYkYoOCOACA7duyhnIGMQAJ_SJB1QRQ,176
 warp_beacon/jobs/upload_job.py,sha256=_ul4psPej1jLEs-BMcMR80GbXDSmm38jE9yoZtecclY,741
@@ -17,12 +17,14 @@ warp_beacon/mediainfo/abstract.py,sha256=ZR2JMuRpoh7nDNov9a8YkAfr6BI2HXnXzQtVrLg
 warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-ZM,651
 warp_beacon/mediainfo/silencer.py,sha256=MgUc9Ibbhjhg9GbJMNfJqrdDkMsQShZkQ1sCwvW_-qI,1647
 warp_beacon/mediainfo/video.py,sha256=AIRy_op_BvehsjarM1rvT5Qo0QWwf-Q6xVVd_aCnbJ4,2505
-warp_beacon/scraper/__init__.py,sha256=tsIk4YlU8l3c4e2qI3KkjMuDpNTXBAUh0Jfp3t6GDRw,14393
+warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+warp_beacon/scheduler/scheduler.py,sha256=SzItuy67UjNlq_77D6n1CoQ7x-o3ItFs5lu9e20yZD8,1450
+warp_beacon/scraper/__init__.py,sha256=fARCkib2rwQGYnhC5Yig_qJWzzBsbVW3pMpgZs8TjZ4,14970
 warp_beacon/scraper/abstract.py,sha256=aNZ9ypF9B8BjflcIwi-7wEzIqF-XPeF0xvfX9CP_iIw,2708
 warp_beacon/scraper/account_selector.py,sha256=9P4Pg4BEJpklWZPZqdQkAloSdJ3geBjQN4hPB7uUNSA,2820
-warp_beacon/scraper/exceptions.py,sha256=TWIMpJeDtt5lxov7XJbMNPPW6NnClFG6TgvLvdaAubE,1325
+warp_beacon/scraper/exceptions.py,sha256=lHsPrYy5iYnHsIdUHqRxZmzlqrkNj_TS4TkiTJfWhTs,1326
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scraper/instagram/instagram.py,sha256=RRNuW8cnfqahNeQw1UM2MGghQBSCJSKL-3OwrS5qx00,13402
+warp_beacon/scraper/instagram/instagram.py,sha256=iT66IM7nsbVDWQzBW-CJUWDBjoBbRZH4BHx6TJw3oqo,13517
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/youtube/abstract.py,sha256=UX6TWEM5x4C_FZpifB8IGW9WRgrxYcctA_HMqR_XEvo,6051
 warp_beacon/scraper/youtube/music.py,sha256=8TguqXc1BC9Slp09BzCU8dtQ6BnyKwzZSLLFHpqDHtg,1498
@@ -30,14 +32,14 @@ warp_beacon/scraper/youtube/shorts.py,sha256=pHfvEBau8Zp7Ar3LBuPmjqYq8fmjJUQvzeZ
 warp_beacon/scraper/youtube/youtube.py,sha256=K98n2TSJaDZt-xT7mADZL1UEf2exIYm0Wnenn2GAYfI,2250
 warp_beacon/storage/__init__.py,sha256=8XsJXq9X7GDlTaWREF4W1PDX9PH5utwhjf5c5M8Bb7o,3378
 warp_beacon/telegram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/telegram/bot.py,sha256=h5yK79a9mxahCGxR0gaRcSdHXXqLhVANDdXZxGUIS2U,13323
+warp_beacon/telegram/bot.py,sha256=DmdR6WMZ1OZ-HuXasgs7dlyRivUc6R1M5n3nbcW8OMw,13210
 warp_beacon/telegram/handlers.py,sha256=MTcHZmWe8RAcZdicnqQewy_SkwujhnaoqJgWHpebfVs,6350
 warp_beacon/telegram/placeholder_message.py,sha256=u5kVfTjGmVYkwA5opniRltHXGpsdSxI41WEde8J5os0,6418
 warp_beacon/telegram/utils.py,sha256=LdCU4ChJHyzpCvyG5v3XcUtUgK3v5by_v8D56VsPeI0,2171
 warp_beacon/uploader/__init__.py,sha256=BQ5rli0soLf0FGqCpivVh9w6lyoGM10-ck2mUjnoOLU,4777
-warp_beacon-2.1.6.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.1.6.dist-info/METADATA,sha256=zDsmACO06KuehBhjotG5oztNO4PDWrg9XGoDWRCql2U,21250
-warp_beacon-2.1.6.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-warp_beacon-2.1.6.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.1.6.dist-info/top_level.txt,sha256=SIMha34qjGXaijMgtzESgQ-Cu-KbLHbmQacwue_ggTY,917
-warp_beacon-2.1.6.dist-info/RECORD,,
+warp_beacon-2.1.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.1.7.dist-info/METADATA,sha256=PYLHaQjTYoquEo-tAKMSaVt_8Zwe3b26H8aD_kLizJ0,21250
+warp_beacon-2.1.7.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+warp_beacon-2.1.7.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.1.7.dist-info/top_level.txt,sha256=ALb_Ft_eG-OY4_m0TWUifNUOZsrx483L-zw7G7vqXoc,971
+warp_beacon-2.1.7.dist-info/RECORD,,

{warp_beacon-2.1.6.dist-info → warp_beacon-2.1.7.dist-info}/top_level.txt

@@ -11,6 +11,8 @@ warp_beacon/mediainfo/abstract
 warp_beacon/mediainfo/audio
 warp_beacon/mediainfo/silencer
 warp_beacon/mediainfo/video
+warp_beacon/scheduler
+warp_beacon/scheduler/scheduler
 warp_beacon/scraper
 warp_beacon/scraper/abstract
 warp_beacon/scraper/account_selector