warp-beacon 2.1.6__tar.gz → 2.1.8__tar.gz

{warp_beacon-2.1.6/warp_beacon.egg-info → warp_beacon-2.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 2.1.6
+Version: 2.1.8
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.1.6 → warp_beacon-2.1.8}/setup.py

@@ -54,7 +54,8 @@ setup(
 		'warp_beacon/scraper/youtube',
 		'warp_beacon/mediainfo',
 		'warp_beacon/jobs',
-		'warp_beacon/compress'
+		'warp_beacon/compress',
+		'warp_beacon/scheduler'
 	],
 	py_modules=[
 		"warp_beacon/__version__",
@@ -71,6 +72,7 @@ setup(
 		"warp_beacon/mediainfo/audio",
 		"warp_beacon/mediainfo/silencer",
 		"warp_beacon/compress/video",
+		"warp_beacon/scheduler/scheduler",
 		"warp_beacon/scraper/abstract",
 		"warp_beacon/scraper/exceptions",
 		"warp_beacon/scraper/types",

warp_beacon-2.1.8/warp_beacon/__init__.py

@@ -0,0 +1,7 @@
+from .scraper import *
+from .compress import *
+from .mediainfo import *
+from .jobs import *
+from .storage import *
+from .telegram import *
+from .uploader import *

warp_beacon-2.1.8/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.1.8"
+

{warp_beacon-2.1.6 → warp_beacon-2.1.8}/warp_beacon/jobs/abstract.py

@@ -36,6 +36,7 @@ class JobSettings(TypedDict):
 	geoblock_error_count: int
 	account_switches: int
 	yt_auth: bool
+	session_validation: bool
 
 class AbstractJob(ABC):
 	job_id: uuid.UUID = None
@@ -66,6 +67,7 @@ class AbstractJob(ABC):
 	geoblock_error_count: int = 0
 	account_switches: int = 0
 	yt_auth: bool = False
+	session_validation: bool = False
 
 	def __init__(self, **kwargs: Unpack[JobSettings]) -> None:
 		if kwargs:

warp_beacon-2.1.8/warp_beacon/scheduler/scheduler.py

@@ -0,0 +1,62 @@
+import threading
+
+import warp_beacon
+
+import logging
+
+class IGScheduler(object):
+	downloader = None
+	running = True
+	thread = None
+	event = None
+
+	def __init__(self, downloader: "warp_beacon.scraper.AsyncDownloader") -> None:
+		self.downloader = downloader
+		self.event = threading.Event()
+
+	def __del__(self) -> None:
+		self.stop()
+
+	def start(self) -> None:
+		self.thread = threading.Thread(target=self.do_work)
+		self.thread.start()
+
+	def stop(self) -> None:
+		self.running = False
+		if self.thread:
+			t_id = self.thread.native_id
+			logging.info("Stopping scheduler thread #'%s'", t_id)
+			self.thread.join()
+			logging.info("Scheduler thread #'%s' stopped", t_id)
+			self.thread = None
+
+	def post_image(self, image) -> None:
+		pass
+
+	def post_story(self) -> None:
+		pass
+
+	def download_random_image(self) -> None:
+		pass
+
+	def validate_ig_session(self) -> bool:
+		try:
+			self.downloader.queue_task(warp_beacon.jobs.download_job.DownloadJob.build(
+				session_validation=True
+			))
+		except Exception as e:
+			logging.warning("An error occurred while validating instagram session!")
+			logging.exception(e)
+
+		return False
+
+	def do_work(self) -> None:
+		logging.info("Scheduler thread started ...")
+		while self.running:
+			try:
+				logging.info("Scheduler waking up")
+				self.validate_ig_session()
+				self.event.wait(timeout=3600)
+			except Exception as e:
+				logging.error("An error occurred in scheduler thread!")
+				logging.exception(e)

{warp_beacon-2.1.6 → warp_beacon-2.1.8}/warp_beacon/scraper/__init__.py

@@ -4,7 +4,7 @@ from typing import Optional
 import multiprocessing
 from queue import Empty
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YotubeLiveError, YotubeAgeRestrictedError, IGRateLimitAccured, CaptchaIssue, AllAccountsFailed
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YotubeLiveError, YotubeAgeRestrictedError, IGRateLimitOccurred, CaptchaIssue, AllAccountsFailed
 from warp_beacon.mediainfo.video import VideoInfo
 from warp_beacon.mediainfo.audio import AudioInfo
 from warp_beacon.mediainfo.silencer import Silencer
@@ -15,10 +15,11 @@ from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.jobs.upload_job import UploadJob
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.account_selector import AccountSelector
+from warp_beacon.scheduler.scheduler import IGScheduler
 
 import logging
 
-ACC_FILE = os.environ.get("SERVICE_ACCOUNTS_FILE", default="/var/warp_beacon/instagram_accounts.json")
+ACC_FILE = os.environ.get("SERVICE_ACCOUNTS_FILE", default="/var/warp_beacon/accounts.json")
 
 class AsyncDownloader(object):
 	__JOE_BIDEN_WAKEUP = None
@@ -29,12 +30,15 @@ class AsyncDownloader(object):
 	workers_count = 0
 	auth_event = multiprocessing.Event()
 	acc_selector = None
+	scheduler = None
 
 	def __init__(self, uploader: AsyncUploader, workers_count: int) -> None:
 		self.allow_loop = multiprocessing.Value('i', 1)
 		self.uploader = uploader
 		self.workers_count = workers_count
 		self.acc_selector = AccountSelector(ACC_FILE)
+		self.scheduler = IGScheduler(self)
+		self.scheduler.start()
 
 	def __del__(self) -> None:
 		self.stop_all()
@@ -104,13 +108,17 @@ class AsyncDownloader(object):
 								elif job.job_origin is Origin.YOUTUBE:
 									from warp_beacon.scraper.youtube.youtube import YoutubeScraper
 									actor = YoutubeScraper(self.acc_selector.get_current())
-								#self.auth_event = multiprocessing.Event()
 								actor.send_message_to_admin_func = self.send_message_to_admin
 								actor.auth_event = self.auth_event
 								while True:
 									try:
-										logging.info("Downloading URL '%s'", job.url)
-										items = actor.download(job.url)
+										if job.session_validation:
+											logging.info("Validating '%s' session ...", job.origin.value)
+											actor.validate_session()
+											logging.info("done")
+										else:
+											logging.info("Downloading URL '%s'", job.url)
+											items = actor.download(job.url)
 										break
 									except NotFound as e:
 										logging.warning("Not found error occurred!")
@@ -159,14 +167,14 @@ class AsyncDownloader(object):
 											f"Task {job.job_id} failed. URL: '{job.url}'. Reason: 'FileTooBig'."
 										)
 										break
-									except IGRateLimitAccured as e:
-										logging.warning("IG ratelimit accured :(")
+									except IGRateLimitOccurred as e:
+										logging.warning("IG ratelimit occurred :(")
 										logging.exception(e)
 										self.try_next_account(job, report_error="rate_limits")
 										self.job_queue.put(job)
 										break
 									except CaptchaIssue as e:
-										logging.warning("Challange accured!")
+										logging.warning("Challange occurred!")
 										logging.exception(e)
 										self.try_next_account(job)
 										self.job_queue.put(job)
@@ -215,6 +223,9 @@ class AsyncDownloader(object):
 													job_failed=True,
 													job_failed_msg="This content does not accessible for all yout bot accounts. Seems like author blocked certain regions.")
 												)
+												self.send_message_to_admin(
+													f"Task {job.job_id} failed. URL: '{job.url}'. Reason: 'geoblock_required'."
+												)
 												break
 											job.geoblock_error_count += 1
 											logging.info("Trying to switch account")
@@ -225,6 +236,10 @@ class AsyncDownloader(object):
 											job_failed=True,
 											job_failed_msg="WOW, unknown error occured! Please [create issue](https://github.com/sb0y/warp_beacon/issues) with service logs.")
 										)
+										self.send_message_to_admin(
+											f"Task {job.job_id} failed. URL: '{job.url}'. Reason: 'UnknownError'."
+											f"Exception:\n```\n{exception_msg}\n```"
+										)
 										break
 
 								if items:
@@ -312,6 +327,7 @@ class AsyncDownloader(object):
 
 	def stop_all(self) -> None:
 		self.allow_loop.value = 0
+		self.scheduler.stop()
 		for proc in self.workers:
 			if proc.is_alive():
 				logging.info("stopping process #%d", proc.pid)

{warp_beacon-2.1.6 → warp_beacon-2.1.8}/warp_beacon/scraper/exceptions.py

@@ -34,7 +34,7 @@ class YotubeLiveError(ScraperError):
 class YotubeAgeRestrictedError(ScraperError):
 	pass
 
-class IGRateLimitAccured(ScraperError):
+class IGRateLimitOccurred(ScraperError):
 	pass
 
 class CaptchaIssue(ScraperError):

{warp_beacon-2.1.6 → warp_beacon-2.1.8}/warp_beacon/scraper/instagram/instagram.py

@@ -19,7 +19,7 @@ from instagrapi import Client
 from instagrapi.mixins.challenge import ChallengeChoice
 from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotFound, ClientNotFoundError, UserNotFound, ChallengeRequired, ChallengeSelfieCaptcha, UnknownError as IGUnknownError
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitAccured, CaptchaIssue, extract_exception_message
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitOccurred, CaptchaIssue, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 from warp_beacon.jobs.types import JobType
 from warp_beacon.telegram.utils import Utils
@@ -85,9 +85,13 @@ class InstagramScraper(ScraperAbstract):
 			self.cl.login(username=username, password=password, verification_code="")
 		self.safe_write_session()
 
-	def scrap(self, url: str) -> tuple[str]:
+	def validate_session(self) -> None:
 		self.load_session()
 		self._download_hndlr(self.cl.get_timeline_feed)
+		self._download_hndlr(self.cl.get_reels_tray_feed)
+
+	def scrap(self, url: str) -> tuple[str]:
+		self.load_session()
 		def _scrap() -> tuple[str]:
 			if "stories" in url:
 				# remove URL options
@@ -258,8 +262,8 @@ class InstagramScraper(ScraperAbstract):
 				logging.warning("Please wait a few minutes error. Trying to relogin ...")
 				logging.exception(e)
 				if please_wait_few_minutes_count >= ratelimit_threshold:
-					logging.warning("IG ratelimit accured")
-					raise IGRateLimitAccured()
+					logging.warning("IG ratelimit occurred")
+					raise IGRateLimitOccurred()
 				wait_timeout += timeout_increment
 				logging.info("Waiting %d seconds according configuration option `IG_WAIT_TIMEOUT` with `IG_TIMEOUT_INCREMENT`", wait_timeout)
 				if res:

{warp_beacon-2.1.6 → warp_beacon-2.1.8}/warp_beacon/telegram/bot.py

@@ -2,7 +2,6 @@ import os, io
 import signal
 
 import uvloop
-import asyncio
 
 from pyrogram import Client, filters
 from pyrogram.enums import ParseMode
@@ -10,15 +9,13 @@ from pyrogram.handlers import MessageHandler, CallbackQueryHandler
 from pyrogram.types import Message, InputMedia, InputMediaAudio, InputMediaPhoto, InputMediaVideo, InputMediaAnimation, InputMediaDocument, InlineKeyboardButton, InlineKeyboardMarkup
 from pyrogram.errors import RPCError, FloodWait, NetworkMigrate, BadRequest, MultiMediaTooLong, MessageIdInvalid
 
+import warp_beacon
 from warp_beacon.__version__ import __version__
 from warp_beacon.telegram.handlers import Handlers
-import warp_beacon.scraper
 from warp_beacon.telegram.placeholder_message import PlaceholderMessage
 from warp_beacon.storage import Storage
 from warp_beacon.uploader import AsyncUploader
-from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.jobs.upload_job import UploadJob
-from warp_beacon.jobs import Origin
 from warp_beacon.jobs.types import JobType
 from warp_beacon.telegram.utils import Utils
 

{warp_beacon-2.1.6 → warp_beacon-2.1.8/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warp_beacon
-Version: 2.1.6
+Version: 2.1.8
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.1.6 → warp_beacon-2.1.8}/warp_beacon.egg-info/SOURCES.txt

@@ -30,6 +30,8 @@ warp_beacon/mediainfo/abstract.py
 warp_beacon/mediainfo/audio.py
 warp_beacon/mediainfo/silencer.py
 warp_beacon/mediainfo/video.py
+warp_beacon/scheduler/__init__.py
+warp_beacon/scheduler/scheduler.py
 warp_beacon/scraper/__init__.py
 warp_beacon/scraper/abstract.py
 warp_beacon/scraper/account_selector.py

{warp_beacon-2.1.6 → warp_beacon-2.1.8}/warp_beacon.egg-info/top_level.txt

@@ -11,6 +11,8 @@ warp_beacon/mediainfo/abstract
 warp_beacon/mediainfo/audio
 warp_beacon/mediainfo/silencer
 warp_beacon/mediainfo/video
+warp_beacon/scheduler
+warp_beacon/scheduler/scheduler
 warp_beacon/scraper
 warp_beacon/scraper/abstract
 warp_beacon/scraper/account_selector

warp_beacon-2.1.6/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.1.6"
-