warp-beacon 2.2.67__tar.gz → 2.3.0__tar.gz

{warp_beacon-2.2.67/warp_beacon.egg-info → warp_beacon-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: warp_beacon
-Version: 2.2.67
+Version: 2.3.0
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -236,6 +236,8 @@ Requires-Dist: urlextract
 Requires-Dist: pillow
 Requires-Dist: pymongo
 Requires-Dist: instagrapi==2.0.0
+Dynamic: author
+Dynamic: home-page
 
 # warp_beacon [![Upload Python Package](https://github.com/sb0y/warp_beacon/actions/workflows/python-publish.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/python-publish.yml) [![Docker Image CI](https://github.com/sb0y/warp_beacon/actions/workflows/docker-image.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/docker-image.yml) [![Build DEB package](https://github.com/sb0y/warp_beacon/actions/workflows/build-deb.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/build-deb.yml)
 

warp_beacon-2.3.0/etc/accounts.json

@@ -0,0 +1,44 @@
+{
+	"instagram":
+	[
+		{
+			"login": "ig_login",
+			"password": "ig_pass",
+			"imap_server": "gmail.com",
+			"imap_login": "your@mail.login",
+			"imap_password": "your_mail_pass",
+			"auth_details":
+			{
+				"delay_range": [1, 3],
+				"country_code": 7,
+				"locale": "en_US",
+				"timezone_offset": 10800,
+				"user_agent": "Barcelona 291.0.0.31.111 Android (33/13; 600dpi; 1440x3044; samsung; SM-G998B; p3s; exynos2100; en_US; 493450264)",
+				"device":
+				{
+					"app_version": "291.0.0.31.111",
+					"android_version": 33,
+					"android_release": "13.0.0",
+					"dpi": "600dpi",
+					"resolution": "1440x3044",
+					"manufacturer": "Samsung",
+					"device": "p3s",
+					"model": "SM-G998B",
+					"cpu": "exynos2100",
+					"version_code": "493450264"
+				}
+			},
+			"account_admins": "@BelisariusCawl",
+			"proxy_id": "ipv6_proxy",
+			"force_ipv6": true
+		}
+	],
+	"youtube":
+	[
+		{
+			"account_admins": "@BelisariusCawl",
+			"force_ipv6": true,
+			"proxy_id": "ipv6_proxy"
+		}
+	]
+}

warp_beacon-2.3.0/etc/proxies.json

@@ -0,0 +1,6 @@
+[
+	{
+		"id": "ipv6_proxy",
+		"dsn": "http://proxy_login:proxy_pass@[fd35:8b7a:1::1]:3129"
+	}
+]

warp_beacon-2.3.0/etc/warp_beacon.conf

@@ -0,0 +1,14 @@
+TG_TOKEN="your_tg_token"
+TG_BOT_NAME="your_bot_name"
+TG_BOT_ADMIN_USERNAME="@BelisariusCawl"
+TG_API_ID="your_api_id"
+TG_API_HASH="your_api_has"
+TG_BOT_ADMIN_USERNAME="@BelisariusCawl"
+MONGODB_HOST="mongodb"
+MONGODB_PORT="27017"
+MONGODB_USER="root"
+MONGODB_PASSWORD="changeme"
+ENABLE_DONATES=true
+SERVICE_ACCOUNTS_FILE=/var/warp_beacon/accounts.json
+PROXY_FILE=/var/warp_beacon/proxies.json
+FORCE_IPV6=true

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/setup.py

@@ -83,6 +83,7 @@ setup(
 		"warp_beacon/scraper/youtube/shorts",
 		"warp_beacon/scraper/youtube/music",
 		"warp_beacon/scraper/fail_handler",
+		"warp_beacon/scraper/link_resolver",
 		"warp_beacon/storage/mongo"
 	],
 	#scripts=['scripts/wait_dc_update.py'],
@@ -98,6 +99,9 @@ setup(
 		),
 		("/var/warp_beacon/",
 			["etc/accounts.json"]
+		),
+		("/var/warp_beacon/",
+			["etc/proxies.json"]
 		)
 	],
 

warp_beacon-2.3.0/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.3.0"
+

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/jobs/abstract.py

@@ -42,6 +42,8 @@ class JobSettings(TypedDict):
 	chat_type: ChatType
 	account_admins: str
 	job_postponed_until: int
+	message_leftover: str
+	replay: bool
 
 class AbstractJob(ABC):
 	job_id: uuid.UUID = None
@@ -76,6 +78,8 @@ class AbstractJob(ABC):
 	chat_type: ChatType = None
 	account_admins: str = None
 	job_postponed_until: int = -1
+	message_leftover: str = ""
+	replay: bool = False
 
 	def __init__(self, **kwargs: Unpack[JobSettings]) -> None:
 		if kwargs:

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/scraper/__init__.py

@@ -5,7 +5,9 @@ from typing import Optional
 import multiprocessing
 from queue import Empty
 
-from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YotubeLiveError, YotubeAgeRestrictedError, IGRateLimitOccurred, CaptchaIssue, AllAccountsFailed
+import logging
+
+from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, Unavailable, FileTooBig, YoutubeLiveError, YotubeAgeRestrictedError, IGRateLimitOccurred, CaptchaIssue, AllAccountsFailed
 from warp_beacon.mediainfo.video import VideoInfo
 from warp_beacon.mediainfo.audio import AudioInfo
 from warp_beacon.mediainfo.silencer import Silencer
@@ -18,10 +20,10 @@ from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.account_selector import AccountSelector
 from warp_beacon.storage.mongo import DBClient
 from warp_beacon.scraper.fail_handler import FailHandler
-
-import logging
+from warp_beacon.scraper.link_resolver import LinkResolver
 
 ACC_FILE = os.environ.get("SERVICE_ACCOUNTS_FILE", default="/var/warp_beacon/accounts.json")
+PROXY_FILE = os.environ.get("PROXY_FILE", default="/var/warp_beacon/proxies.json")
 
 class AsyncDownloader(object):
 	__JOE_BIDEN_WAKEUP = None
@@ -38,7 +40,7 @@ class AsyncDownloader(object):
 	def __init__(self, uploader: AsyncUploader, workers_count: int) -> None:
 		self.manager = multiprocessing.Manager()
 		self.allow_loop = self.manager.Value('i', 1)
-		self.acc_selector = AccountSelector(self.manager, ACC_FILE)
+		self.acc_selector = AccountSelector(self.manager, ACC_FILE, PROXY_FILE)
 		self.uploader = uploader
 		self.workers_count = workers_count
 
@@ -93,13 +95,18 @@ class AsyncDownloader(object):
 				try:
 					job = self.job_queue.get()
 					if job is self.__JOE_BIDEN_WAKEUP:
-						continue
+						break
 					actor = None
 					try:
 						items = []
 						if job.job_origin is Origin.UNKNOWN:
 							logging.warning("Unknown task origin! Skipping.")
 							continue
+						if LinkResolver.resolve_job(job):
+							self.uploader.queue_task(job.to_upload_job(
+								replay=True
+							))
+							continue
 						if not job.in_process:
 							if job.job_postponed_until > 0:
 								if (job.job_postponed_until - time.time()) > 0:
@@ -111,16 +118,16 @@ class AsyncDownloader(object):
 							self.acc_selector.set_module(job.job_origin)
 							if job.job_origin is Origin.INSTAGRAM:
 								from warp_beacon.scraper.instagram.instagram import InstagramScraper
-								actor = InstagramScraper(selector.get_current())
+								actor = InstagramScraper(selector.get_current(), selector.get_account_proxy())
 							elif job.job_origin is Origin.YT_SHORTS:
 								from warp_beacon.scraper.youtube.shorts import YoutubeShortsScraper
-								actor = YoutubeShortsScraper(selector.get_current())
+								actor = YoutubeShortsScraper(selector.get_current(), selector.get_account_proxy())
 							elif job.job_origin is Origin.YT_MUSIC:
 								from warp_beacon.scraper.youtube.music import YoutubeMusicScraper
-								actor = YoutubeMusicScraper(selector.get_current())
+								actor = YoutubeMusicScraper(selector.get_current(), selector.get_account_proxy())
 							elif job.job_origin is Origin.YOUTUBE:
 								from warp_beacon.scraper.youtube.youtube import YoutubeScraper
-								actor = YoutubeScraper(selector.get_current())
+								actor = YoutubeScraper(selector.get_current(), selector.get_account_proxy())
 							actor.send_message_to_admin_func = self.send_message_to_admin
 							actor.auth_event = self.auth_event
 							while True:
@@ -131,7 +138,7 @@ class AsyncDownloader(object):
 										logging.info("done")
 									else:
 										logging.info("Downloading URL '%s'", job.url)
-										items = actor.download(job.url)
+										items = actor.download(job)
 									break
 								except NotFound as e:
 									logging.warning("Not found error occurred!")
@@ -196,7 +203,7 @@ class AsyncDownloader(object):
 									self.try_next_account(selector, job, report_error="captcha")
 									self.job_queue.put(job)
 									break
-								except YotubeLiveError as e:
+								except YoutubeLiveError as e:
 									logging.warning("Youtube Live videos are not supported. Skipping.")
 									logging.exception(e)
 									self.uploader.queue_task(job.to_upload_job(

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/scraper/abstract.py

@@ -18,15 +18,18 @@ class ScraperAbstract(ABC):
 	auth_event = None
 	account = None
 	account_index = 0
+	proxy = None
 
-	def __init__(self, account: tuple) -> None:
+
+	def __init__(self, account: tuple, proxy: dict=None) -> None:
 		self.account_index = account[0]
 		self.account = account[1]
-		if os.environ.get("FORCE_IPV6", default="false") == "true":
+		self.proxy = proxy
+		if self.account.get("force_ipv6", False):
 			self.force_ipv6()
 
 	def __del__(self) -> None:
-		if os.environ.get("FORCE_IPV6", default="false") == "true":
+		if self.account.get("force_ipv6", False):
 			self.restore_gai()
 
 	@abstractmethod

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/scraper/account_selector.py

@@ -1,9 +1,10 @@
-import multiprocessing.managers
 import os
 import json
 import re
+from typing import Optional
 
 import multiprocessing
+import multiprocessing.managers
 
 from warp_beacon.jobs import Origin
 
@@ -11,6 +12,7 @@ import logging
 
 class AccountSelector(object):
 	accounts = []
+	proxies = []
 	current = None
 	current_module_name = None
 	accounts_meta_data = None
@@ -18,7 +20,7 @@ class AccountSelector(object):
 	manager = None
 	account_index = {}
 
-	def __init__(self, manager: multiprocessing.managers.SyncManager, acc_file_path: str) -> None:
+	def __init__(self, manager: multiprocessing.managers.SyncManager, acc_file_path: str, proxy_file_path: str=None) -> None:
 		self.manager = manager
 		self.accounts_meta_data = self.manager.dict()
 		if os.path.exists(acc_file_path):
@@ -29,14 +31,28 @@ class AccountSelector(object):
 				#self.load_yt_sessions()
 				for acc_type, _ in self.accounts.items():
 					self.account_index[acc_type] = self.manager.Value('i', 0)
+			if proxy_file_path:
+				with open(proxy_file_path, 'r', encoding="utf-8") as f:
+					self.proxies = json.loads(f.read())
 		else:
 			raise ValueError("Accounts file not found")
 
 	def __del__(self) -> None:
 		pass
 
-	#def enrich_service_data(self) -> None:
-	#	for k, v in self.accounts.items():
+	def get_account_proxy(self) -> Optional[dict]:
+		if self.proxies:
+			try:
+				current_acc_pid = self.get_current()[1].get("proxy_id", "").strip()
+				for proxy in self.proxies:
+					pid = proxy.get("id", "").strip()
+					if pid and current_acc_pid and pid == current_acc_pid:
+						logging.info("Account proxy matched '%s'", proxy)
+						return proxy
+			except Exception as e:
+				logging.warning("Error on selecting account proxy!")
+				logging.exception(e)
+		return None
 
 	def load_yt_sessions(self) -> None:
 		if "youtube" not in self.accounts:

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/scraper/exceptions.py

@@ -26,7 +26,7 @@ class NotFound(ScraperError):
 class Unavailable(ScraperError):
 	pass
 
-class YotubeLiveError(ScraperError):
+class YoutubeLiveError(ScraperError):
 	pass
 
 class YotubeAgeRestrictedError(ScraperError):

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/scraper/instagram/instagram.py

@@ -17,11 +17,13 @@ from instagrapi.mixins.story import Story
 #from instagrapi.types import Media
 from instagrapi import Client
 from instagrapi.mixins.challenge import ChallengeChoice
-from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotFound, ClientNotFoundError, UserNotFound, ChallengeRequired, ChallengeSelfieCaptcha, UnknownError as IGUnknownError
+from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotFound, ClientNotFoundError, UserNotFound, ChallengeRequired, \
+	ChallengeSelfieCaptcha, ChallengeUnknownStep, UnknownError as IGUnknownError
 
 from warp_beacon.scraper.exceptions import NotFound, UnknownError, TimeOut, IGRateLimitOccurred, CaptchaIssue, extract_exception_message
 from warp_beacon.scraper.abstract import ScraperAbstract
 from warp_beacon.jobs.types import JobType
+from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.telegram.utils import Utils
 
 import logging
@@ -32,11 +34,16 @@ class InstagramScraper(ScraperAbstract):
 	cl = None
 	inst_session_file = ""
 
-	def __init__(self, account: tuple) -> None:
-		super().__init__(account)
+	def __init__(self, account: tuple, proxy: dict=None) -> None:
+		super().__init__(account, proxy)
 		#
 		self.inst_session_file = INST_SESSION_FILE_TPL % self.account_index
 		self.cl = Client()
+		if self.proxy:
+			proxy_dsn = self.proxy.get("dsn", "")
+			if proxy_dsn:
+				self.cl.set_proxy(proxy_dsn)
+				logging.info("Using proxy DSN '%s'", proxy_dsn)
 		#self.cl.logger.setLevel(logging.DEBUG)
 		self.setup_device()
 		self.cl.challenge_code_handler = self.challenge_code_handler
@@ -139,7 +146,7 @@ class InstagramScraper(ScraperAbstract):
 			try:
 				ret_val = func(*args, **kwargs)
 				break
-			except (ChallengeRequired, ChallengeSelfieCaptcha) as e:
+			except (ChallengeRequired, ChallengeSelfieCaptcha, ChallengeUnknownStep) as e:
 				logging.warning("Instagram wants Challange!")
 				logging.exception(e)
 				raise CaptchaIssue("a captcha issue arose")
@@ -235,11 +242,11 @@ class InstagramScraper(ScraperAbstract):
 
 		return {"media_type": JobType.COLLECTION, "items": chunks}
 
-	def download(self, url: str) -> Optional[list[dict]]:
+	def download(self, job: DownloadJob) -> Optional[list[dict]]:
 		res = []
 		while True:
 			try:
-				scrap_type, media_id = self.scrap(url)
+				scrap_type, media_id = self.scrap(job.url)
 				if scrap_type == "media":
 					media_info = self._download_hndlr(self.cl.media_info, media_id)
 					logging.info("media_type is '%d', product_type is '%s'", media_info.media_type, media_info.product_type)

warp_beacon-2.3.0/warp_beacon/scraper/link_resolver.py

@@ -0,0 +1,88 @@
+import os
+import re
+import logging
+import requests
+from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse
+
+from warp_beacon.jobs import Origin
+from warp_beacon.jobs.download_job import DownloadJob
+
+class LinkResolver(object):
+	"Resolve short links"
+	canonical_link_resolve_re = re.compile(r'<link.*rel="canonical".*href="([^"]+)"\s*/?>')
+
+	@staticmethod
+	def extract_youtu_be_link(url: str) -> str:
+		try:
+			response = requests.get(
+					url=url,
+					allow_redirects=False,
+					timeout=int(os.environ.get("REQUESTS_TIMEOUT", default=60))
+				)
+			logging.info("Converted URL '%s' to '%s'", url, response.headers["Location"])
+			return response.headers["Location"]
+		except Exception as e:
+			logging.error("Failed to extract YouTube link!")
+			logging.exception(e)
+
+		return ''
+
+	@staticmethod
+	def extract_youtu_be_link_local(url: str) -> str:
+		try:
+			parsed = urlparse(url)
+			video_id = parsed.path.split('/')[-1] if parsed.path else ''
+			query_params = parse_qsl(parsed.query)
+			query_params.append(('v', video_id))
+			query_params.append(('feature', 'youtu.be'))
+			
+			new_url = urlunparse((
+				parsed.scheme,
+				'www.youtube.com',
+				'/watch',
+				'',  # path parameters
+				urlencode(query_params),
+				''   # archor
+			))
+			logging.info("Locally converted URL '%s' to '%s'", url, new_url)
+			return new_url
+		except Exception as e:
+			logging.error("Failed to extract YouTube link!")
+			logging.exception(e)
+
+		return ''
+	
+	@staticmethod
+	def resolve_ig_share_link(url: str) -> str:
+		'''
+		expected url: https://www.instagram.com/share/reel/BAHtk2AamB
+		result url: https://www.instagram.com/reel/DAKjQgUNzuH/
+		'''
+		try:
+			content = requests.get(
+				url,
+				timeout=int(os.environ.get("REQUESTS_TIMEOUT", default=60)
+			)).text
+			res = re.search(LinkResolver.canonical_link_resolve_re, content)
+			new_url = res.group(1).strip()
+			logging.info("Converted IG share '%s' link to '%s'", url, new_url)
+			return new_url
+		except Exception as e:
+			logging.error("Failed to convert IG share link!")
+			logging.exception(e)
+
+		return url
+
+	@staticmethod
+	def resolve_job(job: DownloadJob) -> bool:
+		ret = False
+		if job.job_origin is Origin.YOUTU_BE:
+			job.url = LinkResolver.extract_youtu_be_link(job.url)
+			job.job_origin = Origin.YOUTUBE
+			ret = True
+		if job.job_origin is Origin.INSTAGRAM:
+			if "share/" in job.url:
+				job.url = LinkResolver.resolve_ig_share_link(job.url)
+				ret = True
+
+		return ret

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/scraper/youtube/abstract.py

@@ -13,6 +13,7 @@ import http.client
 from PIL import Image
 import numpy as np
 
+from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.scraper.abstract import ScraperAbstract
 #from warp_beacon.mediainfo.abstract import MediaInfoAbstract
 from warp_beacon.scraper.exceptions import TimeOut, Unavailable, extract_exception_message
@@ -80,11 +81,11 @@ class YoutubeAbstract(ScraperAbstract):
 	DOWNLOAD_DIR = "/tmp"
 	YT_SESSION_FILE = '/var/warp_beacon/yt_session_%d.json'
 
-	def __init__(self, account: tuple) -> None:
-		super().__init__(account)
+	#def __init__(self, account: tuple, proxy: dict=None) -> None:
+	#	super().__init__(account, proxy)
 
-	def __del__(self) -> None:
-		pass
+	#def __del__(self) -> None:
+	#	pass
 
 	def rename_local_file(self, filename: str) -> str:
 		if not os.path.exists(filename):
@@ -235,4 +236,15 @@ class YoutubeAbstract(ScraperAbstract):
 		yt_opts["use_oauth"] = True
 		yt_opts["allow_oauth_cache"] = True
 		yt_opts["token_file"] = self.YT_SESSION_FILE % self.account_index
+		if self.proxy:
+			proxy_dsn = self.proxy.get("dsn", "")
+			if proxy_dsn:
+				logging.info("Using proxy DSN '%s'", proxy_dsn)
+				yt_opts["proxies"] = {"http": proxy_dsn, "https": proxy_dsn}
 		return YouTube(**yt_opts)
+	
+	def _download(self, url: str) -> list:
+		raise NotImplementedError("Implement _download method")
+
+	def download(self, job: DownloadJob) -> list:
+		return self._download_hndlr(self._download, job.url)

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/scraper/youtube/music.py

@@ -45,6 +45,3 @@ class YoutubeMusicScraper(YoutubeAbstract):
 		})
 
 		return res
-
-	def download(self, url: str) -> list:
-		return self._download_hndlr(self._download, url)

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/scraper/youtube/shorts.py

@@ -42,7 +42,4 @@ class YoutubeShortsScraper(YoutubeAbstract):
 			"media_type": JobType.VIDEO
 		})
 
-		return res
-
-	def download(self, url: str) -> list:
-		return self._download_hndlr(self._download, url)
+		return res

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/scraper/youtube/youtube.py

@@ -1,6 +1,6 @@
 from warp_beacon.jobs.types import JobType
 from warp_beacon.scraper.youtube.abstract import YoutubeAbstract
-from warp_beacon.scraper.exceptions import YotubeLiveError, NotFound, YotubeAgeRestrictedError
+from warp_beacon.scraper.exceptions import YoutubeLiveError, NotFound, YotubeAgeRestrictedError
 
 from pytubefix.exceptions import AgeRestrictedError
 
@@ -35,7 +35,7 @@ class YoutubeScraper(YoutubeAbstract):
 			yt = self.build_yt(url)
 
 			if self.is_live(yt.initial_data):
-				raise YotubeLiveError("Youtube Live is not supported")
+				raise YoutubeLiveError("Youtube Live is not supported")
 
 			if yt:
 				thumbnail = self._download_hndlr(self.download_thumbnail, yt.video_id)
@@ -65,7 +65,4 @@ class YoutubeScraper(YoutubeAbstract):
 		except AgeRestrictedError as e:
 			raise YotubeAgeRestrictedError("Youtube Age Restricted error")
 
-		return res
-
-	def download(self, url: str) -> list:
-		return self._download_hndlr(self._download, url)
+		return res

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/storage/mongo.py

@@ -7,7 +7,7 @@ class DBClient(object):
 
 	def __init__(self) -> None:
 		self.client = MongoClient(
-			host=os.environ.get("MONGODB_HOST", default='127.0.0.1'), 
+			host=os.environ.get("MONGODB_HOST", default='127.0.0.1'),
 			port=int(os.environ.get("MONGODB_PORT", default=27017)),
 			username=os.environ.get("MONGODB_USER", default='root'),
 			password=os.environ.get("MONGODB_PASSWORD", default="changeme")

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/telegram/bot.py

@@ -163,6 +163,9 @@ class Bot(object):
 		if job.chat_type in (ChatType.GROUP, ChatType.SUPERGROUP):
 			if job.canonical_name:
 				caption += "\n—\n"
+			if job.message_leftover:
+				caption += job.message_leftover
+				caption += "\n—\n"
 			if job.source_username:
 				caption += f"Requested by <b>@{job.source_username}</b>"
 			if job.source_username and job.url:

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/telegram/handlers.py

@@ -11,6 +11,7 @@ from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.jobs.upload_job import UploadJob
 from warp_beacon.jobs import Origin
 from warp_beacon.jobs.types import JobType
+from warp_beacon.scraper.link_resolver import LinkResolver
 
 import logging
 
@@ -58,6 +59,73 @@ class Handlers(object):
 			"Currently supported: Instagram, YouTube Shorts and YouTube Music."
 		)
 
+	async def upload_wrapper(self, job: UploadJob) -> None:
+		try:
+			if job.replay:
+				logging.info("Replaying job with URL: '%s'", job.url)
+				return await self.queue_job(job.to_download_job(replay=False))
+
+			if job.job_failed and job.job_failed_msg:
+				if job.placeholder_message_id:
+					await self.bot.placeholder.remove(job.chat_id, job.placeholder_message_id)
+					return await self.bot.send_text(chat_id=job.chat_id, text=job.job_failed_msg, reply_id=job.message_id)
+								
+			if job.job_warning and job.job_warning_msg:
+				return await self.bot.placeholder.update_text(job.chat_id, job.placeholder_message_id, job.job_warning_msg)
+								
+			tg_file_ids = await self.bot.upload_job(job)
+			if tg_file_ids:
+				if job.media_type == JobType.COLLECTION and job.save_items:
+					for chunk in job.media_collection:
+						for i in chunk:
+							self.storage.add_media(
+								tg_file_ids=[i.tg_file_id],
+								media_url=i.effective_url,
+								media_type=i.media_type.value,
+								origin=job.job_origin.value,
+								canonical_name=job.canonical_name
+							)
+				else:
+					self.storage.add_media(
+						tg_file_ids=[','.join(tg_file_ids)],
+						media_url=job.url,
+						media_type=job.media_type.value,
+						origin=job.job_origin.value,
+						canonical_name=job.canonical_name
+					)
+		except Exception as e:
+			logging.error("Exception occurred while performing upload callback!")
+			logging.exception(e)
+
+	async def queue_job(self, job: DownloadJob) -> bool:
+		try:
+			# create placeholder message for long download
+			if not job.placeholder_message_id:
+				job.placeholder_message_id = await self.bot.placeholder.create(
+					chat_id=job.chat_id,
+					reply_id=job.message_id
+				)
+
+			if not job.placeholder_message_id:
+				return await self.bot.send_text(
+					chat_id=job.chat_id,
+					reply_id=job.message_id,
+					text="Failed to create message placeholder. Please check your bot Internet connection."
+				)
+
+			self.bot.uploader.add_callback(
+				job.placeholder_message_id,
+				self.upload_wrapper
+			)
+
+			self.bot.downloader.queue_task(job)
+		except Exception as e:
+			logging.error("Failed to schedule download task!")
+			logging.exception(e)
+			return False
+		
+		return True
+
 	async def handler(self, client: Client, message: Message) -> None:
 		if message is None:
 			return
@@ -66,7 +134,12 @@ class Handlers(object):
 			return
 		chat = message.chat
 		effective_message_id = message.id
-		urls = self.url_extractor.find_urls(message_text)
+		urls_raw = self.url_extractor.find_urls(message_text)
+		urls, msg_leftover = [], ''
+		if urls_raw:
+			msg_leftover = Utils.compute_leftover(urls_raw, message_text)
+			# remove duplicates
+			urls = list(set(urls_raw))
 
 		reply_text = "Wut?"
 		if not urls:
@@ -74,13 +147,11 @@ class Handlers(object):
 		else:
 			for url in urls:
 				origin = Utils.extract_origin(url)
-				if origin is Origin.INSTAGRAM:
-					url = Utils.resolve_ig_share_link(url)
 				if origin is Origin.YOUTU_BE:
-					url = Utils.extract_youtu_be_link(url)
-					if not url:
-						raise ValueError("Failed to extract youtu.be link")
-					origin = Origin.YOUTUBE
+					new_url = LinkResolver.extract_youtu_be_link_local(url)
+					if new_url:
+						url = new_url
+						origin = Origin.YOUTUBE
 				if origin is Origin.UNKNOWN:
 					logging.info("Only Instagram, YouTube Shorts and YouTube Music are now supported. Skipping.")
 					continue
@@ -119,64 +190,13 @@ class Handlers(object):
 								chat_id=chat.id,
 								chat_type=message.chat.type,
 								source_username=Utils.extract_message_author(message),
-								canonical_name=canonical_name
+								canonical_name=canonical_name,
+								message_leftover=msg_leftover
 							)
 						)
 				else:
-					async def upload_wrapper(job: UploadJob) -> None:
-						try:
-							if job.job_failed and job.job_failed_msg:
-								if job.placeholder_message_id:
-									await self.bot.placeholder.remove(chat.id, job.placeholder_message_id)
-								return await self.bot.send_text(chat_id=chat.id, text=job.job_failed_msg, reply_id=job.message_id)
-							if job.job_warning and job.job_warning_msg:
-								return await self.bot.placeholder.update_text(chat.id, job.placeholder_message_id, job.job_warning_msg)
-							tg_file_ids = await self.bot.upload_job(job)
-							if tg_file_ids:
-								if job.media_type == JobType.COLLECTION and job.save_items:
-									for chunk in job.media_collection:
-										for i in chunk:
-											self.storage.add_media(
-												tg_file_ids=[i.tg_file_id],
-												media_url=i.effective_url,
-												media_type=i.media_type.value,
-												origin=job.job_origin.value,
-												canonical_name=job.canonical_name
-											)
-								else:
-									self.storage.add_media(
-										tg_file_ids=[','.join(tg_file_ids)],
-										media_url=job.url,
-										media_type=job.media_type.value,
-										origin=job.job_origin.value,
-										canonical_name=job.canonical_name
-									)
-						except Exception as e:
-							logging.error("Exception occurred while performing upload callback!")
-							logging.exception(e)
-
-					try:
-						# create placeholder message for long download
-						placeholder_message_id = await self.bot.placeholder.create(
-							chat_id=chat.id,
-							reply_id=effective_message_id
-						)
-
-						if not placeholder_message_id:
-							await self.bot.send_text(
-								chat_id=chat.id,
-								reply_id=effective_message_id,
-								text="Failed to create message placeholder. Please check your bot Internet connection.")
-							return
-
-						self.bot.uploader.add_callback(
-							placeholder_message_id,
-							upload_wrapper
-						)
-
-						self.bot.downloader.queue_task(DownloadJob.build(
+					if await self.queue_job(DownloadJob.build(
 							url=url,
-							placeholder_message_id=placeholder_message_id,
 							message_id=effective_message_id,
 							chat_id=chat.id,
 							in_process=self.bot.uploader.is_inprocess(uniq_id),
@@ -184,11 +204,8 @@ class Handlers(object):
 							job_origin=origin,
 							source_username=Utils.extract_message_author(message),
 							chat_type=chat.type
-						))
+						)):
 						self.bot.uploader.set_inprocess(uniq_id)
-					except Exception as e:
-						logging.error("Failed to schedule download task!")
-						logging.exception(e)
 
 		if chat.type not in (ChatType.GROUP, ChatType.SUPERGROUP) and not urls:
 			await self.bot.send_text(text=reply_text, reply_id=effective_message_id, chat_id=chat.id)

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/telegram/placeholder_message.py

@@ -3,13 +3,13 @@ import time
 from enum import Enum
 from typing import Optional
 
-from pyrogram.types import Message
+#from pyrogram.types import Message
 from pyrogram.errors import RPCError, FloodWait
 from pyrogram.enums import ParseMode
 
 import warp_beacon
 from warp_beacon.telegram.utils import Utils
-from warp_beacon.mediainfo.video import VideoInfo
+#from warp_beacon.mediainfo.video import VideoInfo
 
 import logging
 
@@ -131,7 +131,7 @@ class PlaceholderMessage(object):
 							reply = await self.reuse_ph_animation(chat_id, reply_id, text)
 						except ValueError as e:
 							logging.warning("Failed to reuse tg_file_id!")
-							logging.exception(e)
+							#logging.exception(e)
 							reply = await self.reuse_ph_document(chat_id, reply_id, text)
 							self.placeholder.pl_type = PlaceholderType.DOCUMENT
 					elif self.placeholder.pl_type == PlaceholderType.DOCUMENT:
@@ -139,7 +139,7 @@ class PlaceholderMessage(object):
 							reply = await self.reuse_ph_document(chat_id, reply_id, text)
 						except ValueError as e:
 							logging.warning("Failed to reuse tg_file_id!")
-							logging.exception(e)
+							#logging.exception(e)
 							reply = await self.reuse_ph_animation(chat_id, reply_id, text)
 							self.placeholder.pl_type = PlaceholderType.ANIMATION
 					else:

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/telegram/utils.py

@@ -1,7 +1,6 @@
 from typing import Union
 
 import re
-import requests
 
 from pyrogram.types import Message
 
@@ -12,7 +11,6 @@ import logging
 
 class Utils(object):
 	expected_patronum_compiled_re = re.compile(r'Expected ([A-Z]+), got ([A-Z]+) file id instead')
-	canonical_link_resolve_re = re.compile(r'<link.*rel="canonical".*href="([^"]+)"\s*/?>')
 
 	@staticmethod
 	def extract_file_id(message: Message) -> Union[None, str]:
@@ -45,37 +43,6 @@ class Utils(object):
 
 		return Origin.UNKNOWN
 
-	@staticmethod
-	def extract_youtu_be_link(url: str) -> str:
-		try:
-			response = requests.get(
-				url=url,
-				allow_redirects=False
-			)
-			return response.headers["Location"]
-		except Exception as e:
-			logging.error("Failed to extract YouTube link!")
-			logging.exception(e)
-
-		return ''
-	
-	@staticmethod
-	def resolve_ig_share_link(url: str) -> str:
-		# expected url: https://www.instagram.com/share/reel/BAHtk2AamB
-		# result url: https://www.instagram.com/reel/DAKjQgUNzuH/
-		try:
-			if "instagram.com/" in url and "share/" in url:
-				content = requests.get(url).text
-				res = re.search(Utils.canonical_link_resolve_re, content)
-				new_url = res.group(1).strip()
-				logging.info("Converted IG share link to '%s'", new_url)
-				return new_url
-		except Exception as e:
-			logging.error("Failed to convert IG share link!")
-			logging.exception(e)
-
-		return url
-
 	@staticmethod
 	def parse_expected_patronum_error(err_text: str) -> tuple:
 		'''
@@ -111,4 +78,13 @@ class Utils(object):
 				return message.sender_chat.username
 			if message.sender_chat.title:
 				return message.sender_chat.title
-		return ''
+		return ''
+	
+	@staticmethod
+	def compute_leftover(urls: list, message: str) -> str:
+		msg_leftover = ""
+		if len(message) > sum(len(u) for u in urls):
+			msg_leftover = message
+			for u in urls:
+				msg_leftover = msg_leftover.replace(u, '')
+		return msg_leftover.strip()

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon/uploader/__init__.py

@@ -81,10 +81,11 @@ class AsyncUploader(object):
 				try:
 					job = self.job_queue.get()
 					if job is self.__JOE_BIDEN_WAKEUP:
-						continue
+						break
 					if job.is_message_to_admin and job.message_text and self.admin_message_callback:
 						asyncio.ensure_future(self.admin_message_callback(job.message_text, job.account_admins, job.yt_auth), loop=self.loop)
 						continue
+
 					path = ""
 					if job.media_type == JobType.COLLECTION:
 						for i in job.media_collection:
@@ -92,11 +93,14 @@ class AsyncUploader(object):
 								path += "%s; " % j.local_media_path
 					else:
 						path = job.local_media_path
+	
 					in_process = job.in_process
 					uniq_id = job.uniq_id
 					message_id = job.placeholder_message_id
-					if not in_process and not job.job_failed and not job.job_warning:
+
+					if not in_process and not job.job_failed and not job.job_warning and not job.replay:
 						logging.info("Accepted upload job, file(s): '%s'", path)
+
 					try:
 						if message_id in self.callbacks:
 							if job.job_failed:
@@ -106,6 +110,12 @@ class AsyncUploader(object):
 								self.process_done(uniq_id)
 								self.remove_callback(message_id)
 								continue
+							
+							if job.replay:
+								asyncio.ensure_future(self.callbacks[message_id]["callback"](job), loop=self.loop)
+								self.remove_callback(message_id)
+								continue
+
 							if job.job_warning:
 								logging.info("Job warning occurred ...")
 								if job.job_warning_msg:

{warp_beacon-2.2.67 → warp_beacon-2.3.0/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: warp_beacon
-Version: 2.2.67
+Version: 2.3.0
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -236,6 +236,8 @@ Requires-Dist: urlextract
 Requires-Dist: pillow
 Requires-Dist: pymongo
 Requires-Dist: instagrapi==2.0.0
+Dynamic: author
+Dynamic: home-page
 
 # warp_beacon [![Upload Python Package](https://github.com/sb0y/warp_beacon/actions/workflows/python-publish.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/python-publish.yml) [![Docker Image CI](https://github.com/sb0y/warp_beacon/actions/workflows/docker-image.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/docker-image.yml) [![Build DEB package](https://github.com/sb0y/warp_beacon/actions/workflows/build-deb.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/build-deb.yml)
 

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ setup.py
 assets/placeholder.gif
 etc/.gitignore
 etc/accounts.json
+etc/proxies.json
 etc/warp_beacon.conf
 etc/warp_beacon.service
 warp_beacon/__init__.py
@@ -37,6 +38,7 @@ warp_beacon/scraper/abstract.py
 warp_beacon/scraper/account_selector.py
 warp_beacon/scraper/exceptions.py
 warp_beacon/scraper/fail_handler.py
+warp_beacon/scraper/link_resolver.py
 warp_beacon/scraper/instagram/__init__.py
 warp_beacon/scraper/instagram/instagram.py
 warp_beacon/scraper/youtube/__init__.py

{warp_beacon-2.2.67 → warp_beacon-2.3.0}/warp_beacon.egg-info/top_level.txt

@@ -20,6 +20,7 @@ warp_beacon/scraper/exceptions
 warp_beacon/scraper/fail_handler
 warp_beacon/scraper/instagram
 warp_beacon/scraper/instagram/instagram
+warp_beacon/scraper/link_resolver
 warp_beacon/scraper/types
 warp_beacon/scraper/youtube
 warp_beacon/scraper/youtube/abstract

warp_beacon-2.2.67/etc/accounts.json

@@ -1,72 +0,0 @@
-{
-	"instagram":
-	[
-		{
-			"login": "ig_account0",
-			"password": "ig_password",
-			"imap_server": "imap.gmail.com",
-			"imap_login": "user@gmail.com",
-			"imap_password": "",
-			"auth_details":
-			{
-				"delay_range": [1, 3],
-				"country_code": 7,
-				"locale": "en_US",
-				"timezone_offset": 10800,
-				"user_agent": "Barcelona 291.0.0.31.111 Android (33/13; 600dpi; 1440x3044; samsung; SM-G998B; p3s; exynos2100; en_US; 493450264)",
-				"device":
-				{
-					"app_version": "291.0.0.31.111",
-					"android_version": 33,
-					"android_release": "13.0.0",
-					"dpi": "600dpi",
-					"resolution": "1440x3044",
-					"manufacturer": "Samsung",
-					"device": "p3s",
-					"model": "SM-G998B",
-					"cpu": "exynos2100",
-					"version_code": "493450264"
-				}
-			},
-			"account_admins": "@BelisariusCawl"
-		},
-		{
-			"login": "ig_account1",
-			"password": "passwd",
-			"imap_server": "imap.gmail.com",
-			"imap_login": "mail_login1",
-			"imap_password": "imap_password1",
-			"auth_details":
-			{
-				"delay_range": [1, 3],
-				"country_code": 7,
-				"locale": "en_US",
-				"timezone_offset": 10800,
-				"user_agent": "Barcelona 291.0.0.31.111 Android (33/13; 600dpi; 1440x3044; samsung; SM-G998B; p3s; exynos2100; en_US; 493450264)",
-				"device":
-				{
-					"app_version": "291.0.0.31.111",
-					"android_version": 33,
-					"android_release": "13.0.0",
-					"dpi": "600dpi",
-					"resolution": "1440x3044",
-					"manufacturer": "Samsung",
-					"device": "p3s",
-					"model": "SM-G998B",
-					"cpu": "exynos2100",
-					"version_code": "493450264"
-				}
-			},
-			"account_admins": "@BelisariusCawl"
-		}
-	],
-	"youtube":
-	[
-		{
-			"account_admins": "@BelisariusCawl"
-		},
-		{
-			"account_admins": "@Lazyteh"
-		}
-	]
-}

warp_beacon-2.2.67/etc/warp_beacon.conf

@@ -1,15 +0,0 @@
-TG_TOKEN=""
-TG_BOT_NAME=""
-TG_BOT_ADMINS_USERNAMES=""
-TG_API_ID=""
-TG_API_HASH=""
-TG_BOT_NAME=""
-IG_MAX_RETRIES=10
-IG_REQUEST_TIMEOUT=30
-MONGODB_HOST="mongodb"
-MONGODB_PORT="27017"
-MONGODB_USER="root"
-MONGODB_PASSWORD="changeme"
-ENABLE_DONATES=true
-SERVICE_ACCOUNTS_FILE=/var/warp_beacon/accounts.json
-FORCE_IPV6=true

warp_beacon-2.2.67/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.2.67"
-