warp-beacon 2.7.6__tar.gz → 2.7.8__tar.gz

{warp_beacon-2.7.6/warp_beacon.egg-info → warp_beacon-2.7.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.7.6
+Version: 2.7.8
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.7.8/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.7.8"
+

{warp_beacon-2.7.6 → warp_beacon-2.7.8}/warp_beacon/scraper/__init__.py

@@ -290,14 +290,14 @@ class AsyncDownloader(object):
 										break
 									self.send_message_to_admin(
 										f"Task <code>{job.job_id}</code> failed. URL: {job.url}. Reason: '<b>UnknownError</b>'."
-										f"Exception:\n<pre code=\"python\">{exception_msg}<br></pre>"
+										f"Exception:\n<pre code=\"python\">{exception_msg}\n</pre>"
 									)
 									self.uploader.queue_task(job.to_upload_job(
 										job_failed=True,
-										job_failed_msg=f"Unknown error occured. Please <a href=\"https://github.com/sb0y/warp_beacon/issues\">create issue</a> with service logs.<br>"
-										f"Task <code>{job.job_id}</code> failed. URL: {job.url}.<br>"
-										f"Reason: '<b>UnknownError</b>'.<br>"
-										f"Exception:<br><pre code=\"python\">{exception_msg}</pre>"
+										job_failed_msg=f"Unknown error occurred. Please <a href=\"https://github.com/sb0y/warp_beacon/issues\">create issue</a> with service logs.\n"
+										f"Task <code>{job.job_id}</code> failed. URL: {job.url}.\n"
+										f"Reason: '<b>UnknownError</b>'.\n"
+										f"Exception:\n<pre code=\"python\">{exception_msg}</pre>"
 									))
 									break
 								finally:

{warp_beacon-2.7.6 → warp_beacon-2.7.8}/warp_beacon/scraper/instagram/instagram.py

@@ -42,7 +42,7 @@ class InstagramScraper(ScraperAbstract):
 	client_session_id = ""
 
 	def __init__(self, client_session_id: str, account: tuple, proxy: dict=None) -> None:
-		self._download_progress_threshold = 5
+		self._download_progress_threshold = 0
 		self.client_session_id = client_session_id
 		super().__init__(account, proxy)
 		#
@@ -145,17 +145,18 @@ class InstagramScraper(ScraperAbstract):
 	def scrap(self, url: str) -> tuple[str]:
 		self.load_session()
 		def _scrap() -> tuple[str]:
-			if "stories" in url:
+			path = urlparse(url).path
+			slash_count = path.count('/')
+			if "stories" in url or slash_count == 1:
 				# remove URL options
-				_url = urljoin(url, urlparse(url).path)
+				_url = urljoin(url, path)
 				url_last_part = list(filter(None, _url.split('/')))[-1]
 				logging.debug("url last part: '%s'", url_last_part)
 				if url_last_part.isnumeric():
 					return "story", self.scrap_story(url)
-				else:
+				elif slash_count in (1, 2):
 					return "stories", url_last_part
-			else:
-				return "media", self.scrap_media(url)
+			return "media", self.scrap_media(url)
 		try:
 			return _scrap()
 		except exceptions.LoginRequired as e:
@@ -474,7 +475,7 @@ class InstagramScraper(ScraperAbstract):
 				"report_type": ReportType.PROGRESS
 			}
 			self.status_pipe.send(msg)
-			self._download_progress_threshold += 5
+			self._download_progress_threshold += 10
 
 	def report_seen(self, items: dict) -> None:
 		try:

{warp_beacon-2.7.6 → warp_beacon-2.7.8}/warp_beacon/scraper/youtube/abstract.py

@@ -37,7 +37,7 @@ class YoutubeAbstract(ScraperAbstract):
 
 	def __init__(self, account: tuple, proxy: dict=None) -> None:
 		super().__init__(account, proxy)
-		self._download_progress_threshold = 5
+		self._download_progress_threshold = 0
 
 	def validate_session(self) -> int:
 		try:
@@ -245,7 +245,7 @@ class YoutubeAbstract(ScraperAbstract):
 			self.status_pipe.send(msg)
 			logging.debug("[Download worker] Downloaded %d%%", percentage_of_completion)
 			if total_size > 0:
-				self._download_progress_threshold += 5
+				self._download_progress_threshold += 10
 
 	def build_proxies(self, proxy_dsn: str) -> dict:
 		if not proxy_dsn:
@@ -310,7 +310,7 @@ class YoutubeAbstract(ScraperAbstract):
 				self.status_pipe.send(msg)
 				logging.debug("[Download worker][yt_dlp] Downloaded %d%%", percentage_of_completion)
 				if total_size > 0:
-					self._download_progress_threshold += 5
+					self._download_progress_threshold += 10
 
 	def build_yt_dlp(self, timeout: int = 60) -> yt_dlp.YoutubeDL:
 		auth_data = {}

{warp_beacon-2.7.6 → warp_beacon-2.7.8}/warp_beacon/telegram/handlers.py

@@ -148,10 +148,12 @@ class Handlers(object):
 			msg_leftover = await Utils.handle_mentions(chat.id, client, msg_leftover)
 			# remove duplicates
 			urls = list(set(urls_raw))
+			# remove links without paths
+			urls = Utils.remove_links_wo_paths(urls)
 
 		reply_text = "Wut?"
 		if not urls:
-			reply_text = "Your message should contains URLs"
+			reply_text = "Your message should contain URLs"
 		else:
 			for url in urls:
 				origin = Utils.extract_origin(url)

{warp_beacon-2.7.6 → warp_beacon-2.7.8}/warp_beacon/telegram/progress_bar.py

@@ -12,7 +12,7 @@ class ProgressBar(object):
 	MAX_PROGRESS_RENDER_SIZE = 1_500_000 # 1 MB
 
 	def __init__(self, client: Client) -> None:
-		self._next_threshold = 5
+		self._next_threshold = 0
 		self.client = client
 		self.complete = False
 		self.rendered_text = ""
@@ -127,7 +127,7 @@ class ProgressBar(object):
 				logging.warning("An error occurred while setup task to update progress bar")
 				logging.exception(e)
 			if total > 0 and percent != 0:
-				self._next_threshold += 5
+				self._next_threshold += 10
 
 	@staticmethod
 	def make_hash(chat_id: str | int, message_id: int, algorithm: str = 'sha256') -> str:

{warp_beacon-2.7.6 → warp_beacon-2.7.8}/warp_beacon/telegram/utils.py

@@ -1,6 +1,6 @@
-from typing import Union, Optional
-
+from typing import Optional
 import re
+from urllib.parse import urlparse
 
 from pyrogram import Client
 from pyrogram.types import Message, ChatMember
@@ -154,3 +154,18 @@ class Utils(object):
 			return
 		if not client.me or not client.me.is_premium:
 			client.me = await client.get_me()
+
+	@staticmethod
+	def remove_links_wo_paths(urls: list[str]) -> list[str]:
+		try:
+			new_urls = []
+			for url in urls:
+				url = url.rstrip('/')
+				path = urlparse(url).path
+				sc = path.count('/')
+				if sc > 0:
+					new_urls.append(url)
+			return new_urls
+		except Exception as e:
+			logging.warning("Failed to remove links without path!", exc_info=e)
+		return urls

{warp_beacon-2.7.6 → warp_beacon-2.7.8/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.7.6
+Version: 2.7.8
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.7.6/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.7.6"
-