warp-beacon 2.6.62__tar.gz → 2.6.64__tar.gz

{warp_beacon-2.6.62/warp_beacon.egg-info → warp_beacon-2.6.64}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.62
+Version: 2.6.64
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -248,10 +248,10 @@ Dynamic: license-file
 # warp_beacon [![Upload Python Package](https://github.com/sb0y/warp_beacon/actions/workflows/python-publish.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/python-publish.yml) [![Docker Image CI](https://github.com/sb0y/warp_beacon/actions/workflows/docker-image.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/docker-image.yml) [![Build DEB package](https://github.com/sb0y/warp_beacon/actions/workflows/build-deb.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/build-deb.yml)
 > because content should travel freely
 
-Telegram bot for external social networks media links expanding.
-Works with a links in personal messages and also with group chats.
+Telegram bot that expands media links from external social networks.
+Works with links sent in private messages or group chats.
 
-Just send a media link to the chat with bot and get a video or audio reply.
+Just send a media link to the bot, and it will reply with a video or audio file.
 
 | | | |
 |:-------------------------:|:-------------------------:|:-------------------------:|

{warp_beacon-2.6.62 → warp_beacon-2.6.64}/README.md

@@ -1,10 +1,10 @@
 # warp_beacon [![Upload Python Package](https://github.com/sb0y/warp_beacon/actions/workflows/python-publish.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/python-publish.yml) [![Docker Image CI](https://github.com/sb0y/warp_beacon/actions/workflows/docker-image.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/docker-image.yml) [![Build DEB package](https://github.com/sb0y/warp_beacon/actions/workflows/build-deb.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/build-deb.yml)
 > because content should travel freely
 
-Telegram bot for external social networks media links expanding.
-Works with a links in personal messages and also with group chats.
+Telegram bot that expands media links from external social networks.
+Works with links sent in private messages or group chats.
 
-Just send a media link to the chat with bot and get a video or audio reply.
+Just send a media link to the bot, and it will reply with a video or audio file.
 
 | | | |
 |:-------------------------:|:-------------------------:|:-------------------------:|

{warp_beacon-2.6.62 → warp_beacon-2.6.64}/setup.py

@@ -70,6 +70,7 @@ setup(
 		"warp_beacon/telegram/progress_file_reader",
 		"warp_beacon/telegram/edit_message",
 		"warp_beacon/telegram/download_status",
+		"warp_beacon/telegram/types",
 		"warp_beacon/jobs/abstract",
 		"warp_beacon/jobs/download_job",
 		"warp_beacon/jobs/upload_job",
@@ -84,6 +85,7 @@ setup(
 		"warp_beacon/scraper/exceptions",
 		"warp_beacon/scraper/types",
 		"warp_beacon/scraper/instagram/instagram",
+		"warp_beacon/scraper/instagram/wb_instagrapi",
 		"warp_beacon/scraper/account_selector",
 		"warp_beacon/scraper/youtube/abstract",
 		"warp_beacon/scraper/youtube/youtube",

warp_beacon-2.6.64/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.6.64"
+

{warp_beacon-2.6.62 → warp_beacon-2.6.64}/warp_beacon/scraper/abstract.py

@@ -1,4 +1,5 @@
 import os
+import time
 import pathlib
 from abc import ABC, abstractmethod
 from typing import Callable, Union
@@ -77,6 +78,20 @@ class ScraperAbstract(ABC):
 			logging.exception(e)
 
 		return ''
+	
+	def rename_local_file(self, filename: str) -> str:
+		if not os.path.exists(filename):
+			raise NameError("No file provided")
+		path_info = pathlib.Path(filename)
+		ext = path_info.suffix
+		#old_filename = path_info.stem
+		time_name = str(time.time()).replace('.', '_')
+		new_filename = f"{time_name}{ext}"
+		new_filepath = f"{os.path.dirname(filename)}/{new_filename}"
+
+		os.rename(filename, new_filepath)
+
+		return new_filepath
 
 	def force_ipv6(self) -> None:
 		def allowed_gai_family():

{warp_beacon-2.6.62 → warp_beacon-2.6.64}/warp_beacon/scraper/account_selector.py

@@ -267,8 +267,8 @@ class AccountSelector(object):
 			idx = self.account_index[self.current_module_name].value
 			if idx not in self.ig_accounts_session_id:
 				self.ig_accounts_session_id[idx] = str(uuid.uuid4())
-			else:
-				if random.random() > 0.95:
-					self.ig_accounts_session_id[idx] = str(uuid.uuid4())
-					logging.info("Rotated client_session_id — simulating app restart")
+			#else:
+				#if random.random() > 0.95:
+				#	self.ig_accounts_session_id[idx] = str(uuid.uuid4())
+				#	logging.info("Rotated client_session_id — simulating app restart")
 			return self.ig_accounts_session_id[idx]

{warp_beacon-2.6.62 → warp_beacon-2.6.64}/warp_beacon/scraper/instagram/instagram.py

@@ -3,11 +3,12 @@ import time
 import socket
 import ssl
 import re
+from pathlib import Path
+import random
 from typing import Callable, Optional, Union
 
 import logging
 
-import random
 import email
 import imaplib
 import json
@@ -19,7 +20,6 @@ from instagrapi import exceptions
 from instagrapi.exceptions import UnknownError as IGUnknownError
 from instagrapi.mixins.story import Story
 from instagrapi.types import Media
-from instagrapi import Client
 from instagrapi.mixins.challenge import ChallengeChoice
 #from instagrapi.exceptions import LoginRequired, PleaseWaitFewMinutes, MediaNotFound, ClientNotFoundError, UserNotFound, ChallengeRequired, \
 #	ChallengeSelfieCaptcha, ChallengeUnknownStep, UnknownError as IGUnknownError
@@ -29,6 +29,8 @@ from warp_beacon.scraper.abstract import ScraperAbstract
 from warp_beacon.jobs.types import JobType
 from warp_beacon.jobs.download_job import DownloadJob
 from warp_beacon.telegram.utils import Utils
+from warp_beacon.scraper.instagram.wb_instagrapi import WBClient
+from warp_beacon.telegram.types import ReportType
 
 INST_SESSION_FILE_TPL = "/var/warp_beacon/inst_session_account_%d.json"
 
@@ -43,7 +45,7 @@ class InstagramScraper(ScraperAbstract):
 		super().__init__(account, proxy)
 		#
 		self.inst_session_file = INST_SESSION_FILE_TPL % self.account_index
-		self.cl = Client()
+		self.cl = WBClient()
 		if self.proxy:
 			proxy_dsn = self.proxy.get("dsn", "")
 			if proxy_dsn:
@@ -53,6 +55,17 @@ class InstagramScraper(ScraperAbstract):
 		self.setup_device()
 		self.cl.challenge_code_handler = self.challenge_code_handler
 		self.cl.change_password_handler = self.change_password_handler
+		self.cl.public.headers.update({
+			"Connection": "keep-alive",
+			"Accept": "*/*",
+			"Accept-Encoding": "gzip, deflate, br",
+			"Accept-Language": "en-US,en;q=0.9",
+			"User-Agent": (
+				"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+				"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+			)
+		})
+		self.cl.set_progress_callback(self.download_progress)
 
 	def setup_device(self) -> None:
 		if not self.client_session_id:
@@ -227,9 +240,13 @@ class InstagramScraper(ScraperAbstract):
 	def download_video(self, url: str, media_info: Media) -> dict:
 		self.cl.request_timeout = int(os.environ.get("IG_REQUEST_TIMEOUT", default=60))
 		path = self.download_hndlr(self.cl.video_download_by_url, url, folder='/tmp')
-		return {"local_media_path": str(path), "canonical_name": self.extract_canonical_name(media_info), \
-			"media_type": JobType.VIDEO, "last_pk": media_info.pk, \
-			"media_info": {"duration": round(media_info.video_duration)}}
+		return {
+			"local_media_path": self.rename_local_file(str(path)),
+			"canonical_name": self.extract_canonical_name(media_info),
+			"media_type": JobType.VIDEO,
+			"last_pk": media_info.pk,
+			"media_info": {"duration": round(media_info.video_duration)}
+		}
 
 	def download_photo(self, url: str, media_info: Media) -> dict:
 		path = str(self.download_hndlr(self.cl.photo_download_by_url, url, folder='/tmp'))
@@ -238,8 +255,13 @@ class InstagramScraper(ScraperAbstract):
 			path = InstagramScraper.convert_webp_to_png(path)
 		if ".heic" in path_lowered:
 			path = InstagramScraper.convert_heic_to_png(path)
-		return {"local_media_path": path, "canonical_name": self.extract_canonical_name(media_info), "media_type": JobType.IMAGE, "last_pk": media_info.pk}
-	
+		return {
+			"local_media_path": self.rename_local_file(path),
+			"canonical_name": self.extract_canonical_name(media_info),
+			"media_type": JobType.IMAGE,
+			"last_pk": media_info.pk
+		}
+
 	def download_story(self, story_info: Story) -> dict:
 		path, media_type, media_info = "", JobType.UNKNOWN, {}
 		logging.info("Story id is '%s'", story_info.id)
@@ -263,7 +285,7 @@ class InstagramScraper(ScraperAbstract):
 			media_type = JobType.VIDEO
 			media_info["duration"] = story_info.video_duration
 
-		return {"local_media_path": path, "media_type": media_type, "media_info": media_info, "effective_url": effective_url}
+		return {"local_media_path": self.rename_local_file(path), "media_type": media_type, "media_info": media_info, "effective_url": effective_url}
 
 	def download_stories(self, stories: list[Story]) -> dict:
 		chunks = []
@@ -309,8 +331,11 @@ class InstagramScraper(ScraperAbstract):
 			try:
 				scrap_type, media_id = self.scrap(job.url)
 				if scrap_type == "media":
-					#self.status_pipe.send({"action": "report_download_status", "current": 0, "total": 0,
-					#				"message_id": self.job.placeholder_message_id, "chat_id": self.job.chat_id, "label": "Collection meta information ..."})
+					self.status_pipe.send({
+						"action": "report_download_status",
+						"report_type": ReportType.ANNOUNCE,
+						"label": "Collecting meta information ..."
+					})
 					media_info = self.download_hndlr(self.cl.media_info, media_id, use_cache=False)
 					logging.info("media_type is '%d', product_type is '%s'", media_info.media_type, media_info.product_type)
 					if media_info.media_type == 2 and media_info.product_type == "clips": # Reels
@@ -400,4 +425,18 @@ class InstagramScraper(ScraperAbstract):
 		chars = list("abcdefghijklmnopqrstuvwxyz1234567890!&£@#")
 		password = "".join(random.sample(chars, 10))
 		logging.info("Generated new IG password: '%s'", password)
-		return password
+		return password
+	
+	def download_progress(self, total: int | None, bytes_transferred: int, path: Path) -> None:
+		percentage_of_completion = round(bytes_transferred / (total or 1) * 100)
+		logging.debug("[Download] IG file %s, %d", str(path), percentage_of_completion)
+		msg = {
+			"action": "report_download_status",
+			"current": bytes_transferred,
+			"total": total or 0,
+			"message_id": self.job.placeholder_message_id,
+			"chat_id": self.job.chat_id,
+			"completed": bool(total and bytes_transferred >= total),
+			"report_type": ReportType.PROGRESS
+		}
+		self.status_pipe.send(msg)

warp_beacon-2.6.64/warp_beacon/scraper/instagram/wb_instagrapi.py

@@ -0,0 +1,62 @@
+import logging
+from typing import Callable
+from urllib.parse import urlparse
+from pathlib import Path
+import requests
+
+from instagrapi import Client
+from instagrapi.exceptions import VideoNotDownload
+
+class WBClient(Client):
+	"""
+	patched instagrapi
+	"""
+	def __init__(self) -> None:
+		super().__init__()
+		self.progress_callback = None
+
+	def set_progress_callback(self, callback: Callable[[int | None, int, Path], None]) -> None:
+		if not callback or not callable(callback):
+			raise TypeError("Progress callback must be callable")
+		self.progress_callback = callback
+
+	def video_download_by_url(self, url: str, filename: str = "", folder: Path = "") -> Path:
+		fname = urlparse(url).path.rsplit("/", 1)[1]
+		filename = f"{filename}.{fname.rsplit('.', 1)[1]}" if filename else fname
+		path = Path(folder or Path.cwd()) / filename
+
+		logging.info("Downloading video from %s to %s", url, path)
+
+		response = requests.get(url, stream=True, timeout=self.request_timeout)
+		response.raise_for_status()
+
+		content_length = 0
+		try:
+			content_length = int(response.headers.get("Content-Length", 0))
+		except (TypeError, ValueError):
+			logging.warning("Content-Length header is missing or invalid.")
+			content_length = 0
+
+		downloaded = 0
+		with open(path, "wb") as f:
+			for chunk in response.iter_content(chunk_size=8192):
+				if chunk:
+					f.write(chunk)
+					downloaded += len(chunk)
+					if self.progress_callback:
+						try:
+							self.progress_callback(
+								total=content_length or None,
+								bytes_transferred=downloaded,
+								path=path
+							)
+						except Exception as e:
+							logging.warning("Progress callback raised an exception!", exc_info=e)
+
+
+		if content_length and downloaded != content_length:
+			raise VideoNotDownload(
+				f'Broken file "{path}" (expected {content_length}, got {downloaded})'
+			)
+
+		return path.resolve()

{warp_beacon-2.6.62 → warp_beacon-2.6.64}/warp_beacon/scraper/youtube/abstract.py

@@ -4,7 +4,6 @@ import json
 import logging
 import math
 import os
-import pathlib
 import socket
 import ssl
 import time
@@ -65,20 +64,6 @@ class YoutubeAbstract(ScraperAbstract):
 			logging.exception(e)
 
 		return 0
-
-	def rename_local_file(self, filename: str) -> str:
-		if not os.path.exists(filename):
-			raise NameError("No file provided")
-		path_info = pathlib.Path(filename)
-		ext = path_info.suffix
-		#old_filename = path_info.stem
-		time_name = str(time.time()).replace('.', '_')
-		new_filename = f"{time_name}{ext}"
-		new_filepath = f"{os.path.dirname(filename)}/{new_filename}"
-
-		os.rename(filename, new_filepath)
-
-		return new_filepath
 	
 	def get_video_id(self, url: str) -> Optional[str]:
 		parsed_url = urlparse(url)

{warp_beacon-2.6.62 → warp_beacon-2.6.64}/warp_beacon/telegram/download_status.py

@@ -2,6 +2,7 @@ import logging
 from multiprocessing import Pipe
 from pyrogram import Client
 from warp_beacon.telegram.progress_bar import ProgressBar
+from warp_beacon.telegram.types import ReportType
 
 class DownloadStatus(object):
 	status_pipe = None
@@ -15,12 +16,13 @@ class DownloadStatus(object):
 		self.status_pipe, self.child_conn = Pipe()
 
 	async def handle_message(self, msg: dict, progress_bar: ProgressBar) -> None:
-		await progress_bar.progress_callback(
+		progress_bar.progress_callback(
 			current=msg.get("current", 0),
 			total=msg.get("total", 0),
 			message_id=msg.get("message_id", 0),
 			chat_id=msg.get("chat_id", 0),
-			operation="Downloading"
+			operation="Downloading",
+			report_type=msg.get("report_type", ReportType.PROGRESS)
 		)
 
 	def on_status(self) -> None:

{warp_beacon-2.6.62 → warp_beacon-2.6.64}/warp_beacon/telegram/edit_message.py

@@ -93,8 +93,8 @@ class EditMessage(object):
 		file_name: str = None
 	) -> None:
 		progress_bar = ProgressBar(self.client)
-		await progress_bar.progress_callback(current=0, total=0, chat_id=chat_id, message_id=message_id, operation="Uploading", label=file_name)
-		raw_file = await self.client.save_file(path=media.media, progress=progress_bar.progress_callback, progress_args=(chat_id, message_id, "Uploading", file_name,))
+		await progress_bar.progress_callback(current=0, total=0, chat_id=chat_id, message_id=message_id, operation="Uploading", label=file_name, report_type="progress")
+		raw_file = await self.client.save_file(path=media.media, progress=progress_bar.progress_callback, progress_args=(chat_id, message_id, "Uploading", file_name, "progress"))
 
 		caption = media.caption
 		parse_mode = media.parse_mode

{warp_beacon-2.6.62 → warp_beacon-2.6.64}/warp_beacon/telegram/progress_bar.py

@@ -6,6 +6,7 @@ import hashlib
 from pyrogram.enums import ParseMode
 from pyrogram.errors.exceptions.bad_request_400 import MessageNotModified
 from pyrogram import Client
+from warp_beacon.telegram.types import ReportType
 
 class ProgressBar(object):
 	def __init__(self, client: Client) -> None:
@@ -68,7 +69,27 @@ class ProgressBar(object):
 		else:
 			logging.exception("Error in edit_message_caption", exc_info=exc)
 
-	async def progress_callback(self, current: int, total: int, chat_id: int | str, message_id: int, operation: str, label: str = "") -> None:
+	def progress_callback(self, current: int, total: int, chat_id: int | str, message_id: int, operation: str, report_type: ReportType, label: str = "") -> None:
+		if report_type == ReportType.PROGRESS:
+			return self.render_progress_bar(current=current, total=total, chat_id=chat_id, message_id=message_id, operation=operation, label=label)
+		elif report_type == ReportType.ANNOUNCE:
+			return self.render_progress_announce(chat_id=chat_id, message_id=message_id, label=label)
+
+	def render_progress_announce(self, chat_id: int | str, message_id: int, label: str) -> None:
+		try:
+			#await self.client.edit_message_caption(chat_id, message_id, f"{pbar} <b>{operation}</b> {label}", ParseMode.HTML)
+			# we don't need to wait completion, waste of time and resources
+			task = self.client.loop.create_task(
+				self.client.edit_message_caption(chat_id, message_id, f"<b>{label}</b>", ParseMode.HTML)
+			)
+			task.add_done_callback(self._on_edit_done)
+		except MessageNotModified:
+			logging.warning("bad_request_400.MessageNotModified")
+		except Exception as e:
+			logging.warning("An error occurred while setup task to update progress bar")
+			logging.exception(e)
+
+	def render_progress_bar(self, current: int, total: int, chat_id: int | str, message_id: int, operation: str, label: str = "") -> None:
 		percent = 0
 		if total:
 			percent = round(current * 100 / (total or 1))
@@ -81,11 +102,11 @@ class ProgressBar(object):
 			try:
 				#await self.client.edit_message_caption(chat_id, message_id, f"{pbar} <b>{operation}</b> {label}", ParseMode.HTML)
 				# we don't need to wait completion, waste of time and resources
-				text = f"{pbar}\n<b>{operation}</b>"
+				text = f"{pbar}\n{operation}"
 				if label:
-					text += f"\n{label}"
+					text += f"\n<code>{label}</code>"
 				if total:
-					text += f" {self.format_size_si(total)}"
+					text += f" <b>{self.format_size_si(total)}</b>"
 				task = self.client.loop.create_task(
 					self.client.edit_message_caption(chat_id, message_id, text, ParseMode.HTML)
 				)
@@ -97,8 +118,6 @@ class ProgressBar(object):
 				logging.exception(e)
 			if total > 0:
 				self._next_threshold += 20
-		if percent >= 100:
-			self.complete = True
 
 	@staticmethod
 	def make_hash(chat_id: str | int, message_id: int, algorithm: str = 'sha256') -> str:

warp_beacon-2.6.64/warp_beacon/telegram/types.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+class ReportType(Enum):
+	UNKNOWN = 0,
+	PROGRESS = 1,
+	ANNOUNCE = 2

{warp_beacon-2.6.62 → warp_beacon-2.6.64/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.62
+Version: 2.6.64
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev
@@ -248,10 +248,10 @@ Dynamic: license-file
 # warp_beacon [![Upload Python Package](https://github.com/sb0y/warp_beacon/actions/workflows/python-publish.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/python-publish.yml) [![Docker Image CI](https://github.com/sb0y/warp_beacon/actions/workflows/docker-image.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/docker-image.yml) [![Build DEB package](https://github.com/sb0y/warp_beacon/actions/workflows/build-deb.yml/badge.svg)](https://github.com/sb0y/warp_beacon/actions/workflows/build-deb.yml)
 > because content should travel freely
 
-Telegram bot for external social networks media links expanding.
-Works with a links in personal messages and also with group chats.
+Telegram bot that expands media links from external social networks.
+Works with links sent in private messages or group chats.
 
-Just send a media link to the chat with bot and get a video or audio reply.
+Just send a media link to the bot, and it will reply with a video or audio file.
 
 | | | |
 |:-------------------------:|:-------------------------:|:-------------------------:|

{warp_beacon-2.6.62 → warp_beacon-2.6.64}/warp_beacon.egg-info/SOURCES.txt

@@ -45,6 +45,7 @@ warp_beacon/scraper/utils.py
 warp_beacon/scraper/instagram/__init__.py
 warp_beacon/scraper/instagram/captcha.py
 warp_beacon/scraper/instagram/instagram.py
+warp_beacon/scraper/instagram/wb_instagrapi.py
 warp_beacon/scraper/youtube/__init__.py
 warp_beacon/scraper/youtube/abstract.py
 warp_beacon/scraper/youtube/music.py
@@ -61,5 +62,6 @@ warp_beacon/telegram/handlers.py
 warp_beacon/telegram/placeholder_message.py
 warp_beacon/telegram/progress_bar.py
 warp_beacon/telegram/progress_file_reader.py
+warp_beacon/telegram/types.py
 warp_beacon/telegram/utils.py
 warp_beacon/uploader/__init__.py

{warp_beacon-2.6.62 → warp_beacon-2.6.64}/warp_beacon.egg-info/top_level.txt

@@ -21,6 +21,7 @@ warp_beacon/scraper/exceptions
 warp_beacon/scraper/fail_handler
 warp_beacon/scraper/instagram
 warp_beacon/scraper/instagram/instagram
+warp_beacon/scraper/instagram/wb_instagrapi
 warp_beacon/scraper/link_resolver
 warp_beacon/scraper/types
 warp_beacon/scraper/utils
@@ -40,6 +41,7 @@ warp_beacon/telegram/handlers
 warp_beacon/telegram/placeholder_message
 warp_beacon/telegram/progress_bar
 warp_beacon/telegram/progress_file_reader
+warp_beacon/telegram/types
 warp_beacon/telegram/utils
 warp_beacon/uploader
 warp_beacon/warp_beacon

warp_beacon-2.6.62/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.6.62"
-