warp-beacon 2.7.3__py3-none-any.whl → 2.7.5__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.7.3"
+__version__ = "2.7.5"
 

warp_beacon/scraper/__init__.py

@@ -392,6 +392,10 @@ class AsyncDownloader(object):
 											))
 									else:
 										logging.info("Scrolling in progress, ignoring request")
+								
+								# report media seen
+								if job.job_origin is Origin.INSTAGRAM:
+									actor.report_seen(items)
 						else:
 							logging.info("Job already in work in parallel worker. Redirecting job to upload worker.")
 							self.uploader.queue_task(job.to_upload_job())

warp_beacon/scraper/instagram/instagram.py

@@ -256,6 +256,7 @@ class InstagramScraper(ScraperAbstract):
 			"canonical_name": self.extract_canonical_name(media_info),
 			"media_type": JobType.VIDEO,
 			"last_pk": media_info.pk,
+			"last_id": media_info.id,
 			"media_info": {"duration": round(media_info.video_duration)}
 		}
 
@@ -270,7 +271,8 @@ class InstagramScraper(ScraperAbstract):
 			"local_media_path": self.rename_local_file(path),
 			"canonical_name": self.extract_canonical_name(media_info),
 			"media_type": JobType.IMAGE,
-			"last_pk": media_info.pk
+			"last_pk": media_info.pk,
+			"last_id": media_info.id
 		}
 
 	def download_story(self, story_info: Story) -> dict:
@@ -296,7 +298,14 @@ class InstagramScraper(ScraperAbstract):
 			media_type = JobType.VIDEO
 			media_info["duration"] = story_info.video_duration
 
-		return {"local_media_path": self.rename_local_file(path), "media_type": media_type, "media_info": media_info, "effective_url": effective_url}
+		return {
+			"local_media_path": self.rename_local_file(path),
+			"media_type": media_type,
+			"media_info": media_info,
+			"effective_url": effective_url,
+			"last_pk": story_info.pk,
+			"last_id": story_info.id
+		}
 
 	def download_stories(self, stories: list[Story]) -> dict:
 		chunks = []
@@ -306,21 +315,30 @@ class InstagramScraper(ScraperAbstract):
 				chunk.append(self.download_story(story_info=story))
 			chunks.append(chunk)
 
-		return {"media_type": JobType.COLLECTION, "save_items": True, "items": chunks}
+		return {
+			"media_type": JobType.COLLECTION,
+			"save_items": True,
+			"items": chunks
+		}
 
 	def download_album(self, media_info: Media) -> dict:
 		chunks = []
 		for media_chunk in Utils.chunker(media_info.resources, 10):
 			chunk = []
 			for media in media_chunk:
-				_media_info = self.download_hndlr(self.cl.media_info_v1, media.pk)
+				_media_info = self.download_hndlr(self.cl.media_info, media.pk)
 				if media.media_type == 1: # photo
 					chunk.append(self.download_photo(url=_media_info.thumbnail_url, media_info=_media_info))
 				elif media.media_type == 2: # video
 					chunk.append(self.download_video(url=_media_info.video_url, media_info=_media_info))
 			chunks.append(chunk)
 
-		return {"media_type": JobType.COLLECTION, "items": chunks}
+		return {
+			"media_type": JobType.COLLECTION,
+			"items": chunks,
+			"last_pk": media_info.pk,
+			"last_id": media_info.id
+		}
 
 	def extract_canonical_name(self, media: Media) -> str:
 		ret = ""
@@ -349,7 +367,7 @@ class InstagramScraper(ScraperAbstract):
 					#	"chat_id": self.job.chat_id,
 					#	"message_id": self.job.placeholder_message_id
 					#})
-					media_info = self.download_hndlr(self.cl.media_info_v1, media_id)
+					media_info = self.download_hndlr(self.cl.media_info, media_id)
 					logging.info("media_type is '%d', product_type is '%s'", media_info.media_type, media_info.product_type)
 					if media_info.media_type == 2 and media_info.product_type in ("clips", "ad"): # Reels
 						res.append(self.download_video(url=media_info.video_url, media_info=media_info))
@@ -357,10 +375,6 @@ class InstagramScraper(ScraperAbstract):
 						res.append(self.download_photo(url=media_info.thumbnail_url, media_info=media_info))
 					elif media_info.media_type == 8: # Album
 						res.append(self.download_album(media_info=media_info))
-					try:
-						self.cl.media_seen([media_info.id])
-					except Exception as e:
-						logging.warning("Failed to mark seen with id = '%s'", media_info.id, exc_info=e)
 				elif scrap_type == "story":
 					story_info = self.cl.story_info(media_id)
 					logging.info("media_type for story is '%d'", story_info.media_type)
@@ -460,4 +474,27 @@ class InstagramScraper(ScraperAbstract):
 				"report_type": ReportType.PROGRESS
 			}
 			self.status_pipe.send(msg)
-			self._download_progress_threshold += 5
+			self._download_progress_threshold += 5
+
+	def report_seen(self, items: dict) -> None:
+		try:
+			seen = []
+			for item in items:
+				if item["media_type"] == JobType.COLLECTION:
+					if item.get("last_id", None):
+						seen.append(item["last_id"])
+					if item.get("items", None):
+						for chunk in item["items"]:
+							for col_item in chunk:
+								last_id = col_item.get("last_id", None)
+								if last_id:
+									seen.append(last_id)
+				else:
+					last_id = item.get("last_id", None)
+					if last_id:
+						seen.append(last_id)
+
+			if seen:
+				self.download_hndlr(self.cl.media_seen, seen)
+		except Exception as e:
+			logging.error("Failed to report seen media!", exc_info=e)

warp_beacon/telegram/progress_bar.py

@@ -9,7 +9,7 @@ from pyrogram import Client
 from warp_beacon.telegram.types import ReportType
 
 class ProgressBar(object):
-	MAX_PROGRESS_RENDER_SIZE = 1_500_000 # 1 MB
+	MAX_PROGRESS_RENDER_SIZE = 500_000  # 500 kB
 
 	def __init__(self, client: Client) -> None:
 		self._next_threshold = 5

warp_beacon/uploader/__init__.py

@@ -4,6 +4,7 @@ from typing import Callable
 import asyncio
 import threading
 import multiprocessing
+from queue import Empty
 
 from warp_beacon.jobs.types import JobType
 from warp_beacon.jobs.upload_job import UploadJob
@@ -179,7 +180,7 @@ class AsyncUploader(object):
 							logging.info("No callback no call!!")
 					except Exception as e:
 						logging.exception(e)
-				except multiprocessing.Queue.empty:
+				except Empty:
 					pass
 			except Exception as e:
 				logging.error("Exception occurred inside upload worker!")

{warp_beacon-2.7.3.dist-info → warp_beacon-2.7.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.7.3
+Version: 2.7.5
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.7.3.dist-info → warp_beacon-2.7.5.dist-info}/RECORD

@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=zHOuiSy6iXgjI1Uya4AAc0Z69GIAz6P6Pp3YBK0PpoY,23
+warp_beacon/__version__.py,sha256=kvIAlwPLWbb1-76WKES6wpdLYyO-U4V_rglHAGDsFEw,23
 warp_beacon/warp_beacon.py,sha256=ADCR30uGXIsDrt9WoiI9Ghu2QtWs0qZIK6x3pQKM_B4,1109
 warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -22,7 +22,7 @@ warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tE
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scheduler/instagram_human.py,sha256=gqayCcn0xRYSNSFxeSpCGhBJyIwNAbk4Fnt1SCd61iA,11259
 warp_beacon/scheduler/scheduler.py,sha256=Bf4sGXjX75Dox3q-yzUHhagtzUAj3hl5GzfnZya-_io,4995
-warp_beacon/scraper/__init__.py,sha256=k3M0X_m5f7b_DbBB3Ahk62ewEYr5AkqJL0PJXf0G4mI,20140
+warp_beacon/scraper/__init__.py,sha256=KTmch6hUXuXSI7Br8yjgZHf2ET9NRBhcXhl2SQ6De48,20258
 warp_beacon/scraper/abstract.py,sha256=pWbaTu-gDZgi-iFjqMR_uGzPl5KLv-4gTdJ9w6cD4sk,3802
 warp_beacon/scraper/account_selector.py,sha256=vxIVesGPQKVtL3uSn5CFlZB1eAYowUN7lTeiGUFmtDk,10067
 warp_beacon/scraper/exceptions.py,sha256=EKwoF0oH2xZWbNU-v8DOaWK5skKwa3s1yTIBdlcfMpc,1452
@@ -31,7 +31,7 @@ warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5
 warp_beacon/scraper/utils.py,sha256=yL-nrdCOAXNZgmIFPurIYFHfPsxHDl8pqif6BYKrzJI,1218
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/captcha.py,sha256=9UYziuqB3Tsat_ET6ex-cnZDbi6yCnsXHSpmE8MuUHk,4651
-warp_beacon/scraper/instagram/instagram.py,sha256=cGInSO6UzmPcq2DokoL8_jmuIco1PBlVrafywDXazBg,17861
+warp_beacon/scraper/instagram/instagram.py,sha256=6ZEJAoAiB32BywE-On_kQdOoBe2iUWJ6wJqhuGAKmYk,18548
 warp_beacon/scraper/instagram/wb_instagrapi.py,sha256=M5NCtLwdUvByjmDBZMWljgB275R0LSBFblsGpapluD0,5968
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/youtube/abstract.py,sha256=lJCmtXYwEB8exTFr_uSA0MCyjVcoSdVHGDRxwZPvEv8,15276
@@ -47,14 +47,14 @@ warp_beacon/telegram/download_status.py,sha256=N-Qg13LVcPskyQNKG1lw50K1lhFtXu003
 warp_beacon/telegram/edit_message.py,sha256=6DX8eJV70MG_b79WbuJG55qLBs66boElzYdCr8zpMeI,5380
 warp_beacon/telegram/handlers.py,sha256=uvR6TPHSqdSxigp3wR-ewiE6t3TvVcbVLVcYGwkgD2s,9559
 warp_beacon/telegram/placeholder_message.py,sha256=wN9-BRiyrtHG-EvXtZkGJHt2CX71munQ57ITttjt0mw,6400
-warp_beacon/telegram/progress_bar.py,sha256=WLasAysPB_0buh-N6FOuDcz2QHiuaogC9ILAt0a59Lc,4996
+warp_beacon/telegram/progress_bar.py,sha256=dqMZFSqxRCOPPRlahkz0L37mqMhfr5rvg9bg-sqAP4Q,4997
 warp_beacon/telegram/progress_file_reader.py,sha256=e3equyNKlKs764AD-iE9QRsh3YDHTzP78Mx5tdvPPWs,969
 warp_beacon/telegram/types.py,sha256=Kvdng6uCF1HRoqQgGW1ZYYPJoVuYkFb-LDvMBbW5Hjk,89
 warp_beacon/telegram/utils.py,sha256=1Lq67aRylVJzbwSyvAgjPAGjJZFATkICvAj3TJGuJiM,4635
-warp_beacon/uploader/__init__.py,sha256=j3qcuKhpchseZLGzSsSiogqe6WdMbkK8d3I-ConhNRs,5687
-warp_beacon-2.7.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.7.3.dist-info/METADATA,sha256=fW2gRFXVMGTaRSR4EW37Anz92FKujSZKREco4n0bRX0,23214
-warp_beacon-2.7.3.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-warp_beacon-2.7.3.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.7.3.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
-warp_beacon-2.7.3.dist-info/RECORD,,
+warp_beacon/uploader/__init__.py,sha256=ewvR60k9osJxw_kb5U-TlZkCGR_574biq3w_aR1tjIU,5689
+warp_beacon-2.7.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.7.5.dist-info/METADATA,sha256=EAbVMEWj43GP-wOMTBoRFaqD7LSWS90OKImnddQxOMk,23214
+warp_beacon-2.7.5.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+warp_beacon-2.7.5.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.7.5.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
+warp_beacon-2.7.5.dist-info/RECORD,,