warp-beacon 2.6.17__py3-none-any.whl → 2.6.18__py3-none-any.whl

warp_beacon/__version__.py

@@ -1,2 +1,2 @@
-__version__ = "2.6.17"
+__version__ = "2.6.18"
 

warp_beacon/jobs/abstract.py

@@ -47,6 +47,8 @@ class JobSettings(TypedDict):
 	message_leftover: str
 	replay: bool
 	short_text: bool
+	scroll_content: bool
+	last_pk: int
 
 class AbstractJob(ABC):
 	job_id: uuid.UUID = None
@@ -86,6 +88,8 @@ class AbstractJob(ABC):
 	message_leftover: str = ""
 	replay: bool = False
 	short_text: bool = False
+	scroll_content: bool = False
+	last_pk: int = 0
 
 	def __init__(self, **kwargs: Unpack[JobSettings]) -> None:
 		if kwargs:

warp_beacon/scheduler/instagram_human.py

@@ -16,6 +16,32 @@ class InstagramHuman(object):
 		self.scrapler = scrapler
 		self.operations_count = 0
 
+	def watch_content(self, media: list) -> None:
+		for m in media[:random.randint(2, 4)]:
+			try:
+				logging.info("Wathing content with pk '%d'", m.pk)
+				content = self.scrapler.cl.media_info(m.pk)
+				logging.info("Watched content with id '%d'", content.pk)
+				self.operations_count += 1
+				time.sleep(random.uniform(2, 5))
+			except Exception as e:
+				logging.warning("Exception while watching content")
+				logging.exception(e)
+
+	def scroll_content(self, last_pk: int) -> None:
+		if random.random() > 0.2:
+			logging.info("Starting to watch related reels with media_pk '%d'", last_pk)
+			media = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 10), last_media_pk=last_pk)
+			self.operations_count += 1
+			self.watch_content(media)
+		
+		if random.random() > 0.3:
+			time.sleep(random.uniform(2, 5))
+			logging.info("Starting to explore reels with media_pk '%d'", last_pk)
+			media = self.scrapler.download_hndlr(self.scrapler.cl.explore_reels, amount=random.randint(4, 10), last_media_pk=last_pk)
+			self.operations_count += 1
+			self.watch_content(media)
+
 	def simulate_activity(self) -> None:
 		now = datetime.now()
 		hour = now.hour

warp_beacon/scraper/__init__.py

@@ -147,6 +147,13 @@ class AsyncDownloader(object):
 							# job retry loop
 							while self.allow_loop.value == 1:
 								try:
+									if job.scroll_content and job.last_pk and job.job_origin is Origin.INSTAGRAM:
+										logging.info("Scrolling relative content with pk '%s'", job.last_pk)
+										operations = actor.scroll_content(last_pk=job.last_pk)
+										if operations:
+											selector.inc_ig_request_count(amount=operations)
+										logging.info("Scrolling done")
+										break
 									if job.session_validation and job.job_origin in (Origin.INSTAGRAM, Origin.YOUTUBE):
 										if job.job_origin is Origin.INSTAGRAM:
 											if selector.get_ig_request_count() >= int(os.environ.get("IG_REQUESTS_PER_ACCOUNT", default="10")):
@@ -361,6 +368,12 @@ class AsyncDownloader(object):
 										)
 									else:
 										self.uploader.queue_task(upload_job)
+									# watch related reels to simulate human
+									if item.get("last_pk", 0) and "reel/" in job.url:
+										self.queue_task(DownloadJob.build(
+											scroll_content=True,
+											last_pk=int(item.get("last_pk", 0))
+										))
 						else:
 							logging.info("Job already in work in parallel worker. Redirecting job to upload worker.")
 							self.uploader.queue_task(job.to_upload_job())
@@ -384,9 +397,6 @@ class AsyncDownloader(object):
 						logging.error("Error inside download worker!")
 						logging.exception(e)
 						self.notify_task_failed(job)
-					finally:
-						if actor:
-							actor.restore_gai()
 				except Empty:
 					pass
 			except Exception as e:

warp_beacon/scraper/abstract.py

@@ -3,6 +3,7 @@ import pathlib
 
 import socket
 import requests.packages.urllib3.util.connection as urllib3_cn
+import multiprocessing
 
 from abc import ABC, abstractmethod
 from typing import Callable, Union
@@ -22,16 +23,19 @@ class ScraperAbstract(ABC):
 	proxy = None
 
 	def __init__(self, account: tuple, proxy: dict=None) -> None:
+		self._gai_lock = multiprocessing.Lock()
 		self.account_index = account[0]
 		self.account = account[1]
 		self.proxy = proxy
 		if self.account.get("force_ipv6", False):
 			self.force_ipv6()
-
-	def __del__(self) -> None:
-		if self.account.get("force_ipv6", False):
+		else:
 			self.restore_gai()
 
+	#def __del__(self) -> None:
+	#	if self.account.get("force_ipv6", False):
+	#		self.restore_gai()
+
 	@abstractmethod
 	def download(self, url: str) -> bool:
 		raise NotImplementedError
@@ -86,13 +90,15 @@ class ScraperAbstract(ABC):
 			if urllib3_cn.HAS_IPV6:
 				family = socket.AF_INET6 # force ipv6 only if it is available
 			return family
-		if self.original_gai_family is None:
-			self.original_gai_family = urllib3_cn.allowed_gai_family
-		logging.info("Forcing IPv6 ...")
-		urllib3_cn.allowed_gai_family = allowed_gai_family
+		with self._gai_lock:
+			if self.original_gai_family is None:
+				self.original_gai_family = urllib3_cn.allowed_gai_family
+			logging.info("Forcing IPv6 ...")
+			urllib3_cn.allowed_gai_family = allowed_gai_family
 
 	def restore_gai(self) -> None:
-		if self.original_gai_family:
-			logging.info("Restoring normal IP stack ...")
-			urllib3_cn.allowed_gai_family = self.original_gai_family
-			self.original_gai_family = None
+		with self._gai_lock:
+			if self.original_gai_family:
+				logging.info("Restoring normal IP stack ...")
+				urllib3_cn.allowed_gai_family = self.original_gai_family
+				self.original_gai_family = None

warp_beacon/scraper/instagram/instagram.py

@@ -109,6 +109,14 @@ class InstagramScraper(ScraperAbstract):
 		inst_human.simulate_activity()
 		self.safe_write_session()
 		return inst_human.operations_count
+	
+	def scroll_content(self, last_pk: int) -> None:
+		from warp_beacon.scheduler.instagram_human import InstagramHuman
+		self.load_session()
+		inst_human = InstagramHuman(self)
+		inst_human.scroll_content(last_pk)
+		self.safe_write_session()
+		return inst_human.operations_count
 
 	def scrap(self, url: str) -> tuple[str]:
 		self.load_session()
@@ -208,7 +216,8 @@ class InstagramScraper(ScraperAbstract):
 		self.cl.request_timeout = int(os.environ.get("IG_REQUEST_TIMEOUT", default=60))
 		path = self.download_hndlr(self.cl.video_download_by_url, url, folder='/tmp')
 		return {"local_media_path": str(path), "canonical_name": self.extract_canonical_name(media_info), \
-			"media_type": JobType.VIDEO, "media_info": {"duration": round(media_info.video_duration)}}
+			"media_type": JobType.VIDEO, "media_info": {"duration": round(media_info.video_duration), \
+			"last_pk": media_info.pk}}
 
 	def download_photo(self, url: str, media_info: Media) -> dict:
 		path = str(self.download_hndlr(self.cl.photo_download_by_url, url, folder='/tmp'))
@@ -217,7 +226,7 @@ class InstagramScraper(ScraperAbstract):
 			path = InstagramScraper.convert_webp_to_png(path)
 		if ".heic" in path_lowered:
 			path = InstagramScraper.convert_heic_to_png(path)
-		return {"local_media_path": path, "canonical_name": self.extract_canonical_name(media_info), "media_type": JobType.IMAGE}
+		return {"local_media_path": path, "canonical_name": self.extract_canonical_name(media_info), "media_type": JobType.IMAGE, "last_pk": media_info.pk}
 	
 	def download_story(self, story_info: Story) -> dict:
 		path, media_type, media_info = "", JobType.UNKNOWN, {}
@@ -259,7 +268,7 @@ class InstagramScraper(ScraperAbstract):
 		for media_chunk in Utils.chunker(media_info.resources, 10):
 			chunk = []
 			for media in media_chunk:
-				_media_info = self.download_hndlr(self.cl.media_info, media.pk)
+				_media_info = self.download_hndlr(self.cl.media_info, media.pk, use_cache=False)
 				if media.media_type == 1: # photo
 					chunk.append(self.download_photo(url=_media_info.thumbnail_url, media_info=_media_info))
 				elif media.media_type == 2: # video
@@ -287,7 +296,7 @@ class InstagramScraper(ScraperAbstract):
 			try:
 				scrap_type, media_id = self.scrap(job.url)
 				if scrap_type == "media":
-					media_info = self.download_hndlr(self.cl.media_info, media_id)
+					media_info = self.download_hndlr(self.cl.media_info, media_id, use_cache=False)
 					logging.info("media_type is '%d', product_type is '%s'", media_info.media_type, media_info.product_type)
 					if media_info.media_type == 2 and media_info.product_type == "clips": # Reels
 						res.append(self.download_video(url=media_info.video_url, media_info=media_info))

{warp_beacon-2.6.17.dist-info → warp_beacon-2.6.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.17
+Version: 2.6.18
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

{warp_beacon-2.6.17.dist-info → warp_beacon-2.6.18.dist-info}/RECORD

@@ -4,13 +4,13 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
 var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
 var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
 warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
-warp_beacon/__version__.py,sha256=a1aXAI5iMpUmb-fkAT_xmK2FejqLr_zPFFJYf9nelUY,24
+warp_beacon/__version__.py,sha256=Kin5y99GED8zaOzHXCgrFE-8X5r_9lqAz-PnEwlbvZ8,24
 warp_beacon/warp_beacon.py,sha256=ED43vNzdjDUJ_9qLCbri0bjWLWEJ69BENGj9i7G6AvM,342
 warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
 warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/compress/video.py,sha256=_PDMVYCyzLYxHv1uZmmzGcG_8rjaZr7BTXsXTTy_oS4,2846
 warp_beacon/jobs/__init__.py,sha256=ED8_tPle4iL4kqNW0apAVkgNQtRRTnYfAJwBjO1g0JY,180
-warp_beacon/jobs/abstract.py,sha256=IF2bKx2DuuH_AP07YAvFxxqnuxjIwg2Glsr8TpWX6Z0,3179
+warp_beacon/jobs/abstract.py,sha256=x8shgG1So1g-Yqu_uzij7yuqherJTIhVhdslOrq69Z4,3263
 warp_beacon/jobs/download_job.py,sha256=tz_4rKEqnJ9yrtHeDwPGjxoDrePrGhhjCTG5CK5rr-Q,817
 warp_beacon/jobs/types.py,sha256=Ae8zINgbs7cOcYkYoOCOACA7duyhnIGMQAJ_SJB1QRQ,176
 warp_beacon/jobs/upload_job.py,sha256=_ul4psPej1jLEs-BMcMR80GbXDSmm38jE9yoZtecclY,741
@@ -20,17 +20,17 @@ warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-Z
 warp_beacon/mediainfo/silencer.py,sha256=qxMuViOoVwUYb60uCVvqHiGrqByR1_4_rqMT-XdMkwc,1813
 warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tEI,2480
 warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-warp_beacon/scheduler/instagram_human.py,sha256=buF6rJzp8gVZvHBAXEkAL-uJjm5iQkEWnfPgwM2oH8g,6276
+warp_beacon/scheduler/instagram_human.py,sha256=yoOcWlQQVfvJn3nD5u_VnO9jn0e_OWRvIGY9kLjPFUk,7369
 warp_beacon/scheduler/scheduler.py,sha256=9OCh7Ta4wY_aTHGAOOZmaKXg56Ftx1N_aV1g6E3ZLKA,4941
-warp_beacon/scraper/__init__.py,sha256=x5z9NrK_CPXITOR9Uus3U6DvytNThvGt2yDRGlyakJo,18566
-warp_beacon/scraper/abstract.py,sha256=CiOyKCxVYWhPnOUpLAVIRNuHBftN6gmxqATdhjzkaS4,2852
+warp_beacon/scraper/__init__.py,sha256=Pcgg6Dft77_4UM4f4jUQcmfkuf1-MdLhz6tD1swErFo,19127
+warp_beacon/scraper/abstract.py,sha256=HZCglECO13XN3Yjvgt6n9JirxZ1A-8tsE8RudihC4HE,3004
 warp_beacon/scraper/account_selector.py,sha256=wm8XO020QSvJO43TG7G06a17G0lFtNdx17V8mC8vKbs,7783
 warp_beacon/scraper/exceptions.py,sha256=EKwoF0oH2xZWbNU-v8DOaWK5skKwa3s1yTIBdlcfMpc,1452
 warp_beacon/scraper/fail_handler.py,sha256=zcPK3ZVEsu6JmHYcWP7L3naTRK3gWFVRkpP84VBOtJs,964
 warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5ripx7tM,2447
 warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/instagram/captcha.py,sha256=9UYziuqB3Tsat_ET6ex-cnZDbi6yCnsXHSpmE8MuUHk,4651
-warp_beacon/scraper/instagram/instagram.py,sha256=Q08kaTYa1rdQehAdgGftDZog0LAm_nRzErjgIne6KDo,14677
+warp_beacon/scraper/instagram/instagram.py,sha256=Ri0FrwnBQDayRORSF25awtv9Pxdt57MaWvTAXS5QUEo,15046
 warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warp_beacon/scraper/youtube/abstract.py,sha256=QUSHR3aJjIykuRlBpGnmw3tq3m82AQeeZeWQUdJRvgQ,12538
 warp_beacon/scraper/youtube/music.py,sha256=5AeSBQyUgVCJT2hoBCV2WvlyuV9US09SYJhmBG_P9F8,2755
@@ -45,9 +45,9 @@ warp_beacon/telegram/handlers.py,sha256=uvR6TPHSqdSxigp3wR-ewiE6t3TvVcbVLVcYGwkg
 warp_beacon/telegram/placeholder_message.py,sha256=wN9-BRiyrtHG-EvXtZkGJHt2CX71munQ57ITttjt0mw,6400
 warp_beacon/telegram/utils.py,sha256=1Lq67aRylVJzbwSyvAgjPAGjJZFATkICvAj3TJGuJiM,4635
 warp_beacon/uploader/__init__.py,sha256=j3qcuKhpchseZLGzSsSiogqe6WdMbkK8d3I-ConhNRs,5687
-warp_beacon-2.6.17.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warp_beacon-2.6.17.dist-info/METADATA,sha256=oWESJ8WFz9XTMXake_tV2ezRui08bIGVwi9b7m3491c,22706
-warp_beacon-2.6.17.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
-warp_beacon-2.6.17.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
-warp_beacon-2.6.17.dist-info/top_level.txt,sha256=4ML0-mXsezLtRXyxQUntL_ktc5HX9npTeQWzvV8kFvA,1161
-warp_beacon-2.6.17.dist-info/RECORD,,
+warp_beacon-2.6.18.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warp_beacon-2.6.18.dist-info/METADATA,sha256=NNHhNDqvlyumpcAPkopaZlIVn9o8LBnoHmoM9DyDnZE,22706
+warp_beacon-2.6.18.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
+warp_beacon-2.6.18.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
+warp_beacon-2.6.18.dist-info/top_level.txt,sha256=4ML0-mXsezLtRXyxQUntL_ktc5HX9npTeQWzvV8kFvA,1161
+warp_beacon-2.6.18.dist-info/RECORD,,

{warp_beacon-2.6.17.dist-info → warp_beacon-2.6.18.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.0.0)
+Generator: setuptools (80.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any