warp-beacon 2.6.83__tar.gz → 2.6.84__tar.gz

{warp_beacon-2.6.83/warp_beacon.egg-info → warp_beacon-2.6.84}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.83
+Version: 2.6.84
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.6.84/warp_beacon/__version__.py

@@ -0,0 +1,2 @@
+__version__ = "2.6.84"
+

{warp_beacon-2.6.83 → warp_beacon-2.6.84}/warp_beacon/scheduler/scheduler.py

@@ -13,13 +13,13 @@ import warp_beacon
 class IGScheduler(object):
 	state_file = "/var/warp_beacon/scheduler_state.json"
 	yt_sessions_dir = "/var/warp_beacon"
-	downloader = None
-	running = True
-	thread = None
-	event = None
-	state = {"remaining": randrange(8400, 26200), "yt_sess_exp": []}
 
 	def __init__(self, downloader: warp_beacon.scraper.AsyncDownloader) -> None:
+		self.downloader = None
+		self.running = True
+		self.thread = None
+		self.event = None
+		self.state = {"remaining": randrange(8400, 26200), "yt_sess_exp": []}
 		self.downloader = downloader
 		self.event = threading.Event()
 		self.handle_time_planning()
@@ -37,9 +37,7 @@ class IGScheduler(object):
 
 	def load_yt_sessions(self) -> None:
 		try:
-			# old versions migration
-			if not "yt_sess_exp" in self.state:
-				self.state["yt_sess_exp"] = []
+			self.state["yt_sess_exp"] = []
 			
 			for f in os.listdir(self.yt_sessions_dir):
 				if f.startswith("yt_session") and f.endswith(".json"):
@@ -134,6 +132,7 @@ class IGScheduler(object):
 		self.load_state()
 		while self.running:
 			try:
+				self.load_yt_sessions()
 				yt_expires = self.yt_nearest_expire()
 				ig_sched = self.state["remaining"]
 				min_val = min(yt_expires, ig_sched)

{warp_beacon-2.6.83 → warp_beacon-2.6.84}/warp_beacon/scraper/instagram/wb_instagrapi.py

@@ -43,7 +43,19 @@ class WBClient(Client):
 			raise TypeError("Progress callback must be callable")
 		self.progress_callback = callback
 
+	def adaptive_chunk_size(self, content_length: int) -> int:
+		if content_length < 100_000:
+			return 2048
+		elif content_length < 5_000_000:
+			return 8192
+		elif content_length < 100_000_000:
+			return 32768
+		else:
+			return 65536
+
 	def sanitize_instagram_url(self, url: str) -> str:
+		if "oh=" in url: # signed url, do not touch
+				return url
 		parsed = urlparse(url)
 		query = parse_qs(parsed.query)
 		filtered_query = {k: v for k, v in query.items() if k in self.essential_params}
@@ -51,7 +63,7 @@ class WBClient(Client):
 		return urlunparse(parsed._replace(query=new_query))
 
 	def video_download_by_url(self, url: str, filename: str = "", folder: Path = "") -> Path:
-		#url = self.sanitize_instagram_url(url)
+		url = self.sanitize_instagram_url(url)
 		fname = urlparse(url).path.rsplit("/", 1)[1]
 		filename = f"{filename}.{fname.rsplit('.', 1)[1]}" if filename else fname
 		path = Path(folder or Path.cwd()) / filename
@@ -78,7 +90,7 @@ class WBClient(Client):
 
 		downloaded = 0
 		with open(path, "wb") as f:
-			for chunk in response.iter_content(chunk_size=8192):
+			for chunk in response.iter_content(chunk_size=self.adaptive_chunk_size(content_length)):
 				if chunk:
 					f.write(chunk)
 					downloaded += len(chunk)
@@ -103,7 +115,7 @@ class WBClient(Client):
 	def photo_download_by_url(
 		self, url: str, filename: str = "", folder: Path = ""
 	) -> Path:
-		#url = self.sanitize_instagram_url(url)
+		url = self.sanitize_instagram_url(url)
 		fname = urlparse(url).path.rsplit("/", 1)[1]
 		filename = f"{filename}.{(filename, fname.rsplit('.', 1)[1]) if filename else fname}"
 		path = Path(folder) / filename
@@ -136,7 +148,7 @@ class WBClient(Client):
 		downloaded = 0
 		with open(path, "wb") as f:
 			response.raw.decode_content = True
-			for chunk in response.iter_content(chunk_size=4096):
+			for chunk in response.iter_content(chunk_size=self.adaptive_chunk_size(content_length)):
 				if chunk:
 					f.write(chunk)
 					downloaded += len(chunk)

{warp_beacon-2.6.83 → warp_beacon-2.6.84/warp_beacon.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warp_beacon
-Version: 2.6.83
+Version: 2.6.84
 Summary: Telegram bot for expanding external media links
 Home-page: https://github.com/sb0y/warp_beacon
 Author: Andrey Bagrintsev

warp_beacon-2.6.83/warp_beacon/__version__.py

@@ -1,2 +0,0 @@
-__version__ = "2.6.83"
-