warp-beacon 2.6.84__py3-none-any.whl → 2.6.86__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. warp_beacon/__version__.py +1 -1
  2. warp_beacon/scheduler/instagram_human.py +3 -3
  3. warp_beacon/scheduler/scheduler.py +4 -3
  4. warp_beacon/scraper/__init__.py +2 -12
  5. warp_beacon/scraper/abstract.py +11 -6
  6. warp_beacon/scraper/instagram/instagram.py +2 -2
  7. warp_beacon/scraper/youtube/abstract.py +4 -0
  8. {warp_beacon-2.6.84.dist-info → warp_beacon-2.6.86.dist-info}/METADATA +1 -1
  9. {warp_beacon-2.6.84.dist-info → warp_beacon-2.6.86.dist-info}/RECORD +13 -13
  10. {warp_beacon-2.6.84.dist-info → warp_beacon-2.6.86.dist-info}/WHEEL +1 -1
  11. {warp_beacon-2.6.84.dist-info → warp_beacon-2.6.86.dist-info}/entry_points.txt +0 -0
  12. {warp_beacon-2.6.84.dist-info → warp_beacon-2.6.86.dist-info}/licenses/LICENSE +0 -0
  13. {warp_beacon-2.6.84.dist-info → warp_beacon-2.6.86.dist-info}/top_level.txt +0 -0
warp_beacon/__version__.py CHANGED
@@ -1,2 +1,2 @@
1
- __version__ = "2.6.84"
1
+ __version__ = "2.6.86"
2
2
 
warp_beacon/scheduler/instagram_human.py CHANGED
@@ -17,7 +17,7 @@ class InstagramHuman(object):
17
17
  self.operations_count = 0
18
18
 
19
19
  def watch_content(self, media: list) -> None:
20
- for m in media[:random.randint(1, 3)]:
20
+ for m in media[:random.randint(1, 15)]:
21
21
  try:
22
22
  logging.info("Wathing content with pk '%s'", str(m.pk))
23
23
  content = self.scrapler.cl.media_info_v1(m.pk)
@@ -34,7 +34,7 @@ class InstagramHuman(object):
34
34
  timeline_initialized = True
35
35
  self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
36
36
  logging.info("Starting to watch related reels with media_pk '%d'", last_pk)
37
- media = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 10), last_media_pk=last_pk)
37
+ media = self.scrapler.download_hndlr(self.scrapler.cl.reels, amount=random.randint(4, 15), last_media_pk=last_pk)
38
38
  self.operations_count += 1
39
39
  self.watch_content(media)
40
40
 
@@ -43,7 +43,7 @@ class InstagramHuman(object):
43
43
  if not timeline_initialized:
44
44
  self.scrapler.timeline_cursor = self.scrapler.download_hndlr(self.scrapler.cl.get_timeline_feed, reason="cold_start_fetch")
45
45
  logging.info("Starting to explore reels with media_pk '%d'", last_pk)
46
- media = self.scrapler.download_hndlr(self.scrapler.cl.explore_reels, amount=random.randint(4, 10), last_media_pk=last_pk)
46
+ media = self.scrapler.download_hndlr(self.scrapler.cl.explore_reels, amount=random.randint(4, 15), last_media_pk=last_pk)
47
47
  self.operations_count += 1
48
48
  self.watch_content(media)
49
49
 
warp_beacon/scheduler/scheduler.py CHANGED
@@ -15,10 +15,8 @@ class IGScheduler(object):
15
15
  yt_sessions_dir = "/var/warp_beacon"
16
16
 
17
17
  def __init__(self, downloader: warp_beacon.scraper.AsyncDownloader) -> None:
18
- self.downloader = None
19
18
  self.running = True
20
19
  self.thread = None
21
- self.event = None
22
20
  self.state = {"remaining": randrange(8400, 26200), "yt_sess_exp": []}
23
21
  self.downloader = downloader
24
22
  self.event = threading.Event()
@@ -108,6 +106,9 @@ class IGScheduler(object):
108
106
 
109
107
  def validate_yt_session(self) -> bool:
110
108
  try:
109
+ if self.downloader.yt_validate_event.is_set():
110
+ return True
111
+ self.downloader.yt_validate_event.set()
111
112
  logging.info("Setting YT validate task ...")
112
113
  self.downloader.queue_task(warp_beacon.jobs.download_job.DownloadJob.build(
113
114
  session_validation=True,
@@ -147,7 +148,7 @@ class IGScheduler(object):
147
148
  self.handle_time_planning()
148
149
 
149
150
  start_time = time.time()
150
- logging.info("Next scheduler activity in '%s' seconds", int(min_val))
151
+ logging.info("Next scheduler activity in '%d' seconds", int(min_val))
151
152
  logging.info("IG timeout '%d' secs", int(self.state["remaining"]))
152
153
  self.event.wait(timeout=min_val)
153
154
  self.event.clear()
warp_beacon/scraper/__init__.py CHANGED
@@ -33,18 +33,6 @@ PROXY_FILE = os.environ.get("PROXY_FILE", default="/var/warp_beacon/proxies.json
33
33
  class AsyncDownloader(object):
34
34
  TG_FILE_LIMIT = 2147483648 # 2 GiB
35
35
  __JOE_BIDEN_WAKEUP = None
36
- workers = None
37
- allow_loop = None
38
- job_queue = None
39
- uploader = None
40
- workers_count = 0
41
- auth_event = None
42
- manager = None
43
- acc_selector = None
44
- scheduler = None
45
- scrolling_now = None
46
- process_context = None
47
- status_pipe = None
48
36
 
49
37
  def __init__(self, uploader: AsyncUploader, pipe_connection: multiprocessing.connection.Connection, workers_count: int) -> None:
50
38
  self.workers = []
@@ -58,6 +46,7 @@ class AsyncDownloader(object):
58
46
  self.uploader = uploader
59
47
  self.workers_count = workers_count
60
48
  self.status_pipe = pipe_connection
49
+ self.yt_validate_event = multiprocessing.Event()
61
50
  if os.environ.get("TG_PREMIUM", default="false") == "true":
62
51
  self.TG_FILE_LIMIT = 4294967296 # 4 GiB
63
52
 
@@ -160,6 +149,7 @@ class AsyncDownloader(object):
160
149
  actor.request_yt_auth = self.request_yt_auth
161
150
  actor.auth_event = self.auth_event
162
151
  actor.status_pipe = self.status_pipe
152
+ actor.yt_validate_event = self.yt_validate_event
163
153
  # job retry loop
164
154
  while self.allow_loop.value == 1:
165
155
  try:
warp_beacon/scraper/abstract.py CHANGED
@@ -1,23 +1,28 @@
1
- import os
2
- import time
3
- import pathlib
4
- from abc import ABC, abstractmethod
5
- from typing import Callable, Union
6
1
  import logging
7
2
  import multiprocessing
8
3
  import multiprocessing.connection
4
+ import os
9
5
  import socket
10
- import requests.packages.urllib3.util.connection as urllib3_cn
6
+ import pathlib
7
+ import time
8
+ from abc import ABC, abstractmethod
9
+ from typing import TYPE_CHECKING, Callable, Union
11
10
 
12
11
  from PIL import Image
13
12
  from pillow_heif import register_heif_opener
14
13
 
14
+ import requests.packages.urllib3.util.connection as urllib3_cn
15
+
16
+ if TYPE_CHECKING:
17
+ from multiprocessing.synchronize import Event as EventType
18
+
15
19
  class ScraperAbstract(ABC):
16
20
  def __init__(self, account: tuple, proxy: dict = None) -> None:
17
21
  self.original_gai_family = None
18
22
  self.send_message_to_admin_func: Callable = lambda: None
19
23
  self.request_yt_auth: Callable = lambda: None
20
24
  self.status_pipe: multiprocessing.connection.Connection = None
25
+ self.yt_validate_event: EventType = None
21
26
  self.auth_event = None
22
27
  self.account = None
23
28
  self.account_index = 0
warp_beacon/scraper/instagram/instagram.py CHANGED
@@ -12,9 +12,9 @@ import logging
12
12
  import email
13
13
  import imaplib
14
14
  import json
15
+ from urllib.parse import urljoin, urlparse
15
16
  import requests
16
17
  import urllib3
17
- from urllib.parse import urljoin, urlparse
18
18
 
19
19
  from instagrapi import exceptions
20
20
  from instagrapi.exceptions import UnknownError as IGUnknownError
@@ -341,7 +341,7 @@ class InstagramScraper(ScraperAbstract):
341
341
  #})
342
342
  media_info = self.download_hndlr(self.cl.media_info_v1, media_id)
343
343
  logging.info("media_type is '%d', product_type is '%s'", media_info.media_type, media_info.product_type)
344
- if media_info.media_type == 2 and media_info.product_type == "clips": # Reels
344
+ if media_info.media_type == 2 and media_info.product_type in ("clips", "ad"): # Reels
345
345
  res.append(self.download_video(url=media_info.video_url, media_info=media_info))
346
346
  elif media_info.media_type == 1: # Photo
347
347
  res.append(self.download_photo(url=media_info.thumbnail_url, media_info=media_info))
warp_beacon/scraper/youtube/abstract.py CHANGED
@@ -63,6 +63,10 @@ class YoutubeAbstract(ScraperAbstract):
63
63
  logging.error("Failed to refresh Youtube session!")
64
64
  logging.exception(e)
65
65
 
66
+ # avoid task acquiring in parallel worker
67
+ self.yt_validate_event.clear()
68
+ #time.sleep(35)
69
+
66
70
  return 0
67
71
 
68
72
  def get_video_id(self, url: str) -> Optional[str]:
{warp_beacon-2.6.84.dist-info → warp_beacon-2.6.86.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: warp_beacon
3
- Version: 2.6.84
3
+ Version: 2.6.86
4
4
  Summary: Telegram bot for expanding external media links
5
5
  Home-page: https://github.com/sb0y/warp_beacon
6
6
  Author: Andrey Bagrintsev
{warp_beacon-2.6.84.dist-info → warp_beacon-2.6.86.dist-info}/RECORD RENAMED
@@ -4,7 +4,7 @@ var/warp_beacon/accounts.json,sha256=OsXdncs6h88xrF_AP6_WDCK1waGBn9SR-uYdIeK37GM
4
4
  var/warp_beacon/placeholder.gif,sha256=cE5CGJVaop4Sx21zx6j4AyoHU0ncmvQuS2o6hJfEH88,6064
5
5
  var/warp_beacon/proxies.json,sha256=VnjlQDXumOEq72ZFjbh6IqHS1TEHqn8HPYAZqWCeSIA,95
6
6
  warp_beacon/__init__.py,sha256=_rThNODmz0nDp_n4mWo_HKaNFE5jk1_7cRhHyYaencI,163
7
- warp_beacon/__version__.py,sha256=sVfzhpFl0Ti_WQWBe-4XO6fhB_2n-4RIx64CMN0X7PY,24
7
+ warp_beacon/__version__.py,sha256=3jfGBfZMq-huVJ0EkVBkNb6ToGe-hxHYgbahntg0uzI,24
8
8
  warp_beacon/warp_beacon.py,sha256=ED43vNzdjDUJ_9qLCbri0bjWLWEJ69BENGj9i7G6AvM,342
9
9
  warp_beacon/yt_auth.py,sha256=GUTKqYr_tzDC-07Lx_ahWXSag8EyLxXBUnQbDBIkEmk,6022
10
10
  warp_beacon/compress/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -20,10 +20,10 @@ warp_beacon/mediainfo/audio.py,sha256=ous88kwQj4bDIChN5wnGil5LqTs0IQHH0d-nyrL0-Z
20
20
  warp_beacon/mediainfo/silencer.py,sha256=qxMuViOoVwUYb60uCVvqHiGrqByR1_4_rqMT-XdMkwc,1813
21
21
  warp_beacon/mediainfo/video.py,sha256=UBZrhTN5IDI-aYu6tsJEILo9nFkjHhkldGVFmvV7tEI,2480
22
22
  warp_beacon/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
23
- warp_beacon/scheduler/instagram_human.py,sha256=Li8R8Fm9sZCXCJnS4RrQnQJTsPlBPENQowRe_23Bu-Y,8207
24
- warp_beacon/scheduler/scheduler.py,sha256=C-19tIb6iiHfboKNPDjg9tzhJQ3mNEbHywW9foe7nKw,4931
25
- warp_beacon/scraper/__init__.py,sha256=buXmGcJnwCi0qGZAh3srcSbww0XvKyTACp_DW11e86Q,20225
26
- warp_beacon/scraper/abstract.py,sha256=hRyBbdZ5HK7QyyXlbFAgk3FZe2SIil0WBZ9vJziZv_s,3554
23
+ warp_beacon/scheduler/instagram_human.py,sha256=4Cik8bX9QYb3rtJ51matR78xxg8mXpaO9d5tGYVsIlE,8208
24
+ warp_beacon/scheduler/scheduler.py,sha256=cr3oKANu_iOw4vCLzr8Mh0HdZIqYx3I8i4ohfoQkw2U,4995
25
+ warp_beacon/scraper/__init__.py,sha256=2-h_HGb7Gp5H2-qyfONaubTOv1lzR4SWfjLV7iX9cP0,20103
26
+ warp_beacon/scraper/abstract.py,sha256=_Cr_MaSQ56Wp6n00PcqiM2WDa5DlRKXUnQ9C15bPObw,3692
27
27
  warp_beacon/scraper/account_selector.py,sha256=VmXH8SHZqQ6a2mrt2agzDWepE3v46QrFLL3OqVwJ2A4,9731
28
28
  warp_beacon/scraper/exceptions.py,sha256=EKwoF0oH2xZWbNU-v8DOaWK5skKwa3s1yTIBdlcfMpc,1452
29
29
  warp_beacon/scraper/fail_handler.py,sha256=zcPK3ZVEsu6JmHYcWP7L3naTRK3gWFVRkpP84VBOtJs,964
@@ -31,10 +31,10 @@ warp_beacon/scraper/link_resolver.py,sha256=Rc9ZuMyOo3iPywDHwjngy-WRQ2SXhJwxcg-5
31
31
  warp_beacon/scraper/utils.py,sha256=xJfCVhiLjVPoFVupE10sTzX7UpgNWbx2EeYIsoYXGDk,43
32
32
  warp_beacon/scraper/instagram/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
33
33
  warp_beacon/scraper/instagram/captcha.py,sha256=9UYziuqB3Tsat_ET6ex-cnZDbi6yCnsXHSpmE8MuUHk,4651
34
- warp_beacon/scraper/instagram/instagram.py,sha256=3KhxVQSL3lzovu9hjMA83sCv87u4TTDMfDQrxogoWak,17179
34
+ warp_beacon/scraper/instagram/instagram.py,sha256=Z2L9E5dvY2UQGStK1C7MAFmnEPEfH7DwVUObeOc5iNg,17187
35
35
  warp_beacon/scraper/instagram/wb_instagrapi.py,sha256=Dh0SlCZcrgLC4IsIqKUV3xszQXKlrdsll0nCAxoiXHI,5289
36
36
  warp_beacon/scraper/youtube/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
37
- warp_beacon/scraper/youtube/abstract.py,sha256=K2ovycq1lkx0a2R1FddO3B0IcaVsVzni_fdXzajGdwg,13134
37
+ warp_beacon/scraper/youtube/abstract.py,sha256=WLbQKtrKSwpZUQlpjL2XI7z8XxoRCtVxrRjSXFJmXuM,13230
38
38
  warp_beacon/scraper/youtube/music.py,sha256=5AeSBQyUgVCJT2hoBCV2WvlyuV9US09SYJhmBG_P9F8,2755
39
39
  warp_beacon/scraper/youtube/shorts.py,sha256=1GtoYUlxAwcgSQcn80u5ehNJytH5AN5dPOicmX-XD8E,1705
40
40
  warp_beacon/scraper/youtube/youtube.py,sha256=x9v9p1coA9TvBhxjNAofGu4UBkAEdYPE2ePRnU-5tK0,7233
@@ -52,9 +52,9 @@ warp_beacon/telegram/progress_file_reader.py,sha256=e3equyNKlKs764AD-iE9QRsh3YDH
52
52
  warp_beacon/telegram/types.py,sha256=Kvdng6uCF1HRoqQgGW1ZYYPJoVuYkFb-LDvMBbW5Hjk,89
53
53
  warp_beacon/telegram/utils.py,sha256=1Lq67aRylVJzbwSyvAgjPAGjJZFATkICvAj3TJGuJiM,4635
54
54
  warp_beacon/uploader/__init__.py,sha256=j3qcuKhpchseZLGzSsSiogqe6WdMbkK8d3I-ConhNRs,5687
55
- warp_beacon-2.6.84.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
56
- warp_beacon-2.6.84.dist-info/METADATA,sha256=h1DIdxgSyWRQ3H2kNwxfugeIvva3rCmBz_JW1VFlpwg,22706
57
- warp_beacon-2.6.84.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
58
- warp_beacon-2.6.84.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
59
- warp_beacon-2.6.84.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
60
- warp_beacon-2.6.84.dist-info/RECORD,,
55
+ warp_beacon-2.6.86.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
56
+ warp_beacon-2.6.86.dist-info/METADATA,sha256=BQWnSiwqrFa8cs6CBD9xnZwE_XBi0Bq3mRay_GpIPdQ,22706
57
+ warp_beacon-2.6.86.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
58
+ warp_beacon-2.6.86.dist-info/entry_points.txt,sha256=eSB61Rb89d56WY0O-vEIQwkn18J-4CMrJcLA_R_8h3g,119
59
+ warp_beacon-2.6.86.dist-info/top_level.txt,sha256=5YQRN46STNg81V_3jdzZ6bftkMxhe1hTPSFvJugDu84,1405
60
+ warp_beacon-2.6.86.dist-info/RECORD,,
{warp_beacon-2.6.84.dist-info → warp_beacon-2.6.86.dist-info}/WHEEL RENAMED
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (80.3.1)
2
+ Generator: setuptools (80.4.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5