ofscraper 3.10.dev1__tar.gz → 3.10.dev2__tar.gz

{ofscraper-3.10.dev1 → ofscraper-3.10.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ofscraper
-Version: 3.10.dev1
+Version: 3.10.dev2
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net

{ofscraper-3.10.dev1 → ofscraper-3.10.dev2}/ofscraper/classes/sessionmanager.py

@@ -40,11 +40,13 @@ def is_rate_limited(exception):
 
 
 class SessionSleep:
-    def __init__(self, sleep=None):
-        self._sleep = sleep
-        self._init_sleep = sleep
+    def __init__(self, sleep=None,difmin=None):
+        self._sleep = None
+        self._init_sleep = sleep 
         self._last_date = arrow.now()
+        self._difmin=difmin if difmin!=None else constants.getattr("SESSION_SLEEP_INCREASE_TIME_DIFF")
         self._alock = asyncio.Lock()
+    
 
     def reset_sleep(self):
         self._sleep = self._init_sleep
@@ -56,13 +58,12 @@ class SessionSleep:
 
     def toomany_req(self):
         log = logging.getLogger("shared")
-        dif_min = constants.getattr("SESSION_SLEEP_INCREASE_TIME_DIFF")
-        if self._sleep is None:
-            self._sleep = constants.getattr("SESSION_SLEEP_INIT")
+        if not self._sleep:
+            self._sleep = self._init_sleep if self._init_sleep else constants.getattr("SESSION_SLEEP_INIT")
             log.debug(f"too many req => setting sleep to init [{self._sleep} seconds]")
-        elif arrow.now().float_timestamp - self._last_date.float_timestamp < dif_min:
+        elif arrow.now().float_timestamp - self._last_date.float_timestamp < self._difmin:
             log.debug(
-                f"too many req => not changing sleep [{self._sleep} seconds] because last call less than {dif_min} seconds"
+                f"too many req => not changing sleep [{self._sleep} seconds] because last call less than {self._difmin} seconds"
             )
             return self._sleep
         else:
@@ -357,6 +358,7 @@ class sessionManager:
                 except Exception as E:
                     if is_rate_limited(E):
                         sleeper.toomany_req()
+
                     log.traceback_(E)
                     log.traceback_(traceback.format_exc())
                     sync_sem.release()

{ofscraper-3.10.dev1 → ofscraper-3.10.dev2}/ofscraper/commands/scraper/actions/like.py

@@ -112,18 +112,24 @@ def _like(model_id, ids: list, like_action: bool):
 
             [tasks.append(functools.partial(like_func, c, id, model_id)) for id in ids]
             count = 1
+
+            stable_sleep_duration=constants.getattr("STABLE_SLEEP_DURATION_LIKE")
+            sleep_duration_50=constants.getattr("SLEEP_DURATION_LIKE_50")
+            sleep_duration_60=constants.getattr("SLEEP_DURATION_LIKE_60")
+            sleep_duration_common=constants.getattr("COMMON_MULTIPLE_SLEEP_DURATION_LIKE")
+
             for count, func in enumerate(tasks):
                 out = func()
                 if out == 0:
                     sleep_duration = 0
                 elif count + 1 % 60 == 0 and count + 1 % 50 == 0:
-                    sleep_duration = 40
+                    sleep_duration = sleep_duration_common
                 elif count % 60 == 0:
-                    sleep_duration = 1  # Divisible by 60 - 1 second sleep
+                    sleep_duration = sleep_duration_60  # Divisible by 60 - 1 second sleep
                 elif count % 50 == 0:
-                    sleep_duration = 30  # Divisible by 50 - 30 seconds sleep
+                    sleep_duration = sleep_duration_50   # Divisible by 50 - 30 seconds sleep
                 else:
-                    sleep_duration = 5
+                    sleep_duration =  stable_sleep_duration
                 if out == 1:
                     progress_utils.increment_like_task(task2)
                 progress_utils.increment_like_task(task)
@@ -132,24 +138,34 @@ def _like(model_id, ids: list, like_action: bool):
 
 
 def _toggle_like_requests(c, id, model_id):
+
+
+    sleeper=sessionManager.SessionSleep(
+        sleep=constants.getattr("SESSION_429_SLEEP_STARTER_VAL"),
+        difmin=constants.getattr("SESSION_429_LIKE_INCREASE_SLEEP_TIME_DIF"),
+    )
     if not read_args.retriveArgs().force_like and cache.get(f"liked_status_{id}", None):
         log.debug(f"ID: {id} marked as liked in cache")
         return 0
-    sleep_duration = 5
-    favorited, id = _like_request(c, id, model_id)
+    sleep_duration = constants.getattr("DOUBLE_TOGGLE_SLEEP_DURATION_LIKE")
+    favorited, id = _like_request(c, id, model_id,sleeper)
     if favorited:
         log.debug(f"ID: {id} changed to liked")
         out = 1
     else:
         log.debug(f"ID: {id} restored to liked")
         time.sleep(sleep_duration)
-        _like_request(c, id, model_id)
+        _like_request(c, id, model_id,sleeper)
         out = 2
     cache.set(f"liked_status_{id}", True)
     return out
 
 
 def _toggle_unlike_requests(c, id, model_id):
+    sleeper=sessionManager.SessionSleep(
+        sleep=constants.getattr("SESSION_429_SLEEP_STARTER_VAL"),
+        difmin=constants.getattr("SESSION_429_LIKE_INCREASE_SLEEP_TIME_DIF")
+    )
     if (
         not read_args.retriveArgs().force_like
         and cache.get(f"liked_status_{id}", None) == False
@@ -157,22 +173,25 @@ def _toggle_unlike_requests(c, id, model_id):
         log.debug(f"ID: {id} marked as unliked in cache")
 
         return 0
-    sleep_duration = 5
-    favorited, id = _like_request(c, id, model_id)
+    sleep_duration = constants.getattr("DOUBLE_TOGGLE_SLEEP_DURATION_LIKE")
+    favorited, id = _like_request(c, id, model_id,sleeper)
     if not favorited:
         log.debug(f"ID: {id} changed to unliked")
         out = 1
     else:
         log.debug(f"ID: {id} restored to unlike")
         time.sleep(sleep_duration)
-        _like_request(c, id, model_id)
+        _like_request(c, id, model_id,sleeper)
         out = 2
     cache.set(f"liked_status_{id}", False)
     return out
 
 
-def _like_request(c, id, model_id):
+def _like_request(c, id, model_id,sleeper):
+   
     with c.requests(
-        constants.getattr("favoriteEP").format(id, model_id), method="post"
+        constants.getattr("favoriteEP").format(id, model_id),
+        method="post",sleeper=sleeper,
+        retries=constants.getattr("LIKE_MAX_RETRIES")
     ) as r:
         return r.json_()["isFavorite"], r.json_()["id"]

{ofscraper-3.10.dev1 → ofscraper-3.10.dev2}/ofscraper/const/constants.py

@@ -25,3 +25,5 @@ from ofscraper.const.values.table import *
 from ofscraper.const.values.test_constants import *
 from ofscraper.const.values.time import *
 from ofscraper.const.values.url import *
+from ofscraper.const.values.like import *
+

ofscraper-3.10.dev2/ofscraper/const/values/like.py

@@ -0,0 +1,15 @@
+
+
+
+
+
+STABLE_SLEEP_DURATION_LIKE=5
+SLEEP_DURATION_LIKE_60=1
+SLEEP_DURATION_LIKE_50=30
+COMMON_MULTIPLE_SLEEP_DURATION_LIKE=40
+DOUBLE_TOGGLE_SLEEP_DURATION_LIKE=5
+SESSION_429_LIKE_INCREASE_SLEEP_TIME_DIF=0
+SESSION_429_SLEEP_STARTER_VAL=8
+LIKE_MAX_RETRIES=5
+
+

{ofscraper-3.10.dev1 → ofscraper-3.10.dev2}/ofscraper/filters/media/helpers.py

@@ -1,4 +1,5 @@
 import logging
+from collections import defaultdict
 import random
 import re
 
@@ -22,13 +23,13 @@ def sort_by_date(media):
 
 
 def dupefilter(media):
-    output = []
-    ids = set()
+    output =defaultdict(lambda:None)
     for item in media:
-        if not item.id or item.id not in ids:
-            output.append(item)
-            ids.add(item.id)
-    return output
+        if not output[item.id]:
+            output[item.id]=item
+        elif item.canview and not output[item.id].canview:
+             output[item.id]=item
+    return output.values()
 
 
 def post_datesorter(output):

{ofscraper-3.10.dev1 → ofscraper-3.10.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ofscraper"
-version = "3.10.dev1"
+version = "3.10.dev2"
 description = "automatically scrape onlyfans"
 authors = ["datawhores <datawhores@riseup.net>"]
 readme = "README.md"