ofscraper 3.9.0.dev13__tar.gz → 3.9.2__tar.gz

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ofscraper
-Version: 3.9.0.dev13
+Version: 3.9.2
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/actions/like.py

@@ -53,7 +53,7 @@ async def get_posts(model_id, username):
             retries=constants.getattr("API_NUM_TRIES"),
             wait_min=constants.getattr("OF_MIN_WAIT_API"),
             wait_max=constants.getattr("OF_MAX_WAIT_API"),
-            new_request_auth=True
+            new_request_auth=True,
         ) as c:
             while True:
                 max_count = min(

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/actions/scraper.py

@@ -385,15 +385,8 @@ async def process_profile(username) -> list:
 async def process_all_paid():
     with stdout.lowstdout():
         paid_content = await paid.get_all_paid_posts()
-        user_dict = {}
-
-        for ele in paid_content:
-            user_id = ele.get("fromUser", {}).get("id") or ele.get("author", {}).get(
-                "id"
-            )
-            user_dict.setdefault(user_id, []).append(ele)
         output = {}
-        for model_id, value in user_dict.items():
+        for model_id, value in paid_content.items():
             username = profile.scrape_profile(model_id).get("username")
             if username == constants.getattr(
                 "DELETED_MODEL_PLACEHOLDER"
@@ -521,8 +514,7 @@ async def process_task(model_id, username, ele):
         wait_min=constants.getattr("OF_MIN_WAIT_API"),
         wait_max=constants.getattr("OF_MAX_WAIT_API"),
         total_timeout=constants.getattr("API_TIMEOUT_PER_TASK"),
-                    new_request_auth=True
-
+        new_request_auth=True,
     ) as c:
         while True:
             max_count = min(

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/archive.py

@@ -22,6 +22,7 @@ import ofscraper.utils.constants as constants
 import ofscraper.utils.progress as progress_utils
 import ofscraper.utils.settings as settings
 from ofscraper.utils.context.run_async import run
+from ofscraper.utils.logs.helpers import is_trace
 
 log = logging.getLogger("shared")
 
@@ -131,7 +132,7 @@ async def process_tasks(tasks, model_id, after):
 
 def get_split_array(oldarchived, after):
     if len(oldarchived) == 0:
-        return 0
+        return []
     min_posts = max(
         len(oldarchived) // constants.getattr("REASONABLE_MAX_PAGE"),
         constants.getattr("MIN_PAGE_POST_COUNT"),
@@ -230,7 +231,7 @@ def set_check(unduped, model_id, after):
         cache.set(
             f"archived_check_{model_id}",
             all_posts,
-            expire=constants.getattr("DAY_SECONDS"),
+            expire=constants.getattr("THREE_DAY_SECONDS"),
         )
         cache.close()
 
@@ -377,6 +378,8 @@ async def scrape_archived_posts(
 
 
 def trace_log_task(responseArray):
+    if not is_trace():
+        return
     chunk_size = constants.getattr("LARGE_TRACE_CHUNK_SIZE")
     for i in range(1, len(responseArray) + 1, chunk_size):
         # Calculate end index considering potential last chunk being smaller
@@ -394,6 +397,8 @@ def trace_log_task(responseArray):
 
 
 def trace_log_old(responseArray):
+    if not is_trace():
+        return
     chunk_size = constants.getattr("LARGE_TRACE_CHUNK_SIZE")
     for i in range(1, len(responseArray) + 1, chunk_size):
         # Calculate end index considering potential last chunk being smaller

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/highlights.py

@@ -436,8 +436,7 @@ def get_individual_stories(id, c=None):
         retries=constants.getattr("API_INDVIDIUAL_NUM_TRIES"),
         wait_min=constants.getattr("OF_MIN_WAIT_API"),
         wait_max=constants.getattr("OF_MAX_WAIT_API"),
-        new_request_auth=True
-
+        new_request_auth=True,
     ) as c:
         with c.requests_async(constants.getattr("storiesSPECIFIC").format(id)) as r:
             log.trace(f"highlight raw highlight individua; {r.json_()}")

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/labels.py

@@ -21,6 +21,7 @@ import ofscraper.utils.cache as cache
 import ofscraper.utils.constants as constants
 import ofscraper.utils.progress as progress_utils
 from ofscraper.utils.context.run_async import run
+from ofscraper.utils.logs.helpers import is_trace
 
 log = logging.getLogger("shared")
 
@@ -394,12 +395,14 @@ def set_check(unduped, model_id):
     cache.set(
         f"labels_check_{model_id}",
         list(unduped),
-        expire=constants.getattr("DAY_SECONDS"),
+        expire=constants.getattr("THREE_DAY_SECONDS"),
     )
     cache.close()
 
 
 def trace_log_task(responseArray, header=None):
+    if not is_trace():
+        return
     chunk_size = constants.getattr("LARGE_TRACE_CHUNK_SIZE")
     for i in range(1, len(responseArray) + 1, chunk_size):
         # Calculate end index considering potential last chunk being smaller

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/me.py

@@ -28,8 +28,7 @@ def scrape_user():
         retries=constants.getattr("API_INDVIDIUAL_NUM_TRIES"),
         wait_min=constants.getattr("OF_AUTH_MIN_WAIT"),
         wait_max=constants.getattr("OF_AUTH_MAX_WAIT"),
-        new_request_auth=True
-
+        new_request_auth=True,
     ) as c:
         return _scraper_user_helper(c)
 
@@ -62,8 +61,7 @@ def parse_subscriber_count():
         retries=constants.getattr("API_INDVIDIUAL_NUM_TRIES"),
         wait_min=constants.getattr("OF_MIN_WAIT_API"),
         wait_max=constants.getattr("OF_MAX_WAIT_API"),
-                    new_request_auth=True
-
+        new_request_auth=True,
     ) as c:
         try:
             with c.requests(constants.getattr("subscribeCountEP")) as r:

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/messages.py

@@ -12,7 +12,6 @@ r"""
 """
 
 import asyncio
-import contextvars
 import logging
 import traceback
 
@@ -27,6 +26,8 @@ import ofscraper.utils.constants as constants
 import ofscraper.utils.progress as progress_utils
 import ofscraper.utils.settings as settings
 from ofscraper.utils.context.run_async import run
+from ofscraper.utils.logs.helpers import is_trace
+
 
 log = logging.getLogger("shared")
 
@@ -306,7 +307,7 @@ def set_check(unduped, model_id, after):
         cache.set(
             f"message_check_{model_id}",
             list(all_posts),
-            expire=constants.getattr("DAY_SECONDS"),
+            expire=constants.getattr("THREE_DAY_SECONDS"),
         )
         cache.close()
 
@@ -326,7 +327,6 @@ async def scrape_messages(
     await asyncio.sleep(1)
     try:
         async with c.requests_async(url=url) as r:
-
             task = (
                 job_progress.add_task(
                     f": Message ID-> {message_id if message_id else 'initial'}"
@@ -422,8 +422,7 @@ def get_individual_post(model_id, postid):
         retries=constants.getattr("API_INDVIDIUAL_NUM_TRIES"),
         wait_min=constants.getattr("OF_MIN_WAIT_API"),
         wait_max=constants.getattr("OF_MAX_WAIT_API"),
-                    new_request_auth=True
-
+        new_request_auth=True,
     ) as c:
         with c.requests(
             url=constants.getattr("messageSPECIFIC").format(model_id, postid)
@@ -473,6 +472,8 @@ async def get_after(model_id, username, forced_after=None):
 
 
 def trace_log_task(responseArray):
+    if not is_trace():
+        return
     chunk_size = constants.getattr("LARGE_TRACE_CHUNK_SIZE")
     for i in range(1, len(responseArray) + 1, chunk_size):
         # Calculate end index considering potential last chunk being smaller
@@ -490,6 +491,8 @@ def trace_log_task(responseArray):
 
 
 def trace_log_old(responseArray):
+    if not is_trace():
+        return
     chunk_size = constants.getattr("LARGE_TRACE_CHUNK_SIZE")
     for i in range(1, len(responseArray) + 1, chunk_size):
         # Calculate end index considering potential last chunk being smaller

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/paid.py

@@ -108,25 +108,10 @@ async def process_tasks(tasks, model_id):
         paid_str += f"{common_logs.RAW_INNER} {post}\n\n"
     log.trace(f"{common_logs.FINAL_RAW.format('Paid')}".format(posts=paid_str))
     log.debug(f"{common_logs.FINAL_COUNT.format('Paid')} {len(responseArray)}")
-    set_check(responseArray, model_id)
+    set_check(model_id, responseArray)
     return responseArray
 
 
-def set_check(unduped, model_id):
-    seen = set()
-    all_posts = [
-        post
-        for post in cache.get(f"purchase_check_{model_id}", default=[]) + unduped
-        if post["id"] not in seen and not seen.add(post["id"])
-    ]
-    cache.set(
-        f"purchased_check_{model_id}",
-        all_posts,
-        expire=constants.getattr("DAY_SECONDS"),
-    )
-    cache.close()
-
-
 @run
 async def scrape_paid(c, username, job_progress=None, offset=0):
     """Takes headers to access onlyfans as an argument and then checks the purchased content
@@ -189,6 +174,11 @@ async def scrape_paid(c, username, job_progress=None, offset=0):
 
 @run
 async def get_all_paid_posts():
+    data = await process_and_create_tasks()
+    return create_all_paid_dict(data)
+
+
+async def process_and_create_tasks():
     with ThreadPoolExecutor(
         max_workers=constants.getattr("MAX_THREAD_WORKERS")
     ) as executor:
@@ -206,8 +196,7 @@ async def get_all_paid_posts():
                 retries=constants.getattr("API_PAID_NUM_TRIES"),
                 wait_min=constants.getattr("OF_MIN_WAIT_API"),
                 wait_max=constants.getattr("OF_MAX_WAIT_API"),
-                            new_request_auth=True
-
+                new_request_auth=True,
             ) as c:
                 allpaid = cache.get("purchased_all", default=[])
                 log.debug(f"[bold]All Paid Cache[/bold] {len(allpaid)} found")
@@ -220,7 +209,7 @@ async def get_all_paid_posts():
                                 scrape_all_paid(
                                     c,
                                     job_progress,
-                                    required=100,
+                                    required=min_posts,
                                     offset=splitArrays[i],
                                 )
                             )
@@ -268,7 +257,7 @@ async def get_all_paid_posts():
                             await asyncio.sleep(1)
                             log.traceback_(E)
                             log.traceback_(traceback.format_exc())
-                            continue
+                            u
                     tasks = new_tasks
                 overall_progress.remove_task(page_task)
 
@@ -290,6 +279,15 @@ async def get_all_paid_posts():
         return output
 
 
+def create_all_paid_dict(paid_content):
+    user_dict = {}
+    for ele in paid_content:
+        user_id = ele.get("fromUser", {}).get("id") or ele.get("author", {}).get("id")
+        user_dict.setdefault(str(user_id), []).append(ele)
+    [set_check(key, val) for key, val in user_dict.items()]
+    return user_dict
+
+
 async def scrape_all_paid(c, job_progress=None, offset=0, required=None):
     """Takes headers to access onlyfans as an argument and then checks the purchased content
     url to look for any purchased content. If it finds some it will return it as a list.
@@ -355,6 +353,21 @@ async def scrape_all_paid(c, job_progress=None, offset=0, required=None):
         job_progress.remove_task(task)
 
 
+def set_check(model_id, unduped):
+    seen = set()
+    all_posts = [
+        post
+        for post in cache.get(f"purchase_check_{model_id}", default=[]) + unduped
+        if post["id"] not in seen and not seen.add(post["id"])
+    ]
+    cache.set(
+        f"purchased_check_{model_id}",
+        all_posts,
+        expire=constants.getattr("THREE_DAY_SECONDS"),
+    )
+    cache.close()
+
+
 def get_individual_post(username, model_id, postid):
     data = get_paid_posts_progress(username, model_id)
     postid = int(postid)

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/pinned.py

@@ -149,7 +149,7 @@ def set_check(unduped, model_id):
         cache.set(
             f"pinned_check_{model_id}",
             all_posts,
-            expire=constants.getattr("DAY_SECONDS"),
+            expire=constants.getattr("THREE_DAY_SECONDS"),
         )
         cache.close()
 

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/profile.py

@@ -38,8 +38,7 @@ def scrape_profile(username: Union[int, str]) -> dict:
         retries=constants.getattr("API_INDVIDIUAL_NUM_TRIES"),
         wait_min=constants.getattr("OF_MIN_WAIT_API"),
         wait_max=constants.getattr("OF_MAX_WAIT_API"),
-                    new_request_auth=True
-
+        new_request_auth=True,
     ) as c:
         return scrape_profile_helper(c, username)
 

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/subscriptions/individual.py

@@ -49,8 +49,7 @@ async def get_subscription(accounts=None):
                 retries=constants.getattr("API_INDVIDIUAL_NUM_TRIES"),
                 wait_min=constants.getattr("OF_MIN_WAIT_API"),
                 wait_max=constants.getattr("OF_MAX_WAIT_API"),
-                            new_request_auth=True
-
+                new_request_auth=True,
             ) as c:
                 out = await get_subscription_helper(c, accounts)
                 job_progress.remove_task(task1)

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/subscriptions/lists.py

@@ -28,6 +28,8 @@ import ofscraper.utils.args.read as read_args
 import ofscraper.utils.console as console
 import ofscraper.utils.constants as constants
 from ofscraper.utils.context.run_async import run
+from ofscraper.utils.logs.helpers import is_trace
+
 
 log = logging.getLogger("shared")
 attempt = contextvars.ContextVar("attempt")
@@ -100,8 +102,7 @@ async def get_lists():
                 retries=constants.getattr("API_INDVIDIUAL_NUM_TRIES"),
                 wait_min=constants.getattr("OF_MIN_WAIT_API"),
                 wait_max=constants.getattr("OF_MAX_WAIT_API"),
-                            new_request_auth=True
-
+                new_request_auth=True,
             ) as c:
                 tasks.append(asyncio.create_task(scrape_for_list(c, job_progress)))
                 page_task = overall_progress.add_task(
@@ -125,13 +126,34 @@ async def get_lists():
                             continue
                     tasks = new_tasks
     overall_progress.remove_task(page_task)
-    log.trace(
+    trace_log_list(output)
+    
+    log.debug(f"[bold]lists name count without Dupes[/bold] {len(output)} found")
+    return output
+
+def trace_log_list(responseArray):
+    if not is_trace():
+        return
+    chunk_size = constants.getattr("LARGE_TRACE_CHUNK_SIZE")
+    for i in range(1, len(responseArray) + 1, chunk_size):
+        # Calculate end index considering potential last chunk being smaller
+        end_index = min(
+            i + chunk_size - 1, len(responseArray)
+        )  # Adjust end_index calculation
+        chunk = responseArray[i - 1 : end_index]  # Adjust slice to start at i-1
+        log.trace(
         "list unduped {posts}".format(
-            posts="\n\n".join(map(lambda x: f" list data raw:{x}", output))
+            posts="\n\n".join(map(lambda x: f" list data raw:{x}", chunk))
         )
     )
-    log.debug(f"[bold]lists name count without Dupes[/bold] {len(output)} found")
-    return output
+        # Check if there are more elements remaining after this chunk
+        if i + chunk_size > len(responseArray):
+            break  # Exit the loop if we've processed all elements
+
+
+
+
+
 
 
 async def scrape_for_list(c, job_progress, offset=0):
@@ -155,7 +177,7 @@ async def scrape_for_list(c, job_progress, offset=0):
                 f"offset:{offset} -> hasMore value in json {data.get('hasMore','undefined') }"
             )
             log.trace(
-                "offset:{offset} -> label names raw: {posts}".format(
+                "offset:{offset} -> list names raw: {posts}".format(
                     offset=offset, posts=data
                 )
             )
@@ -203,8 +225,7 @@ async def get_list_users(lists):
                 retries=constants.getattr("API_INDVIDIUAL_NUM_TRIES"),
                 wait_min=constants.getattr("OF_MIN_WAIT_API"),
                 wait_max=constants.getattr("OF_MAX_WAIT_API"),
-                            new_request_auth=True
-
+                new_request_auth=True,
             ) as c:
                 [
                     tasks.append(
@@ -237,15 +258,31 @@ async def get_list_users(lists):
     outdict = {}
     for ele in output:
         outdict[ele["id"]] = ele
-    log.trace(
-        "users found {users}".format(
-            users="\n\n".join(map(lambda x: f"user data: {str(x)}", outdict.values()))
-        )
-    )
+    trace_log_usernames(outdict.values())
     log.debug(f"[bold]users count without Dupes[/bold] {len(outdict.values())} found")
     return outdict.values()
 
 
+def trace_log_usernames(responseArray):
+    if not is_trace():
+        return
+    chunk_size = constants.getattr("LARGE_TRACE_CHUNK_SIZE")
+    for i in range(1, len(responseArray) + 1, chunk_size):
+        # Calculate end index considering potential last chunk being smaller
+        end_index = min(
+            i + chunk_size - 1, len(responseArray)
+        )  # Adjust end_index calculation
+        chunk = responseArray[i - 1 : end_index]  # Adjust slice to start at i-1
+        log.trace(
+        "users found {users}".format(
+            users="\n\n".join(map(lambda x: f"user data: {str(x)}", chunk))
+        )
+    
+    )
+        # Check if there are more elements remaining after this chunk
+        if i + chunk_size > len(responseArray):
+            break  # Exit the loop if we've processed all elements
+
 async def scrape_list_members(c, item, job_progress, offset=0):
     users = None
     attempt.set(0)

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/subscriptions/subscriptions.py

@@ -47,8 +47,7 @@ async def get_subscriptions(subscribe_count, account="active"):
                 retries=constants.getattr("API_INDVIDIUAL_NUM_TRIES"),
                 wait_min=constants.getattr("OF_MIN_WAIT_API"),
                 wait_max=constants.getattr("OF_MAX_WAIT_API"),
-                            new_request_auth=True
-
+                new_request_auth=True,
             ) as c:
                 if account == "active":
                     out = await activeHelper(subscribe_count, c)
@@ -173,7 +172,7 @@ async def scrape_subscriptions_active(c, offset=0, num=0, recur=False) -> list:
                 f"usernames retrived -> {list(map(lambda x:x.get('username'),subscriptions))}"
             )
             if len(subscriptions) == 0:
-                return subscriptions
+                return subscriptions,new_tasks
             elif recur is False:
                 pass
             elif (await r.json_())["hasMore"] is True:
@@ -208,7 +207,7 @@ async def scrape_subscriptions_disabled(c, offset=0, num=0, recur=False) -> list
             )
 
             if len(subscriptions) == 0:
-                return subscriptions
+                return subscriptions,new_tasks
             elif recur is False:
                 pass
             elif (await r.json_())["hasMore"] is True:

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/api/timeline.py

@@ -23,6 +23,7 @@ import ofscraper.utils.constants as constants
 import ofscraper.utils.progress as progress_utils
 import ofscraper.utils.settings as settings
 from ofscraper.utils.context.run_async import run
+from ofscraper.utils.logs.helpers import is_trace
 
 log = logging.getLogger("shared")
 
@@ -245,7 +246,7 @@ def set_check(unduped, model_id, after):
         cache.set(
             f"timeline_check_{model_id}",
             all_posts,
-            expire=constants.getattr("DAY_SECONDS"),
+            expire=constants.getattr("THREE_DAY_SECONDS"),
         )
         cache.close()
 
@@ -256,8 +257,7 @@ def get_individual_post(id):
         retries=constants.getattr("API_INDVIDIUAL_NUM_TRIES"),
         wait_min=constants.getattr("OF_MIN_WAIT_API"),
         wait_max=constants.getattr("OF_MAX_WAIT_API"),
-                    new_request_auth=True
-
+        new_request_auth=True,
     ) as c:
         with c.requests(constants.getattr("INDIVIDUAL_TIMELINE").format(id)) as r:
             log.trace(f"post raw individual {r.json()}")
@@ -406,6 +406,8 @@ async def scrape_timeline_posts(
 
 
 def trace_log_task(responseArray):
+    if not is_trace():
+        return
     chunk_size = constants.getattr("LARGE_TRACE_CHUNK_SIZE")
     for i in range(1, len(responseArray) + 1, chunk_size):
         # Calculate end index considering potential last chunk being smaller
@@ -423,6 +425,8 @@ def trace_log_task(responseArray):
 
 
 def trace_log_old(responseArray):
+    if not is_trace():
+        return
     chunk_size = constants.getattr("LARGE_TRACE_CHUNK_SIZE")
     for i in range(1, len(responseArray) + 1, chunk_size):
         # Calculate end index considering potential last chunk being smaller

{ofscraper-3.9.0.dev13 → ofscraper-3.9.2}/ofscraper/classes/media.py

@@ -34,6 +34,7 @@ class Media(base.base):
         self._post = post
         self._final_url = None
         self._cached_parse_mpd = None
+        self._mpd=None
 
     def __eq__(self, other):
         return self.postid == other.postid
@@ -181,7 +182,9 @@ class Media(base.base):
 
     @property
     def mpd(self):
-        if self.protected == False:
+        if self._mpd:
+            return self._mpd
+        elif self.protected == False:
             return None
         return (
             self._media.get("files", {}).get("drm", {}).get("manifest", {}).get("dash")
@@ -326,8 +329,7 @@ class Media(base.base):
             retries=constants.getattr("MPD_NUM_TRIES"),
             wait_min=constants.getattr("OF_MIN_WAIT_API"),
             wait_max=constants.getattr("OF_MAX_WAIT_API"),
-                        new_request_auth=True,
-
+            new_request_auth=True,
             semaphore=semaphore,
         ) as c:
             async with c.requests_async(url=self.mpd, params=params) as r:
@@ -362,6 +364,12 @@ class Media(base.base):
     def mediatype(self, val):
         self._media["type"] = val
 
+    @url.setter
+    def url(self, val):
+        self._final_url=val
+    @mpd.setter
+    def mpd(self, val):
+        self._mpd=val
     @property
     async def selected_quality(self):
         if self.protected == False: