ofscraper 3.9.0.dev11__tar.gz → 3.9.0.dev12__tar.gz

{ofscraper-3.9.0.dev11 → ofscraper-3.9.0.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ofscraper
-Version: 3.9.0.dev11
+Version: 3.9.0.dev12
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net
@@ -35,6 +35,7 @@ Requires-Dist: poetry-dynamic-versioning (>=1.2.0,<2.0.0)
 Requires-Dist: prompt-toolkit (>=3.0.43,<4.0.0)
 Requires-Dist: psutil (>=5.9.8,<6.0.0)
 Requires-Dist: pycryptodome (>=3.20.0,<4.0.0)
+Requires-Dist: pynumparser (>=1.4.1,<2.0.0)
 Requires-Dist: pywidevine (>=1.8.0,<2.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: rich (>=13.7.1,<14.0.0)

{ofscraper-3.9.0.dev11 → ofscraper-3.9.0.dev12}/ofscraper/actions/like.py

@@ -11,9 +11,10 @@ r"""
                                                                                       
 """
 
+import functools
 import asyncio
+import time
 import logging
-import traceback
 
 from rich.progress import (
     BarColumn,
@@ -23,31 +24,21 @@ from rich.progress import (
     TextColumn,
 )
 from rich.style import Style
-from tenacity import (
-    AsyncRetrying,
-    retry,
-    retry_if_not_exception_type,
-    stop_after_attempt,
-    wait_random,
-)
 
 import ofscraper.api.archive as archive
 import ofscraper.api.labels as labels_api
 import ofscraper.api.pinned as pinned
 import ofscraper.api.timeline as timeline
-import ofscraper.classes.labels as labels
 import ofscraper.classes.posts as posts_
-import ofscraper.classes.sessionbuilder as sessionbuilder
+import ofscraper.classes.sessionmanager as sessionManager
 import ofscraper.utils.args.areas as areas
 import ofscraper.utils.args.read as read_args
 import ofscraper.utils.console as console
 import ofscraper.utils.constants as constants
 import ofscraper.utils.progress as progress_utils
 import ofscraper.utils.system.system as system
-from ofscraper.classes.semaphoreDelayed import semaphoreDelayed
 from ofscraper.utils.context.run_async import run
 
-sem = semaphoreDelayed(1)
 log = logging.getLogger("shared")
 
 
@@ -57,7 +48,12 @@ async def get_posts(model_id, username):
     final_post_areas = set(areas.get_like_area())
     tasks = []
     with progress_utils.setup_api_split_progress_live():
-        async with sessionbuilder.sessionBuilder() as c:
+        async with sessionManager.sessionManager(
+            sem=constants.getattr("LIKE_MAX_SEMS"),
+            retries=constants.getattr("API_NUM_TRIES"),
+            wait_min=constants.getattr("OF_MIN_WAIT_API"),
+            wait_max=constants.getattr("OF_MAX_WAIT_API"),
+        ) as c:
             while True:
                 max_count = min(
                     constants.getattr("API_MAX_AREAS"),
@@ -149,13 +145,13 @@ def get_post_for_like(model_id, username):
 
 
 def filter_for_unfavorited(posts: list) -> list:
-    output = list(filter(lambda x: x.favorited == False, posts))
+    output = list(filter(lambda x: x.favorited == False and x.opened, posts))
     log.debug(f"[bold]Number of unliked post[/bold] {len(output)}")
     return output
 
 
 def filter_for_favorited(posts: list) -> list:
-    output = list(filter(lambda x: x.favorited == True, posts))
+    output = list(filter(lambda x: x.favorited == True and x.opened, posts))
     log.debug(f"[bold]Number of liked post[/bold] {len(output)}")
     return output
 
@@ -172,19 +168,16 @@ def get_post_ids(posts: list) -> list:
     return list(map(lambda x: x.id, posts))
 
 
-def like(model_id, username, ids: list):
-    _like(model_id, username, ids, True)
+def like(model_id, ids: list):
+    _like(model_id, ids, True)
 
 
-def unlike(model_id, username, ids: list):
-    _like(model_id, username, ids, False)
+def unlike(model_id, ids: list):
+    _like(model_id, ids, False)
 
 
-@run
-async def _like(model_id, username, ids: list, like_action: bool):
+def _like(model_id, ids: list, like_action: bool):
     title = "Liking" if like_action else "Unliking"
-    global sem
-    sem.delay = 3
     with Progress(
         SpinnerColumn(style=Style(color="blue")),
         TextColumn("{task.description}"),
@@ -192,59 +185,39 @@ async def _like(model_id, username, ids: list, like_action: bool):
         MofNCompleteColumn(),
         console=console.get_shared_console(),
     ) as overall_progress:
-        async with sessionbuilder.sessionBuilder() as c:
+        with sessionManager.sessionManager(
+            sem=1,
+            backend="httpx",
+            retries=constants.getattr("API_LIKE_NUM_TRIES"),
+            wait_min=constants.getattr("OF_MIN_WAIT_API"),
+            wait_max=constants.getattr("OF_MAX_WAIT_API"),
+        ) as c:
             tasks = []
             task1 = overall_progress.add_task(f"{title} posts...\n", total=len(ids))
 
             [
-                tasks.append(asyncio.create_task(_like_request(c, id, model_id)))
+                tasks.append(functools.partial(_like_request,c, id, model_id))
                 for id in ids
             ]
-            for count, coro in enumerate(asyncio.as_completed(tasks)):
-                id = await coro
+            sleep_duration = 3
+            for count, func in enumerate(tasks):
+                id = func()
                 log.debug(
                     f"ID: {id} Performed {'like' if like_action==True else 'unlike'} action"
                 )
-
                 if count + 1 % 60 == 0 and count + 1 % 50 == 0:
-                    sem.delay = 15
-                elif count + 1 % 60 == 0:
-                    sem.delay = 3
-                elif count + 1 % 50 == 0:
-                    sem.delay = 30
-
+                    sleep_duration=40
+                elif count % 60 == 0:
+                    sleep_duration = 1  # Divisible by 60 - 1 second sleep
+                elif count % 50 == 0:
+                    sleep_duration = 30  # Divisible by 50 - 30 seconds sleep
                 overall_progress.update(task1, advance=1, refresh=True)
+                time.sleep(sleep_duration)
 
 
-async def _like_request(c, id, model_id):
-    global sem
-    async for _ in AsyncRetrying(
-        retry=retry_if_not_exception_type(KeyboardInterrupt),
-        stop=stop_after_attempt(constants.getattr("NUM_TRIES")),
-        wait=wait_random(
-            min=constants.getattr("OF_MIN"),
-            max=constants.getattr("OF_MAX"),
-        ),
-        reraise=True,
-    ):
-        with _:
-            await sem.acquire()
-            try:
-                async with c.requests(
-                    constants.getattr("favoriteEP").format(id, model_id), "post"
-                )() as r:
-                    if r.ok:
-                        return id
-                    else:
-                        log.debug(
-                            f"[bold]timeline response status code:[/bold]{r.status}"
-                        )
-                        log.debug(f"[bold]timeline response:[/bold] {await r.text_()}")
-                        log.debug(f"[bold]timeline headers:[/bold] {r.headers}")
-                        r.raise_for_status()
-            except Exception as E:
-                log.traceback_(E)
-                log.traceback_(traceback.format_exc())
-                raise E
-            finally:
-                sem.release()
+
+def _like_request(c, id, model_id):
+    with c.requests(
+        constants.getattr("favoriteEP").format(id, model_id),method="post"
+    ) as _:
+        return id

{ofscraper-3.9.0.dev11 → ofscraper-3.9.0.dev12}/ofscraper/actions/process.py

@@ -191,12 +191,14 @@ def process_like():
                 )
                 model_id = ele.id
                 operations.table_init_create(model_id=model_id, username=ele.name)
-                unfavorited_posts = like.get_post_for_like(model_id=model_id, username=ele.name)
+                unfavorited_posts = like.get_post_for_like(
+                    model_id=model_id, username=ele.name
+                )
                 unfavorited_posts = filters.post_filter_for_like(
                     unfavorited_posts, like=True
                 )
                 post_ids = like.get_post_ids(unfavorited_posts)
-                like.like(model_id, ele.name, post_ids)
+                like.like(model_id, post_ids)
 
 
 @exit.exit_wrapper
@@ -223,7 +225,7 @@ def process_unlike():
                     favorited_posts, like=False
                 )
                 post_ids = like.get_post_ids(favorited_posts)
-                like.unlike(model_id, ele.name, post_ids)
+                like.unlike(model_id, post_ids)
 
 
 def add_selected_areas():

{ofscraper-3.9.0.dev11 → ofscraper-3.9.0.dev12}/ofscraper/actions/scraper.py

@@ -28,7 +28,7 @@ import ofscraper.api.timeline as timeline
 import ofscraper.classes.labels as labels
 import ofscraper.classes.media as media
 import ofscraper.classes.posts as posts_
-import ofscraper.classes.sessionbuilder as sessionbuilder
+import ofscraper.classes.sessionmanager as sessionManager
 import ofscraper.db.operations as operations
 import ofscraper.filters.media.main as filters
 import ofscraper.utils.args.areas as areas
@@ -63,7 +63,7 @@ async def process_messages(model_id, username, c):
             )
             log.debug("Removing locked messages media")
             output = []
-            [output.extend(message.media) for message in messages_]
+            [output.extend(message.all_media) for message in messages_]
             log.debug(f"[bold]Messages media count[/bold] {len(output)}")
             # Update after database
             cache.set(
@@ -98,7 +98,7 @@ async def process_paid_post(model_id, username, c):
                 username=username,
             )
             output = []
-            [output.extend(post.media) for post in paid_content]
+            [output.extend(post.all_media) for post in paid_content]
             log.debug(f"[bold]Paid media count without locked[/bold] {len(output)}")
 
             await operations.batch_mediainsert(
@@ -143,7 +143,7 @@ async def process_stories(model_id, username, c):
                 f"[bold]Story media count[/bold] {sum(map(lambda x:len(x.post_media), stories))}"
             )
             output = []
-            [output.extend(stories.media) for stories in stories]
+            [output.extend(stories.all_media) for stories in stories]
             await operations.batch_mediainsert(
                 output,
                 model_id=model_id,
@@ -183,7 +183,7 @@ async def process_highlights(model_id, username, c):
                 f"[bold]highlight media count[/bold] {sum(map(lambda x:len(x.post_media), highlights_))}"
             )
             output = []
-            [output.extend(stories.media) for stories in highlights_]
+            [output.extend(stories.all_media) for stories in highlights_]
             await operations.batch_mediainsert(
                 output,
                 model_id=model_id,
@@ -231,7 +231,7 @@ async def process_timeline_posts(model_id, username, c):
             )
             log.debug("Removing locked timeline media")
             output = []
-            [output.extend(post.media) for post in timeline_only_posts]
+            [output.extend(post.all_media) for post in timeline_only_posts]
             log.debug(f"[bold]Timeline media count without locked[/bold] {len(output)}")
 
             await operations.batch_mediainsert(
@@ -281,7 +281,7 @@ async def process_archived_posts(model_id, username, c):
             )
             log.debug("Removing locked archived media")
             output = []
-            [output.extend(post.media) for post in archived_posts]
+            [output.extend(post.all_media) for post in archived_posts]
             log.debug(f"[bold]Archived media count without locked[/bold] {len(output)}")
 
             await operations.batch_mediainsert(
@@ -325,7 +325,7 @@ async def process_pinned_posts(model_id, username, c):
             )
             log.debug("Removing locked pinned media")
             output = []
-            [output.extend(post.media) for post in pinned_posts]
+            [output.extend(post.all_media) for post in pinned_posts]
             log.debug(f"[bold]Pinned media count without locked[/bold] {len(output)}")
 
             await operations.batch_mediainsert(
@@ -386,6 +386,7 @@ async def process_all_paid():
     with stdout.lowstdout():
         paid_content = await paid.get_all_paid_posts()
         user_dict = {}
+
         for ele in paid_content:
             user_id = ele.get("fromUser", {}).get("id") or ele.get("author", {}).get(
                 "id"
@@ -394,11 +395,10 @@ async def process_all_paid():
         output = {}
         for model_id, value in user_dict.items():
             username = profile.scrape_profile(model_id).get("username")
-            if (
-                username == "modeldeleted"
-                and await operations.check_profile_table_exists(
-                    model_id=model_id, username=username
-                )
+            if username == constants.getattr(
+                "DELETED_MODEL_PLACEHOLDER"
+            ) and await operations.check_profile_table_exists(
+                model_id=model_id, username=username
             ):
                 username = (
                     await operations.get_profile_info(
@@ -421,7 +421,7 @@ async def process_all_paid():
                 for post in all_posts
                 if post.id not in seen and not seen.add(post.id)
             ]
-            new_medias = [item for post in new_posts for item in post.media]
+            new_medias = [item for post in new_posts for item in post.all_media]
             new_medias = filters.filterMedia(new_medias)
             new_posts = filters.filterPost(new_posts)
             await operations.make_post_table_changes(
@@ -455,22 +455,22 @@ async def process_labels(model_id, username, c):
         with stdout.lowstdout():
             labelled_posts_ = await labels_api.get_labels_progress(model_id, c=c)
 
-            labelled_posts_ = list(
+            labelled_posts_labels = list(
                 map(lambda x: labels.Label(x, model_id, username), labelled_posts_)
             )
             await operations.make_label_table_changes(
-                    labelled_posts_,
-                    model_id=model_id,
-                    username=username,
+                labelled_posts_labels,
+                model_id=model_id,
+                username=username,
             )
 
             log.debug(
-                f"[bold]Label media count with locked[/bold] {sum(map(lambda x:len(x),[post.post_media for labelled_post in labelled_posts_ for post in labelled_post.posts]))}"
+                f"[bold]Label media count with locked[/bold] {sum(map(lambda x:len(x),[post.post_media for labelled_post in labelled_posts_labels for post in labelled_post.posts]))}"
             )
             log.debug("Removing locked messages media")
             output = [
-                post.media
-                for labelled_post in labelled_posts_
+                post.all_media
+                for labelled_post in labelled_posts_labels
                 for post in labelled_post.posts
             ]
             log.debug(
@@ -478,7 +478,7 @@ async def process_labels(model_id, username, c):
             )
 
             return [item for sublist in output for item in sublist], [
-                post for ele in labelled_posts_ for post in ele.posts
+                post for ele in labelled_posts_labels for post in ele.posts
             ]
     except Exception as E:
         try:
@@ -513,7 +513,12 @@ async def process_task(model_id, username, ele):
     postObjs = []
     final_post_areas = set(areas.get_download_area())
     tasks = []
-    async with sessionbuilder.sessionBuilder() as c:
+    async with sessionManager.sessionManager(
+        sem=constants.getattr("API_REQ_SEM_MAX"),
+        retries=constants.getattr("API_NUM_TRIES"),
+        wait_min=constants.getattr("OF_MIN_WAIT_API"),
+        wait_max=constants.getattr("OF_MAX_WAIT_API"),
+    ) as c:
         while True:
             max_count = min(
                 constants.getattr("API_MAX_AREAS"),