ofscraper 3.9.0.dev7__tar.gz → 3.9.0.dev10__tar.gz

{ofscraper-3.9.0.dev7 → ofscraper-3.9.0.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ofscraper
-Version: 3.9.0.dev7
+Version: 3.9.0.dev10
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net
@@ -17,6 +17,7 @@ Requires-Dist: arrow (>=1.3.0,<2.0.0)
 Requires-Dist: browser-cookie3 (==0.19.1)
 Requires-Dist: bs4 (>=0.0.2,<0.0.3)
 Requires-Dist: certifi (>=2024.2.2,<2025.0.0)
+Requires-Dist: cloup (>=3.0.5,<4.0.0)
 Requires-Dist: dill (>=0.3.8,<0.4.0)
 Requires-Dist: diskcache (>=5.6.3,<6.0.0)
 Requires-Dist: dunamai (>=1.19.2,<2.0.0)
@@ -68,8 +69,8 @@ Docker and binary releases also availible
 ## Dev
 
 <div style="display: inline-block">
-<a href="https://pypi.org/project/ofscraper/3.7.0.dev2/">
-<img src="https://img.shields.io/badge/Pre--Release-v3.70.dev-dark_green.svg" alt="drawing" style="height:75px"/>
+<a href="https://pypi.org/project/ofscraper/3.9.0.dev5/">
+<img src="https://img.shields.io/badge/Pre--Release-v3.90.dev-dark_green.svg" alt="drawing" style="height:75px"/>
 </a>
 </div>
 
@@ -116,6 +117,11 @@ For detailed instructions on:
 
 Open a issue in this repo, or you can mention your issue in the [Discord](#discord)
 
+## Private Reports
+
+A ticket can be created in the ticket channel
+only you and admins have access to ticket discussions
+
 # Feature Requests
 
 [ClearFlask Feedback](https://ofscraper.clearflask.com/feedback) or [Discord](#discord)

{ofscraper-3.9.0.dev7 → ofscraper-3.9.0.dev10}/README.md

@@ -17,8 +17,8 @@ Docker and binary releases also availible
 ## Dev
 
 <div style="display: inline-block">
-<a href="https://pypi.org/project/ofscraper/3.7.0.dev2/">
-<img src="https://img.shields.io/badge/Pre--Release-v3.70.dev-dark_green.svg" alt="drawing" style="height:75px"/>
+<a href="https://pypi.org/project/ofscraper/3.9.0.dev5/">
+<img src="https://img.shields.io/badge/Pre--Release-v3.90.dev-dark_green.svg" alt="drawing" style="height:75px"/>
 </a>
 </div>
 
@@ -65,6 +65,11 @@ For detailed instructions on:
 
 Open a issue in this repo, or you can mention your issue in the [Discord](#discord)
 
+## Private Reports
+
+A ticket can be created in the ticket channel
+only you and admins have access to ticket discussions
+
 # Feature Requests
 
 [ClearFlask Feedback](https://ofscraper.clearflask.com/feedback) or [Discord](#discord)

{ofscraper-3.9.0.dev7 → ofscraper-3.9.0.dev10}/ofscraper/actions/like.py

@@ -34,45 +34,108 @@ from tenacity import (
 import ofscraper.api.archive as archive
 import ofscraper.api.labels as labels_api
 import ofscraper.api.pinned as pinned
+import ofscraper.api.timeline as timeline
 import ofscraper.classes.labels as labels
+import ofscraper.classes.posts as posts_
 import ofscraper.classes.sessionbuilder as sessionbuilder
+import ofscraper.utils.args.areas as areas
+import ofscraper.utils.args.read as read_args
 import ofscraper.utils.console as console
 import ofscraper.utils.constants as constants
+import ofscraper.utils.progress as progress_utils
+import ofscraper.utils.system.system as system
 from ofscraper.classes.semaphoreDelayed import semaphoreDelayed
 from ofscraper.utils.context.run_async import run
 
-from ..api import timeline
-
 sem = semaphoreDelayed(1)
 log = logging.getLogger("shared")
-import ofscraper.utils.args.read as read_args
 
 
-def get_posts(model_id, username):
-    args = read_args.retriveArgs()
-    pinned_posts = []
-    timeline_posts = []
-    archived_posts = []
-    labelled_posts = []
-    options = args.like_area
-    if "Pinned" in options or "All" in options:
-        pinned_posts = pinned.get_pinned_post(model_id)
-    if "Timeline" in options or "All" in options:
-        timeline_posts = timeline.get_timeline_media_progress(
-            model_id, username, forced_after=0
-        )
-    if "Archived" in options or "All" in options:
-        archived_posts = archive.get_archived_media(model_id, username, forced_after=0)
-    if "Labels" in options or "All" in options:
-        labels_ = labels_api.get_labels(model_id)
-        labelled_posts_ = labels_api.get_labelled_posts(labels_, model_id)
-        labelled_posts_ = list(
-            map(lambda x: labels.Label(x, model_id, username), labelled_posts_)
+@run
+async def get_posts(model_id, username):
+    responses = []
+    final_post_areas = set(areas.get_like_area())
+    tasks = []
+    with progress_utils.setup_api_split_progress_live():
+        async with sessionbuilder.sessionBuilder() as c:
+            while True:
+                max_count = min(
+                    constants.getattr("API_MAX_AREAS"),
+                    system.getcpu_count(),
+                    len(final_post_areas),
+                )
+                if not bool(tasks) and not bool(final_post_areas):
+                    break
+                for _ in range(max_count - len(tasks)):
+                    if "Pinned" in final_post_areas:
+                        tasks.append(
+                            asyncio.create_task(
+                                pinned.get_pinned_posts_progress(model_id, c)
+                            )
+                        )
+                        progress_utils.pinned_layout.visible = True
+                        final_post_areas.remove("Pinned")
+                    elif "Timeline" in final_post_areas:
+                        tasks.append(
+                            asyncio.create_task(
+                                timeline.get_timeline_posts_progress(
+                                    model_id=model_id,
+                                    username=username,
+                                    c=c,
+                                    forced_after=read_args.retriveArgs().after or 0,
+                                )
+                            )
+                        )
+                        progress_utils.timeline_layout.visible = True
+                        final_post_areas.remove("Timeline")
+                    if "Archived" in final_post_areas:
+                        tasks.append(
+                            asyncio.create_task(
+                                archive.get_archived_posts_progress(
+                                    model_id=model_id,
+                                    username=username,
+                                    c=c,
+                                    forced_after=read_args.retriveArgs().after or 0,
+                                )
+                            )
+                        )
+                        progress_utils.archived_layout.visible = True
+                        final_post_areas.remove("Archived")
+
+                    elif "Labels" in final_post_areas:
+                        tasks.append(
+                            asyncio.create_task(
+                                labels_api.get_labels_posts_progress(
+                                    model_id=model_id, c=c
+                                )
+                            )
+                        )
+                        progress_utils.labelled_layout.visible = True
+                        final_post_areas.remove("Labels")
+                if not bool(tasks):
+                    break
+                done, pending = await asyncio.wait(
+                    tasks, return_when=asyncio.FIRST_COMPLETED
+                )
+                await asyncio.sleep(1)
+                tasks = list(pending)
+                for results in done:
+                    try:
+                        data = await results or []
+                        responses.extend(data)
+                        await asyncio.sleep(1)
+                    except Exception as E:
+                        await asyncio.sleep(1)
+                        log.debug(E)
+                        continue
+    return pre_filter(
+        list(
+            map(
+                lambda x: posts_.Post(x, model_id=model_id, username=username),
+                responses,
+            )
         )
-    log.debug(
-        f"[bold]Number of Post Found[/bold] {len(pinned_posts) + len(timeline_posts) + len(archived_posts)}"
     )
-    return pinned_posts + timeline_posts + archived_posts + labelled_posts
 
 
 def get_posts_for_unlike(model_id, username):
@@ -86,20 +149,27 @@ def get_post_for_like(model_id, username):
 
 
 def filter_for_unfavorited(posts: list) -> list:
-    output = list(filter(lambda x: x.get("isFavorite") == False, posts))
+    output = list(filter(lambda x: x.favorited == False, posts))
     log.debug(f"[bold]Number of unliked post[/bold] {len(output)}")
     return output
 
 
 def filter_for_favorited(posts: list) -> list:
-    output = list(filter(lambda x: x.get("isFavorite") == True, posts))
+    output = list(filter(lambda x: x.favorited == True, posts))
     log.debug(f"[bold]Number of liked post[/bold] {len(output)}")
     return output
 
 
+def pre_filter(posts):
+    valid_post = list(filter(lambda x: x.opened, posts))
+    seen = set()
+    return [
+        post for post in valid_post if post.id not in seen and not seen.add(post.id)
+    ]
+
+
 def get_post_ids(posts: list) -> list:
-    valid_post = list(filter(lambda x: x.get("isOpened") == True, posts))
-    return list(map(lambda x: x.get("id"), valid_post))
+    return list(map(lambda x: x.id, posts))
 
 
 def like(model_id, username, ids: list):

{ofscraper-3.9.0.dev7 → ofscraper-3.9.0.dev10}/ofscraper/actions/process.py

@@ -38,7 +38,6 @@ import ofscraper.utils.constants as constants
 import ofscraper.utils.context.exit as exit
 import ofscraper.utils.context.stdout as stdout
 import ofscraper.utils.profiles.tools as profile_tools
-from ofscraper.utils.text import get_text
 
 log = logging.getLogger("shared")
 
@@ -76,7 +75,7 @@ def process_post():
 
 
 @exit.exit_wrapper
-async def process_post_user_first():
+def process_post_user_first():
     with scrape_context_manager():
         if not placeholder.check_uniquename():
             log.warning(
@@ -96,7 +95,7 @@ async def process_post_user_first():
         # log.info(f"Data retrival progressing on model {count+1}/{length}")
 
 
-async def process_user_first_helper(ele):
+def process_user_first_helper(ele):
     if constants.getattr("SHOW_AVATAR") and ele.avatar:
         log.warning(f"Avatar : {ele.avatar}")
     if bool(areas.get_download_area()):
@@ -116,29 +115,22 @@ async def process_user_first_helper(ele):
         log.traceback_(traceback.format_exc())
 
 
-def scrape_paid(user_dict=None):
-    output = []
-    user_dict = user_dict or {}
-    output.extend(OF.process_all_paid())
-    user_dict = user_dict or {}
-    [
-        user_dict.update(
-            {ele.post.model_id: user_dict.get(ele.post.model_id, []) + [ele]}
-        )
-        for ele in output
-    ]
+def scrape_paid_all(user_dict=None):
+    user_dict = OF.process_all_paid()
     oldUsers = selector.get_ALL_SUBS_DICT()
-    for value in user_dict.values():
-        model_id = value[0].post.model_id
-        username = value[0].post.username
+    length = len(list(user_dict.keys()))
+    for count, value in enumerate(user_dict.values()):
+        model_id = value["model_id"]
+        username = value["username"]
+        posts = value["posts"]
+        medias = value["medias"]
+        log.warning(
+            f"Download paid content for {model_id}_{username} number:{count+1}/{length} models "
+        )
         selector.set_ALL_SUBS_DICTVManger(
             {username: models.Model(profile.scrape_profile(model_id))}
         )
-        download.download_process(
-            username,
-            model_id,
-            value,
-        )
+        download.download_process(username, model_id, medias, posts=posts)
     # restore og users
     selector.set_ALL_SUBS_DICT(oldUsers)
 
@@ -168,7 +160,7 @@ def normal_post_process():
             )
             try:
                 model_id = ele.id
-                operations.table_init_create(model_id, ele.name)
+                operations.table_init_create(model_id=model_id, username=ele.name)
                 combined_urls, posts = asyncio.run(OF.process_areas(ele, model_id))
                 download.download_process(
                     ele.name, model_id, combined_urls, posts=posts
@@ -198,9 +190,9 @@ def process_like():
                     f"Getting {','.join(areas.get_like_area())} for [bold]{ele.name}[/bold]\n[bold]Subscription Active:[/bold] {ele.active}"
                 )
                 model_id = ele.id
-                operations.table_init_create(model_id, ele.name)
-                unfavorited_posts = like.get_post_for_like(model_id, ele.name)
-                unfavorited_posts = filters.media_filter_for_like(
+                operations.table_init_create(model_id=model_id, username=ele.name)
+                unfavorited_posts = like.get_post_for_like(model_id=model_id, username=ele.name)
+                unfavorited_posts = filters.post_filter_for_like(
                     unfavorited_posts, like=True
                 )
                 post_ids = like.get_post_ids(unfavorited_posts)
@@ -225,9 +217,9 @@ def process_unlike():
                     f"Getting {','.join(areas.get_like_area())} for [bold]{ele.name}[/bold]\n[bold]Subscription Active:[/bold] {ele.active}"
                 )
                 model_id = profile.get_id(ele.name)
-                operations.table_init_create(model_id, ele.name)
+                operations.table_init_create(model_id=model_id, username=ele.name)
                 favorited_posts = like.get_posts_for_unlike(model_id, ele.name)
-                favorited_posts = filters.media_filter_for_like(
+                favorited_posts = filters.post_filter_for_like(
                     favorited_posts, like=False
                 )
                 post_ids = like.get_post_ids(favorited_posts)
@@ -255,5 +247,5 @@ def add_selected_areas():
         actions.select_areas()
         functs.append(process_unlike)
     if read_args.retriveArgs().scrape_paid:
-        functs.append(scrape_paid)
+        functs.append(scrape_paid_all)
     return functs