ofscraper 3.10.dev5__tar.gz → 3.10.dev7__tar.gz

{ofscraper-3.10.dev5 → ofscraper-3.10.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ofscraper
-Version: 3.10.dev5
+Version: 3.10.dev7
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net

{ofscraper-3.10.dev5 → ofscraper-3.10.dev7}/ofscraper/api/labels.py

@@ -27,8 +27,8 @@ log = logging.getLogger("shared")
 
 
 @run
-async def get_labels_progress(model_id, c=None):
-    labels_ = await get_labels_data_progress(model_id, c=c)
+async def get_labels(model_id, c=None):
+    labels_ = await get_labels_data(model_id, c=c)
     labels_ = (
         labels_
         if not read_args.retriveArgs().label
@@ -39,19 +39,18 @@ async def get_labels_progress(model_id, c=None):
             )
         )
     )
-    return await get_posts_for_labels_progress(labels_, model_id, c=c)
+    return await get_posts_for_labels(labels_, model_id, c=c)
 
 
 @run
-async def get_labels_data_progress(model_id, c=None):
+async def get_labels_data(model_id, c=None):
     tasks = []
     tasks.append(asyncio.create_task(scrape_labels(c, model_id)))
-    progress_utils.labelled_layout.visible = False
     return await process_tasks_labels(tasks)
 
 
 @run
-async def get_posts_for_labels_progress(labels, model_id, c=None):
+async def get_posts_for_labels(labels, model_id, c=None):
     tasks = []
 
     [
@@ -59,43 +58,11 @@ async def get_posts_for_labels_progress(labels, model_id, c=None):
         for label in labels
     ]
     labels_final = await process_tasks_get_posts_for_labels(tasks, labels, model_id)
-    progress_utils.labelled_layout.visible = False
     return labels_final
 
 
-@run
-async def get_labels(model_id, c=None):
-    labels_ = await get_labels_data(model_id, c=c)
-    labels_ = (
-        labels_
-        if not read_args.retriveArgs().label
-        else list(
-            filter(
-                lambda x: x.get("name").lower() in read_args.retriveArgs().label,
-                labels_,
-            )
-        )
-    )
-    return await get_posts_for_labels(labels_, model_id, c=c)
 
 
-@run
-async def get_labels_data(model_id, c=None):
-    with progress_utils.set_up_api_labels():
-        tasks = []
-        tasks.append(asyncio.create_task(scrape_labels(c, model_id)))
-        return await process_tasks_labels(tasks)
-
-
-@run
-async def get_posts_for_labels(labels, model_id, c=None):
-    with progress_utils.set_up_api_posts_labels():
-        tasks = []
-        [
-            tasks.append(asyncio.create_task(scrape_posts_labels(c, label, model_id)))
-            for label in labels
-        ]
-        return await process_tasks_get_posts_for_labels(tasks, labels, model_id)
 
 
 async def process_tasks_labels(tasks):

{ofscraper-3.10.dev5 → ofscraper-3.10.dev7}/ofscraper/classes/sessionmanager.py

@@ -60,29 +60,29 @@ class SessionSleep:
         log = logging.getLogger("shared")
         if not self._sleep:
             self._sleep = self._init_sleep if self._init_sleep else constants.getattr("SESSION_SLEEP_INIT")
-            log.debug(f"too many req => setting sleep to init [{self._sleep} seconds]")
+            log.debug(f"too many req => setting sleep to init \\[{self._sleep} seconds]")
         elif arrow.now().float_timestamp - self._last_date.float_timestamp < self._difmin:
             log.debug(
-                f"too many req => not changing sleep [{self._sleep} seconds] because last call less than {self._difmin} seconds"
+                f"too many req => not changing sleep \\[{self._sleep} seconds] because last call less than {self._difmin} seconds"
             )
             return self._sleep
         else:
             self._sleep = self._sleep * 2
-            log.debug(f"too many req => setting sleep to [{self._sleep} seconds]")
+            log.debug(f"too many req => setting sleep to \\[{self._sleep} seconds]")
         self._last_date = arrow.now()
         return self._sleep
 
     async def async_do_sleep(self):
         if self._sleep:
             logging.getLogger("shared").debug(
-                f"too many req => waiting [{self._sleep} seconds] before next req"
+                f"too many req => waiting \\[{self._sleep} seconds] before next req"
             )
             await asyncio.sleep(self._sleep)
 
     def do_sleep(self):
         if self._sleep:
             logging.getLogger("shared").debug(
-                f"too many req => waiting [{self._sleep} seconds] before next req"
+                f"too many req => waiting \\[{self._sleep} seconds] before next req"
             )
             time.sleep(self._sleep)
 

{ofscraper-3.10.dev5 → ofscraper-3.10.dev7}/ofscraper/commands/check.py

@@ -35,7 +35,7 @@ import ofscraper.utils.system.network as network
 from ofscraper.classes.table.row_names import row_names_all
 from ofscraper.commands.helpers.strings import check_str
 from ofscraper.db.operations_.media import batch_mediainsert, get_media_ids_downloaded
-from ofscraper.download.shared.utils.text import textDownloader
+from ofscraper.download.shared.text import textDownloader
 from ofscraper.utils.context.run_async import run
 
 log = logging.getLogger("shared")
@@ -161,7 +161,7 @@ def post_checker():
 @run
 async def post_check_runner():
     async for user_name, model_id, final_post_array in post_check_retriver():
-        with progress_utils.setup_api_split_progress_live(stop=True):
+        with progress_utils.setup_api_split_progress_live(revert=True):
             progress_utils.update_activity_task(
                 description=check_str.format(
                     username=user_name, activity="Timeline posts"
@@ -331,7 +331,7 @@ def message_checker():
 @run
 async def message_checker_runner():
     async for user_name, model_id, final_post_array in message_check_retriver():
-        with progress_utils.setup_api_split_progress_live(stop=True):
+        with progress_utils.setup_api_split_progress_live(revert=True):
             progress_utils.update_activity_task(
                 description=check_str.format(username=user_name, activity="Messages")
             )
@@ -418,7 +418,7 @@ def purchase_checker():
 @run
 async def purchase_checker_runner():
     async for user_name, model_id, final_post_array in purchase_check_retriver():
-        with progress_utils.setup_api_split_progress_live(stop=True):
+        with progress_utils.setup_api_split_progress_live(revert=True):
             progress_utils.update_activity_task(
                 description=check_str.format(
                     username=user_name, activity="Purchased posts"
@@ -482,7 +482,7 @@ def stories_checker():
 @run
 async def stories_checker_runner():
     async for user_name, model_id, final_post_array in stories_check_retriver():
-        with progress_utils.setup_api_split_progress_live(stop=True):
+        with progress_utils.setup_api_split_progress_live(revert=True):
             progress_utils.update_activity_task(
                 description=check_str.format(
                     username=user_name, activity="Stories posts"

ofscraper-3.10.dev7/ofscraper/commands/helpers/data.py

@@ -0,0 +1,39 @@
+
+import logging
+
+import ofscraper.models.selector as selector
+import ofscraper.utils.args.helpers.areas as areas
+import ofscraper.utils.constants as constants
+import ofscraper.utils.live.screens as progress_utils
+from ofscraper.commands.scraper.post import post_media_process
+
+from ofscraper.commands.helpers.strings import (
+    area_str,
+    avatar_str,
+    data_str,
+    progress_str,
+)
+log = logging.getLogger("shared")
+def data_helper(user):
+    avatar = user.avatar
+    username = user.name
+    active = user.active
+    final_post_areas = areas.get_final_posts_area()
+    length = selector.get_num_selected()
+    count = progress_utils.get_user_task_obj().completed
+
+
+    logging.getLogger("shared_other").warning(
+        progress_str.format(count=count + 1, length=length)
+    )
+    logging.getLogger("shared_other").warning(data_str.format(name=username))
+    if constants.getattr("SHOW_AVATAR") and avatar:
+        logging.getLogger("shared_other").warning(avatar_str.format(avatar=avatar))
+    progress_utils.update_activity_task(
+        description=area_str.format(
+            areas=",".join(final_post_areas), name=username, active=active
+        )
+    )
+    logging.getLogger("shared_other").info(
+        area_str.format(areas=",".join(final_post_areas), name=username, active=active)
+    )

ofscraper-3.10.dev7/ofscraper/commands/helpers/final_log.py

@@ -0,0 +1,14 @@
+import logging
+import time
+import ofscraper.utils.constants as constants
+
+def final_log(data):
+    log = logging.getLogger("shared") 
+    if len(data)<3:
+        return
+    elif constants.getattr("SHOW_RESULTS_LOG"):
+        log.warning("[bold yellow]Final Results Logs[/bold yellow]")
+        for record in data:
+            log.warning(record)
+    #give time for last long to process
+    time.sleep(3)

ofscraper-3.10.dev7/ofscraper/commands/helpers/normal.py

@@ -0,0 +1,52 @@
+import logging
+import traceback
+
+
+import ofscraper.utils.args.read as read_args
+import ofscraper.utils.constants as constants
+import ofscraper.utils.live.screens as progress_utils
+from ofscraper.commands.scraper.post import post_media_process
+
+from ofscraper.commands.helpers.strings import (
+    avatar_str,
+)
+log = logging.getLogger("shared")
+from ofscraper.commands.helpers.data import data_helper
+
+
+def get_user_action_function(func):
+    async def wrapper(userdata,session,*args, **kwargs):
+        async with session as c:
+            data=["[bold yellow]Normal Mode Results[/bold yellow]"]
+            for ele in userdata:
+                try:
+                    with progress_utils.setup_api_split_progress_live():
+                        data_helper(ele)
+                        all_media, posts, like_posts = await post_media_process(ele, c=c)
+                    with progress_utils.setup_activity_group_live(revert=False):
+                        avatar = ele.avatar
+                        if (
+                            constants.getattr("SHOW_AVATAR")
+                            and avatar
+                            and read_args.retriveArgs ().userfirst
+                        ):
+                            logging.getLogger("shared_other").warning(avatar_str.format(avatar=avatar))
+                        data.extend(await func(media=all_media, posts=posts, like_posts=like_posts,ele=ele))
+                except Exception as e:
+
+                    log.traceback_(f"failed with exception: {e}")
+                    log.traceback_(traceback.format_exc())
+
+                    if isinstance(e, KeyboardInterrupt):
+                        raise e
+                finally:
+                    progress_utils.increment_user_activity()
+            return data
+    return wrapper
+
+
+
+
+
+
+

ofscraper-3.10.dev7/ofscraper/commands/helpers/scrape_paid.py

@@ -0,0 +1,69 @@
+import logging
+
+import ofscraper.api.profile as profile
+import ofscraper.classes.models as models
+import ofscraper.commands.scraper.post as OF
+import ofscraper.download.download as download
+import ofscraper.models.selector as userselector
+import ofscraper.utils.live.screens as progress_utils
+from ofscraper.commands.helpers.strings import all_paid_download_str,metadata_activity_str, all_paid_progress_metadata_str
+
+from ofscraper.utils.context.run_async import run
+
+log = logging.getLogger("shared")
+
+@run
+async def process_scrape_paid():
+    progress_utils.update_activity_task(
+                    description="Scraping Entire Paid page"
+    )
+    with progress_utils.setup_all_paid_database_live():
+        async for ele in process_paid_dict():
+            yield ele
+        
+
+async def process_paid_dict():
+    user_dict = await OF.process_all_paid()
+    length = len(list(user_dict.keys()))
+    progress_utils.update_activity_count(totat=length, completed=0)
+
+
+    for count, value in enumerate(user_dict.values()):
+        yield count,value,length
+
+
+
+def process_user_info_printer(value,length,count,all_paid_activity=None,all_paid_update=None,log_progress=None):
+    model_id = value["model_id"]
+    username = value["username"]
+
+    all_paid_update=all_paid_update or all_paid_download_str
+    all_paid_activity=all_paid_activity or metadata_activity_str
+    log_progress=log_progress or all_paid_progress_metadata_str
+
+    progress_utils.update_activity_count(
+                totat=length,
+                description=all_paid_update.format(username=username),
+    )
+    progress_utils.update_activity_task(
+        description=(all_paid_activity.format(username=username,model_id=model_id,count=count+1,length=length)
+    ))
+    logging.getLogger("shared_other").warning(
+        log_progress.format(username=username,model_id=model_id,count=count+1,length=length)
+    )
+
+async def process_user(value,length):
+    model_id = value["model_id"]
+    username = value["username"]
+    posts = value["posts"]
+    medias = value["medias"]
+
+    userselector.set_ALL_SUBS_DICTVManger(
+        {
+            username: models.Model(
+                profile.scrape_profile(model_id, refresh=False)
+            )
+        }
+    )
+    progress_utils.increment_activity_count(total=length)
+    return (await download.download_process(username, model_id, medias, posts=posts))

{ofscraper-3.10.dev5 → ofscraper-3.10.dev7}/ofscraper/commands/helpers/shared.py

@@ -3,3 +3,8 @@ import ofscraper.utils.args.read as read_args
 
 def run_action_bool():
     return len(read_args.retriveArgs().action) > 0
+
+
+
+def run_metadata_bool():
+    return read_args.retriveArgs().metadata

{ofscraper-3.10.dev5 → ofscraper-3.10.dev7}/ofscraper/commands/helpers/strings.py

@@ -4,9 +4,18 @@ avatar_str = "Avatar : {avatar}"
 area_str = "Getting \[[bold blue]{areas}[/bold blue]] for [bold]{name}[/bold]\n[bold]Subscription Active:[/bold] {active}"
 all_paid_model_id_str = "getting username data for {model_id}"
 all_paid_str = "Creating databases for {username}"
-download_str = "Performing Downloading Action on {name}"
+download_activity_str = "Performing Downloading Action on {username}"
 all_paid_download_str = "Users With paid content downloaded"
-metadata_str = "Updating Metadata for {username}"
+all_paid_metadata_str = "Users With metadata updated downloaded"
+
+metadata_activity_str = "Updating Metadata for {username}"
+
 post_str_manual = "Manual Mode: Getting media from {url}"
 download_manual_str = "Downloading individual media for {username}"
 check_str = "Checking {activity} for {username}"
+
+all_paid_progress_download_str="\\[{model_id}_{username}] Downloading Progress :{count}/{length} models"
+
+all_paid_progress_metadata_str="\\[{model_id}_{username}] Metadata Update Progress :{count}/{length} models"
+
+mark_stray_str="Marking Stray Items for {username} as downloaded"

ofscraper-3.10.dev5/ofscraper/commands/helpers/context.py → ofscraper-3.10.dev7/ofscraper/commands/helpers/user_first.py

@@ -1,63 +1,17 @@
 import logging
 import traceback
-from itertools import chain
 
 
-
-import ofscraper.models.selector as selector
-import ofscraper.utils.args.helpers.areas as areas
 import ofscraper.utils.args.read as read_args
 import ofscraper.utils.constants as constants
 import ofscraper.utils.live.screens as progress_utils
 from ofscraper.commands.scraper.post import post_media_process
 
 from ofscraper.commands.helpers.strings import (
-    area_str,
     avatar_str,
-    data_str,
-    progress_str,
 )
 log = logging.getLogger("shared")
-
-
-def get_user_action_function(func):
-    async def wrapper(userdata,session,*args, **kwargs):
-        async with session as c:
-            data=[]
-            for ele in userdata:
-                try:
-                    with progress_utils.setup_api_split_progress_live():
-                        all_media, posts, like_posts = await post_media_process(ele, c=c)
-                    with progress_utils.setup_activity_counter_live(revert=False):
-                        avatar = ele.avatar
-                        if (
-                            constants.getattr("SHOW_AVATAR")
-                            and avatar
-                            and read_args.retriveArgs ().userfirst
-                        ):
-                            logging.getLogger("shared_other").warning(avatar_str.format(avatar=avatar))
-                        data_helper(ele)
-                        data.append(await func(all_media, posts, like_posts,ele=ele))
-                except Exception as e:
-
-                    log.traceback_(f"failed with exception: {e}")
-                    log.traceback_(traceback.format_exc())
-
-                    if isinstance(e, KeyboardInterrupt):
-                        raise e
-                finally:
-                    progress_utils.increment_user_activity()
-            records=list(chain.from_iterable(data))
-
-            if len(records)<2:
-                return
-            for record in records:
-                log.warning(record)
-
-    return wrapper
-
-
-
+from ofscraper.commands.helpers.data import data_helper
 
 def get_userfirst_data_function(funct):
     async def wrapper(userdata, session, *args, **kwargs):
@@ -86,6 +40,7 @@ def get_userfirst_data_function(funct):
 
 def get_userfirst_action_execution_function(funct):
     async def wrapper(data,*args, **kwargs):
+        out=["[bold yellow]User First Results[/bold yellow]"]
         progress_utils.increment_activity_count(total=2)
         try:
              progress_utils.update_user_activity(total=len(data.items()))
@@ -103,7 +58,7 @@ def get_userfirst_action_execution_function(funct):
                     logging.getLogger("shared_other").warning(avatar_str.format(avatar=avatar))
                 try:
                     with progress_utils.setup_activity_counter_live(revert=False):
-                        await funct(all_media, posts, like_posts,*args, ele=ele,**kwargs)
+                        out.extend(await funct(posts, like_posts,*args,media=all_media, ele=ele,**kwargs))
                 except Exception as e:
                     log.traceback_(f"failed with exception: {e}")
                     log.traceback_(traceback.format_exc())
@@ -118,31 +73,6 @@ def get_userfirst_action_execution_function(funct):
                 raise e
         finally:
             progress_utils.increment_activity_count(description="Overall progress", total=2)
-
+        return out
     return wrapper
 
-
-
-def data_helper(user):
-    avatar = user.avatar
-    username = user.name
-    active = user.active
-    final_post_areas = areas.get_final_posts_area()
-    length = selector.get_num_selected()
-    count = progress_utils.get_user_task_obj().completed
-
-
-    logging.getLogger("shared_other").warning(
-        progress_str.format(count=count + 1, length=length)
-    )
-    logging.getLogger("shared_other").warning(data_str.format(name=username))
-    if constants.getattr("SHOW_AVATAR") and avatar:
-        logging.getLogger("shared_other").warning(avatar_str.format(avatar=avatar))
-    progress_utils.update_activity_task(
-        description=area_str.format(
-            areas=",".join(final_post_areas), name=username, active=active
-        )
-    )
-    logging.getLogger("shared_other").info(
-        area_str.format(areas=",".join(final_post_areas), name=username, active=active)
-    )