ofscraper 3.10.dev10__tar.gz → 3.10.2__tar.gz

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ofscraper
-Version: 3.10.dev10
+Version: 3.10.2
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net
@@ -119,6 +119,9 @@ For detailed instructions on:
 
 Open a issue in this repo, or you can mention your issue in the [Discord](#discord)
 
+**Feature request issues are fine**
+**Bug Report Issues without required material will be closed**
+
 ### Private Reports
 
 A ticket can be created in the ticket channel

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/README.md

@@ -65,6 +65,9 @@ For detailed instructions on:
 
 Open a issue in this repo, or you can mention your issue in the [Discord](#discord)
 
+**Feature request issues are fine**
+**Bug Report Issues without required material will be closed**
+
 ### Private Reports
 
 A ticket can be created in the ticket channel

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/__version__.py

@@ -22,7 +22,7 @@ try:
 
     __hardcoded__ = None
     __version__ = __hardcoded__ or Version.from_git(
-        pattern="(?P<base>\d+\.\d+\.((\d+\.\w+)|\w+))"
+        pattern="(?P<base>\d+\.\d+(\.((\d+\.\w+)|\w+)|))"
     ).serialize(format="{base}+{branch}.{commit}", metadata=False)
 except:
     import pkg_resources

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/api/subscriptions/individual.py

@@ -14,7 +14,6 @@ r"""
 import asyncio
 import logging
 import traceback
-from concurrent.futures import ThreadPoolExecutor
 
 import ofscraper.api.profile as profile
 import ofscraper.classes.sessionmanager as sessionManager

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/api/subscriptions/lists.py

@@ -15,17 +15,9 @@ import asyncio
 import contextvars
 import logging
 import traceback
-from concurrent.futures import ThreadPoolExecutor
-
-from rich.console import Group
-from rich.live import Live
-from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TextColumn
-from rich.style import Style
 
 import ofscraper.classes.sessionmanager as sessionManager
 import ofscraper.utils.args.read as read_args
-import ofscraper.utils.console as console
 import ofscraper.utils.constants as constants
 import ofscraper.utils.live.screens as progress_utils
 from ofscraper.utils.context.run_async import run
@@ -108,7 +100,7 @@ async def get_lists():
                     result, new_tasks_batch = await task
                     new_tasks.extend(new_tasks_batch)
                     page_count = page_count + 1
-                    progress_utils.userlist_overall_progress.update(
+                    progress_utils.update_userlist_task(
                         page_task,
                         description=f"UserList Pages Progress: {page_count}",
                     )
@@ -209,7 +201,7 @@ async def get_list_users(lists):
                     result, new_tasks_batch = await task
                     new_tasks.extend(new_tasks_batch)
                     page_count = page_count + 1
-                    progress_utils.userlist_overall_progress.update(
+                    progress_utils.update_userlist_task(
                         page_task,
                         description=f"UserList Users Pages Progress: {page_count}",
                     )

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/api/subscriptions/subscriptions.py

@@ -14,7 +14,6 @@ r"""
 import asyncio
 import logging
 import traceback
-from concurrent.futures import ThreadPoolExecutor
 
 from rich.console import Console
 

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/classes/media.py

@@ -402,25 +402,6 @@ class Media(base.base):
             return False
         return True
 
-    # for use in dynamic names
-    @property
-    def needs_count(self):
-        if set(["text", "postid", "post_id"]).isdisjoint(
-            [
-                name
-                for text, name, spec, conv in list(
-                    string.Formatter().parse(data.get_fileformat())
-                )
-            ]
-        ):
-            return False
-        elif len(self._post.post_media) > 1 or self.responsetype in [
-            "stories",
-            "highlights",
-        ]:
-            return True
-        return False
-
     @property
     def username(self):
         return self._post.username

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/classes/placeholder.py

@@ -17,6 +17,7 @@ import ofscraper.utils.paths.common as common_paths
 import ofscraper.utils.paths.paths as paths
 import ofscraper.utils.profiles.data as profile_data
 import ofscraper.utils.settings as settings
+from ofscraper.utils.string import parse_safe
 
 log = logging.getLogger("shared")
 
@@ -342,7 +343,7 @@ class Placeholders(basePlaceholder):
     def _addcount(self, ele, out):
         if not constants.getattr("FILE_COUNT_PLACEHOLDER"):
             return
-        elif not ele.needs_count:
+        elif not self._needs_count(ele):
             return out
         out = re.sub(" $", "", out)
         # insert count
@@ -352,6 +353,19 @@ class Placeholders(basePlaceholder):
             out = f"{out}_{ele.count}"
         return out
 
+    def _needs_count(self,ele):
+        non_unique=set(["text", "postid", "post_id","ext"])
+        file_format=parse_safe(data.get_fileformat())
+
+        if len(file_format)!=len((non_unique&file_format)):
+            return False
+        elif len(ele._post.post_media) > 1 or ele.responsetype in [
+            "stories",
+            "highlights",
+        ]:
+            return True
+        return False
+
     @property
     def filepath(self):
         return pathlib.Path(self._filepath)

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/commands/helpers/normal.py

@@ -12,6 +12,8 @@ from ofscraper.commands.helpers.strings import (
 )
 log = logging.getLogger("shared")
 from ofscraper.commands.helpers.data import data_helper
+import ofscraper.filters.media.main as filters
+
 
 
 def get_user_action_function(func):
@@ -19,10 +21,16 @@ def get_user_action_function(func):
         async with session as c:
             data=["[bold yellow]Normal Mode Results[/bold yellow]"]
             for ele in userdata:
+                username=ele.name
+                model_id=ele.id
                 try:
                     with progress_utils.setup_api_split_progress_live():
                         data_helper(ele)
                         all_media, posts, like_posts = await post_media_process(ele, c=c)
+                        all_media=filters.filtermediaFinal(all_media,username,model_id)
+                        posts=  filters.filterPostFinal(posts)
+                        like_posts=filters.post_filter_for_like(like_posts)
+
                     with progress_utils.setup_activity_group_live(revert=False):
                         avatar = ele.avatar
                         if (
@@ -47,6 +55,36 @@ def get_user_action_function(func):
 
 
 
+def get_user_action_function_meta(func):
+    async def wrapper(userdata,session,*args, **kwargs):
+        async with session as c:
+            data=["[bold yellow]Normal Mode Results[/bold yellow]"]
+            for ele in userdata:
+                try:
+                    with progress_utils.setup_api_split_progress_live():
+                        data_helper(ele)
+                        all_media, posts, like_posts = await post_media_process(ele, c=c)
+                    with progress_utils.setup_activity_group_live(revert=False):
+                        avatar = ele.avatar
+                        if (
+                            constants.getattr("SHOW_AVATAR")
+                            and avatar
+                            and read_args.retriveArgs ().userfirst
+                        ):
+                            logging.getLogger("shared_other").warning(avatar_str.format(avatar=avatar))
+                        data.extend(await func(media=all_media, posts=posts, like_posts=like_posts,ele=ele))
+                except Exception as e:
+
+                    log.traceback_(f"failed with exception: {e}")
+                    log.traceback_(traceback.format_exc())
+
+                    if isinstance(e, KeyboardInterrupt):
+                        raise e
+                finally:
+                    progress_utils.increment_user_activity()
+            return data
+    return wrapper
+
 
 
 

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/commands/metadata.py

@@ -29,7 +29,7 @@ import ofscraper.utils.live.screens as progress_utils
 import ofscraper.utils.profiles.tools as profile_tools
 from ofscraper.__version__ import __version__
 from ofscraper.commands.helpers.normal import (
-    get_user_action_function,
+    get_user_action_function_meta,
 )
 from ofscraper.commands.helpers.user_first import (
     get_userfirst_action_execution_function,
@@ -38,7 +38,7 @@ from ofscraper.commands.helpers.user_first import (
 from ofscraper.commands.helpers.shared import run_metadata_bool
 from ofscraper.commands.helpers.strings import metadata_activity_str,mark_stray_str,all_paid_metadata_str,all_paid_progress_metadata_str
 
-from ofscraper.commands.scraper.post import process_areas_helper
+from ofscraper.commands.scraper.post import process_areas
 from ofscraper.commands.scraper.scrape_context import scrape_context_manager
 from ofscraper.db.operations_.media import (
     batch_set_media_downloaded,
@@ -53,7 +53,7 @@ log = logging.getLogger("shared")
 
 
 def metadata():
-    with progress_utils.setup_activity_progress_live(revert=True):
+    with progress_utils.setup_activity_progress_live(revert=True,stop=True,setup=True):
         scrape_paid_data=[]
         userfirst_data=[]
         normal_data=[]
@@ -73,7 +73,7 @@ def metadata():
 
 @run
 async def process_users_metadata_normal(userdata, session):
-    user_action_funct = get_user_action_function(execute_metadata_action_on_user)
+    user_action_funct = get_user_action_function_meta(execute_metadata_action_on_user)
     progress_utils.update_user_activity(description="Users with Updated Metadata")
     return await user_action_funct(userdata, session)
 
@@ -143,11 +143,10 @@ async def execute_metadata_action_on_user(*args,ele=None, media=None,**kwargs):
 
     model_id = ele.id
     await operations.table_init_create(model_id=model_id, username=username)
-    filterMedia = filters.filterMedia(media, username=username, model_id=model_id)
     progress_utils.update_activity_task(
         description=metadata_activity_str.format(username=username)
     )
-    data=await download.download_process(username, model_id, filterMedia)
+    data=await download.download_process(username, model_id, media)
     metadata_stray_media(username, model_id, media)
     return [data]
 
@@ -185,7 +184,7 @@ async def process_ele_user_first_data_retriver(ele=None, session=None):
         model_id = ele.id
         username = ele.name
         await operations.table_init_create(model_id=model_id, username=username)
-        media, _, _ = await process_areas_helper(ele, model_id, c=session)
+        media, _, _ = await process_areas(ele, model_id, c=session)
         return {
             model_id: {
                 "username": username,

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/commands/scraper/actions/download.py

@@ -10,6 +10,8 @@ r"""_____  _______         _______  _______  _______  _______  _______  _______
 
 import logging
 import time
+import pathlib
+
 
 import ofscraper.classes.placeholder as placeholder
 import ofscraper.download.download as download
@@ -18,6 +20,10 @@ import ofscraper.utils.live.screens as progress_utils
 from ofscraper.commands.helpers.strings import  download_activity_str,all_paid_progress_download_str,all_paid_download_str
 from ofscraper.commands.helpers.scrape_paid import process_scrape_paid,process_user_info_printer,process_user
 from ofscraper.utils.context.run_async import run
+import ofscraper.utils.config.data as config_data
+import ofscraper.utils.paths.common as common_paths
+from ofscraper.utils.string import format_safe
+
 
 
 log = logging.getLogger("shared")
@@ -26,7 +32,10 @@ log = logging.getLogger("shared")
 async def downloader(ele=None, posts=None, media=None, **kwargs):
     model_id = ele.id
     username = ele.name
-    progress_utils.update_activity_task(description=download_activity_str.format(username=username))
+    download_str=download_activity_str.format(username=username)
+    path_str=format_safe(f"\nSaving files to [deep_sky_blue2]{str(pathlib.Path(common_paths.get_save_location(),config_data.get_dirformat(),config_data.get_fileformat()))}[/deep_sky_blue2]",username=username,model_id=model_id,model_username=username,modelusername=username,modelid=model_id)
+
+    progress_utils.update_activity_task(description=download_str+path_str)
     logging.getLogger("shared_other").warning(download_activity_str.format(username=username))
     return await download.download_process(username, model_id, media, posts=posts)
 

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/commands/scraper/actions/like.py

@@ -14,6 +14,7 @@ r"""
 import functools
 import logging
 import time
+import random
 
 import ofscraper.classes.sessionmanager as sessionManager
 import ofscraper.utils.args.read as read_args
@@ -22,12 +23,12 @@ import ofscraper.utils.constants as constants
 import ofscraper.utils.context.exit as exit
 import ofscraper.utils.live.screens as progress_utils
 from rich.markup import escape
-from rich.text import Text
 
 
 log = logging.getLogger("shared")
-like_str = "Performing Like Action on {name}"
-unlike_str = "Performing Unlike Action on {name}"
+warning_str="\n[red]Current Like rate can eat through the ~1000 like limit in ~1 hour\nIncreasing the rate could lead to getting logged out[/red]"
+like_str = "Performing Like Action on {name}" + warning_str
+unlike_str = "Performing Unlike Action on {name}"+warning_str
 
 
 @exit.exit_wrapper
@@ -114,30 +115,20 @@ def _like(model_id, username,ids: list, like_action: bool):
             task2 = progress_utils.add_like_task(like_str, total=None)
 
             [tasks.append(functools.partial(like_func, c, id, model_id)) for id in ids]
-            count = 1
-
-            stable_sleep_duration=constants.getattr("STABLE_SLEEP_DURATION_LIKE")
-            sleep_duration_50=constants.getattr("SLEEP_DURATION_LIKE_50")
-            sleep_duration_60=constants.getattr("SLEEP_DURATION_LIKE_60")
-            sleep_duration_common=constants.getattr("COMMON_MULTIPLE_SLEEP_DURATION_LIKE")
+            max_duration=constants.getattr("MAX_SLEEP_DURATION_LIKE")
+            min_duration=constants.getattr("MIN_SLEEP_DURATION_LIKE")
             failed=0
             post=0
             liked=0
 
-            for count, func in enumerate(tasks):
+            for _, func in enumerate(tasks):
                 out = func()
                 post+=1
                 #sleep
                 if out == 0:
                     sleep_duration = 0
-                elif count + 1 % 60 == 0 and count + 1 % 50 == 0:
-                    sleep_duration = sleep_duration_common
-                elif count % 60 == 0:
-                    sleep_duration = sleep_duration_60  # Divisible by 60 - 1 second sleep
-                elif count % 50 == 0:
-                    sleep_duration = sleep_duration_50   # Divisible by 50 - 30 seconds sleep
                 else:
-                    sleep_duration =  stable_sleep_duration
+                    sleep_duration=random.uniform(min_duration, max_duration)
                 #values
                 if out == 1:
                     liked=+1
@@ -178,7 +169,10 @@ def _toggle_like_requests(c, id, model_id):
     if not read_args.retriveArgs().force_like and cache.get(f"liked_status_{id}", None):
         log.debug(f"ID: {id} marked as liked in cache")
         return 0
-    sleep_duration = constants.getattr("DOUBLE_TOGGLE_SLEEP_DURATION_LIKE")
+    max_duration=constants.getattr("MAX_SLEEP_DURATION_LIKE")
+    min_duration=constants.getattr("MIN_SLEEP_DURATION_LIKE")
+
+    sleep_duration = random.uniform(min_duration, max_duration)
     favorited, id = _like_request(c, id, model_id,sleeper)
     if favorited==None:
         return 3

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/commands/scraper/post.py

@@ -27,7 +27,6 @@ import ofscraper.api.timeline as timeline
 import ofscraper.classes.labels as labels
 import ofscraper.classes.media as media
 import ofscraper.classes.posts as posts_
-import ofscraper.classes.sessionmanager as sessionManager
 import ofscraper.db.operations as operations
 import ofscraper.filters.media.main as filters
 import ofscraper.utils.args.read as read_args
@@ -58,13 +57,13 @@ async def post_media_process(ele, c=None):
 
     username = ele.name
     model_id = ele.id
-    console.get_shared_console().clear()
-    console.get_shared_console().clear_live()
     await operations.table_init_create(model_id=model_id, username=username)
     data = await process_areas(ele, model_id, username, c=c)
     return data
 
 
+
+
 @free.space_checker
 async def process_messages(model_id, username, c):
     try:
@@ -331,28 +330,29 @@ async def process_all_paid():
                 for post in all_posts
                 if post.id not in seen and not seen.add(post.id)
             ]
-            new_medias = [item for post in new_posts for item in post.all_media]
-            new_medias = filters.filterMedia(
-                new_medias, model_id=model_id, username=username
+            all_medias = [item for post in new_posts for item in post.all_media]
+            insert_media = filters.filtermediaAreas(
+                all_medias, model_id=model_id, username=username
             )
-            new_posts = filters.filterPost(new_posts)
+            new_posts = filters.filterPostFinal(new_posts)
             await operations.make_post_table_changes(
                 new_posts,
                 model_id=model_id,
                 username=username,
             )
             await batch_mediainsert(
-                new_medias,
+                insert_media,
                 model_id=model_id,
                 username=username,
                 downloaded=False,
             )
+            final_medias=filters.filtermediaFinal(insert_media,username,model_id)
 
             output[model_id] = dict(
-                model_id=model_id, username=username, posts=new_posts, medias=new_medias
+                model_id=model_id, username=username, posts=new_posts, medias=final_medias
             )
             log.debug(
-                f"[bold]Paid media count {username}_{model_id}[/bold] {len(new_medias)}"
+                f"[bold]Paid media count {username}_{model_id}[/bold] {len(final_medias)}"
             )
             progress_utils.increment_activity_count(total=None)
 
@@ -408,35 +408,22 @@ async def process_labels(model_id, username, c):
         log.traceback_(traceback.format_exc())
 
 
+
+
+
 @run
-async def process_areas_helper(ele, model_id, c=None) -> list:
+async def process_areas(ele, model_id, username, c=None):
     try:
         username = ele.name
-        output = []
         medias, posts, like_post = await process_tasks(model_id, username, ele, c=c)
+        insert_medias=filters.filtermediaAreas(medias)
         await batch_mediainsert(
-            medias,
+            insert_medias,
             model_id=model_id,
             username=username,
             downloaded=False,
         )
-        output.extend(medias)
-        return (medias, posts, like_post)
-    except Exception as E:
-        log.traceback_(E)
-        log.traceback_(traceback.format_exc())
-
-
-@run
-async def process_areas(ele, model_id, username, c=None):
-    
-    media, posts, like_post = await process_areas_helper(ele, model_id, c=c)
-    try:
-        return (
-            filters.filterMedia(media, model_id=model_id, username=username),
-            filters.filterPost(posts),
-            filters.post_filter_for_like(like_post),
-        )
+        return (insert_medias, posts, like_post)
     except Exception as E:
         log.traceback_(E)
         log.traceback_(traceback.format_exc())

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/commands/scraper/runner.py

@@ -43,7 +43,7 @@ def runner():
         user_first_data=[
         ]
         scrape_paid_data=[]
-        with progress_utils.setup_activity_group_live(setup=True,revert=False):
+        with progress_utils.setup_activity_group_live(setup=True,revert=False,stop=True):
             if read_args.retriveArgs().scrape_paid:
                 scrape_paid_data=download_action.scrape_paid_all()
 

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/const/values/config.py

@@ -9,7 +9,7 @@ KEY_DEFAULT = "cdrm"
 DIR_FORMAT_DEFAULT = "{model_username}/{responsetype}/{mediatype}/"
 FILE_FORMAT_DEFAULT = "{filename}.{ext}"
 METADATA_DEFAULT = "{configpath}/{profile}/.data/{model_username}_{model_id}"
-FILE_SIZE_LIMIT_DEFAULT = 0
+FILE_SIZE_MAX_DEFAULT = 0
 FILE_SIZE_MIN_DEFAULT = 0
 TEXTLENGTH_DEFAULT = 0
 SPACE_REPLACER_DEFAULT = " "

ofscraper-3.10.2/ofscraper/const/values/like.py

@@ -0,0 +1,6 @@
+MAX_SLEEP_DURATION_LIKE=1.5
+MIN_SLEEP_DURATION_LIKE=.9
+SESSION_429_LIKE_INCREASE_SLEEP_TIME_DIF=0
+SESSION_429_SLEEP_STARTER_VAL=8
+LIKE_MAX_RETRIES=5
+

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/const/values/test_constants.py

@@ -7321,7 +7321,7 @@ DATE_ALT = "YY"
 CONFIG_KEYS = [
     "main_profile",
     "save_location",
-    "file_size_limit",
+    "file_size_max",
     "dir_format",
     "file_format",
     "textlength",

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/const/values/url.py

@@ -38,6 +38,7 @@ DIGITALCRIMINALS = (
 
 DEVIINT = "https://raw.githubusercontent.com/deviint/onlyfans-dynamic-rules/main/dynamicRules.json"
 SNEAKY = "https://raw.githubusercontent.com/SneakyOvis/onlyfans-dynamic-rules/main/rules.json"
+GROWIK="https://raw.githubusercontent.com/Growik/onlyfans-dynamic-rules/main/rules.json"
 
 donateEP = "https://www.buymeacoffee.com/excludedBittern"
 

{ofscraper-3.10.dev10 → ofscraper-3.10.2}/ofscraper/db/operations.py

@@ -158,12 +158,12 @@ def get_group_difference(model_id=None, username=None, db_path=None):
     groupB = [
         "profile_username_constraint_modified",
         "stories_model_id_constraint_added",
-        "media_model_id_constraint_added",
         "labels_model_id_constraint_added",
         "posts_model_id_constraint_added",
         "others_model_id_constraint_added",
         "products_model_id_constraint_added",
         "messages_model_id_constraint_added",
+        "media_post_id_constraint_added",
         "media_bool_changes",
     ]
     return set((groupA + groupB)).difference(set(changes))
@@ -189,6 +189,8 @@ def create_backup_transition(model_id=None, username=None, db_path=None, **kwarg
         return new_backup_path
 
 
+
+
 async def add_column_tables(model_id=None, username=None, db_path=None, **kwargs):
     missing = get_group_difference(
         model_id=model_id, username=username, db_path=db_path
@@ -300,11 +302,10 @@ async def modify_tables_constraints_and_columns(
             username=username,
             db_path=db_path,
         )
-    # only do one
-    if "media_model_id_constraint_added" in missing or "media_bool_changes" in missing:
+    if "media_post_id_constraint_added" in missing or "media_bool_changes" in missing:
         await rebuild_media_table(model_id=model_id, username=username, db_path=db_path)
         await add_flag_schema(
-            "media_model_id_constraint_added",
+            "media_post_id_constraint_added",
             model_id=model_id,
             username=username,
             db_path=db_path,
@@ -365,7 +366,6 @@ async def modify_tables_constraints_and_columns(
             db_path=db_path,
         )
 
-
 def check_backup(model_id, username, new_path, db_path=None, **kwargs):
     if not new_path.absolute().exists():
         raise Exception("Backup db was not created")