ofscraper 3.13.dev6__tar.gz → 3.13.dev7__tar.gz

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ofscraper
-Version: 3.13.dev6
+Version: 3.13.dev7
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net
@@ -71,6 +71,9 @@ As the API has changed older version are no longer viable
 
 https://github.com/datawhores/OF-Scraper/issues/442
 
+# For Issues related to Downloading/ No Models Retrieved error
+https://github.com/datawhores/OF-Scraper/issues/542
+
 # ReadMe
 
 ## Releases

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/README.md

@@ -11,6 +11,9 @@ As the API has changed older version are no longer viable
 
 https://github.com/datawhores/OF-Scraper/issues/442
 
+# For Issues related to Downloading/ No Models Retrieved error
+https://github.com/datawhores/OF-Scraper/issues/542
+
 # ReadMe
 
 ## Releases

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/classes/sessionmanager/sessionmanager.py

@@ -587,7 +587,6 @@ class sessionManager:
                     elif not r.ok:
                         log.debug(f"[bold]failed: [bold] {r.url}")
                         log.debug(f"[bold]status: [bold] {r.status}")
-                        log.debug(f"[bold]response text [/bold]: {await r.text_()}")
                         log.debug(f"response headers {dict(r.headers)}")
                         log.debug(f"requests headers mode{dict(r.request.headers)}")
                         r.raise_for_status()

ofscraper-3.13.dev7/ofscraper/data/api/common/after.py

@@ -0,0 +1,25 @@
+import logging
+
+import ofscraper.utils.settings as settings
+from ofscraper.data.api.common.cache.read import read_full_after_scan_check
+
+log= logging.getLogger("shared")
+def get_after_pre_checks(model_id, api):
+    if settings.get_settings().after is not None:
+        log.debug(f"{api}: using provided value for after")
+        if settings.get_settings().after !=0:
+            return settings.get_settings().after.float_timestamp
+        else:
+            return 0
+    elif not settings.get_settings().auto_after:
+        log.debug(f"{api}: auto-after is disabled using 0")
+        return 0
+    elif read_full_after_scan_check(model_id, api):
+        log.debug(f"{api}: full scan has been trigger")
+        return 0
+    #scan action
+    for setting in settings.get_settings().action:
+        if setting in {"like","unlike"}:
+            log.debug(f"{api}: doing full scan for action like/unlike")
+            return 0
+    

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/data/models/utils/retriver.py

@@ -23,10 +23,10 @@ async def get_models() -> list:
         return await get_via_list(count)
     elif "ALL" in settings.get_settings().usernames:
         return await get_via_list(count)
-    elif settings.get_settings().individual:
+    elif settings.get_settings().username_search=="indvidual":
         return await get_via_individual()
-    elif settings.get_settings().list:
-        return get_via_list(count)
+    elif settings.get_settings().username_search=="list":
+        return await get_via_list(count)
     elif (sum(count) // 12) >= len(settings.get_settings().usernames):
         return await get_via_individual()
     else:

ofscraper-3.13.dev7/ofscraper/utils/args/callbacks/arguments/post.py

@@ -0,0 +1,6 @@
+def set_download_type_flag(ctx, param, value):
+    # 'value' will be True if --protected-only is used,
+    # False if --normal-only is used, and None otherwise.
+    if value is None:
+        return None
+    return "protected" if value else "normal"

ofscraper-3.13.dev7/ofscraper/utils/args/callbacks/arguments/username.py

@@ -0,0 +1,9 @@
+
+
+def set_search_strategy_flag(ctx, param, value):
+    # 'value' is True for --individual, False for --list, None by default.
+    if value is True:
+        return "individual"
+    # For both --list (value=False) and the default case (value=None),
+    # we'll return "list".
+    return "list"

{ofscraper-3.13.dev6/ofscraper/utils/args/callbacks → ofscraper-3.13.dev7/ofscraper/utils/args/callbacks/parse}/file.py

@@ -1,6 +1,6 @@
 import re
 
-from ofscraper.utils.args.callbacks.string import StringSplitParse
+from ofscraper.utils.args.callbacks.parse.string import StringSplitParse
 
 
 def FileCallback(ctx, param, values):

{ofscraper-3.13.dev6/ofscraper/utils/args/callbacks → ofscraper-3.13.dev7/ofscraper/utils/args/callbacks/parse}/string.py

@@ -1,6 +1,6 @@
 import re
 
-from ofscraper.utils.args.callbacks.choice import MultiChoiceCallback
+from ofscraper.utils.args.callbacks.parse.choice import MultiChoiceCallback
 
 
 def StringSplitParse(ctx, param, value):

{ofscraper-3.13.dev6/ofscraper/utils/args/callbacks → ofscraper-3.13.dev7/ofscraper/utils/args/callbacks/parse}/username.py

@@ -1,7 +1,6 @@
-from ofscraper.utils.args.callbacks.string import StringSplitParse
-
+from ofscraper.utils.args.callbacks.parse.string import StringSplitParse
 
 def UsernameParse(ctx, param, value):
     out = StringSplitParse(ctx, param, value)
     out = list(map(lambda y: y.lower() if not y == "ALL" else y, out))
-    return out
+    return out

ofscraper-3.13.dev7/ofscraper/utils/args/parse/arguments/advanced_processing.py

@@ -0,0 +1,33 @@
+import cloup as click
+from ofscraper.utils.args.callbacks.arguments.username import set_search_strategy_flag
+
+
+users_first_option = click.option(
+    "-uf",
+    "--users-first",
+    "--user-first",
+    "--usersfirst",
+    "--userfirst",
+    "users_first",
+    help="Process all users first rather than one at a time (affects --action)",
+    default=False,
+    is_flag=True,  # Shorthand for action="store_true"
+)
+
+username_search_option = click.option(
+    "-fi/-fl",
+    "--username-individual-search/--username-list-search",
+    "--user-individual-search/--user-list-search",
+    "username_search", # The new, shared destination
+    help="""
+        \b
+        Set the search strategy when --username is provided.
+        --individual: Search each username as a separate request.
+        --list: Search entire enabled lists before filtering (default).
+    """,
+    # The default from the callback will be used.
+    # We set default=None here so the callback is triggered correctly.
+    default=None,
+    callback=set_search_strategy_flag,
+    is_flag=True
+)

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/arguments/advanced_program.py

@@ -81,7 +81,7 @@ update_profile_option = click.option(
 )
 
 download_script_option = click.option(
-    "-ds",
+    "-dls",
     "--download-script",
     "download_script",
     help="""

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/arguments/automatic.py

@@ -1,6 +1,6 @@
 import cloup as click
 
-from ofscraper.utils.args.callbacks.string import (
+from ofscraper.utils.args.callbacks.parse.string import (
     StringSplitParse,
 )
 from ofscraper.utils.args.types.choice import MutuallyExclusiveMultichoice

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/arguments/check.py

@@ -1,8 +1,8 @@
 # checkmode  args
 import cloup as click
 
-from ofscraper.utils.args.callbacks.file import FileCallback
-from ofscraper.utils.args.callbacks.string import (
+from ofscraper.utils.args.callbacks.parse.file import FileCallback
+from ofscraper.utils.args.callbacks.parse.string import (
     StringSplitParse,
     StringSplitParseTitle,
 )

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/arguments/media_content.py

@@ -3,7 +3,7 @@ import cloup as click
 # import click
 from humanfriendly import parse_size
 
-from ofscraper.utils.args.callbacks.string import (
+from ofscraper.utils.args.callbacks.parse.string import (
     StringSplitParse,
     StringSplitParseTitle,
 )
@@ -47,7 +47,7 @@ min_size_option = click.option(
 
 
 media_id_filter = click.option(
-    "-md",
+    "-mid",
     "--media-id",
     help="Filter media based on media id",
     required=False,
@@ -73,7 +73,7 @@ length_min = click.option(
 
 
 max_media_count_option = click.option(
-    "-xc",
+    "-mxc",
     "--max-media-count",
     "max_count",
     help="Maximum number of media to process",

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/arguments/metadata_filters.py

@@ -15,7 +15,7 @@ check_post_id_filter_option = click.option(
 
 
 check_media_id_filter_option = click.option(
-    "-md",
+    "-mid",
     "--media-id",
     help="Filter media based on media id",
     required=False,
@@ -23,7 +23,7 @@ check_media_id_filter_option = click.option(
 )
 
 lock_option = click.option(
-    "-ul/-l",
+    "-ucl/-lc",
     "--unlocked/--locked",
     "unlocked",
     help="""
@@ -36,7 +36,7 @@ lock_option = click.option(
 )
 
 downloaded_option = click.option(
-    "-dl/-ndl",
+    "-dwl/-ndw",
     "--downloaded/--not-downloaded",
     "downloaded",
     help="""
@@ -77,7 +77,7 @@ created_before = click.option(
 
 
 preview_option = click.option(
-    "-p/-np",
+    "-pv/-npv",
     "--preview/--no-preview",
     "preview",
     help="""
@@ -88,15 +88,3 @@ preview_option = click.option(
     required=False,
     is_flag=True,
 )
-
-
-download_type_option = preview_option = click.option(
-    "-no/-to",
-    "--normal-only/--protected_only",
-    "download_type_filter",
-    help="""
-        \b
-        Type of downloda to retrict to """,
-    default=None,
-    required=False,
-)

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/arguments/post_content.py

@@ -1,11 +1,12 @@
 import cloup as click
 
-from ofscraper.utils.args.callbacks.string import (
+from ofscraper.utils.args.callbacks.parse.string import (
     StringSplitNormalizeParse,
     StringSplitParse,
     StringSplitParseTitle,
     StringTupleList,
 )
+from ofscraper.utils.args.callbacks.arguments.post import set_download_type_flag
 from ofscraper.utils.args.types.choice import MultiChoice, MultiChoicePost
 
 # Define individual options
@@ -316,22 +317,21 @@ timeline_strict = click.option(
     is_flag=True,
 )
 
-
-protected_only = click.option(
-    "-to",
-    "--protected-only",
-    help="Restricts downloads to content that requires decryption.",
-    required=False,
-    is_flag=True,
+download_type_option = click.option(
+    "-to/-no",
+    "--protected-only/--normal-only",
+    "download_type",
+    help="""
+        \b
+        Filter downloads by type.
+        --protected-only: For content that requires decryption.
+        --normal-only: For content that does not require decryption.
+    """,
+    default=None,
+    callback=set_download_type_flag,
+    is_flag=True # is_flag is still needed for the split flag to work
 )
 
-normal_only = click.option(
-    "-no",
-    "--normal-only",
-    help="Restricts downloads to content that does not require decryption.",
-    required=False,
-    is_flag=True,
-)
 
 max_post_count_option = click.option(
     "-xc",
@@ -341,4 +341,3 @@ max_post_count_option = click.option(
     default=0,
     type=int,
 )
-download_type_option = click.constraints.mutually_exclusive(normal_only, protected_only)

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/arguments/user_select.py

@@ -3,7 +3,7 @@ import re
 
 import cloup as click
 
-from ofscraper.utils.args.callbacks.username import UsernameParse
+from ofscraper.utils.args.callbacks.parse.username import UsernameParse
 
 usernames_option = click.option(
     "-u",

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/commands/db.py

@@ -6,6 +6,8 @@ from ofscraper.utils.args.parse.group_bundles.db import db_args
 @db_args
 @click.pass_context
 def db(ctx, *args, **kwargs):
-    ctx.params["post_id"] = set(ctx.params["post_id"])
-    ctx.params["media_id"] = set(ctx.params["media_id"])
+    if ctx.params["post_id"]:
+        ctx.params["post_id"]=set(ctx.params["post_id"])
+    if ctx.params["media_id"]:
+            ctx.params["media_id"] = set(ctx.params["media_id"])
     return ctx.params, ctx.info_name

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/commands/main.py

@@ -1,7 +1,6 @@
 import cloup as click
 
 from ofscraper.utils.args.parse.group_bundles.advanced_common import advanced_args
-from ofscraper.utils.args.parse.group_bundles.common import common_args
 from ofscraper.utils.args.parse.group_bundles.main import main_program_args
 import ofscraper.utils.args.parse.arguments.utils.retry as retry_helper
 
@@ -12,7 +11,6 @@ import ofscraper.utils.args.parse.arguments.utils.retry as retry_helper
     invoke_without_command=True,
     show_constraints=True,
 )
-@common_args
 @main_program_args
 @advanced_args
 @click.pass_context

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/group_bundles/check.py

@@ -30,9 +30,10 @@ from ofscraper.utils.args.parse.groups.program import program_options
 from ofscraper.utils.args.parse.arguments.metadata_filters import (
     posted_after,
     posted_before,
-    download_type_option,
 )
-
+from ofscraper.utils.args.parse.arguments.post_content import (
+    download_type_option
+)
 
 def main_check(func):
     @force

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/group_bundles/db.py

@@ -31,7 +31,6 @@ from ofscraper.utils.args.parse.arguments.metadata_filters import (
     created_before,
     posted_after,
     posted_before,
-    download_type_option,
 )
 
 
@@ -45,6 +44,9 @@ from ofscraper.utils.args.parse.groups.advanced_program import (
     advanced_options_help,
     update_profile_option,
 )
+from ofscraper.utils.args.parse.arguments.post_content import (
+    download_type_option,
+)
 
 
 def db_args(func):

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/group_bundles/main.py

@@ -9,17 +9,17 @@ from ofscraper.utils.args.parse.groups.advanced_user_filter import (
     advanced_userfilters_options,
 )
 from ofscraper.utils.args.parse.groups.automatic import automatic_options
-from ofscraper.utils.args.parse.groups.media_filter import media_filter_options
 from ofscraper.utils.args.parse.groups.post_filter import post_filter_options
 from ofscraper.utils.args.parse.groups.user_list import userlist_options
 from ofscraper.utils.args.parse.groups.user_select import user_select_options
 from ofscraper.utils.args.parse.groups.user_sort import user_sorting_options
+from ofscraper.utils.args.parse.group_bundles.common import common_args
 
 # import click
 
 
 def main_program_args(func):
-    @media_filter_options
+    @common_args
     @post_filter_options
     @automatic_options
     @user_select_options

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/groups/advanced_processing.py

@@ -1,9 +1,8 @@
 import cloup as click
 
 from ofscraper.utils.args.parse.arguments.advanced_processing import (
-    individual_search_option,
-    search_entire_list_option,
-    users_first_option,
+   username_search_option,
+    users_first_option
 )
 
 advanced_processing_desc = "Advanced Search & Processing Options"
@@ -11,9 +10,7 @@ advanced_processing_desc = "Advanced Search & Processing Options"
 advanced_processing_help = "Choose how usernames are searched, and define the order in which users are processed for actions"
 advanced_processing_tuple = (
     users_first_option,
-    click.constraints.mutually_exclusive(
-        individual_search_option, search_entire_list_option
-    ),
+    username_search_option
 )
 advanced_processing_options = click.option_group(
     advanced_processing_desc, *advanced_processing_tuple, help=advanced_processing_help

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/args/parse/groups/post_filter.py

@@ -30,7 +30,6 @@ post_filter_options_tuple = (
     label_option,
     mass_msg_option,
     timed_only_option,
-    max_post_count_option,
 )
 # Create the option group
 post_filter_options = click.option_group(

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/ofscraper/utils/settings.py

@@ -97,7 +97,8 @@ def setup_settings(mediatype):
         or config_data.get_post_download_script()
     )
     merged.auto_resume = get_auto_resume(mediatype)
-    merged.cached_disabled = get_auto_after_enabled()
+    merged.auto_after = get_auto_after_enabled()
+    merged.cached_disabled=get_cached_disabled()
     merged.logs_expire_time=config_data.get_logs_expire()
 
     return merged
@@ -120,13 +121,16 @@ def get_ffmpeg():
 
 
 def get_auto_after_enabled():
-    if read_args.retriveArgs().no_cache:
-        return False
-    if read_args.retriveArgs().no_api_cache:
+    if get_cached_disabled():
         return False
     return config_data.get_enable_after()
 
-
+def get_cached_disabled():
+    if read_args.retriveArgs().no_cache:
+        return True
+    if read_args.retriveArgs().no_api_cache:
+        return True
+    return False
 def get_auto_resume(mediatype=None):
     if read_args.retriveArgs().no_auto_resume:
         return False

{ofscraper-3.13.dev6 → ofscraper-3.13.dev7}/pyproject.toml

@@ -1,12 +1,12 @@
 [tool.poetry]
 name = "ofscraper"
-version = "3.13.dev6"
+version = "3.13.dev7"
 description = "automatically scrape onlyfans"
 authors = ["datawhores <datawhores@riseup.net>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = ">=3.11,<3.14"
+python = "^3.11,<3.14"
 
 
 httpx = {extras = ["http2"], version = "^0.28.1"}
@@ -72,7 +72,7 @@ random-emoji = "^1.0.15"
 pytest-mock = "^3.12.0"
 
 [tool.poetry.group.docs.dependencies]
-sphinx-argparse = "^0.4.0"
+sphinx-argparse = "^0.5.2"
 
 
 [tool.poetry.extras]

ofscraper-3.13.dev6/ofscraper/data/api/common/after.py

@@ -1,34 +0,0 @@
-import logging
-
-import ofscraper.utils.settings as settings
-from ofscraper.data.api.common.cache.read import read_full_after_scan_check
-
-
-def get_after_pre_checks(model_id, api):
-    val = None
-    if settings.get_settings().after == 0:
-        val = 0
-    elif settings.get_settings().after is not None:
-        val = settings.get_settings().after.float_timestamp
-    elif not settings.get_settings().auto_after:
-        val = 0
-    elif read_full_after_scan_check(model_id, api):
-        val = 0
-    elif "like" in settings.get_settings().action:
-        val = 0
-    elif "unlike" in settings.get_settings().action:
-        val = 0
-    return _return_val(val, api)
-
-
-def _return_val(val, api):
-    if val is None:
-        logging.getLogger("shared").info(
-            f"precheck failed for setting after {api} using db"
-        )
-        return val
-    else:
-        logging.getLogger("shared").info(
-            f"precheck success for setting after {api} skipping db"
-        )
-        return val

ofscraper-3.13.dev6/ofscraper/utils/args/parse/arguments/advanced_processing.py

@@ -1,29 +0,0 @@
-import cloup as click
-
-users_first_option = click.option(
-    "-uf",
-    "--users-first",
-    "--user-first",
-    "--usersfirst",
-    "--userfirst",
-    "users_first",
-    help="Process all users first rather than one at a time (affects --action)",
-    default=False,
-    is_flag=True,  # Shorthand for action="store_true"
-)
-
-individual_search_option = click.option(
-    "-fi",
-    "--individual",
-    help="Search each username as a separate request when --username is provided",
-    default=False,
-    is_flag=True,
-)
-
-search_entire_list_option = click.option(
-    "-fl",
-    "--list",
-    help="Search entire enabled lists before filtering for usernames when --username is provided",
-    default=False,
-    is_flag=True,
-)

ofscraper-3.13.dev6/ofscraper/utils/args/parse/arguments/db.py

@@ -1,15 +0,0 @@
-# db args
-# import cloup as click
-
-# preview_option = click.option(
-#     "-p/-np",
-#     "--preview/--no-preview",
-#     "preview",
-#     help="""
-#         \b
-#         Flag for filtering accounts based on preview status.
-#         [select one --preview or --no-preview]""",
-#     default=None,
-#     required=False,
-#     is_flag=True,
-# )