ofscraper 3.13.dev2__tar.gz → 3.13.dev4__tar.gz

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ofscraper
-Version: 3.13.dev2
+Version: 3.13.dev4
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net
@@ -51,6 +51,7 @@ Requires-Dist: speedtest-cli (>=2.1.3,<3.0.0)
 Requires-Dist: tenacity (>=8.2.3,<9.0.0)
 Requires-Dist: textual (==1.0.0)
 Requires-Dist: tqdm (>=4.66.4,<5.0.0)
+Requires-Dist: ua-generator (>=2.0.5,<3.0.0)
 Requires-Dist: uvloop (>=0.19.0,<0.20.0) ; sys_platform == "linux" or sys_platform == "linux2"
 Requires-Dist: win32-setctime (>=1.1.0,<2.0.0)
 Requires-Dist: xxhash (>=3.4.1,<4.0.0)
@@ -78,7 +79,7 @@ https://github.com/datawhores/OF-Scraper/issues/442
 
 ```Text
 
-Docker and binary releases also availible
+Docker and binary releases also available
 ```
 
 ## Stable

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/README.md

@@ -19,7 +19,7 @@ https://github.com/datawhores/OF-Scraper/issues/442
 
 ```Text
 
-Docker and binary releases also availible
+Docker and binary releases also available
 ```
 
 ## Stable

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/ofscraper/classes/sessionmanager/sessionmanager.py

@@ -7,7 +7,6 @@ import time
 import traceback
 
 import aiohttp
-import aiohttp.client_exceptions
 import arrow
 import certifi
 import httpx
@@ -18,6 +17,9 @@ import ofscraper.utils.auth.request as auth_requests
 import ofscraper.utils.config.data as data
 import ofscraper.utils.constants as constants
 from ofscraper.utils.auth.utils.warning.print import print_auth_warning
+import ofscraper.utils.settings as settings
+import ua_generator
+
 
 TOO_MANY = "too_many"
 AUTH = "auth"
@@ -25,6 +27,7 @@ FORCED_NEW = "get_new_sign"
 SIGN = "get_sign"
 COOKIES = "get_cookies_str"
 HEADERS = "create_header"
+AGENT=str(ua_generator.generate())
 
 
 def is_rate_limited(exception, sleeper):
@@ -343,17 +346,21 @@ class sessionManager:
                 try:
                     sleeper.do_sleep()
                     # remake each time
-                    headers = (
-                        self._create_headers(
-                            headers, url, SIGN in actions, FORCED_NEW in actions
-                        )
-                        if (
+                    if (
                             SIGN in actions
                             or FORCED_NEW in actions
                             or HEADERS in actions
-                        )
-                        else None
-                    )
+                       ):
+                            headers = self._create_headers(
+                                    headers, url, SIGN in actions, FORCED_NEW in actions
+                                )
+                    elif not headers or not "user-agent" in headers:
+                        headers=headers or {}
+                        headers.update({"user-agent":AGENT})
+                        
+                        
+                       
+                    
                     cookies = self._create_cookies() if COOKIES in actions else None
                     r = self._httpx_funct(
                         method,
@@ -448,17 +455,21 @@ class sessionManager:
                 await sem.acquire()
                 try:
                     await sleeper.async_do_sleep()
-                    headers = (
-                        self._create_headers(
-                            headers, url, SIGN in actions, FORCED_NEW in actions
-                        )
-                        if (
+                    if (
                             SIGN in actions
                             or FORCED_NEW in actions
                             or HEADERS in actions
+                    ): 
+                        headers = (
+                        self._create_headers(
+                            headers, url, SIGN in actions, FORCED_NEW in actions
                         )
-                        else headers
+                       
                     )
+                    elif not headers or not "user-agent" in headers:
+                        headers= headers or {}
+                        headers.update({"user-agent":AGENT})
+                        
 
                     cookies = self._create_cookies() if COOKIES in actions else None
                     json = json
@@ -481,7 +492,7 @@ class sessionManager:
                             params=params,
                             json=json,
                             data=data,
-                            ssl=ssl.create_default_context(cafile=certifi.where()),
+                            ssl=False if not settings.get_settings().ssl_validation else ssl.create_default_context(cafile=certifi.where()),
                         )
                     else:
                         r = await self._httpx_funct_async(

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/ofscraper/commands/utils/scrape_context.py

@@ -12,7 +12,7 @@ r"""
 """
 
 import logging
-import timeit
+from datetime import datetime
 from contextlib import contextmanager
 
 import arrow
@@ -24,7 +24,7 @@ log = logging.getLogger("shared")
 def scrape_context_manager():
     # reset stream if needed
     # Before yield as the enter method
-    start = timeit.default_timer()
+    start = datetime.now()
     log.warning(
         """
 ==============================                            
@@ -33,12 +33,14 @@ def scrape_context_manager():
 """
     )
     yield
-    end = timeit.default_timer()
+    end = datetime.now()
     log.warning(
         f"""
 ===========================
 [bold] Script Finished [/bold]
 Run Time:  [bold]{str(arrow.get(end)-arrow.get(start)).split(".")[0]}[/bold]
+Started At:  [bold]{str(arrow.get(start).format("YYYY-MM-DD hh:mm:ss"))}[/bold]
+Finished At:  [bold]{str(arrow.get(end).format("YYYY-MM-DD hh:mm:ss"))}[/bold]
 ===========================
 """
     )

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/ofscraper/data/api/common/after.py

@@ -10,7 +10,7 @@ def get_after_pre_checks(model_id, api):
         val = 0
     elif settings.get_settings().after is not None:
         val = settings.get_settings().after.float_timestamp
-    elif not settings.get_settings().auto_after():
+    elif not settings.get_settings().auto_after:
         val = 0
     elif read_full_after_scan_check(model_id, api):
         val = 0

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/ofscraper/prompts/prompt_groups/config.py

@@ -104,7 +104,8 @@ def funct(prompt_):
     enable_auto_after: whether to dynamically set --after: default True
     default_user_list: default user list for --action
     default_black_list: default black list for --action
-    logs_expire_time: max age for a log before it is autodeleted
+    logs_expire_time: max age for a log before it is autodelete
+    ssl_validation: option for whether to validate ssl certificates
     remove_hash_match: remove files if hash matches
     ----------------------------------------------------------
     [Response Type]
@@ -646,13 +647,20 @@ List are case insensitive\n
                 "type": "input",
                 "name": "logs_expire_time",
                 "message": "Logs expire Time",
-                "default": data.get_logs_expire_time(),
+                "default": data.get_logs_expire(),
                 "option_instruction": """
-A comma seperated list of userlists to set as black listed
-Main user list with all active+expired users can be called main or ofscraper.main
-Active user list can be called active or ofscraper.active
-Expired user list can be called expired or ofscraper.expired
-List are case insensitive\n
+Logs will auto delete after the selected amount of time has passed
+If value is 'None' or '0' no logs will be touched
+""",
+            },
+
+             {
+                "type": "input",
+                "name": "ssl_validation",
+                "message": "ssl validation",
+                "default": data.get_ssl_validation(),
+                "option_instruction": """
+Option to disable ssl certificate validation
 """,
             },
             {

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/ofscraper/utils/config/data.py

@@ -268,12 +268,20 @@ def get_default_userlist(config=None):
 
 @wrapper.config_reader
 def get_logs_expire(config=None):
-    val = (
+   return(
         config.get("logs_expire_time")
         if not any(x == config.get("logs_expire_time") for x in [None, ""])
         else config.get("advanced_options", {}).get("logs_expire_time")
     )
-    return val
+
+@wrapper.config_reader
+def get_ssl_validation(config=None):
+    val = (
+        config.get("ssl_validation")
+        if not any(x == config.get("ssl_validation") for x in [None, ""])
+        else config.get("advanced_options", {}).get("ssl_validation")
+    )
+    return val if val else True
 @wrapper.config_reader
 def get_post_download_script(config=None):
     if config is False:
@@ -301,8 +309,8 @@ def get_post_script(config=None):
         val = config.get("post_script")
     elif config.get("advanced_options", {}).get("post_script") is not None:
         val = config.get("advanced_options", {}).get("post_script")
-    elif config.get("scripts", {}).get("post_download_script") is not None:
-        val = config.get("scripts", {}).get("post_download_script")
+    elif config.get("scripts", {}).get("post_script") is not None:
+        val = config.get("scripts", {}).get("post_script")
     elif config.get("script_options", {}).get("post_script") is not None:
         val = config.get("script_options", {}).get("post_script")
     return val if val is not None else constants_attr.getattr("POST_SCRIPT_DEFAULT")

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/ofscraper/utils/config/schema.py

@@ -64,9 +64,10 @@ def get_current_config_schema(config: dict = None) -> dict:
             "enable_auto_after": data.get_enable_after(config=config),
             "default_user_list": data.get_default_userlist(config=config),
             "default_black_list": data.get_default_blacklist(config=config),
-            "logs_expire_time":data.get_logs_expire(config=config)
+            "logs_expire_time":data.get_logs_expire(config=config),
+            "ssl_validation":data.get_ssl_validation(config=config)
         },
-        "scripts_options": {
+        "script_options": {
             "post_download_script": data.get_post_download_script(config=config),
             "post_script": data.get_post_script(config=config),
         },
@@ -98,20 +99,27 @@ def config_diff(config):
     if config.get("config"):
         config = config["config"]
     schema = get_current_config_schema()
-    return config_diff_helper(config, schema)
+    return _config_diff_helper(config, schema)
 
+def _config_diff_helper(config, schema):
+    # Check for keys in schema but missing in config
+    if set(schema.keys()) - set(config.keys()):
+        return True
 
-def config_diff_helper(config, schema, key=None):
-    if key:
-        config = config[key]
-        schema = schema[key]
-    diff = set(schema.keys()) - set(config.keys())
-    if len(diff) > 0:
+    # Check for keys in config but missing in schema
+    if set(config.keys()) - set(schema.keys()):
         return True
-    for key in schema.keys():
-        if not isinstance(schema[key], dict):
-            continue
-        elif not isinstance(config[key], dict):
-            return True
-        elif config_diff_helper(config, schema, key):
-            return True
+
+    for key, schema_value in schema.items():
+        if key in config:
+            config_value = config[key]
+            if isinstance(schema_value, dict):
+                if not isinstance(config_value, dict):
+                    return True
+                if _config_diff_helper(config_value, schema_value):
+                    return True
+            elif schema_value != config_value:  # Simple value comparison
+                return True
+        # Keys missing from config are already handled above
+
+    return False

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/ofscraper/utils/hash.py

@@ -18,7 +18,7 @@ fileHashes = {}
 def get_hash(file_data, mediatype=None):
     global fileHashes
     hash = None
-    if settings.get_hash(mediatype=mediatype) is None:
+    if settings.get_settings().hash is None:
         return
     if isinstance(file_data, placeholder.Placeholders):
         file_data = file_data.trunicated_filepath

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/ofscraper/utils/live/updater.py

@@ -137,7 +137,7 @@ downloads_pending = set()
 def add_download_job_task(*args, **kwargs):
     max_visible = constants.getattr("MAX_PROGRESS_BARS")
     visible = (
-        settings.get_settings().download_bars()
+        settings.get_settings().download_bars
         and len(download_job_progress.tasks) < max_visible
     )
     task = download_job_progress.add_task(*args, visible=visible, **kwargs)
@@ -160,7 +160,7 @@ def update_download_task(*args, **kwargs):
 
 
 def update_download_job_task(*args, **kwargs):
-    if not settings.get_settings().download_bars():
+    if not settings.get_settings().download_bars:
         return
     download_job_progress.update(*args, **kwargs)
 

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/ofscraper/utils/logs/logs.py

@@ -102,7 +102,7 @@ def print_latest_version():
                     "[bold yellow]OF-Scraper can't check version (probably from zip)[/bold yellow]"
                 )
             elif ".dev" in __version__:
-                log.error("OF-Scraper up to date[/bold yellow]")
+                log.error("[bold yellow]OF-Scraper up to date[/bold yellow]")
             else:
                 log.error(
                     f"[bold yellow]new version of OF-Scraper available[/bold yellow]: [bold]{new_version}[/bold]"

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/ofscraper/utils/settings.py

@@ -99,6 +99,7 @@ def setup_settings(mediatype):
     merged.auto_resume = get_auto_resume(mediatype)
     merged.cached_disabled = get_auto_after_enabled()
     merged.logs_expire_time=config_data.get_logs_expire()
+    merged.ssl_validation=config_data.get_ssl_validation()
 
     return merged
 

{ofscraper-3.13.dev2 → ofscraper-3.13.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ofscraper"
-version = "3.13.dev2"
+version = "3.13.dev4"
 description = "automatically scrape onlyfans"
 authors = ["datawhores <datawhores@riseup.net>"]
 readme = "README.md"
@@ -53,6 +53,7 @@ async-property = "^0.2.2"
 limiter = "^0.5.0"
 aiolimiter = "^1.1.0"
 textual = "1.0.0"
+ua-generator = "^2.0.5"
 
 
 [tool.poetry.group.dev.dependencies]

ofscraper-3.13.dev2/ofscraper/commands/scraper/actions/utils/send/send_bar_msg.py

@@ -1,16 +0,0 @@
-# import asyncio
-
-# import ofscraper.commands.scraper.actions.utils.globals as common_globals
-# import ofscraper.utils.settings as settings
-
-
-# async def send_bar_msg_batch(msg, count, update_count):
-#     if not settings.get_settings().download_bars():
-#         return
-#     await send_msg_alt(msg)
-
-
-# async def send_bar_msg(func, count, update_count):
-#     if not settings.get_settings().download_bars():
-#         return
-#     await asyncio.get_event_loop().run_in_executor(common_globals.thread, func)