ofscraper 3.13.dev0__tar.gz → 3.13.dev3__tar.gz

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ofscraper
-Version: 3.13.dev0
+Version: 3.13.dev3
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net
@@ -78,7 +78,7 @@ https://github.com/datawhores/OF-Scraper/issues/442
 
 ```Text
 
-Docker and binary releases also availible
+Docker and binary releases also available
 ```
 
 ## Stable

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/README.md

@@ -19,7 +19,7 @@ https://github.com/datawhores/OF-Scraper/issues/442
 
 ```Text
 
-Docker and binary releases also availible
+Docker and binary releases also available
 ```
 
 ## Stable

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/ofscraper/classes/sessionmanager/sessionmanager.py

@@ -7,7 +7,6 @@ import time
 import traceback
 
 import aiohttp
-import aiohttp.client_exceptions
 import arrow
 import certifi
 import httpx
@@ -18,6 +17,7 @@ import ofscraper.utils.auth.request as auth_requests
 import ofscraper.utils.config.data as data
 import ofscraper.utils.constants as constants
 from ofscraper.utils.auth.utils.warning.print import print_auth_warning
+import ofscraper.utils.settings as settings
 
 TOO_MANY = "too_many"
 AUTH = "auth"
@@ -481,7 +481,7 @@ class sessionManager:
                             params=params,
                             json=json,
                             data=data,
-                            ssl=ssl.create_default_context(cafile=certifi.where()),
+                            ssl=False if not settings.get_settings().ssl_validation else ssl.create_default_context(cafile=certifi.where()),
                         )
                     else:
                         r = await self._httpx_funct_async(

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/ofscraper/commands/utils/scrape_context.py

@@ -12,7 +12,7 @@ r"""
 """
 
 import logging
-import timeit
+from datetime import datetime
 from contextlib import contextmanager
 
 import arrow
@@ -24,7 +24,7 @@ log = logging.getLogger("shared")
 def scrape_context_manager():
     # reset stream if needed
     # Before yield as the enter method
-    start = timeit.default_timer()
+    start = datetime.now()
     log.warning(
         """
 ==============================                            
@@ -33,12 +33,14 @@ def scrape_context_manager():
 """
     )
     yield
-    end = timeit.default_timer()
+    end = datetime.now()
     log.warning(
         f"""
 ===========================
 [bold] Script Finished [/bold]
 Run Time:  [bold]{str(arrow.get(end)-arrow.get(start)).split(".")[0]}[/bold]
+Started At:  [bold]{str(arrow.get(start).format("YYYY-MM-DD hh:mm:ss"))}[/bold]
+Finished At:  [bold]{str(arrow.get(end).format("YYYY-MM-DD hh:mm:ss"))}[/bold]
 ===========================
 """
     )

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/ofscraper/main/open/load.py

@@ -49,7 +49,7 @@ def setLogger():
     logger.get_shared_logger()
     logs.discord_warning()
     logger.start_threads()
-
+    paths_manage.cleanup_logs()
 
 def systemSet():
     system.setName()

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/ofscraper/prompts/prompt_groups/config.py

@@ -104,6 +104,8 @@ def funct(prompt_):
     enable_auto_after: whether to dynamically set --after: default True
     default_user_list: default user list for --action
     default_black_list: default black list for --action
+    logs_expire_time: max age for a log before it is autodelete
+    ssl_validation: option for whether to validate ssl certificates
     remove_hash_match: remove files if hash matches
     ----------------------------------------------------------
     [Response Type]
@@ -638,6 +640,27 @@ Main user list with all active+expired users can be called main or ofscraper.mai
 Active user list can be called active or ofscraper.active
 Expired user list can be called expired or ofscraper.expired
 List are case insensitive\n
+""",
+            },
+
+                        {
+                "type": "input",
+                "name": "logs_expire_time",
+                "message": "Logs expire Time",
+                "default": data.get_logs_expire(),
+                "option_instruction": """
+Logs will auto delete after the selected amount of time has passed
+If value is 'None' or '0' no logs will be touched
+""",
+            },
+
+             {
+                "type": "input",
+                "name": "ssl_validation",
+                "message": "ssl validation",
+                "default": data.get_ssl_validation(),
+                "option_instruction": """
+Option to disable ssl certificate validation
 """,
             },
             {

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/ofscraper/utils/config/data.py

@@ -266,6 +266,22 @@ def get_default_userlist(config=None):
     )
 
 
+@wrapper.config_reader
+def get_logs_expire(config=None):
+   return(
+        config.get("logs_expire_time")
+        if not any(x == config.get("logs_expire_time") for x in [None, ""])
+        else config.get("advanced_options", {}).get("logs_expire_time")
+    )
+
+@wrapper.config_reader
+def get_ssl_validation(config=None):
+    val = (
+        config.get("ssl_validation")
+        if not any(x == config.get("ssl_validation") for x in [None, ""])
+        else config.get("advanced_options", {}).get("ssl_validation")
+    )
+    return val if val else True
 @wrapper.config_reader
 def get_post_download_script(config=None):
     if config is False:
@@ -293,8 +309,8 @@ def get_post_script(config=None):
         val = config.get("post_script")
     elif config.get("advanced_options", {}).get("post_script") is not None:
         val = config.get("advanced_options", {}).get("post_script")
-    elif config.get("scripts", {}).get("post_download_script") is not None:
-        val = config.get("scripts", {}).get("post_download_script")
+    elif config.get("scripts", {}).get("post_script") is not None:
+        val = config.get("scripts", {}).get("post_script")
     elif config.get("script_options", {}).get("post_script") is not None:
         val = config.get("script_options", {}).get("post_script")
     return val if val is not None else constants_attr.getattr("POST_SCRIPT_DEFAULT")

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/ofscraper/utils/config/schema.py

@@ -64,8 +64,10 @@ def get_current_config_schema(config: dict = None) -> dict:
             "enable_auto_after": data.get_enable_after(config=config),
             "default_user_list": data.get_default_userlist(config=config),
             "default_black_list": data.get_default_blacklist(config=config),
+            "logs_expire_time":data.get_logs_expire(config=config),
+            "ssl_validation":data.get_ssl_validation(config=config)
         },
-        "scripts_options": {
+        "script_options": {
             "post_download_script": data.get_post_download_script(config=config),
             "post_script": data.get_post_script(config=config),
         },
@@ -97,20 +99,27 @@ def config_diff(config):
     if config.get("config"):
         config = config["config"]
     schema = get_current_config_schema()
-    return config_diff_helper(config, schema)
+    return _config_diff_helper(config, schema)
 
+def _config_diff_helper(config, schema):
+    # Check for keys in schema but missing in config
+    if set(schema.keys()) - set(config.keys()):
+        return True
 
-def config_diff_helper(config, schema, key=None):
-    if key:
-        config = config[key]
-        schema = schema[key]
-    diff = set(schema.keys()) - set(config.keys())
-    if len(diff) > 0:
+    # Check for keys in config but missing in schema
+    if set(config.keys()) - set(schema.keys()):
         return True
-    for key in schema.keys():
-        if not isinstance(schema[key], dict):
-            continue
-        elif not isinstance(config[key], dict):
-            return True
-        elif config_diff_helper(config, schema, key):
-            return True
+
+    for key, schema_value in schema.items():
+        if key in config:
+            config_value = config[key]
+            if isinstance(schema_value, dict):
+                if not isinstance(config_value, dict):
+                    return True
+                if _config_diff_helper(config_value, schema_value):
+                    return True
+            elif schema_value != config_value:  # Simple value comparison
+                return True
+        # Keys missing from config are already handled above
+
+    return False

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/ofscraper/utils/hash.py

@@ -18,7 +18,7 @@ fileHashes = {}
 def get_hash(file_data, mediatype=None):
     global fileHashes
     hash = None
-    if settings.get_hash(mediatype=mediatype) is None:
+    if settings.get_settings().hash is None:
         return
     if isinstance(file_data, placeholder.Placeholders):
         file_data = file_data.trunicated_filepath

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/ofscraper/utils/live/updater.py

@@ -137,7 +137,7 @@ downloads_pending = set()
 def add_download_job_task(*args, **kwargs):
     max_visible = constants.getattr("MAX_PROGRESS_BARS")
     visible = (
-        settings.get_settings().download_bars()
+        settings.get_settings().download_bars
         and len(download_job_progress.tasks) < max_visible
     )
     task = download_job_progress.add_task(*args, visible=visible, **kwargs)
@@ -160,7 +160,7 @@ def update_download_task(*args, **kwargs):
 
 
 def update_download_job_task(*args, **kwargs):
-    if not settings.get_settings().download_bars():
+    if not settings.get_settings().download_bars:
         return
     download_job_progress.update(*args, **kwargs)
 

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/ofscraper/utils/logs/logs.py

@@ -102,7 +102,7 @@ def print_latest_version():
                     "[bold yellow]OF-Scraper can't check version (probably from zip)[/bold yellow]"
                 )
             elif ".dev" in __version__:
-                log.error("OF-Scraper up to date[/bold yellow]")
+                log.error("[bold yellow]OF-Scraper up to date[/bold yellow]")
             else:
                 log.error(
                     f"[bold yellow]new version of OF-Scraper available[/bold yellow]: [bold]{new_version}[/bold]"

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/ofscraper/utils/paths/common.py

@@ -78,15 +78,13 @@ def getlogpath():
     path = None
     if not data.get_appendlog():
         path = (
-            get_config_home()
-            / "logging"
+            get_log_folder()
             / f'{data.get_main_profile()}_{dates_manager.getLogDate().get("day")}'
             / f'ofscraper_{data.get_main_profile()}_{dates_manager.getLogDate().get("now")}.log'
         )
     else:
         path = (
-            get_config_home()
-            / "logging"
+           get_log_folder()
             / f'ofscraper_{data.get_main_profile()}_{dates_manager.getLogDate().get("day")}.log'
         )
     path = pathlib.Path(path).resolve()
@@ -94,6 +92,11 @@ def getlogpath():
     return path
 
 
+def get_log_folder():
+    return (
+            get_config_home()
+            / "logging")
+
 def get_profile_path(name=None):
     if name:
         profile = get_config_home() / name

ofscraper-3.13.dev3/ofscraper/utils/paths/manage.py

@@ -0,0 +1,77 @@
+import logging
+import shutil
+import arrow
+import traceback
+import ofscraper.utils.paths.common as common_paths
+import ofscraper.utils.settings as settings
+import ofscraper.utils.context.stdout as stdout
+import ofscraper.utils.console as console_
+
+console=console_.get_shared_console()
+
+
+def make_folders():
+    common_paths.get_config_folder()
+    create_profile_path()
+
+
+def create_profile_path(name=None):
+    out = common_paths.get_profile_path(name)
+    out.mkdir(exist_ok=True, parents=True)
+    return out
+
+
+def copy_path(source, dst):
+    try:
+        shutil.copy2(source, dst)
+    except OSError as e:
+        logging.getLogger("shared").debug("failed to copy with copy2 using copy")
+        logging.getLogger("shared").traceback_(e)
+        logging.getLogger("shared").traceback_(traceback.format_exc())
+        shutil.copy(source, dst)
+    except Exception as e:
+        raise e
+
+
+def cleanup_logs():
+    """
+    Performs actions to cleanup logs
+    """
+    with stdout.lowstdout():
+        delete_old_logs()
+        delete_empty_folders()
+
+def delete_old_logs():
+    """
+    Recursively finds and deletes .log files in the specified folder
+    that are older than the given maximum age
+
+    Also cleans up folder
+    """
+    log_path = common_paths.get_log_folder()
+    now = arrow.now().float_timestamp
+    if not settings.get_settings().logs_expire_time:
+        return
+    for log_file in log_path.rglob("*.log"):  # rglob for recursive globbing
+        try:
+            if (now - log_file.stat().st_mtime) > settings.get_settings().logs_expire_time:
+                log_file.unlink()  # pathlib's way to delete a file
+                console.print(f"Deleted old log file: {log_file}")
+        except OSError as e:
+            console.print(f"Error deleting file {log_file}: {e}")
+def delete_empty_folders():
+    """
+    Recursively finds and deletes empty folders within thelog folder.
+
+    """
+    log_path = common_paths.get_log_folder()
+
+    # Iterate through all directories within the log folder and its subdirectories
+    for folder in sorted(log_path.rglob("*"), key=lambda p: len(str(p)), reverse=True):
+        if folder.is_dir() and not any(folder.iterdir()):
+            try:
+                folder.rmdir()
+                console.print(f"Deleted empty folder: {folder}")
+            except OSError as e:
+                console.print(f"Error deleting folder {folder}: {e}")
+

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/ofscraper/utils/settings.py

@@ -98,6 +98,8 @@ def setup_settings(mediatype):
     )
     merged.auto_resume = get_auto_resume(mediatype)
     merged.cached_disabled = get_auto_after_enabled()
+    merged.logs_expire_time=config_data.get_logs_expire()
+    merged.ssl_validation=config_data.get_ssl_validation()
 
     return merged
 

{ofscraper-3.13.dev0 → ofscraper-3.13.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ofscraper"
-version = "3.13.dev"
+version = "3.13.dev3"
 description = "automatically scrape onlyfans"
 authors = ["datawhores <datawhores@riseup.net>"]
 readme = "README.md"

ofscraper-3.13.dev0/ofscraper/commands/scraper/actions/utils/send/send_bar_msg.py

@@ -1,16 +0,0 @@
-# import asyncio
-
-# import ofscraper.commands.scraper.actions.utils.globals as common_globals
-# import ofscraper.utils.settings as settings
-
-
-# async def send_bar_msg_batch(msg, count, update_count):
-#     if not settings.get_settings().download_bars():
-#         return
-#     await send_msg_alt(msg)
-
-
-# async def send_bar_msg(func, count, update_count):
-#     if not settings.get_settings().download_bars():
-#         return
-#     await asyncio.get_event_loop().run_in_executor(common_globals.thread, func)

ofscraper-3.13.dev0/ofscraper/utils/paths/manage.py

@@ -1,28 +0,0 @@
-import logging
-import shutil
-import traceback
-
-import ofscraper.utils.paths.common as common_paths
-
-
-def make_folders():
-    common_paths.get_config_folder()
-    create_profile_path()
-
-
-def create_profile_path(name=None):
-    out = common_paths.get_profile_path(name)
-    out.mkdir(exist_ok=True, parents=True)
-    return out
-
-
-def copy_path(source, dst):
-    try:
-        shutil.copy2(source, dst)
-    except OSError as e:
-        logging.getLogger("shared").debug("failed to copy with copy2 using copy")
-        logging.getLogger("shared").traceback_(e)
-        logging.getLogger("shared").traceback_(traceback.format_exc())
-        shutil.copy(source, dst)
-    except Exception as e:
-        raise e