ofscraper 3.13.4__tar.gz → 3.13.5.dev2__tar.gz

{ofscraper-3.13.4 → ofscraper-3.13.5.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ofscraper
-Version: 3.13.4
+Version: 3.13.5.dev2
 Summary: automatically scrape onlyfans
 Project-URL: Homepage, https://github.com/datawhores/OF-Scraper
 Author-email: datawhores <datawhores@riseup.net>

{ofscraper-3.13.4 → ofscraper-3.13.5.dev2}/ofscraper/classes/of/media.py

@@ -4,6 +4,8 @@ import asyncio
 import logging
 import re
 import warnings
+from typing import Union
+from pathlib import Path
 
 import arrow
 from async_property import async_cached_property
@@ -37,52 +39,46 @@ class Media(base.base):
         self._lock = asyncio.Lock()
         self._cached_mpd = None
 
-        # ==> STATE TRACKING ATTRIBUTES <==
         self.download_attempted = False
         self.download_succeeded = (
             None  # Using tri-state: None (not attempted), True, False
         )
-        self.like_attempted = False
-        self.like_succeeded = None  # Using tri-state: None (not attempted), True, False
 
-    # ==> RECOMMENDED CHANGE for __eq__ <==
-    # Two different media items can have the same postid.
-    # It's safer to check for media id equality.
     def __eq__(self, other):
         if not isinstance(other, Media):
             return NotImplemented
         return self.id == other.id
 
-    # ==> STATE MANAGEMENT METHODS (New Additions) <==
     def mark_download_attempt(self):
         """Marks that a download has been attempted."""
         self.download_attempted = True
 
-    def mark_download_success(self):
-        """Marks a download as successful."""
+    def mark_download_finished(self,success=True):
+        """Marks a download as successful/failed."""
         self.download_attempted = True
-        self.download_succeeded = True
-
-    def mark_download_failure(self):
-        """Marks a download as failed."""
-        self.download_attempted = True
-        self.download_succeeded = False
-
-    def mark_like_attempt(self):
-        """Marks that a like has been attempted."""
-        self.like_attempted = True
-
-    def mark_like_success(self):
-        """Marks a like as successful."""
-        self.like_attempted = True
-        self.like_succeeded = True
-
-    def mark_like_failure(self):
-        """Marks a like as failed."""
-        self.like_attempted = True
-        self.like_succeeded = False
-
-    # --- Start of your original properties and methods ---
+        self.download_succeeded =(success==True)
+        self.update_status()
+
+    def update_status(self):
+        self.media["download_status"] = "skipped"
+        if self.download_attempted == True and self.download_succeeded == False:
+            self.media["download_status"] = "failed"
+        elif self.download_attempted == True and self.download_succeeded == True:
+            self.media["download_status"] = "succeed"
+
+    def add_filepath(self, path: Union[str | Path]):
+        path = str(path)
+        self.media["filepath"] = path
+
+    # only use if content type can't be found from request
+    @property
+    def content_type(self):
+        if self.mediatype.lower() == "videos":
+            return "mp4"
+        elif self.mediatype.lower() == "images":
+            return "jpg"
+        elif self.mediatype.lower() == "audios":
+            return "mp3"
 
     @property
     def expires(self):

{ofscraper-3.13.4 → ofscraper-3.13.5.dev2}/ofscraper/classes/of/postcollection.py

@@ -26,7 +26,7 @@ class PostCollection:
         self.mode = mode
         # Internal dictionary to store posts, using post_id as the key for de-duplication
         self._posts_map = {}
-        log.info(f"Initialized PostCollection for {username} in '{self.mode}' mode.")
+        log.info(f"Initialized PostCollection for {username}")
 
     @property
     def posts(self) -> list[Post]:

{ofscraper-3.13.4 → ofscraper-3.13.5.dev2}/ofscraper/classes/of/posts.py

@@ -102,16 +102,35 @@ class Post(base.base):
         if media_item in self.media_to_download:
             self.media_to_download.remove(media_item)
 
-    def mark_post_liked(self, success):
+    def mark_post_liked(self, success=True):
         """
-        Updates the status of the post after a like/unlike attempt.
+        Updates the success status of the post after a like/unlike attempt.
         """
         self.like_attempted = True
-        self.like_success = success
+        self.like_success =(success==True)
         if success:
             self._post["isFavorite"] = True
             self.is_actionable_like = False
 
+    def mark_post_unliked(self, success=True):
+        """
+        Updates the success status of the post after a like/unlike attempt.
+        """
+        self.like_attempted = True
+        self.like_success = (success==True)
+        if success:
+            self._post["isFavorite"] = False
+            self.is_actionable_like = False
+    
+
+    def mark_like_attempt(self):
+        """
+        Updates the status of the post before a like/unlike attempt.
+        """
+        self.like_attempted = True
+
+    
+    
     @property
     def missed_downloads(self):
         """Convenience property to see what failed to download."""

{ofscraper-3.13.4 → ofscraper-3.13.5.dev2}/ofscraper/classes/placeholder.py

@@ -4,7 +4,6 @@ import pathlib
 import re
 import arrow
 
-import ofscraper.utils.args.accessors.read as read_args
 import ofscraper.utils.cache as cache
 import ofscraper.utils.config.data as data
 import ofscraper.utils.config.file as config_file
@@ -236,7 +235,7 @@ class Placeholders(basePlaceholder):
         self._variables.update({"only_filename": ele.no_quality_final_filename})
         self._variables.update({"text": ele.file_text})
         self._variables.update({"config": config_file.open_config()})
-        self._variables.update({"args": read_args.retriveArgs()})
+        self._variables.update({"args": settings.get_args()})
 
     @basePlaceholder.async_wrapper
     async def getmediadir(self, root=None, create=True):
@@ -432,7 +431,7 @@ class Textholders(basePlaceholder):
         )
         self._variables.update({"text": ele.text_trunicate(ele.file_sanitized_text)})
         self._variables.update({"config": config_file.open_config()})
-        self._variables.update({"args": read_args.retriveArgs()})
+        self._variables.update({"args": settings.get_args()})
 
         self._variables.update({"quality": "source"})
         self._variables.update(

{ofscraper-3.13.4 → ofscraper-3.13.5.dev2}/ofscraper/classes/sessionmanager/sessionmanager.py

@@ -19,7 +19,7 @@ import ofscraper.utils.of_env.of_env as of_env
 from ofscraper.utils.auth.utils.warning.print import print_auth_warning
 import ofscraper.utils.settings as settings
 from ofscraper.classes.sessionmanager.cert import create_custom_ssl_context
- 
+
 # Constants for request actions
 TOO_MANY = "too_many"
 AUTH = "auth"
@@ -64,10 +64,10 @@ class SessionSleep:
         self._init_sleep = (
             sleep if sleep is not None else of_env.getattr("SESSION_SLEEP_INIT")
         )
-        self._min_sleep=(
+        self._min_sleep = (
             min_sleep if min_sleep is not None else of_env.getattr("SESSION_SLEEP_MIN")
         )
-        #set sleep to min sleep always
+        # set sleep to min sleep always
         self._sleep = self._min_sleep
 
         self._difmin = (
@@ -102,15 +102,15 @@ class SessionSleep:
             new_sleep = self._sleep / self._decay_factor
             self._sleep = max(self._min_sleep, new_sleep)
             self._last_date = arrow.now()
-            logging.getLogger("shared").debug(f"[{self._error_name}] Sleep decay => Reducing sleep to [{self._sleep:.2f} seconds]")
-
-
+            logging.getLogger("shared").debug(
+                f"[{self.error_name}] Sleep decay => Reducing sleep to [{self._sleep:.2f} seconds]"
+            )
 
     async def async_do_sleep(self):
         async with self._alock:
             self._maybe_decay_sleep()
-        if self._sleep and self._sleep<=self._min_sleep:
-             logging.getLogger("shared").debug(
+        if self._sleep and self._sleep <= self._min_sleep:
+            logging.getLogger("shared").debug(
                 f"SessionSleep: Waiting [{self._sleep:.2f} seconds] due to {self.error_name} min sleep value"
             )
         elif self._sleep and self._sleep > 0:
@@ -124,7 +124,7 @@ class SessionSleep:
     def do_sleep(self):
         with self._lock:
             self._maybe_decay_sleep()
-        if self._sleep and self._sleep<=self._min_sleep:
+        if self._sleep and self._sleep <= self._min_sleep:
             logging.getLogger("shared").debug(
                 f"SessionSleep: Waiting [{self._sleep:.2f} seconds] due to {self.error_name} min sleep value"
             )
@@ -175,9 +175,6 @@ class SessionSleep:
         self._sleep = val
 
 
-
-
-
 class sessionManager:
     """
     Manages HTTP sessions, handling pooling, signing, retries, and dual backoff systems.
@@ -386,27 +383,27 @@ class sessionManager:
             #         max_connections=self._connect_limit,
             #         keepalive_expiry=self._keep_alive_exp,
             #     ),
-                    
+
             #     ),
             # )
 
-            self._session=aiohttp.ClientSession(
-                        connector=aiohttp.TCPConnector(
-                            limit=self._connect_limit,
-                            ssl=(
-                                False
-                                if not settings.get_settings().ssl_verify
-                                else create_custom_ssl_context()
-                            ),
-                        ),
-                          timeout=aiohttp.ClientTimeout(
-                                total= self._total_timeout,
-                                connect= self._connect_timeout,
-                                sock_connect=self._pool_connect_timeout,
-                                sock_read= self._read_timeout,
+            self._session = aiohttp.ClientSession(
+                connector=aiohttp.TCPConnector(
+                    limit=self._connect_limit,
+                    ssl=(
+                        False
+                        if not settings.get_settings().ssl_verify
+                        else create_custom_ssl_context()
+                    ),
+                ),
+                timeout=aiohttp.ClientTimeout(
+                    total=self._total_timeout,
+                    connect=self._connect_timeout,
+                    sock_connect=self._pool_connect_timeout,
+                    sock_read=self._read_timeout,
                 ),
             )
-            
+
         else:
             self._session = httpx.Client(
                 http2=True,
@@ -500,7 +497,6 @@ class sessionManager:
                             pass
                         else:
                             self._forbidden_sleeper.do_sleep()
-              
 
                         if (
                             SIGN in actions
@@ -525,8 +521,7 @@ class sessionManager:
                             timeout=httpx.Timeout(
                                 total_timeout or self._total_timeout,
                                 connect=connect_timeout,
-                                pool=pool_connect_timeout
-                                or self._pool_connect_timeout,
+                                pool=pool_connect_timeout or self._pool_connect_timeout,
                                 read=read_timeout or self._read_timeout,
                             ),
                         )
@@ -556,14 +551,14 @@ class sessionManager:
         json: dict = None,
         params: dict = None,
         data: dict = None,
-        wait_min:float=None,
-        wait_max:float=None,
-        retries:bool=None,
-        redirects:bool=True,
-        total_timeout:bool=None,
-        connect_timeout:float=None,
-        pool_connect_timeout:float=None,
-        read_timeout:float=None,
+        wait_min: float = None,
+        wait_max: float = None,
+        retries: bool = None,
+        redirects: bool = True,
+        total_timeout: bool = None,
+        connect_timeout: float = None,
+        pool_connect_timeout: float = None,
+        read_timeout: float = None,
         actions: Optional[list] = None,
         exceptions: Optional[list] = None,
         **kwargs,
@@ -572,15 +567,11 @@ class sessionManager:
         exceptions = exceptions or []
         retries = kwargs.get("retries") or self._retries
         wait_min = kwargs.get("wait_min") or self._wait_min
-        wait_max = (
-            kwargs.get("wait_max_exponential") or self._wait_max_exponential
-        )
+        wait_max = kwargs.get("wait_max_exponential") or self._wait_max_exponential
 
         async for _ in AsyncRetrying(
             stop=tenacity.stop.stop_after_attempt(retries),
-            wait=tenacity.wait_random(
-                min=wait_min, max=wait_max
-            ),
+            wait=tenacity.wait_random(min=wait_min, max=wait_max),
             retry=retry_if_not_exception_type((KeyboardInterrupt, SystemExit)),
             reraise=True,
             before=lambda x: (
@@ -595,7 +586,7 @@ class sessionManager:
                     if await self._rate_limit_sleeper.async_do_sleep():
                         pass
                     else:
-                        self._forbidden_sleeper.async_do_sleep()
+                        await self._forbidden_sleeper.async_do_sleep()
                     if SIGN in actions or FORCED_NEW in actions or HEADERS in actions:
                         headers = self._create_headers(
                             headers, url, SIGN in actions, FORCED_NEW in actions
@@ -604,24 +595,24 @@ class sessionManager:
                         cookies = self._create_cookies()
 
                     r = await self._aio_funct(
-                            method,
-                            url,
-                            timeout=aiohttp.ClientTimeout(
-                                total=total_timeout or self._total_timeout,
-                                connect=connect_timeout or self._connect_timeout,
-                                sock_connect=pool_connect_timeout
-                                or self._pool_connect_timeout,
-                                sock_read=read_timeout or self._read_timeout,
-                            ),
-                            headers=headers,
-                            cookies=cookies,
-                            allow_redirects=redirects,
-                            proxy=self._proxy,
-                            params=params,
-                            json=json,
-                            data=data,
-                            ssl=ssl.create_default_context(cafile=certifi.where()),
-                        )
+                        method,
+                        url,
+                        timeout=aiohttp.ClientTimeout(
+                            total=total_timeout or self._total_timeout,
+                            connect=connect_timeout or self._connect_timeout,
+                            sock_connect=pool_connect_timeout
+                            or self._pool_connect_timeout,
+                            sock_read=read_timeout or self._read_timeout,
+                        ),
+                        headers=headers,
+                        cookies=cookies,
+                        allow_redirects=redirects,
+                        proxy=self._proxy,
+                        params=params,
+                        json=json,
+                        data=data,
+                        ssl=ssl.create_default_context(cafile=certifi.where()),
+                    )
 
                     if not r.ok and r.status_code != 404:
                         self._log.debug(f"[bold]failed: [bold] {r.url}")
@@ -640,7 +631,6 @@ class sessionManager:
                     self._sem.release()
                     raise E
 
-
     @property
     def sleep(self):
         return self._rate_limit_sleeper.sleep
@@ -649,7 +639,6 @@ class sessionManager:
     def sleep(self, val):
         self._rate_limit_sleeper.sleep = val
 
-
     def _httpx_funct(self, method, **kwargs):
         t = self._session.request(method.upper(), **kwargs)
         t.ok = not t.is_error
@@ -659,6 +648,7 @@ class sessionManager:
         t.iter_chunked = t.iter_bytes
         t.read_ = t.read
         return t
+
     async def _aio_funct(self, method, *args, **kwargs):
         # public function forces context manager use
         r = await self._session._request(method, *args, **kwargs)
@@ -670,4 +660,4 @@ class sessionManager:
         r.status_code = r.status
         r.read_ = r.content.read
         r.eof = r.content.at_eof
-        return r
+        return r

{ofscraper-3.13.4 → ofscraper-3.13.5.dev2}/ofscraper/classes/table/fields/sizefield.py

@@ -1,5 +1,4 @@
 from ofscraper.classes.table.fields.numfield import PostiveNumField
-import ofscraper.utils.args.accessors.read as read_args
 import ofscraper.utils.settings as settings
 
 
@@ -13,7 +12,7 @@ class SizeMaxField(PostiveNumField):
 
     def on_input_changed(self, event):
         value = event.value if event.value else "0"
-        args = read_args.retriveArgs()
+        args = settings.get_args()
         args.size_max = int(value)
         settings.update_args(args)
 
@@ -28,6 +27,6 @@ class SizeMinField(PostiveNumField):
 
     def on_input_changed(self, event):
         value = event.value if event.value else "0"
-        args = read_args.retriveArgs()
+        args = settings.get_args()
         args.size_min = int(value)
         settings.update_args(args)

{ofscraper-3.13.4 → ofscraper-3.13.5.dev2}/ofscraper/commands/check.py

@@ -22,7 +22,6 @@ import ofscraper.data.api.timeline as timeline
 import ofscraper.classes.of.posts as posts_
 import ofscraper.classes.table.app as app
 import ofscraper.db.operations as operations
-import ofscraper.utils.args.accessors.read as read_args
 import ofscraper.utils.settings as settings
 import ofscraper.utils.auth.request as auth_requests
 import ofscraper.utils.console as console_
@@ -40,8 +39,8 @@ from ofscraper.db.operations_.media import (
 )
 from ofscraper.utils.checkers import check_auth
 from ofscraper.utils.context.run_async import run
-from ofscraper.scripts.final_user_script import post_user_script
-from ofscraper.main.close.final.final import final
+from ofscraper.scripts.after_download_action_script import after_download_action_script
+from ofscraper.main.close.final.final import final_action
 import ofscraper.main.manager as manager
 import ofscraper.filters.media.main as filters
 from ofscraper.commands.scraper.actions.download.download import process_dicts
@@ -73,11 +72,13 @@ def process_download_cart():
                     None
             if len(cart_dict.keys()) > 0:
                 for val in cart_dict.values():
-                    post_user_script(val["userdata"], val["media"], val["post"])
+                    after_download_action_script(
+                        val["userdata"], val["media"], val["post"]
+                    )
                 results = ["check cart results"] + list(
                     map(lambda x: x["results"], cart_dict.values())
                 )
-                final(normal_data=results)
+                final_action(normal_data=results)
             time.sleep(5)
         except Exception as e:
             log.traceback_(f"Error in process_item: {e}")
@@ -110,7 +111,7 @@ def process_item():
                 raise Exception(f"No data for {media_id}_{post_id}_{username}")
             log.info(f"Added url {media.url or media.mpd}")
             log.info("Sending URLs to OF-Scraper")
-            manager.Manager.model_manager.set_data_all_subs_model_dict(username)
+            manager.Manager.model_manager.add_model(username)
             post = media.post
             model_id = media.post.model_id
             username = media.post.username
@@ -136,6 +137,7 @@ def process_item():
                 app.app.table.update_cell_at_key(
                     key, "download_cart", Text("[downloaded]", style="bold green")
                 )
+                #might not be needed because of lack of final_script
                 manager.Manager.model_manager.mark_as_processed(username)
                 break
         except Exception as E:
@@ -190,7 +192,7 @@ async def data_refill(media_id, post_id, target_name, model_id):
 
 
 def allow_check_dupes():
-    args = read_args.retriveArgs()
+    args = settings.get_args()
     args.force_all = True
     settings.update_args(args)
 
@@ -219,7 +221,7 @@ def checker():
 
 
 def set_after_check_mode():
-    args = read_args.retriveArgs()
+    args = settings.get_args()
     args.after = 0
     settings.update_args(args)
 
@@ -413,7 +415,7 @@ async def post_check_retriver():
 
 def reset_data():
     # clean up args once check modes are ready to launch
-    args = read_args.retriveArgs()
+    args = settings.get_args()
     if args.username:
         args.username = settings.get_settings().usernames = None
     settings.update_args(args)

{ofscraper-3.13.4 → ofscraper-3.13.5.dev2}/ofscraper/commands/db.py

@@ -11,7 +11,6 @@ from humanfriendly import format_size
 
 import ofscraper.utils.console as console
 
-import ofscraper.utils.args.accessors.read as read_args
 from ofscraper.db.operations_.media import (
     get_timeline_media,
     get_archived_media,
@@ -54,7 +53,7 @@ class DBManager:
 
     @run_async
     async def get_all_media(self):
-        args = read_args.retriveArgs()
+        args = settings.get_args()
         timeline = []
         archived = []
         messages = []
@@ -108,7 +107,7 @@ class DBManager:
 
     def filter_media(self):
         self.log.info(f"filtering media for {self.username}_{self.model_id}")
-        args = read_args.retriveArgs()
+        args = settings.get_args()
         medias = self.media
         # downloaded
         if args.downloaded:

{ofscraper-3.13.4 → ofscraper-3.13.5.dev2}/ofscraper/commands/manual.py

@@ -10,8 +10,6 @@ import ofscraper.classes.of.media as media_
 import ofscraper.classes.of.posts as posts_
 import ofscraper.db.operations as operations
 import ofscraper.commands.scraper.actions.download.download as download
-import ofscraper.utils.args.accessors.read as read_args
-import ofscraper.utils.args.mutators.write as write_args
 import ofscraper.utils.of_env.of_env as of_env
 import ofscraper.utils.live.screens as progress_utils
 import ofscraper.utils.live.updater as progress_updater
@@ -23,7 +21,7 @@ from ofscraper.db.operations import make_changes_to_content_tables
 from ofscraper.db.operations_.media import batch_mediainsert
 from ofscraper.utils.checkers import check_auth
 from ofscraper.utils.context.run_async import run
-from ofscraper.main.close.final.final import final
+from ofscraper.main.close.final.final import final_action
 from ofscraper.commands.scraper.actions.download.utils.text import textDownloader
 import ofscraper.main.manager as manager
 import ofscraper.utils.settings as settings
@@ -95,7 +93,7 @@ def manual_download(urls=None):
 def final_action(results):
     normal_data = ["Manual Mode Results"]
     normal_data.extend(results)
-    final(
+    final_action(
         normal_data=normal_data,
         scrape_paid_data=None,
         user_first_data=None,
@@ -103,13 +101,13 @@ def final_action(results):
 
 
 def allow_manual_dupes():
-    args = read_args.retriveArgs()
+    args = settings.get_args()
     args.force_all = True
-    write_args.setArgs(args)
+    settings.update_args(args)
 
 
 def set_user_data(url_dicts):
-    manager.Manager.model_manager.set_data_all_subs_model_dict(
+    manager.Manager.model_manager.add_model(
         [nested_dict.get("username") for nested_dict in url_dicts.values()]
     )
 
@@ -297,7 +295,7 @@ def get_info(url):
 
 
 def url_helper(urls):
-    args = read_args.retriveArgs()
+    args = settings.get_args()
     out = []
     out.extend(args.get("file", []) or [])
     out.extend(args.get("url", []) or [])

{ofscraper-3.13.4 → ofscraper-3.13.5.dev2}/ofscraper/commands/metadata/consumer.py

@@ -1,14 +1,14 @@
 r"""
-                                                             
- _______  _______         _______  _______  _______  _______  _______  _______  _______ 
+
+ _______  _______         _______  _______  _______  _______  _______  _______  _______
 (  ___  )(  ____ \       (  ____ \(  ____ \(  ____ )(  ___  )(  ____ )(  ____ \(  ____ )
 | (   ) || (    \/       | (    \/| (    \/| (    )|| (   ) || (    )|| (    \/| (    )|
 | |   | || (__     _____ | (_____ | |      | (____)|| (___) || (____)|| (__    | (____)|
 | |   | ||  __)   (_____)(_____  )| |      |     __)|  ___  ||  _____)|  __)   |     __)
-| |   | || (                   ) || |      | (\ (   | (   ) || (      | (      | (\ (   
+| |   | || (                   ) || |      | (\ (   | (   ) || (      | (      | (\ (
 | (___) || )             /\____) || (____/\| ) \ \__| )   ( || )      | (____/\| ) \ \__
 (_______)|/              \_______)(_______/|/   \__/|/     \||/       (_______/|/   \__/
-                                                                                      
+
 """
 
 import asyncio
@@ -23,6 +23,7 @@ from ofscraper.commands.scraper.actions.utils.log import get_medialog
 
 from ofscraper.commands.metadata.manager import MetaDataManager
 from ofscraper.commands.metadata.desc import desc
+from ofscraper.classes.of.media import Media
 
 
 async def consumer(aws, task1, medialist, lock):
@@ -36,7 +37,7 @@ async def consumer(aws, task1, medialist, lock):
             break
         else:
             try:
-                ele = data[1]
+                ele: Media = data[1]
                 media_type = await MetaDataManager().metadata(*data)
             except Exception as e:
                 common_globals.log.info(
@@ -47,13 +48,17 @@ async def consumer(aws, task1, medialist, lock):
             try:
                 if media_type == "images":
                     common_globals.photo_count += 1
+                    ele.mark_download_finished()
 
                 elif media_type == "videos":
                     common_globals.video_count += 1
+                    ele.mark_download_finished()
                 elif media_type == "audios":
                     common_globals.audio_count += 1
+                    ele.mark_download_finished()
                 elif media_type == "skipped":
                     common_globals.skipped += 1
+                    ele.mark_download_finished(success=False)
                 elif media_type == "forced_skipped":
                     common_globals.forced_skipped += 1
                 sum_count = (
@@ -64,6 +69,7 @@ async def consumer(aws, task1, medialist, lock):
                     + common_globals.forced_skipped
                 )
                 log_download_progress(media_type)
+
                 progress_updater.update_metadata_task(
                     task1,
                     description=desc.format(