ofscraper 3.13.dev5__tar.gz → 3.13.dev7__tar.gz

{ofscraper-3.13.dev5 → ofscraper-3.13.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ofscraper
-Version: 3.13.dev5
+Version: 3.13.dev7
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net
@@ -27,6 +27,7 @@ Requires-Dist: faust-cchardet (>=2.1.19,<3.0.0)
 Requires-Dist: filelock (>=3.13.1,<4.0.0)
 Requires-Dist: httpx-aiohttp (>=0.1.2,<0.2.0)
 Requires-Dist: httpx-curl-cffi (>=0.1.3,<0.2.0)
+Requires-Dist: httpx[http2] (>=0.28.1,<0.29.0)
 Requires-Dist: humanfriendly (>=10.0,<11.0)
 Requires-Dist: idna (==3.7)
 Requires-Dist: inquirerpy (>=0.3.4,<0.4.0)
@@ -70,6 +71,9 @@ As the API has changed older version are no longer viable
 
 https://github.com/datawhores/OF-Scraper/issues/442
 
+# For Issues related to Downloading/ No Models Retrieved error
+https://github.com/datawhores/OF-Scraper/issues/542
+
 # ReadMe
 
 ## Releases

{ofscraper-3.13.dev5 → ofscraper-3.13.dev7}/README.md

@@ -11,6 +11,9 @@ As the API has changed older version are no longer viable
 
 https://github.com/datawhores/OF-Scraper/issues/442
 
+# For Issues related to Downloading/ No Models Retrieved error
+https://github.com/datawhores/OF-Scraper/issues/542
+
 # ReadMe
 
 ## Releases

{ofscraper-3.13.dev5 → ofscraper-3.13.dev7}/ofscraper/classes/sessionmanager/download.py

@@ -14,24 +14,30 @@ from ofscraper.classes.sessionmanager.sessionmanager import (
 )
 from ofscraper.commands.scraper.actions.download.utils.leaky import LeakyBucket
 import ofscraper.utils.settings as settings
+from ofscraper.commands.scraper.actions.download.utils.chunk import get_chunk_timeout
 
 
 class download_session(sessionManager.sessionManager):
     def __init__(
         self, sem_count=None, retries=None, wait_min=None, wait_max=None, log=None
     ) -> None:
-        sem_count = sem_count or common_globals.sem
         retries = retries or get_download_req_retries()
         wait_min = wait_min or constants.getattr("OF_MIN_WAIT_API")
         wait_max = wait_max or constants.getattr("OF_MAX_WAIT_API")
+        read_timeout=get_chunk_timeout()
         log = log or common_globals.log
         self.leaky_bucket = LeakyBucket(settings.get_settings().download_limit, 1)
+
+
+
+
         super().__init__(
             sem_count=sem_count,
             retries=retries,
             wait_min=wait_min,
             wait_max=wait_max,
             log=log,
+            read_timeout=read_timeout
         )
 
     @contextlib.asynccontextmanager
@@ -56,19 +62,6 @@ class download_session(sessionManager.sessionManager):
         t.request = t.request
         return t
 
-    async def _aio_funct(self, method, *args, **kwargs):
-        # public function forces context manager use
-        r = await self._session._request(method, *args, **kwargs)
-        r.text_ = r.text
-        r.json_ = r.json
-        r.iter_chunked = self.chunk_with_limit(r.content.iter_chunked)
-        r.iter_chunks = self.chunk_with_limit(r.content.iter_chunks)
-        r.request = r.request_info
-        r.status_code = r.status
-        r.read_ = r.content.read
-        r.eof = r.content.at_eof
-        return r
-
     async def factoryasync(self, input):
         if callable(input):
             return input()
@@ -88,11 +81,9 @@ class download_session(sessionManager.sessionManager):
 
 class cdm_session(sessionManager.sessionManager):
     def __init__(self,  sem_count=None) -> None:
-        sem_count = sem_count or common_globals.sem
         super().__init__(sem_count=sem_count)
 
 
 class cdm_session_manual(ofsessionmanager.OFSessionManager):
     def __init__(self,  sem_count=None) -> None:
-        sem_count = sem_count or common_globals.sem
         super().__init__(sem_count=sem_count)

{ofscraper-3.13.dev5 → ofscraper-3.13.dev7}/ofscraper/classes/sessionmanager/sessionmanager.py

@@ -50,8 +50,15 @@ def check_400(exception):
         time.sleep(8)
 
 
+
 def is_provided_exception_number(exception, *numbers):
     return (
+        isinstance(exception, aiohttp.ClientResponseError)
+        and (
+            getattr(exception, "status_code", None)
+            or getattr(exception, "status", None) in numbers
+        )
+    ) or (
         isinstance(exception, httpx.HTTPStatusError)
         and (
             (
@@ -62,7 +69,6 @@ def is_provided_exception_number(exception, *numbers):
         )
     )
 
-
 class SessionSleep:
     def __init__(self, sleep=None, difmin=None):
         self._sleep = None
@@ -179,7 +185,6 @@ class sessionManager:
     ):
         connect_timeout = connect_timeout or constants.getattr("CONNECT_TIMEOUT")
         total_timeout = total_timeout or constants.getattr("TOTAL_TIMEOUT")
-        read_timeout = read_timeout or constants.getattr("CHUNK_READ_TIMEOUT")
         pool_timeout = pool_timeout or constants.getattr("POOL_CONNECT_TIMEOUT")
         limit = limit or constants.getattr("MAX_CONNECTIONS")
         keep_alive = keep_alive or constants.getattr("KEEP_ALIVE")
@@ -188,7 +193,7 @@ class sessionManager:
         proxy_auth = proxy_auth or constants.getattr("PROXY_AUTH")
         self._connect_timeout = connect_timeout
         self._total_timeout = total_timeout
-        self._read_timeout = read_timeout
+        self._read_timeout=read_timeout
         self._pool_connect_timeout = pool_timeout
         self._connect_limit = limit
         self._keep_alive = keep_alive
@@ -217,7 +222,7 @@ class sessionManager:
             return
         if async_:
             self._session = httpx.AsyncClient(
-                # http2=True,
+                http2=True,
                 proxy=self._proxy,
                 limits=httpx.Limits(
                     max_keepalive_connections=self._keep_alive,
@@ -232,7 +237,7 @@ class sessionManager:
                             )
         else:
             self._session = httpx.Client(
-                # http2=True,
+                http2=True,
                 proxy=self._proxy,
                 limits=httpx.Limits(
                     max_keepalive_connections=self._keep_alive,
@@ -347,7 +352,7 @@ class sessionManager:
                             total_timeout or self._total_timeout,
                             connect=connect_timeout or self._connect_timeout,
                             pool=pool_connect_timeout or self._pool_connect_timeout,
-                            read=read_timeout or self._read_timeout,
+                            read=read_timeout or self._read_timeout
                         ),
                         url=url,
                         follow_redirects=redirects,
@@ -429,9 +434,9 @@ class sessionManager:
             ),
             reraise=True,
         ):
+            await sem.acquire()
             with _:
                 r = None
-                await sem.acquire()
                 try:
                     await sleeper.async_do_sleep()
                     if (
@@ -456,7 +461,8 @@ class sessionManager:
                             total_timeout or self._total_timeout,
                             connect=connect_timeout or self._connect_timeout,
                             pool=pool_connect_timeout or self._pool_connect_timeout,
-                            read=read_timeout or self._read_timeout,
+                            read=read_timeout or self._read_timeout
+
                         ),
                         follow_redirects=redirects,
                         url=url,
@@ -487,6 +493,118 @@ class sessionManager:
         yield r
         sem.release()
 
+    @contextlib.asynccontextmanager
+    async def requests_async_stream(
+        self,
+        url=None,
+        wait_min=None,
+        wait_max=None,
+        wait_min_exponential=None,
+        wait_max_exponential=None,
+        retries=None,
+        method="get",
+        headers=None,
+        cookies=None,
+        json=None,
+        params=None,
+        redirects=True,
+        data=None,
+        log=None,
+        sem=None,
+        total_timeout=None,
+        connect_timeout=None,
+        pool_connect_timeout=None,
+        read_timeout=None,
+        sleeper=None,
+        exceptions=[],
+        actions=[],
+        *args,
+        **kwargs,
+    ):
+        wait_min = wait_min or self._wait_min
+        wait_max = wait_max or self._wait_max
+        wait_min_exponential = wait_min_exponential or self._wait_min_exponential
+        wait_max_exponential = wait_max_exponential or self._wait_max_exponential
+        log = log or self._log
+        retries = retries or self._retries
+        sem = sem or self._sem
+        sleeper = sleeper or self._sleeper
+        exceptions = exceptions or []
+        actions = actions or []
+        async for _ in CustomTenacity(
+            wait_exponential=tenacity.wait.wait_exponential(
+                multiplier=2, min=wait_min_exponential, max=wait_max_exponential
+            ),
+            retry=retry_if_not_exception_type((KeyboardInterrupt)),
+            wait_random=tenacity.wait_random(min=wait_min, max=wait_max),
+            stop=tenacity.stop.stop_after_attempt(retries),
+            before=lambda x: (
+                log.debug(f"[bold]attempt: {x.attempt_number}[bold] for {url}")
+                if x.attempt_number > 1
+                else None
+            ),
+            reraise=True,
+        ):
+            with _:
+                r = None
+                try:
+                    await sem.acquire()
+                    await sleeper.async_do_sleep()
+                    if (
+                            SIGN in actions
+                            or FORCED_NEW in actions
+                            or HEADERS in actions
+                    ): 
+                        headers = (
+                        self._create_headers(
+                            headers, url, SIGN in actions, FORCED_NEW in actions
+                        )
+                       
+                    )
+
+
+                    cookies = self._create_cookies() if COOKIES in actions else None
+                    json = json
+                    params = params
+                    r = await self._httpx_funct_async_stream(
+                        method,
+                        timeout=httpx.Timeout(
+                            total_timeout or self._total_timeout,
+                            connect=connect_timeout or self._connect_timeout,
+                            pool=pool_connect_timeout or self._pool_connect_timeout,
+                            read=read_timeout or self._read_timeout
+                        ),
+                        follow_redirects=redirects,
+                        url=url,
+                        cookies=cookies,
+                        headers=headers,
+                        json=json,
+                        params=params,
+                        data=data,
+                    )
+                    if r.status_code == 404:
+                        pass
+                    elif not r.ok:
+                        log.debug(f"[bold]failed: [bold] {r.url}")
+                        log.debug(f"[bold]status: [bold] {r.status}")
+                        log.debug(f"response headers {dict(r.headers)}")
+                        log.debug(f"requests headers mode{dict(r.request.headers)}")
+                        r.raise_for_status()
+                    yield r
+                except Exception as E:
+                    # only call from sync req like "me"
+                    # check_400(E)
+                    log.traceback_(E)
+                    log.traceback_(traceback.format_exc())
+                    if TOO_MANY in exceptions:
+                        await async_is_rate_limited(E, sleeper)
+                    raise E
+                finally:
+                    sem.release()
+                    await r.aclose()
+
+
+
     @property
     def sleep(self):
         return self._sleeper._sleep
@@ -506,6 +624,27 @@ class sessionManager:
         t.read_ = t.aread
         t.request = t.request
         return t
+    
+    async def _httpx_funct_async_stream(self, *args, **kwargs):
+        auth=kwargs.pop("auth",None)
+        follow_redirects=kwargs.pop("follow_redirects",None)
+        req=self._session.build_request(*args, **kwargs)
+        t = await self._session.send(
+            request=req,
+            follow_redirects=follow_redirects,
+            stream=True,
+            auth=auth,
+        )
+        t.ok = not t.is_error
+        t.json_ = lambda: self.factoryasync(t.json)
+        t.text_ = lambda: self.factoryasync(t.text)
+        t.status = t.status_code
+        t.iter_chunked = t.aiter_bytes
+        t.iter_chunks = t.aiter_bytes
+        t.read_ = t.aread
+        t.request = t.request
+        return t
+
 
     def _httpx_funct(self, method, **kwargs):
         t = self._session.request(method.upper(), **kwargs)

{ofscraper-3.13.dev5 → ofscraper-3.13.dev7}/ofscraper/commands/metadata/metadata.py

@@ -48,7 +48,6 @@ import ofscraper.utils.context.exit as exit
 from ofscraper.classes.sessionmanager.download import download_session
 from ofscraper.commands.scraper.actions.utils.log import final_log, final_log_text
 from ofscraper.commands.scraper.actions.utils.paths.paths import setDirectoriesDate
-from ofscraper.commands.scraper.actions.utils.buffer import download_log_clear_helper
 
 from ofscraper.commands.scraper.actions.utils.workers import get_max_workers
 from ofscraper.utils.context.run_async import run
@@ -192,7 +191,7 @@ class metadataCommandManager(commmandManager):
                                 and avatar
                                 and settings.get_settings().userfirst
                             ):
-                                logging.getLogger("shared_other").warning(
+                                logging.getLogger("shared").warning(
                                     avatar_str.format(avatar=avatar)
                                 )
                             data.extend(
@@ -294,7 +293,6 @@ def metadata():
 
 @run
 async def process_dicts(username, model_id, medialist):
-    download_log_clear_helper()
     task1 = None
     with progress_utils.setup_metadata_progress_live():
         common_globals.mainProcessVariableInit()
@@ -337,7 +335,6 @@ async def process_dicts(username, model_id, medialist):
             common_globals.thread.shutdown()
 
         setDirectoriesDate()
-        download_log_clear_helper()
         progress_updater.remove_metadata_task(task1)
         final_log(username)
         return final_log_text(username)

{ofscraper-3.13.dev5 → ofscraper-3.13.dev7}/ofscraper/commands/scraper/actions/download/download.py

@@ -22,7 +22,6 @@ from ofscraper.classes.sessionmanager.download import download_session
 from ofscraper.commands.scraper.actions.utils.log import final_log, final_log_text
 
 from ofscraper.commands.scraper.actions.utils.paths.paths import setDirectoriesDate
-from ofscraper.commands.scraper.actions.utils.buffer import download_log_clear_helper
 
 from ofscraper.commands.scraper.actions.utils.workers import get_max_workers
 from ofscraper.utils.context.run_async import run
@@ -45,7 +44,7 @@ async def downloader(username=None, model_id=None, posts=None, media=None, **kwa
     )
 
     progress_updater.update_activity_task(description=download_str + path_str)
-    logging.getLogger("shared_other").warning(
+    logging.getLogger("shared").warning(
         download_activity_str.format(username=username)
     )
     progress_updater.update_activity_task(description="")
@@ -71,7 +70,6 @@ async def download_model_deleted_process(
 @run
 async def process_dicts(username, model_id, medialist, posts):
     log_text_array = []
-    download_log_clear_helper()
     log_text_array.append(await textDownloader(posts, username=username))
     logging.getLogger("shared").info("Downloading in single thread mode")
     common_globals.mainProcessVariableInit()
@@ -103,7 +101,6 @@ async def process_dicts(username, model_id, medialist, posts):
         try:
 
             aws = []
-
             async with download_session() as c:
                 for ele in medialist:
                     aws.append((c, ele, model_id, username))
@@ -139,7 +136,6 @@ async def process_dicts(username, model_id, medialist, posts):
             common_globals.thread.shutdown()
 
         setDirectoriesDate()
-        download_log_clear_helper()
         final_log(username, log=logging.getLogger("shared"))
         progress_updater.remove_download_task(task1)
         log_text_array.append(final_log_text(username))

{ofscraper-3.13.dev5 → ofscraper-3.13.dev7}/ofscraper/commands/scraper/actions/download/managers/alt_download.py

@@ -36,7 +36,7 @@ from ofscraper.commands.scraper.actions.utils.log import (
     temp_file_logger,
 )
 from ofscraper.commands.scraper.actions.download.utils.chunk import (
-    get_chunk_size,
+    get_chunk_size
 )
 from ofscraper.commands.scraper.actions.utils.retries import get_download_retries
 from ofscraper.commands.scraper.actions.utils.send.chunk import send_chunk_msg
@@ -65,6 +65,7 @@ from ofscraper.commands.scraper.actions.utils.send.message import send_msg
 class AltDownloadManager(DownloadManager):
 
     async def alt_download(self, c, ele, username, model_id):
+        await common_globals.sem.acquire()
         common_globals.log.debug(
             f"{get_medialog(ele)} Downloading with protected media downloader"
         )
@@ -184,8 +185,9 @@ class AltDownloadManager(DownloadManager):
             common_globals.log.debug(
                 f"{get_medialog(ele)} [attempt {self._alt_attempt_get(item).get()}/{get_download_retries()}] Downloading media with url  {ele.mpd}"
             )
-            async with c.requests_async(
+            async with c.requests_async_stream(
                 url=url,
+                stream=True,
                 headers=headers,
                 params=params,
                 # action=[FORCED_NEW,SIGN] if constants.getattr("ALT_FORCE_KEY") else None
@@ -222,17 +224,20 @@ class AltDownloadManager(DownloadManager):
         except Exception as E:
             await self._total_change_helper(0)
             raise E
+        finally:
+            common_globals.sem.release()
 
     async def _download_fileobject_writer(self, total, l, ele, placeholderObj, item):
         common_globals.log.debug(
             f"{get_medialog(ele)} [attempt {self._alt_attempt_get(item).get()}/{get_download_retries()}] writing media to disk"
         )
-        if total > constants.getattr("MAX_READ_SIZE"):
-            await self._download_fileobject_writer_streamer(
-                ele, total, l, placeholderObj
-            )
-        else:
-            await self._download_fileobject_writer_reader(ele, total, l, placeholderObj)
+        await self._download_fileobject_writer_streamer(
+            ele, total, l, placeholderObj
+        )
+        # if total > constants.getattr("MAX_READ_SIZE"):
+        
+        # else:
+        #     await self._download_fileobject_writer_reader(ele, total, l, placeholderObj)
         common_globals.log.debug(
             f"{get_medialog(ele)} [attempt {self._alt_attempt_get(item).get()}/{get_download_retries()}] finished writing media to disk"
         )
@@ -258,30 +263,60 @@ class AltDownloadManager(DownloadManager):
             except Exception as E:
                 raise E
 
-    async def _download_fileobject_writer_streamer(
-        self, ele, total, res, placeholderObj
-    ):
+    async def _download_fileobject_writer_streamer(self, ele, total, res, placeholderObj):
+        common_globals.log.info(f"Starting download for {ele}. Tracking memory usage...")
+        initial_memory = self.process.memory_info().rss / (1024 * 1024) # RSS in MB
+        common_globals.log.info(f"Initial memory usage for {ele}: {initial_memory:.2f} MB")
 
         task1 = await self._add_download_job_task(ele, total, placeholderObj)
-        fileobject = await aiofiles.open(placeholderObj.tempfilepath, "ab").__aenter__()
-        chunk_size = get_chunk_size()
+        fileobject = None # Initialize to None for finally block
         try:
-            async for chunk in res.iter_chunked(chunk_size):
-                await fileobject.write(chunk)
-                send_chunk_msg(ele, total, placeholderObj)
+            # Use asyncio.timeout as a context manager for the entire download process
+            async with asyncio.timeout(None):
+                fileobject = await aiofiles.open(
+                    placeholderObj.tempfilepath, "ab"
+                ).__aenter__()
+                chunk_count = 0
+                chunk_iter = res.iter_chunked(get_chunk_size())
+
+                while True:
+                    try:
+                        chunk = await chunk_iter.__anext__()
+                        await fileobject.write(chunk)
+                        send_chunk_msg(ele, total, placeholderObj)
+                        chunk_count += 1
+                        # Log memory usage periodically, e.g., every 10 chunks
+                        if chunk_count % 10 == 0:
+                            current_memory = self.process.memory_info().rss / (1024 * 1024)
+                            common_globals.log.debug(f"Memory usage for {ele} after {chunk_count} chunks: {current_memory:.2f} MB")
+                    except StopAsyncIteration:
+                        break # Exit loop when no more chunks
+        except asyncio.TimeoutError:
+            # This catches the timeout for the entire async with block
+            common_globals.log.warning(f"{common_logs.get_medialog(ele)}⚠️ No chunk received in {get_chunk_timeout()} seconds or download timed out!")
+            return # Exit the function on timeout
         except Exception as E:
-            raise E
+            # Catch other potential exceptions during file operations or chunk iteration
+            common_globals.log.error(f"An error occurred during download for {ele}: {E}")
+            raise E # Re-raise the exception after logging
         finally:
-            # Close file if needed
-            try:
-                await fileobject.close()
-            except Exception as E:
-                raise E
+            # Log final memory usage
+            final_memory = self.process.memory_info().rss / (1024 * 1024)
+            common_globals.log.info(f"Final memory usage for {ele}: {final_memory:.2f} MB")
+            common_globals.log.info(f"Memory change for {ele}: {(final_memory - initial_memory):.2f} MB")
 
+            # Close file if needed
+            if fileobject: # Ensure fileobject was successfully opened
+                try:
+                    await fileobject.close()
+                except Exception as E:
+                    common_globals.log.error(f"Error closing file for {ele}: {E}")
+                    raise E # Re-raise if closing fails
             try:
                 await self._remove_download_job_task(task1, ele)
             except Exception as E:
-                raise E
+                common_globals.log.error(f"Error removing download job task for {ele}: {E}")
+                raise E # Re-raise if task removal fails
 
     async def _handle_result_alt(
         self, sharedPlaceholderObj, ele, audio, video, username, model_id

{ofscraper-3.13.dev5 → ofscraper-3.13.dev7}/ofscraper/commands/scraper/actions/download/managers/downloadmanager.py

@@ -11,9 +11,10 @@ r"""
                                                                                       
 """
 
-from functools import partial
 import pathlib
+import os
 from humanfriendly import format_size
+import psutil
 
 import ofscraper.utils.constants as constants
 import ofscraper.utils.live.updater as progress_updater
@@ -30,6 +31,7 @@ from ofscraper.db.operations_.media import download_media_update
 class DownloadManager:
     def __init__(self):
         self.total = None
+        self.process= psutil.Process(os.getpid())
 
     async def _add_download_job_task(
         self, ele, total=None, placeholderObj=None, tempholderObj=None