ofscraper 3.11.dev1__tar.gz → 3.11.2__tar.gz

{ofscraper-3.11.dev1 → ofscraper-3.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ofscraper
-Version: 3.11.dev1
+Version: 3.11.2
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net
@@ -11,9 +11,11 @@ Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: pyinstaller
 Requires-Dist: aiofiles (>=23.2.1,<24.0.0)
 Requires-Dist: aiohttp (==3.9.4)
+Requires-Dist: aiolimiter (>=1.1.0,<2.0.0)
 Requires-Dist: aioprocessing (>=2.0.1,<3.0.0)
 Requires-Dist: aiosqlite (>=0.20.0,<0.21.0)
 Requires-Dist: arrow (>=1.3.0,<2.0.0)
+Requires-Dist: async-property (>=0.2.2,<0.3.0)
 Requires-Dist: browser-cookie3 (==0.19.1)
 Requires-Dist: bs4 (>=0.0.2,<0.0.3)
 Requires-Dist: certifi (>=2024.2.2,<2025.0.0)
@@ -27,6 +29,7 @@ Requires-Dist: httpx[http2] (>=0.27.0,<0.28.0)
 Requires-Dist: humanfriendly (>=10.0,<11.0)
 Requires-Dist: idna (==3.7)
 Requires-Dist: inquirerpy (>=0.3.4,<0.4.0)
+Requires-Dist: limiter (>=0.5.0,<0.6.0)
 Requires-Dist: lxml (>=5.1.0,<6.0.0)
 Requires-Dist: more-itertools (>=10.2.0,<11.0.0)
 Requires-Dist: mpegdash (>=0.4.0,<0.5.0)
@@ -42,7 +45,7 @@ Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: rich (>=13.7.1,<14.0.0)
 Requires-Dist: schedule (>=1.2.1,<2.0.0)
 Requires-Dist: setproctitle (>=1.3.3,<2.0.0)
-Requires-Dist: setuptools (>=69.1.1,<70.0.0)
+Requires-Dist: setuptools (>=69.1.1,<71.0.0)
 Requires-Dist: speedtest-cli (>=2.1.3,<3.0.0)
 Requires-Dist: tenacity (>=8.2.3,<9.0.0)
 Requires-Dist: textual (==0.52.1)
@@ -53,13 +56,9 @@ Requires-Dist: xxhash (>=3.4.1,<4.0.0)
 Project-URL: Homepage, https://github.com/datawhores/OF-Scraper
 Description-Content-Type: text/markdown
 
-# FOR NOW ALL AUTH ISSUE REPORTS WILL BE CLOSED
+# For auth issues
 
-# USE THE PIN POST
-
-https://github.com/datawhores/OF-Scraper/issues/402
-
-# Chances are you will find your solution there anyways
+https://github.com/datawhores/OF-Scraper/issues/442
 
 # ReadMe
 

{ofscraper-3.11.dev1 → ofscraper-3.11.2}/README.md

@@ -1,10 +1,6 @@
-# FOR NOW ALL AUTH ISSUE REPORTS WILL BE CLOSED
+# For auth issues
 
-# USE THE PIN POST
-
-https://github.com/datawhores/OF-Scraper/issues/402
-
-# Chances are you will find your solution there anyways
+https://github.com/datawhores/OF-Scraper/issues/442
 
 # ReadMe
 

{ofscraper-3.11.dev1 → ofscraper-3.11.2}/ofscraper/__main__.py

@@ -1,7 +1,7 @@
 #!/root/OF-Scraper/.venv/bin/python
 import multiprocessing
 
-import ofscraper.runner.load as load
+import ofscraper.runner.open.load as load
 import ofscraper.utils.system.system as system
 
 

ofscraper-3.11.2/ofscraper/actions/actions/download/batch/downloadbatch.py

@@ -0,0 +1,224 @@
+import asyncio
+import logging
+import logging.handlers
+import os
+import platform
+import threading
+
+import aioprocessing
+from aioprocessing import AioPipe
+
+import ofscraper.actions.utils.globals as common_globals
+import ofscraper.data.models.selector as selector
+import ofscraper.utils.args.accessors.read as read_args
+import ofscraper.utils.cache as cache
+import ofscraper.utils.context.exit as exit
+import ofscraper.utils.dates as dates
+import ofscraper.utils.live.screens as progress_utils
+import ofscraper.utils.live.updater as progress_updater
+
+import ofscraper.utils.system.system as system
+import ofscraper.utils.system.priority as priority
+
+from ofscraper.classes.sessionmanager.download import download_session
+from ofscraper.actions.utils.globals import subProcessVariableInit
+from ofscraper.actions.utils.log import (
+    final_log,
+    final_log_text,
+)
+
+from ofscraper.actions.utils.paths.paths import setDirectoriesDate
+from ofscraper.actions.utils.send.message import send_msg
+from ofscraper.actions.utils.workers import get_max_workers
+from ofscraper.utils.context.run_async import run
+import ofscraper.utils.logs.stdout as stdout_logs
+import ofscraper.utils.logs.other as other_logs
+
+from ofscraper.utils.system.speed import add_pids_to_download_obj
+from ofscraper.actions.utils.buffer import download_log_clear_helper
+from ofscraper.actions.utils.mediasplit import get_mediasplits
+from ofscraper.actions.actions.download.batch.utils.consumer  import consumer
+from ofscraper.actions.utils.threads import handle_threads,start_threads
+from ofscraper.actions.actions.download.batch.utils.queue import queue_process
+from  ofscraper.actions.actions.download.utils.desc import desc
+platform_name = platform.system()
+
+
+
+def process_dicts(username, model_id, filtered_medialist):
+    log = logging.getLogger("shared")
+    try:
+        common_globals.main_globals()
+        download_log_clear_helper()
+        with progress_utils.setup_download_progress_live(multi=True):
+            mediasplits = get_mediasplits(filtered_medialist)
+            num_proc = len(mediasplits)
+            log.debug(f"Number of download threads: {num_proc}")
+            connect_tuples = [AioPipe() for _ in range(num_proc)]
+            stdout_logqueues = [AioPipe() for _ in range(num_proc)]
+    
+            #start stdout/main queues consumers
+            log_threads=[]
+            for i in range(num_proc):
+                thread=stdout_logs.start_stdout_logthread(
+                input_=stdout_logqueues[i][0],
+                name=f"ofscraper_{model_id}_{i+1}",
+                count=1,
+                )
+                log_threads.append(thread)
+
+
+            processes = [
+                aioprocessing.AioProcess(
+                    target=process_dict_starter,
+                    args=(
+                        username,
+                        model_id,
+                        mediasplits[i],
+                        stdout_logqueues[i][1],
+                        connect_tuples[i][1],
+                        dates.getLogDate(),
+                        selector.get_ALL_SUBS_DICT(),
+                        read_args.retriveArgs(),
+                    ),
+
+                )
+                for i in range(num_proc)
+            ]
+            add_pids_to_download_obj(map(lambda x: x.pid, processes))
+
+            task1 = progress_updater.add_download_task(
+                desc.format(
+                    p_count=0,
+                    v_count=0,
+                    a_count=0,
+                    skipped=0,
+                    mediacount=len(filtered_medialist),
+                    forced_skipped=0,
+                    sumcount=0,
+                    total_bytes_download=0,
+                    total_bytes=0,
+                ),
+                total=len(filtered_medialist),
+                visible=True,
+            )
+            queue_threads = [
+                threading.Thread(
+                    target=queue_process,
+                    args=(
+                        connect_tuples[i][0],
+                        task1,
+                        len(filtered_medialist),
+                    ),
+                    daemon=True,
+                )
+                for i in range(num_proc)
+            ]
+            start_threads(queue_threads,processes)
+            handle_threads(queue_threads,processes,log_threads)
+            # for reading completed downloads
+        download_log_clear_helper()
+        progress_updater.remove_download_task(task1)
+        setDirectoriesDate(log)
+        final_log(username)
+        return final_log_text(username)
+    except KeyboardInterrupt as E:
+        try:
+            with exit.DelayedKeyboardInterrupt():
+                [process.terminate() for process in processes or []]
+                raise KeyboardInterrupt
+        except KeyboardInterrupt:
+            with exit.DelayedKeyboardInterrupt():
+                raise E
+        except:
+            with exit.DelayedKeyboardInterrupt():
+                raise E
+    except Exception as E:
+        try:
+            with exit.DelayedKeyboardInterrupt():
+                [process.terminate() for process in processes or []]
+                raise E
+        except KeyboardInterrupt:
+            with exit.DelayedKeyboardInterrupt():
+                raise E
+        except Exception:
+            with exit.DelayedKeyboardInterrupt():
+                raise E
+
+
+
+def process_dict_starter(
+    username,
+    model_id,
+    ele,
+    stdout_logqueue,
+    pipe_,
+    dateDict,
+    userNameList,
+    argsCopy,
+):
+    #queue for file and discord
+    file_logqueue=AioPipe()
+    subProcessVariableInit(
+        dateDict,
+        userNameList,
+        pipe_,
+        argsCopy,
+        stdout_logqueue,
+        file_logqueue[0]
+    )
+    common_globals.log.debug(f"{pid_log_helper()} preparing thread")
+    priority.setpriority()
+    system.setNameAlt()
+    plat = platform.system()
+    if plat == "Linux":
+        import uvloop
+        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+    other_logs.start_other_thread(
+        input_=file_logqueue[1], name=str(os.getpid()), count=1
+    )
+    try:
+        process_dicts_split(username, model_id, ele)
+    except KeyboardInterrupt as E:
+        with exit.DelayedKeyboardInterrupt():
+            try:
+                pipe_.send(None)
+                common_globals.log.log(100,None)
+                raise E
+            except Exception as E:
+                raise E
+
+
+
+
+@run
+async def process_dicts_split(username, model_id, medialist):
+    medialist = list(medialist)
+    common_globals.log.debug(f"{pid_log_helper()} starting process")
+    common_globals.log.debug(
+        f"{pid_log_helper()} process mediasplit from total {len(medialist)}"
+    )
+    aws = []
+    async with download_session() as c:
+        for ele in medialist:
+            aws.append((c, ele, model_id, username))
+        concurrency_limit = get_max_workers()
+        lock = asyncio.Lock()
+        consumers = [
+            asyncio.create_task(consumer(lock,aws)) for _ in range(concurrency_limit)
+        ]
+
+        await asyncio.gather(*consumers)
+    common_globals.log.debug(f"{pid_log_helper()} download process thread closing")
+    # send message directly
+    await asyncio.get_event_loop().run_in_executor(common_globals.thread, cache.close)
+    common_globals.thread.shutdown()
+    common_globals.log.log(100,None)
+    await send_msg({"dir_update": common_globals.localDirSet})
+    await send_msg(None)
+
+
+def pid_log_helper():
+    return f"PID: {os.getpid()}"
+
+

ofscraper-3.11.2/ofscraper/actions/actions/download/batch/utils/consumer.py

@@ -0,0 +1,29 @@
+import platform
+import traceback
+
+from ofscraper.actions.utils.send.message import send_msg
+from ofscraper.actions.actions.download.runners.download import download
+import ofscraper.actions.utils.globals as common_globals
+
+
+platform_name = platform.system()
+async def consumer(lock,aws):
+    while True:
+        async with lock:
+            if not(bool(aws)):
+                break
+            data = aws.pop()
+        if data is None:
+            break
+        else:
+            try:
+                pack = await download(*data,multi=True)
+                common_globals.log.debug(f"unpack {pack} count {len(pack)}")
+                media_type, num_bytes_downloaded = pack
+                await send_msg((media_type, num_bytes_downloaded, 0))
+            except Exception as e:
+                common_globals.log.info(f"Download Failed because\n{e}")
+                common_globals.log.traceback_(traceback.format_exc())
+                media_type = "skipped"
+                num_bytes_downloaded = 0
+                await send_msg((media_type, num_bytes_downloaded, 0))

ofscraper-3.11.2/ofscraper/actions/actions/download/batch/utils/queue.py

@@ -0,0 +1,123 @@
+import asyncio
+import logging
+import logging.handlers
+import platform
+import traceback
+
+import ofscraper.actions.utils.globals as common_globals
+
+import ofscraper.utils.live.updater as progress_updater
+from ofscraper.actions.utils.log import (
+    log_download_progress,
+)
+
+from ofscraper.actions.utils.paths.paths import addGlobalDir
+from ofscraper.actions.utils.progress.convert import convert_num_bytes
+from  ofscraper.actions.actions.download.utils.desc import desc
+
+platform_name = platform.system()
+def queue_process(pipe_, task1, total):
+    count = 0
+    while True:
+        try:
+            if count == 1:
+                break
+            try:
+                if not pipe_.poll(timeout=1) :
+                    continue
+                results = pipe_.recv()
+                if not isinstance(results, list):
+                    results = [results]
+            except Exception as E:
+                common_globals.log.traceback_(E)
+                common_globals.log.traceback_(traceback.format_exc())
+                continue
+            for result in results:
+                try:
+                    if isinstance(result, list) or isinstance(result, tuple):
+                        media_type, num_bytes_downloaded, total_size = result
+                        with common_globals.count_lock:
+                            common_globals.total_bytes_downloaded = (
+                                common_globals.total_bytes_downloaded + num_bytes_downloaded
+                            )
+                            common_globals.total_bytes = (
+                                common_globals.total_bytes + total_size
+                            )
+                            if media_type == "images":
+                                common_globals.photo_count += 1
+
+                            elif media_type == "videos":
+                                common_globals.video_count += 1
+                            elif media_type == "audios":
+                                common_globals.audio_count += 1
+                            elif media_type == "skipped":
+                                common_globals.skipped += 1
+                            elif media_type == "forced_skipped":
+                                common_globals.forced_skipped += 1
+                            log_download_progress(media_type)
+                            progress_updater.update_download_task(
+                                task1,
+                                description=desc.format(
+                                    p_count=common_globals.photo_count,
+                                    v_count=common_globals.video_count,
+                                    a_count=common_globals.audio_count,
+                                    skipped=common_globals.skipped,
+                                    forced_skipped=common_globals.forced_skipped,
+                                    total_bytes_download=convert_num_bytes(
+                                        common_globals.total_bytes_downloaded
+                                    ),
+                                    total_bytes=convert_num_bytes(
+                                        common_globals.total_bytes
+                                    ),
+                                    mediacount=total,
+                                    sumcount=common_globals.video_count
+                                    + common_globals.audio_count
+                                    + common_globals.photo_count
+                                    + common_globals.skipped
+                                    + common_globals.forced_skipped,
+                                ),
+                                refresh=True,
+                                completed=common_globals.video_count
+                                + common_globals.audio_count
+                                + common_globals.photo_count
+                                + common_globals.skipped
+                                + common_globals.forced_skipped,
+                            )
+
+                    elif result is None or result == "None":
+                        count = count + 1
+                    elif isinstance(result, dict) and "dir_update" in result:
+                        addGlobalDir(result["dir_update"])
+                    elif callable(result):
+                        job_progress_helper(result)
+                except Exception as E:
+                    common_globals.log.traceback_(E)
+                    common_globals.log.traceback_(traceback.format_exc())
+                    #increase skipped
+                    common_globals.skipped += 1
+        except Exception as E:
+                    common_globals.log.traceback_(E)
+                    common_globals.log.traceback_(traceback.format_exc())
+                    #increase skipped
+
+def job_progress_helper(funct):
+    try:
+        funct()
+    #probably handle by other thread
+    except KeyError:
+        pass
+    except Exception as E:
+        logging.getLogger("shared").debug(E)
+
+
+async def ajob_progress_helper(funct):
+    try:
+        await asyncio.get_event_loop().run_in_executor(
+                None,
+                funct,
+        )
+    #probably handle by other thread
+    except KeyError:
+        pass
+    except Exception as E:
+        logging.getLogger("shared").debug(E)

ofscraper-3.11.2/ofscraper/actions/actions/download/download.py

@@ -0,0 +1,74 @@
+import logging
+import pathlib
+
+import ofscraper.actions.actions.download.batch.downloadbatch as batch
+import ofscraper.actions.actions.download.normal.downloadnormal as normal
+import ofscraper.utils.args.accessors.read as read_args
+import ofscraper.utils.constants as constants
+import ofscraper.utils.hash as hash
+import ofscraper.utils.settings as settings
+import ofscraper.utils.system.system as system
+from ofscraper.actions.utils.log import empty_log
+from ofscraper.actions.actions.download.utils.text import textDownloader
+from ofscraper.utils.context.run_async import run as run_async
+from ofscraper.runner.close.final.final_user import post_user_process
+from ofscraper.commands.utils.strings import (
+    download_activity_str,
+)
+import ofscraper.utils.live.updater as progress_updater
+import ofscraper.utils.config.data as config_data
+import ofscraper.utils.paths.common as common_paths
+from ofscraper.utils.string import format_safe
+
+
+async def downloader(ele=None, posts=None, media=None, **kwargs):
+    model_id = ele.id
+    username = ele.name
+    download_str = download_activity_str.format(username=username)
+    path_str = format_safe(
+        f"\nSaving files to [deep_sky_blue2]{str(pathlib.Path(common_paths.get_save_location(),config_data.get_dirformat(),config_data.get_fileformat()))}[/deep_sky_blue2]",
+        username=username,
+        model_id=model_id,
+        model_username=username,
+        modelusername=username,
+        modelid=model_id,
+    )
+
+    progress_updater.update_activity_task(description=download_str + path_str)
+    logging.getLogger("shared_other").warning(
+        download_activity_str.format(username=username)
+    )
+    progress_updater.update_activity_task(description="")
+    return await download_process(username, model_id, media, posts=posts)
+
+@run_async
+async def download_process(username, model_id, medialist, posts=None):
+    if not read_args.retriveArgs().command == "metadata":
+        await textDownloader(posts, username=username)
+    data = await download_picker(username, model_id, medialist)
+    post_user_process(username, model_id, medialist, posts)
+    return data
+
+
+async def download_picker(username, model_id, medialist):
+    if len(medialist) == 0:
+        out = empty_log(username)
+        logging.getLogger("shared").error(out)
+        return out
+    elif (
+        system.getcpu_count() > 1
+        and (
+            len(medialist)
+            >= settings.get_threads() * constants.getattr("DOWNLOAD_THREAD_MIN")
+        )
+        and settings.not_solo_thread()
+    ):
+        return batch.process_dicts(username, model_id, medialist)
+    else:
+        return await normal.process_dicts(username, model_id, medialist)
+
+
+def remove_downloads_with_hashes(username, model_id):
+    hash.remove_dupes_hash(username, model_id, "audios")
+    hash.remove_dupes_hash(username, model_id, "images")
+    hash.remove_dupes_hash(username, model_id, "videos")