ofscraper 3.10.dev7__tar.gz → 3.10.dev8__tar.gz

{ofscraper-3.10.dev7 → ofscraper-3.10.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ofscraper
-Version: 3.10.dev7
+Version: 3.10.dev8
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net

{ofscraper-3.10.dev7 → ofscraper-3.10.dev8}/ofscraper/const/values/config.py

@@ -57,5 +57,5 @@ DEFAULT_LOG_LEVEL = "DEBUG"
 INCLUDE_LABELS_ALL = False
 DISCORD_THREAD_OVERRIDE = False
 DISCORD_ASYNC = False
-USE_CACHE_KEY = True
+USE_WIV_CACHE_KEY = True
 POST_SCRIPT_DEFAULT = None

{ofscraper-3.10.dev7 → ofscraper-3.10.dev8}/ofscraper/const/values/other_url.py

@@ -8,6 +8,6 @@ FFMPEG_WINDOWS = "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest
 FFMPEG_MAC = "https://evermeet.cx/ffmpeg/#api-download"
 CDRM2 = "http://172.106.17.134:8080/wv"
 KEYDB = "https://keysdb.net/api"
-CDRM = "https://cdrm-project.com/wv"
+CDRM = "https://old.cdrm-project.com/"
 LARGEZIP = "https://proof.ovh.net/files/100Mb.dat"
 BAD_URL_HOST = {"us.upload.onlyfans.com", "of2transcoder.s3-accelerate.amazonaws.com"}

{ofscraper-3.10.dev7 → ofscraper-3.10.dev8}/ofscraper/download/alt_download.py

@@ -140,23 +140,29 @@ async def alt_download_downloader(item, c, ele):
 
 
 async def resume_data_handler(data, item, c, ele, placeholderObj):
-    item["total"] = (
+    common_globals.log.debug(f"{get_medialog(ele)} Resume cached data {data}")
+    total=(
         int(data.get("content-total")) if data.get("content-total") else None
     )
+    item["total"]=total
+
     resume_size = get_resume_size(placeholderObj, mediatype=ele.mediatype)
-    if await check_forced_skip(ele, item["total"]) == 0:
+    common_globals.log.debug(f"{get_medialog(ele)} resume_size: {resume_size}  and total: {total }")
+
+    if await check_forced_skip(ele, total) == 0:
         item["total"] = 0
         return item
-    elif item["total"] == resume_size:
+    elif total == resume_size:
+        common_globals.log.debug(f"{get_medialog(ele)} total==resume_size skipping download")
+
         temp_file_logger(placeholderObj, ele)
-        total = item["total"]
         (
             await common.total_change_helper(None, total)
             if common.alt_attempt_get(item).get() == 1
             else None
         )
         return item
-    elif item["total"] != resume_size:
+    elif total!= resume_size:
         return await alt_download_sendreq(item, c, ele, placeholderObj)
 
 
@@ -189,6 +195,8 @@ async def alt_download_sendreq(item, c, ele, placeholderObj):
 
 async def send_req_inner(c, ele, item, placeholderObj):
     try:
+        common_globals.log.debug(f"{get_medialog(ele)} writing item to disk")
+
         resume_size = get_resume_size(placeholderObj, mediatype=ele.mediatype)
         headers = None if not resume_size else {"Range": f"bytes={resume_size}-"}
         params = {

{ofscraper-3.10.dev7 → ofscraper-3.10.dev8}/ofscraper/download/alt_downloadbatch.py

@@ -57,8 +57,17 @@ async def alt_download(c, ele, username, model_id):
     )
     async for _ in download_retry():
         with _:
-            sharedPlaceholderObj = await placeholder.Placeholders(ele, "mp4").init()
-            audio, video = await ele.mpd_dict
+            try:
+                sharedPlaceholderObj = await placeholder.Placeholders(ele, "mp4").init()
+                audio, video = await ele.mpd_dict
+            except Exception as e:
+                common_globals.log.handlers[1].queue.put(
+                list(common_globals.innerlog.get().handlers[1].queue.queue)
+                )
+                common_globals.log.handlers[0].queue.put(
+                list(common_globals.innerlog.get().handlers[0].queue.queue)
+                )
+                raise e
     path_to_file_logger(sharedPlaceholderObj, ele, common_globals.innerlog.get())
     audio = await alt_download_downloader(audio, c, ele)
     video = await alt_download_downloader(video, c, ele)
@@ -121,24 +130,37 @@ async def alt_download_downloader(
                 common_globals.innerlog.get().traceback_(
                     f"{get_medialog(ele)} [attempt {_attempt.get()}/{constants.getattr('DOWNLOAD_FILE_NUM_TRIES')}] {E}"
                 )
+                common_globals.log.handlers[1].queue.put(
+                list(common_globals.innerlog.get().handlers[1].queue.queue)
+                )
+                common_globals.log.handlers[0].queue.put(
+                list(common_globals.innerlog.get().handlers[0].queue.queue)
+                )
                 raise E
 
 
 async def resume_data_handler(data, item, c, ele, placeholderObj):
-    item["total"] = (
+    common_globals.log.debug(f"{get_medialog(ele)} Resume cached data {data}")
+    total=(
         int(data.get("content-total")) if data.get("content-total") else None
     )
+    item["total"] = total
+
     resume_size = get_resume_size(placeholderObj, mediatype=ele.mediatype)
-    if await check_forced_skip(ele, item["total"]) == 0:
+    common_globals.log.debug(f"{get_medialog(ele)} resume_size: {resume_size}  and total: {total }")
+    if await check_forced_skip(ele,total) == 0:
         item["total"] = 0
         return item
-    elif item["total"] == resume_size:
-        total = item["total"]
+    elif total == resume_size:
+        common_globals.log.debug(f"{get_medialog(ele)} total==resume_size skipping download")
+        
         (
+
             await common.batch_total_change_helper(None, total)
             if common.alt_attempt_get(item).get() == 1
             else None
         )
+        return item
     elif item["total"] != resume_size:
         return await alt_download_sendreq(item, c, ele, placeholderObj)
 
@@ -181,6 +203,8 @@ async def alt_download_sendreq(item, c, ele, placeholderObj):
 
 async def send_req_inner(c, ele, item, placeholderObj):
     try:
+        common_globals.log.debug(f"{get_medialog(ele)} writing item to disk")
+        
         resume_size = get_resume_size(placeholderObj, mediatype=ele.mediatype)
         headers = None if not resume_size else {"Range": f"bytes={resume_size}-"}
         params = {

{ofscraper-3.10.dev7 → ofscraper-3.10.dev8}/ofscraper/download/main_download.py

@@ -124,15 +124,19 @@ async def fresh_data_handler(c, tempholderObj, ele):
 
 
 async def resume_data_handler(data, c, tempholderObj, ele):
+    common_globals.log.debug(f"{get_medialog(ele)} Resume cached data {data}")
     content_type = data.get("content-type").split("/")[-1]
     total = int(data.get("content-total")) if data.get("content-total") else None
     placeholderObj = await placeholder.Placeholders(ele, content_type).init()
     resume_size = get_resume_size(tempholderObj, mediatype=ele.mediatype)
+    common_globals.log.debug(f"{get_medialog(ele)} resume_size: {resume_size}  and total: {total}")
+
     # other
     if await check_forced_skip(ele, total) == 0:
         path_to_file_logger(placeholderObj, ele)
         return [0]
     elif total == resume_size:
+        common_globals.log.debug(f"{get_medialog(ele)} total==resume_size skipping download")
         (
             await common.total_change_helper(None, total)
             if common_globals.attempt.get() == 1
@@ -178,6 +182,8 @@ async def main_download_sendreq(c, ele, tempholderObj, total=None, placeholderOb
 
 async def send_req_inner(c, ele, tempholderObj, placeholderObj=None, total=None):
     try:
+        common_globals.log.debug(f"{get_medialog(ele)} writing item to disk")
+
         resume_size = get_resume_size(tempholderObj, mediatype=ele.mediatype)
         headers = None if not resume_size else {"Range": f"bytes={resume_size}-"}
         common_globals.log.debug(
@@ -227,7 +233,6 @@ async def download_fileobject_writer(r, ele, tempholderObj, placeholderObj, tota
         total=total,
     )
     try:
-        loop = asyncio.get_event_loop()
         fileobject = await aiofiles.open(tempholderObj.tempfilepath, "ab").__aenter__()
         download_sleep = constants.getattr("DOWNLOAD_SLEEP")
         chunk_size = get_ideal_chunk_size(total, tempholderObj.tempfilepath)

{ofscraper-3.10.dev7 → ofscraper-3.10.dev8}/ofscraper/download/main_downloadbatch.py

@@ -28,7 +28,6 @@ import ofscraper.download.shared.globals.globals as common_globals
 import ofscraper.utils.cache as cache
 import ofscraper.utils.constants as constants
 import ofscraper.utils.live.screens as progress_utils
-import ofscraper.utils.settings as settings
 import ofscraper.utils.system.system as system
 from ofscraper.download.shared.classes.retries import download_retry
 from ofscraper.download.shared.general import (
@@ -113,6 +112,12 @@ async def main_download_downloader(c, ele):
                 common_globals.innerlog.get().traceback_(
                     f"{get_medialog(ele)} [attempt {common_globals.attempt.get()}/{constants.getattr('DOWNLOAD_FILE_NUM_TRIES')}] {E}"
                 )
+                common_globals.log.handlers[1].queue.put(
+                list(common_globals.innerlog.get().handlers[1].queue.queue)
+                )
+                common_globals.log.handlers[0].queue.put(
+                list(common_globals.innerlog.get().handlers[0].queue.queue)
+                )
                 raise E
 
 
@@ -128,10 +133,13 @@ async def fresh_data_handler(c, ele, tempholderObj):
 
 
 async def resume_data_handler(data, c, ele, tempholderObj):
+    common_globals.log.debug(f"{get_medialog(ele)} Resume cached data {data}")
     content_type = data.get("content-type").split("/")[-1]
     total = int(data.get("content-total")) if data.get("content-total") else None
     placeholderObj = await placeholder.Placeholders(ele, content_type).init()
     resume_size = get_resume_size(tempholderObj, mediatype=ele.mediatype)
+    common_globals.log.debug(f"{get_medialog(ele)} resume_size: {resume_size}  and total: {total}")
+
     # other
     if await check_forced_skip(ele, total) == 0:
         path_to_file_logger(placeholderObj, ele, common_globals.innerlog.get())
@@ -141,6 +149,7 @@ async def resume_data_handler(data, c, ele, tempholderObj):
             placeholderObj,
         )
     elif total == resume_size:
+        common_globals.log.debug(f"{get_medialog(ele)} total==resume_size skipping download")
         path_to_file_logger(placeholderObj, ele, common_globals.innerlog.get())
         (
             await common.batch_total_change_helper(None, total)
@@ -178,6 +187,8 @@ async def main_download_sendreq(c, ele, tempholderObj, placeholderObj=None, tota
 
 async def send_req_inner(c, ele, tempholderObj, placeholderObj=None, total=None):
     try:
+        common_globals.log.debug(f"{get_medialog(ele)} writing item to disk")
+
         resume_size = get_resume_size(tempholderObj, mediatype=ele.mediatype)
         headers = None if not resume_size else {"Range": f"bytes={resume_size}-"}
         common_globals.log.debug(

{ofscraper-3.10.dev7 → ofscraper-3.10.dev8}/ofscraper/download/shared/keyhelpers.py

@@ -37,7 +37,7 @@ async def un_encrypt(item, c, ele, input_=None):
         await asyncio.get_event_loop().run_in_executor(
             common_globals.thread, partial(cache.get, ele.license)
         )
-        if constants.getattr("USE_CACHE_KEY")
+        if constants.getattr("USE_WIV_CACHE_KEY")
         else None
     )
     if past_key:

{ofscraper-3.10.dev7 → ofscraper-3.10.dev8}/ofscraper/download/shared/log.py

@@ -63,27 +63,27 @@ def final_log(username, log=None):
 
 def final_log_text(username):
     total_count=common_globals.audio_count+common_globals.photo_count+common_globals.video_count
-    size_log=f"[green]{format_size(common_globals.total_bytes )}[/green]"
+    size_log=f"[bold green]{format_size(common_globals.total_bytes )}[/bold green]"
 
     
-    photo_log=f"[green]{common_globals.photo_count} photos[/green]"if common_globals.photo_count>0 else f"{common_globals.photo_count} photos"
+    photo_log=f"[bold green]{common_globals.photo_count} photos[/bold green]"if common_globals.photo_count>0 else f"{common_globals.photo_count} photos"
 
-    audio_log=f"[green]{common_globals.audio_count} audios[/green]"if common_globals.audio_count>0 else f"{common_globals.audio_count} audios"
+    audio_log=f"[bold green]{common_globals.audio_count} audios[/bold green]"if common_globals.audio_count>0 else f"{common_globals.audio_count} audios"
 
 
-    video_log=f"[green]{common_globals.video_count} videos[/green]"if common_globals.video_count>0 else f"{common_globals.video_count} videos"
+    video_log=f"[bold green]{common_globals.video_count} videos[/bold green]"if common_globals.video_count>0 else f"{common_globals.video_count} videos"
 
 
-    failed_log=f"[red]{common_globals.video_count} failed[/red]"if common_globals.skipped>0 else f"{common_globals.skipped} failed"
+    failed_log=f"[bold red]{common_globals.skipped} failed[/bold red]"if common_globals.skipped>0 else f"{common_globals.skipped} failed"
 
     log_format=None
     skipped_log=""
     if read_args.retriveArgs().metadata:
-        log_format="[blue][bold]\\[{username}][/bold] [bold]\\[Action Metadata][/bold] ({size_log}) ([green]{total_count} changed media item total[/green]\\[{video_log}, {audio_log}, {photo_log}], {skipped_log}, {failed_log}))[/blue]"
-        skipped_log=f"[yellow]{common_globals.skipped} metadata unchanged[/yellow]"if common_globals.skipped>0 else f"{common_globals.forced_skipped} items unchanged"
+        log_format="[blue][bold]\\[{username}][/bold] [bold]\\[Action Metadata][/bold] ({size_log}) ([bold green]{total_count} changed media item total [/bold green]\\[{video_log}, {audio_log}, {photo_log}], {skipped_log}, {failed_log}))[/blue]"
+        skipped_log=f"[bold yellow]{common_globals.forced_skipped} metadata unchanged[/bold yellow]"if common_globals.forced_skipped>0 else f"{common_globals.forced_skipped} items unchanged"
     else:
-        log_format="[blue][bold]\\[{username}][/bold][bold]\\[Action Download][/bold] ({size_log}) ([green]{total_count} downloads total[/green]\\[{video_log}, {audio_log}, {photo_log}], {skipped_log}, {failed_log}))[/blue]"
-        skipped_log=f"[yellow]{common_globals.video_count} skipped[/yellow]"if common_globals.skipped>0 else f"{common_globals.forced_skipped} skipped"
+        log_format="[blue][bold]\\[{username}][/bold] [bold]\\[Action Download][/bold] ({size_log}) ([bold green]{total_count} downloads total [/bold green]\\[{video_log}, {audio_log}, {photo_log}], {skipped_log}, {failed_log}))[/blue]"
+        skipped_log=f"[bold yellow]{common_globals.forced_skipped} skipped[/bold yellow]"if common_globals.forced_skipped>0 else f"{common_globals.forced_skipped} skipped"
 
     return log_format.format(username=username,total_count=total_count,video_log=video_log,audio_log=audio_log,skipped_log=skipped_log,failed_log=failed_log,photo_log=photo_log,size_log=size_log)
 

{ofscraper-3.10.dev7 → ofscraper-3.10.dev8}/ofscraper/utils/logs/classes.py

@@ -267,12 +267,13 @@ class DiscordFormatter(SensitiveFormatter):
     @staticmethod
     def _filter(s):
         t = SensitiveFormatter._filter(s)
-        t=re.sub("\\\\+","",t)
-        t=re.sub(r"\[(bold|/?bold(?:\s\w+\s\w+)?)\]","**",t)
+        t=re.sub("\\\\+","\\\\",t)
+        t=re.sub("\[bold[^\]]*]","**",t)
+        t=re.sub("\[\/bold[^\]]*]","**",t)
         t=Text.from_markup(Text(t).plain).plain
+        t=re.sub("  +"," ",t)
         t=re.sub("\*\*+","**",t)
-        return t
-
+        t=re.sub("\\\\+","",t)
 
 
 class LogFileFormatter(SensitiveFormatter):

{ofscraper-3.10.dev7 → ofscraper-3.10.dev8}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ofscraper"
-version = "3.10.dev7"
+version = "3.10.dev8"
 description = "automatically scrape onlyfans"
 authors = ["datawhores <datawhores@riseup.net>"]
 readme = "README.md"