ofscraper 3.9.7__tar.gz → 3.10.dev1__tar.gz

{ofscraper-3.9.7 → ofscraper-3.10.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ofscraper
-Version: 3.9.7
+Version: 3.10.dev1
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net
@@ -72,12 +72,9 @@ Docker and binary releases also availible
 </a>
 
 ## Dev
+None
+
 
-<div style="display: inline-block">
-<a href="https://pypi.org/project/ofscraper/3.9.0.dev5/">
-<img src="https://img.shields.io/badge/Pre--Release-v3.90.dev-dark_green.svg" alt="drawing" style="height:75px"/>
-</a>
-</div>
 
 ## Table-of-contents
 

{ofscraper-3.9.7 → ofscraper-3.10.dev1}/README.md

@@ -18,12 +18,9 @@ Docker and binary releases also availible
 </a>
 
 ## Dev
+None
+
 
-<div style="display: inline-block">
-<a href="https://pypi.org/project/ofscraper/3.9.0.dev5/">
-<img src="https://img.shields.io/badge/Pre--Release-v3.90.dev-dark_green.svg" alt="drawing" style="height:75px"/>
-</a>
-</div>
 
 ## Table-of-contents
 

{ofscraper-3.9.7 → ofscraper-3.10.dev1}/ofscraper/api/archive.py

@@ -22,9 +22,12 @@ import ofscraper.api.common.logs as common_logs
 import ofscraper.utils.args.read as read_args
 import ofscraper.utils.cache as cache
 import ofscraper.utils.constants as constants
-import ofscraper.utils.progress as progress_utils
+import ofscraper.utils.live.screens as progress_utils
 import ofscraper.utils.settings as settings
-from ofscraper.db.operations_.media import get_media_ids_downloaded_model,get_archived_media
+from ofscraper.db.operations_.media import (
+    get_archived_media,
+    get_media_ids_downloaded_model,
+)
 from ofscraper.db.operations_.posts import (
     get_archived_post_info,
     get_youngest_archived_date,
@@ -39,7 +42,7 @@ sem = None
 
 
 @run
-async def get_archived_posts_progress(model_id, username, forced_after=None, c=None):
+async def get_archived_posts(model_id, username, forced_after=None, c=None):
 
     oldarchived = None
     if not settings.get_api_cache_disabled():
@@ -55,45 +58,26 @@ async def get_archived_posts_progress(model_id, username, forced_after=None, c=N
     splitArrays = get_split_array(oldarchived, after)
     tasks = get_tasks(splitArrays, c, model_id, after)
     data = await process_tasks(tasks, model_id, after)
-    progress_utils.archived_layout.visible = False
     return data
 
 
-@run
-async def get_archived_posts(model_id, username, forced_after=None, c=None):
-    if not settings.get_api_cache_disabled():
-        oldarchived = await get_archived_post_info(model_id=model_id, username=username)
-    else:
-        oldarchived = []
-    trace_log_old(oldarchived)
-
-    log.debug(f"[bold]Archived Cache[/bold] {len(oldarchived)} found")
-    oldarchived = list(filter(lambda x: x is not None, oldarchived))
-    after = await get_after(model_id, username, forced_after)
-    time_log(username, after)
-    with progress_utils.set_up_api_archived():
-        splitArrays = get_split_array(oldarchived, after)
-        tasks = get_tasks(splitArrays, c, model_id, after)
-        return await process_tasks(tasks, model_id, after)
-
-
 async def process_tasks(tasks, model_id, after):
     responseArray = []
     page_count = 0
-    overall_progress = progress_utils.overall_progress
 
-    page_task = overall_progress.add_task(
+    page_task = progress_utils.add_api_task(
         f"Archived Content Pages Progress: {page_count}", visible=True
     )
     seen = set()
     while tasks:
         new_tasks = []
+
         for task in asyncio.as_completed(tasks):
             try:
                 result, new_tasks_batch = await task
                 new_tasks.extend(new_tasks_batch)
                 page_count = page_count + 1
-                overall_progress.update(
+                progress_utils.update_api_task(
                     page_task,
                     description=f"Archived Content Pages Progress: {page_count}",
                 )
@@ -123,9 +107,9 @@ async def process_tasks(tasks, model_id, after):
                 log.traceback_(E)
                 log.traceback_(traceback.format_exc())
                 continue
-            tasks = new_tasks
+        tasks = new_tasks
 
-    overall_progress.remove_task(page_task)
+    progress_utils.remove_api_task(page_task)
 
     log.debug(
         f"{common_logs.FINAL_IDS.format('Archived')} {list(map(lambda x:x['id'],responseArray))}"
@@ -158,15 +142,13 @@ def get_split_array(oldarchived, after):
 
 def get_tasks(splitArrays, c, model_id, after):
     tasks = []
-    job_progress = progress_utils.archived_progress
-    before=arrow.get(read_args.retriveArgs().before or arrow.now()).float_timestamp
+    before = arrow.get(read_args.retriveArgs().before or arrow.now()).float_timestamp
     if len(splitArrays) > 2:
         tasks.append(
             asyncio.create_task(
                 scrape_archived_posts(
                     c,
                     model_id,
-                    job_progress=job_progress,
                     required_ids=set([ele.get("created_at") for ele in splitArrays[0]]),
                     timestamp=splitArrays[0][0].get("created_at"),
                     offset=True,
@@ -179,7 +161,6 @@ def get_tasks(splitArrays, c, model_id, after):
                     scrape_archived_posts(
                         c,
                         model_id,
-                        job_progress=job_progress,
                         required_ids=set(
                             [ele.get("created_at") for ele in splitArrays[i]]
                         ),
@@ -196,11 +177,9 @@ def get_tasks(splitArrays, c, model_id, after):
                 scrape_archived_posts(
                     c,
                     model_id,
-                    job_progress=job_progress,
                     timestamp=splitArrays[-1][0].get("created_at"),
                     offset=True,
-                    required_ids=set([before])
-
+                    required_ids=set([before]),
                 )
             )
         )
@@ -211,11 +190,9 @@ def get_tasks(splitArrays, c, model_id, after):
                 scrape_archived_posts(
                     c,
                     model_id,
-                    job_progress=job_progress,
                     timestamp=splitArrays[0][0].get("created_at"),
                     offset=True,
-                    required_ids=set([before])
-
+                    required_ids=set([before]),
                 )
             )
         )
@@ -224,8 +201,11 @@ def get_tasks(splitArrays, c, model_id, after):
         tasks.append(
             asyncio.create_task(
                 scrape_archived_posts(
-                    c, model_id, job_progress=job_progress, timestamp=after, offset=True,
-                    required_ids=set([before])
+                    c,
+                    model_id,
+                    timestamp=after,
+                    offset=True,
+                    required_ids=set([before]),
                 )
             )
         )
@@ -254,9 +234,7 @@ async def get_after(model_id, username, forced_after=None):
         return forced_after
     elif not settings.get_after_enabled():
         return 0
-    elif read_args.retriveArgs().after == 0:
-        return 0
-    elif read_args.retriveArgs().after:
+    elif read_args.retriveArgs().after != None:
         return read_args.retriveArgs().after.float_timestamp
 
     elif cache.get(f"{model_id}_full_archived_scrape"):
@@ -265,12 +243,21 @@ async def get_after(model_id, username, forced_after=None):
         )
         return 0
 
-    curr=await get_archived_media(model_id=model_id, username=username)
+    curr = await get_archived_media(model_id=model_id, username=username)
     if len(curr) == 0:
         log.debug("Setting date to zero because database is empty")
         return 0
-    curr_downloaded = await get_media_ids_downloaded_model(model_id=model_id, username=username)
-    missing_items = list(filter(lambda x: x.get("downloaded") != 1 and x.get("post_id") not in curr_downloaded and x.get("unlocked") != 0, curr))
+    curr_downloaded = await get_media_ids_downloaded_model(
+        model_id=model_id, username=username
+    )
+    missing_items = list(
+        filter(
+            lambda x: x.get("downloaded") != 1
+            and x.get("post_id") not in curr_downloaded
+            and x.get("unlocked") != 0,
+            curr,
+        )
+    )
     missing_items = list(sorted(missing_items, key=lambda x: x.get("posted_at") or 0))
     if len(missing_items) == 0:
         log.debug(
@@ -287,14 +274,14 @@ async def get_after(model_id, username, forced_after=None):
 
 
 async def scrape_archived_posts(
-    c, model_id, job_progress=None, timestamp=None, required_ids=None, offset=False
+    c, model_id, timestamp=None, required_ids=None, offset=False
 ) -> list:
     global sem
     posts = None
     if timestamp and (
         float(timestamp) > (read_args.retriveArgs().before).float_timestamp
     ):
-        return [],[]
+        return [], []
     timestamp = float(timestamp) - 1000 if timestamp and offset else timestamp
     url = (
         constants.getattr("archivedNextEP").format(model_id, str(timestamp))
@@ -302,25 +289,22 @@ async def scrape_archived_posts(
         else constants.getattr("archivedEP").format(model_id)
     )
     log.debug(url)
-    
+
     new_tasks = []
-    posts=[]
+    posts = []
     try:
-        task = (
-            job_progress.add_task(
-                f"Timestamp -> {arrow.get(math.trunc(float(timestamp))).format(constants.getattr('API_DATE_FORMAT')) if timestamp is not None  else 'initial'}",
-                visible=True,
-            )
-            if job_progress
-            else None
+        task = progress_utils.add_api_job_task(
+            f"[Archived] Timestamp -> {arrow.get(math.trunc(float(timestamp))).format(constants.getattr('API_DATE_FORMAT')) if timestamp is not None  else 'initial'}",
+            visible=True,
         )
         async with c.requests_async(url) as r:
+
             posts = (await r.json_())["list"]
             log_id = f"timestamp:{arrow.get(math.trunc(float(timestamp))).format(constants.getattr('API_DATE_FORMAT')) if timestamp is not None  else 'initial'}"
             if not bool(posts):
                 log.debug(f"{log_id} -> no posts found")
                 return [], []
-            
+
             log.debug(f"{log_id} -> number of archived post found {len(posts)}")
             log.debug(
                 f"{log_id} -> first date {posts[0].get('createdAt') or posts[0].get('postedAt')}"
@@ -350,9 +334,9 @@ async def scrape_archived_posts(
             elif float(timestamp or 0) >= max(required_ids):
                 pass
             else:
-                log.debug(f"{log_id} Required before {required_ids}")
+                log.debug(f"{log_id} Required before change:  {required_ids}")
                 [required_ids.discard(float(ele["postedAtPrecise"])) for ele in posts]
-                log.debug(f"{log_id} Required after {required_ids}")
+                log.debug(f"{log_id} Required after change: {required_ids}")
 
                 if len(required_ids) > 0:
                     new_tasks.append(
@@ -360,7 +344,6 @@ async def scrape_archived_posts(
                             scrape_archived_posts(
                                 c,
                                 model_id,
-                                job_progress=job_progress,
                                 timestamp=posts[-1]["postedAtPrecise"],
                                 required_ids=required_ids,
                                 offset=False,
@@ -374,9 +357,9 @@ async def scrape_archived_posts(
         log.traceback_(traceback.format_exc())
         raise E
     finally:
-        job_progress.remove_task(task) if job_progress and task else None
-    return posts, new_tasks
+        progress_utils.remove_api_job_task(task)
 
+    return posts, new_tasks
 
 
 def trace_log_task(responseArray):

{ofscraper-3.9.7 → ofscraper-3.10.dev1}/ofscraper/api/highlights.py

@@ -18,7 +18,7 @@ import traceback
 import ofscraper.api.common.logs as common_logs
 import ofscraper.classes.sessionmanager as sessionManager
 import ofscraper.utils.constants as constants
-import ofscraper.utils.progress as progress_utils
+import ofscraper.utils.live.screens as progress_utils
 from ofscraper.utils.context.run_async import run
 
 log = logging.getLogger("shared")
@@ -29,51 +29,37 @@ log = logging.getLogger("shared")
 ####
 ##############################################################################
 @run
-async def get_stories_post_progress(model_id, c=None):
+async def get_stories_post(model_id, c=None):
     tasks = []
-    job_progress = progress_utils.stories_progress
 
-    tasks.append(
-        asyncio.create_task(scrape_stories(c, model_id, job_progress=job_progress))
-    )
+    tasks.append(asyncio.create_task(scrape_stories(c, model_id)))
 
     data = await process_stories_tasks(tasks)
 
-    progress_utils.stories_layout.visible = False
     return data
 
 
 @run
 async def get_stories_post(model_id, c=None):
     tasks = []
-    with progress_utils.set_up_api_stories():
-        tasks.append(
-            asyncio.create_task(
-                scrape_stories(
-                    c, model_id, job_progress=progress_utils.stories_progress
-                )
-            )
-        )
-        return await process_stories_tasks(tasks)
+    tasks.append(asyncio.create_task(scrape_stories(c, model_id)))
+    return await process_stories_tasks(tasks)
 
 
-async def scrape_stories(c, user_id, job_progress=None) -> list:
+async def scrape_stories(c, user_id) -> list:
     stories = None
     new_tasks = []
     task = None
 
-    await asyncio.sleep(1)
+    # await asyncio.sleep(1)
     url = constants.getattr("highlightsWithAStoryEP").format(user_id)
     try:
-        task = (
-            job_progress.add_task(
-                f"user id -> {user_id}",
-                visible=True,
-            )
-            if job_progress
-            else None
+        task = progress_utils.add_api_job_task(
+            f"[Stories] user id -> {user_id}",
+            visible=True,
         )
         async with c.requests_async(url=url) as r:
+
             stories = await r.json_()
             log.debug(
                 f"stories: -> found stories ids {list(map(lambda x:x.get('id'),stories))}"
@@ -91,13 +77,13 @@ async def scrape_stories(c, user_id, job_progress=None) -> list:
     except asyncio.TimeoutError as _:
         raise Exception(f"Task timed out {url}")
     except Exception as E:
-        await asyncio.sleep(1)
+        # await asyncio.sleep(1)
         log.traceback_(E)
         log.traceback_(traceback.format_exc())
         raise E
 
     finally:
-        (job_progress.remove_task(task) if job_progress and task is not None else None)
+        progress_utils.remove_api_job_task(task)
 
     return stories, new_tasks
 
@@ -105,8 +91,7 @@ async def scrape_stories(c, user_id, job_progress=None) -> list:
 async def process_stories_tasks(tasks):
     responseArray = []
     page_count = 0
-    overall_progress = progress_utils.overall_progress
-    page_task = overall_progress.add_task(
+    page_task = progress_utils.add_api_task(
         f"Stories Pages Progress: {page_count}", visible=True
     )
 
@@ -118,7 +103,7 @@ async def process_stories_tasks(tasks):
                 result, new_tasks_batch = await task
                 new_tasks.extend(new_tasks_batch)
                 page_count = page_count + 1
-                overall_progress.update(
+                progress_utils.update_api_task(
                     page_task,
                     description=f"Stories Content Pages Progress: {page_count}",
                 )
@@ -148,7 +133,7 @@ async def process_stories_tasks(tasks):
                 continue
 
         tasks = new_tasks
-    overall_progress.remove_task(page_task)
+    progress_utils.remove_api_task(page_task)
     log.debug(
         f"{common_logs.FINAL_IDS.format('Stories')} {list(map(lambda x:x['id'],responseArray))}"
     )
@@ -171,79 +156,29 @@ async def process_stories_tasks(tasks):
 
 
 @run
-async def get_highlight_post_progress(model_id, c=None):
-    highlightLists = await get_highlight_list_progress(model_id, c)
-    return await get_highlights_via_list_progress(highlightLists, c)
+async def get_highlight_post(model_id, c=None):
+    highlightLists = await get_highlight_list(model_id, c)
+    return await get_highlights_via_list(highlightLists, c)
 
 
-async def get_highlight_list_progress(model_id, c=None):
+async def get_highlight_list(model_id, c=None):
     tasks = []
-    tasks.append(
-        asyncio.create_task(
-            scrape_highlight_list(
-                c, model_id, job_progress=progress_utils.highlights_progress
-            )
-        )
-    )
+    tasks.append(asyncio.create_task(scrape_highlight_list(c, model_id)))
     return await process_task_get_highlight_list(tasks)
 
 
-async def get_highlights_via_list_progress(highlightLists, c=None):
+async def get_highlights_via_list(highlightLists, c=None):
     tasks = []
-    [
-        tasks.append(
-            asyncio.create_task(
-                scrape_highlights(c, i, job_progress=progress_utils.highlights_progress)
-            )
-        )
-        for i in highlightLists
-    ]
+    [tasks.append(asyncio.create_task(scrape_highlights(c, i))) for i in highlightLists]
     return await process_task_highlights(tasks)
 
 
-@run
-async def get_highlight_post(model_id, c=None):
-    highlightList = await get_highlight_list(model_id, c)
-    return await get_highlights_via_list(highlightList, c)
-
-
-async def get_highlight_list(model_id, c=None):
-    with progress_utils.set_up_api_highlights_lists():
-        tasks = []
-        tasks.append(
-            asyncio.create_task(
-                scrape_highlight_list(
-                    c, model_id, job_progress=progress_utils.highlights_progress
-                )
-            )
-        )
-        return await process_task_get_highlight_list(tasks)
-
-
-async def get_highlights_via_list(highlightLists, c):
-    tasks = []
-    with progress_utils.set_up_api_highlights():
-
-        [
-            tasks.append(
-                asyncio.create_task(
-                    scrape_highlights(
-                        c, i, job_progress=progress_utils.highlights_progress
-                    )
-                )
-            )
-            for i in highlightLists
-        ]
-        return await process_task_highlights(tasks)
-
-
 async def process_task_get_highlight_list(tasks):
     highlightLists = []
 
     page_count = 0
-    overall_progress = progress_utils.overall_progress
 
-    page_task = overall_progress.add_task(
+    page_task = progress_utils.add_api_task(
         f"Highlights List Pages Progress: {page_count}", visible=True
     )
     seen = set()
@@ -254,7 +189,7 @@ async def process_task_get_highlight_list(tasks):
                 result, new_tasks_batch = await task
                 new_tasks.extend(new_tasks_batch)
                 page_count = page_count + 1
-                overall_progress.update(
+                progress_utils.update_api_task(
                     page_task,
                     description=f"Highlights List Pages Progress: {page_count}",
                 )
@@ -272,7 +207,7 @@ async def process_task_get_highlight_list(tasks):
                 continue
         tasks = new_tasks
 
-    overall_progress.remove_task(page_task)
+    progress_utils.remove_api_task(page_task)
     log.trace(
         f"{common_logs.FINAL_IDS.format('Highlight List')} {map(lambda x:x,highlightLists)}"
     )
@@ -286,9 +221,8 @@ async def process_task_get_highlight_list(tasks):
 async def process_task_highlights(tasks):
     highlightResponse = []
     page_count = 0
-    overall_progress = progress_utils.overall_progress
-    page_task = overall_progress.add_task(
-        f"Highlight Content via List Pages Progress: {page_count}", visible=True
+    page_task = progress_utils.add_api_task(
+        f"Highlight Content via list pages progress: {page_count}", visible=True
     )
     seen = set()
     while tasks:
@@ -298,9 +232,9 @@ async def process_task_highlights(tasks):
                 result, new_tasks_batch = await task
                 new_tasks.extend(new_tasks_batch)
                 page_count = page_count + 1
-                overall_progress.update(
+                progress_utils.update_api_task(
                     page_task,
-                    description=f"Highlights Content via list Pages Progress: {page_count}",
+                    description=f"Highlights content via list pages progress: {page_count}",
                 )
                 new_posts = [
                     post
@@ -327,6 +261,7 @@ async def process_task_highlights(tasks):
                 log.traceback_(traceback.format_exc())
                 continue
         tasks = new_tasks
+
         log.debug(
             f"{common_logs.FINAL_IDS.format('Highlight List Posts')} {list(map(lambda x:x['id'],highlightResponse))}"
         )
@@ -340,25 +275,23 @@ async def process_task_highlights(tasks):
         log.debug(
             f"{common_logs.FINAL_COUNT.format('Highlight List Posts')} {len(highlightResponse)}"
         )
+    progress_utils.remove_api_task(page_task)
     return highlightResponse
 
 
-async def scrape_highlight_list(c, user_id, job_progress=None, offset=0) -> list:
+async def scrape_highlight_list(c, user_id, offset=0) -> list:
     new_tasks = []
-    await asyncio.sleep(1)
+    # await asyncio.sleep(1)
     url = constants.getattr("highlightsWithStoriesEP").format(user_id, offset)
     task = None
 
     try:
-        task = (
-            job_progress.add_task(
-                f"scraping highlight list  offset-> {offset}",
-                visible=True,
-            )
-            if job_progress
-            else None
+        task = progress_utils.add_api_job_task(
+            f"[Highlights] scraping highlight list  offset-> {offset}",
+            visible=True,
         )
         async with c.requests_async(url) as r:
+
             resp_data = await r.json_()
             log.trace(f"highlights list: -> found highlights list data {resp_data}")
             data = get_highlightList(resp_data)
@@ -367,33 +300,30 @@ async def scrape_highlight_list(c, user_id, job_progress=None, offset=0) -> list
     except asyncio.TimeoutError:
         raise Exception(f"Task timed out {url}")
     except Exception as E:
-        await asyncio.sleep(1)
+        # await asyncio.sleep(1)
         log.traceback_(E)
         log.traceback_(traceback.format_exc())
         raise E
 
     finally:
-        (job_progress.remove_task(task) if job_progress and task is not None else None)
+        progress_utils.remove_api_job_task(task)
 
     return data, new_tasks
 
 
-async def scrape_highlights(c, id, job_progress=None) -> list:
+async def scrape_highlights(c, id) -> list:
     new_tasks = []
-    await asyncio.sleep(1)
+    # await asyncio.sleep(1)
     url = constants.getattr("storyEP").format(id)
     task = None
 
     try:
-        task = (
-            job_progress.add_task(
-                f"highlights id -> {id}",
-                visible=True,
-            )
-            if job_progress
-            else None
+        task = progress_utils.add_api_job_task(
+            f"[Highlights]  highlights id -> {id}",
+            visible=True,
         )
         async with c.requests_async(url=url) as r:
+
             resp_data = await r.json_()
             log.trace(f"highlights: -> found highlights data {resp_data}")
             log.debug(
@@ -403,13 +333,13 @@ async def scrape_highlights(c, id, job_progress=None) -> list:
         raise Exception(f"Task timed out {url}")
 
     except Exception as E:
-        await asyncio.sleep(1)
+        # await asyncio.sleep(1)
         log.traceback_(E)
         log.traceback_(traceback.format_exc())
         raise E
 
     finally:
-        (job_progress.remove_task(task) if job_progress and task is not None else None)
+        progress_utils.remove_api_job_task(task)
 
     return resp_data["stories"], new_tasks
 
@@ -442,7 +372,6 @@ def get_individual_stories(id, c=None):
         retries=constants.getattr("API_INDVIDIUAL_NUM_TRIES"),
         wait_min=constants.getattr("OF_MIN_WAIT_API"),
         wait_max=constants.getattr("OF_MAX_WAIT_API"),
-        new_request_auth=True,
     ) as c:
         with c.requests_async(constants.getattr("storiesSPECIFIC").format(id)) as r:
             log.trace(f"highlight raw highlight individua; {r.json_()}")

{ofscraper-3.9.7 → ofscraper-3.10.dev1}/ofscraper/api/init.py

@@ -14,24 +14,19 @@ r"""
 import logging
 import traceback
 
-from rich.console import Console
-
-import ofscraper.utils.context.stdout as stdout
+import ofscraper.utils.console as console
 
 from . import me
 
 log = logging.getLogger("shared")
 
-console = Console()
-
 
 def print_sign_status():
-    with stdout.lowstdout():
-        status = getstatus()
-        if status == "UP":
-            print("Status - \033[32mUP\033[0m")
-        else:
-            print("Status - \033[31mDOWN\033[0m")
+    status = getstatus()
+    if status == "UP":
+        console.get_shared_console().print("Status - [bold green]UP[/bold green]")
+    else:
+        console.get_shared_console().print("Status - [bold red]DOWN[/bold red]")
 
 
 def getstatus():