ofscraper 3.9.0.dev5__tar.gz → 3.9.0.dev7__tar.gz

{ofscraper-3.9.0.dev5 → ofscraper-3.9.0.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ofscraper
-Version: 3.9.0.dev5
+Version: 3.9.0.dev7
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net

{ofscraper-3.9.0.dev5 → ofscraper-3.9.0.dev7}/ofscraper/actions/process.py

@@ -107,7 +107,7 @@ async def process_user_first_helper(ele):
         model_id = ele.id
         username = ele.name
         operations.table_init_create(model_id=model_id, username=username)
-        results, posts = OF.process_areas(ele, model_id, progress=False)
+        results, posts = OF.process_areas(ele, model_id, job_progress=False)
         return results
     except Exception as e:
         if isinstance(e, KeyboardInterrupt):

{ofscraper-3.9.0.dev5 → ofscraper-3.9.0.dev7}/ofscraper/actions/scraper.py

@@ -54,12 +54,9 @@ async def process_messages(model_id, username, c):
             messages_ = list(
                 map(lambda x: posts_.Post(x, model_id, username), messages_)
             )
-            curr = set(
-                operations.get_all_messages_ids(model_id=model_id, username=username)
-            )
             [
-                operations.write_messages_table(
-                    list(filter(lambda x: x.id not in curr, messages_)),
+                operations.make_messages_table_changes(
+                    messages_,
                     model_id=model_id,
                     username=username,
                 )
@@ -75,7 +72,6 @@ async def process_messages(model_id, username, c):
 
             await operations.batch_mediainsert(
                 output,
-                operations.write_media_table_batch,
                 model_id=model_id,
                 username=username,
                 downloaded=False,
@@ -107,11 +103,8 @@ async def process_paid_post(model_id, username, c):
                     paid_content,
                 )
             )
-            curr = set(
-                operations.get_all_post_ids(model_id=model_id, username=username)
-            )
-            operations.write_post_table(
-                list(filter(lambda x: x.id not in curr, paid_content)),
+            operations.make_post_table_changes(
+                paid_content,
                 model_id=model_id,
                 username=username,
             )
@@ -122,7 +115,6 @@ async def process_paid_post(model_id, username, c):
 
             await operations.batch_mediainsert(
                 output,
-                operations.write_media_table_batch,
                 model_id=model_id,
                 username=username,
                 downloaded=False,
@@ -153,11 +145,8 @@ async def process_stories(model_id, username, c):
                     stories,
                 )
             )
-            curr = set(
-                operations.get_all_stories_ids(model_id=model_id, username=username)
-            )
-            operations.write_stories_table(
-                list(filter(lambda x: x.id not in curr, stories)),
+            operations.make_stories_tables_changes(
+                stories,
                 model_id=model_id,
                 username=username,
             )
@@ -169,7 +158,6 @@ async def process_stories(model_id, username, c):
             [output.extend(stories.media) for stories in stories]
             await operations.batch_mediainsert(
                 output,
-                operations.write_media_table_batch,
                 model_id=model_id,
                 username=username,
                 downloaded=False,
@@ -197,11 +185,8 @@ async def process_highlights(model_id, username, c):
                     highlights_,
                 )
             )
-            curr = set(
-                operations.get_all_stories_ids(model_id=model_id, username=username)
-            )
-            operations.write_stories_table(
-                list(filter(lambda x: x.id not in curr, highlights_)),
+            operations.make_stories_tables_changes(
+                highlights_,
                 model_id=model_id,
                 username=username,
             )
@@ -213,7 +198,6 @@ async def process_highlights(model_id, username, c):
             [output.extend(stories.media) for stories in highlights_]
             await operations.batch_mediainsert(
                 output,
-                operations.write_media_table_batch,
                 model_id=model_id,
                 username=username,
                 downloaded=False,
@@ -244,15 +228,12 @@ async def process_timeline_posts(model_id, username, c):
                     timeline_posts,
                 )
             )
-
-            curr = set(
-                operations.get_all_post_ids(model_id=model_id, username=username)
-            )
-            operations.write_post_table(
-                list(filter(lambda x: x.id not in curr, timeline_posts)),
+            operations.make_post_table_changes(
+                timeline_posts,
                 model_id=model_id,
                 username=username,
             )
+
             log.debug(
                 f"[bold]Timeline media count with locked[/bold] {sum(map(lambda x:len(x.post_media),timeline_posts))}"
             )
@@ -263,14 +244,12 @@ async def process_timeline_posts(model_id, username, c):
 
             await operations.batch_mediainsert(
                 output,
-                operations.write_media_table_batch,
                 model_id=model_id,
                 username=username,
                 downloaded=False,
             )
             await operations.batch_mediainsert(
                 output,
-                operations.update_response_media_table,
                 model_id=model_id,
                 username=username,
                 downloaded=False,
@@ -303,11 +282,8 @@ async def process_archived_posts(model_id, username, c):
                 )
             )
 
-            curr = set(
-                operations.get_all_post_ids(model_id=model_id, username=username)
-            )
-            operations.write_post_table(
-                list(filter(lambda x: x.id not in curr, archived_posts)),
+            operations.make_post_table_changes(
+                archived_posts,
                 model_id=model_id,
                 username=username,
             )
@@ -322,7 +298,6 @@ async def process_archived_posts(model_id, username, c):
 
             await operations.batch_mediainsert(
                 output,
-                operations.write_media_table_batch,
                 model_id=model_id,
                 username=username,
                 downloaded=False,
@@ -353,11 +328,8 @@ async def process_pinned_posts(model_id, username, c):
                     lambda x: posts_.Post(x, model_id, username, "pinned"), pinned_posts
                 )
             )
-            curr = set(
-                operations.get_all_post_ids(model_id=model_id, username=username)
-            )
-            operations.write_post_table(
-                list(filter(lambda x: x.id not in curr, pinned_posts)),
+            operations.make_post_table_changes(
+                pinned_posts,
                 model_id=model_id,
                 username=username,
             )
@@ -372,14 +344,12 @@ async def process_pinned_posts(model_id, username, c):
 
             await operations.batch_mediainsert(
                 output,
-                operations.write_media_table_batch,
                 model_id=model_id,
                 username=username,
                 downloaded=False,
             )
             await operations.batch_mediainsert(
                 output,
-                operations.update_response_media_table,
                 model_id=model_id,
                 username=username,
                 downloaded=False,
@@ -475,11 +445,8 @@ def process_all_paid():
             for ele in all_posts:
                 new_dict[ele.id] = ele
             new_posts = new_dict.values()
-            curr = set(
-                operations.get_all_post_ids(model_id=model_id, username=username) or []
-            )
-            operations.write_post_table(
-                list(filter(lambda x: x.id not in curr, new_posts)),
+            operations.make_post_table_changes(
+                new_posts,
                 model_id=model_id,
                 username=username,
             )
@@ -572,8 +539,7 @@ async def process_areas(ele, model_id) -> list:
             asyncio.get_event_loop().set_default_executor(executor)
             username = ele.name
             output = []
-            group = progress_utils.get_api_progress_Group()
-            with Live(group, console=console_.get_shared_console()):
+            with progress_utils.setup_api_progress_live():
                 medias, posts = await process_task(model_id, username, ele)
                 output.extend(medias)
         return filters.filterMedia(output), filters.filterPost(posts)

{ofscraper-3.9.0.dev5 → ofscraper-3.9.0.dev7}/ofscraper/api/archive.py

@@ -218,13 +218,13 @@ def get_after(model_id, username, forced_after=None):
             "Used --after previously. Scraping all archived posts required to make sure content is not missing"
         )
         return 0
-    curr = operations.get_archived_postinfo(model_id=model_id, username=username)
+    curr = operations.get_archived_media(model_id=model_id, username=username)
 
     if len(curr) == 0:
         log.debug("Setting date to zero because database is empty")
         return 0
-    missing_items = list(filter(lambda x: x[-2] == 0, curr))
-    missing_items = list(sorted(missing_items, key=lambda x: arrow.get(x[-1])))
+    missing_items = list(filter(lambda x: x[10] != 1, curr))
+    missing_items = list(sorted(missing_items, key=lambda x: arrow.get(x[12])))
     if len(missing_items) == 0:
         log.debug("Using last db date because,all downloads in db marked as downloaded")
         return (
@@ -235,7 +235,7 @@ def get_after(model_id, username, forced_after=None):
         log.debug(
             f"Setting date slightly before earliest missing item\nbecause {len(missing_items)} posts in db are marked as undownloaded"
         )
-        return arrow.get(missing_items[0][-1]).float_timestamp - 1000
+        return arrow.get(missing_items[0][12]).float_timestamp - 1000
 
 
 @run
@@ -322,7 +322,7 @@ async def scrape_archived_posts(
                                         scrape_archived_posts(
                                             c,
                                             model_id,
-                                            job_progress=None,
+                                            job_progress=job_progress,
                                             timestamp=posts[-1]["postedAtPrecise"],
                                         )
                                     )
@@ -341,7 +341,7 @@ async def scrape_archived_posts(
                                             scrape_archived_posts(
                                                 c,
                                                 model_id,
-                                                job_progress=None,
+                                                job_progress=job_progress,
                                                 timestamp=posts[-1]["postedAtPrecise"],
                                                 required_ids=required_ids,
                                             )
@@ -361,5 +361,7 @@ async def scrape_archived_posts(
 
             finally:
                 sem.release()
-                job_progress.remove_task(task) if job_progress and task else None
+                job_progress.remove_task(
+                    task
+                ) if job_progress and task != None else None
             return posts, new_tasks

{ofscraper-3.9.0.dev5 → ofscraper-3.9.0.dev7}/ofscraper/api/highlights.py

@@ -213,7 +213,9 @@ async def scrape_stories(c, user_id, job_progress=None) -> list:
 
             finally:
                 sem.release()
-                job_progress.remove_task(task) if job_progress and task else None
+                job_progress.remove_task(
+                    task
+                ) if job_progress and task != None else None
 
             return stories, new_tasks
 
@@ -340,7 +342,7 @@ async def get_highlights_via_list_progress(highlightLists, c=None):
 @run
 async def get_highlight_post(model_id, c=None):
     highlightList = await get_highlight_list(model_id, c)
-    return await get_highlights_via_list(highlightList)
+    return await get_highlights_via_list(highlightList, c)
 
 
 async def get_highlight_list(model_id, c=None):
@@ -382,9 +384,12 @@ async def get_highlight_list(model_id, c=None):
 
 
 async def get_highlights_via_list(highlightLists, c):
+    job_progress = None
     tasks = []
     [
-        tasks.append(asyncio.create_task(scrape_highlights(c, i, job_progress=None)))
+        tasks.append(
+            asyncio.create_task(scrape_highlights(c, i, job_progress=job_progress))
+        )
         for i in highlightLists
     ]
 
@@ -487,7 +492,9 @@ async def scrape_highlight_list(c, user_id, job_progress=None, offset=0) -> list
 
             finally:
                 sem.release()
-                job_progress.remove_task(task) if job_progress and task else None
+                job_progress.remove_task(
+                    task
+                ) if job_progress and task != None else None
 
             return data, new_tasks
 
@@ -540,7 +547,9 @@ async def scrape_highlights(c, id, job_progress=None) -> list:
 
             finally:
                 sem.release()
-                job_progress.remove_task(task) if job_progress and task else None
+                job_progress.remove_task(
+                    task
+                ) if job_progress and task != None else None
 
             return resp_data["stories"], new_tasks
 

{ofscraper-3.9.0.dev5 → ofscraper-3.9.0.dev7}/ofscraper/api/labels.py

@@ -173,7 +173,9 @@ async def scrape_labels(c, model_id, job_progress=None, offset=0):
 
             finally:
                 sem.release()
-                job_progress.remove_task(task) if job_progress and task else None
+                job_progress.remove_task(
+                    task
+                ) if job_progress and task != None else None
 
 
 @run
@@ -344,7 +346,9 @@ async def scrape_labelled_posts(c, label, model_id, job_progress=None, offset=0)
 
             finally:
                 sem.release()
-                job_progress.remove_task(task) if job_progress and task else None
+                job_progress.remove_task(
+                    task
+                ) if job_progress and task != None else None
 
             return label, posts, new_tasks
 

{ofscraper-3.9.0.dev5 → ofscraper-3.9.0.dev7}/ofscraper/api/messages.py

@@ -136,7 +136,7 @@ Setting initial message scan date for {username} to {arrow.get(after).format('YY
     if len(IDArray) <= 2:
         tasks.append(
             asyncio.create_task(
-                scrape_messages(c, model_id, job_progress, message_id=None)
+                scrape_messages(c, model_id, job_progress=job_progress, message_id=None)
             )
         )
 
@@ -156,7 +156,7 @@ Setting initial message scan date for {username} to {arrow.get(after).format('YY
                     scrape_messages(
                         c,
                         model_id,
-                        job_progress,
+                        job_progress=job_progress,
                         message_id=None,
                         required_ids=set(splitArraysTime[0]),
                     )
@@ -168,7 +168,7 @@ Setting initial message scan date for {username} to {arrow.get(after).format('YY
                     scrape_messages(
                         c,
                         model_id,
-                        job_progress,
+                        job_progress=job_progress,
                         message_id=splitArraysID[0][0],
                         required_ids=set(splitArraysTime[0]),
                     )
@@ -181,7 +181,7 @@ Setting initial message scan date for {username} to {arrow.get(after).format('YY
                         scrape_messages(
                             c,
                             model_id,
-                            job_progress,
+                            job_progress=job_progress,
                             required_ids=set(splitArraysTime[i]),
                             message_id=splitArraysID[i - 1][-1],
                         )
@@ -195,7 +195,7 @@ Setting initial message scan date for {username} to {arrow.get(after).format('YY
                     scrape_messages(
                         c,
                         model_id,
-                        job_progress,
+                        job_progress=job_progress,
                         message_id=splitArraysID[-2][-1],
                     )
                 )
@@ -206,7 +206,7 @@ Setting initial message scan date for {username} to {arrow.get(after).format('YY
                     scrape_messages(
                         c,
                         model_id,
-                        job_progress,
+                        job_progress=job_progress,
                         message_id=splitArraysID[-1][-1],
                     )
                 )
@@ -217,7 +217,7 @@ Setting initial message scan date for {username} to {arrow.get(after).format('YY
                 scrape_messages(
                     c,
                     model_id,
-                    job_progress,
+                    job_progress=job_progress,
                     message_id=IDArray[0],
                     required_ids=set(postedAtArray[1:]),
                 )
@@ -282,7 +282,7 @@ async def get_messages(model_id, username, forced_after=None, c=None):
     global sem
     sem = sems.get_req_sem()
     global after
-
+    job_progress = None
     tasks = []
     responseArray = []
     # require a min num of posts to be returned
@@ -371,7 +371,7 @@ Setting initial message scan date for {username} to {arrow.get(after).format('YY
     if len(IDArray) <= 2:
         tasks.append(
             asyncio.create_task(
-                scrape_messages(c, model_id, job_progress=None, message_id=None)
+                scrape_messages(c, model_id, job_progress=job_progress, message_id=None)
             )
         )
 
@@ -512,7 +512,7 @@ def set_check(unduped, model_id, after):
 
 
 async def scrape_messages(
-    c, model_id, progress=None, message_id=None, required_ids=None
+    c, model_id, job_progress=None, message_id=None, required_ids=None
 ) -> list:
     global sem
     global tasks
@@ -543,10 +543,10 @@ async def scrape_messages(
                     attempt.set(attempt.get(0) + 1)
 
                     task = (
-                        progress.add_task(
+                        job_progress.add_task(
                             f"Attempt {attempt.get()}/{constants.getattr('NUM_TRIES')}: Message ID-> {message_id if message_id else 'initial'}"
                         )
-                        if progress
+                        if job_progress
                         else None
                     )
                     if r.ok:
@@ -596,7 +596,7 @@ async def scrape_messages(
                                         scrape_messages(
                                             c,
                                             model_id,
-                                            progress,
+                                            job_progress=job_progress,
                                             message_id=messages[-1]["id"],
                                         )
                                     )
@@ -618,7 +618,7 @@ async def scrape_messages(
                                             scrape_messages(
                                                 c,
                                                 model_id,
-                                                progress,
+                                                job_progress=job_progress,
                                                 message_id=messages[-1]["id"],
                                                 required_ids=required_ids,
                                             )
@@ -640,7 +640,9 @@ async def scrape_messages(
                 raise E
             finally:
                 sem.release()
-                progress.remove_task(task) if progress and task else None
+                job_progress.remove_task(
+                    task
+                ) if job_progress and task != None else None
             return messages, new_tasks
 
 
@@ -682,8 +684,8 @@ def get_after(model_id, username, forced_after=None):
     if len(curr) == 0:
         log.debug("Setting date to zero because database is empty")
         return 0
-    missing_items = list(filter(lambda x: x[-2] == 0, curr))
-    missing_items = list(sorted(missing_items, key=lambda x: arrow.get(x[-1])))
+    missing_items = list(filter(lambda x: x[10] != 1, curr))
+    missing_items = list(sorted(missing_items, key=lambda x: arrow.get(x[12])))
     if len(missing_items) == 0:
         log.debug(
             "Using last db date because,all downloads in db are marked as downloaded"
@@ -695,4 +697,4 @@ def get_after(model_id, username, forced_after=None):
         log.debug(
             f"Setting date slightly before earliest missing item\nbecause {len(missing_items)} messages in db are marked as undownloaded"
         )
-        return arrow.get(missing_items[0][-1]).float_timestamp - 1000
+        return arrow.get(missing_items[0][12]).float_timestamp - 1000

{ofscraper-3.9.0.dev5 → ofscraper-3.9.0.dev7}/ofscraper/api/paid.py

@@ -101,11 +101,15 @@ async def get_paid_posts(model_id, username, c=None):
     sem = sems.get_req_sem()
     responseArray = []
     tasks = []
+    job_progress = None
+    page_count = 0
 
     # async with c or sessionbuilder.sessionBuilder(
     #     limit=constants.getattr("API_MAX_CONNECTION")
     # ) as c:
-    tasks.append(asyncio.create_task(scrape_paid(c, username, job_progress=None)))
+    tasks.append(
+        asyncio.create_task(scrape_paid(c, username, job_progress=job_progress))
+    )
     while bool(tasks):
         new_tasks = []
         try:
@@ -247,25 +251,20 @@ async def scrape_paid(c, username, job_progress=None, offset=0):
 @run
 async def get_all_paid_posts():
     global sem
-    sem = semaphoreDelayed(constants.getattr("MAX_SEMAPHORE"))
+    sem = sems.get_req_sem()
     with ThreadPoolExecutor(
         max_workers=constants.getattr("MAX_REQUEST_WORKERS")
     ) as executor:
         asyncio.get_event_loop().set_default_executor(executor)
-        overall_progress = Progress(
-            SpinnerColumn(style=Style(color="blue")),
-            TextColumn("Getting all paid media...\n{task.description}"),
-        )
-        job_progress = Progress("{task.description}")
-        progress_group = Group(overall_progress, Panel(Group(job_progress)))
 
         output = []
         min_posts = 100
         tasks = []
         page_count = 0
-        with Live(
-            progress_group, refresh_per_second=5, console=console.get_shared_console()
-        ):
+        with progress_utils.setup_all_paid_live():
+            job_progress = progress_utils.all_paid_progress
+            overall_progress = progress_utils.overall_progress
+
             async with sessionbuilder.sessionBuilder() as c:
                 allpaid = cache.get(f"purchased_all", default=[])
                 log.debug(f"[bold]All Paid Cache[/bold] {len(allpaid)} found")

{ofscraper-3.9.0.dev5 → ofscraper-3.9.0.dev7}/ofscraper/api/pinned.py

@@ -49,7 +49,7 @@ async def get_pinned_post(model_id, c=None):
             scrape_pinned_posts(
                 c,
                 model_id,
-                job_progress,
+                job_progress=job_progress,
                 timestamp=read_args.retriveArgs().after.float_timestamp
                 if read_args.retriveArgs().after
                 else None,
@@ -102,7 +102,9 @@ async def get_pinned_post(model_id, c=None):
     return list(outdict.values())
 
 
-async def scrape_pinned_posts(c, model_id, progress, timestamp=None, count=0) -> list:
+async def scrape_pinned_posts(
+    c, model_id, job_progress=None, timestamp=None, count=0
+) -> list:
     global sem
     sem = semaphoreDelayed(constants.getattr("AlT_SEM"))
     posts = None
@@ -131,7 +133,7 @@ async def scrape_pinned_posts(c, model_id, progress, timestamp=None, count=0) ->
             await asyncio.sleep(1)
             try:
                 attempt.set(attempt.get(0) + 1)
-                task = progress.add_task(
+                task = job_progress.add_task(
                     f"Attempt {attempt.get()}/{constants.getattr('NUM_TRIES')}: Timestamp -> {arrow.get(math.trunc(float(timestamp))) if timestamp!=None  else 'initial'}",
                     visible=True,
                 )
@@ -184,7 +186,7 @@ async def scrape_pinned_posts(c, model_id, progress, timestamp=None, count=0) ->
                                     scrape_pinned_posts(
                                         c,
                                         model_id,
-                                        progress,
+                                        job_progress=job_progress,
                                         timestamp=posts[-1]["postedAtPrecise"],
                                         count=count + len(posts),
                                     )
@@ -205,5 +207,5 @@ async def scrape_pinned_posts(c, model_id, progress, timestamp=None, count=0) ->
 
             finally:
                 sem.release()
-                progress.remove_task(task)
+                job_progress.remove_task(task)
             return posts, new_tasks

{ofscraper-3.9.0.dev5 → ofscraper-3.9.0.dev7}/ofscraper/api/subscriptions/individual.py

@@ -43,13 +43,13 @@ async def get_subscription(accounts=None):
 
         with Progress(
             SpinnerColumn(style=Style(color="blue")), TextColumn("{task.description}")
-        ) as progress:
-            task1 = progress.add_task(
+        ) as job_progress:
+            task1 = job_progress.add_task(
                 f"Getting the following accounts => {accounts} (this may take awhile)..."
             )
             async with sessionbuilder.sessionBuilder() as c:
                 out = await get_subscription_helper(c, accounts)
-                progress.remove_task(task1)
+                job_progress.remove_task(task1)
         outdict = {}
         for ele in filter(lambda x: x["username"] != "modeldeleted", out):
             outdict[ele["id"]] = ele

{ofscraper-3.9.0.dev5 → ofscraper-3.9.0.dev7}/ofscraper/api/subscriptions/subscriptions.py

@@ -51,8 +51,8 @@ async def get_subscriptions(subscribe_count, account="active"):
 
         with Progress(
             SpinnerColumn(style=Style(color="blue")), TextColumn("{task.description}")
-        ) as progress:
-            task1 = progress.add_task(
+        ) as job_progress:
+            task1 = job_progress.add_task(
                 f"Getting your {account} subscriptions (this may take awhile)..."
             )
             async with sessionbuilder.sessionBuilder() as c:
@@ -60,7 +60,7 @@ async def get_subscriptions(subscribe_count, account="active"):
                     out = await activeHelper(subscribe_count, c)
                 else:
                     out = await expiredHelper(subscribe_count, c)
-                progress.remove_task(task1)
+                job_progress.remove_task(task1)
         outdict = {}
         for ele in out:
             outdict[ele["id"]] = ele