ofscraper 3.8.4__tar.gz → 3.9.0.dev0__tar.gz

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ofscraper
-Version: 3.8.4
+Version: 3.9.0.dev0
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/ofscraper/actions/process.py

@@ -37,6 +37,7 @@ import ofscraper.utils.constants as constants
 import ofscraper.utils.context.exit as exit
 import ofscraper.utils.context.stdout as stdout
 import ofscraper.utils.profiles.tools as profile_tools
+from ofscraper.utils.text import get_text
 
 log = logging.getLogger("shared")
 

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/ofscraper/classes/media.py

@@ -37,6 +37,9 @@ class Media:
         self._final_url = None
         self._cached_parse_mpd = None
 
+    def __eq__(self, other):
+        return self.postid == other.postid
+
     @property
     def expires(self):
         return self._post.expires

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/ofscraper/classes/placeholder.py

@@ -90,9 +90,9 @@ class tempFilePlaceholder(basePlaceholder):
         self._tempname = tempname
 
     async def init(self):
-        self._tempfilepath = paths.truncate(
-            pathlib.Path(await self.gettempDir(self._ele), self._tempname)
-        )
+        dir = await self.gettempDir(self._ele)
+        file = self._tempname
+        self._tempfilepath = paths.truncate(pathlib.Path(dir, file))
 
     @basePlaceholder.async_wrapper
     async def gettempDir(self, ele, create=True):
@@ -190,12 +190,9 @@ class Placeholders(basePlaceholder):
         self._ext = ext
 
     async def init(self):
-        self._filepath = paths.truncate(
-            pathlib.Path(
-                await self.getmediadir(),
-                await self.createfilename(),
-            )
-        )
+        dir = await self.getmediadir()
+        file = await self.createfilename()
+        self._filepath = paths.truncate(pathlib.Path(dir, file))
 
     def add_price_variables(self, username):
         modelObj = selector.get_model_fromParsed(username)

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/ofscraper/download/common/common.py

@@ -32,6 +32,7 @@ from ofscraper.download.common.metadata import *
 from ofscraper.download.common.paths import *
 from ofscraper.download.common.progress import *
 from ofscraper.download.common.sem import sem_wrapper
+from ofscraper.download.common.text import *
 from ofscraper.utils.context.run_async import run
 
 

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/ofscraper/download/common/text.py

@@ -2,6 +2,7 @@ import logging
 import traceback
 
 import ofscraper.utils.settings as settings
+import ofscraper.utils.text as text
 
 
 def textDownloader(mediadicts):

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/ofscraper/download/download.py

@@ -11,6 +11,7 @@ import ofscraper.utils.hash as hash
 import ofscraper.utils.separate as seperate
 import ofscraper.utils.settings as settings
 import ofscraper.utils.system.system as system
+from ofscraper.download.common.common import textDownloader
 
 
 def medialist_filter(medialist, model_id, username):
@@ -36,6 +37,7 @@ def medialist_filter(medialist, model_id, username):
 def download_process(username, model_id, medialist):
     medialist = medialist_filter(medialist, model_id, username)
     medialist = helpers.post_count_filter(medialist)
+    textDownloader(medialist)
     download_picker(username, model_id, medialist)
     remove_downloads_with_hashes(username, model_id)
 

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/ofscraper/download/downloadbatch.py

@@ -381,11 +381,9 @@ async def process_dicts_split(username, model_id, medialist):
     )
 
     aws = []
-
     async with sessionbuilder.sessionBuilder() as c:
         for ele in medialist:
             aws.append(asyncio.create_task(download(c, ele, model_id, username)))
-
         for coro in asyncio.as_completed(aws):
             try:
                 pack = await coro

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/ofscraper/utils/binaries.py

@@ -15,7 +15,6 @@ import ofscraper.utils.paths.common as common_paths
 
 
 def mp4decrypt_download():
-    return mp4_decrypt_mac()
     if platform.system() == "Windows":
         return mp4_decrypt_windows()
     elif platform.system() == "Linux":

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/ofscraper/utils/config/data.py

@@ -681,6 +681,15 @@ def get_images_overwrites(config=None):
     )
 
 
+@wrapper.config_reader
+def get_text_overwrites(config=None):
+    if config == False:
+        return constants.EMPTY_MEDIA_DEFAULT
+    return config.get("overwrites", {}).get("text") or constants_attr.getattr(
+        "EMPTY_MEDIA_DEFAULT"
+    )
+
+
 @wrapper.config_reader
 def get_max_post_count(config=None):
     if config == False:

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/ofscraper/utils/config/schema.py

@@ -75,6 +75,7 @@ def get_current_config_schema(config: dict = None) -> dict:
             "audios": data.get_audios_overwrites(config=config),
             "videos": data.get_videos_overwrites(config=config),
             "images": data.get_images_overwrites(config=config),
+            "text": data.get_text_overwrites(config=config),
         },
     }
     return new_config

ofscraper-3.9.0.dev0/ofscraper/utils/text.py

@@ -0,0 +1,51 @@
+import asyncio
+import copy
+import logging
+import pathlib
+import textwrap
+import traceback
+from concurrent.futures import ThreadPoolExecutor
+
+import aiofiles
+
+import ofscraper.classes.placeholder as placeholder
+import ofscraper.utils.args.read as read_args
+import ofscraper.utils.constants as constants
+import ofscraper.utils.settings as settings
+from ofscraper.utils.context.run_async import run
+
+
+@run
+async def get_text(values):
+    with ThreadPoolExecutor(
+        max_workers=constants.getattr("MAX_TEXT_WORKER")
+    ) as executor:
+        asyncio.get_event_loop().set_default_executor(executor)
+        dupe = read_args.retriveArgs().dupe
+        async with asyncio.TaskGroup() as tg:
+            [tg.create_task(get_text_process(value, dupe=dupe)) for value in values]
+
+
+async def get_text_process(ele, dupe=None):
+    log = logging.getLogger("shared")
+    try:
+        if "Text" not in settings.get_mediatypes():
+            return
+        elif bool(ele.text) == False:
+            return
+        # make new text mediatype
+        new_ele = copy.deepcopy(ele)
+        new_ele.mediatype = "text"
+
+        placeholderObj = placeholder.Placeholders(new_ele, "txt")
+        await placeholderObj.init()
+        if pathlib.Path(placeholderObj.filepath).exists() and not dupe:
+            return
+        wrapped_text = textwrap.wrap(
+            new_ele.text, width=constants.getattr("MAX_TEXT_LENGTH")
+        )
+        async with aiofiles.open(placeholderObj.filepath, "w") as p:
+            await p.writelines(wrapped_text)
+    except Exception as E:
+        log.traceback_(f"{E}")
+        log.traceback_(f"{traceback.format_exc()}")

{ofscraper-3.8.4 → ofscraper-3.9.0.dev0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ofscraper"
-version = "3.8.4"
+version = "3.9.0dev0"
 description = "automatically scrape onlyfans"
 authors = ["datawhores <datawhores@riseup.net>"]
 readme = "README.md"