ofscraper 3.12.8__tar.gz → 3.13.dev0__tar.gz

{ofscraper-3.12.8 → ofscraper-3.13.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: ofscraper
-Version: 3.12.8
+Version: 3.13.dev0
 Summary: automatically scrape onlyfans
 Author: datawhores
 Author-email: datawhores@riseup.net
@@ -8,9 +8,10 @@ Requires-Python: >=3.11,<3.14
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Provides-Extra: pyinstaller
 Requires-Dist: aiofiles (>=23.2.1,<24.0.0)
-Requires-Dist: aiohttp (==3.10.2)
+Requires-Dist: aiohttp (==3.10.11)
 Requires-Dist: aiolimiter (>=1.1.0,<2.0.0)
 Requires-Dist: aioprocessing (>=2.0.1,<3.0.0)
 Requires-Dist: aiosqlite (>=0.20.0,<0.21.0)
@@ -48,7 +49,7 @@ Requires-Dist: setproctitle (>=1.3.3,<2.0.0)
 Requires-Dist: setuptools (>=69.1.1,<71.0.0)
 Requires-Dist: speedtest-cli (>=2.1.3,<3.0.0)
 Requires-Dist: tenacity (>=8.2.3,<9.0.0)
-Requires-Dist: textual (==0.52.1)
+Requires-Dist: textual (==1.0.0)
 Requires-Dist: tqdm (>=4.66.4,<5.0.0)
 Requires-Dist: uvloop (>=0.19.0,<0.20.0) ; sys_platform == "linux" or sys_platform == "linux2"
 Requires-Dist: win32-setctime (>=1.1.0,<2.0.0)
@@ -56,10 +57,13 @@ Requires-Dist: xxhash (>=3.4.1,<4.0.0)
 Project-URL: Homepage, https://github.com/datawhores/OF-Scraper
 Description-Content-Type: text/markdown
 
-# Notice 
+# Notice
+
 1. OF-Scraper can not bypass pay walls
 2. OF-Scraper requires a subscription to the model, models can not be scraped anonymously whether free or paid
+
 # Please Update to 3.11.6
+
 As the API has changed older version are no longer viable
 
 # For auth issues

{ofscraper-3.12.8 → ofscraper-3.13.dev0}/README.md

@@ -1,7 +1,10 @@
-# Notice 
+# Notice
+
 1. OF-Scraper can not bypass pay walls
 2. OF-Scraper requires a subscription to the model, models can not be scraped anonymously whether free or paid
+
 # Please Update to 3.11.6
+
 As the API has changed older version are no longer viable
 
 # For auth issues

{ofscraper-3.12.8 → ofscraper-3.13.dev0}/ofscraper/__main__.py

@@ -1,7 +1,7 @@
 #!/root/OF-Scraper/.venv/bin/python
 import multiprocessing
 
-import ofscraper.runner.open.load as load
+import ofscraper.main.open.load as load
 import ofscraper.utils.system.system as system
 
 

{ofscraper-3.12.8 → ofscraper-3.13.dev0}/ofscraper/__version__.py

@@ -25,9 +25,9 @@ try:
     __version__ = __hardcoded__ or Version.from_git(
         pattern="(?P<base>\d+\.\d+(\.((\d+\.\w+)|\w+)|))"
     ).serialize(format="{base}+{branch}.{commit}", metadata=False)
-    if __version__ == "0.0.0":
-        print(__version__)
-        # raise Exception("pipx error")
+    # if __version__ == "0.0.0":
+    #     print(__version__)
+    # raise Exception("pipx error")
 except:
     import importlib.metadata
 

{ofscraper-3.12.8 → ofscraper-3.13.dev0}/ofscraper/classes/labels.py

@@ -1,4 +1,4 @@
-import ofscraper.classes.posts as posts_
+import ofscraper.classes.of.posts as posts_
 
 
 class Label:

{ofscraper-3.12.8/ofscraper/classes → ofscraper-3.13.dev0/ofscraper/classes/of}/base.py

@@ -18,10 +18,10 @@ class base:
             return "None"
         if len(text) == 0:
             return text
-        length = int(settings.get_textlength(mediatype=self.mediatype))
+        length = int(settings.get_settings(mediatype=self.mediatype).text_length)
         if length == 0:
             return text
-        elif settings.get_text_type() == "letter":
+        elif settings.get_settings(mediatype=self.mediatype).text_type == "letter":
             return f"{''.join(list(text)[:length])}"
         else:
             # split and reduce
@@ -37,7 +37,9 @@ class base:
         text = re.sub('[\n<>:"/\|?*:;]+', "", text)
         text = re.sub("-+", "_", text)
         text = re.sub(" +", " ", text)
-        text = re.sub(" ", settings.get_space_replacer(mediatype=mediatype), text)
+        text = re.sub(
+            " ", settings.get_settings(mediatype=mediatype).space_replacer, text
+        )
         return text
 
     def db_cleanup(self, string):

{ofscraper-3.12.8/ofscraper/classes → ofscraper-3.13.dev0/ofscraper/classes/of}/media.py

@@ -10,13 +10,13 @@ from bs4 import MarkupResemblesLocatorWarning
 from mpegdash.parser import MPEGDASHParser
 from async_property import async_cached_property
 
-import ofscraper.classes.base as base
+import ofscraper.classes.of.base as base
 import ofscraper.utils.args.accessors.quality as quality
 import ofscraper.utils.config.data as data
 import ofscraper.utils.constants as constants
 import ofscraper.utils.dates as dates
 import ofscraper.utils.logs.utils.level as log_helpers
-import  ofscraper.runner.manager as manager
+import ofscraper.main.manager as manager
 
 
 warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)
@@ -50,11 +50,13 @@ class Media(base.base):
     @property
     def media_source(self):
         return self._media.get("source", {})
-    
 
     @property
     def files_source(self):
-        return {key: (inner_dict or {}).get("url") for key, inner_dict in self._media.get("files",{}).items()}
+        return {
+            key: (inner_dict or {}).get("url")
+            for key, inner_dict in self._media.get("files", {}).items()
+        }
 
     @property
     def quality(self):
@@ -101,16 +103,15 @@ class Media(base.base):
 
     def _url_quality_picker(self):
         quality = self.normal_quality_helper()
-        out=None
+        out = None
         if quality != "source":
-            out=self._media.get("videoSources", {}).get(quality)
+            out = self._media.get("videoSources", {}).get(quality)
         elif out is None:
-            out=self.files_source.get("full")
-        
+            out = self.files_source.get("full")
+
         elif out is None:
-            out=self.media_source.get("source")
+            out = self.media_source.get("source")
         return out
-        
 
     @property
     def post(self):
@@ -309,7 +310,7 @@ class Media(base.base):
         text = self.file_cleanup(text, mediatype=self.mediatype)
         text = self.text_trunicate(text)
         if not text:
-            return  self.id
+            return self.id
         return text
 
     @property
@@ -372,7 +373,11 @@ class Media(base.base):
 
     @property
     def linked(self):
-        return None
+        return (self.url or self.mpd) is not None
+
+    @property
+    def link(self):
+        return self.url or self.mpd
 
     @property
     def media(self):

{ofscraper-3.12.8/ofscraper/classes → ofscraper-3.13.dev0/ofscraper/classes/of}/posts.py

@@ -2,8 +2,8 @@ import logging
 
 import arrow
 
-import ofscraper.classes.base as base
-import ofscraper.classes.media as Media
+import ofscraper.classes.of.base as base
+import ofscraper.classes.of.media as Media
 import ofscraper.utils.config.data as data
 
 log = logging.getLogger("shared")

{ofscraper-3.12.8 → ofscraper-3.13.dev0}/ofscraper/classes/placeholder.py

@@ -17,31 +17,12 @@ import ofscraper.utils.paths.paths as paths
 import ofscraper.utils.profiles.data as profile_data
 import ofscraper.utils.settings as settings
 from ofscraper.utils.string import parse_safe
-import  ofscraper.runner.manager as manager
+import ofscraper.main.manager as manager
 
 
 log = logging.getLogger("shared")
 
 
-def check_uniquename():
-    format = data.get_fileformat()
-    if re.search("text", format):
-        return True
-    elif re.search("filename", format):
-        return True
-    elif re.search("post_id", format):
-        return True
-    elif re.search("postid", format):
-        return True
-    elif re.search("media_id", format):
-        return True
-    elif re.search("mediaid", format):
-        return True
-    elif re.search("custom", format):
-        return True
-    return False
-
-
 class basePlaceholder:
     def __init__(self) -> None:
         self._ele = None
@@ -179,6 +160,7 @@ class databasePlaceholder(basePlaceholder):
     def metadata(self, input):
         self._matadata = input
 
+
 class Placeholders(basePlaceholder):
     def __init__(self, ele, ext) -> None:
         super().__init__()
@@ -189,7 +171,7 @@ class Placeholders(basePlaceholder):
         self._ele = ele
         self._ext = ext
 
-    async def init(self,create=True):
+    async def init(self, create=True):
         dir = await self.getmediadir(create=create)
         file = await self.createfilename()
         self._filepath = paths.truncate(pathlib.Path(dir, file))
@@ -258,7 +240,9 @@ class Placeholders(basePlaceholder):
         self._variables.update({"response_type": ele.modified_responsetype})
         self._variables.update({"label": ele.label_string})
         self._variables.update({"download_type": ele.downloadtype})
-        self._variables.update({"modelObj": manager.Manager.model_manager.get_model(username)})
+        self._variables.update(
+            {"modelObj": manager.Manager.model_manager.get_model(username)}
+        )
         self._variables.update({"quality": await ele.selected_quality_placeholder})
         self._variables.update({"file_name": await ele.final_filename})
         self._variables.update({"original_filename": ele.filename})
@@ -397,7 +381,7 @@ class Placeholders(basePlaceholder):
 
     @property
     def trunicated_filepath(self):
-        if settings.get_trunication(mediatype=self._ele.mediatype):
+        if settings.get_settings(mediatype=self._ele.mediatype).trunicate:
             return pathlib.Path(paths.truncate(self._filepath))
         return self._filepath
 
@@ -425,7 +409,7 @@ class Textholders(basePlaceholder):
         self._ele = ele
         self._ext = ext
 
-    async def init(self,create=True):
+    async def init(self, create=True):
         dir = await self.getmediadir(create=create)
         file = await self.createfilename()
         self._filepath = paths.truncate(pathlib.Path(dir, file))
@@ -488,7 +472,9 @@ class Textholders(basePlaceholder):
         self._variables.update({"media_type": "Text"})
         self._variables.update({"response_type": ele.modified_responsetype})
         self._variables.update({"label": ele.label_string})
-        self._variables.update({"modelObj": manager.Manager.model_manager.get_model(username)})
+        self._variables.update(
+            {"modelObj": manager.Manager.model_manager.get_model(username)}
+        )
         self._variables.update({"text": ele.text_trunicate(ele.file_sanitized_text)})
         self._variables.update({"config": config_file.open_config()})
         self._variables.update({"args": read_args.retriveArgs()})
@@ -594,7 +580,7 @@ class Textholders(basePlaceholder):
 
     @property
     def trunicated_filepath(self):
-        if settings.get_trunication(mediatype=self._ele.mediatype):
+        if settings.get_settings(mediatype=self._ele.mediatype).trunicate:
             return pathlib.Path(paths.truncate(self._filepath))
         return self._filepath
 

{ofscraper-3.12.8 → ofscraper-3.13.dev0}/ofscraper/classes/sessionmanager/download.py

@@ -1,11 +1,9 @@
 import contextlib
-import time
-import asyncio
 import ofscraper.classes.sessionmanager.ofsession as ofsessionmanager
 import ofscraper.classes.sessionmanager.sessionmanager as sessionManager
-import ofscraper.actions.utils.globals as common_globals
+import ofscraper.commands.scraper.actions.utils.globals as common_globals
 import ofscraper.utils.constants as constants
-from ofscraper.actions.utils.retries import get_download_req_retries
+from ofscraper.commands.scraper.actions.utils.retries import get_download_req_retries
 from ofscraper.classes.sessionmanager.sessionmanager import (
     AUTH,
     COOKIES,
@@ -14,34 +12,10 @@ from ofscraper.classes.sessionmanager.sessionmanager import (
     SIGN,
     TOO_MANY,
 )
-from ofscraper.classes.sessionmanager.leaky import LeakyBucket
+from ofscraper.commands.scraper.actions.download.utils.leaky import LeakyBucket
 import ofscraper.utils.settings as settings
 
 
-class TokenBucket:
-    def __init__(self, capacity, fill_rate):
-        self.capacity = capacity
-        self.fill_rate = fill_rate
-        self.tokens = 0
-        self.last_update = time.time()
-
-    async def consume(self, tokens):
-        if self.capacity <= 0:
-            return True
-        while True:
-            now = time.time()
-            delta = now - self.last_update
-            self.last_update = now
-            self.tokens = min(self.capacity, self.tokens + delta * self.fill_rate)
-
-            if self.tokens >= tokens:
-                self.tokens -= tokens
-                return True
-
-            # Not enough tokens, wait for refill
-            await asyncio.sleep(0.01)
-
-
 class download_session(sessionManager.sessionManager):
     def __init__(
         self, sem_count=None, retries=None, wait_min=None, wait_max=None, log=None
@@ -51,7 +25,7 @@ class download_session(sessionManager.sessionManager):
         wait_min = wait_min or constants.getattr("OF_MIN_WAIT_API")
         wait_max = wait_max or constants.getattr("OF_MAX_WAIT_API")
         log = log or common_globals.log
-        self.leaky_bucket = LeakyBucket(settings.get_download_limit(), 1)
+        self.leaky_bucket = LeakyBucket(settings.get_settings().download_limit, 1)
         super().__init__(
             sem_count=sem_count,
             retries=retries,
@@ -105,11 +79,10 @@ class download_session(sessionManager.sessionManager):
             async for chunk in funct(*args, **kwargs):
                 await self.get_token(chunk)
                 yield chunk
+
         return wrapper
 
     async def get_token(self, chunk):
-        if settings.get_download_limit() <= 0:
-            return
         await self.leaky_bucket.acquire(chunk)