scrapeMM 0.4.2__tar.gz → 0.4.3__tar.gz

{scrapemm-0.4.2/scrapeMM.egg-info → scrapemm-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.4.2
+Version: 0.4.3
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.4.2 → scrapemm-0.4.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.4.2"
+version = "0.4.3"
 description = "Flexible multimodal scraper for social media and the open web."
 readme = "README.md"
 requires-python = ">=3.10"

{scrapemm-0.4.2 → scrapemm-0.4.3/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.4.2
+Version: 0.4.3
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.4.2 → scrapemm-0.4.3}/scrapemm/__init__.py

@@ -2,7 +2,7 @@ import logging
 import sys
 
 import scrapemm.common
-from .common import APP_NAME
+from .common import APP_NAME, set_wait_on_rate_limit, RateLimitError, ContentNotFoundError
 from .integrations import Telegram, X
 from .retrieval import retrieve
 from .secrets import configure_secrets

{scrapemm-0.4.2 → scrapemm-0.4.3}/scrapemm/common/__init__.py

@@ -5,6 +5,7 @@ import yaml
 from platformdirs import user_config_dir
 
 from .scraping_response import ScrapingResponse
+from .exceptions import RateLimitError, ContentNotFoundError
 
 APP_NAME = "scrapeMM"
 
@@ -13,6 +14,7 @@ CONFIG_DIR = Path(user_config_dir(APP_NAME))
 os.makedirs(CONFIG_DIR, exist_ok=True)
 CONFIG_PATH = CONFIG_DIR / "config.yaml"
 
+WAIT_ON_RATE_LIMIT = False
 
 def load_config() -> dict:
     if os.path.exists(CONFIG_PATH):
@@ -31,5 +33,12 @@ def get_config_var(name: str, default=None) -> str:
     return _config.get(name, default)
 
 
+def set_wait_on_rate_limit(wait: bool):
+    """Set whether to wait on rate limits (particularly relevant for X API).
+    Will result in a RateLimitError otherwise."""
+    global WAIT_ON_RATE_LIMIT
+    WAIT_ON_RATE_LIMIT = wait
+
+
 # Load config
 _config = load_config()

{scrapemm-0.4.2 → scrapemm-0.4.3}/scrapemm/integrations/fb.py

@@ -1,6 +1,6 @@
 import logging
 import re
-from urllib.parse import urlparse
+from urllib.parse import urlparse, unquote
 
 from ezmm import MultimodalSequence
 
@@ -24,6 +24,8 @@ class Facebook(RetrievalIntegration):
 
     async def _get(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook post URL."""
+        url = self._normalize_url(url)
+
         # Determine if this is a video or photo URL, act accordingly
         if self._is_video_url(url):
             return await self._get_video(url, **kwargs)
@@ -50,6 +52,13 @@ class Facebook(RetrievalIntegration):
         """Retrieves content from a Facebook user profile URL."""
         raise NotImplementedError("No available method to retrieve Facebook profiles.")
 
+    def _normalize_url(self, url: str) -> str:
+        """If the URL is a login Facebook URL, i.e., of the form https://www.facebook.com/login/?next=...
+        extracts the actual post's URL."""
+        if url.startswith("https://www.facebook.com/login/?next="):
+            url = unquote(url.split("?next=")[1])
+        return url
+
     def _is_video_url(self, url: str) -> bool:
         """Checks if the URL is a Facebook video URL."""
         # video URLS are in the format: https://www.facebook.com/watch?v=VIDEO_ID or fb.watch/...

{scrapemm-0.4.2 → scrapemm-0.4.3}/scrapemm/integrations/x.py

@@ -1,14 +1,17 @@
 import asyncio
 import logging
 import re
+import traceback
 from typing import Optional
 from urllib.parse import urlparse
 
 import aiohttp
 from ezmm import MultimodalSequence, download_video, download_image
-from tweepy import Tweet, User
+from tweepy import Tweet, User, BadRequest, TooManyRequests
 from tweepy.asynchronous import AsyncClient
 
+import scrapemm.common
+from scrapemm.common.exceptions import RateLimitError
 from scrapemm.secrets import get_secret
 from scrapemm.integrations.base import RetrievalIntegration
 
@@ -40,7 +43,7 @@ class X(RetrievalIntegration):
     async def _connect(self):
         bearer_token = get_secret("x_bearer_token")
         if bearer_token:
-            self.client = AsyncClient(bearer_token=bearer_token, wait_on_rate_limit=True)
+            self.client = AsyncClient(bearer_token=bearer_token, wait_on_rate_limit=scrapemm.common.WAIT_ON_RATE_LIMIT)
             self.connected = True
             logger.info("✅ Successfully connected to X.")
         else:
@@ -51,12 +54,16 @@ class X(RetrievalIntegration):
         session = kwargs.get("session")
         max_video_size = kwargs.get("max_video_size")
         tweet_id = extract_tweet_id_from_url(url)
-        if tweet_id:
-            return await self._get_tweet(tweet_id, session, max_video_size)
-        else:
-            username = extract_username_from_url(url)
-            if username:
-                return await self._get_user(username, session)
+        try:
+            if tweet_id:
+                return await self._get_tweet(tweet_id, session, max_video_size)
+            else:
+                username = extract_username_from_url(url)
+                if username:
+                    return await self._get_user(username, session)
+        except TooManyRequests:
+            raise RateLimitError("X API rate limit reached.")
+
 
     async def _get_tweet(self, tweet_id: int, session: aiohttp.ClientSession, max_video_size: int = None) -> Optional[MultimodalSequence]:
         """Returns a MultimodalSequence containing the tweet's text and media

{scrapemm-0.4.2 → scrapemm-0.4.3}/scrapemm/integrations/ytdlp.py

@@ -2,7 +2,6 @@ import asyncio
 import logging
 import sys
 import tempfile
-import traceback
 from datetime import datetime
 from typing import Any, Optional
 
@@ -10,7 +9,7 @@ import aiohttp
 from ezmm import MultimodalSequence, download_image, Video, Image
 from yt_dlp import YoutubeDL
 
-from scrapemm.common.exceptions import IPBannedError, ContentNotFoundError
+from scrapemm.common.exceptions import ContentNotFoundError
 
 logger = logging.getLogger("scrapeMM")
 
@@ -79,7 +78,8 @@ async def download_video_with_ytdlp(
         if "The following content is not available on this app" in str(e):
             logger.warning(f"You should update yt-dlp to re-enable YouTube downloads.")
             raise e
-        elif "Video unavailable" in str(e):
+        elif ("Video unavailable" in str(e)
+              or "HTTP Error 404: Not Found" in str(e)):
             raise ContentNotFoundError(f"Video is unavailable.")
         elif "Cannot parse data; please report this issue" in str(e):
             raise RuntimeError(f"yt-dlp is unable to parse the received video metadata.")

{scrapemm-0.4.2 → scrapemm-0.4.3}/scripts/example.py

@@ -2,7 +2,7 @@ from scrapemm import retrieve
 import asyncio
 
 if __name__ == "__main__":
-    url = "https://www.awesomescreenshot.com/image/43774028?key=0de2147873b73ed468d07aeb93512c51"
+    url = "https://x.com/Shayan86/status/1891673533601780022"
     result = asyncio.run(retrieve(url))
     if result.errors:
         print(result.errors)