scrapeMM 0.4.2__tar.gz → 0.4.4__tar.gz

{scrapemm-0.4.2/scrapeMM.egg-info → scrapemm-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.4.2
+Version: 0.4.4
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.4.2 → scrapemm-0.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.4.2"
+version = "0.4.4"
 description = "Flexible multimodal scraper for social media and the open web."
 readme = "README.md"
 requires-python = ">=3.10"

{scrapemm-0.4.2 → scrapemm-0.4.4/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.4.2
+Version: 0.4.4
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.4.2 → scrapemm-0.4.4}/scrapemm/__init__.py

@@ -2,7 +2,7 @@ import logging
 import sys
 
 import scrapemm.common
-from .common import APP_NAME
+from .common import APP_NAME, set_wait_on_rate_limit, RateLimitError, ContentNotFoundError
 from .integrations import Telegram, X
 from .retrieval import retrieve
 from .secrets import configure_secrets

{scrapemm-0.4.2 → scrapemm-0.4.4}/scrapemm/common/__init__.py

@@ -5,6 +5,7 @@ import yaml
 from platformdirs import user_config_dir
 
 from .scraping_response import ScrapingResponse
+from .exceptions import RateLimitError, ContentNotFoundError
 
 APP_NAME = "scrapeMM"
 
@@ -13,6 +14,7 @@ CONFIG_DIR = Path(user_config_dir(APP_NAME))
 os.makedirs(CONFIG_DIR, exist_ok=True)
 CONFIG_PATH = CONFIG_DIR / "config.yaml"
 
+WAIT_ON_RATE_LIMIT = False
 
 def load_config() -> dict:
     if os.path.exists(CONFIG_PATH):
@@ -31,5 +33,12 @@ def get_config_var(name: str, default=None) -> str:
     return _config.get(name, default)
 
 
+def set_wait_on_rate_limit(wait: bool):
+    """Set whether to wait on rate limits (particularly relevant for X API).
+    Will result in a RateLimitError otherwise."""
+    global WAIT_ON_RATE_LIMIT
+    WAIT_ON_RATE_LIMIT = wait
+
+
 # Load config
 _config = load_config()

scrapemm-0.4.4/scrapemm/common/exceptions.py

@@ -0,0 +1,21 @@
+class RateLimitError(Exception):
+    pass
+
+
+class IPBannedError(Exception):
+    pass
+
+
+class ContentNotFoundError(Exception):
+    pass
+
+
+class ContentBlockedError(Exception):
+    """The content was found but is blocked by content moderation, prohibiting
+    automated access (manual access might work, though)."""
+    pass
+
+
+class UnsupportedDomainError(Exception):
+    """The domain is not supported by the scraper."""
+    pass

{scrapemm-0.4.2 → scrapemm-0.4.4}/scrapemm/integrations/fb.py

@@ -1,12 +1,15 @@
 import logging
 import re
-from urllib.parse import urlparse
+from urllib.parse import urlparse, parse_qs
 
 from ezmm import MultimodalSequence
 
+from scrapemm import RateLimitError
+from scrapemm.common import CONFIG_DIR
+from scrapemm.common.exceptions import ContentBlockedError
 from scrapemm.integrations.base import RetrievalIntegration
 from scrapemm.integrations.ytdlp import get_content_with_ytdlp
-from scrapemm.util import get_domain
+from scrapemm.secrets import get_secret
 
 logger = logging.getLogger("scrapeMM")
 
@@ -16,17 +19,38 @@ VIDEO_URL_REGEX = r"facebook\.com/\d+/videos/\d+/?"
 class Facebook(RetrievalIntegration):
     name = "Facebook"
     domains = ["facebook.com", "fb.watch"]
+    cookie_file = CONFIG_DIR / "facebook_cookie.txt"
 
     async def _connect(self):
         self.api_available = False  # TODO
+
+        cookie = get_secret("facebook_cookie")
+        if cookie:
+            # Save the cookie in a .txt file next to the secrets file
+            with open(self.cookie_file, "w") as f:
+                f.write(cookie)
+            logger.info(f"✅ Using cookie to connect to Facebook.")
+        else:
+            logger.warning(f"⚠️ Missing Facebook cookie. Won't be able to download videos that require login.")
+
         logger.info(f"✅ Facebook integration ready (yt-dlp only mode).")
         self.connected = True
 
     async def _get(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook post URL."""
+        url = self._normalize_url(url)
+
         # Determine if this is a video or photo URL, act accordingly
         if self._is_video_url(url):
-            return await self._get_video(url, **kwargs)
+            try:
+                return await self._get_video(url, **kwargs)
+            except Exception as e:
+                if "No video formats found" in str(e):
+                    raise ContentBlockedError(f"Video is blocked by Facebook.")
+                elif "This video is only available for registered users" in str(e):
+                    raise RateLimitError(f"Facebook is rate-limiting your IP address. Set a 'facebook_cookie' in ScrapeMM.")
+                else:
+                    raise e
         elif self._is_photo_url(url):
             return await self._get_photo(url, **kwargs)
 
@@ -40,7 +64,10 @@ class Facebook(RetrievalIntegration):
         if self.api_available:
             raise NotImplementedError("Facebook video retrieval through API not yet supported.")
         else:
-            return await get_content_with_ytdlp(url, platform="Facebook", **kwargs)
+            return await get_content_with_ytdlp(url,
+                                                platform="Facebook",
+                                                cookie_file=self.cookie_file.as_posix(),
+                                                **kwargs)
 
     async def _get_photo(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook photo URL."""
@@ -50,6 +77,17 @@ class Facebook(RetrievalIntegration):
         """Retrieves content from a Facebook user profile URL."""
         raise NotImplementedError("No available method to retrieve Facebook profiles.")
 
+    def _normalize_url(self, url: str) -> str:
+        """If the URL is a login Facebook URL, i.e., of the form https://www.facebook.com/login/?next=...
+        or https://www.facebook.com/plugins/post.php?href=..., extracts the actual post's URL."""
+        if url.startswith("https://www.facebook.com/login/?next="):  # Login redirect URLs
+            query = urlparse(url).query
+            return parse_qs(query).get("next", [])[0] or url
+        elif url.startswith("https://www.facebook.com/plugins/post.php?href="):  # Post embedding links
+            query = urlparse(url).query
+            return parse_qs(query).get("href", [])[0] or url
+        return url
+
     def _is_video_url(self, url: str) -> bool:
         """Checks if the URL is a Facebook video URL."""
         # video URLS are in the format: https://www.facebook.com/watch?v=VIDEO_ID or fb.watch/...

{scrapemm-0.4.2 → scrapemm-0.4.4}/scrapemm/integrations/tiktok.py

@@ -9,7 +9,7 @@ from ezmm import MultimodalSequence, download_image
 from ezmm.common.items import Video, Image
 from tiktok_research_api import TikTokResearchAPI, QueryVideoRequest, QueryUserInfoRequest, Criteria, Query
 
-from scrapemm.common.exceptions import RateLimitError
+from scrapemm.common.exceptions import RateLimitError, ContentBlockedError
 from scrapemm.integrations.base import RetrievalIntegration
 from scrapemm.integrations.ytdlp import download_video_with_ytdlp
 from scrapemm.secrets import get_secret
@@ -68,8 +68,11 @@ class TikTok(RetrievalIntegration):
             else:
                 return await self._get_user_profile(url, session)
         except Exception as e:
-            if "quota" in str(e).lower():
-                raise RateLimitError(f"TikTok rate limit likely reached: {e}")
+            if "Your IP address is blocked from accessing this post":
+                raise ContentBlockedError("Video is blocked by TikTok.")
+            elif "This post may not be comfortable for some audiences" in str(e):
+                raise ContentBlockedError("Video is blocked by TikTok for being 'uncomfortable for some audiences'. "
+                                          "Set a 'tiktok_cookie' in ScrapeMM to download this video.")
             else:
                 raise e
 

{scrapemm-0.4.2 → scrapemm-0.4.4}/scrapemm/integrations/x.py

@@ -2,15 +2,17 @@ import asyncio
 import logging
 import re
 from typing import Optional
-from urllib.parse import urlparse
+from urllib.parse import urlparse, unquote, urljoin, parse_qs
 
 import aiohttp
 from ezmm import MultimodalSequence, download_video, download_image
-from tweepy import Tweet, User
+from tweepy import Tweet, User, TooManyRequests
 from tweepy.asynchronous import AsyncClient
 
-from scrapemm.secrets import get_secret
+import scrapemm.common
+from scrapemm.common.exceptions import RateLimitError
 from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.secrets import get_secret
 
 logger = logging.getLogger("scrapeMM")
 
@@ -40,7 +42,7 @@ class X(RetrievalIntegration):
     async def _connect(self):
         bearer_token = get_secret("x_bearer_token")
         if bearer_token:
-            self.client = AsyncClient(bearer_token=bearer_token, wait_on_rate_limit=True)
+            self.client = AsyncClient(bearer_token=bearer_token, wait_on_rate_limit=scrapemm.common.WAIT_ON_RATE_LIMIT)
             self.connected = True
             logger.info("✅ Successfully connected to X.")
         else:
@@ -50,13 +52,25 @@ class X(RetrievalIntegration):
     async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         session = kwargs.get("session")
         max_video_size = kwargs.get("max_video_size")
+        url = self._normalize(url)
         tweet_id = extract_tweet_id_from_url(url)
-        if tweet_id:
-            return await self._get_tweet(tweet_id, session, max_video_size)
-        else:
-            username = extract_username_from_url(url)
-            if username:
-                return await self._get_user(username, session)
+        try:
+            if tweet_id:
+                return await self._get_tweet(tweet_id, session, max_video_size)
+            else:
+                username = extract_username_from_url(url)
+                if username:
+                    return await self._get_user(username, session)
+        except TooManyRequests:
+            raise RateLimitError("X API rate limit reached.")
+
+    def _normalize(self, url: str) -> str:
+        """Turns URLs of the form https://publish.twitter.com/?query=...
+        into the bare Twitter URL."""
+        if url.startswith("https://publish.twitter.com/?query="):
+            query = urlparse(url).query
+            return parse_qs(query).get("query", [])[0] or url
+        return url
 
     async def _get_tweet(self, tweet_id: int, session: aiohttp.ClientSession, max_video_size: int = None) -> Optional[MultimodalSequence]:
         """Returns a MultimodalSequence containing the tweet's text and media

{scrapemm-0.4.2 → scrapemm-0.4.4}/scrapemm/integrations/ytdlp.py

@@ -2,7 +2,6 @@ import asyncio
 import logging
 import sys
 import tempfile
-import traceback
 from datetime import datetime
 from typing import Any, Optional
 
@@ -10,7 +9,7 @@ import aiohttp
 from ezmm import MultimodalSequence, download_image, Video, Image
 from yt_dlp import YoutubeDL
 
-from scrapemm.common.exceptions import IPBannedError, ContentNotFoundError
+from scrapemm.common.exceptions import ContentNotFoundError
 
 logger = logging.getLogger("scrapeMM")
 
@@ -79,7 +78,8 @@ async def download_video_with_ytdlp(
         if "The following content is not available on this app" in str(e):
             logger.warning(f"You should update yt-dlp to re-enable YouTube downloads.")
             raise e
-        elif "Video unavailable" in str(e):
+        elif ("Video unavailable" in str(e)
+              or "HTTP Error 404: Not Found" in str(e)):
             raise ContentNotFoundError(f"Video is unavailable.")
         elif "Cannot parse data; please report this issue" in str(e):
             raise RuntimeError(f"yt-dlp is unable to parse the received video metadata.")

{scrapemm-0.4.2 → scrapemm-0.4.4}/scrapemm/retrieval.py

@@ -7,13 +7,28 @@ import aiohttp
 from ezmm import MultimodalSequence, download_item
 
 from scrapemm.common import ScrapingResponse
-from scrapemm.common.exceptions import IPBannedError
+from scrapemm.common.exceptions import IPBannedError, UnsupportedDomainError
 from scrapemm.integrations import retrieve_via_integration, fire, decodo
-from scrapemm.util import run_with_semaphore
+from scrapemm.util import run_with_semaphore, get_domain
 
 logger = logging.getLogger("scrapeMM")
 METHODS = ["integrations", "firecrawl", "decodo"]
 
+UNSUPPORTED_DOMAINS = [
+    "archive.today",
+    "archive.is",
+    "archive.ph",
+    "archive.vn",
+    "archive.li",
+    "archive.fo",
+    "archive.md",
+    "perma.cc",
+    "ghostarchive.org",
+    "archive.org",
+    "mvau.lt",
+    "archive.st",
+]
+
 
 async def retrieve(
         urls: str | Collection[str],
@@ -103,6 +118,10 @@ async def _retrieve_single(
 ) -> ScrapingResponse:
     logger.debug(f"Retrieving {url}")
 
+    if get_domain(url) in UNSUPPORTED_DOMAINS:
+        return ScrapingResponse(url=url, content=None,
+                                errors=dict(scrapemm=UnsupportedDomainError("Unsupported domain.")))
+
     if methods is None:
         methods = METHODS.copy()
 

{scrapemm-0.4.2 → scrapemm-0.4.4}/scrapemm/secrets.py

@@ -25,7 +25,8 @@ SECRETS = {
     "tiktok_client_secret": "TikTok client secret",
     "decodo_username": "Decodo Web Scraping API username",
     "decodo_password": "Decodo Web Scraping API password",
-    "youtube_cookie": "YouTube cookie string"
+    "youtube_cookie": "YouTube cookie string",
+    "facebook_cookie": "Facebook cookie string",
 }
 
 SALT = b'\xa4\x93\xf1\x88\x13\x88'
@@ -142,7 +143,7 @@ def override_secret(key_name: str):
     """Prompts the user to enter a new value for the given secret key. Does nothing
     when nothing entered."""
     description = SECRETS[key_name]
-    if key_name == "youtube_cookie":
+    if key_name in ["youtube_cookie", "facebook_cookie"]:
         user_input = get_multiline_user_input(f"Please enter the {description}. Hit Ctrl-D or Ctrl-Z to save it.")
     else:
         user_input = getpass(f"Please enter the {description} (leave empty to skip): ", stream=sys.stdout)

{scrapemm-0.4.2 → scrapemm-0.4.4}/scripts/example.py

@@ -2,7 +2,7 @@ from scrapemm import retrieve
 import asyncio
 
 if __name__ == "__main__":
-    url = "https://www.awesomescreenshot.com/image/43774028?key=0de2147873b73ed468d07aeb93512c51"
+    url = "https://www.facebook.com/login/?next=https%3A%2F%2Fwww.facebook.com%2Fphoto%3Ffbid%3D860758296160977%26set%3Da.513585680878242"
     result = asyncio.run(retrieve(url))
     if result.errors:
         print(result.errors)

scrapemm-0.4.2/scrapemm/common/exceptions.py

@@ -1,8 +0,0 @@
-class RateLimitError(Exception):
-    pass
-
-class IPBannedError(Exception):
-    pass
-
-class ContentNotFoundError(Exception):
-    pass