scrapeMM 0.4.3__tar.gz → 0.4.4__tar.gz

{scrapemm-0.4.3/scrapeMM.egg-info → scrapemm-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.4.3
+Version: 0.4.4
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.4.3 → scrapemm-0.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.4.3"
+version = "0.4.4"
 description = "Flexible multimodal scraper for social media and the open web."
 readme = "README.md"
 requires-python = ">=3.10"

{scrapemm-0.4.3 → scrapemm-0.4.4/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.4.3
+Version: 0.4.4
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

scrapemm-0.4.4/scrapemm/common/exceptions.py

@@ -0,0 +1,21 @@
+class RateLimitError(Exception):
+    pass
+
+
+class IPBannedError(Exception):
+    pass
+
+
+class ContentNotFoundError(Exception):
+    pass
+
+
+class ContentBlockedError(Exception):
+    """The content was found but is blocked by content moderation, prohibiting
+    automated access (manual access might work, though)."""
+    pass
+
+
+class UnsupportedDomainError(Exception):
+    """The domain is not supported by the scraper."""
+    pass

{scrapemm-0.4.3 → scrapemm-0.4.4}/scrapemm/integrations/fb.py

@@ -1,12 +1,15 @@
 import logging
 import re
-from urllib.parse import urlparse, unquote
+from urllib.parse import urlparse, parse_qs
 
 from ezmm import MultimodalSequence
 
+from scrapemm import RateLimitError
+from scrapemm.common import CONFIG_DIR
+from scrapemm.common.exceptions import ContentBlockedError
 from scrapemm.integrations.base import RetrievalIntegration
 from scrapemm.integrations.ytdlp import get_content_with_ytdlp
-from scrapemm.util import get_domain
+from scrapemm.secrets import get_secret
 
 logger = logging.getLogger("scrapeMM")
 
@@ -16,9 +19,20 @@ VIDEO_URL_REGEX = r"facebook\.com/\d+/videos/\d+/?"
 class Facebook(RetrievalIntegration):
     name = "Facebook"
     domains = ["facebook.com", "fb.watch"]
+    cookie_file = CONFIG_DIR / "facebook_cookie.txt"
 
     async def _connect(self):
         self.api_available = False  # TODO
+
+        cookie = get_secret("facebook_cookie")
+        if cookie:
+            # Save the cookie in a .txt file next to the secrets file
+            with open(self.cookie_file, "w") as f:
+                f.write(cookie)
+            logger.info(f"✅ Using cookie to connect to Facebook.")
+        else:
+            logger.warning(f"⚠️ Missing Facebook cookie. Won't be able to download videos that require login.")
+
         logger.info(f"✅ Facebook integration ready (yt-dlp only mode).")
         self.connected = True
 
@@ -28,7 +42,15 @@ class Facebook(RetrievalIntegration):
 
         # Determine if this is a video or photo URL, act accordingly
         if self._is_video_url(url):
-            return await self._get_video(url, **kwargs)
+            try:
+                return await self._get_video(url, **kwargs)
+            except Exception as e:
+                if "No video formats found" in str(e):
+                    raise ContentBlockedError(f"Video is blocked by Facebook.")
+                elif "This video is only available for registered users" in str(e):
+                    raise RateLimitError(f"Facebook is rate-limiting your IP address. Set a 'facebook_cookie' in ScrapeMM.")
+                else:
+                    raise e
         elif self._is_photo_url(url):
             return await self._get_photo(url, **kwargs)
 
@@ -42,7 +64,10 @@ class Facebook(RetrievalIntegration):
         if self.api_available:
             raise NotImplementedError("Facebook video retrieval through API not yet supported.")
         else:
-            return await get_content_with_ytdlp(url, platform="Facebook", **kwargs)
+            return await get_content_with_ytdlp(url,
+                                                platform="Facebook",
+                                                cookie_file=self.cookie_file.as_posix(),
+                                                **kwargs)
 
     async def _get_photo(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook photo URL."""
@@ -54,9 +79,13 @@ class Facebook(RetrievalIntegration):
 
     def _normalize_url(self, url: str) -> str:
         """If the URL is a login Facebook URL, i.e., of the form https://www.facebook.com/login/?next=...
-        extracts the actual post's URL."""
-        if url.startswith("https://www.facebook.com/login/?next="):
-            url = unquote(url.split("?next=")[1])
+        or https://www.facebook.com/plugins/post.php?href=..., extracts the actual post's URL."""
+        if url.startswith("https://www.facebook.com/login/?next="):  # Login redirect URLs
+            query = urlparse(url).query
+            return parse_qs(query).get("next", [])[0] or url
+        elif url.startswith("https://www.facebook.com/plugins/post.php?href="):  # Post embedding links
+            query = urlparse(url).query
+            return parse_qs(query).get("href", [])[0] or url
         return url
 
     def _is_video_url(self, url: str) -> bool:

{scrapemm-0.4.3 → scrapemm-0.4.4}/scrapemm/integrations/tiktok.py

@@ -9,7 +9,7 @@ from ezmm import MultimodalSequence, download_image
 from ezmm.common.items import Video, Image
 from tiktok_research_api import TikTokResearchAPI, QueryVideoRequest, QueryUserInfoRequest, Criteria, Query
 
-from scrapemm.common.exceptions import RateLimitError
+from scrapemm.common.exceptions import RateLimitError, ContentBlockedError
 from scrapemm.integrations.base import RetrievalIntegration
 from scrapemm.integrations.ytdlp import download_video_with_ytdlp
 from scrapemm.secrets import get_secret
@@ -68,8 +68,11 @@ class TikTok(RetrievalIntegration):
             else:
                 return await self._get_user_profile(url, session)
         except Exception as e:
-            if "quota" in str(e).lower():
-                raise RateLimitError(f"TikTok rate limit likely reached: {e}")
+            if "Your IP address is blocked from accessing this post":
+                raise ContentBlockedError("Video is blocked by TikTok.")
+            elif "This post may not be comfortable for some audiences" in str(e):
+                raise ContentBlockedError("Video is blocked by TikTok for being 'uncomfortable for some audiences'. "
+                                          "Set a 'tiktok_cookie' in ScrapeMM to download this video.")
             else:
                 raise e
 

{scrapemm-0.4.3 → scrapemm-0.4.4}/scrapemm/integrations/x.py

@@ -1,19 +1,18 @@
 import asyncio
 import logging
 import re
-import traceback
 from typing import Optional
-from urllib.parse import urlparse
+from urllib.parse import urlparse, unquote, urljoin, parse_qs
 
 import aiohttp
 from ezmm import MultimodalSequence, download_video, download_image
-from tweepy import Tweet, User, BadRequest, TooManyRequests
+from tweepy import Tweet, User, TooManyRequests
 from tweepy.asynchronous import AsyncClient
 
 import scrapemm.common
 from scrapemm.common.exceptions import RateLimitError
-from scrapemm.secrets import get_secret
 from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.secrets import get_secret
 
 logger = logging.getLogger("scrapeMM")
 
@@ -53,6 +52,7 @@ class X(RetrievalIntegration):
     async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         session = kwargs.get("session")
         max_video_size = kwargs.get("max_video_size")
+        url = self._normalize(url)
         tweet_id = extract_tweet_id_from_url(url)
         try:
             if tweet_id:
@@ -64,6 +64,13 @@ class X(RetrievalIntegration):
         except TooManyRequests:
             raise RateLimitError("X API rate limit reached.")
 
+    def _normalize(self, url: str) -> str:
+        """Turns URLs of the form https://publish.twitter.com/?query=...
+        into the bare Twitter URL."""
+        if url.startswith("https://publish.twitter.com/?query="):
+            query = urlparse(url).query
+            return parse_qs(query).get("query", [])[0] or url
+        return url
 
     async def _get_tweet(self, tweet_id: int, session: aiohttp.ClientSession, max_video_size: int = None) -> Optional[MultimodalSequence]:
         """Returns a MultimodalSequence containing the tweet's text and media

{scrapemm-0.4.3 → scrapemm-0.4.4}/scrapemm/retrieval.py

@@ -7,13 +7,28 @@ import aiohttp
 from ezmm import MultimodalSequence, download_item
 
 from scrapemm.common import ScrapingResponse
-from scrapemm.common.exceptions import IPBannedError
+from scrapemm.common.exceptions import IPBannedError, UnsupportedDomainError
 from scrapemm.integrations import retrieve_via_integration, fire, decodo
-from scrapemm.util import run_with_semaphore
+from scrapemm.util import run_with_semaphore, get_domain
 
 logger = logging.getLogger("scrapeMM")
 METHODS = ["integrations", "firecrawl", "decodo"]
 
+UNSUPPORTED_DOMAINS = [
+    "archive.today",
+    "archive.is",
+    "archive.ph",
+    "archive.vn",
+    "archive.li",
+    "archive.fo",
+    "archive.md",
+    "perma.cc",
+    "ghostarchive.org",
+    "archive.org",
+    "mvau.lt",
+    "archive.st",
+]
+
 
 async def retrieve(
         urls: str | Collection[str],
@@ -103,6 +118,10 @@ async def _retrieve_single(
 ) -> ScrapingResponse:
     logger.debug(f"Retrieving {url}")
 
+    if get_domain(url) in UNSUPPORTED_DOMAINS:
+        return ScrapingResponse(url=url, content=None,
+                                errors=dict(scrapemm=UnsupportedDomainError("Unsupported domain.")))
+
     if methods is None:
         methods = METHODS.copy()
 

{scrapemm-0.4.3 → scrapemm-0.4.4}/scrapemm/secrets.py

@@ -25,7 +25,8 @@ SECRETS = {
     "tiktok_client_secret": "TikTok client secret",
     "decodo_username": "Decodo Web Scraping API username",
     "decodo_password": "Decodo Web Scraping API password",
-    "youtube_cookie": "YouTube cookie string"
+    "youtube_cookie": "YouTube cookie string",
+    "facebook_cookie": "Facebook cookie string",
 }
 
 SALT = b'\xa4\x93\xf1\x88\x13\x88'
@@ -142,7 +143,7 @@ def override_secret(key_name: str):
     """Prompts the user to enter a new value for the given secret key. Does nothing
     when nothing entered."""
     description = SECRETS[key_name]
-    if key_name == "youtube_cookie":
+    if key_name in ["youtube_cookie", "facebook_cookie"]:
         user_input = get_multiline_user_input(f"Please enter the {description}. Hit Ctrl-D or Ctrl-Z to save it.")
     else:
         user_input = getpass(f"Please enter the {description} (leave empty to skip): ", stream=sys.stdout)

{scrapemm-0.4.3 → scrapemm-0.4.4}/scripts/example.py

@@ -2,7 +2,7 @@ from scrapemm import retrieve
 import asyncio
 
 if __name__ == "__main__":
-    url = "https://x.com/Shayan86/status/1891673533601780022"
+    url = "https://www.facebook.com/login/?next=https%3A%2F%2Fwww.facebook.com%2Fphoto%3Ffbid%3D860758296160977%26set%3Da.513585680878242"
     result = asyncio.run(retrieve(url))
     if result.errors:
         print(result.errors)

scrapemm-0.4.3/scrapemm/common/exceptions.py

@@ -1,8 +0,0 @@
-class RateLimitError(Exception):
-    pass
-
-class IPBannedError(Exception):
-    pass
-
-class ContentNotFoundError(Exception):
-    pass