scrapeMM 0.4.0__tar.gz → 0.4.2__tar.gz

{scrapemm-0.4.0/scrapeMM.egg-info → scrapemm-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.4.0
+Version: 0.4.2
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -50,12 +50,21 @@ The `MultimodalSequence` is a sequence of Markdown-formatted text and media prov
 Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) and [Decodo](https://decodo.com/).
 
 ## Supported Platforms
+### Social Media
 - ✅ X/Twitter
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
 - ✅ YouTube
-- (✅️) Instagram (works for most content)
-- ⚠️ Facebook (done for videos but not for images yet)
-- ⏳ Threads (TBD)
-- ⏳ Reddit (TBD)
+- (✅️) Instagram: works for most content
+- ⏳ Facebook: done for videos but not for images yet
+- ❌ Threads: TBD
+- ❌ Reddit: TBD
+
+### Archiving Services
+- ❌ Perma.cc
+- ❌ Archive.today
+- ❌ Wayback Machine, Internet Archive (web.archive.org)
+- ❌ AwesomeScreenshot.com
+- ⏳ MediaVault (mvau.lt): Works for images but not for videos yet
+- ❌ Ghost Archive (ghostarchive.org)

{scrapemm-0.4.0 → scrapemm-0.4.2}/README.md

@@ -26,12 +26,21 @@ The `MultimodalSequence` is a sequence of Markdown-formatted text and media prov
 Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) and [Decodo](https://decodo.com/).
 
 ## Supported Platforms
+### Social Media
 - ✅ X/Twitter
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
 - ✅ YouTube
-- (✅️) Instagram (works for most content)
-- ⚠️ Facebook (done for videos but not for images yet)
-- ⏳ Threads (TBD)
-- ⏳ Reddit (TBD)
+- (✅️) Instagram: works for most content
+- ⏳ Facebook: done for videos but not for images yet
+- ❌ Threads: TBD
+- ❌ Reddit: TBD
+
+### Archiving Services
+- ❌ Perma.cc
+- ❌ Archive.today
+- ❌ Wayback Machine, Internet Archive (web.archive.org)
+- ❌ AwesomeScreenshot.com
+- ⏳ MediaVault (mvau.lt): Works for images but not for videos yet
+- ❌ Ghost Archive (ghostarchive.org)

{scrapemm-0.4.0 → scrapemm-0.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.4.0"
+version = "0.4.2"
 description = "Flexible multimodal scraper for social media and the open web."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -33,3 +33,9 @@ namespaces = false  # Disable implicit namespaces
 
 [tool.setuptools.package-data]
 "*" = ["*.txt"]  # Include all .txt files contained in any (sub)package in the build
+
+[tool.pytest.ini_options]
+testpaths = [
+    "testing/",
+]
+asyncio_default_fixture_loop_scope = "session"

{scrapemm-0.4.0 → scrapemm-0.4.2/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.4.0
+Version: 0.4.2
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -50,12 +50,21 @@ The `MultimodalSequence` is a sequence of Markdown-formatted text and media prov
 Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) and [Decodo](https://decodo.com/).
 
 ## Supported Platforms
+### Social Media
 - ✅ X/Twitter
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
 - ✅ YouTube
-- (✅️) Instagram (works for most content)
-- ⚠️ Facebook (done for videos but not for images yet)
-- ⏳ Threads (TBD)
-- ⏳ Reddit (TBD)
+- (✅️) Instagram: works for most content
+- ⏳ Facebook: done for videos but not for images yet
+- ❌ Threads: TBD
+- ❌ Reddit: TBD
+
+### Archiving Services
+- ❌ Perma.cc
+- ❌ Archive.today
+- ❌ Wayback Machine, Internet Archive (web.archive.org)
+- ❌ AwesomeScreenshot.com
+- ⏳ MediaVault (mvau.lt): Works for images but not for videos yet
+- ❌ Ghost Archive (ghostarchive.org)

{scrapemm-0.4.0 → scrapemm-0.4.2}/scrapeMM.egg-info/SOURCES.txt

@@ -13,6 +13,7 @@ scrapemm/retrieval.py
 scrapemm/secrets.py
 scrapemm/util.py
 scrapemm/common/__init__.py
+scrapemm/common/exceptions.py
 scrapemm/common/scraping_response.py
 scrapemm/integrations/__init__.py
 scrapemm/integrations/base.py

scrapemm-0.4.2/scrapemm/common/exceptions.py

@@ -0,0 +1,8 @@
+class RateLimitError(Exception):
+    pass
+
+class IPBannedError(Exception):
+    pass
+
+class ContentNotFoundError(Exception):
+    pass

{scrapemm-0.4.0 → scrapemm-0.4.2}/scrapemm/common/scraping_response.py

@@ -8,7 +8,7 @@ from ezmm import MultimodalSequence
 class ScrapingResponse:
     url: str  # The input URL that was scraped
     content: Optional[MultimodalSequence | str]  # The retrieved content. None if retrieval failed.
-    errors: dict[str, Optional[Exception]] = None  # The exceptions raised during retrieval for each method
+    errors: dict[str, Optional[Exception]] | None = None  # The exceptions raised during retrieval for each method
     method: Optional[str] = None  # The successful method used to retrieve the content
 
     @property

{scrapemm-0.4.0 → scrapemm-0.4.2}/scrapemm/integrations/base.py

@@ -15,7 +15,7 @@ class RetrievalIntegration(ABC):
 
     name: str
     domains: list[str]  # The domains supported by this integration
-    connected: bool = None
+    connected: bool | None = None
 
     @abstractmethod
     async def _connect(self):
@@ -25,7 +25,7 @@ class RetrievalIntegration(ABC):
 
     async def get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         """Ensures connectivity before invoking the service for retrieval."""
-        assert get_domain(url) in self.domains
+        assert get_domain(url) in self.domains, f"Invalid domain {get_domain(url)} for integration {self.name}."
         if self.connected is None:
             await self._connect()
             if not self.connected:

{scrapemm-0.4.0 → scrapemm-0.4.2}/scrapemm/integrations/bluesky.py

@@ -34,14 +34,20 @@ class Bluesky(RetrievalIntegration):
             return None
 
         session = kwargs.get("session")
+        max_video_size = kwargs.get("max_video_size")
         if "post" in url:
-            result = await self._retrieve_post(url, session)
+            result = await self._retrieve_post(url, session, max_video_size)
         else:
             result = await self._retrieve_profile(url, session)
 
         return result
 
-    async def _retrieve_post(self, url: str, session: aiohttp.ClientSession = None) -> Optional[MultimodalSequence]:
+    async def _retrieve_post(
+        self,
+        url: str,
+        session: aiohttp.ClientSession | None = None,
+        max_video_size: int | None = None
+    ) -> Optional[MultimodalSequence]:
         """Retrieve a post from the given Bluesky URL."""
         uri = self._construct_uri(url)
         if not uri:
@@ -94,7 +100,11 @@ class Bluesky(RetrievalIntegration):
                 elif hasattr(embed, 'py_type') and getattr(embed, 'py_type') == 'app.bsky.embed.video#view':
                     video = await download_video(embed.playlist, session)
                     if video:
-                        media.append(video)
+                        if max_video_size is None or video.size <= max_video_size:
+                            media.append(video)
+                        else:
+                            logger.info(f"Removing video {video.reference} because it exceeds the maximum size "
+                                        f"of {max_video_size / 1024 / 1024:.2f} MB.")
 
                         # Extract hashtags and mentions
             hashtags, mentions, external_links = [], [], []

{scrapemm-0.4.0 → scrapemm-0.4.2}/scrapemm/integrations/fb.py

@@ -15,7 +15,7 @@ VIDEO_URL_REGEX = r"facebook\.com/\d+/videos/\d+/?"
 
 class Facebook(RetrievalIntegration):
     name = "Facebook"
-    domains = ["facebook.com", "www.facebook.com"]
+    domains = ["facebook.com", "fb.watch"]
 
     async def _connect(self):
         self.api_available = False  # TODO
@@ -24,10 +24,6 @@ class Facebook(RetrievalIntegration):
 
     async def _get(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook post URL."""
-        if get_domain(url) not in self.domains:
-            logger.error(f"❌ Invalid domain for Facebook: {get_domain(url)}")
-            return None
-
         # Determine if this is a video or photo URL, act accordingly
         if self._is_video_url(url):
             return await self._get_video(url, **kwargs)
@@ -42,23 +38,26 @@ class Facebook(RetrievalIntegration):
     async def _get_video(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook video URL."""
         if self.api_available:
-            raise NotImplementedError("❌ Facebook video retrieval through API not yet supported.")
+            raise NotImplementedError("Facebook video retrieval through API not yet supported.")
         else:
             return await get_content_with_ytdlp(url, platform="Facebook", **kwargs)
 
     async def _get_photo(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook photo URL."""
-        raise NotImplementedError("❌ No available method to retrieve Facebook photos.")
+        raise NotImplementedError("No available method to retrieve Facebook photos.")
 
     async def _get_user_profile(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook user profile URL."""
-        raise NotImplementedError("❌ No available method to retrieve Facebook profiles.")
+        raise NotImplementedError("No available method to retrieve Facebook profiles.")
 
     def _is_video_url(self, url: str) -> bool:
         """Checks if the URL is a Facebook video URL."""
-        # video URLS are in the format: https://www.facebook.com/watch?v=VIDEO_ID
+        # video URLS are in the format: https://www.facebook.com/watch?v=VIDEO_ID or fb.watch/...
         # or Reels: https://www.facebook.com/reel/REEL_ID
-        return "facebook.com/watch" in url or "facebook.com/reel" in url or bool(re.search(VIDEO_URL_REGEX, url))
+        return ("facebook.com/watch" in url
+                or "facebook.com/reel" in url
+                or bool(re.search(VIDEO_URL_REGEX, url))
+                or "fb.watch" in url)
 
     def _extract_video_id(self, url: str) -> str:
         """Extracts the video ID from a Facebook video URL."""

{scrapemm-0.4.0 → scrapemm-0.4.2}/scrapemm/integrations/instagram.py

@@ -4,6 +4,7 @@ from urllib.parse import urlparse
 from ezmm import MultimodalSequence
 from yt_dlp import DownloadError
 
+from scrapemm.common.exceptions import RateLimitError
 from scrapemm.integrations.base import RetrievalIntegration
 from scrapemm.integrations.ytdlp import get_content_with_ytdlp
 from scrapemm.util import get_domain
@@ -22,16 +23,15 @@ class Instagram(RetrievalIntegration):
 
     async def _get(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from an Instagram post URL."""
-        if get_domain(url) not in self.domains:
-            logger.error(f"❌ Invalid domain for Instagram: {get_domain(url)}")
-            return None
-
         # Determine if this is a video or profile URL
         if self._is_video_url(url):
             return await self._get_video(url, **kwargs)
         elif self._is_photo_url(url):
             # /p/ URLs can also be reels, so try both
-            content = await self._get_video(url, **kwargs)
+            try:
+                content = await self._get_video(url, **kwargs)
+            except Exception:
+                content = None
             if content and content.has_videos():
                 return content
             else:
@@ -48,16 +48,13 @@ class Instagram(RetrievalIntegration):
                 return await get_content_with_ytdlp(url, platform="Instagram", **kwargs)
             except DownloadError as e:
                 if "rate-limit reached" in str(e):
-                    msg = "Instagram rate limit likely reached!"
-                    logger.error(msg)
-                    raise RuntimeError(msg) from e
+                    raise RateLimitError(f"Instagram rate limit likely reached: {e}")
                 else:
                     raise e
 
     async def _get_photo(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from an Instagram photo URL (can also be a reel)."""
-        logger.warning("❌ Native Instagram photo download not yet supported. Use Decodo for that.")
-        return None
+        raise NotImplementedError("Native Instagram photo download not yet supported. Use Decodo for that.")
 
     async def _get_user_profile(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from an Instagram user profile URL."""

{scrapemm-0.4.0 → scrapemm-0.4.2}/scrapemm/integrations/telegram.py

@@ -31,11 +31,12 @@ class Telegram(RetrievalIntegration):
         if api_id and api_hash and bot_token:
             self.client = TelegramClient(self.session_path, api_id, api_hash)
             try:
-                self.client.start(bot_token=bot_token)
+                await self.client.start(bot_token=bot_token)  # Returns a coroutine b/c event loop exists already
             except sqlite3.OperationalError:  # Database is locked from an interrupted previous session
                 # Remove the database file and try again
                 journal_path = Path(self.session_path + ".session-journal")
                 journal_path.unlink(missing_ok=True)
+                await self.client.start(bot_token=bot_token)  # Returns a coroutine b/c event loop exists already
             self.connected = True
             logger.info("✅ Successfully connected to Telegram.")
         else:
@@ -44,8 +45,6 @@ class Telegram(RetrievalIntegration):
 
     async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         """Retrieves content from a Telegram post URL."""
-        assert get_domain(url) in self.domains
-
         # Parse the URL to get channel/group name and post ID
         parsed = urlparse(url)
         path_parts = parsed.path.strip("/").split("/")
@@ -56,40 +55,44 @@ class Telegram(RetrievalIntegration):
         channel_name = path_parts[0]
         post_id = int(path_parts[1])
 
-        try:
-            # Get the message
-            channel = await self.client.get_entity(channel_name)
-            message = await self.client.get_messages(channel, ids=post_id)
-
-            if not message:
-                return None
-
-            # Handle media
-            media = await self._get_media_from_message(channel, message)
-
-            author = message.sender
-            author_type = type(author).__name__
-            if isinstance(author, Channel):
-                name = f'"{author.title}"'
-                if author.username:
-                    name += f" (@{author.username})"
-            elif isinstance(author, User):
-                if author.bot:
-                    author_type = "Bot"
-                name = f"{author.first_name} {author.last_name}" if author.last_name else author.first_name
-                if author.username:
-                    name += f" (@{author.username})"
-                if author.phone:
-                    name += f", Phone: {author.phone}"
-                if author.verified:
-                    name += " (Verified)"
-            else:
-                name = "Unknown"
-
-            edit_text = "\nEdit date: " + message.edit_date.strftime("%B %d, %Y at %H:%M") if message.edit_date else ""
-            reactions_text = "\nReactions: " + message.reactions.stringify() if message.reactions else ""
-
-            text = f"""**Telegram Post**
+        # Get the message
+        channel = await self.client.get_entity(channel_name)
+        if not channel:
+            return None
+
+        assert isinstance(channel, Channel), f"Can only retrieve posts from Telegram channels, but got {type(channel).__name__}."
+
+        message = await self.client.get_messages(channel, ids=post_id)
+        if not message:
+            return None
+
+        # Handle media
+        max_video_size = kwargs.get("max_video_size")
+        media = await self._get_media_from_message(channel, message, max_video_size=max_video_size)
+
+        author = message.sender
+        author_type = type(author).__name__
+        if isinstance(author, Channel):
+            name = f'"{author.title}"'
+            if author.username:
+                name += f" (@{author.username})"
+        elif isinstance(author, User):
+            if author.bot:
+                author_type = "Bot"
+            name = f"{author.first_name} {author.last_name}" if author.last_name else author.first_name
+            if author.username:
+                name += f" (@{author.username})"
+            if author.phone:
+                name += f", Phone: {author.phone}"
+            if author.verified:
+                name += " (Verified)"
+        else:
+            name = "Unknown"
+
+        edit_text = "\nEdit date: " + message.edit_date.strftime("%B %d, %Y at %H:%M") if message.edit_date else ""
+        reactions_text = "\nReactions: " + message.reactions.stringify() if message.reactions else ""
+
+        text = f"""**Telegram Post**
 Author: {author_type} {name}
 Date: {message.date.strftime("%B %d, %Y at %H:%M")}{edit_text}
 Views: {message.views}
@@ -98,13 +101,9 @@ Forwards: {message.forwards}{reactions_text}
 {' '.join(m.reference for m in media)}
 {message.text}"""
 
-            return MultimodalSequence(text)
-
-        except Exception as e:
-            logger.warning(f"Error retrieving Telegram content: {e}")
-            return None
+        return MultimodalSequence(text)
 
-    async def _get_media_from_message(self, chat, original_post, max_amp=10) -> list[Item]:
+    async def _get_media_from_message(self, chat, original_post, max_amp=10, max_video_size=None) -> list[Item]:
         """
         Searches for Telegram posts that are part of the same group of uploads.
         The search is conducted around the id of the original post with an amplitude
@@ -129,6 +128,10 @@ Forwards: {message.forwards}{reactions_text}
                         item = Image(binary_data=medium_bytes, source_url=post_url)
                     elif hasattr(medium, "video"):
                         item = Video(binary_data=medium_bytes, source_url=post_url)
+                        if max_video_size is not None and item.size > max_video_size:
+                            logger.info(f"Removing video {item.reference} because it exceeds the maximum size "
+                                        f"of {max_video_size / 1024 / 1024:.2f} MB.")
+                            item = None
                     else:
                         raise ValueError(f"Unsupported medium: {medium.__dict__}")
                     media.append(item)