scrapeMM 0.3.3__tar.gz → 0.3.5__tar.gz

{scrapemm-0.3.3 → scrapemm-0.3.5}/.gitignore

@@ -4,4 +4,7 @@
 /dist/
 /config.yaml
 __pycache__/
-.DS_Store
+.DS_Store
+build/
+.pytest_cache/
+.claude/

{scrapemm-0.3.3/scrapeMM.egg-info → scrapemm-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.3
+Version: 0.3.5
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -54,8 +54,8 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) a
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
-- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ✅ YouTube (working sometimes)
+- ⚠️ Facebook (done for videos but not for images yet)
 - ⚠️ Instagram (done for videos but not for images yet)
-- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit

{scrapemm-0.3.3 → scrapemm-0.3.5}/README.md

@@ -30,8 +30,8 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) a
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
-- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ✅ YouTube (working sometimes)
+- ⚠️ Facebook (done for videos but not for images yet)
 - ⚠️ Instagram (done for videos but not for images yet)
-- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit

{scrapemm-0.3.3 → scrapemm-0.3.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.3.3"
+version = "0.3.5"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"

{scrapemm-0.3.3 → scrapemm-0.3.5}/requirements.txt

@@ -2,7 +2,7 @@ pytest~=8.4.1
 aiohttp~=3.12.13
 numpy~=1.26.4
 requests~=2.32.4
-ezMM~=0.3.4
+ezMM~=0.3.9
 Telethon~=1.40.0
 PyYAML~=6.0.1
 tweepy~=4.16.0

{scrapemm-0.3.3 → scrapemm-0.3.5/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.3
+Version: 0.3.5
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -54,8 +54,8 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) a
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
-- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ✅ YouTube (working sometimes)
+- ⚠️ Facebook (done for videos but not for images yet)
 - ⚠️ Instagram (done for videos but not for images yet)
-- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit

{scrapemm-0.3.3 → scrapemm-0.3.5}/scrapemm/integrations/__init__.py

@@ -18,9 +18,9 @@ DOMAIN_TO_INTEGRATION = {domain: integration
                          for domain in integration.domains}
 
 
-async def retrieve_via_integration(url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+async def retrieve_via_integration(url: str, **kwargs) -> Optional[MultimodalSequence]:
     domain = get_domain(url)
     if domain in DOMAIN_TO_INTEGRATION:
         integration = DOMAIN_TO_INTEGRATION[domain]
         if integration.connected or integration.connected is None:
-            return await integration.get(url, session)
+            return await integration.get(url, **kwargs)

{scrapemm-0.3.3 → scrapemm-0.3.5}/scrapemm/integrations/base.py

@@ -1,7 +1,6 @@
 from abc import ABC, abstractmethod
 from typing import Optional
 
-import aiohttp
 from ezmm import MultimodalSequence
 
 from scrapemm.common import logger
@@ -22,7 +21,7 @@ class RetrievalIntegration(ABC):
         Must set self.connet = True if connection was successful, else False."""
         raise NotImplementedError
 
-    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         """Ensures connectivity before invoking the service for retrieval."""
         assert get_domain(url) in self.domains
         if self.connected is None:
@@ -31,9 +30,9 @@ class RetrievalIntegration(ABC):
                 logger.warning(f"❌ Connection to {self.name} service could not be established.")
         if self.connected:
             logger.debug(f"Calling {self.name} service for {url}")
-            return await self._get(url, session)
+            return await self._get(url, **kwargs)
 
     @abstractmethod
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         """Retrieves the contents present at the given URL."""
         raise NotImplementedError

{scrapemm-0.3.3 → scrapemm-0.3.5}/scrapemm/integrations/bluesky.py

@@ -28,11 +28,12 @@ class Bluesky(RetrievalIntegration):
         self.client = Client()
         self._authenticate()
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Bluesky: {get_domain(url)}")
             return None
 
+        session = kwargs.get("session")
         if "post" in url:
             result = await self._retrieve_post(url, session)
         else:

{scrapemm-0.3.3 → scrapemm-0.3.5}/scrapemm/integrations/fb.py

@@ -1,79 +1,68 @@
 import logging
+import re
 from urllib.parse import urlparse
+
 import aiohttp
-from ezmm import MultimodalSequence, download_image
+from ezmm import MultimodalSequence
 
-from scrapemm.scraping.ytdlp import check_ytdlp_available,  get_video_with_ytdlp
 from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.scraping.ytdlp import get_content_with_ytdlp
 from scrapemm.util import get_domain
 
 logger = logging.getLogger("scrapeMM")
 
+VIDEO_URL_REGEX = r"facebook\.com/\d+/videos/\d+/?"
+
 
 class Facebook(RetrievalIntegration):
     name = "Facebook"
     domains = ["facebook.com", "www.facebook.com"]
 
     async def _connect(self):
-        # Check if yt-dlp is available
-        self.ytdlp_available = check_ytdlp_available()
-
-        if self.ytdlp_available:
-            self.connected = True
-            mode = "yt-dlp only"
-            logger.info(f"✅ Facebook integration ready ({mode} mode).")
-        else:
-            self.connected = False
-            logger.warning("❌ Facebook integration not available: Neither API credentials nor yt-dlp found.")
+        self.api_available = False  # TODO
+        logger.info(f"✅ Facebook integration ready (yt-dlp only mode).")
+        self.connected = True
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook post URL."""
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Facebook: {get_domain(url)}")
             return None
-        
-        # Determine if this is a video or profile URL
+
+        # Determine if this is a video or photo URL, act accordingly
         if self._is_video_url(url):
-            return await self._get_video(url, session)
+            return await self._get_video(url, **kwargs)
         elif self._is_photo_url(url):
-            return await self._get_photo(url, session)
-        else:
-            return await self._get_user_profile(url, session)
-        
-    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
-        """Retrieves content from a TikTok video URL."""
+            return await self._get_photo(url, **kwargs)
 
-        # Fallback to yt-dlp mode
-        if self.ytdlp_available:
-            return await get_video_with_ytdlp(url, session, platform="Facebook")
+        # The URL is not indicative, so try all methods
+        return (await self._get_video(url, **kwargs) or
+                await self._get_photo(url, **kwargs) or
+                await self._get_user_profile(url, **kwargs))
 
-        logger.error("❌ No available method to retrieve Facebook video.")
-        return None
-    
-    async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get_video(self, url: str, **kwargs) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook video URL."""
+        if self.api_available:
+            raise NotImplementedError
+        else:
+            return await get_content_with_ytdlp(url, platform="Facebook", **kwargs)
+
+    async def _get_photo(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook photo URL."""
         logger.error("❌ No available method to retrieve Facebook photo.")
         return None
 
-    async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get_user_profile(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook user profile URL."""
-        username = self._extract_username(url)
-        if username:
-            text = f"""**Facebook Profile**
-Username: @{username}
-URL: {url}
-
-Note: Profile details require Facebook API access.
-Configure API credentials for full profile information."""
-            return MultimodalSequence([text])
-        
+        logger.error("❌ No available method to retrieve Facebook profiles.")
         return None
 
     def _is_video_url(self, url: str) -> bool:
         """Checks if the URL is a Facebook video URL."""
         # video URLS are in the format: https://www.facebook.com/watch?v=VIDEO_ID
-        return "facebook.com/watch" in url
-    
+        # or Reels: https://www.facebook.com/reel/REEL_ID
+        return "facebook.com/watch" in url or "facebook.com/reel" in url or bool(re.search(VIDEO_URL_REGEX, url))
+
     def _extract_video_id(self, url: str) -> str:
         """Extracts the video ID from a Facebook video URL."""
         parsed_url = urlparse(url)
@@ -82,11 +71,11 @@ Configure API credentials for full profile information."""
             if param.startswith('v='):
                 return param.split('=')[1]
         return ""
-    
+
     def _is_photo_url(self, url: str) -> bool:
         """Checks if the URL is a Facebook photo URL."""
         return "facebook.com/photo" in url or "facebook.com/photos" in url
-    
+
     def _extract_username(self, url: str) -> str:
         """Extracts the username from a Facebook profile URL."""
         # url format: https://www.facebook.com/username<?...>

{scrapemm-0.3.3 → scrapemm-0.3.5}/scrapemm/integrations/instagram.py

@@ -1,10 +1,11 @@
 import logging
 from urllib.parse import urlparse
+
 import aiohttp
-from ezmm import MultimodalSequence, download_image
+from ezmm import MultimodalSequence
 
-from scrapemm.scraping.ytdlp import check_ytdlp_available,  get_video_with_ytdlp
 from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.scraping.ytdlp import get_content_with_ytdlp
 from scrapemm.util import get_domain
 
 logger = logging.getLogger("scrapeMM")
@@ -15,45 +16,42 @@ class Instagram(RetrievalIntegration):
     domains = ["instagram.com", "www.instagram.com"]
 
     async def _connect(self):
-        # Check if yt-dlp is available
-        self.ytdlp_available = check_ytdlp_available()
-
-        if self.ytdlp_available:
-            self.connected = True
-            mode = "yt-dlp only"
-            logger.info(f"✅ Instagram integration ready ({mode} mode).")
-        else:
-            self.connected = False
-            logger.warning("❌ Instagram integration not available: Neither API credentials nor yt-dlp found.")
+        self.api_available = False
+        logger.info(f"✅ Instagram integration ready (yt-dlp only mode).")
+        self.connected = True
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from an Instagram post URL."""
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Instagram: {get_domain(url)}")
             return None
-        
+
         # Determine if this is a video or profile URL
         if self._is_video_url(url):
-            return await self._get_video(url, session)
+            return await self._get_video(url, **kwargs)
         elif self._is_photo_url(url):
-            return await self._get_photo(url, session)
+            # /p/ URLs can also be reels, so try both
+            content = await self._get_video(url, **kwargs)
+            if content and content.has_videos():
+                return content
+            else:
+                return await self._get_photo(url, **kwargs)
         else:
-            return await self._get_user_profile(url, session)
-        
-    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+            return await self._get_user_profile(url, **kwargs)
+
+    async def _get_video(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from an Instagram video URL."""
-        if self.ytdlp_available:
-            return await get_video_with_ytdlp(url, session, platform="Instagram")
+        if self.api_available:
+            raise NotImplementedError
+        else:
+            return await get_content_with_ytdlp(url, platform="Instagram", **kwargs)
 
-        logger.error("❌ No available method to retrieve Instagram video.")
+    async def _get_photo(self, url: str, **kwargs) -> MultimodalSequence | None:
+        """Retrieves content from an Instagram photo URL (can also be a reel)."""
+        logger.warning("❌ Native Instagram photo download not yet supported. Use Decodo for that.")
         return None
-    
-    async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
-        """Retrieves content from an Instagram photo URL."""
-        logger.error("❌ No available method to retrieve Instagram photo.")
-        return None
-    
-    async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+
+    async def _get_user_profile(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from an Instagram user profile URL."""
         username = self._extract_username(url)
         if username:

{scrapemm-0.3.3 → scrapemm-0.3.5}/scrapemm/integrations/telegram.py

@@ -42,7 +42,7 @@ class Telegram(RetrievalIntegration):
             self.connected = False
             logger.warning("❌ Telegram integration not configured: Missing API keys.")
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         """Retrieves content from a Telegram post URL."""
         assert get_domain(url) in self.domains
 

{scrapemm-0.3.3 → scrapemm-0.3.5}/scrapemm/integrations/tiktok.py

@@ -15,7 +15,6 @@ from ezmm.common.items import Video, Image
 from tiktok_research_api import TikTokResearchAPI, QueryVideoRequest, QueryUserInfoRequest, Criteria, Query
 
 from scrapemm.integrations.base import RetrievalIntegration
-from scrapemm.scraping.ytdlp import check_ytdlp_available
 from scrapemm.secrets import get_secret
 
 logger = logging.getLogger("scrapeMM")
@@ -45,6 +44,7 @@ class TikTok(RetrievalIntegration):
 
         if client_key and client_secret:
             try:
+                # TODO: Replace with async version
                 self.api = TikTokResearchAPI(
                     client_key=client_key,
                     client_secret=client_secret,
@@ -57,18 +57,13 @@ class TikTok(RetrievalIntegration):
             except Exception as e:
                 logger.info(f"⚠️ TikTok Research API connection failed: {e}. Using fallback mode.")
 
-        # Check if yt-dlp is available
-        self.ytdlp_available = check_ytdlp_available()
+        mode = "API" if self.api_available else "yt-dlp only"
+        logger.info(f"✅ TikTok integration ready ({mode} mode).")
+        self.connected = True
 
-        if self.api_available or self.ytdlp_available:
-            self.connected = True
-            mode = "API + yt-dlp" if self.api_available else "yt-dlp only"
-            logger.info(f"✅ TikTok integration ready ({mode} mode).")
-        else:
-            self.connected = False
-            logger.warning("❌ TikTok integration not available: Neither API credentials nor yt-dlp found.")
+    async def _get(self, url: str, **kwargs) -> MultimodalSequence | None:
+        session = kwargs.get('session')
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         # Determine if this is a video or profile URL
         if self._is_video_url(url):
             return await self._get_video(url, session)
@@ -80,19 +75,16 @@ class TikTok(RetrievalIntegration):
 
         # Try API mode first if available
         if self.api_available:
-            result = await self._get_video_with_api(url, session)
+            result = await self._get_video_with_api(url)
             if result:
                 return result
             logger.warning("API method failed, falling back to yt-dlp...")
 
         # Fallback to yt-dlp mode
-        if self.ytdlp_available:
+        else:
             return await self._get_video_with_ytdlp(url, session)
 
-        logger.error("❌ No available method to retrieve TikTok video.")
-        return None
-
-    async def _get_video_with_api(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get_video_with_api(self, url: str) -> MultimodalSequence | None:
         """Retrieves video using TikTok Research API."""
         video_id = self._extract_video_id(url)
         if not video_id:

{scrapemm-0.3.3 → scrapemm-0.3.5}/scrapemm/integrations/x.py

@@ -47,7 +47,8 @@ class X(RetrievalIntegration):
             self.connected = False
             logger.warning("❌ X (Twitter) integration not configured: Missing bearer token.")
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
+        session = kwargs.get("session")
         tweet_id = extract_tweet_id_from_url(url)
         if tweet_id:
             return await self._get_tweet(tweet_id, session)

scrapemm-0.3.5/scrapemm/integrations/youtube.py

@@ -0,0 +1,43 @@
+import logging
+from typing import Optional
+
+import aiohttp
+from ezmm import MultimodalSequence
+
+from scrapemm.scraping.ytdlp import get_content_with_ytdlp
+from .base import RetrievalIntegration
+from scrapemm.secrets import get_secret
+from ..common import CONFIG_DIR
+
+logger = logging.getLogger("scrapeMM")
+
+
+class YouTube(RetrievalIntegration):
+    """YouTube integration for downloading videos and shorts using yt-dlp.
+    YouTube is rate-limited to 333 videos per hour."""
+
+    name = "YouTube"
+    domains = [
+        "youtube.com",
+        "youtu.be",
+    ]
+    cookie_file = CONFIG_DIR / "youtube_cookie.txt"
+
+    async def _connect(self):
+        self.connected = True  # Connect always by default
+
+        cookie = get_secret("youtube_cookie")
+        if cookie:
+            # Save the cookie in a .txt file next to the secrets file
+            with open(self.cookie_file, "w") as f:
+                f.write(cookie)
+            logger.info(f"✅ Using cookie to connect to YouTube.")
+        else:
+            logger.warning(f"⚠️ Missing YouTube cookie. Won't be able to download videos, only thumbnails and metadata.")
+
+    async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
+        """Downloads YouTube video or short using yt-dlp."""
+        return await get_content_with_ytdlp(url,
+                                            platform="YouTube",
+                                            cookie_file=self.cookie_file.as_posix(),
+                                            **kwargs)

{scrapemm-0.3.3 → scrapemm-0.3.5}/scrapemm/retrieval.py

@@ -1,6 +1,6 @@
 import logging
-from typing import Optional, Collection
 from traceback import format_exc
+from typing import Optional, Collection
 
 import aiohttp
 from ezmm import MultimodalSequence, download_item
@@ -20,6 +20,7 @@ async def retrieve(
         actions: list[dict] = None,
         methods: list[str] = None,
         format: str = "multimodal_sequence",
+        max_video_size: int = None
 ) -> Optional[MultimodalSequence | str] | list[Optional[MultimodalSequence | str]]:
     """Main function of this repository. Downloads the contents present at the given URL(s).
     For each URL, returns a MultimodalSequence containing text, images, and videos.
@@ -41,6 +42,7 @@ async def retrieve(
     :param format: The format of the output. Available formats:
         - "multimodal_sequence" (MultimodalSequence containing parsed and downloaded media from the page)
         - "html" (string containing the raw HTML code of the page, not compatible with 'integrations' method)
+    :param max_video_size: Maximum size of videos to download, in MB. If None, no limit is applied.
     """
     if methods is None:
         methods = METHODS
@@ -66,7 +68,8 @@ async def retrieve(
         urls_unique = set(urls_to_retrieve)
 
         # Retrieve URLs concurrently
-        tasks = [_retrieve_single(url, remove_urls, session, methods, actions, format) for url in urls_unique]
+        tasks = [_retrieve_single(url, remove_urls, session, methods, actions, format, max_video_size) for url in
+                 urls_unique]
         results = await run_with_semaphore(tasks, limit=20, show_progress=show_progress and len(urls_to_retrieve) > 1,
                                            progress_description="Retrieving URLs...")
 
@@ -85,6 +88,7 @@ async def _retrieve_single(
         methods: list[str],
         actions: list[dict] = None,
         format: str = "multimodal_sequence",
+        max_video_size: int = None,
 ) -> Optional[MultimodalSequence | str]:
     try:
         # Ensure URL is a string
@@ -101,9 +105,9 @@ async def _retrieve_single(
 
         # Define available retrieval methods
         method_map = {
-            "integrations": lambda: retrieve_via_integration(url, session),
+            "integrations": lambda: retrieve_via_integration(url, session=session, max_video_size=max_video_size),
             "firecrawl": lambda: fire.scrape(url, remove_urls=remove_urls,
-                                                  session=session, format=format, actions=actions),
+                                             session=session, format=format, actions=actions),
             "decodo": lambda: decodo.scrape(url, remove_urls, session, format=format),
         }