scrapeMM 0.2.1__tar.gz → 0.3.0__tar.gz

{scrapemm-0.2.1 → scrapemm-0.3.0}/.gitignore

@@ -3,4 +3,5 @@
 /scrapeMM.egg-info/
 /dist/
 /config.yaml
-__pycache__/
+__pycache__/
+.DS_Store

{scrapemm-0.2.1/scrapeMM.egg-info → scrapemm-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.2.1
+Version: 0.3.0
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -46,14 +46,15 @@ URL (string)   -->   retrieve()   -->   MultimodalSequence
 ```
 The `MultimodalSequence` is a sequence of Markdown-formatted text and media provided by the [ezMM](https://github.com/multimodal-ai-lab/ezmm) library.
 
-Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
+Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) and [Decodo](https://decodo.com/).
 
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
+- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ⚠️ Instagram (done for videos but not for images yet)
+- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit
-- ⏳ Facebook
-- ⏳ Instagram

{scrapemm-0.2.1 → scrapemm-0.3.0}/README.md

@@ -23,14 +23,15 @@ URL (string)   -->   retrieve()   -->   MultimodalSequence
 ```
 The `MultimodalSequence` is a sequence of Markdown-formatted text and media provided by the [ezMM](https://github.com/multimodal-ai-lab/ezmm) library.
 
-Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
+Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) and [Decodo](https://decodo.com/).
 
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
+- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ⚠️ Instagram (done for videos but not for images yet)
+- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit
-- ⏳ Facebook
-- ⏳ Instagram

{scrapemm-0.2.1 → scrapemm-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.2.1"
+version = "0.3.0"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"
@@ -24,7 +24,7 @@ Homepage = "https://github.com/multimodal-ai-lab/scrapeMM"
 Issues = "https://github.com/multimodal-ai-lab/scrapeMM"
 
 [build-system]
-requires = ["setuptools>=60", "setuptools-scm>=8.0"]
+requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools.packages.find]

{scrapemm-0.2.1 → scrapemm-0.3.0}/requirements.txt

@@ -2,7 +2,7 @@ pytest~=8.4.1
 aiohttp~=3.12.13
 numpy~=1.26.4
 requests~=2.32.4
-ezMM~=0.3.1
+ezMM~=0.3.4
 Telethon~=1.40.0
 PyYAML~=6.0.1
 tweepy~=4.16.0
@@ -15,4 +15,5 @@ platformdirs~=4.3.8
 atproto
 TikTokResearchApi
 yt-dlp
-cryptography
+cryptography
+firecrawl-py~=4.6.0

{scrapemm-0.2.1 → scrapemm-0.3.0/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.2.1
+Version: 0.3.0
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -46,14 +46,15 @@ URL (string)   -->   retrieve()   -->   MultimodalSequence
 ```
 The `MultimodalSequence` is a sequence of Markdown-formatted text and media provided by the [ezMM](https://github.com/multimodal-ai-lab/ezmm) library.
 
-Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
+Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) and [Decodo](https://decodo.com/).
 
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
+- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ⚠️ Instagram (done for videos but not for images yet)
+- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit
-- ⏳ Facebook
-- ⏳ Instagram

{scrapemm-0.2.1 → scrapemm-0.3.0}/scrapeMM.egg-info/SOURCES.txt

@@ -16,13 +16,18 @@ scrapemm/util.py
 scrapemm/integrations/__init__.py
 scrapemm/integrations/base.py
 scrapemm/integrations/bluesky.py
+scrapemm/integrations/fb.py
+scrapemm/integrations/instagram.py
 scrapemm/integrations/telegram.py
 scrapemm/integrations/tiktok.py
 scrapemm/integrations/x.py
+scrapemm/integrations/youtube.py
 scrapemm/scraping/__init__.py
+scrapemm/scraping/decodo.py
 scrapemm/scraping/firecrawl.py
 scrapemm/scraping/no_bot_domains.txt
 scrapemm/scraping/util.py
+scrapemm/scraping/ytdlp.py
 scripts/example.py
 testing/test_retrieval.py
 testing/test_utils.py

{scrapemm-0.2.1 → scrapemm-0.3.0}/scrapemm/integrations/__init__.py

@@ -1,8 +1,3 @@
-import logging
-
-logger = logging.getLogger("scrapeMM")
-logger.debug("Starting integrations...")
-
 from typing import Optional
 
 import aiohttp
@@ -10,19 +5,22 @@ from ezmm import MultimodalSequence
 
 from scrapemm.util import get_domain
 from .bluesky import Bluesky
+from .fb import Facebook
+from .instagram import Instagram
 from .telegram import Telegram
 from .tiktok import TikTok
 from .x import X
+from .youtube import YouTube
 
-RETRIEVAL_INTEGRATIONS = [X(), Telegram(), Bluesky(), TikTok()]
+RETRIEVAL_INTEGRATIONS = [X(), Telegram(), Bluesky(), TikTok(), Instagram(), Facebook(), YouTube()]
 DOMAIN_TO_INTEGRATION = {domain: integration
                          for integration in RETRIEVAL_INTEGRATIONS
-                         for domain in integration.domains
-                         if integration.connected}
+                         for domain in integration.domains}
 
 
 async def retrieve_via_integration(url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
     domain = get_domain(url)
     if domain in DOMAIN_TO_INTEGRATION:
         integration = DOMAIN_TO_INTEGRATION[domain]
-        return await integration.get(url, session)
+        if integration.connected or integration.connected is None:
+            return await integration.get(url, session)

scrapemm-0.3.0/scrapemm/integrations/base.py

@@ -0,0 +1,39 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+import aiohttp
+from ezmm import MultimodalSequence
+
+from scrapemm.common import logger
+from scrapemm.util import get_domain
+
+
+class RetrievalIntegration(ABC):
+    """Any integration used to retrieve information via an external API. Typically
+     used when direct URL scraping is not possible."""
+
+    name: str
+    domains: list[str]  # The domains supported by this integration
+    connected: bool = None
+
+    @abstractmethod
+    async def _connect(self):
+        """Establish a connection to the external service. Invoked upon the first get() call.
+        Must set self.connet = True if connection was successful, else False."""
+        raise NotImplementedError
+
+    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        """Ensures connectivity before invoking the service for retrieval."""
+        assert get_domain(url) in self.domains
+        if self.connected is None:
+            await self._connect()
+            if not self.connected:
+                logger.warning(f"❌ Connection to {self.name} service could not be established.")
+        if self.connected:
+            logger.debug(f"Calling {self.name} service for {url}")
+            return await self._get(url, session)
+
+    @abstractmethod
+    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        """Retrieves the contents present at the given URL."""
+        raise NotImplementedError

{scrapemm-0.2.1 → scrapemm-0.3.0}/scrapemm/integrations/bluesky.py

@@ -2,8 +2,6 @@ import logging
 from typing import Optional
 
 import aiohttp
-from atproto_client.exceptions import RequestErrorBase
-from atproto_client.models.common import XrpcError
 from ezmm import MultimodalSequence, download_image, download_video
 
 from scrapemm.secrets import get_secret
@@ -14,9 +12,10 @@ logger = logging.getLogger("scrapeMM")
 
 
 class Bluesky(RetrievalIntegration):
+    name = "Bluesky"
     domains = ["bsky.app"]
 
-    def __init__(self):
+    async def _connect(self):
         self.username = get_secret("bluesky_username")
         self.password = get_secret("bluesky_password")
 
@@ -27,14 +26,9 @@ class Bluesky(RetrievalIntegration):
 
         from atproto import Client
         self.client = Client()
-        self.authenticated = False
         self._authenticate()
 
-    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
-        if not self.authenticated:
-            logger.error("❌ Not authenticated with Bluesky. Cannot retrieve content.")
-            return None
-
+    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Bluesky: {get_domain(url)}")
             return None
@@ -169,7 +163,7 @@ Metrics:
         """Authenticate with Bluesky using provided credentials."""
         try:
             self.client.login(self.username, self.password)
-            self.authenticated = True
+            self.connected = True
             logger.info(f"✅ Successfully authenticated with Bluesky as {self.username}")
             return True
         except Exception as e:
@@ -221,12 +215,14 @@ Metrics:
             return handle  # Return the handle itself as fallback
 
 
-def error_to_string(error: RequestErrorBase | Exception) -> str:
+def error_to_string(error: Exception) -> str:
     """Takes an Error object containing a response and prints the contents."""
+    from atproto_client.exceptions import RequestErrorBase
     if isinstance(error, RequestErrorBase):
         response = error.response
         code = response.status_code
         content = response.content
+        from atproto_client.models.common import XrpcError
         if isinstance(content, XrpcError):
             error_type = content.error
             msg = content.message

scrapemm-0.3.0/scrapemm/integrations/fb.py

@@ -0,0 +1,97 @@
+import logging
+from urllib.parse import urlparse
+import aiohttp
+from ezmm import MultimodalSequence, download_image
+
+from scrapemm.scraping.ytdlp import check_ytdlp_available,  get_video_with_ytdlp
+from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.util import get_domain
+
+logger = logging.getLogger("scrapeMM")
+
+
+class Facebook(RetrievalIntegration):
+    name = "Facebook"
+    domains = ["facebook.com", "www.facebook.com"]
+
+    async def _connect(self):
+        # Check if yt-dlp is available
+        self.ytdlp_available = check_ytdlp_available()
+
+        if self.ytdlp_available:
+            self.connected = True
+            mode = "yt-dlp only"
+            logger.info(f"✅ Facebook integration ready ({mode} mode).")
+        else:
+            self.connected = False
+            logger.warning("❌ Facebook integration not available: Neither API credentials nor yt-dlp found.")
+
+    async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook post URL."""
+        if get_domain(url) not in self.domains:
+            logger.error(f"❌ Invalid domain for Facebook: {get_domain(url)}")
+            return None
+        
+        # Determine if this is a video or profile URL
+        if self._is_video_url(url):
+            return await self._get_video(url, session)
+        elif self._is_photo_url(url):
+            return await self._get_photo(url, session)
+        else:
+            return await self._get_user_profile(url, session)
+        
+    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a TikTok video URL."""
+
+        # Fallback to yt-dlp mode
+        if self.ytdlp_available:
+            return await get_video_with_ytdlp(url, session, platform="Facebook")
+
+        logger.error("❌ No available method to retrieve Facebook video.")
+        return None
+    
+    async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook photo URL."""
+        logger.error("❌ No available method to retrieve Facebook photo.")
+        return None
+
+    async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook user profile URL."""
+        username = self._extract_username(url)
+        if username:
+            text = f"""**Facebook Profile**
+Username: @{username}
+URL: {url}
+
+Note: Profile details require Facebook API access.
+Configure API credentials for full profile information."""
+            return MultimodalSequence([text])
+        
+        return None
+
+    def _is_video_url(self, url: str) -> bool:
+        """Checks if the URL is a Facebook video URL."""
+        # video URLS are in the format: https://www.facebook.com/watch?v=VIDEO_ID
+        return "facebook.com/watch" in url
+    
+    def _extract_video_id(self, url: str) -> str:
+        """Extracts the video ID from a Facebook video URL."""
+        parsed_url = urlparse(url)
+        query_params = parsed_url.query
+        for param in query_params.split('&'):
+            if param.startswith('v='):
+                return param.split('=')[1]
+        return ""
+    
+    def _is_photo_url(self, url: str) -> bool:
+        """Checks if the URL is a Facebook photo URL."""
+        return "facebook.com/photo" in url or "facebook.com/photos" in url
+    
+    def _extract_username(self, url: str) -> str:
+        """Extracts the username from a Facebook profile URL."""
+        # url format: https://www.facebook.com/username<?...>
+        parsed_url = urlparse(url)
+        path_parts = parsed_url.path.strip('/').split('/')
+        if len(path_parts) > 0:
+            return path_parts[0]
+        return ""

scrapemm-0.3.0/scrapemm/integrations/instagram.py

@@ -0,0 +1,84 @@
+import logging
+from urllib.parse import urlparse
+import aiohttp
+from ezmm import MultimodalSequence, download_image
+
+from scrapemm.scraping.ytdlp import check_ytdlp_available,  get_video_with_ytdlp
+from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.util import get_domain
+
+logger = logging.getLogger("scrapeMM")
+
+
+class Instagram(RetrievalIntegration):
+    name = "Instagram"
+    domains = ["instagram.com", "www.instagram.com"]
+
+    async def _connect(self):
+        # Check if yt-dlp is available
+        self.ytdlp_available = check_ytdlp_available()
+
+        if self.ytdlp_available:
+            self.connected = True
+            mode = "yt-dlp only"
+            logger.info(f"✅ Instagram integration ready ({mode} mode).")
+        else:
+            self.connected = False
+            logger.warning("❌ Instagram integration not available: Neither API credentials nor yt-dlp found.")
+
+    async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from an Instagram post URL."""
+        if get_domain(url) not in self.domains:
+            logger.error(f"❌ Invalid domain for Instagram: {get_domain(url)}")
+            return None
+        
+        # Determine if this is a video or profile URL
+        if self._is_video_url(url):
+            return await self._get_video(url, session)
+        elif self._is_photo_url(url):
+            return await self._get_photo(url, session)
+        else:
+            return await self._get_user_profile(url, session)
+        
+    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from an Instagram video URL."""
+        if self.ytdlp_available:
+            return await get_video_with_ytdlp(url, session, platform="Instagram")
+
+        logger.error("❌ No available method to retrieve Instagram video.")
+        return None
+    
+    async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from an Instagram photo URL."""
+        logger.error("❌ No available method to retrieve Instagram photo.")
+        return None
+    
+    async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from an Instagram user profile URL."""
+        username = self._extract_username(url)
+        if username:
+            text = f"""**Instagram Profile**
+Username: @{username}
+URL: {url}
+
+Note: Profile details require Instagram API access.
+Configure API credentials for full profile information."""
+            return MultimodalSequence([text])
+
+        return None
+
+    def _is_video_url(self, url: str) -> bool:
+        """Checks if the URL is an Instagram video URL."""
+        return "instagram.com/reels" in url or "instagram.com/reel/" in url
+
+    def _is_photo_url(self, url: str) -> bool:
+        """Checks if the URL is an Instagram photo URL."""
+        return "instagram.com/p/" in url
+
+    def _extract_username(self, url: str) -> str:
+        """Extracts the username from an Instagram profile URL."""
+        parsed_url = urlparse(url)
+        path_parts = parsed_url.path.strip('/').split('/')
+        if len(path_parts) > 0:
+            return path_parts[0]
+        return ""

{scrapemm-0.2.1 → scrapemm-0.3.0}/scrapemm/integrations/telegram.py

@@ -1,6 +1,8 @@
 import logging
+from pathlib import Path
 from typing import Optional
 from urllib.parse import urlparse
+import sqlite3
 
 import aiohttp
 from ezmm import MultimodalSequence, Image, Item, Video
@@ -17,23 +19,30 @@ logger = logging.getLogger("scrapeMM")
 class Telegram(RetrievalIntegration):
     """The Telegram integration for retrieving post contents from Telegram channels and groups."""
 
+    name = "Telegram"
     domains = ["t.me", "telegram.me"]
     session_path = "temp/telegram"
 
-    def __init__(self):
+    async def _connect(self):
         api_id = int(get_secret("telegram_api_id")) if get_secret("telegram_api_id") else None
         api_hash = get_secret("telegram_api_hash")
         bot_token = get_secret("telegram_bot_token")
 
         if api_id and api_hash and bot_token:
             self.client = TelegramClient(self.session_path, api_id, api_hash)
-            self.client.start(bot_token=bot_token)
+            try:
+                self.client.start(bot_token=bot_token)
+            except sqlite3.OperationalError:  # Database is locked from an interrupted previous session
+                # Remove the database file and try again
+                journal_path = Path(self.session_path + ".session-journal")
+                journal_path.unlink(missing_ok=True)
             self.connected = True
             logger.info("✅ Successfully connected to Telegram.")
         else:
+            self.connected = False
             logger.warning("❌ Telegram integration not configured: Missing API keys.")
 
-    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
         """Retrieves content from a Telegram post URL."""
         assert get_domain(url) in self.domains
 
@@ -92,8 +101,7 @@ Forwards: {message.forwards}{reactions_text}
             return MultimodalSequence(text)
 
         except Exception as e:
-            print(f"Error retrieving Telegram content: {e}")
-            # raise
+            logger.warning(f"Error retrieving Telegram content: {e}")
             return None
 
     async def _get_media_from_message(self, chat, original_post, max_amp=10) -> list[Item]: