scrapeMM 0.2.2__tar.gz → 0.3.1__tar.gz

{scrapemm-0.2.2 → scrapemm-0.3.1}/.gitignore

@@ -3,4 +3,5 @@
 /scrapeMM.egg-info/
 /dist/
 /config.yaml
-__pycache__/
+__pycache__/
+.DS_Store

{scrapemm-0.2.2/scrapeMM.egg-info → scrapemm-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.2.2
+Version: 0.3.1
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -19,6 +19,7 @@ Requires-Dist: atproto
 Requires-Dist: TikTokResearchApi
 Requires-Dist: yt-dlp
 Requires-Dist: cryptography
+Requires-Dist: firecrawl-py
 Dynamic: license-file
 
 # scrapeMM: Multimodal Web Retrieval
@@ -46,14 +47,15 @@ URL (string)   -->   retrieve()   -->   MultimodalSequence
 ```
 The `MultimodalSequence` is a sequence of Markdown-formatted text and media provided by the [ezMM](https://github.com/multimodal-ai-lab/ezmm) library.
 
-Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
+Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) and [Decodo](https://decodo.com/).
 
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
+- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ⚠️ Instagram (done for videos but not for images yet)
+- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit
-- ⏳ Facebook
-- ⏳ Instagram

{scrapemm-0.2.2 → scrapemm-0.3.1}/README.md

@@ -23,14 +23,15 @@ URL (string)   -->   retrieve()   -->   MultimodalSequence
 ```
 The `MultimodalSequence` is a sequence of Markdown-formatted text and media provided by the [ezMM](https://github.com/multimodal-ai-lab/ezmm) library.
 
-Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
+Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) and [Decodo](https://decodo.com/).
 
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
+- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ⚠️ Instagram (done for videos but not for images yet)
+- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit
-- ⏳ Facebook
-- ⏳ Instagram

{scrapemm-0.2.2 → scrapemm-0.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.2.2"
+version = "0.3.1"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"
@@ -17,6 +17,7 @@ dependencies = [
     "TikTokResearchApi",
     "yt-dlp",
     "cryptography",
+    "firecrawl-py"
 ]
 
 [project.urls]

{scrapemm-0.2.2 → scrapemm-0.3.1}/requirements.txt

@@ -2,7 +2,7 @@ pytest~=8.4.1
 aiohttp~=3.12.13
 numpy~=1.26.4
 requests~=2.32.4
-ezMM~=0.3.1
+ezMM~=0.3.4
 Telethon~=1.40.0
 PyYAML~=6.0.1
 tweepy~=4.16.0
@@ -15,4 +15,5 @@ platformdirs~=4.3.8
 atproto
 TikTokResearchApi
 yt-dlp
-cryptography
+cryptography
+firecrawl-py~=4.6.0

{scrapemm-0.2.2 → scrapemm-0.3.1/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.2.2
+Version: 0.3.1
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -19,6 +19,7 @@ Requires-Dist: atproto
 Requires-Dist: TikTokResearchApi
 Requires-Dist: yt-dlp
 Requires-Dist: cryptography
+Requires-Dist: firecrawl-py
 Dynamic: license-file
 
 # scrapeMM: Multimodal Web Retrieval
@@ -46,14 +47,15 @@ URL (string)   -->   retrieve()   -->   MultimodalSequence
 ```
 The `MultimodalSequence` is a sequence of Markdown-formatted text and media provided by the [ezMM](https://github.com/multimodal-ai-lab/ezmm) library.
 
-Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
+Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) and [Decodo](https://decodo.com/).
 
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
+- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ⚠️ Instagram (done for videos but not for images yet)
+- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit
-- ⏳ Facebook
-- ⏳ Instagram

{scrapemm-0.2.2 → scrapemm-0.3.1}/scrapeMM.egg-info/SOURCES.txt

@@ -21,7 +21,9 @@ scrapemm/integrations/instagram.py
 scrapemm/integrations/telegram.py
 scrapemm/integrations/tiktok.py
 scrapemm/integrations/x.py
+scrapemm/integrations/youtube.py
 scrapemm/scraping/__init__.py
+scrapemm/scraping/decodo.py
 scrapemm/scraping/firecrawl.py
 scrapemm/scraping/no_bot_domains.txt
 scrapemm/scraping/util.py

{scrapemm-0.2.2 → scrapemm-0.3.1}/scrapeMM.egg-info/requires.txt

@@ -8,3 +8,4 @@ atproto
 TikTokResearchApi
 yt-dlp
 cryptography
+firecrawl-py

{scrapemm-0.2.2 → scrapemm-0.3.1}/scrapemm/integrations/__init__.py

@@ -1,8 +1,3 @@
-import logging
-
-logger = logging.getLogger("scrapeMM")
-logger.debug("Starting integrations...")
-
 from typing import Optional
 
 import aiohttp
@@ -15,16 +10,17 @@ from .instagram import Instagram
 from .telegram import Telegram
 from .tiktok import TikTok
 from .x import X
+from .youtube import YouTube
 
-RETRIEVAL_INTEGRATIONS = [X(), Telegram(), Bluesky(), TikTok(), Instagram(), Facebook()]
+RETRIEVAL_INTEGRATIONS = [X(), Telegram(), Bluesky(), TikTok(), Instagram(), Facebook(), YouTube()]
 DOMAIN_TO_INTEGRATION = {domain: integration
                          for integration in RETRIEVAL_INTEGRATIONS
-                         for domain in integration.domains
-                         if integration.connected}
+                         for domain in integration.domains}
 
 
 async def retrieve_via_integration(url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
     domain = get_domain(url)
     if domain in DOMAIN_TO_INTEGRATION:
         integration = DOMAIN_TO_INTEGRATION[domain]
-        return await integration.get(url, session)
+        if integration.connected or integration.connected is None:
+            return await integration.get(url, session)

scrapemm-0.3.1/scrapemm/integrations/base.py

@@ -0,0 +1,39 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+import aiohttp
+from ezmm import MultimodalSequence
+
+from scrapemm.common import logger
+from scrapemm.util import get_domain
+
+
+class RetrievalIntegration(ABC):
+    """Any integration used to retrieve information via an external API. Typically
+     used when direct URL scraping is not possible."""
+
+    name: str
+    domains: list[str]  # The domains supported by this integration
+    connected: bool = None
+
+    @abstractmethod
+    async def _connect(self):
+        """Establish a connection to the external service. Invoked upon the first get() call.
+        Must set self.connet = True if connection was successful, else False."""
+        raise NotImplementedError
+
+    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        """Ensures connectivity before invoking the service for retrieval."""
+        assert get_domain(url) in self.domains
+        if self.connected is None:
+            await self._connect()
+            if not self.connected:
+                logger.warning(f"❌ Connection to {self.name} service could not be established.")
+        if self.connected:
+            logger.debug(f"Calling {self.name} service for {url}")
+            return await self._get(url, session)
+
+    @abstractmethod
+    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        """Retrieves the contents present at the given URL."""
+        raise NotImplementedError

{scrapemm-0.2.2 → scrapemm-0.3.1}/scrapemm/integrations/bluesky.py

@@ -2,8 +2,6 @@ import logging
 from typing import Optional
 
 import aiohttp
-from atproto_client.exceptions import RequestErrorBase
-from atproto_client.models.common import XrpcError
 from ezmm import MultimodalSequence, download_image, download_video
 
 from scrapemm.secrets import get_secret
@@ -14,9 +12,10 @@ logger = logging.getLogger("scrapeMM")
 
 
 class Bluesky(RetrievalIntegration):
+    name = "Bluesky"
     domains = ["bsky.app"]
 
-    def __init__(self):
+    async def _connect(self):
         self.username = get_secret("bluesky_username")
         self.password = get_secret("bluesky_password")
 
@@ -27,14 +26,9 @@ class Bluesky(RetrievalIntegration):
 
         from atproto import Client
         self.client = Client()
-        self.authenticated = False
         self._authenticate()
 
-    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
-        if not self.authenticated:
-            logger.error("❌ Not authenticated with Bluesky. Cannot retrieve content.")
-            return None
-
+    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Bluesky: {get_domain(url)}")
             return None
@@ -169,7 +163,7 @@ Metrics:
         """Authenticate with Bluesky using provided credentials."""
         try:
             self.client.login(self.username, self.password)
-            self.authenticated = True
+            self.connected = True
             logger.info(f"✅ Successfully authenticated with Bluesky as {self.username}")
             return True
         except Exception as e:
@@ -221,12 +215,14 @@ Metrics:
             return handle  # Return the handle itself as fallback
 
 
-def error_to_string(error: RequestErrorBase | Exception) -> str:
+def error_to_string(error: Exception) -> str:
     """Takes an Error object containing a response and prints the contents."""
+    from atproto_client.exceptions import RequestErrorBase
     if isinstance(error, RequestErrorBase):
         response = error.response
         code = response.status_code
         content = response.content
+        from atproto_client.models.common import XrpcError
         if isinstance(content, XrpcError):
             error_type = content.error
             msg = content.message

{scrapemm-0.2.2 → scrapemm-0.3.1}/scrapemm/integrations/fb.py

@@ -11,10 +11,10 @@ logger = logging.getLogger("scrapeMM")
 
 
 class Facebook(RetrievalIntegration):
-
+    name = "Facebook"
     domains = ["facebook.com", "www.facebook.com"]
 
-    def __init__(self):
+    async def _connect(self):
         # Check if yt-dlp is available
         self.ytdlp_available = check_ytdlp_available()
 
@@ -26,12 +26,8 @@ class Facebook(RetrievalIntegration):
             self.connected = False
             logger.warning("❌ Facebook integration not available: Neither API credentials nor yt-dlp found.")
 
-    async def get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from a Facebook post URL."""
-        if not self.connected:
-            logger.error("❌ Facebook integration not connected. Cannot retrieve content.")
-            return None
-            
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Facebook: {get_domain(url)}")
             return None

{scrapemm-0.2.2 → scrapemm-0.3.1}/scrapemm/integrations/instagram.py

@@ -11,9 +11,10 @@ logger = logging.getLogger("scrapeMM")
 
 
 class Instagram(RetrievalIntegration):
+    name = "Instagram"
     domains = ["instagram.com", "www.instagram.com"]
 
-    def __init__(self):
+    async def _connect(self):
         # Check if yt-dlp is available
         self.ytdlp_available = check_ytdlp_available()
 
@@ -25,12 +26,8 @@ class Instagram(RetrievalIntegration):
             self.connected = False
             logger.warning("❌ Instagram integration not available: Neither API credentials nor yt-dlp found.")
 
-    async def get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from an Instagram post URL."""
-        if not self.connected:
-            logger.error("❌ Instagram integration not connected. Cannot retrieve content.")
-            return None
-            
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Instagram: {get_domain(url)}")
             return None
@@ -72,7 +69,7 @@ Configure API credentials for full profile information."""
 
     def _is_video_url(self, url: str) -> bool:
         """Checks if the URL is an Instagram video URL."""
-        return "instagram.com/reels" in url
+        return "instagram.com/reels" in url or "instagram.com/reel/" in url
 
     def _is_photo_url(self, url: str) -> bool:
         """Checks if the URL is an Instagram photo URL."""

{scrapemm-0.2.2 → scrapemm-0.3.1}/scrapemm/integrations/telegram.py

@@ -1,6 +1,8 @@
 import logging
+from pathlib import Path
 from typing import Optional
 from urllib.parse import urlparse
+import sqlite3
 
 import aiohttp
 from ezmm import MultimodalSequence, Image, Item, Video
@@ -17,23 +19,30 @@ logger = logging.getLogger("scrapeMM")
 class Telegram(RetrievalIntegration):
     """The Telegram integration for retrieving post contents from Telegram channels and groups."""
 
+    name = "Telegram"
     domains = ["t.me", "telegram.me"]
     session_path = "temp/telegram"
 
-    def __init__(self):
+    async def _connect(self):
         api_id = int(get_secret("telegram_api_id")) if get_secret("telegram_api_id") else None
         api_hash = get_secret("telegram_api_hash")
         bot_token = get_secret("telegram_bot_token")
 
         if api_id and api_hash and bot_token:
             self.client = TelegramClient(self.session_path, api_id, api_hash)
-            self.client.start(bot_token=bot_token)
+            try:
+                self.client.start(bot_token=bot_token)
+            except sqlite3.OperationalError:  # Database is locked from an interrupted previous session
+                # Remove the database file and try again
+                journal_path = Path(self.session_path + ".session-journal")
+                journal_path.unlink(missing_ok=True)
             self.connected = True
             logger.info("✅ Successfully connected to Telegram.")
         else:
+            self.connected = False
             logger.warning("❌ Telegram integration not configured: Missing API keys.")
 
-    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
         """Retrieves content from a Telegram post URL."""
         assert get_domain(url) in self.domains
 
@@ -92,8 +101,7 @@ Forwards: {message.forwards}{reactions_text}
             return MultimodalSequence(text)
 
         except Exception as e:
-            print(f"Error retrieving Telegram content: {e}")
-            # raise
+            logger.warning(f"Error retrieving Telegram content: {e}")
             return None
 
     async def _get_media_from_message(self, chat, original_post, max_amp=10) -> list[Item]: