scrapeMM 0.3.3__tar.gz → 0.3.4__tar.gz

{scrapemm-0.3.3 → scrapemm-0.3.4}/.gitignore

@@ -4,4 +4,7 @@
 /dist/
 /config.yaml
 __pycache__/
-.DS_Store
+.DS_Store
+build/
+.pytest_cache/
+.claude/

{scrapemm-0.3.3/scrapeMM.egg-info → scrapemm-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.3
+Version: 0.3.4
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -54,8 +54,8 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) a
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
-- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ✅ YouTube (working sometimes)
+- ⚠️ Facebook (done for videos but not for images yet)
 - ⚠️ Instagram (done for videos but not for images yet)
-- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit

{scrapemm-0.3.3 → scrapemm-0.3.4}/README.md

@@ -30,8 +30,8 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) a
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
-- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ✅ YouTube (working sometimes)
+- ⚠️ Facebook (done for videos but not for images yet)
 - ⚠️ Instagram (done for videos but not for images yet)
-- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit

{scrapemm-0.3.3 → scrapemm-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.3.3"
+version = "0.3.4"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"

{scrapemm-0.3.3 → scrapemm-0.3.4/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.3
+Version: 0.3.4
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -54,8 +54,8 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) a
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
-- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ✅ YouTube (working sometimes)
+- ⚠️ Facebook (done for videos but not for images yet)
 - ⚠️ Instagram (done for videos but not for images yet)
-- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit

{scrapemm-0.3.3 → scrapemm-0.3.4}/scrapemm/integrations/fb.py

@@ -1,55 +1,52 @@
 import logging
+import re
 from urllib.parse import urlparse
+
 import aiohttp
-from ezmm import MultimodalSequence, download_image
+from ezmm import MultimodalSequence
 
-from scrapemm.scraping.ytdlp import check_ytdlp_available,  get_video_with_ytdlp
 from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.scraping.ytdlp import get_content_with_ytdlp
 from scrapemm.util import get_domain
 
 logger = logging.getLogger("scrapeMM")
 
+VIDEO_URL_REGEX = r"facebook\.com/\d+/videos/\d+/?"
+
 
 class Facebook(RetrievalIntegration):
     name = "Facebook"
     domains = ["facebook.com", "www.facebook.com"]
 
     async def _connect(self):
-        # Check if yt-dlp is available
-        self.ytdlp_available = check_ytdlp_available()
-
-        if self.ytdlp_available:
-            self.connected = True
-            mode = "yt-dlp only"
-            logger.info(f"✅ Facebook integration ready ({mode} mode).")
-        else:
-            self.connected = False
-            logger.warning("❌ Facebook integration not available: Neither API credentials nor yt-dlp found.")
+        self.api_available = False  # TODO
+        logger.info(f"✅ Facebook integration ready (yt-dlp only mode).")
+        self.connected = True
 
     async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from a Facebook post URL."""
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Facebook: {get_domain(url)}")
             return None
-        
-        # Determine if this is a video or profile URL
+
+        # Determine if this is a video or photo URL, act accordingly
         if self._is_video_url(url):
             return await self._get_video(url, session)
         elif self._is_photo_url(url):
             return await self._get_photo(url, session)
-        else:
-            return await self._get_user_profile(url, session)
-        
-    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
-        """Retrieves content from a TikTok video URL."""
 
-        # Fallback to yt-dlp mode
-        if self.ytdlp_available:
-            return await get_video_with_ytdlp(url, session, platform="Facebook")
+        # The URL is not indicative, so try all methods
+        return (await self._get_video(url, session) or
+                await self._get_photo(url, session) or
+                await self._get_user_profile(url, session))
+
+    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook video URL."""
+        if self.api_available:
+            raise NotImplementedError
+        else:
+            return await get_content_with_ytdlp(url, session, platform="Facebook")
 
-        logger.error("❌ No available method to retrieve Facebook video.")
-        return None
-    
     async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from a Facebook photo URL."""
         logger.error("❌ No available method to retrieve Facebook photo.")
@@ -57,23 +54,15 @@ class Facebook(RetrievalIntegration):
 
     async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from a Facebook user profile URL."""
-        username = self._extract_username(url)
-        if username:
-            text = f"""**Facebook Profile**
-Username: @{username}
-URL: {url}
-
-Note: Profile details require Facebook API access.
-Configure API credentials for full profile information."""
-            return MultimodalSequence([text])
-        
+        logger.error("❌ No available method to retrieve Facebook profiles.")
         return None
 
     def _is_video_url(self, url: str) -> bool:
         """Checks if the URL is a Facebook video URL."""
         # video URLS are in the format: https://www.facebook.com/watch?v=VIDEO_ID
-        return "facebook.com/watch" in url
-    
+        # or Reels: https://www.facebook.com/reel/REEL_ID
+        return "facebook.com/watch" in url or "facebook.com/reel" in url or bool(re.search(VIDEO_URL_REGEX, url))
+
     def _extract_video_id(self, url: str) -> str:
         """Extracts the video ID from a Facebook video URL."""
         parsed_url = urlparse(url)
@@ -82,11 +71,11 @@ Configure API credentials for full profile information."""
             if param.startswith('v='):
                 return param.split('=')[1]
         return ""
-    
+
     def _is_photo_url(self, url: str) -> bool:
         """Checks if the URL is a Facebook photo URL."""
         return "facebook.com/photo" in url or "facebook.com/photos" in url
-    
+
     def _extract_username(self, url: str) -> str:
         """Extracts the username from a Facebook profile URL."""
         # url format: https://www.facebook.com/username<?...>

{scrapemm-0.3.3 → scrapemm-0.3.4}/scrapemm/integrations/instagram.py

@@ -1,10 +1,11 @@
 import logging
 from urllib.parse import urlparse
+
 import aiohttp
-from ezmm import MultimodalSequence, download_image
+from ezmm import MultimodalSequence
 
-from scrapemm.scraping.ytdlp import check_ytdlp_available,  get_video_with_ytdlp
 from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.scraping.ytdlp import get_content_with_ytdlp
 from scrapemm.util import get_domain
 
 logger = logging.getLogger("scrapeMM")
@@ -15,44 +16,37 @@ class Instagram(RetrievalIntegration):
     domains = ["instagram.com", "www.instagram.com"]
 
     async def _connect(self):
-        # Check if yt-dlp is available
-        self.ytdlp_available = check_ytdlp_available()
-
-        if self.ytdlp_available:
-            self.connected = True
-            mode = "yt-dlp only"
-            logger.info(f"✅ Instagram integration ready ({mode} mode).")
-        else:
-            self.connected = False
-            logger.warning("❌ Instagram integration not available: Neither API credentials nor yt-dlp found.")
+        self.api_available = False
+        logger.info(f"✅ Instagram integration ready (yt-dlp only mode).")
+        self.connected = True
 
     async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from an Instagram post URL."""
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Instagram: {get_domain(url)}")
             return None
-        
+
         # Determine if this is a video or profile URL
         if self._is_video_url(url):
             return await self._get_video(url, session)
         elif self._is_photo_url(url):
-            return await self._get_photo(url, session)
+            # /p/ URLs can also be reels, so try both
+            return await self._get_video(url, session) or await self._get_photo(url, session)
         else:
             return await self._get_user_profile(url, session)
-        
+
     async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from an Instagram video URL."""
-        if self.ytdlp_available:
-            return await get_video_with_ytdlp(url, session, platform="Instagram")
+        if self.api_available:
+            raise NotImplementedError
+        else:
+            return await get_content_with_ytdlp(url, session, platform="Instagram")
 
-        logger.error("❌ No available method to retrieve Instagram video.")
-        return None
-    
     async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
-        """Retrieves content from an Instagram photo URL."""
+        """Retrieves content from an Instagram photo URL (can also be a reel)."""
         logger.error("❌ No available method to retrieve Instagram photo.")
         return None
-    
+
     async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from an Instagram user profile URL."""
         username = self._extract_username(url)

{scrapemm-0.3.3 → scrapemm-0.3.4}/scrapemm/integrations/tiktok.py

@@ -15,7 +15,6 @@ from ezmm.common.items import Video, Image
 from tiktok_research_api import TikTokResearchAPI, QueryVideoRequest, QueryUserInfoRequest, Criteria, Query
 
 from scrapemm.integrations.base import RetrievalIntegration
-from scrapemm.scraping.ytdlp import check_ytdlp_available
 from scrapemm.secrets import get_secret
 
 logger = logging.getLogger("scrapeMM")
@@ -57,16 +56,9 @@ class TikTok(RetrievalIntegration):
             except Exception as e:
                 logger.info(f"⚠️ TikTok Research API connection failed: {e}. Using fallback mode.")
 
-        # Check if yt-dlp is available
-        self.ytdlp_available = check_ytdlp_available()
-
-        if self.api_available or self.ytdlp_available:
-            self.connected = True
-            mode = "API + yt-dlp" if self.api_available else "yt-dlp only"
-            logger.info(f"✅ TikTok integration ready ({mode} mode).")
-        else:
-            self.connected = False
-            logger.warning("❌ TikTok integration not available: Neither API credentials nor yt-dlp found.")
+        mode = "API" if self.api_available else "yt-dlp only"
+        logger.info(f"✅ TikTok integration ready ({mode} mode).")
+        self.connected = True
 
     async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         # Determine if this is a video or profile URL
@@ -86,12 +78,9 @@ class TikTok(RetrievalIntegration):
             logger.warning("API method failed, falling back to yt-dlp...")
 
         # Fallback to yt-dlp mode
-        if self.ytdlp_available:
+        else:
             return await self._get_video_with_ytdlp(url, session)
 
-        logger.error("❌ No available method to retrieve TikTok video.")
-        return None
-
     async def _get_video_with_api(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves video using TikTok Research API."""
         video_id = self._extract_video_id(url)

{scrapemm-0.3.3 → scrapemm-0.3.4}/scrapemm/integrations/youtube.py

@@ -4,7 +4,7 @@ from typing import Optional
 import aiohttp
 from ezmm import MultimodalSequence
 
-from scrapemm.scraping.ytdlp import get_video_with_ytdlp, check_ytdlp_available
+from scrapemm.scraping.ytdlp import get_content_with_ytdlp
 from .base import RetrievalIntegration
 
 logger = logging.getLogger("scrapeMM")
@@ -15,20 +15,14 @@ class YouTube(RetrievalIntegration):
 
     name = "YouTube"
     domains = [
-        "youtube.com", 
-        "www.youtube.com", 
-        "youtu.be", 
-        "m.youtube.com"
+        "youtube.com",
+        "youtu.be",
     ]
 
     async def _connect(self):
-        self.connected = check_ytdlp_available()
-        if not self.connected:
-            logger.warning("❌ YouTube integration disabled: yt-dlp not available")
-        else:
-            logger.info("✅ YouTube integration enabled")
+        self.connected = True
 
     async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
         """Downloads YouTube video or short using yt-dlp."""
         logger.debug(f"📺 Downloading YouTube content: {url}")
-        return await get_video_with_ytdlp(url, session, "YouTube")
+        return await get_content_with_ytdlp(url, session, "YouTube")

scrapemm-0.3.4/scrapemm/scraping/decodo.py

@@ -0,0 +1,216 @@
+import asyncio
+import logging
+from typing import Optional
+
+import aiohttp
+from ezmm import MultimodalSequence
+
+from scrapemm.secrets import get_secret
+from scrapemm.scraping.util import to_multimodal_sequence, get_domain_root
+
+logger = logging.getLogger("scrapeMM")
+
+
+class Decodo:
+    """Scrapes web content using Decodo's Web Scraping API with proxy support
+    and JavaScript rendering capabilities."""
+
+    DECODO_API_URL = "https://scraper-api.decodo.com/v2/scrape"
+
+    def __init__(self):
+        self.username = None
+        self.password = None
+        self.n_scrapes = 0
+
+    def _load_credentials(self):
+        """Loads Decodo credentials from the secrets manager."""
+        self.username = get_secret("decodo_username")
+        self.password = get_secret("decodo_password")
+
+        if self.username and self.password:
+            logger.info("✅ Decodo credentials loaded successfully.")
+        else:
+            logger.warning("⚠️ Decodo credentials not found. Please configure them in secrets.")
+
+    def _has_credentials(self) -> bool:
+        """Checks if Decodo credentials are available."""
+        return bool(self.username and self.password)
+
+    async def scrape(
+        self, url: str,
+        remove_urls: bool,
+        session: aiohttp.ClientSession,
+        format: str,
+        enable_js: bool = True,
+        timeout: int = 30,
+    ) -> Optional[MultimodalSequence | str]:
+        """Downloads the contents of the specified webpage using Decodo's API.
+
+        Args:
+            url: The URL to scrape
+            remove_urls: Whether to remove URLs from hyperlinks in the result
+            session: The aiohttp ClientSession to use
+            enable_js: Whether to enable JavaScript rendering (default: True)
+            timeout: Request timeout in seconds (default: 30)
+
+        Returns:
+            MultimodalSequence containing the scraped content, or None if scraping failed
+        """
+        if not self._has_credentials():
+            self._load_credentials()
+
+        if not self._has_credentials():
+            logger.warning("Cannot scrape with Decodo: credentials not configured.")
+            return None
+
+        # Try with JS rendering first if enabled
+        html = await self._call_decodo(url, session, enable_js, timeout)
+
+        # If it failed with JS and we got a 400 error, try without JS
+        # (400 might mean the plan doesn't support headless rendering)
+        if html is None and enable_js:
+            logger.debug("Retrying without JavaScript rendering...")
+            html = await self._call_decodo(url, session, enable_js=False, timeout=timeout)
+
+        if html:
+            if format == "html":
+                return html
+            else:
+                domain_root = get_domain_root(url)
+                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session, domain_root=domain_root)
+        return None
+
+    async def _call_decodo(
+        self, url: str,
+        session: aiohttp.ClientSession,
+        enable_js: bool = True,
+        timeout: int = 30,
+        max_retries: int = 5
+    ) -> Optional[str]:
+        """Calls the Decodo API to scrape the given URL with exponential backoff retry logic.
+
+        Args:
+            url: The URL to scrape
+            session: The aiohttp ClientSession to use
+            enable_js: Whether to enable JavaScript rendering
+            timeout: Request timeout in seconds
+            max_retries: Maximum number of retry attempts for rate limits (default: 5)
+
+        Returns:
+            HTML content as a string, or None if scraping failed
+        """
+        headers = {
+            'Content-Type': 'application/json',
+        }
+
+        # Build request payload
+        # Note: For simple URL scraping, we just provide the URL
+        # The "target" parameter is only used for specific templates like "google_search"
+        payload = {
+            "url": url,
+        }
+
+        # Enable JavaScript rendering if requested
+        # Note: This requires an Advanced plan subscription
+        if enable_js:
+            payload["headless"] = "html"
+
+        # Create basic auth
+        auth = aiohttp.BasicAuth(self.username, self.password)
+
+        # Retry loop with exponential backoff
+        for attempt in range(max_retries + 1):
+            try:
+                async with session.post(
+                    self.DECODO_API_URL,
+                    json=payload,
+                    headers=headers,
+                    auth=auth,
+                    timeout=aiohttp.ClientTimeout(total=timeout)
+                ) as response:
+                    # Validate response health
+                    if response.status != 200:
+                        logger.debug("Communication with Decodo API failed.")
+
+                        # Handle rate limiting and 613 errors with retry
+                        if response.status in (429, 613):
+                            if attempt < max_retries:
+                                # Exponential backoff: 2^attempt seconds (1s, 2s, 4s, 8s, 16s)
+                                wait_time = 2 ** attempt
+                                error_msg = "Rate limit exceeded" if response.status == 429 else "Error 613"
+                                logger.warning(f"Error {response.status}: {error_msg}. Retrying in {wait_time}s (attempt {attempt + 1}/{max_retries})...")
+                                await asyncio.sleep(wait_time)
+                                continue
+                            else:
+                                logger.warning(f"Error {response.status}: Maximum retries reached.")
+                                return None
+
+                        # Log other errors
+                        match response.status:
+                            case 400:
+                                logger.debug("Error 400: Bad request. If you use JavaScript, make sure you have the "
+                                             "Advanced plan subscription.")
+                            case 401:
+                                logger.error("Error 401: Unauthorized. Check your Decodo credentials.")
+                            case 402:
+                                logger.error("Error 402: Payment required. Check your Decodo subscription.")
+                            case 403:
+                                logger.debug("Error 403: Forbidden.")
+                            case 408:
+                                logger.warning("Error 408: Timeout! Website did not respond in time.")
+                            case 500:
+                                logger.debug("Error 500: Server error.")
+                            case _:
+                                logger.debug(f"Error {response.status}: {response.reason}.")
+
+                        return None
+
+                    # Parse response
+                    json_response = await response.json()
+
+                    # Validate if scrape was successful
+                    if json_response.get("status") == "failed":
+                        status_code = json_response.get("status_code")
+                        message = json_response.get("message")
+                        logger.info(f"Decodo failed to scrape. Error {status_code}: {message}")
+                        return None
+
+                    # Extract HTML content from results
+                    if "results" in json_response and len(json_response["results"]) > 0:
+                        result = json_response["results"][0]
+
+                        # Check status code from the actual request
+                        status_code = result.get("status_code")
+                        if status_code and status_code >= 400:
+                            logger.warning(f"Target website returned status {status_code} for {url}")
+                            return None
+
+                        html_content = result.get("content")
+                        if html_content:
+                            self.n_scrapes += 1
+                            logger.debug(f"Successfully scraped {url} with Decodo (scrape #{self.n_scrapes})")
+                            return html_content
+                        else:
+                            logger.warning(f"No content in Decodo response for {url}")
+                            return None
+                    else:
+                        logger.warning(f"No results in Decodo response for {url}")
+                        logger.debug(f"Response: {json_response}")
+                        return None
+
+            except aiohttp.ClientError as e:
+                logger.error(f"Network error while scraping {url} with Decodo: {repr(e)}")
+                return None
+            except asyncio.TimeoutError:
+                logger.warning(f"Timeout while scraping {url} with Decodo. Skipping...")
+                return None
+            except Exception as e:
+                logger.error(f"Unexpected error while scraping {url} with Decodo: {repr(e)}")
+                return None
+
+        # Should not reach here, but return None as fallback
+        return None
+
+
+# Create a singleton instance
+decodo = Decodo()

scrapemm-0.3.4/scrapemm/scraping/ytdlp.py

@@ -0,0 +1,114 @@
+import logging
+import tempfile
+import traceback
+from datetime import datetime
+from typing import Any, Optional
+
+import asyncio
+import aiohttp
+from ezmm import MultimodalSequence, download_image, Video, Image
+from yt_dlp import YoutubeDL
+
+logger = logging.getLogger("scrapeMM")
+
+
+async def _download_with_ytdlp(
+        url: str,
+        session: aiohttp.ClientSession
+) -> tuple[Optional[Video], Optional[Image], Optional[dict[str, Any]]]:
+    """Downloads a video, its thumbnail, and the metadata using yt-dlp."""
+    try:
+        with tempfile.NamedTemporaryFile() as temp_file:
+            temp_path = temp_file.name
+
+        ydl_opts: dict[str, Any] = dict(
+            outtmpl=f'{temp_path}.%(ext)s',  # Output filename format
+            format='best[ext=mp4]/best',  # Download the best video/audio quality
+            quiet=False,  # Show progress in terminal
+            noplaylist=True,  # Disable playlist downloading
+            retries=3,
+            cookies_from_browser='firefox',
+            cookies='cookies.txt',  # Use cookies from Firefox to bypass sign-in requirements
+        )
+
+        if "youtube" in url or "youtu.be" in url:
+            # YouTube delivers video and audio separately when downloaded above 720p.
+            # This would require FFmpeg to merge them. Restrict to 720p to avoid that.
+            ydl_opts['format'] = 'best[height<=720][acodec!=none]'
+            ydl_opts['extractor_args'] = dict(youtube=dict(player_client=["default"]))
+
+        with YoutubeDL(ydl_opts) as ydl:
+            metadata = ydl.extract_info(url, download=True)
+
+        ext = metadata.get("ext")
+
+        try:
+            video = Video(file_path=temp_path + f".{ext}", source_url=url)
+            video.relocate(move_not_copy=True)
+        except Exception as e:
+            logger.warning(f"Could not load downloaded video: {e}\n{traceback.format_exc()}")
+            video = None
+
+        thumbnail = None
+        if thumbnail_url := metadata.get('thumbnail'):
+            thumbnail = await download_image(thumbnail_url, session)
+
+        return video, thumbnail, metadata
+
+    except Exception as e:
+        logger.warning(f"Could not download video with yt-dlp: {e}\n{traceback.format_exc()}")
+        return None, None, None
+
+
+def fmt_count(v):
+    return f"{v:,}" if isinstance(v, int) else "Unknown"
+
+
+async def compose_data_to_sequence(metadata: dict, video: Video | None, thumbnail: Image | None,
+                                   platform: str) -> MultimodalSequence:
+    """Creates a MultimodalSequence from the yt-dlp metadata."""
+    # title = metadata.get('title', '')
+    uploader = metadata.get('uploader', 'Unknown')
+    upload_date = metadata.get('upload_date', '')
+    duration = metadata.get('duration', 0)
+    view_count = metadata.get('view_count', 0)
+    like_count = metadata.get('like_count', 0)
+    comment_count = metadata.get('comment_count', 0)
+    description = metadata.get('description', '')
+
+    # Format upload date
+    formatted_date = upload_date
+    if upload_date and len(upload_date) == 8:
+        try:
+            date_obj = datetime.strptime(upload_date, '%Y%m%d')
+            formatted_date = date_obj.strftime('%Y-%m-%d')
+        except ValueError:
+            pass
+
+    text = f"""**{platform} Video**
+Author: @{uploader}
+Posted: {formatted_date}
+Duration: {duration}s
+Views: {fmt_count(view_count)} - Likes: {fmt_count(like_count)} - Comments: {fmt_count(comment_count)}
+
+{description}"""
+
+    items: list = [text]
+    if thumbnail:
+        items.append(thumbnail)
+    if video:
+        items.append(video)
+
+    return MultimodalSequence(items)
+
+
+async def get_content_with_ytdlp(url: str, session: aiohttp.ClientSession, platform: str) -> MultimodalSequence | None:
+    """Retrieves video, thumbnail, and metadata using the powerful yt-dlp package."""
+    # Run the download in a separate thread to avoid blocking the event loop
+    coroutine = await asyncio.to_thread(_download_with_ytdlp, url, session)
+    video, thumbnail, metadata = await coroutine
+    if not video:
+        logger.warning(f"Video download failed for {url}")
+    if metadata:
+        return await compose_data_to_sequence(metadata, video, thumbnail, platform)
+    return None

{scrapemm-0.3.3 → scrapemm-0.3.4}/scrapemm/secrets.py

@@ -20,8 +20,8 @@ SECRETS = {
     "bluesky_password": "Bluesky password",
     "tiktok_client_key": "TikTok client key",
     "tiktok_client_secret": "TikTok client secret",
-    "decodo_username": "Decodo API username",
-    "decodo_password": "Decodo API password",
+    "decodo_username": "Decodo Web Scraping API username",
+    "decodo_password": "Decodo Web Scraping API password",
 }
 
 SALT = b'\xa4\x93\xf1\x88\x13\x88'

scrapemm-0.3.4/scripts/example.py

@@ -0,0 +1,7 @@
+from scrapemm import retrieve
+import asyncio
+
+if __name__ == "__main__":
+    url = "https://www.youtube.com/shorts/cE0zgN6pYOc"
+    result = asyncio.run(retrieve(url))
+    print(result)

{scrapemm-0.3.3 → scrapemm-0.3.4}/testing/test_retrieval.py

@@ -12,6 +12,7 @@ from scrapemm import retrieve
     "https://assamese.factcrescendo.com/viral-claim-that-the-video-shows-the-incident-from-uttar-pradesh-and-the-youth-on-the-bike-and-the-youth-being-beaten-and-taken-away-by-the-police-are-the-same-youth-named-abdul-is-false/",
     "https://factuel.afp.com/doc.afp.com.43ZN7NP",
     "https://leadstories.com/365cb414b83e29d26fecae374d55c743a3eac4c7.png",
+    "https://leadstories.com/assets_c/2025/08/193f14f06dd6f15b89bf8050e553ad7fb1be6530-thumb-900xauto-3165872.png"
 ])
 @pytest.mark.parametrize("method", ["firecrawl", "decodo"])
 async def test_generic_retrieval(url, method):
@@ -44,7 +45,7 @@ async def test_html_retrieval(url, method):
     "https://t.me/tglobaleye/6289",  # GIF (treated as video)
     "https://t.me/tglobaleye/16192",  # Images and video
 ])
-async def test_telegram_retrieval(url):
+async def test_telegram(url):
     result = await retrieve(url)
     print(result)
     assert result
@@ -55,7 +56,7 @@ async def test_telegram_retrieval(url):
     "https://www.tiktok.com/@realdonaldtrump/video/7433870905635409198",
     "https://www.tiktok.com/@xxxx.xxxx5743/video/7521704371109793046"
 ])
-async def test_tiktok_retrieval(url):
+async def test_tiktok(url):
     result = await retrieve(url)
     print(result)
     assert result
@@ -66,7 +67,7 @@ async def test_tiktok_retrieval(url):
     "https://x.com/PopBase/status/1938496291908030484",
     "https://x.com/realDonaldTrump"
 ])
-async def test_x_retrieval(url):
+async def test_x(url):
     result = await retrieve(url)
     print(result)
     assert result
@@ -74,35 +75,60 @@ async def test_x_retrieval(url):
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("url", [
-    "https://www.instagram.com/p/CqJDbyOP839",  # image (post)
-    "https://www.instagram.com/reel/DKqPQqpTDW4",  # video (Reel)
-    "https://www.instagram.com/reel/C75nh7Lvo8F",  # video (Reel)
-    "https://www.instagram.com/p/DMuOe6th94D",  # video (Reel)
+    "https://www.instagram.com/p/CqJDbyOP839",
 ])
-async def test_instagram_retrieval(url):
+async def test_instagram_images(url):
     result = await retrieve(url)
     print(result)
     assert result
+    assert result.has_images()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("url", [
+    "https://www.instagram.com/reel/DKqPQqpTDW4",
+    "https://www.instagram.com/reel/C75nh7Lvo8F",
+    "https://www.instagram.com/p/DMuOe6th94D",  # yes, this is a video
+])
+async def test_instagram_videos(url):
+    result = await retrieve(url)
+    print(result)
+    assert result
+    assert result.has_videos()
 
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("url", [
-    "https://www.facebook.com/reel/1089214926521000",  # video
-    "https://www.facebook.com/reel/3466446073497470",  # video, restricted for misinformation
-    "https://www.facebook.com/groups/1973976962823632/posts/3992825270938781/",  # video, restricted for misinformation
-    "https://www.facebook.com/photo/?fbid=1721085455188778&set=a.107961589834514&_rdc=1&_rdr",  # image
+    "https://www.facebook.com/reel/1089214926521000",
+    "https://www.facebook.com/reel/3466446073497470",  # restricted for misinformation
+    "https://www.facebook.com/61561558177010/videos/1445957793080961/",
+    "https://www.facebook.com/watch/?v=1445957793080961",
+    "https://www.facebook.com/groups/1973976962823632/posts/3992825270938781/",  # restricted for misinformation, yt-dlp fails here
 ])
-async def test_facebook_retrieval(url):
+async def test_facebook_videos(url):
     result = await retrieve(url)
     print(result)
     assert result
+    assert result.has_videos()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("url", [
+    "https://www.facebook.com/photo/?fbid=1721085455188778&set=a.107961589834514&_rdc=1&_rdr",
+])
+async def test_facebook_images(url):
+    result = await retrieve(url)
+    print(result)
+    assert result
+    assert result.has_images()
 
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("url", [
-    "https://www.youtube.com/watch?v=ykQZPdaXTJc",
+    "https://www.youtube.com/watch?v=A4dVOznX6Kk",
+    "https://www.youtube.com/shorts/cE0zgN6pYOc",
 ])
-async def test_youtube_retrieval(url):
+async def test_youtube(url):
     result = await retrieve(url)
     print(result)
     assert result

scrapemm-0.3.3/scrapemm/scraping/decodo.py

@@ -1,192 +0,0 @@
-import asyncio
-import logging
-from typing import Optional
-
-import aiohttp
-from ezmm import MultimodalSequence
-
-from scrapemm.secrets import get_secret
-from scrapemm.scraping.util import to_multimodal_sequence, get_domain_root
-
-logger = logging.getLogger("scrapeMM")
-
-
-class Decodo:
-    """Scrapes web content using Decodo's Web Scraping API with proxy support
-    and JavaScript rendering capabilities."""
-
-    DECODO_API_URL = "https://scraper-api.decodo.com/v2/scrape"
-
-    def __init__(self):
-        self.username = None
-        self.password = None
-        self.n_scrapes = 0
-
-    def _load_credentials(self):
-        """Loads Decodo credentials from the secrets manager."""
-        self.username = get_secret("decodo_username")
-        self.password = get_secret("decodo_password")
-
-        if self.username and self.password:
-            logger.info("✅ Decodo credentials loaded successfully.")
-        else:
-            logger.warning("⚠️ Decodo credentials not found. Please configure them in secrets.")
-
-    def _has_credentials(self) -> bool:
-        """Checks if Decodo credentials are available."""
-        return bool(self.username and self.password)
-
-    async def scrape(self, url: str,
-                     remove_urls: bool,
-                     session: aiohttp.ClientSession,
-                     format: str,
-                     enable_js: bool = True,
-                     timeout: int = 30) -> Optional[MultimodalSequence | str]:
-        """Downloads the contents of the specified webpage using Decodo's API.
-
-        Args:
-            url: The URL to scrape
-            remove_urls: Whether to remove URLs from hyperlinks in the result
-            session: The aiohttp ClientSession to use
-            enable_js: Whether to enable JavaScript rendering (default: True)
-            timeout: Request timeout in seconds (default: 30)
-
-        Returns:
-            MultimodalSequence containing the scraped content, or None if scraping failed
-        """
-        if not self._has_credentials():
-            self._load_credentials()
-
-        if not self._has_credentials():
-            logger.warning("Cannot scrape with Decodo: credentials not configured.")
-            return None
-
-        # Try with JS rendering first if enabled
-        html = await self._call_decodo(url, session, enable_js, timeout)
-
-        # If it failed with JS and we got a 400 error, try without JS
-        # (400 might mean the plan doesn't support headless rendering)
-        if html is None and enable_js:
-            logger.debug("Retrying without JavaScript rendering...")
-            html = await self._call_decodo(url, session, enable_js=False, timeout=timeout)
-
-        if html:
-            if format == "html":
-                return html
-            else:
-                domain_root = get_domain_root(url)
-                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session, domain_root=domain_root)
-        return None
-
-    async def _call_decodo(self, url: str,
-                          session: aiohttp.ClientSession,
-                          enable_js: bool = True,
-                          timeout: int = 30) -> Optional[str]:
-        """Calls the Decodo API to scrape the given URL.
-
-        Args:
-            url: The URL to scrape
-            session: The aiohttp ClientSession to use
-            enable_js: Whether to enable JavaScript rendering
-            timeout: Request timeout in seconds
-
-        Returns:
-            HTML content as a string, or None if scraping failed
-        """
-        headers = {
-            'Content-Type': 'application/json',
-        }
-
-        # Build request payload
-        # Note: For simple URL scraping, we just provide the URL
-        # The "target" parameter is only used for specific templates like "google_search"
-        payload = {
-            "url": url,
-        }
-
-        # Enable JavaScript rendering if requested
-        # Note: This requires an Advanced plan subscription
-        if enable_js:
-            payload["headless"] = "html"
-
-        # Create basic auth
-        auth = aiohttp.BasicAuth(self.username, self.password)
-
-        try:
-            async with session.post(
-                self.DECODO_API_URL,
-                json=payload,
-                headers=headers,
-                auth=auth,
-                timeout=aiohttp.ClientTimeout(total=timeout)
-            ) as response:
-                # Validate response health
-                if response.status != 200:
-                    logger.debug("Communication with Decodo API failed.")
-                    match response.status:
-                        case 400:
-                            logger.debug("Error 400: Bad request. If you use JavaScript, make sure you have the "
-                                         "Advanced plan subscription.")
-                        case 401:
-                            logger.error("Error 401: Unauthorized. Check your Decodo credentials.")
-                        case 402:
-                            logger.error("Error 402: Payment required. Check your Decodo subscription.")
-                        case 403:
-                            logger.debug("Error 403: Forbidden.")
-                        case 408:
-                            logger.warning("Error 408: Timeout! Website did not respond in time.")
-                        case 429:
-                            logger.warning("Error 429: Rate limit exceeded. Too many requests.")
-                        case 500:
-                            logger.debug("Error 500: Server error.")
-                        case _:
-                            logger.debug(f"Error {response.status}: {response.reason}.")
-
-                    return None
-
-                # Parse response
-                json_response = await response.json()
-
-                # Validate if scrape was successful
-                if json_response.get("status") == "failed":
-                    status_code = json_response.get("status_code")
-                    message = json_response.get("message")
-                    logger.info(f"Decodo failed to scrape. Error {status_code}: {message}")
-                    return None
-
-                # Extract HTML content from results
-                if "results" in json_response and len(json_response["results"]) > 0:
-                    result = json_response["results"][0]
-
-                    # Check status code from the actual request
-                    status_code = result.get("status_code")
-                    if status_code and status_code >= 400:
-                        logger.warning(f"Target website returned status {status_code} for {url}")
-                        return None
-
-                    html_content = result.get("content")
-                    if html_content:
-                        self.n_scrapes += 1
-                        logger.debug(f"Successfully scraped {url} with Decodo (scrape #{self.n_scrapes})")
-                        return html_content
-                    else:
-                        logger.warning(f"No content in Decodo response for {url}")
-                        return None
-                else:
-                    logger.warning(f"No results in Decodo response for {url}")
-                    logger.debug(f"Response: {json_response}")
-                    return None
-
-        except aiohttp.ClientError as e:
-            logger.error(f"Network error while scraping {url} with Decodo: {repr(e)}")
-            return None
-        except asyncio.TimeoutError:
-            logger.warning(f"Timeout while scraping {url} with Decodo. Skipping...")
-            return None
-        except Exception as e:
-            logger.error(f"Unexpected error while scraping {url} with Decodo: {repr(e)}")
-            return None
-
-
-# Create a singleton instance
-decodo = Decodo()

scrapemm-0.3.3/scrapemm/scraping/ytdlp.py

@@ -1,203 +0,0 @@
-import sys
-from typing import Any, cast
-
-import aiohttp
-import asyncio
-from datetime import datetime
-import json
-import logging
-import os
-import subprocess
-import tempfile
-
-from ezmm import MultimodalSequence, download_image
-from ezmm.common.items import Video, Image
-
-logger = logging.getLogger("scrapeMM")
-
-
-def check_ytdlp_available() -> bool:
-    """Returns True if yt-dlp is available, else False."""
-    try:
-        # Run yt-dlp --version to check if it's installed and working'
-        subprocess.run([sys.executable, '-m', 'yt_dlp', '--version'],
-                     capture_output=True, check=True, timeout=5)
-        return True
-    except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired) as e:
-        return False
-
-
-async def extract_metadata_with_ytdlp(url: str) -> dict[str, Any] | None:
-    """Extracts metadata using yt-dlp without downloading the video."""
-    try:
-        cmd = [
-            sys.executable,  # Ensure to use the same Python interpreter as the calling process
-            '-m',
-            'yt_dlp',
-            '--no-download',
-            '--print-json',
-            '--no-warnings',
-            '--quiet',
-            url
-        ]
-
-        process = await asyncio.create_subprocess_exec(
-            *cmd,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE
-        )
-        
-        stdout, stderr = await process.communicate()
-        
-        if process.returncode != 0:
-            logger.error(f"❌ yt-dlp metadata extraction failed: {stderr.decode()}")
-            return None
-
-        # Parse JSON output
-        metadata = json.loads(stdout.decode())
-        return metadata
-
-    except Exception as e:
-        logger.error(f"❌ Error extracting metadata with yt-dlp: {e}")
-        return None
-    
-
-async def download_video_with_ytdlp(url: str) -> Video | None:
-    """Downloads a video using yt-dlp."""
-    try:
-        with tempfile.NamedTemporaryFile(suffix='.%(ext)s', delete=False) as temp_file:
-            temp_path = temp_file.name
-
-        cmd = [
-            sys.executable,  # Ensure to use the same Python interpreter as the calling process
-            "-m",
-            "yt_dlp",
-            "--no-playlist",
-            "--no-warnings",
-            "--quiet",
-            "--retries", "10",
-        ]
-
-        if "youtube" in url or "youtu.be" in url:
-            cmd.extend([
-                "-f", "bv*[vcodec~='^avc1']+ba/bv*+ba/b",
-                "--merge-output-format", "mp4",
-            ])
-        else:
-            cmd.extend(['--format', 'best[ext=mp4]/best'])
-
-        cmd.extend([
-            "--output", temp_path,
-            url
-        ])
-
-        process = await asyncio.create_subprocess_exec(
-            *cmd,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE
-        )
-        
-        _, stderr = await process.communicate()
-        
-        if process.returncode != 0:
-            logger.warning(f"yt-dlp video download failed: {stderr.decode()}")
-            return None
-
-        # Find the actual downloaded file (yt-dlp changes extension)
-        downloaded_file = None
-        for ext in ['.mp4', '.webm', '.mkv']:
-            potential_file = temp_path.replace('.%(ext)s', ext)
-            if os.path.exists(potential_file) and os.path.getsize(potential_file) > 0:
-                downloaded_file = potential_file
-                break
-
-        if downloaded_file and os.path.exists(downloaded_file):
-            with open(downloaded_file, 'rb') as f:
-                video_data = f.read()
-            
-            os.unlink(downloaded_file)
-            
-            video = Video(binary_data=video_data, source_url=url)
-            video.relocate(move_not_copy=True)
-            return cast(Video, video)
-
-        return None
-
-    except Exception as e:
-        logger.error(f"❌ Error downloading video with yt-dlp: {e}")
-        return None
-    
-
-async def download_thumbnail_with_ytdlp(url: str, session: aiohttp.ClientSession) -> Image | None:
-    """Downloads thumbnail using yt-dlp metadata."""
-    try:
-        metadata = await extract_metadata_with_ytdlp(url)
-        if not metadata:
-            return None
-
-        thumbnail_url = metadata.get('thumbnail')
-        if thumbnail_url:
-            return await download_image(thumbnail_url, session)
-
-    except Exception as e:
-        logger.error(f"❌ Error downloading thumbnail: {e}")
-    
-    return None
-
-def fmt_count(v):
-    return f"{v:,}" if isinstance(v, int) else "Unknown"
-
-async def create_video_sequence_from_ytdlp(metadata: dict, url: str, video: Video | None, thumbnail: Image | None, platform: str) -> MultimodalSequence:
-    """Creates MultimodalSequence from yt-dlp metadata."""
-    # title = metadata.get('title', '')
-    uploader = metadata.get('uploader', 'Unknown')
-    upload_date = metadata.get('upload_date', '')
-    duration = metadata.get('duration', 0)
-    view_count = metadata.get('view_count', 0)
-    like_count = metadata.get('like_count', 0)
-    comment_count = metadata.get('comment_count', 0)
-    description = metadata.get('description', '')
-    
-    # Format upload date
-    formatted_date = upload_date
-    if upload_date and len(upload_date) == 8:
-        try:
-            date_obj = datetime.strptime(upload_date, '%Y%m%d')
-            formatted_date = date_obj.strftime('%Y-%m-%d')
-        except ValueError:
-            pass
-
-    text = f"""**{platform} Video**
-Author: @{uploader}
-Posted: {formatted_date}
-Duration: {duration}s
-Views: {fmt_count(view_count)} - Likes: {fmt_count(like_count)} - Comments: {fmt_count(comment_count)}
-
-{description}"""
-
-    items: list = [text]
-    if thumbnail:
-        items.append(thumbnail)
-    if video:
-        items.append(video)
-        
-    return MultimodalSequence(items)
-
-async def get_video_with_ytdlp(url: str, session: aiohttp.ClientSession, platform: str) -> MultimodalSequence | None:
-    """Retrieves video using only yt-dlp (no API required)."""
-    try:
-        # Get metadata and video using yt-dlp
-        metadata = await extract_metadata_with_ytdlp(url)
-        if not metadata:
-            return None
-
-        video = await download_video_with_ytdlp(url)
-        thumbnail = await download_thumbnail_with_ytdlp(url, session)
-        
-        return await create_video_sequence_from_ytdlp(metadata, url, video, thumbnail, platform)
-
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        logger.error(f"❌ Error retrieving video with yt-dlp: {e}")
-        return None

scrapemm-0.3.3/scripts/example.py

@@ -1,7 +0,0 @@
-from scrapemm import retrieve
-import asyncio
-
-if __name__ == "__main__":
-    url = "https://factuel.afp.com/doc.afp.com.43ZN7NP"
-    result = asyncio.run(retrieve(url, methods=["decodo"]))
-    print(result)