scrapeMM 0.3.2__tar.gz → 0.3.4__tar.gz

{scrapemm-0.3.2 → scrapemm-0.3.4}/.gitignore

@@ -4,4 +4,7 @@
 /dist/
 /config.yaml
 __pycache__/
-.DS_Store
+.DS_Store
+build/
+.pytest_cache/
+.claude/

{scrapemm-0.3.2/scrapeMM.egg-info → scrapemm-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.2
+Version: 0.3.4
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -54,8 +54,8 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) a
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
-- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ✅ YouTube (working sometimes)
+- ⚠️ Facebook (done for videos but not for images yet)
 - ⚠️ Instagram (done for videos but not for images yet)
-- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit

{scrapemm-0.3.2 → scrapemm-0.3.4}/README.md

@@ -30,8 +30,8 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) a
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
-- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ✅ YouTube (working sometimes)
+- ⚠️ Facebook (done for videos but not for images yet)
 - ⚠️ Instagram (done for videos but not for images yet)
-- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit

{scrapemm-0.3.2 → scrapemm-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.3.2"
+version = "0.3.4"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"

{scrapemm-0.3.2 → scrapemm-0.3.4/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.2
+Version: 0.3.4
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -54,8 +54,8 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl) a
 - ✅ Telegram
 - ✅ Bluesky
 - ✅ TikTok
-- ⚠️ Facebook (working only sometimes and only with yt-dlp and Decodo)
+- ✅ YouTube (working sometimes)
+- ⚠️ Facebook (done for videos but not for images yet)
 - ⚠️ Instagram (done for videos but not for images yet)
-- ⚠️ YouTube (working sometimes)
 - ⏳ Threads
 - ⏳ Reddit

{scrapemm-0.3.2 → scrapemm-0.3.4}/scrapemm/integrations/fb.py

@@ -1,55 +1,52 @@
 import logging
+import re
 from urllib.parse import urlparse
+
 import aiohttp
-from ezmm import MultimodalSequence, download_image
+from ezmm import MultimodalSequence
 
-from scrapemm.scraping.ytdlp import check_ytdlp_available,  get_video_with_ytdlp
 from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.scraping.ytdlp import get_content_with_ytdlp
 from scrapemm.util import get_domain
 
 logger = logging.getLogger("scrapeMM")
 
+VIDEO_URL_REGEX = r"facebook\.com/\d+/videos/\d+/?"
+
 
 class Facebook(RetrievalIntegration):
     name = "Facebook"
     domains = ["facebook.com", "www.facebook.com"]
 
     async def _connect(self):
-        # Check if yt-dlp is available
-        self.ytdlp_available = check_ytdlp_available()
-
-        if self.ytdlp_available:
-            self.connected = True
-            mode = "yt-dlp only"
-            logger.info(f"✅ Facebook integration ready ({mode} mode).")
-        else:
-            self.connected = False
-            logger.warning("❌ Facebook integration not available: Neither API credentials nor yt-dlp found.")
+        self.api_available = False  # TODO
+        logger.info(f"✅ Facebook integration ready (yt-dlp only mode).")
+        self.connected = True
 
     async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from a Facebook post URL."""
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Facebook: {get_domain(url)}")
             return None
-        
-        # Determine if this is a video or profile URL
+
+        # Determine if this is a video or photo URL, act accordingly
         if self._is_video_url(url):
             return await self._get_video(url, session)
         elif self._is_photo_url(url):
             return await self._get_photo(url, session)
-        else:
-            return await self._get_user_profile(url, session)
-        
-    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
-        """Retrieves content from a TikTok video URL."""
 
-        # Fallback to yt-dlp mode
-        if self.ytdlp_available:
-            return await get_video_with_ytdlp(url, session, platform="Facebook")
+        # The URL is not indicative, so try all methods
+        return (await self._get_video(url, session) or
+                await self._get_photo(url, session) or
+                await self._get_user_profile(url, session))
+
+    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook video URL."""
+        if self.api_available:
+            raise NotImplementedError
+        else:
+            return await get_content_with_ytdlp(url, session, platform="Facebook")
 
-        logger.error("❌ No available method to retrieve Facebook video.")
-        return None
-    
     async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from a Facebook photo URL."""
         logger.error("❌ No available method to retrieve Facebook photo.")
@@ -57,23 +54,15 @@ class Facebook(RetrievalIntegration):
 
     async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from a Facebook user profile URL."""
-        username = self._extract_username(url)
-        if username:
-            text = f"""**Facebook Profile**
-Username: @{username}
-URL: {url}
-
-Note: Profile details require Facebook API access.
-Configure API credentials for full profile information."""
-            return MultimodalSequence([text])
-        
+        logger.error("❌ No available method to retrieve Facebook profiles.")
         return None
 
     def _is_video_url(self, url: str) -> bool:
         """Checks if the URL is a Facebook video URL."""
         # video URLS are in the format: https://www.facebook.com/watch?v=VIDEO_ID
-        return "facebook.com/watch" in url
-    
+        # or Reels: https://www.facebook.com/reel/REEL_ID
+        return "facebook.com/watch" in url or "facebook.com/reel" in url or bool(re.search(VIDEO_URL_REGEX, url))
+
     def _extract_video_id(self, url: str) -> str:
         """Extracts the video ID from a Facebook video URL."""
         parsed_url = urlparse(url)
@@ -82,11 +71,11 @@ Configure API credentials for full profile information."""
             if param.startswith('v='):
                 return param.split('=')[1]
         return ""
-    
+
     def _is_photo_url(self, url: str) -> bool:
         """Checks if the URL is a Facebook photo URL."""
         return "facebook.com/photo" in url or "facebook.com/photos" in url
-    
+
     def _extract_username(self, url: str) -> str:
         """Extracts the username from a Facebook profile URL."""
         # url format: https://www.facebook.com/username<?...>

{scrapemm-0.3.2 → scrapemm-0.3.4}/scrapemm/integrations/instagram.py

@@ -1,10 +1,11 @@
 import logging
 from urllib.parse import urlparse
+
 import aiohttp
-from ezmm import MultimodalSequence, download_image
+from ezmm import MultimodalSequence
 
-from scrapemm.scraping.ytdlp import check_ytdlp_available,  get_video_with_ytdlp
 from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.scraping.ytdlp import get_content_with_ytdlp
 from scrapemm.util import get_domain
 
 logger = logging.getLogger("scrapeMM")
@@ -15,44 +16,37 @@ class Instagram(RetrievalIntegration):
     domains = ["instagram.com", "www.instagram.com"]
 
     async def _connect(self):
-        # Check if yt-dlp is available
-        self.ytdlp_available = check_ytdlp_available()
-
-        if self.ytdlp_available:
-            self.connected = True
-            mode = "yt-dlp only"
-            logger.info(f"✅ Instagram integration ready ({mode} mode).")
-        else:
-            self.connected = False
-            logger.warning("❌ Instagram integration not available: Neither API credentials nor yt-dlp found.")
+        self.api_available = False
+        logger.info(f"✅ Instagram integration ready (yt-dlp only mode).")
+        self.connected = True
 
     async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from an Instagram post URL."""
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Instagram: {get_domain(url)}")
             return None
-        
+
         # Determine if this is a video or profile URL
         if self._is_video_url(url):
             return await self._get_video(url, session)
         elif self._is_photo_url(url):
-            return await self._get_photo(url, session)
+            # /p/ URLs can also be reels, so try both
+            return await self._get_video(url, session) or await self._get_photo(url, session)
         else:
             return await self._get_user_profile(url, session)
-        
+
     async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from an Instagram video URL."""
-        if self.ytdlp_available:
-            return await get_video_with_ytdlp(url, session, platform="Instagram")
+        if self.api_available:
+            raise NotImplementedError
+        else:
+            return await get_content_with_ytdlp(url, session, platform="Instagram")
 
-        logger.error("❌ No available method to retrieve Instagram video.")
-        return None
-    
     async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
-        """Retrieves content from an Instagram photo URL."""
+        """Retrieves content from an Instagram photo URL (can also be a reel)."""
         logger.error("❌ No available method to retrieve Instagram photo.")
         return None
-    
+
     async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from an Instagram user profile URL."""
         username = self._extract_username(url)

{scrapemm-0.3.2 → scrapemm-0.3.4}/scrapemm/integrations/tiktok.py

@@ -15,7 +15,6 @@ from ezmm.common.items import Video, Image
 from tiktok_research_api import TikTokResearchAPI, QueryVideoRequest, QueryUserInfoRequest, Criteria, Query
 
 from scrapemm.integrations.base import RetrievalIntegration
-from scrapemm.scraping.ytdlp import check_ytdlp_available
 from scrapemm.secrets import get_secret
 
 logger = logging.getLogger("scrapeMM")
@@ -57,16 +56,9 @@ class TikTok(RetrievalIntegration):
             except Exception as e:
                 logger.info(f"⚠️ TikTok Research API connection failed: {e}. Using fallback mode.")
 
-        # Check if yt-dlp is available
-        self.ytdlp_available = check_ytdlp_available()
-
-        if self.api_available or self.ytdlp_available:
-            self.connected = True
-            mode = "API + yt-dlp" if self.api_available else "yt-dlp only"
-            logger.info(f"✅ TikTok integration ready ({mode} mode).")
-        else:
-            self.connected = False
-            logger.warning("❌ TikTok integration not available: Neither API credentials nor yt-dlp found.")
+        mode = "API" if self.api_available else "yt-dlp only"
+        logger.info(f"✅ TikTok integration ready ({mode} mode).")
+        self.connected = True
 
     async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         # Determine if this is a video or profile URL
@@ -86,12 +78,9 @@ class TikTok(RetrievalIntegration):
             logger.warning("API method failed, falling back to yt-dlp...")
 
         # Fallback to yt-dlp mode
-        if self.ytdlp_available:
+        else:
             return await self._get_video_with_ytdlp(url, session)
 
-        logger.error("❌ No available method to retrieve TikTok video.")
-        return None
-
     async def _get_video_with_api(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves video using TikTok Research API."""
         video_id = self._extract_video_id(url)

{scrapemm-0.3.2 → scrapemm-0.3.4}/scrapemm/integrations/youtube.py

@@ -4,7 +4,7 @@ from typing import Optional
 import aiohttp
 from ezmm import MultimodalSequence
 
-from scrapemm.scraping.ytdlp import get_video_with_ytdlp, check_ytdlp_available
+from scrapemm.scraping.ytdlp import get_content_with_ytdlp
 from .base import RetrievalIntegration
 
 logger = logging.getLogger("scrapeMM")
@@ -15,20 +15,14 @@ class YouTube(RetrievalIntegration):
 
     name = "YouTube"
     domains = [
-        "youtube.com", 
-        "www.youtube.com", 
-        "youtu.be", 
-        "m.youtube.com"
+        "youtube.com",
+        "youtu.be",
     ]
 
     async def _connect(self):
-        self.connected = check_ytdlp_available()
-        if not self.connected:
-            logger.warning("❌ YouTube integration disabled: yt-dlp not available")
-        else:
-            logger.info("✅ YouTube integration enabled")
+        self.connected = True
 
     async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
         """Downloads YouTube video or short using yt-dlp."""
         logger.debug(f"📺 Downloading YouTube content: {url}")
-        return await get_video_with_ytdlp(url, session, "YouTube")
+        return await get_content_with_ytdlp(url, session, "YouTube")

{scrapemm-0.3.2 → scrapemm-0.3.4}/scrapemm/retrieval.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Optional
+from typing import Optional, Collection
 from traceback import format_exc
 
 import aiohttp
@@ -14,7 +14,7 @@ METHODS = ["integrations", "firecrawl", "decodo"]
 
 
 async def retrieve(
-        urls: str | list[str],
+        urls: str | Collection[str],
         remove_urls: bool = False,
         show_progress: bool = True,
         actions: list[dict] = None,

scrapemm-0.3.4/scrapemm/scraping/decodo.py

@@ -0,0 +1,216 @@
+import asyncio
+import logging
+from typing import Optional
+
+import aiohttp
+from ezmm import MultimodalSequence
+
+from scrapemm.secrets import get_secret
+from scrapemm.scraping.util import to_multimodal_sequence, get_domain_root
+
+logger = logging.getLogger("scrapeMM")
+
+
+class Decodo:
+    """Scrapes web content using Decodo's Web Scraping API with proxy support
+    and JavaScript rendering capabilities."""
+
+    DECODO_API_URL = "https://scraper-api.decodo.com/v2/scrape"
+
+    def __init__(self):
+        self.username = None
+        self.password = None
+        self.n_scrapes = 0
+
+    def _load_credentials(self):
+        """Loads Decodo credentials from the secrets manager."""
+        self.username = get_secret("decodo_username")
+        self.password = get_secret("decodo_password")
+
+        if self.username and self.password:
+            logger.info("✅ Decodo credentials loaded successfully.")
+        else:
+            logger.warning("⚠️ Decodo credentials not found. Please configure them in secrets.")
+
+    def _has_credentials(self) -> bool:
+        """Checks if Decodo credentials are available."""
+        return bool(self.username and self.password)
+
+    async def scrape(
+        self, url: str,
+        remove_urls: bool,
+        session: aiohttp.ClientSession,
+        format: str,
+        enable_js: bool = True,
+        timeout: int = 30,
+    ) -> Optional[MultimodalSequence | str]:
+        """Downloads the contents of the specified webpage using Decodo's API.
+
+        Args:
+            url: The URL to scrape
+            remove_urls: Whether to remove URLs from hyperlinks in the result
+            session: The aiohttp ClientSession to use
+            enable_js: Whether to enable JavaScript rendering (default: True)
+            timeout: Request timeout in seconds (default: 30)
+
+        Returns:
+            MultimodalSequence containing the scraped content, or None if scraping failed
+        """
+        if not self._has_credentials():
+            self._load_credentials()
+
+        if not self._has_credentials():
+            logger.warning("Cannot scrape with Decodo: credentials not configured.")
+            return None
+
+        # Try with JS rendering first if enabled
+        html = await self._call_decodo(url, session, enable_js, timeout)
+
+        # If it failed with JS and we got a 400 error, try without JS
+        # (400 might mean the plan doesn't support headless rendering)
+        if html is None and enable_js:
+            logger.debug("Retrying without JavaScript rendering...")
+            html = await self._call_decodo(url, session, enable_js=False, timeout=timeout)
+
+        if html:
+            if format == "html":
+                return html
+            else:
+                domain_root = get_domain_root(url)
+                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session, domain_root=domain_root)
+        return None
+
+    async def _call_decodo(
+        self, url: str,
+        session: aiohttp.ClientSession,
+        enable_js: bool = True,
+        timeout: int = 30,
+        max_retries: int = 5
+    ) -> Optional[str]:
+        """Calls the Decodo API to scrape the given URL with exponential backoff retry logic.
+
+        Args:
+            url: The URL to scrape
+            session: The aiohttp ClientSession to use
+            enable_js: Whether to enable JavaScript rendering
+            timeout: Request timeout in seconds
+            max_retries: Maximum number of retry attempts for rate limits (default: 5)
+
+        Returns:
+            HTML content as a string, or None if scraping failed
+        """
+        headers = {
+            'Content-Type': 'application/json',
+        }
+
+        # Build request payload
+        # Note: For simple URL scraping, we just provide the URL
+        # The "target" parameter is only used for specific templates like "google_search"
+        payload = {
+            "url": url,
+        }
+
+        # Enable JavaScript rendering if requested
+        # Note: This requires an Advanced plan subscription
+        if enable_js:
+            payload["headless"] = "html"
+
+        # Create basic auth
+        auth = aiohttp.BasicAuth(self.username, self.password)
+
+        # Retry loop with exponential backoff
+        for attempt in range(max_retries + 1):
+            try:
+                async with session.post(
+                    self.DECODO_API_URL,
+                    json=payload,
+                    headers=headers,
+                    auth=auth,
+                    timeout=aiohttp.ClientTimeout(total=timeout)
+                ) as response:
+                    # Validate response health
+                    if response.status != 200:
+                        logger.debug("Communication with Decodo API failed.")
+
+                        # Handle rate limiting and 613 errors with retry
+                        if response.status in (429, 613):
+                            if attempt < max_retries:
+                                # Exponential backoff: 2^attempt seconds (1s, 2s, 4s, 8s, 16s)
+                                wait_time = 2 ** attempt
+                                error_msg = "Rate limit exceeded" if response.status == 429 else "Error 613"
+                                logger.warning(f"Error {response.status}: {error_msg}. Retrying in {wait_time}s (attempt {attempt + 1}/{max_retries})...")
+                                await asyncio.sleep(wait_time)
+                                continue
+                            else:
+                                logger.warning(f"Error {response.status}: Maximum retries reached.")
+                                return None
+
+                        # Log other errors
+                        match response.status:
+                            case 400:
+                                logger.debug("Error 400: Bad request. If you use JavaScript, make sure you have the "
+                                             "Advanced plan subscription.")
+                            case 401:
+                                logger.error("Error 401: Unauthorized. Check your Decodo credentials.")
+                            case 402:
+                                logger.error("Error 402: Payment required. Check your Decodo subscription.")
+                            case 403:
+                                logger.debug("Error 403: Forbidden.")
+                            case 408:
+                                logger.warning("Error 408: Timeout! Website did not respond in time.")
+                            case 500:
+                                logger.debug("Error 500: Server error.")
+                            case _:
+                                logger.debug(f"Error {response.status}: {response.reason}.")
+
+                        return None
+
+                    # Parse response
+                    json_response = await response.json()
+
+                    # Validate if scrape was successful
+                    if json_response.get("status") == "failed":
+                        status_code = json_response.get("status_code")
+                        message = json_response.get("message")
+                        logger.info(f"Decodo failed to scrape. Error {status_code}: {message}")
+                        return None
+
+                    # Extract HTML content from results
+                    if "results" in json_response and len(json_response["results"]) > 0:
+                        result = json_response["results"][0]
+
+                        # Check status code from the actual request
+                        status_code = result.get("status_code")
+                        if status_code and status_code >= 400:
+                            logger.warning(f"Target website returned status {status_code} for {url}")
+                            return None
+
+                        html_content = result.get("content")
+                        if html_content:
+                            self.n_scrapes += 1
+                            logger.debug(f"Successfully scraped {url} with Decodo (scrape #{self.n_scrapes})")
+                            return html_content
+                        else:
+                            logger.warning(f"No content in Decodo response for {url}")
+                            return None
+                    else:
+                        logger.warning(f"No results in Decodo response for {url}")
+                        logger.debug(f"Response: {json_response}")
+                        return None
+
+            except aiohttp.ClientError as e:
+                logger.error(f"Network error while scraping {url} with Decodo: {repr(e)}")
+                return None
+            except asyncio.TimeoutError:
+                logger.warning(f"Timeout while scraping {url} with Decodo. Skipping...")
+                return None
+            except Exception as e:
+                logger.error(f"Unexpected error while scraping {url} with Decodo: {repr(e)}")
+                return None
+
+        # Should not reach here, but return None as fallback
+        return None
+
+
+# Create a singleton instance
+decodo = Decodo()

{scrapemm-0.3.2 → scrapemm-0.3.4}/scrapemm/scraping/firecrawl.py

@@ -7,7 +7,7 @@ from ezmm import MultimodalSequence
 
 from scrapemm.common import get_config_var, update_config
 from scrapemm.util import read_urls_from_file, get_domain
-from scrapemm.scraping.util import find_firecrawl, to_multimodal_sequence, firecrawl_is_running
+from scrapemm.scraping.util import find_firecrawl, to_multimodal_sequence, firecrawl_is_running, get_domain_root
 
 logger = logging.getLogger("scrapeMM")
 
@@ -58,6 +58,7 @@ class Firecrawl:
 
     def connect(self):
         from firecrawl import AsyncFirecrawl
+        logging.getLogger("firecrawl").setLevel(logging.WARNING)
         self.firecrawl_url = locate_firecrawl()
         self._firecrawl = AsyncFirecrawl(api_url=self.firecrawl_url)
 
@@ -88,7 +89,9 @@ class Firecrawl:
             if format == "html":
                 return html
             else:
-                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session)
+                domain_root = get_domain_root(url)
+                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session, domain_root=domain_root)
+        return None
 
 
 fire = Firecrawl()

{scrapemm-0.3.2 → scrapemm-0.3.4}/scrapemm/scraping/util.py

@@ -54,6 +54,7 @@ def postprocess_scraped(text: str) -> str:
 
 async def resolve_media_hyperlinks(
         text: str, session: aiohttp.ClientSession,
+        domain_root: str = None,
         remove_urls: bool = False,
 ) -> Optional[MultimodalSequence]:
     """Downloads all media that are hyperlinked in the provided Markdown text.
@@ -65,19 +66,21 @@ async def resolve_media_hyperlinks(
 
     # Extract URLs and base64-encoded data from the text
     hyperlinks = get_markdown_hyperlinks(text)
-    urls = set()
+    hrefs_urls = dict()
     data_uris = set()
     for _, _, href in hyperlinks:
         if is_url(href):
-            urls.add(href)
+            hrefs_urls[href] = href
+        elif domain_root and is_root_relative_url(href):
+            hrefs_urls[href] = f"{domain_root}{href}"
         elif is_data_uri(href):
             data_uris.add(href)
 
     # Try to download media for each URL
-    tasks = [download_item(url, session=session) for url in urls]
+    tasks = [download_item(url, session=session) for url in hrefs_urls.values()]
     media: list[Item | None] = await run_with_semaphore(tasks, limit=100, show_progress=False)
 
-    href_media = dict(zip(urls, media))
+    href_media = dict(zip(hrefs_urls.keys(), media))
 
     # Convert each base64-encoded data to the respective medium
     for data_uri in data_uris:
@@ -102,15 +105,29 @@ async def resolve_media_hyperlinks(
 
 
 def is_url(href: str) -> bool:
-    """Returns True iff the given string is a valid URL."""
+    """Returns True iff the given string is an absolute HTTP URL."""
     return re.match(URL_REGEX, href) is not None
 
 
+def is_root_relative_url(href: str) -> bool:
+    """Returns True iff the given string is a root-relative URL."""
+    return href.startswith("/")
+
+
 def is_data_uri(href: str) -> bool:
     """Returns True iff the given string is a valid data URI."""
     return re.match(DATA_URI_REGEX, href) is not None
 
 
+def get_domain_root(url: str) -> Optional[str]:
+    """Extracts the domain root from the given URL. Allows for missing http(s) prefix."""
+    match = re.match(r"(:?https?://)?([^/]+)", url)
+    if match:
+        return match.group(0)
+    else:
+        return None
+
+
 def get_markdown_hyperlinks(text: str) -> list[tuple[str, str, str]]:
     """Extracts all web hyperlinks from the given markdown-formatted string. Returns
     a list of fullmatch-hypertext-URL-triples."""