scrapeMM 0.7.3__tar.gz → 0.7.4__tar.gz

{scrapemm-0.7.3/scrapeMM.egg-info → scrapemm-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.7.3
+Version: 0.7.4
 Summary: Multimodal scraper for social media, internet archives, and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -11,7 +11,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp~=3.14.1
 Requires-Dist: async_lru
-Requires-Dist: atproto>=0.0.69
+Requires-Dist: atproto~=0.0.69
 Requires-Dist: beautifulsoup4
 Requires-Dist: backports.zstd
 Requires-Dist: brotli
@@ -24,7 +24,7 @@ Requires-Dist: pip-system-certs
 Requires-Dist: platformdirs~=4.10.0
 Requires-Dist: playwright
 Requires-Dist: playwright-stealth
-Requires-Dist: seleniumbase~=4.51.1
+Requires-Dist: seleniumbase~=4.44.20
 Requires-Dist: prompt_toolkit
 Requires-Dist: PyYAML~=6.0.1
 Requires-Dist: requests~=2.34.2

{scrapemm-0.7.3 → scrapemm-0.7.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.7.3"
+version = "0.7.4"
 description = "Multimodal scraper for social media, internet archives, and the open web."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -9,7 +9,7 @@ authors = [{name = "Mark Rothermel", email = "mark.rothermel@tu-darmstadt.de"}]
 dependencies = [
     "aiohttp~=3.14.1",
     "async_lru",
-    "atproto>=0.0.69",
+    "atproto~=0.0.69",
     "beautifulsoup4",
     "backports.zstd",  # For better content parsing with Firecrawl
     "brotli",  # Required to read brotli-encoded media
@@ -22,7 +22,7 @@ dependencies = [
     "platformdirs~=4.10.0",
     "playwright",
     "playwright-stealth",
-    "seleniumbase~=4.51.1",
+    "seleniumbase~=4.44.20",  # Package conflict with atproto on the websockets package. Upgrade only when atproto supports websockets>=16.1
     "prompt_toolkit",  # For handling user terminal input
     "PyYAML~=6.0.1",
     "requests~=2.34.2",

{scrapemm-0.7.3 → scrapemm-0.7.4/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.7.3
+Version: 0.7.4
 Summary: Multimodal scraper for social media, internet archives, and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -11,7 +11,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp~=3.14.1
 Requires-Dist: async_lru
-Requires-Dist: atproto>=0.0.69
+Requires-Dist: atproto~=0.0.69
 Requires-Dist: beautifulsoup4
 Requires-Dist: backports.zstd
 Requires-Dist: brotli
@@ -24,7 +24,7 @@ Requires-Dist: pip-system-certs
 Requires-Dist: platformdirs~=4.10.0
 Requires-Dist: playwright
 Requires-Dist: playwright-stealth
-Requires-Dist: seleniumbase~=4.51.1
+Requires-Dist: seleniumbase~=4.44.20
 Requires-Dist: prompt_toolkit
 Requires-Dist: PyYAML~=6.0.1
 Requires-Dist: requests~=2.34.2

{scrapemm-0.7.3 → scrapemm-0.7.4}/scrapeMM.egg-info/requires.txt

@@ -1,6 +1,6 @@
 aiohttp~=3.14.1
 async_lru
-atproto>=0.0.69
+atproto~=0.0.69
 beautifulsoup4
 backports.zstd
 brotli
@@ -13,7 +13,7 @@ pip-system-certs
 platformdirs~=4.10.0
 playwright
 playwright-stealth
-seleniumbase~=4.51.1
+seleniumbase~=4.44.20
 prompt_toolkit
 PyYAML~=6.0.1
 requests~=2.34.2

{scrapemm-0.7.3 → scrapemm-0.7.4}/scrapemm/common/exceptions.py

@@ -23,3 +23,7 @@ class UnsupportedDomainError(Exception):
 
 class TargetUnavailableError(Exception):
     """The target website is not available for scraping."""
+
+
+class DiskFull(Exception):
+    """No space left on disk. The application should be aborted immediately before continuing."""

scrapemm-0.7.4/scrapemm/download/requests.py

@@ -0,0 +1,172 @@
+import logging
+from typing import Optional, Union, TYPE_CHECKING
+
+import aiohttp
+
+if TYPE_CHECKING:
+    from playwright.async_api import APIRequestContext
+
+from scrapemm.download.common import ssl_context, RELAXED_SSL_DOMAINS
+from scrapemm.download.util import stream
+
+logger = logging.getLogger("scrapeMM")
+
+# Cloudflare (and similar) bot gates reject aiohttp's TLS fingerprint with HTTP 403
+# while accepting real browser JA3/JA4 profiles. Tried in order until one succeeds.
+_CURL_CFFI_IMPERSONATIONS = ("chrome124", "safari17_2_ios", "chrome110")
+
+
+async def _request_via_curl_cffi(
+        url: str,
+        headers: Optional[dict] = None,
+) -> Optional[tuple[int, dict, bytes]]:
+    """GET ``url`` with browser TLS impersonation. Returns (status, headers, body) or None."""
+    try:
+        from curl_cffi.requests import AsyncSession
+    except ImportError:
+        logger.debug("curl_cffi not available; cannot bypass bot-gated 403 for %s", url)
+        return None
+
+    async with AsyncSession() as session:
+        for impersonate in _CURL_CFFI_IMPERSONATIONS:
+            try:
+                response = await session.get(
+                    url,
+                    impersonate=impersonate,
+                    headers=headers or {},
+                    allow_redirects=True,
+                )
+                status = response.status_code
+                hdrs = dict(response.headers)
+                body = response.content
+                if status == 200:
+                    logger.debug(
+                        "Retrieved %s via curl_cffi impersonate=%s (%s bytes)",
+                        url, impersonate, len(body),
+                    )
+                    return status, hdrs, body
+                if status != 403:
+                    # Real client/server error — further fingerprints won't help.
+                    return status, hdrs, body
+                logger.debug(
+                    "curl_cffi impersonate=%s still got 403 for %s; trying next profile",
+                    impersonate, url,
+                )
+            except Exception:
+                logger.debug(
+                    "curl_cffi impersonate=%s failed for %s",
+                    impersonate, url, exc_info=True,
+                )
+    return None
+
+
+def _merge_request_headers(
+        session: Union[aiohttp.ClientSession, "APIRequestContext"],
+        kwargs: dict,
+) -> dict:
+    """Combine session-default headers with per-request overrides for curl_cffi."""
+    merged: dict = {}
+    if isinstance(session, aiohttp.ClientSession):
+        merged.update({str(k): str(v) for k, v in session.headers.items()})
+    extra = kwargs.get("headers") or {}
+    merged.update({str(k): str(v) for k, v in extra.items()})
+    return merged
+
+
+async def fetch_headers(url, session: Union[aiohttp.ClientSession, "APIRequestContext"], **kwargs) -> dict:
+    """Fetch only HTTP headers for a URL."""
+    from scrapemm.util import get_domain
+    ssl = None if get_domain(str(url)) in RELAXED_SSL_DOMAINS else ssl_context
+
+    async def _headers_via_curl_cffi() -> dict:
+        result = await _request_via_curl_cffi(str(url), _merge_request_headers(session, kwargs))
+        if result and result[0] == 200:
+            return result[1]
+        raise RuntimeError(f"Forbidden (403) fetching headers for {url}")
+
+    if hasattr(session, "head"):  # aiohttp.ClientSession or Playwright APIRequestContext
+        try:
+            # Playwright APIRequestContext.head exists and returns APIResponse
+            if not isinstance(session, aiohttp.ClientSession):
+                # Playwright
+                response = await session.head(str(url), headers=kwargs.get("headers"), timeout=kwargs.get("timeout"))
+                if getattr(response, "status", None) == 403:
+                    return await _headers_via_curl_cffi()
+                return response.headers
+            else:
+                # aiohttp
+                async with session.head(url, ssl=ssl, **kwargs) as response:
+                    response.raise_for_status()
+                    return dict(response.headers)
+        except Exception:
+            logger.debug(f"HEAD failed for {url}, falling back to GET", exc_info=True)
+            # Fallback to GET
+            try:
+                if not isinstance(session, aiohttp.ClientSession):
+                    response = await session.get(str(url), headers=kwargs.get("headers"), timeout=kwargs.get("timeout"))
+                    if getattr(response, "status", None) == 403:
+                        return await _headers_via_curl_cffi()
+                    return response.headers
+                else:
+                    async with session.get(url, ssl=ssl, **kwargs) as response:
+                        response.raise_for_status()
+                        return dict(response.headers)
+            except aiohttp.ClientResponseError as e:
+                if e.status == 403:
+                    return await _headers_via_curl_cffi()
+                raise
+    raise ValueError(f"Unsupported session type: {type(session)}")
+
+
+async def request_static(url: str,
+                         session: Union[aiohttp.ClientSession, "APIRequestContext"],
+                         get_text: bool = True,
+                         **kwargs) -> Optional[str | bytes]:
+    """Downloads the static page from the given URL using aiohttp or Playwright.
+
+    On HTTP 403 (common Cloudflare bot gate), retries with curl_cffi browser
+    TLS impersonation so hosts that accept browsers but reject aiohttp still work.
+    """
+    if not url:
+        return None
+
+    url = str(url)
+    from scrapemm.util import get_domain
+    ssl = None if get_domain(url) in RELAXED_SSL_DOMAINS else ssl_context
+
+    async def _from_curl_cffi() -> Optional[str | bytes]:
+        result = await _request_via_curl_cffi(url, _merge_request_headers(session, kwargs))
+        if not result or result[0] != 200:
+            return None
+        _, _, content = result
+        if get_text:
+            return content.decode("utf-8", errors="replace")
+        return content
+
+    try:
+        if not isinstance(session, aiohttp.ClientSession):
+            # Playwright APIRequestContext
+            response = await session.get(url, **kwargs)
+            if response.ok:
+                return await response.text() if get_text else await response.body()
+            if response.status == 403:
+                return await _from_curl_cffi()
+            return None
+        else:
+            # aiohttp
+            async with session.get(url, timeout=10, allow_redirects=True,
+                                   raise_for_status=True, ssl=ssl, **kwargs) as response:
+                if get_text:
+                    return await response.text()
+                else:
+                    return await stream(response)
+    except aiohttp.ClientResponseError as e:
+        if e.status == 403:
+            content = await _from_curl_cffi()
+            if content is not None:
+                return content
+        logger.debug(f"Error requesting {url}", exc_info=True)
+        return None
+    except Exception:
+        logger.debug(f"Error requesting {url}", exc_info=True)
+        return None

{scrapemm-0.7.3 → scrapemm-0.7.4}/scrapemm/integrations/__init__.py

@@ -41,10 +41,20 @@ DOMAIN_TO_INTEGRATION = {
     for domain in integration.domains
 }
 
+NAME_TO_INTEGRATION = {integration.name: integration for integration in RETRIEVAL_INTEGRATIONS}
 
-async def retrieve_via_integration(url: str, **kwargs) -> Optional[MultimodalSequence]:
+INTEGRATION_NAMES = [integration.name for integration in RETRIEVAL_INTEGRATIONS]
+
+
+async def retrieve_via_integration(url: str, integration_name: str, **kwargs) -> Optional[MultimodalSequence]:
+    integration = NAME_TO_INTEGRATION[integration_name]
+    if integration.connected or integration.connected is None:
+        return await integration.get(url, **kwargs)
+
+
+def get_integrations_for_url(url: str) -> list[str]:
+    """Returns the list of integration names that support the given domain."""
     domain = get_domain(url)
-    if domain in DOMAIN_TO_INTEGRATION:
-        integration = DOMAIN_TO_INTEGRATION[domain]
-        if integration.connected or integration.connected is None:
-            return await integration.get(url, **kwargs)
+    return [integration.name
+            for integration in RETRIEVAL_INTEGRATIONS
+            if domain in integration.domains]

{scrapemm-0.7.3 → scrapemm-0.7.4}/scrapemm/integrations/decodo/decodo.py

@@ -224,7 +224,7 @@ class Decodo:
             except (RateLimitError, asyncio.TimeoutError):
                 raise
             except Exception as e:
-                logger.error(f"Unexpected error while scraping with Decodo: {e}")
+                logger.debug(f"Error while scraping with Decodo: {e}")
                 raise
 
             await backoff(attempt)  # Wait before retrying

{scrapemm-0.7.3 → scrapemm-0.7.4}/scrapemm/integrations/tiktok.py

@@ -70,7 +70,7 @@ class TikTok(RetrievalIntegration):
                 return await self._get_user_profile(url, session)
         except Exception as e:
             if "Your IP address is blocked from accessing this post" in str(e):
-                raise IPBannedError(f"TikTok prevents your IP address from accessing this post: {str(e)}")
+                raise IPBannedError(f"TikTok prevents your IP address from accessing the post {url}")
             elif "This post may not be comfortable for some audiences" in str(e):
                 raise ContentBlockedError("Video is blocked by TikTok for being 'uncomfortable for some audiences'.")
             else:

{scrapemm-0.7.3 → scrapemm-0.7.4}/scrapemm/retrieval.py

@@ -9,19 +9,21 @@ from ezmm import MultimodalSequence
 from playwright.async_api import TimeoutError as PlaywrightTimeoutError
 
 from scrapemm.common import ScrapingResponse
-from scrapemm.common.exceptions import RetrievalFailed, IPBannedError, UnsupportedDomainError
-from scrapemm.download import download_medium, download_image, download_video
+from scrapemm.common.exceptions import RetrievalFailed, IPBannedError, UnsupportedDomainError, DiskFull
+from scrapemm.download import download_image, download_video
 from scrapemm.download.common import HEADERS
 from scrapemm.download.util import looks_like_image_file_url, looks_like_video_file_url, looks_like_hls_url
-from scrapemm.integrations import retrieve_via_integration, fire, decodo
+from scrapemm.integrations import retrieve_via_integration, fire, decodo, get_integrations_for_url, INTEGRATION_NAMES
 from scrapemm.util import run_with_semaphore, get_domain, normalize_video
 
 logger = logging.getLogger("scrapeMM")
 METHODS = ["integrations", "firecrawl", "decodo"]
+ALL_METHODS = METHODS + INTEGRATION_NAMES
 
 UNSUPPORTED_DOMAINS = []
 
 BEST_METHODS = {
+    # Social media platforms:
     "instagram.com": ["integrations", "decodo"],
     "facebook.com": ["integrations"],
     "fb.watch": ["integrations"],
@@ -34,24 +36,24 @@ BEST_METHODS = {
     "bsky.app": ["integrations"],
     "truthsocial.com": ["firecrawl"],
     "reddit.com": ["integrations"],
-    "awesomescreenshot.com": ["integrations"],
     "youtube.com": ["integrations"],
     "youtu.be": ["integrations"],
+    # Archiving services:
+    "archive.today": ["integrations"],
+    "archive.is": ["integrations"],
+    "archive.ph": ["integrations"],
+    "archive.vn": ["integrations"],
+    "archive.li": ["integrations"],
+    "archive.fo": ["integrations"],
+    "archive.md": ["integrations"],
     "perma.cc": ["integrations"],
-    "mvau.lt": ["integrations"],
     "archive.org": ["integrations"],
+    "mvau.lt": ["integrations"],
+    "awesomescreenshot.com": ["integrations"],
+    # Miscellaneous:
+    "washingtonpost.com": ["decodo"],
     "verafiles.org": ["decodo", "firecrawl"],
     "youturn.in": ["decodo"],
-    # Archive.today:
-    "archive.today": ["integrations", "decodo"],
-    "archive.is": ["integrations", "decodo"],
-    "archive.ph": ["integrations", "decodo"],
-    "archive.vn": ["integrations", "decodo"],
-    "archive.li": ["integrations", "decodo"],
-    "archive.fo": ["integrations", "decodo"],
-    "archive.md": ["integrations", "decodo"],
-    # More special cases
-    "washingtonpost.com": ["decodo"],
 }
 
 
@@ -59,8 +61,8 @@ async def retrieve(
         urls: str | Collection[str],
         show_progress: bool = True,
         actions: list[dict] | None = None,
-        methods: Literal["auto"] | list[str] | list[Literal["auto"] | list[str]] | None = "auto",
-        format: str = "multimodal_sequence",
+        methods: Literal["auto"] | list[str] | list[Literal["auto"] | list[str]] = "auto",
+        format: Literal["multimodal_sequence", "html"] = "multimodal_sequence",
         include_media: bool = True,
         max_video_size: int | None = None,
         prioritize: Literal["completeness", "speed"] = "completeness"
@@ -80,12 +82,12 @@ async def retrieve(
         You can specify any subset in any order, e.g., ["decodo", "firecrawl"] or ["integrations"]. If provided
         a list of strings, that order of methods will be applied to all submitted URLs. In contrast, if provided
         a list of lists, each list will be applied to the corresponding URL in the batch. If provided "auto",
-        will determine the best method based on the URL's domain. If None, will use the default order.
+        will determine the best method based on the URL's domain. If "auto", will use the default order.
     :param format: The format of the output. Available formats:
         - "multimodal_sequence" (MultimodalSequence containing parsed and downloaded media from the page)
         - "html" (string containing the raw HTML code of the page, not compatible with 'integrations' method)
     :param include_media: If True (default), download and embed images/videos. If False, return text only
-        (media elements are omitted).
+        (media elements are omitted). This parameter is ignored for HTML format.
     :param max_video_size: Maximum size of videos to download, in bytes. If None, no limit is applied.
     :param prioritize: Prioritization strategy for retrieval. Available options:
         - "completeness": Higher timeout limits and more retries.
@@ -110,7 +112,7 @@ async def retrieve(
     elif isinstance(methods, list):
         assert len(methods) >= 1, "'methods' cannot be an empty list."
 
-    # Build per-URL method dict according to the provided 'methods'
+    # Unfold methods list to build per-URL method dict according to the provided 'methods'
     if isinstance(methods[0], str) and methods[0] != "auto":
         # methods: list[str] → apply same order to all URLs
         url_to_methods = {url: methods[:] for url in urls_to_retrieve}
@@ -141,9 +143,9 @@ async def retrieve(
 async def _retrieve_single(
         url: str,
         session: aiohttp.ClientSession,
-        methods: Literal["auto"] | list[str] | None = "auto",
+        methods: Literal["auto"] | list[str] = "auto",
         actions: list[dict] | None = None,
-        format: str = "multimodal_sequence",
+        format: Literal["multimodal_sequence", "html"] = "multimodal_sequence",
         include_media: bool = True,
         max_video_size: int | None = None,
         prioritize: Literal["completeness", "speed"] = "completeness"
@@ -156,22 +158,26 @@ async def _retrieve_single(
                                 errors=dict(scrapemm=UnsupportedDomainError("Unsupported domain.")),
                                 retrieval_time=time.time() - start_time)
 
-    if methods is None:
-        methods = METHODS.copy()
-    elif methods == "auto":
-        methods = get_optimal_methods(url)
+    # Ensure URL is a string
+    url = str(url)
 
-    try:
-        # Ensure URL is a string
-        url = str(url)
+    # Resolve the best methods to try in order
+    methods: list[str] = resolve_best_methods(url, methods)
+
+    if len(methods) == 0:
+        raise AssertionError("No retrieval methods were resolved for the given URL.")
 
+    try:
         # Validate methods
         for method in methods:
-            assert method in METHODS, f"Unknown method '{method}'. Allowed: {METHODS}"
+            assert method in ALL_METHODS, f"Unknown method '{method}'. Allowed: {ALL_METHODS}"
 
-        # Ensure compatibility with methods (local list only — never mutate shared globals)
-        if format == "html":
-            methods = [m for m in methods if m != "integrations"]
+            # Ensure compatibility with methods (local list only — never mutate shared globals)
+            if format == "html" and method not in ["decodo", "firecrawl"]:
+                raise AssertionError(
+                    "'html' format is only compatible with 'firecrawl' and 'decodo' method. "
+                    "Please choose a different format or use a different method."
+                )
 
         # Shortcut to download as medium
         if format != "html" and include_media:
@@ -184,44 +190,39 @@ async def _retrieve_single(
                 return ScrapingResponse(url=url, content=MultimodalSequence(medium), method="Direct download",
                                         retrieval_time=time.time() - start_time)
 
-        # Find available integrations TODO: Propagate name of actual integration to ScrapingResponse
-        method_map = {
-            "integrations": lambda: retrieve_via_integration(
-                url, session=session, max_video_size=max_video_size, include_media=include_media
-            ),
-            "firecrawl": lambda: fire.scrape(
-                url, session=session, format=format, actions=actions, include_media=include_media
-            ),
-            "decodo": lambda: decodo.scrape(
-                url, session,
-                format=format,
-                timeout=15 if prioritize == "speed" else 60,
-                max_retries=1 if prioritize == "speed" else 5,
-                include_media=include_media),
-        }
-
-        result = None
-        errors = {}
+        # Map method names to their corresponding retrieval functions
+        method_retrieval_tasks = []
+        for method in methods:
+            if method == "firecrawl":
+                method_retrieval_tasks.append(fire.scrape(url, session=session, format=format, actions=actions,
+                                                include_media=include_media))
+            elif method == "decodo":
+                method_retrieval_tasks.append(decodo.scrape(url, session, format=format,
+                                                  timeout=15 if prioritize == "speed" else 60,
+                                                  max_retries=1 if prioritize == "speed" else 5,
+                                                  include_media=include_media))
+            else:
+                method_retrieval_tasks.append(retrieve_via_integration(url, integration_name=method, session=session,
+                                                             max_video_size=max_video_size,
+                                                             include_media=include_media))
 
     except Exception as e:
         logger.error(f"Error while preparing retrieval for '{url}'.\n" + format_exc())
-        return ScrapingResponse(url=url, content=None, errors=dict(all=e), retrieval_time=time.time() - start_time)
+        return ScrapingResponse(url=url, content=None, errors=dict(scrapemm=e), retrieval_time=time.time() - start_time)
 
     # Try each method in the specified order until one succeeds
-    for method_name in methods:
-        if method_name not in method_map:
-            logger.warning(f"Unknown retrieval method '{method_name}'. Skipping...")
-            continue
-
+    result = None
+    errors = {}
+    logger.debug(f"Trying methods in order: {', '.join(methods)}")
+    for method_name, retrieval_task in zip(methods, method_retrieval_tasks):
         logger.debug(f"Trying method: {method_name}")
 
         try:
-            result = await method_map[method_name]()
+            result = await retrieval_task
 
         except NotImplementedError as e:
             logger.info("Reached a method that is not implemented.", exc_info=True)
-            errors[method_name] = e
-            result = None
+            errors[method_name] = f"{type(e).__name__}: {e}"
 
         except sqlite3.OperationalError as e:
             if str(e) == "attempt to write a readonly database":
@@ -229,29 +230,29 @@ async def _retrieve_single(
                 raise
             else:
                 logger.warning(f"Error while retrieving with method '{method_name}': {e}")
-                errors[method_name] = e
-                result = None
+                errors[method_name] = f"{type(e).__name__}: {e}"
 
         except (TimeoutError, PlaywrightTimeoutError) as e:
             logger.warning(f"Timeout while retrieving with method '{method_name}': {e}")
-            errors[method_name] = e
-            result = None
+            errors[method_name] = f"{type(e).__name__}: {e}"
 
         except IPBannedError as e:
-            logger.error(e)
-            errors[method_name] = e
-            result = None
+            logger.info(e)
+            errors[method_name] = f"{type(e).__name__}: {e}"
+
+        except OSError as e:
+            if "Disk is full" in str(e):
+                logger.critical("❌ Disk is full! Please free up space and try again. Aborting.")
+                raise DiskFull()
 
         except Exception as e:
             logger.warning(f"Error while retrieving with method '{method_name}': {e}")
             errors[method_name] = f"{type(e).__name__}: {e}"
-            result = None
 
         if result is not None:
             logger.debug(f"Successfully retrieved with method: {method_name}")
             if isinstance(result, MultimodalSequence):
-                if include_media:
-                    postprocess_media(result)
+                postprocess_media(result)
             return ScrapingResponse(url=url, content=result, method=method_name,
                                     retrieval_time=time.time() - start_time)
 
@@ -263,11 +264,25 @@ async def _retrieve_single(
 
     # All methods failed
     logger.warning(f"All retrieval methods failed for URL: {url}")
-    if not errors:
-        errors = dict(scrapemm=RetrievalFailed(f"No retrieval method produced content for '{url}'."))
     return ScrapingResponse(url=url, content=None, errors=errors, retrieval_time=time.time() - start_time)
 
 
+def resolve_best_methods(url: str, allowed_methods: Literal["auto"] | list[str]) -> list[str]:
+    """Returns the best retrieval methods for the given URL."""
+    # Initialize methods list
+    methods = get_optimal_methods(url) if allowed_methods == "auto" else allowed_methods
+
+    # Resolve 'integrations' method to specific, applicable integrations, maintaining order
+    methods_resolved = []
+    for method in methods:
+        if method == "integrations":
+            methods_resolved.extend(get_integrations_for_url(url))
+        else:
+            methods_resolved.append(method)
+
+    return methods_resolved
+
+
 def postprocess_media(result: MultimodalSequence):
     """Ensure all media are located in the default ezmm directory (no temp files)
     and transcode all videos into a format suitable for browser playback."""

{scrapemm-0.7.3 → scrapemm-0.7.4}/scripts/example.py

@@ -2,8 +2,8 @@ from scrapemm import retrieve
 import asyncio
 
 if __name__ == "__main__":
-    url = "https://srilanka.newschecker.co/en/fact-check/videos-of-2022-aragalaya-revived-shared-as-recent"
-    result = asyncio.run(retrieve(url))
+    url = "https://www.vishvasnews.com/viral/fact-check-upsc-has-not-reduced-the-maximum-age-limit-for-ias-and-ips-exams/"
+    result = asyncio.run(retrieve(url, format="html", methods=["firecrawl"]))
     print(f"Retrieval time: {result.retrieval_time:.2f} seconds")
     if result.errors:
         print(result.errors)

{scrapemm-0.7.3 → scrapemm-0.7.4}/testing/test_retrieval.py

@@ -7,16 +7,16 @@ from scrapemm import retrieve
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("url", [
+    "https://factcheckarabic.afp.com/doc.afp.com.9DD6J8",
     "https://www.vishvasnews.com/viral/fact-check-upsc-has-not-reduced-the-maximum-age-limit-for-ias-and-ips-exams/",
     "https://health.medicaldialogues.in/fact-check/brain-health-fact-check/fact-check-is-sprite-the-best-remedy-for-headaches-in-the-world-140368",
     "https://assamese.factcrescendo.com/viral-claim-that-the-video-shows-the-incident-from-uttar-pradesh-and-the-youth-on-the-bike-and-the-youth-being-beaten-and-taken-away-by-the-police-are-the-same-youth-named-abdul-is-false/",
-    # "https://factuel.afp.com/doc.afp.com.43ZN7NP",
+    "https://factuel.afp.com/doc.afp.com.43ZN7NP",
     "https://leadstories.com/365cb414b83e29d26fecae374d55c743a3eac4c7.png",
     "https://leadstories.com/assets_c/2025/08/193f14f06dd6f15b89bf8050e553ad7fb1be6530-thumb-900xauto-3165872.png"
 ])
-@pytest.mark.parametrize("method", ["firecrawl", "decodo"])
-async def test_generic_retrieval(url, method):
-    result = await retrieve(url, methods=[method])
+async def test_generic_retrieval(url):
+    result = await retrieve(url)
     assert isinstance(result, ScrapingResponse)
     print(result)
     assert result
@@ -27,6 +27,7 @@ async def test_generic_retrieval(url, method):
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("url", [
+    "https://www.vishvasnews.com/viral/fact-check-upsc-has-not-reduced-the-maximum-age-limit-for-ias-and-ips-exams/",
     "https://www.zeit.de/politik/deutschland/2025-07/spionage-iran-festnahme-anschlag-juden-berlin-daenemark",
     "https://factnameh.com/fa/fact-checks/2025-04-16-araghchi-witkoff-fake-photo",
     "https://www.thip.media/health-news-fact-check/fact-check-can-a-kalava-on-the-wrist-prevent-paralysis/74724/",
@@ -35,6 +36,7 @@ async def test_generic_retrieval(url, method):
 async def test_html_retrieval(url, method):
     result = await retrieve(url, format="html", methods=[method])
     assert isinstance(result, ScrapingResponse)
+    print(result)
     content = result.content
     print(content)
     assert content
@@ -66,13 +68,13 @@ async def test_max_video_size(url, max_video_size, download_expected):
     ([
          "https://www.youtube.com/shorts/cE0zgN6pYOc"
      ], [
-         "integrations"
+         "YouTube"
      ]),
     ([
          "https://www.facebook.com/reel/1089214926521000",
          "https://www.zeit.de/politik/deutschland/2025-07/spionage-iran-festnahme-anschlag-juden-berlin-daenemark",
      ], [
-         ["integrations"],
+         ["Facebook"],
          ["firecrawl"]
      ]),
     ([
@@ -99,6 +101,7 @@ async def test_max_video_size(url, max_video_size, download_expected):
 async def test_methods(urls: list[str], methods: list[str] | list[list[str]] | None):
     results = await retrieve(urls, methods=methods)
     assert results
+    print(results)
     if methods and methods != "auto":
         if isinstance(methods[0], str):
             methods = [methods] * len(urls)

{scrapemm-0.7.3 → scrapemm-0.7.4}/testing/test_utils.py

@@ -44,18 +44,3 @@ def test_extract_skips_emoji_background_image():
     )
     soup = BeautifulSoup(html, "html.parser")
     assert _extract_media_elements(soup) == []
-
-
-def test_extract_skips_background_on_content_wrapper():
-    """Archive.today wraps the whole snapshot in a div with a decorative background-image.
-    That wrapper must not be treated as media, or decompose() would wipe the article image."""
-    html = (
-        '<div class="html1" style="background-image:url(\'/Edqcv/bg.png\')">'
-        '<img src="/Edqcv/article.webp"/>'
-        '</div>'
-    )
-    soup = BeautifulSoup(html, "html.parser")
-    elements = _extract_media_elements(soup)
-    assert len(elements) == 1
-    assert elements[0].name == "img"
-    assert elements[0].get("src") == "/Edqcv/article.webp"

scrapemm-0.7.3/scrapemm/download/requests.py

@@ -1,74 +0,0 @@
-import logging
-from typing import Optional, Union, TYPE_CHECKING
-
-import aiohttp
-
-if TYPE_CHECKING:
-    from playwright.async_api import APIRequestContext
-
-from scrapemm.download.common import ssl_context, RELAXED_SSL_DOMAINS
-from scrapemm.download.util import stream
-
-logger = logging.getLogger("scrapeMM")
-
-
-async def fetch_headers(url, session: Union[aiohttp.ClientSession, "APIRequestContext"], **kwargs) -> dict:
-    """Fetch only HTTP headers for a URL."""
-    from scrapemm.util import get_domain
-    ssl = None if get_domain(str(url)) in RELAXED_SSL_DOMAINS else ssl_context
-
-    if hasattr(session, "head"):  # aiohttp.ClientSession or Playwright APIRequestContext
-        try:
-            # Playwright APIRequestContext.head exists and returns APIResponse
-            if not isinstance(session, aiohttp.ClientSession):
-                # Playwright
-                response = await session.head(str(url), headers=kwargs.get("headers"), timeout=kwargs.get("timeout"))
-                return response.headers
-            else:
-                # aiohttp
-                async with session.head(url, ssl=ssl, **kwargs) as response:
-                    response.raise_for_status()
-                    return dict(response.headers)
-        except Exception:
-            logger.debug(f"HEAD failed for {url}, falling back to GET", exc_info=True)
-            # Fallback to GET
-            if not isinstance(session, aiohttp.ClientSession):
-                response = await session.get(str(url), headers=kwargs.get("headers"), timeout=kwargs.get("timeout"))
-                return response.headers
-            else:
-                async with session.get(url, ssl=ssl, **kwargs) as response:
-                    response.raise_for_status()
-                    return dict(response.headers)
-    raise ValueError(f"Unsupported session type: {type(session)}")
-
-
-async def request_static(url: str,
-                         session: Union[aiohttp.ClientSession, "APIRequestContext"],
-                         get_text: bool = True,
-                         **kwargs) -> Optional[str | bytes]:
-    """Downloads the static page from the given URL using aiohttp or Playwright."""
-    if not url:
-        return None
-
-    url = str(url)
-    from scrapemm.util import get_domain
-    ssl = None if get_domain(url) in RELAXED_SSL_DOMAINS else ssl_context
-
-    try:
-        if not isinstance(session, aiohttp.ClientSession):
-            # Playwright APIRequestContext
-            response = await session.get(url, **kwargs)
-            if response.ok:
-                return await response.text() if get_text else await response.body()
-            return None
-        else:
-            # aiohttp
-            async with session.get(url, timeout=10, allow_redirects=True,
-                                   raise_for_status=True, ssl=ssl, **kwargs) as response:
-                if get_text:
-                    return await response.text()
-                else:
-                    return await stream(response)
-    except Exception:
-        logger.debug(f"Error requesting {url}", exc_info=True)
-        return None