scrapeMM 0.5.2__tar.gz → 0.5.4__tar.gz

{scrapemm-0.5.2/scrapeMM.egg-info → scrapemm-0.5.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.5.2
+Version: 0.5.4
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.5.2 → scrapemm-0.5.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.5.2"
+version = "0.5.4"
 description = "Flexible multimodal scraper for social media and the open web."
 readme = "README.md"
 requires-python = ">=3.10"

{scrapemm-0.5.2 → scrapemm-0.5.4/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.5.2
+Version: 0.5.4
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.5.2 → scrapemm-0.5.4}/scrapeMM.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ scrapemm/download/requests.py
 scrapemm/download/util.py
 scrapemm/download/videos.py
 scrapemm/integrations/__init__.py
+scrapemm/integrations/archive_org.py
 scrapemm/integrations/base.py
 scrapemm/integrations/bluesky.py
 scrapemm/integrations/decodo.py

{scrapemm-0.5.2 → scrapemm-0.5.4}/scrapemm/download/requests.py

@@ -55,6 +55,8 @@ async def request_static(url: str,
             pass  # Page not readable
         except (aiohttp.ClientOSError, aiohttp.ClientConnectorError):
             pass  # Page not available anymore
+        except aiohttp.ServerDisconnectedError:
+            pass  # Server aborted the connection
         except aiohttp.ClientResponseError as e:
             if e.status in [403, 404, 429, 500, 502, 503]:
                 # 403: Forbidden access

{scrapemm-0.5.2 → scrapemm-0.5.4}/scrapemm/integrations/__init__.py

@@ -3,6 +3,7 @@ from typing import Optional
 from ezmm import MultimodalSequence
 
 from scrapemm.util import get_domain
+from .archive_org import ArchiveOrg
 from .bluesky import Bluesky
 from .decodo import Decodo, decodo
 from .fb import Facebook
@@ -14,7 +15,7 @@ from .x import X
 from .youtube import YouTube
 from .perma_cc import PermaCC
 
-RETRIEVAL_INTEGRATIONS = [X(), Telegram(), Bluesky(), TikTok(), Instagram(), Facebook(), YouTube(), PermaCC()]
+RETRIEVAL_INTEGRATIONS = [X(), Telegram(), Bluesky(), TikTok(), Instagram(), Facebook(), YouTube(), PermaCC(), ArchiveOrg()]
 DOMAIN_TO_INTEGRATION = {domain: integration
                          for integration in RETRIEVAL_INTEGRATIONS
                          for domain in integration.domains}

scrapemm-0.5.4/scrapemm/integrations/archive_org.py

@@ -0,0 +1,26 @@
+import logging
+
+from ezmm import MultimodalSequence
+
+from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.integrations.ytdlp import get_content_with_ytdlp
+
+logger = logging.getLogger("scrapeMM")
+
+
+class ArchiveOrg(RetrievalIntegration):
+    """Integration for retrieving content from archive.org.
+    TODO: Not working yet, but yt-dlp might support this soon:
+    https://github.com/yt-dlp/yt-dlp-master-builds/releases/tag/2025.12.12.222600"""
+    name = "Internet Archive"
+    domains = ["archive.org"]
+
+    async def _connect(self):
+        self.connected = True
+
+    async def _get(self, url: str, **kwargs) -> MultimodalSequence | None:
+        return await get_content_with_ytdlp(
+            url,
+            platform="Internet Archive",
+            **kwargs
+        )

{scrapemm-0.5.2 → scrapemm-0.5.4}/scrapemm/integrations/decodo.py

@@ -137,14 +137,14 @@ class Decodo:
                         if response.status == 429:  # Rate limit
                             if attempt >= max_retries:
                                 logger.warning(f"Error 429: Rate limit hit and maximum retries reached.")
-                                raise RateLimitError(f"Decodo rate limit reached.")
+                                raise RateLimitError(f"Decodo rate limit hit (despite {max_retries} retries).")
                         elif response.status == 613:
                             if attempt >= max_retries:
-                                raise RuntimeError("Decodo API error 613.")
+                                raise RuntimeError(f"Decodo API error 613 (despite {max_retries} retries).")
                         elif response.status == 502:  # Bad gateway
                             if attempt >= max_retries:
                                 logger.warning(f"Error 502: Bad gateway and maximum retries reached.")
-                                raise RuntimeError("Decodo API error 502: Bad gateway.")
+                                raise RuntimeError(f"Decodo API error 502: Bad gateway (despite {max_retries} retries).")
 
                         else:  # Other errors that don't go away on retry
                             match response.status:

{scrapemm-0.5.2 → scrapemm-0.5.4}/scrapemm/integrations/firecrawl/firecrawl.py

@@ -71,6 +71,7 @@ class Firecrawl:
                      remove_urls: bool,
                      session: aiohttp.ClientSession,
                      format: str,
+                     max_attempts: int = 3,
                      **kwargs) -> Optional[MultimodalSequence | str]:
         if is_no_bot_site(url):
             raise ValueError(f"Firecrawl cannot scrape sites from {get_domain(url)}")
@@ -78,25 +79,35 @@ class Firecrawl:
         if not self._firecrawl:
             await self.connect()
 
-        try:
-            document = await self._firecrawl.scrape(
-                url,
-                formats=["html"],
-                only_main_content=False,
-                remove_base64_images=False,
-                exclude_tags=["script", "style", "noscript", "footer", "aside"],
-                timeout=30_000,
-                wait_for=1_000,
-                store_in_cache=False,
-                **kwargs
-            )
-        except Exception as e:
-            # Ensure firecrawl is still running
-            if not await firecrawl_is_running(self.firecrawl_url):
-                logger.error(f"❌ Firecrawl stopped running at {self.firecrawl_url}.")
-                raise RuntimeError("Firecrawl stopped running.")
-            else:
-                raise e
+        document = None
+        for attempt in range(max_attempts):
+            try:
+                document = await self._firecrawl.scrape(
+                    url,
+                    formats=["html"],
+                    only_main_content=False,
+                    remove_base64_images=False,
+                    exclude_tags=["script", "style", "noscript", "footer", "aside"],
+                    timeout=30_000,
+                    wait_for=1_000,
+                    store_in_cache=False,
+                    **kwargs
+                )
+                break
+            except Exception as e:
+                # Ensure firecrawl is still running
+                state = await get_firecrawl_state(self.firecrawl_url)
+                if state == "unavailable":
+                    logger.error(f"❌ Firecrawl stopped running at {self.firecrawl_url}.")
+                    raise RuntimeError("Firecrawl stopped running.")
+                elif state == "busy":
+                    if attempt < max_attempts - 1:
+                        logger.warning(f"⚠️ Firecrawl seems busy. Retrying in 10 seconds...")
+                        await asyncio.sleep(10)
+                    else:
+                        raise e
+                else:
+                    raise e
 
         self.n_scrapes += 1
         html = document.html
@@ -128,15 +139,23 @@ async def find_firecrawl(urls):
 
 async def firecrawl_is_running(url: str) -> bool:
     """Returns True iff Firecrawl can be successfully pinged at the specified URL."""
+    return await get_firecrawl_state(url) == "running"
+
+
+async def get_firecrawl_state(url: str) -> str | None:
+    """Returns the state of Firecrawl at the specified URL."""
     if not url:
-        return False
+        return None
     if not url.startswith("http"):
         url = "https://" + url
 
     async with aiohttp.ClientSession(headers=HEADERS) as session:
         # Retrieve the head of the homepage
         try:
-            async with session.head(url, timeout=1) as response:
-                return 200 <= response.status < 400
-        except (aiohttp.ClientError, ConnectionError, ReadTimeout, RetryError, asyncio.TimeoutError):
-            return False
+            async with session.head(url, timeout=2) as response:
+                if 200 <= response.status < 400:
+                    return "running"
+        except (ReadTimeout, asyncio.TimeoutError):
+            return "busy"
+        except (aiohttp.ClientError, ConnectionError, RetryError):
+            return "unavailable"

{scrapemm-0.5.2 → scrapemm-0.5.4}/scrapemm/integrations/tiktok.py

@@ -28,7 +28,7 @@ class TikTok(RetrievalIntegration):
     """
 
     name = "TikTok"
-    domains = ["tiktok.com", "vm.tiktok.com"]
+    domains = ["tiktok.com"]
 
     async def _connect(self):
         # Try to initialize TikTok Research API

{scrapemm-0.5.2 → scrapemm-0.5.4}/scrapemm/retrieval.py

@@ -30,11 +30,16 @@ BEST_METHODS = {
     "twitter.com": ["integrations"],
     "t.co": ["integrations"],
     "t.me": ["integrations"],
+    "tiktok.com": ["integrations"],
     "telegram.me": ["integrations"],
+    "bsky.app": ["integrations"],
     "truthsocial.com": ["firecrawl"],
     "reddit.com": ["integrations"],
     "awesomescreenshot.com": ["firecrawl"],
     "snopes.com": ["decodo"],
+    "youtube.com": ["integrations"],
+    "youtu.be": ["integrations"],
+    "perma.cc": ["integrations"],
     # Archive.today:
     "archive.today": ["decodo"],
     "archive.is": ["decodo"],
@@ -53,6 +58,7 @@ async def retrieve(
         actions: list[dict] | None = None,
         methods: Literal["auto"] | list[str] | list[Literal["auto"] | list[str]] | None = "auto",
         format: str = "multimodal_sequence",
+        include_media: bool = True,  # TODO
         max_video_size: int | None = None,
 ) -> ScrapingResponse | list[ScrapingResponse]:
     """Main function of this repository. Downloads the contents present at the given URL(s).

{scrapemm-0.5.2 → scrapemm-0.5.4}/scrapemm/util.py

@@ -240,9 +240,9 @@ def normalize_video(video: Video):
     input_path = video.file_path
     output_path = input_path.with_suffix(".normalized.mp4")
 
-    meta = probe_video(input_path)
-
     try:
+        meta = probe_video(input_path)
+
         # Case 1: fully browser-safe → only ensure faststart
         if is_browser_safe(meta):
             run_command([

{scrapemm-0.5.2 → scrapemm-0.5.4}/scripts/example.py

@@ -2,7 +2,7 @@ from scrapemm import retrieve
 import asyncio
 
 if __name__ == "__main__":
-    url = "https://factuel.afp.com/doc.afp.com.43ZN7NP"
+    url = "https://web.archive.org/web/20210604181412/https://www.tiktok.com/@realstewpeters/video/6969789589590379781?is_copy_url=1"
     result = asyncio.run(retrieve(url))
     if result.errors:
         print(result.errors)