scrapeMM 0.3.0__tar.gz → 0.3.2__tar.gz

{scrapemm-0.3.0/scrapeMM.egg-info → scrapemm-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.0
+Version: 0.3.2
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -19,6 +19,7 @@ Requires-Dist: atproto
 Requires-Dist: TikTokResearchApi
 Requires-Dist: yt-dlp
 Requires-Dist: cryptography
+Requires-Dist: firecrawl-py
 Dynamic: license-file
 
 # scrapeMM: Multimodal Web Retrieval

{scrapemm-0.3.0 → scrapemm-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.3.0"
+version = "0.3.2"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"
@@ -17,6 +17,7 @@ dependencies = [
     "TikTokResearchApi",
     "yt-dlp",
     "cryptography",
+    "firecrawl-py"
 ]
 
 [project.urls]

{scrapemm-0.3.0 → scrapemm-0.3.2/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.0
+Version: 0.3.2
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -19,6 +19,7 @@ Requires-Dist: atproto
 Requires-Dist: TikTokResearchApi
 Requires-Dist: yt-dlp
 Requires-Dist: cryptography
+Requires-Dist: firecrawl-py
 Dynamic: license-file
 
 # scrapeMM: Multimodal Web Retrieval

{scrapemm-0.3.0 → scrapemm-0.3.2}/scrapeMM.egg-info/requires.txt

@@ -8,3 +8,4 @@ atproto
 TikTokResearchApi
 yt-dlp
 cryptography
+firecrawl-py

{scrapemm-0.3.0 → scrapemm-0.3.2}/scrapemm/retrieval.py

@@ -18,8 +18,9 @@ async def retrieve(
         remove_urls: bool = False,
         show_progress: bool = True,
         actions: list[dict] = None,
-        methods: list[str] = None
-) -> Optional[MultimodalSequence] | list[Optional[MultimodalSequence]]:
+        methods: list[str] = None,
+        format: str = "multimodal_sequence",
+) -> Optional[MultimodalSequence | str] | list[Optional[MultimodalSequence | str]]:
     """Main function of this repository. Downloads the contents present at the given URL(s).
     For each URL, returns a MultimodalSequence containing text, images, and videos.
     Returns None if the corresponding URL is not supported or if retrieval failed.
@@ -37,6 +38,9 @@ async def retrieve(
         - "firecrawl" (Firecrawl scraping service)
         - "decodo" (Decodo Web Scraping API)
         You can specify any subset in any order, e.g., ["decodo", "firecrawl"] or ["integrations"].
+    :param format: The format of the output. Available formats:
+        - "multimodal_sequence" (MultimodalSequence containing parsed and downloaded media from the page)
+        - "html" (string containing the raw HTML code of the page, not compatible with 'integrations' method)
     """
     if methods is None:
         methods = METHODS
@@ -62,7 +66,7 @@ async def retrieve(
         urls_unique = set(urls_to_retrieve)
 
         # Retrieve URLs concurrently
-        tasks = [_retrieve_single(url, remove_urls, session, methods, actions) for url in urls_unique]
+        tasks = [_retrieve_single(url, remove_urls, session, methods, actions, format) for url in urls_unique]
         results = await run_with_semaphore(tasks, limit=20, show_progress=show_progress and len(urls_to_retrieve) > 1,
                                            progress_description="Retrieving URLs...")
 
@@ -80,21 +84,27 @@ async def _retrieve_single(
         session: aiohttp.ClientSession,
         methods: list[str],
         actions: list[dict] = None,
-) -> Optional[MultimodalSequence]:
+        format: str = "multimodal_sequence",
+) -> Optional[MultimodalSequence | str]:
     try:
         # Ensure URL is a string
         url = str(url)
 
+        # Ensure compatibility with methods
+        if format == "html" and "integrations" in methods:
+            methods.remove("integrations")
+
         # Try to download as medium
-        if medium := await download_item(url, session=session):
-            return MultimodalSequence(medium)
+        if format != "html":
+            if medium := await download_item(url, session=session):
+                return MultimodalSequence(medium)
 
         # Define available retrieval methods
         method_map = {
             "integrations": lambda: retrieve_via_integration(url, session),
             "firecrawl": lambda: fire.scrape(url, remove_urls=remove_urls,
-                                                  session=session, actions=actions),
-            "decodo": lambda: decodo.scrape(url, remove_urls, session),
+                                                  session=session, format=format, actions=actions),
+            "decodo": lambda: decodo.scrape(url, remove_urls, session, format=format),
         }
 
         # Try each method in the specified order until one succeeds
@@ -104,7 +114,11 @@ async def _retrieve_single(
                 continue
 
             logger.debug(f"Trying method: {method_name}")
-            result = await method_map[method_name]()
+            try:
+                result = await method_map[method_name]()
+            except Exception as e:
+                logger.warning(f"Error while retrieving with method '{method_name}': {e}")
+                result = None
 
             if result is not None:
                 logger.debug(f"Successfully retrieved with method: {method_name}")

{scrapemm-0.3.0 → scrapemm-0.3.2}/scrapemm/scraping/decodo.py

@@ -39,8 +39,9 @@ class Decodo:
     async def scrape(self, url: str,
                      remove_urls: bool,
                      session: aiohttp.ClientSession,
+                     format: str,
                      enable_js: bool = True,
-                     timeout: int = 30) -> Optional[MultimodalSequence]:
+                     timeout: int = 30) -> Optional[MultimodalSequence | str]:
         """Downloads the contents of the specified webpage using Decodo's API.
 
         Args:
@@ -70,7 +71,10 @@ class Decodo:
             html = await self._call_decodo(url, session, enable_js=False, timeout=timeout)
 
         if html:
-            return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session)
+            if format == "html":
+                return html
+            else:
+                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session)
         return None
 
     async def _call_decodo(self, url: str,
@@ -107,8 +111,6 @@ class Decodo:
         # Create basic auth
         auth = aiohttp.BasicAuth(self.username, self.password)
 
-        # logger.debug(f"Decodo request payload: {payload}")
-
         try:
             async with session.post(
                 self.DECODO_API_URL,
@@ -117,26 +119,17 @@ class Decodo:
                 auth=auth,
                 timeout=aiohttp.ClientTimeout(total=timeout)
             ) as response:
-
+                # Validate response health
                 if response.status != 200:
-                    # Try to get the error message from response
-                    try:
-                        error_body = await response.json()
-                        error_msg = error_body.get('message', error_body)
-                    except:
-                        error_msg = await response.text()
-
-                    logger.info(
-                        f"Failed to scrape {url} with Decodo\n"
-                        f"Status code: {response.status} - Reason: {response.reason}\n"
-                        f"Error details: {error_msg}"
-                    )
-
+                    logger.debug("Communication with Decodo API failed.")
                     match response.status:
+                        case 400:
+                            logger.debug("Error 400: Bad request. If you use JavaScript, make sure you have the "
+                                         "Advanced plan subscription.")
                         case 401:
                             logger.error("Error 401: Unauthorized. Check your Decodo credentials.")
                         case 402:
-                            logger.debug("Error 402: Payment required. Check your Decodo subscription.")
+                            logger.error("Error 402: Payment required. Check your Decodo subscription.")
                         case 403:
                             logger.debug("Error 403: Forbidden.")
                         case 408:
@@ -153,6 +146,13 @@ class Decodo:
                 # Parse response
                 json_response = await response.json()
 
+                # Validate if scrape was successful
+                if json_response.get("status") == "failed":
+                    status_code = json_response.get("status_code")
+                    message = json_response.get("message")
+                    logger.info(f"Decodo failed to scrape. Error {status_code}: {message}")
+                    return None
+
                 # Extract HTML content from results
                 if "results" in json_response and len(json_response["results"]) > 0:
                     result = json_response["results"][0]

{scrapemm-0.3.0 → scrapemm-0.3.2}/scrapemm/scraping/firecrawl.py

@@ -65,7 +65,8 @@ class Firecrawl:
                      url: str,
                      remove_urls: bool,
                      session: aiohttp.ClientSession,
-                     **kwargs) -> Optional[MultimodalSequence]:
+                     format: str,
+                     **kwargs) -> Optional[MultimodalSequence | str]:
         if is_no_bot_site(url):
             return None
 
@@ -84,7 +85,10 @@ class Firecrawl:
         html = document.html
 
         if html:
-            return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session)
+            if format == "html":
+                return html
+            else:
+                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session)
 
 
 fire = Firecrawl()

{scrapemm-0.3.0 → scrapemm-0.3.2}/scrapemm/scraping/util.py

@@ -6,6 +6,7 @@ from typing import Optional
 
 import aiohttp
 import requests
+from PIL import UnidentifiedImageError
 from ezmm import MultimodalSequence, download_item, Item, Image, Video
 from markdownify import markdownify as md
 from requests.exceptions import ReadTimeout, ConnectionError, RetryError
@@ -159,7 +160,8 @@ def from_base64(b64_data: str, mime_type: str = "image/jpeg") -> Optional[Item]:
             else:
                 raise ValueError(f"Unsupported media type: {mime_type}")
     except binascii.Error:  # base64 validation failed
-        return
+        return None
+    except UnidentifiedImageError:  # Pillow could not identify image format
+        return None
     except Exception as e:
-        logger.warning(f"Error decoding base64 data!\n"
-                       f"{type(e).__name__}: {e}")
+        logger.debug(f"Error decoding {mime_type} base64 data. \n {type(e).__name__}: {e}")

scrapemm-0.3.2/scripts/example.py

@@ -0,0 +1,7 @@
+from scrapemm import retrieve
+import asyncio
+
+if __name__ == "__main__":
+    url = "https://verafiles.org"
+    result = asyncio.run(retrieve(url))
+    print(result)

{scrapemm-0.3.0 → scrapemm-0.3.2}/testing/test_retrieval.py

@@ -1,25 +1,39 @@
 import pytest
+from ezmm import MultimodalSequence
 
 from scrapemm import retrieve
 
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("url", [
-    "https://www.zeit.de/politik/deutschland/2025-07/spionage-iran-festnahme-anschlag-juden-berlin-daenemark",
-    "https://factnameh.com/fa/fact-checks/2025-04-16-araghchi-witkoff-fake-photo",
-    "https://www.thip.media/health-news-fact-check/fact-check-can-a-kalava-on-the-wrist-prevent-paralysis/74724/",
     "https://www.vishvasnews.com/viral/fact-check-upsc-has-not-reduced-the-maximum-age-limit-for-ias-and-ips-exams/",
-    "https://www.thip.media/health-news-fact-check/fact-check-does-wrapping-body-with-banana-leaves-help-with-obesity-and-indigestion/71333/",
     "https://health.medicaldialogues.in/fact-check/brain-health-fact-check/fact-check-is-sprite-the-best-remedy-for-headaches-in-the-world-140368",
     "https://www.washingtonpost.com/politics/2024/05/15/bidens-false-claim-that-inflation-was-9-percent-when-he-took-office/",
     "https://assamese.factcrescendo.com/viral-claim-that-the-video-shows-the-incident-from-uttar-pradesh-and-the-youth-on-the-bike-and-the-youth-being-beaten-and-taken-away-by-the-police-are-the-same-youth-named-abdul-is-false/",
     "https://factuel.afp.com/doc.afp.com.43ZN7NP",
     "https://leadstories.com/365cb414b83e29d26fecae374d55c743a3eac4c7.png",
 ])
-async def test_generic_retrieval(url):
-    result = await retrieve(url)
+@pytest.mark.parametrize("method", ["firecrawl", "decodo"])
+async def test_generic_retrieval(url, method):
+    result = await retrieve(url, methods=[method])
+    print(result)
+    assert result
+    assert isinstance(result, MultimodalSequence)
+    assert result.has_images()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("url", [
+    "https://www.zeit.de/politik/deutschland/2025-07/spionage-iran-festnahme-anschlag-juden-berlin-daenemark",
+    "https://factnameh.com/fa/fact-checks/2025-04-16-araghchi-witkoff-fake-photo",
+    "https://www.thip.media/health-news-fact-check/fact-check-can-a-kalava-on-the-wrist-prevent-paralysis/74724/",
+])
+@pytest.mark.parametrize("method", ["firecrawl", "decodo"])
+async def test_html_retrieval(url, method):
+    result = await retrieve(url, format="html", methods=[method])
     print(result)
     assert result
+    assert isinstance(result, str)
 
 
 @pytest.mark.asyncio
@@ -75,7 +89,7 @@ async def test_instagram_retrieval(url):
 @pytest.mark.parametrize("url", [
     "https://www.facebook.com/reel/1089214926521000",  # video
     "https://www.facebook.com/reel/3466446073497470",  # video, restricted for misinformation
-    "https://www.facebook.com/groups/1973976962823632/posts/3992825270938781/", # video, restricted for misinformation
+    "https://www.facebook.com/groups/1973976962823632/posts/3992825270938781/",  # video, restricted for misinformation
     "https://www.facebook.com/photo/?fbid=1721085455188778&set=a.107961589834514&_rdc=1&_rdr",  # image
 ])
 async def test_facebook_retrieval(url):

scrapemm-0.3.0/scripts/example.py

@@ -1,8 +0,0 @@
-from scrapemm import retrieve
-import asyncio
-
-if __name__ == "__main__":
-    url = "https://www.tiktok.com/@xxxx.xxxx5743/video/7521704371109793046"
-    loop = asyncio.get_event_loop()
-    result = loop.run_until_complete(retrieve(url))
-    print(result)