scrapeMM 0.3.1__tar.gz → 0.3.3__tar.gz

{scrapemm-0.3.1/scrapeMM.egg-info → scrapemm-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.1
+Version: 0.3.3
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.3.1 → scrapemm-0.3.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.3.1"
+version = "0.3.3"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"

{scrapemm-0.3.1 → scrapemm-0.3.3/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.1
+Version: 0.3.3
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.3.1 → scrapemm-0.3.3}/scrapemm/retrieval.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Optional
+from typing import Optional, Collection
 from traceback import format_exc
 
 import aiohttp
@@ -14,7 +14,7 @@ METHODS = ["integrations", "firecrawl", "decodo"]
 
 
 async def retrieve(
-        urls: str | list[str],
+        urls: str | Collection[str],
         remove_urls: bool = False,
         show_progress: bool = True,
         actions: list[dict] = None,
@@ -114,7 +114,11 @@ async def _retrieve_single(
                 continue
 
             logger.debug(f"Trying method: {method_name}")
-            result = await method_map[method_name]()
+            try:
+                result = await method_map[method_name]()
+            except Exception as e:
+                logger.warning(f"Error while retrieving with method '{method_name}': {e}")
+                result = None
 
             if result is not None:
                 logger.debug(f"Successfully retrieved with method: {method_name}")

{scrapemm-0.3.1 → scrapemm-0.3.3}/scrapemm/scraping/decodo.py

@@ -6,7 +6,7 @@ import aiohttp
 from ezmm import MultimodalSequence
 
 from scrapemm.secrets import get_secret
-from scrapemm.scraping.util import to_multimodal_sequence
+from scrapemm.scraping.util import to_multimodal_sequence, get_domain_root
 
 logger = logging.getLogger("scrapeMM")
 
@@ -74,7 +74,8 @@ class Decodo:
             if format == "html":
                 return html
             else:
-                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session)
+                domain_root = get_domain_root(url)
+                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session, domain_root=domain_root)
         return None
 
     async def _call_decodo(self, url: str,
@@ -111,8 +112,6 @@ class Decodo:
         # Create basic auth
         auth = aiohttp.BasicAuth(self.username, self.password)
 
-        # logger.debug(f"Decodo request payload: {payload}")
-
         try:
             async with session.post(
                 self.DECODO_API_URL,
@@ -121,26 +120,17 @@ class Decodo:
                 auth=auth,
                 timeout=aiohttp.ClientTimeout(total=timeout)
             ) as response:
-
+                # Validate response health
                 if response.status != 200:
-                    # Try to get the error message from response
-                    try:
-                        error_body = await response.json()
-                        error_msg = error_body.get('message', error_body)
-                    except:
-                        error_msg = await response.text()
-
-                    logger.info(
-                        f"Failed to scrape {url} with Decodo\n"
-                        f"Status code: {response.status} - Reason: {response.reason}\n"
-                        f"Error details: {error_msg}"
-                    )
-
+                    logger.debug("Communication with Decodo API failed.")
                     match response.status:
+                        case 400:
+                            logger.debug("Error 400: Bad request. If you use JavaScript, make sure you have the "
+                                         "Advanced plan subscription.")
                         case 401:
                             logger.error("Error 401: Unauthorized. Check your Decodo credentials.")
                         case 402:
-                            logger.debug("Error 402: Payment required. Check your Decodo subscription.")
+                            logger.error("Error 402: Payment required. Check your Decodo subscription.")
                         case 403:
                             logger.debug("Error 403: Forbidden.")
                         case 408:
@@ -157,6 +147,13 @@ class Decodo:
                 # Parse response
                 json_response = await response.json()
 
+                # Validate if scrape was successful
+                if json_response.get("status") == "failed":
+                    status_code = json_response.get("status_code")
+                    message = json_response.get("message")
+                    logger.info(f"Decodo failed to scrape. Error {status_code}: {message}")
+                    return None
+
                 # Extract HTML content from results
                 if "results" in json_response and len(json_response["results"]) > 0:
                     result = json_response["results"][0]

{scrapemm-0.3.1 → scrapemm-0.3.3}/scrapemm/scraping/firecrawl.py

@@ -7,7 +7,7 @@ from ezmm import MultimodalSequence
 
 from scrapemm.common import get_config_var, update_config
 from scrapemm.util import read_urls_from_file, get_domain
-from scrapemm.scraping.util import find_firecrawl, to_multimodal_sequence, firecrawl_is_running
+from scrapemm.scraping.util import find_firecrawl, to_multimodal_sequence, firecrawl_is_running, get_domain_root
 
 logger = logging.getLogger("scrapeMM")
 
@@ -58,6 +58,7 @@ class Firecrawl:
 
     def connect(self):
         from firecrawl import AsyncFirecrawl
+        logging.getLogger("firecrawl").setLevel(logging.WARNING)
         self.firecrawl_url = locate_firecrawl()
         self._firecrawl = AsyncFirecrawl(api_url=self.firecrawl_url)
 
@@ -88,7 +89,9 @@ class Firecrawl:
             if format == "html":
                 return html
             else:
-                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session)
+                domain_root = get_domain_root(url)
+                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session, domain_root=domain_root)
+        return None
 
 
 fire = Firecrawl()

{scrapemm-0.3.1 → scrapemm-0.3.3}/scrapemm/scraping/util.py

@@ -6,6 +6,7 @@ from typing import Optional
 
 import aiohttp
 import requests
+from PIL import UnidentifiedImageError
 from ezmm import MultimodalSequence, download_item, Item, Image, Video
 from markdownify import markdownify as md
 from requests.exceptions import ReadTimeout, ConnectionError, RetryError
@@ -53,6 +54,7 @@ def postprocess_scraped(text: str) -> str:
 
 async def resolve_media_hyperlinks(
         text: str, session: aiohttp.ClientSession,
+        domain_root: str = None,
         remove_urls: bool = False,
 ) -> Optional[MultimodalSequence]:
     """Downloads all media that are hyperlinked in the provided Markdown text.
@@ -64,19 +66,21 @@ async def resolve_media_hyperlinks(
 
     # Extract URLs and base64-encoded data from the text
     hyperlinks = get_markdown_hyperlinks(text)
-    urls = set()
+    hrefs_urls = dict()
     data_uris = set()
     for _, _, href in hyperlinks:
         if is_url(href):
-            urls.add(href)
+            hrefs_urls[href] = href
+        elif domain_root and is_root_relative_url(href):
+            hrefs_urls[href] = f"{domain_root}{href}"
         elif is_data_uri(href):
             data_uris.add(href)
 
     # Try to download media for each URL
-    tasks = [download_item(url, session=session) for url in urls]
+    tasks = [download_item(url, session=session) for url in hrefs_urls.values()]
     media: list[Item | None] = await run_with_semaphore(tasks, limit=100, show_progress=False)
 
-    href_media = dict(zip(urls, media))
+    href_media = dict(zip(hrefs_urls.keys(), media))
 
     # Convert each base64-encoded data to the respective medium
     for data_uri in data_uris:
@@ -101,15 +105,29 @@ async def resolve_media_hyperlinks(
 
 
 def is_url(href: str) -> bool:
-    """Returns True iff the given string is a valid URL."""
+    """Returns True iff the given string is an absolute HTTP URL."""
     return re.match(URL_REGEX, href) is not None
 
 
+def is_root_relative_url(href: str) -> bool:
+    """Returns True iff the given string is a root-relative URL."""
+    return href.startswith("/")
+
+
 def is_data_uri(href: str) -> bool:
     """Returns True iff the given string is a valid data URI."""
     return re.match(DATA_URI_REGEX, href) is not None
 
 
+def get_domain_root(url: str) -> Optional[str]:
+    """Extracts the domain root from the given URL. Allows for missing http(s) prefix."""
+    match = re.match(r"(:?https?://)?([^/]+)", url)
+    if match:
+        return match.group(0)
+    else:
+        return None
+
+
 def get_markdown_hyperlinks(text: str) -> list[tuple[str, str, str]]:
     """Extracts all web hyperlinks from the given markdown-formatted string. Returns
     a list of fullmatch-hypertext-URL-triples."""
@@ -159,7 +177,8 @@ def from_base64(b64_data: str, mime_type: str = "image/jpeg") -> Optional[Item]:
             else:
                 raise ValueError(f"Unsupported media type: {mime_type}")
     except binascii.Error:  # base64 validation failed
-        return
+        return None
+    except UnidentifiedImageError:  # Pillow could not identify image format
+        return None
     except Exception as e:
-        logger.warning(f"Error decoding base64 data!\n"
-                       f"{type(e).__name__}: {e}")
+        logger.debug(f"Error decoding {mime_type} base64 data. \n {type(e).__name__}: {e}")

scrapemm-0.3.3/scripts/example.py

@@ -0,0 +1,7 @@
+from scrapemm import retrieve
+import asyncio
+
+if __name__ == "__main__":
+    url = "https://factuel.afp.com/doc.afp.com.43ZN7NP"
+    result = asyncio.run(retrieve(url, methods=["decodo"]))
+    print(result)

scrapemm-0.3.1/scripts/example.py

@@ -1,8 +0,0 @@
-from scrapemm import retrieve
-import asyncio
-
-if __name__ == "__main__":
-    url = "https://www.tiktok.com/@xxxx.xxxx5743/video/7521704371109793046"
-    loop = asyncio.get_event_loop()
-    result = loop.run_until_complete(retrieve(url))
-    print(result)