scrapeMM 0.3.1__tar.gz → 0.3.2__tar.gz

{scrapemm-0.3.1/scrapeMM.egg-info → scrapemm-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.1
+Version: 0.3.2
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.3.1 → scrapemm-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.3.1"
+version = "0.3.2"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"

{scrapemm-0.3.1 → scrapemm-0.3.2/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.1
+Version: 0.3.2
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.3.1 → scrapemm-0.3.2}/scrapemm/retrieval.py

@@ -114,7 +114,11 @@ async def _retrieve_single(
                 continue
 
             logger.debug(f"Trying method: {method_name}")
-            result = await method_map[method_name]()
+            try:
+                result = await method_map[method_name]()
+            except Exception as e:
+                logger.warning(f"Error while retrieving with method '{method_name}': {e}")
+                result = None
 
             if result is not None:
                 logger.debug(f"Successfully retrieved with method: {method_name}")

{scrapemm-0.3.1 → scrapemm-0.3.2}/scrapemm/scraping/decodo.py

@@ -111,8 +111,6 @@ class Decodo:
         # Create basic auth
         auth = aiohttp.BasicAuth(self.username, self.password)
 
-        # logger.debug(f"Decodo request payload: {payload}")
-
         try:
             async with session.post(
                 self.DECODO_API_URL,
@@ -121,26 +119,17 @@ class Decodo:
                 auth=auth,
                 timeout=aiohttp.ClientTimeout(total=timeout)
             ) as response:
-
+                # Validate response health
                 if response.status != 200:
-                    # Try to get the error message from response
-                    try:
-                        error_body = await response.json()
-                        error_msg = error_body.get('message', error_body)
-                    except:
-                        error_msg = await response.text()
-
-                    logger.info(
-                        f"Failed to scrape {url} with Decodo\n"
-                        f"Status code: {response.status} - Reason: {response.reason}\n"
-                        f"Error details: {error_msg}"
-                    )
-
+                    logger.debug("Communication with Decodo API failed.")
                     match response.status:
+                        case 400:
+                            logger.debug("Error 400: Bad request. If you use JavaScript, make sure you have the "
+                                         "Advanced plan subscription.")
                         case 401:
                             logger.error("Error 401: Unauthorized. Check your Decodo credentials.")
                         case 402:
-                            logger.debug("Error 402: Payment required. Check your Decodo subscription.")
+                            logger.error("Error 402: Payment required. Check your Decodo subscription.")
                         case 403:
                             logger.debug("Error 403: Forbidden.")
                         case 408:
@@ -157,6 +146,13 @@ class Decodo:
                 # Parse response
                 json_response = await response.json()
 
+                # Validate if scrape was successful
+                if json_response.get("status") == "failed":
+                    status_code = json_response.get("status_code")
+                    message = json_response.get("message")
+                    logger.info(f"Decodo failed to scrape. Error {status_code}: {message}")
+                    return None
+
                 # Extract HTML content from results
                 if "results" in json_response and len(json_response["results"]) > 0:
                     result = json_response["results"][0]

{scrapemm-0.3.1 → scrapemm-0.3.2}/scrapemm/scraping/util.py

@@ -6,6 +6,7 @@ from typing import Optional
 
 import aiohttp
 import requests
+from PIL import UnidentifiedImageError
 from ezmm import MultimodalSequence, download_item, Item, Image, Video
 from markdownify import markdownify as md
 from requests.exceptions import ReadTimeout, ConnectionError, RetryError
@@ -159,7 +160,8 @@ def from_base64(b64_data: str, mime_type: str = "image/jpeg") -> Optional[Item]:
             else:
                 raise ValueError(f"Unsupported media type: {mime_type}")
     except binascii.Error:  # base64 validation failed
-        return
+        return None
+    except UnidentifiedImageError:  # Pillow could not identify image format
+        return None
     except Exception as e:
-        logger.warning(f"Error decoding base64 data!\n"
-                       f"{type(e).__name__}: {e}")
+        logger.debug(f"Error decoding {mime_type} base64 data. \n {type(e).__name__}: {e}")

scrapemm-0.3.2/scripts/example.py

@@ -0,0 +1,7 @@
+from scrapemm import retrieve
+import asyncio
+
+if __name__ == "__main__":
+    url = "https://verafiles.org"
+    result = asyncio.run(retrieve(url))
+    print(result)

scrapemm-0.3.1/scripts/example.py

@@ -1,8 +0,0 @@
-from scrapemm import retrieve
-import asyncio
-
-if __name__ == "__main__":
-    url = "https://www.tiktok.com/@xxxx.xxxx5743/video/7521704371109793046"
-    loop = asyncio.get_event_loop()
-    result = loop.run_until_complete(retrieve(url))
-    print(result)