scrapeMM 0.3.2__tar.gz → 0.3.3__tar.gz

{scrapemm-0.3.2/scrapeMM.egg-info → scrapemm-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.2
+Version: 0.3.3
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.3.2 → scrapemm-0.3.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.3.2"
+version = "0.3.3"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"

{scrapemm-0.3.2 → scrapemm-0.3.3/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.2
+Version: 0.3.3
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.3.2 → scrapemm-0.3.3}/scrapemm/retrieval.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Optional
+from typing import Optional, Collection
 from traceback import format_exc
 
 import aiohttp
@@ -14,7 +14,7 @@ METHODS = ["integrations", "firecrawl", "decodo"]
 
 
 async def retrieve(
-        urls: str | list[str],
+        urls: str | Collection[str],
         remove_urls: bool = False,
         show_progress: bool = True,
         actions: list[dict] = None,

{scrapemm-0.3.2 → scrapemm-0.3.3}/scrapemm/scraping/decodo.py

@@ -6,7 +6,7 @@ import aiohttp
 from ezmm import MultimodalSequence
 
 from scrapemm.secrets import get_secret
-from scrapemm.scraping.util import to_multimodal_sequence
+from scrapemm.scraping.util import to_multimodal_sequence, get_domain_root
 
 logger = logging.getLogger("scrapeMM")
 
@@ -74,7 +74,8 @@ class Decodo:
             if format == "html":
                 return html
             else:
-                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session)
+                domain_root = get_domain_root(url)
+                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session, domain_root=domain_root)
         return None
 
     async def _call_decodo(self, url: str,

{scrapemm-0.3.2 → scrapemm-0.3.3}/scrapemm/scraping/firecrawl.py

@@ -7,7 +7,7 @@ from ezmm import MultimodalSequence
 
 from scrapemm.common import get_config_var, update_config
 from scrapemm.util import read_urls_from_file, get_domain
-from scrapemm.scraping.util import find_firecrawl, to_multimodal_sequence, firecrawl_is_running
+from scrapemm.scraping.util import find_firecrawl, to_multimodal_sequence, firecrawl_is_running, get_domain_root
 
 logger = logging.getLogger("scrapeMM")
 
@@ -58,6 +58,7 @@ class Firecrawl:
 
     def connect(self):
         from firecrawl import AsyncFirecrawl
+        logging.getLogger("firecrawl").setLevel(logging.WARNING)
         self.firecrawl_url = locate_firecrawl()
         self._firecrawl = AsyncFirecrawl(api_url=self.firecrawl_url)
 
@@ -88,7 +89,9 @@ class Firecrawl:
             if format == "html":
                 return html
             else:
-                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session)
+                domain_root = get_domain_root(url)
+                return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session, domain_root=domain_root)
+        return None
 
 
 fire = Firecrawl()

{scrapemm-0.3.2 → scrapemm-0.3.3}/scrapemm/scraping/util.py

@@ -54,6 +54,7 @@ def postprocess_scraped(text: str) -> str:
 
 async def resolve_media_hyperlinks(
         text: str, session: aiohttp.ClientSession,
+        domain_root: str = None,
         remove_urls: bool = False,
 ) -> Optional[MultimodalSequence]:
     """Downloads all media that are hyperlinked in the provided Markdown text.
@@ -65,19 +66,21 @@ async def resolve_media_hyperlinks(
 
     # Extract URLs and base64-encoded data from the text
     hyperlinks = get_markdown_hyperlinks(text)
-    urls = set()
+    hrefs_urls = dict()
     data_uris = set()
     for _, _, href in hyperlinks:
         if is_url(href):
-            urls.add(href)
+            hrefs_urls[href] = href
+        elif domain_root and is_root_relative_url(href):
+            hrefs_urls[href] = f"{domain_root}{href}"
         elif is_data_uri(href):
             data_uris.add(href)
 
     # Try to download media for each URL
-    tasks = [download_item(url, session=session) for url in urls]
+    tasks = [download_item(url, session=session) for url in hrefs_urls.values()]
     media: list[Item | None] = await run_with_semaphore(tasks, limit=100, show_progress=False)
 
-    href_media = dict(zip(urls, media))
+    href_media = dict(zip(hrefs_urls.keys(), media))
 
     # Convert each base64-encoded data to the respective medium
     for data_uri in data_uris:
@@ -102,15 +105,29 @@ async def resolve_media_hyperlinks(
 
 
 def is_url(href: str) -> bool:
-    """Returns True iff the given string is a valid URL."""
+    """Returns True iff the given string is an absolute HTTP URL."""
     return re.match(URL_REGEX, href) is not None
 
 
+def is_root_relative_url(href: str) -> bool:
+    """Returns True iff the given string is a root-relative URL."""
+    return href.startswith("/")
+
+
 def is_data_uri(href: str) -> bool:
     """Returns True iff the given string is a valid data URI."""
     return re.match(DATA_URI_REGEX, href) is not None
 
 
+def get_domain_root(url: str) -> Optional[str]:
+    """Extracts the domain root from the given URL. Allows for missing http(s) prefix."""
+    match = re.match(r"(:?https?://)?([^/]+)", url)
+    if match:
+        return match.group(0)
+    else:
+        return None
+
+
 def get_markdown_hyperlinks(text: str) -> list[tuple[str, str, str]]:
     """Extracts all web hyperlinks from the given markdown-formatted string. Returns
     a list of fullmatch-hypertext-URL-triples."""

scrapemm-0.3.3/scripts/example.py

@@ -0,0 +1,7 @@
+from scrapemm import retrieve
+import asyncio
+
+if __name__ == "__main__":
+    url = "https://factuel.afp.com/doc.afp.com.43ZN7NP"
+    result = asyncio.run(retrieve(url, methods=["decodo"]))
+    print(result)

scrapemm-0.3.2/scripts/example.py

@@ -1,7 +0,0 @@
-from scrapemm import retrieve
-import asyncio
-
-if __name__ == "__main__":
-    url = "https://verafiles.org"
-    result = asyncio.run(retrieve(url))
-    print(result)