scrapeMM 0.1.0__py3-none-any.whl

scrapemm/__init__.py

@@ -0,0 +1,2 @@
+from .retrieval import retrieve
+from .integrations import Telegram, X

scrapemm/common.py

@@ -0,0 +1,30 @@
+import sys
+from pathlib import Path
+
+from scrapemm.util import get_domain
+import logging
+
+logger = logging.getLogger("Retriever")
+logger.setLevel(logging.DEBUG)
+
+# Only add handler if none exists (avoid duplicate logs on rerun)
+if not logger.hasHandlers():
+    handler = logging.StreamHandler(sys.stdout)
+    formatter = logging.Formatter('[%(levelname)s]: %(message)s')
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+
+def is_unsupported_site(url: str) -> bool:
+    """Checks if the URL belongs to a known unsupported website."""
+    domain = get_domain(url)
+    return domain is None or domain.endswith(".gov") or domain in unsupported_domains
+
+
+def read_urls_from_file(file_path):
+    with open(file_path, 'r') as f:
+        return f.read().splitlines()
+
+
+unsupported_domains_file = Path(__file__).parent / "unsupported_domains.txt"
+unsupported_domains = read_urls_from_file(unsupported_domains_file)

scrapemm/integrations/__init__.py

@@ -0,0 +1,20 @@
+from typing import Optional
+
+import aiohttp
+from ezmm import MultimodalSequence
+
+from scrapemm.util import get_domain
+from .telegram import Telegram
+from .x import X
+
+RETRIEVAL_INTEGRATIONS = [X(), Telegram()]
+DOMAIN_TO_INTEGRATION = {domain: integration
+                         for integration in RETRIEVAL_INTEGRATIONS
+                         for domain in integration.domains}
+
+
+async def retrieve_via_integration(url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    domain = get_domain(url)
+    if domain in DOMAIN_TO_INTEGRATION:
+        integration = DOMAIN_TO_INTEGRATION[domain]
+        return await integration.get(url, session)

scrapemm/integrations/base.py

@@ -0,0 +1,13 @@
+import aiohttp
+from ezmm import MultimodalSequence
+
+
+class RetrievalIntegration:
+    """Any integration used to retrieve information via a proprietary API, i.e., where
+    direct URL scraping is not possible."""
+
+    domains: list[str]  # The domains supported by this integration
+
+    async def get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence:
+        """Retrieves the contents present at the given URL."""
+        raise NotImplementedError

scrapemm/integrations/telegram.py

@@ -0,0 +1,124 @@
+import asyncio
+import logging
+from typing import Optional
+from urllib.parse import urlparse
+
+import aiohttp
+from ezmm import MultimodalSequence, Image, Item, Video
+from telethon import TelegramClient
+from telethon.tl.types import Channel, User
+
+from config import telegram_api_id, telegram_api_hash, telegram_bot_token
+from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.util import get_domain
+
+logger = logging.getLogger("Retriever")
+
+
+class Telegram(RetrievalIntegration):
+    """The Telegram integration for retrieving post contents from Telegram channels and groups."""
+
+    domains = ["t.me", "telegram.me"]
+    session_path = "temp/telegram"
+
+    def __init__(self):
+        self.api_id = telegram_api_id
+        self.api_hash = telegram_api_hash
+
+        self.client = TelegramClient(self.session_path, self.api_id, self.api_hash)
+        self.client.start(bot_token=telegram_bot_token)
+        logger.info("✅ Successfully connected to Telegram...")
+
+    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        """Retrieves content from a Telegram post URL."""
+        assert get_domain(url) in self.domains
+
+        # Parse the URL to get channel/group name and post ID
+        parsed = urlparse(url)
+        path_parts = parsed.path.strip("/").split("/")
+
+        if len(path_parts) < 2:
+            return None
+
+        channel_name = path_parts[0]
+        post_id = int(path_parts[1])
+
+        try:
+            # Get the message
+            channel = await self.client.get_entity(channel_name)
+            message = await self.client.get_messages(channel, ids=post_id)
+
+            if not message:
+                return None
+
+            # Handle media
+            media = await self._get_media_from_message(channel, message)
+
+            author = message.sender
+            author_type = type(author).__name__
+            if isinstance(author, Channel):
+                name = f'"{author.title}"'
+                if author.username:
+                    name += f" (@{author.username})"
+            elif isinstance(author, User):
+                if author.bot:
+                    author_type = "Bot"
+                name = f"{author.first_name} {author.last_name}" if author.last_name else author.first_name
+                if author.username:
+                    name += f" (@{author.username})"
+                if author.phone:
+                    name += f", Phone: {author.phone}"
+                if author.verified:
+                    name += " (Verified)"
+            else:
+                name = "Unknown"
+
+            edit_text = "\nEdit date: " + message.edit_date.strftime("%B %d, %Y at %H:%M") if message.edit_date else ""
+            reactions_text = "\nReactions: " + message.reactions.stringify() if message.reactions else ""
+
+            text = f"""**Telegram Post**
+Author: {author_type} {name}
+Date: {message.date.strftime("%B %d, %Y at %H:%M")}{edit_text}
+Views: {message.views}
+Forwards: {message.forwards}{reactions_text}
+
+{' '.join(m.reference for m in media)}
+{message.text}"""
+
+            return MultimodalSequence(text)
+
+        except Exception as e:
+            print(f"Error retrieving Telegram content: {e}")
+            # raise
+            return None
+
+    async def _get_media_from_message(self, chat, original_post, max_amp=10) -> list[Item]:
+        """
+        Searches for Telegram posts that are part of the same group of uploads.
+        The search is conducted around the id of the original post with an amplitude
+        of `max_amp` both ways.
+        Returns a list of [post] where each post has media and is in the same grouped_id.
+        """
+        # Gather posts that may belong to the same group
+        if original_post.grouped_id is None:
+            posts = [original_post]
+        else:
+            search_ids = list(range(original_post.id - max_amp, original_post.id + max_amp + 1))
+            posts = await self.client.get_messages(chat, ids=search_ids)
+
+        # Download media of posts that belong to the same group
+        media = []
+        for post in posts:
+            if post is not None and post.grouped_id == original_post.grouped_id:
+                if medium := post.media:
+                    post_url = f"https://t.me/{chat.username}/{post.id}"
+                    medium_bytes = await self.client.download_media(post, file=bytes)
+                    if hasattr(medium, "photo"):
+                        item = Image(binary_data=medium_bytes, source_url=post_url)
+                    elif hasattr(medium, "video"):
+                        item = Video(binary_data=medium_bytes, source_url=post_url)
+                    else:
+                        raise ValueError(f"Unsupported medium: {medium.__dict__}")
+                    media.append(item)
+
+        return media

scrapemm/integrations/x.py

@@ -0,0 +1,198 @@
+import asyncio
+import re
+from typing import Optional
+from urllib.parse import urlparse
+
+import aiohttp
+from ezmm import MultimodalSequence, download_video, download_image
+from tweepy import Tweet
+from tweepy.asynchronous import AsyncClient
+
+from config import x_bearer_token
+from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.util import get_domain
+
+
+class X(RetrievalIntegration):
+    """The X (Twitter) integration. Requires "Basic" API access to work. For more info, see
+    https://developer.x.com/en/docs/twitter-api/getting-started/about-twitter-api#v2-access-level
+    "Free" API access does NOT include reading Tweets."""
+    domains = ["twitter.com", "x.com", "t.co"]
+
+    account_explanation = """X accounts having a "blue" verification fulfill a basic set of criteria,
+    such as having a confirmed phone number. At time of the verification, the account must be not
+    deceptive.
+    
+    An account with "gold" verification belongs to an "official organization" verified through X,
+    costing about $1000 per month.
+    
+    If an account is "protected", it means that it was set private by the user.
+    
+    A "withheld" account is a user who got restricted by X.
+    
+    A "parody" account is an explicit, user-provided indication of being a parody (of someone or something).
+    
+    The "location" of a user profile is a user-provided string and is not guaranteed to be accurate."""
+
+    def __init__(self):
+        if not x_bearer_token: raise ValueError("No X bearer token provided. Add it to config/secrets.yaml")
+        self.client = AsyncClient(bearer_token=x_bearer_token)
+
+    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        assert get_domain(url) in self.domains
+        tweet_id = extract_tweet_id_from_url(url)
+        if tweet_id:
+            return await self._get_tweet(tweet_id, session)
+        else:
+            username = extract_username_from_url(url)
+            if username:
+                return await self._get_user(username, session)
+
+    async def _get_tweet(self, tweet_id: int, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        """Returns a MultimodalSequence containing the tweet's text and media
+        along with information like metrics, etc."""
+
+        response = await self.client.get_tweet(
+            id=tweet_id,
+            expansions=["author_id", "attachments.media_keys", "geo.place_id",
+                        "edit_history_tweet_ids"],
+            media_fields=["url", "variants"],
+            tweet_fields=["created_at", "public_metrics"],
+        )
+        tweet: Tweet = response.data
+
+        if tweet:
+            media_raw = response.includes.get("media")
+            author = response.includes.get("users")[0]
+            metrics = tweet.public_metrics
+
+            # Post-process text
+            text = tweet.text
+            text = re.sub(r"https?://t\.co/\S+", "", text).strip()
+
+            # Download the media
+            media = []
+            for medium_raw in media_raw:
+                if medium_raw.type == "photo":
+                    url = medium_raw.url
+                    medium = await download_image(url, session=session)
+                elif medium_raw.type in ["video", "animated_gif"]:
+                    # Get the variant with the highest bitrate
+                    url = _get_best_quality_video_url(medium_raw.variants)
+                    medium = await download_video(url, session=session)
+                else:
+                    raise ValueError(f"Unsupported media type: {medium_raw.type}")
+                if medium:
+                    media.append(medium)
+
+            tweet_str = f"""**Post on X**
+Author: {author.name}, @{author.username}
+Posted on: {tweet.created_at.strftime("%B %d, %Y at %H:%M")}
+Likes: {metrics['like_count']} - Retweets: {metrics['retweet_count']} - Replies: {metrics['reply_count']} - Views: {metrics['impression_count']}
+
+{text}"""  # TODO: Add edit history
+            return MultimodalSequence([tweet_str, *media])
+
+    async def _get_user(self, username: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        """Returns a MultimodalSequence containing the user's profile information
+        incl. profile image and profile banner."""
+
+        # The fields "parody" and "verified_followers_count" are fairly new. See
+        # https://x.com/Safety/status/1877581125608153389
+        # and https://x.com/XDevelopers/status/1865180409425715202
+        response = await self.client.get_user(username=username, user_fields=[
+            "created_at", "description", "location", "parody", "profile_banner_url", "profile_image_url",
+            "protected", "public_metrics", "url", "verified", "verified_followers_count", "verified_type", "withheld"
+        ])
+        user = response.data
+
+        if user:
+            # Turn all the data into a multimodal sequence
+
+            profile_image_url = user.profile_image_url.replace("_normal", "")  # Use the original picture variant
+            profile_image = await download_image(profile_image_url, session)
+            profile_banner = await download_image(user.profile_banner_url, session)
+
+            verification_status_text = f"{'Verified' if user.verified else 'Not verified'}"
+            if user.verified:
+                verification_status_text += f" ({user.verified_type})"
+
+            metrics = [f" - {k.capitalize().replace('_', ' ')}: {v}"
+                       for k, v in user.public_metrics.items()]
+            metrics_text = "\n".join(metrics)
+            if hasattr(user, "verified_followers_count"):
+                metrics_text += f"\n - Verified followers count: {user.verified_followers_count}"
+
+            properties_text = f"- {verification_status_text}"
+            if user.protected:
+                properties_text += "\n- Protected"
+            if user.withheld:
+                properties_text += "\n- Withheld"
+            if user.parody:
+                properties_text += "\n- Marked as parody"
+
+            text = f"""**Profile on X**
+User: {user.name}, @{user.username}
+Created on: {user.created_at.strftime("%B %d, %Y")}
+Profile image: {profile_image.reference}
+Profile banner: {profile_banner.reference}
+
+URL: {user.url}
+Location: {user.location}
+Description: {user.description}
+
+Metrics:
+{metrics_text}
+
+Account properties:
+{properties_text}"""
+
+            return MultimodalSequence(text)
+
+
+def extract_username_from_url(url: str) -> Optional[str]:
+    # TODO: Users may change their username, invalidating corresponding URLs. Handle this
+    # by retrieving the author's ID of the linked tweet.
+    parsed = urlparse(url)
+    try:
+        candidate = parsed.path.strip("/").split("/")[0]
+        if candidate and len(candidate) >= 3:
+            return candidate
+    except IndexError:
+        return None
+
+
+def extract_tweet_id_from_url(url: str) -> Optional[int]:
+    parsed = urlparse(url)
+    id_candidate = parsed.path.strip("/").split("/")[-1]  # Takes variants (like short links) into account
+    try:
+        return int(id_candidate)
+    except ValueError:
+        return None
+
+
+def _get_best_quality_video_url(variants: list) -> Optional[str]:
+    """Returns the URL of the video variant that has the highest bitrate."""
+    bitrate = -1
+    best_url = None
+    for variant in variants:
+        if content_type := variant.get("content_type"):
+            if content_type.startswith("video/") and variant["bit_rate"] > bitrate:
+                bitrate = variant["bit_rate"]
+                best_url = variant["url"]
+    return best_url
+
+
+if __name__ == "__main__":
+    urls = [
+        # "https://x.com/thinking_panda/status/1939348093155344491",  # Image
+        # "https://x.com/PopBase/status/1938496291908030484",  # Multiple images
+        # "https://x.com/AMAZlNGNATURE"  # Profile
+        # "https://x.com/AMAZlNGNATURE/status/1917939518000210352",  # Video
+        "https://x.com/GiFShitposting/status/1936904802082161085",  # GIF
+    ]
+    x = X()
+    for url in urls:
+        task = x.get(url)
+        out = asyncio.run(task)
+        print(out)

scrapemm/retrieval.py

@@ -0,0 +1,65 @@
+import logging
+from typing import Optional
+
+import aiohttp
+from ezmm import MultimodalSequence
+
+from scrapemm.common import is_unsupported_site
+from scrapemm.integrations import retrieve_via_integration
+from scrapemm.scraping.firecrawl import firecrawl
+from scrapemm.util import run_with_semaphore
+
+logger = logging.getLogger("Retriever")
+
+
+async def retrieve(urls: str | list[str], remove_urls: bool = True) -> Optional[MultimodalSequence] | list[Optional[MultimodalSequence]]:
+    """Main function of this repository. Downloads the contents present at the given URL(s).
+    For each URL, returns a MultimodalSequence containing text, images, and videos.
+    Returns None if the corresponding URL is not supported or if retrieval failed.
+
+    :param urls: The URL(s) to retrieve.
+    :param remove_urls: Whether to remove URLs from hyperlinks contained in the
+        retrieved text (and only keep the hypertext).
+    """
+
+    async with aiohttp.ClientSession() as session:
+        if isinstance(urls, str):
+            return await _retrieve_single(urls, remove_urls, session)
+
+        elif isinstance(urls, list):
+            if len(urls) == 1:
+                return await _retrieve_single(urls[0], remove_urls, session)
+
+            # Remove duplicates
+            urls_unique = set(urls)
+
+            # Retrieve URLs concurrently
+            tasks = [_retrieve_single(url, remove_urls, session) for url in urls_unique]
+            results = await run_with_semaphore(tasks, limit=20, show_progress=True,
+                                               progress_description="Retrieving URLs...")
+
+            # Reconstruct output list
+            results = dict(zip(urls_unique, results))
+            return [results[url] for url in urls]
+
+        else:
+            raise ValueError("'urls' must be a string or a list of strings.")
+
+
+async def _retrieve_single(url: str, remove_urls: bool,
+                           session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    try:
+        # Ensure URL is a string
+        url = str(url)
+
+        # Skip URLs from domains that are not supported
+        if is_unsupported_site(url):
+            return None
+
+        # First, try to use a matching API, otherwise scrape directly
+        return ((await retrieve_via_integration(url, session)) or
+                (await firecrawl.scrape(url, remove_urls, session)))
+
+    except Exception as e:
+        logger.error(f"Error while retrieving URL '{url}'.\n"
+                     f"{type(e).__name__}: {e}")

scrapemm/scraping/firecrawl.py

@@ -0,0 +1,126 @@
+import logging
+from typing import Optional
+
+import aiohttp
+import requests
+from ezmm import MultimodalSequence
+
+from config import firecrawl_url
+from scrapemm.scraping.util import find_firecrawl, to_multimodal_sequence
+
+logger = logging.getLogger("Retriever")
+
+FIRECRAWL_URLS = [
+    firecrawl_url,
+    "http://localhost:3002",
+    "http://firecrawl:3002",
+    "http://0.0.0.0:3002",
+]
+
+
+class Firecrawl:
+    """Takes any URL and tries to scrape its contents. If the URL belongs to a platform
+    requiring an API and the API integration is implemented (e.g. X, Reddit etc.), the
+    respective API will be used instead of direct HTTP requests."""
+
+    firecrawl_url: Optional[str]
+
+    def __init__(self):
+        self.locate_firecrawl()
+        self.n_scrapes = 0
+
+    def locate_firecrawl(self):
+        """Scans a list of URLs (included the user-specified one) to find a
+        running Firecrawl instance."""
+        self.firecrawl_url = find_firecrawl(FIRECRAWL_URLS)
+        if self.firecrawl_url:
+            logger.info(f"✅ Detected Firecrawl running at {self.firecrawl_url}.")
+        else:
+            logger.warning(f"❌ Unable to locate Firecrawl! It is not running at: {firecrawl_url}")
+
+    async def scrape(self, url: str,
+                     remove_urls: bool,
+                     session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        """Downloads the contents of the specified webpages dynamically or statically.
+        Resolves up to MAX_MEDIA_PER_PAGE image URLs and replaces them with their
+        respective reference."""
+        html = await self._call_firecrawl(url, session)
+        if html:
+            return await to_multimodal_sequence(html, remove_urls=remove_urls, session=session)
+
+    async def _call_firecrawl(self, url: str, session: aiohttp.ClientSession,
+                              format: str = "html", timeout: int = 30_000) -> Optional[str]:
+        """Scrapes the given URL using Firecrawl. Returns a Markdown-formatted string
+        of the webpage's contents."""
+        assert self.firecrawl_url is not None
+
+        headers = {
+            'Content-Type': 'application/json',
+        }
+        json_data = {
+            "url": url,
+            "formats": [format],
+            "onlyMainContent": False,
+            "removeBase64Images": False,
+            # "includeTags": [
+            #     # Text
+            #     "p", "h1", "h2", "h3", "h4", "h5", "h6", "span", "a", "div",
+            #     "li", "blockquote", "figcaption", "article", "header", "section", "ul", "ol",
+            #     "pre", "code", "table", "tbody", "tr", "td", "th", "thead",
+            #     # Media
+            #     "img", "picture", "video", "audio", "source", "iframe", "embed", "object",
+            # ],
+            "excludeTags": ["script", "style", "noscript", "footer", "aside"],
+            "timeout": timeout,  # Max. duration in ms that the scraper will wait for the page to respond
+        }
+
+        try:
+            async with session.post(self.firecrawl_url + "/v1/scrape",
+                                    json=json_data,
+                                    headers=headers,
+                                    timeout=10 * 60) as response:  # Firecrawl scrapes usually take 2 to 4s, but a 1700-page PDF takes 5 min
+
+                if response.status != 200:
+                    logger.warning(
+                        f"Failed to scrape {url}\nStatus code: {response.status} - Reason: {response.reason}")
+                    match response.status:
+                        case 402:
+                            logger.debug(f"Error 402: Access denied.")
+                        case 403:
+                            logger.debug(f"Error 403: Forbidden.")
+                        case 408:
+                            logger.warning(f"Error 408: Timeout! Firecrawl overloaded or Webpage did not respond.")
+                        case 409:
+                            logger.debug(f"Error 409: Access denied.")
+                        case 500:
+                            logger.debug(f"Error 500: Server error.")
+                        case _:
+                            logger.debug(f"Error {response.status}: {response.reason}.")
+                    logger.debug("Skipping that URL.")
+                    return None
+
+                json = await response.json()
+                success = json["success"]
+                if success and "data" in json:
+                    data = json["data"]
+                    text = data.get(format)
+                    return text
+                else:
+                    logger.warning(f"Unable to read {url}. No usable data in response. Skipping it.")
+                    logger.debug(str(json))
+                    return None
+
+        except (requests.exceptions.RetryError, requests.exceptions.ConnectionError):
+            logger.error(f"Firecrawl is not running!")
+            return None
+        except requests.exceptions.Timeout:
+            error_message = "Firecrawl failed to respond in time! This can be due to server overload."
+            logger.warning(f"{error_message}\nSkipping the URL {url}.")
+            return None
+        except Exception as e:
+            error_message = f"Exception: {repr(e)}"
+            logger.warning(f"{error_message}\nUnable to scrape {url} with Firecrawl. Skipping...")
+            return None
+
+
+firecrawl = Firecrawl()

scrapemm/scraping/util.py

@@ -0,0 +1,159 @@
+from typing import Optional
+
+import aiohttp
+import requests
+from ezmm import MultimodalSequence, download_item, Item, Image, Video
+from markdownify import markdownify as md
+import re
+import base64
+
+from scrapemm.util import run_with_semaphore
+
+MAX_MEDIA_PER_PAGE = 32
+
+URL_REGEX = r"https?:\/\/(?:www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b(?:[-a-zA-Z0-9@:%_\+.~#?&//=]*)"
+DATA_URI_REGEX = r"data:([\w/+.-]+/[\w.+-]+);base64,([A-Za-z0-9+/=]+)"
+MD_HYPERLINK_REGEX = rf'(!?\[([^]^[]*)\]\((.*?)(?: "[^"]*")?\))'
+
+
+def find_firecrawl(urls):
+    for url in urls:
+        if firecrawl_is_running(url):
+            return url
+    return None
+
+
+def firecrawl_is_running(url):
+    """Returns True iff Firecrawl is running at the specified URL."""
+    try:
+        response = requests.get(url, timeout=0.1)
+    except (requests.exceptions.ConnectionError, requests.exceptions.RetryError):
+        return False
+    return response.status_code == 200
+
+
+def postprocess_scraped(text: str) -> str:
+    # Remove any excess whitespaces
+    text = re.sub(r' {2,}', ' ', text)
+
+    # Remove any excess newlines
+    text = re.sub(r'(\n *){3,}', '\n\n', text)
+
+    return sanitize(text.strip())
+
+
+async def resolve_media_hyperlinks(
+        text: str, session: aiohttp.ClientSession,
+        remove_urls: bool = False,
+) -> Optional[MultimodalSequence]:
+    """Identifies up to MAX_MEDIA_PER_PAGE image URLs, downloads images that
+    have substantial size (larger than 256 x 256) and replaces the
+    respective Markdown hyperlinks with their proper image reference."""
+
+    if text is None:
+        return None
+
+    # Extract URLs and base64-encoded data from the text
+    hyperlinks = get_markdown_hyperlinks(text)
+    urls = set()
+    data_uris = set()
+    for _, _, href in hyperlinks:
+        if is_url(href):
+            urls.add(href)
+        elif is_data_uri(href):
+            data_uris.add(href)
+
+    # Try to download media for each URL
+    tasks = [download_item(url, session=session) for url in urls]
+    media: list[Item | None] = await run_with_semaphore(tasks, limit=100, show_progress=False)
+
+    href_media = dict(zip(urls, media))
+
+    # Convert each base64-encoded data to the respective medium
+    for data_uri in data_uris:
+        mime_type, base64_encoding = decompose_data_uri(data_uri)
+        href_media[data_uri] = from_base64(base64_encoding, mime_type=mime_type)
+
+    # Replace hyperlinks with their respective media reference
+    media_count = 0
+    for full_match, hypertext, href in hyperlinks:
+        medium = href_media.get(href)
+        if medium:
+            # Ignore small images
+            to_ignore = isinstance(medium, Image) and (medium.width < 256 or medium.height < 256)
+            reference = "" if to_ignore else medium.reference
+            replacement = f"{hypertext} {reference}" if hypertext else reference
+            text = text.replace(full_match, replacement)
+            media_count += 1 if not to_ignore else 0
+        elif remove_urls:
+            text = text.replace(full_match, hypertext)
+
+        if media_count >= MAX_MEDIA_PER_PAGE:
+            break
+
+    return MultimodalSequence(text)
+
+
+def is_url(href: str) -> bool:
+    """Returns True iff the given string is a valid URL."""
+    return re.match(URL_REGEX, href) is not None
+
+
+def is_data_uri(href: str) -> bool:
+    """Returns True iff the given string is a valid data URI."""
+    return re.match(DATA_URI_REGEX, href) is not None
+
+
+# def md(soup, **kwargs):
+#     """Converts a BeautifulSoup object into Markdown."""
+#     if soup is None:
+#         return None
+#     return MarkdownConverter(**kwargs).convert_soup(soup)
+
+
+def get_markdown_hyperlinks(text: str) -> list[tuple[str, str, str]]:
+    """Extracts all web hyperlinks from the given markdown-formatted string. Returns
+    a list of fullmatch-hypertext-URL-triples."""
+    pattern = re.compile(MD_HYPERLINK_REGEX, re.DOTALL)
+    hyperlinks = re.findall(pattern, text)
+    return hyperlinks
+
+
+def decompose_data_uri(href: str) -> Optional[tuple[str, str]]:
+    """Extracts the mime type and base64-encoded data from a data URI."""
+    match = re.match(DATA_URI_REGEX, href)
+    if match:
+        return match.group(1), match.group(2)
+    else:
+        return None
+
+
+async def to_multimodal_sequence(
+        html: str | None,
+        **kwargs
+) -> Optional[MultimodalSequence]:
+    """Turns a scraped output into the corresponding MultimodalSequences
+    by converting the HTML into Markdown and resolving media hyperlinks."""
+    text = md(html, heading_style="ATX")
+    text = postprocess_scraped(text)
+    return await resolve_media_hyperlinks(text, **kwargs)
+
+
+def sanitize(text: str) -> str:
+    """Post-processes scraped text, removing invalid characters."""
+    return text.replace("\u0000", "")
+
+
+def from_base64(b64_data: str, mime_type: str = "image/jpeg") -> Optional[Item]:
+    """Converts a base64-encoded image to an Item object."""
+    try:
+        binary_data = base64.b64decode(b64_data)
+        if binary_data:
+            if mime_type.startswith("image/"):
+                return Image(binary_data=binary_data)
+            elif mime_type.startswith("video/"):
+                return Video(binary_data=binary_data)
+            else:
+                raise ValueError(f"Unsupported media type: {mime_type}")
+    except ValueError as e:
+        print(f"Error decoding base64 data: {e}")

scrapemm/util.py

@@ -0,0 +1,69 @@
+import asyncio
+import logging
+import re
+import sys
+from typing import Optional, Awaitable, Iterable
+
+import tqdm
+from pydantic import HttpUrl
+
+logger = logging.getLogger("Retriever")
+
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                  "AppleWebKit/537.36 (KHTML, like Gecko) "
+                  "Chrome/123.0.0.0 Safari/537.36",
+}
+
+DOMAIN_REGEX = r"(?:https?:\/\/)?(?:www\.)?([-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6})/?"
+
+
+def get_domain(url: str | HttpUrl, keep_subdomain: bool = False) -> Optional[str]:
+    """Uses regex to get out the domain from the given URL. The output will be
+    of the form 'example.com'. No 'www', no 'http'."""
+    url = str(url)
+    match = re.search(DOMAIN_REGEX, url)
+    if match:
+        domain = match.group(1)
+        if not keep_subdomain:
+            # Keep only second-level and top-level domain
+            domain = '.'.join(domain.split('.')[-2:])
+        return domain
+
+
+async def run_with_semaphore(tasks: Iterable[Awaitable],
+                             limit: int,
+                             show_progress: bool = True,
+                             progress_description: str = None) -> list:
+    """
+    Runs asynchronous tasks with a concurrency limit.
+
+    Args:
+        tasks: The tasks to execute concurrently.
+        limit: The maximum number of coroutines to run concurrently.
+        show_progress: Whether to show a progress bar while executing tasks.
+        progress_description: The message to display in the progress bar.
+
+    Returns:
+        list: A list of results returned by the tasks, order-preserved.
+    """
+    semaphore = asyncio.Semaphore(limit)  # Limit concurrent executions
+
+    async def limited_coroutine(t: Awaitable):
+        async with semaphore:
+            return await t
+
+    print(progress_description, end="\r")
+
+    tasks = [asyncio.create_task(limited_coroutine(task)) for task in tasks]
+
+    # Report completion status of tasks (if more than one task)
+    if show_progress:
+        progress = tqdm.tqdm(total=len(tasks), desc=progress_description, file=sys.stdout)
+        while progress.n < len(tasks):
+            progress.n = sum(task.done() for task in tasks)
+            progress.refresh()
+            await asyncio.sleep(0.1)
+        progress.close()
+
+    return await asyncio.gather(*tasks)

scrapemm-0.1.0.dist-info/METADATA

@@ -0,0 +1,47 @@
+Metadata-Version: 2.4
+Name: scrapeMM
+Version: 0.1.0
+Summary: LLM-friendly scraper for media and text from social media and the open web.
+Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/multimodal-ai-lab/scrapeMM
+Project-URL: Bug Tracker, https://github.com/multimodal-ai-lab/scrapeMM
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: ezmm
+Requires-Dist: telethon
+Requires-Dist: tweepy
+Requires-Dist: markdownify
+Dynamic: license-file
+
+# scrapeMM: Multimodal Web Retrieval
+Simple web scraper to asynchronously retrieve webpages and access social media contents, fetching text along with media, i.e., images and videos.
+
+This library aims to help developers and researchers to easily access multimodal data from the web and use it for LLM processing.
+
+## Usage
+```python
+from scrapemm import retrieve
+
+url = "https://example.com"
+result = retrieve(url)
+result.render()
+```
+
+## How it works
+```
+Input:                                  Output:
+URL (string)   -->   retrieve()   -->   MultimodalSequence
+```
+The `MultimodalSequence` is a sequence of Markdown-formatted text and media provided by the [ezMM](https://github.com/multimodal-ai-lab/ezmm) library.
+
+Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
+
+## Supported Proprietary APIs
+- ✅ X/Twitter
+- ✅ Telegram
+- ⏳ Facebook
+- ⏳ Instagram
+- ⏳ Threads
+- ⏳ TikTok

scrapemm-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+scrapemm/__init__.py,sha256=G1hiuY8OvUdmamcHJTc9Ov2yK7tY8SVXQO8q6UBJxT4,70
+scrapemm/common.py,sha256=jwgvuVHg-7iAUEOQt-pLvod4QrhUZIrBqvhjac99e0M,954
+scrapemm/retrieval.py,sha256=3J8gfVcIwhggomN4HkT8y6_Oz8k9Nk2ZlG9ihwHraOA,2578
+scrapemm/util.py,sha256=iUBb9kqmoHOr71RqfaXIJLHINNqdMlNd1gj75WTuwFU,2406
+scrapemm/integrations/__init__.py,sha256=xU58ouFwKUbr4rnLHOwQhgawttcZZoEJZtSC8pgCYXM,686
+scrapemm/integrations/base.py,sha256=5gyf4zyMufcmvv0hVr4p5Q8bnIvrN8829nz1LNHmdrI,481
+scrapemm/integrations/telegram.py,sha256=8qTN6UGoKjkkIVAE9YHetJ-2f38W0_5Ktf76yYb1C9s,4947
+scrapemm/integrations/x.py,sha256=8x8_KyVl9RhKkocr9EjqHwPjhIvkfUyge2DPr5O9whU,8315
+scrapemm/scraping/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scrapemm/scraping/firecrawl.py,sha256=pT0Se11aq7ayalfZBKDMgEj1-V56aCKBDyx958ED_uk,5514
+scrapemm/scraping/util.py,sha256=5kiNn9pDpiJTVVlk8AfOns2MzD4V1EfXQ2hmJzM7-EE,5538
+scrapemm-0.1.0.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+scrapemm-0.1.0.dist-info/METADATA,sha256=Zs3E04B33rzNVcEBSBs1xrzVAz1Tekhiz7gvReDdygo,1539
+scrapemm-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+scrapemm-0.1.0.dist-info/top_level.txt,sha256=ftpWMzYGpbXKbz6etSQCOFbSqoiTxQN3eLkHHlhBXJA,9
+scrapemm-0.1.0.dist-info/RECORD,,

scrapemm-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

scrapemm-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

scrapemm-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+scrapemm