scrapeMM 0.6.2__tar.gz → 0.6.4__tar.gz

{scrapemm-0.6.2/scrapeMM.egg-info → scrapemm-0.6.4}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.6.2
-Summary: Flexible multimodal scraper for social media and the open web.
+Version: 0.6.4
+Summary: Multimodal scraper for social media, internet archives, and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://github.com/multimodal-ai-lab/scrapeMM
@@ -9,18 +9,33 @@ Project-URL: Issues, https://github.com/multimodal-ai-lab/scrapeMM
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ezmm
-Requires-Dist: telethon
-Requires-Dist: tweepy
-Requires-Dist: markdownify
-Requires-Dist: platformdirs
-Requires-Dist: PyYAML
+Requires-Dist: aiohttp~=3.14.1
+Requires-Dist: async_lru
 Requires-Dist: atproto
-Requires-Dist: TikTokResearchApi
-Requires-Dist: yt-dlp
+Requires-Dist: beautifulsoup4
+Requires-Dist: backports.zstd
+Requires-Dist: brotli
 Requires-Dist: cryptography
-Requires-Dist: firecrawl-py
+Requires-Dist: curl_cffi~=0.15.0
+Requires-Dist: ezmm~=0.5.1
+Requires-Dist: firecrawl-py~=4.32.0
+Requires-Dist: markdownify~=1.2.3
+Requires-Dist: oauthlib
+Requires-Dist: pip-system-certs
+Requires-Dist: platformdirs~=4.10.0
 Requires-Dist: playwright
+Requires-Dist: playwright-stealth
+Requires-Dist: prompt_toolkit
+Requires-Dist: PyYAML~=6.0.1
+Requires-Dist: requests~=2.34.2
+Requires-Dist: telethon~=1.44.0
+Requires-Dist: TikTokResearchApi~=1.0.4
+Requires-Dist: tqdm~=4.68.4
+Requires-Dist: tweepy~=4.17.0
+Requires-Dist: yt-dlp[curl-cffi,default]
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-asyncio; extra == "test"
 Dynamic: license-file
 
 # scrapeMM: Multimodal Web Retrieval
@@ -33,6 +48,19 @@ This library aims to help developers and researchers to easily access multimodal
 In Conda, you can install it with `conda install -c conda-forge ffmpeg`.
 * **If you want to scrape Perma.cc archive records or Facebook photos**, you'll need to install playwright with `pip install playwright` and running `playwright install`.
 
+## Configure
+To set the API secrets, run
+```python
+from scrapemm import configure_secrets
+configure_secrets()
+```
+
+To set the Firecrawl URL, run
+```python
+from scrapemm import update_config
+update_config(firecrawl_url="your_url")
+```
+
 ## Usage
 ```python
 from scrapemm import retrieve

{scrapemm-0.6.2 → scrapemm-0.6.4}/README.md

@@ -8,6 +8,19 @@ This library aims to help developers and researchers to easily access multimodal
 In Conda, you can install it with `conda install -c conda-forge ffmpeg`.
 * **If you want to scrape Perma.cc archive records or Facebook photos**, you'll need to install playwright with `pip install playwright` and running `playwright install`.
 
+## Configure
+To set the API secrets, run
+```python
+from scrapemm import configure_secrets
+configure_secrets()
+```
+
+To set the Firecrawl URL, run
+```python
+from scrapemm import update_config
+update_config(firecrawl_url="your_url")
+```
+
 ## Usage
 ```python
 from scrapemm import retrieve

scrapemm-0.6.4/pyproject.toml

@@ -0,0 +1,62 @@
+[project]
+name = "scrapeMM"
+version = "0.6.4"
+description = "Multimodal scraper for social media, internet archives, and the open web."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+authors = [{name = "Mark Rothermel", email = "mark.rothermel@tu-darmstadt.de"}]
+dependencies = [
+    "aiohttp~=3.14.1",
+    "async_lru",
+    "atproto",
+    "beautifulsoup4",
+    "backports.zstd",  # For better content parsing with Firecrawl
+    "brotli",  # Required to read brotli-encoded media
+    "cryptography",
+    "curl_cffi~=0.15.0",  # Provides browser binaries for impersonation
+    "ezmm~=0.5.1",
+    "firecrawl-py~=4.32.0",
+    "markdownify~=1.2.3",
+    "oauthlib",
+    "pip-system-certs",  # For better SSL certificate handling
+    "platformdirs~=4.10.0",
+    "playwright",
+    "playwright-stealth",
+    "prompt_toolkit",  # For handling user terminal input
+    "PyYAML~=6.0.1",
+    "requests~=2.34.2",
+    "telethon~=1.44.0",
+    "TikTokResearchApi~=1.0.4",
+    "tqdm~=4.68.4",
+    "tweepy~=4.17.0",
+    "yt-dlp[default,curl-cffi]",
+]
+
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "pytest-asyncio",
+]
+
+[project.urls]
+Homepage = "https://github.com/multimodal-ai-lab/scrapeMM"
+Issues = "https://github.com/multimodal-ai-lab/scrapeMM"
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+namespaces = false  # Disable implicit namespaces
+
+[tool.setuptools.package-data]
+"*" = ["*.txt"]  # Include all .txt files contained in any (sub)package in the build
+
+[tool.pytest.ini_options]
+testpaths = [
+    "testing/",
+]
+asyncio_mode = "auto"
+asyncio_default_test_loop_scope = "session"
+asyncio_default_fixture_loop_scope = "session"

{scrapemm-0.6.2 → scrapemm-0.6.4/scrapeMM.egg-info}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.6.2
-Summary: Flexible multimodal scraper for social media and the open web.
+Version: 0.6.4
+Summary: Multimodal scraper for social media, internet archives, and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://github.com/multimodal-ai-lab/scrapeMM
@@ -9,18 +9,33 @@ Project-URL: Issues, https://github.com/multimodal-ai-lab/scrapeMM
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: ezmm
-Requires-Dist: telethon
-Requires-Dist: tweepy
-Requires-Dist: markdownify
-Requires-Dist: platformdirs
-Requires-Dist: PyYAML
+Requires-Dist: aiohttp~=3.14.1
+Requires-Dist: async_lru
 Requires-Dist: atproto
-Requires-Dist: TikTokResearchApi
-Requires-Dist: yt-dlp
+Requires-Dist: beautifulsoup4
+Requires-Dist: backports.zstd
+Requires-Dist: brotli
 Requires-Dist: cryptography
-Requires-Dist: firecrawl-py
+Requires-Dist: curl_cffi~=0.15.0
+Requires-Dist: ezmm~=0.5.1
+Requires-Dist: firecrawl-py~=4.32.0
+Requires-Dist: markdownify~=1.2.3
+Requires-Dist: oauthlib
+Requires-Dist: pip-system-certs
+Requires-Dist: platformdirs~=4.10.0
 Requires-Dist: playwright
+Requires-Dist: playwright-stealth
+Requires-Dist: prompt_toolkit
+Requires-Dist: PyYAML~=6.0.1
+Requires-Dist: requests~=2.34.2
+Requires-Dist: telethon~=1.44.0
+Requires-Dist: TikTokResearchApi~=1.0.4
+Requires-Dist: tqdm~=4.68.4
+Requires-Dist: tweepy~=4.17.0
+Requires-Dist: yt-dlp[curl-cffi,default]
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-asyncio; extra == "test"
 Dynamic: license-file
 
 # scrapeMM: Multimodal Web Retrieval
@@ -33,6 +48,19 @@ This library aims to help developers and researchers to easily access multimodal
 In Conda, you can install it with `conda install -c conda-forge ffmpeg`.
 * **If you want to scrape Perma.cc archive records or Facebook photos**, you'll need to install playwright with `pip install playwright` and running `playwright install`.
 
+## Configure
+To set the API secrets, run
+```python
+from scrapemm import configure_secrets
+configure_secrets()
+```
+
+To set the Firecrawl URL, run
+```python
+from scrapemm import update_config
+update_config(firecrawl_url="your_url")
+```
+
 ## Usage
 ```python
 from scrapemm import retrieve

{scrapemm-0.6.2 → scrapemm-0.6.4}/scrapeMM.egg-info/SOURCES.txt

@@ -2,7 +2,6 @@
 LICENSE
 README.md
 pyproject.toml
-requirements.txt
 scrapeMM.egg-info/PKG-INFO
 scrapeMM.egg-info/SOURCES.txt
 scrapeMM.egg-info/dependency_links.txt

scrapemm-0.6.4/scrapeMM.egg-info/requires.txt

@@ -0,0 +1,28 @@
+aiohttp~=3.14.1
+async_lru
+atproto
+beautifulsoup4
+backports.zstd
+brotli
+cryptography
+curl_cffi~=0.15.0
+ezmm~=0.5.1
+firecrawl-py~=4.32.0
+markdownify~=1.2.3
+oauthlib
+pip-system-certs
+platformdirs~=4.10.0
+playwright
+playwright-stealth
+prompt_toolkit
+PyYAML~=6.0.1
+requests~=2.34.2
+telethon~=1.44.0
+TikTokResearchApi~=1.0.4
+tqdm~=4.68.4
+tweepy~=4.17.0
+yt-dlp[curl-cffi,default]
+
+[test]
+pytest
+pytest-asyncio

{scrapemm-0.6.2 → scrapemm-0.6.4}/scrapemm/__init__.py

@@ -1,4 +1,4 @@
-from .common import APP_NAME, set_wait_on_rate_limit, RateLimitError, ContentNotFoundError, logger
+from .common import APP_NAME, set_wait_on_rate_limit, RateLimitError, ContentNotFoundError, logger, update_config
 from .integrations import Telegram, X
 from .retrieval import retrieve
 from .secrets import configure_secrets

{scrapemm-0.6.2 → scrapemm-0.6.4}/scrapemm/integrations/decodo/decodo.py

@@ -26,23 +26,21 @@ class Decodo:
     DECODO_API_URL = "https://scraper-api.decodo.com/v2/scrape"
 
     def __init__(self):
-        self.username = None
-        self.password = None
+        self.basic_auth_token = None
         self.n_scrapes = 0
 
-    def _load_credentials(self):
+    def _load_token(self):
         """Loads Decodo credentials from the secrets manager."""
-        self.username = get_secret("decodo_username")
-        self.password = get_secret("decodo_password")
+        self.basic_auth_token = get_secret("decodo_token")
 
-        if self.username and self.password:
-            logger.info("✅ Decodo credentials loaded successfully.")
+        if self.basic_auth_token:
+            logger.info("✅ Decodo token set.")
         else:
-            logger.warning("⚠️ Decodo credentials not found. Please configure them in secrets.")
+            logger.warning("⚠️ Decodo auth token not found. Please configure it in secrets.")
 
-    def _has_credentials(self) -> bool:
+    def _has_token(self) -> bool:
         """Checks if Decodo credentials are available."""
-        return bool(self.username and self.password)
+        return bool(self.basic_auth_token)
 
     async def scrape(
             self, url: str,
@@ -66,10 +64,10 @@ class Decodo:
         Returns:
             MultimodalSequence containing the scraped content, or None if scraping failed
         """
-        if not self._has_credentials():
-            self._load_credentials()
+        if not self._has_token():
+            self._load_token()
 
-        if not self._has_credentials():
+        if not self._has_token():
             logger.warning("Cannot scrape with Decodo: credentials not configured.")
             return None
 
@@ -124,6 +122,7 @@ class Decodo:
         """
         headers = {
             'Content-Type': 'application/json',
+            'Authorization': f'Basic {self.basic_auth_token}',
         }
 
         # Build request payload
@@ -141,9 +140,6 @@ class Decodo:
         if use_premium_proxy:
             payload["proxy_pool"] = "premium"
 
-        # Create basic auth
-        auth = aiohttp.BasicAuth(self.username, self.password)
-
         # Retry loop with exponential backoff
         for attempt in range(max_retries + 1):
             try:
@@ -151,12 +147,11 @@ class Decodo:
                         self.DECODO_API_URL,
                         json=payload,
                         headers=headers,
-                        auth=auth,
                         timeout=aiohttp.ClientTimeout(total=timeout)
                 ) as response:
                     # Validate response health
                     if response.status != 200:
-                        logger.debug("Communication with Decodo API failed.")
+                        logger.debug(f"Communication with Decodo API failed. Status code: {response.status}")
 
                         if response.status == 429:  # Rate limit
                             if attempt >= max_retries:
@@ -233,12 +228,7 @@ class Decodo:
             except aiohttp.ClientError:
                 logger.error(f"Network error while scraping with Decodo.")
                 raise
-            except asyncio.TimeoutError:
-                if attempt >= max_retries:
-                    raise
-                else:
-                    logger.debug(f"Timeout while scraping with Decodo. Retrying...")
-            except RateLimitError:
+            except (RateLimitError, asyncio.TimeoutError):
                 raise
             except Exception as e:
                 logger.error(f"Unexpected error while scraping with Decodo: {e}")

{scrapemm-0.6.2 → scrapemm-0.6.4}/scrapemm/integrations/fb.py

@@ -28,17 +28,42 @@ LIKE_COMMENT_SHARE_SVG_REGEX = (
 )
 FB_PHOTO_HREF_REGEX = r'href="(https://www\.facebook\.com/photo/[^"]*)"'
 
-JS_GET_MEDIA_VC_IMAGE = """
-    () => {
-        const img = document.querySelector('img[data-visualcompletion="media-vc-image"]');
-        return img ? img.getAttribute('src') : null;
-    }
-""".strip()
-
-JS_GET_OG_IMAGE = """
+JS_GET_PHOTO_IMAGE = """
     () => {
+        // Strategy 1: data-visualcompletion attribute (legacy)
+        let img = document.querySelector('img[data-visualcompletion="media-vc-image"]');
+        if (img) return img.getAttribute('src');
+
+        // Strategy 2: Large image inside the photo theater/spotlight viewer
+        const viewerSelectors = [
+            '[role="dialog"] img[src*="scontent"]',
+            '[data-pagelet="MediaViewerPhoto"] img',
+            '[role="main"] img[src*="scontent"]',
+            'img[alt][src*="fbcdn"]',
+            'img[alt][src*="scontent"]',
+        ];
+        for (const sel of viewerSelectors) {
+            const candidates = document.querySelectorAll(sel);
+            for (const c of candidates) {
+                const src = c.getAttribute('src') || '';
+                const w = c.naturalWidth || c.width || 0;
+                if (src && w > 200) return src;
+            }
+        }
+
+        // Strategy 3: Largest image on the page with a CDN src
+        const allImgs = Array.from(document.querySelectorAll('img[src*="scontent"], img[src*="fbcdn"]'));
+        if (allImgs.length > 0) {
+            allImgs.sort((a, b) => (b.naturalWidth || 0) - (a.naturalWidth || 0));
+            const best = allImgs[0];
+            if (best && (best.naturalWidth || 0) > 100) return best.getAttribute('src');
+        }
+
+        // Strategy 4: og:image meta tag
         const og = document.querySelector('meta[property="og:image"]');
-        return og ? og.getAttribute('content') : null;
+        if (og) return og.getAttribute('content');
+
+        return null;
     }
 """.strip()
 
@@ -118,7 +143,7 @@ class Facebook(RetrievalIntegration):
                 url,
                 platform="Facebook",
                 # cookiefile=self.cookie_file.as_posix(),
-                impersonate=ImpersonateTarget("chrome", "133"),
+                impersonate=ImpersonateTarget("chrome", "146"),
                 **kwargs,
             )
 
@@ -148,13 +173,21 @@ class Facebook(RetrievalIntegration):
                 try:
                     await page.goto(url, timeout=30000)
                     await page.wait_for_load_state("domcontentloaded")
+                    # Wait for network to settle so images are rendered
+                    try:
+                        await page.wait_for_load_state("networkidle", timeout=10000)
+                    except PlaywrightTimeoutError:
+                        pass
+                    # Give React/JS a moment to render the photo
+                    await page.wait_for_timeout(2000)
                 except PlaywrightTimeoutError:
-                    raise RuntimeError("Timed out loading Facebook photo page.")
+                    raise TimeoutError("Timed out loading Facebook photo page.")
+
+                image_url = await page.evaluate(JS_GET_PHOTO_IMAGE)
 
-                image_url = await page.evaluate(JS_GET_MEDIA_VC_IMAGE)
+            except TimeoutError:
+                raise
 
-                if not image_url:
-                    image_url = await page.evaluate(JS_GET_OG_IMAGE)
             finally:
                 html = await page.content()
                 await browser.close()

{scrapemm-0.6.2 → scrapemm-0.6.4}/scrapemm/integrations/instagram.py

@@ -7,7 +7,6 @@ from yt_dlp import DownloadError
 from scrapemm.common.exceptions import RateLimitError
 from scrapemm.integrations.base import RetrievalIntegration
 from scrapemm.integrations.ytdlp import get_content_with_ytdlp
-from scrapemm.util import get_domain
 
 logger = logging.getLogger("scrapeMM")
 

{scrapemm-0.6.2 → scrapemm-0.6.4}/scrapemm/integrations/perma_cc.py

@@ -1,22 +1,96 @@
 import logging
+import re
 from typing import Optional
+from urllib.parse import urlparse, parse_qs
 
 import aiohttp
 from ezmm import MultimodalSequence
 from playwright.async_api import Error, TimeoutError, async_playwright
+from playwright_stealth import Stealth
 
 from scrapemm.download.common import HEADERS
 from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.integrations.decodo import decodo
+from scrapemm.secrets import get_secret
 from scrapemm.util import to_multimodal_sequence
 
 logger = logging.getLogger("scrapeMM")
 
-
 # Limits for inlining media as data URIs to avoid excessive memory usage
 MAX_IMAGE_BYTES = 15 * 1024 * 1024  # 15 MB
 MAX_VIDEO_BYTES = 25 * 1024 * 1024  # 25 MB
 INLINE_CONCURRENCY = 6
 
+PERMA_CC_API_BASE = "https://api.perma.cc/v1/public/archives"
+
+
+def _extract_guid(url: str) -> Optional[str]:
+    """Extract the Perma.cc GUID (e.g. 'N8MR-NS96') from a perma.cc URL."""
+    match = re.search(r"perma\.cc/([A-Za-z0-9]+-[A-Za-z0-9]+)", url)
+    return match.group(1) if match else None
+
+
+def _is_screenshot_request(url: str) -> bool:
+    """Check if the URL explicitly requests the screenshot view (?type=image)."""
+    parsed = urlparse(url)
+    params = parse_qs(parsed.query)
+    return params.get("type", [None])[0] == "image"
+
+
+async def _get_archive_metadata(guid: str, session: aiohttp.ClientSession) -> Optional[dict]:
+    """Fetch archive metadata from Perma.cc's public API."""
+    api_url = f"{PERMA_CC_API_BASE}/{guid}/"
+    try:
+        async with session.get(api_url, timeout=aiohttp.ClientTimeout(total=15)) as resp:
+            if resp.status == 200:
+                return await resp.json()
+            else:
+                logger.debug(f"Perma.cc API returned status {resp.status} for GUID {guid}")
+    except Exception as e:
+        logger.debug(f"Failed to fetch Perma.cc API metadata: {e}")
+    return None
+
+
+def _extract_iframe_src(html: str) -> Optional[str]:
+    """Extract the rejouer.perma.cc iframe src URL from the Perma.cc wrapper HTML."""
+    # Match iframe with class "archive-iframe" or src containing rejouer.perma.cc
+    patterns = [
+        re.compile(r'<iframe[^>]+class=["\'][^"\']*archive-iframe[^"\']*["\'][^>]+src=["\']([^"\']+)["\']', re.IGNORECASE),
+        re.compile(r'<iframe[^>]+src=["\']([^"\']*rejouer\.perma\.cc[^"\']+)["\']', re.IGNORECASE),
+    ]
+    for pattern in patterns:
+        match = pattern.search(html)
+        if match:
+            url = match.group(1)
+            # Unescape HTML entities
+            url = url.replace("&amp;", "&")
+            return url
+    return None
+
+
+def _extract_screenshot_img_url(html: str, guid: str) -> Optional[str]:
+    """Extract the actual screenshot image URL from the Perma.cc screenshot page HTML.
+
+    The page embeds the screenshot in an <img> tag whose src points to
+    something like: https://rejouer.perma.cc/replay-web-page/w/.../mp_/file:///GUID/cap.png
+    """
+    # Look for img src containing the GUID or cap.png
+    patterns = [
+        # Match src with the GUID in the path
+        re.compile(r'<img[^>]+src=["\']([^"\']*' + re.escape(guid) + r'[^"\']*cap\.png[^"\']*)["\']', re.IGNORECASE),
+        # Match any src on rejouer.perma.cc
+        re.compile(r'<img[^>]+src=["\']([^"\']*rejouer\.perma\.cc[^"\']*)["\']', re.IGNORECASE),
+        # Match any src containing cap.png
+        re.compile(r'<img[^>]+src=["\']([^"\']*cap\.png[^"\']*)["\']', re.IGNORECASE),
+        # Fallback: any img src that looks like a full URL
+        re.compile(r'<img[^>]+src=["\'](https?://[^"\']+)["\']', re.IGNORECASE),
+    ]
+    for pattern in patterns:
+        match = pattern.search(html)
+        if match:
+            return match.group(1)
+    return None
+
 
 class PermaCC(RetrievalIntegration):
     name = "Perma.cc"
@@ -27,14 +101,95 @@ class PermaCC(RetrievalIntegration):
         self.connected = True
 
     async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
-        html = await get_record_html(url)
-        if html:
-            async with aiohttp.ClientSession(headers=HEADERS) as session:
-                return await to_multimodal_sequence(
-                    html, remove_urls=False, session=session, url=url
-                )
-        else:
-            raise RuntimeError("Failed to retrieve Perma.cc record HTML.")
+        guid = _extract_guid(url)
+        if not guid:
+            raise RuntimeError(f"Could not extract Perma.cc GUID from URL: {url}")
+
+        async with aiohttp.ClientSession(headers=HEADERS) as session:
+            # Fetch archive metadata to get the original URL
+            metadata = await _get_archive_metadata(guid, session)
+            original_url = None
+            if metadata:
+                # Top-level "url" field
+                original_url = metadata.get("url")
+                # Fallback: extract from captures array (role=primary)
+                if not original_url:
+                    for capture in metadata.get("captures", []):
+                        if capture.get("role") == "primary" and capture.get("url"):
+                            original_url = capture["url"]
+                            break
+                logger.debug(f"Perma.cc metadata keys: {list(metadata.keys())}, original_url={original_url}")
+
+            # Strategy 1: Scrape the ORIGINAL URL via Decodo (media URLs will be normal)
+            if original_url:
+                logger.debug(f"Perma.cc archive of: {original_url}")
+                result = await _scrape_original_via_decodo(original_url, session)
+                if result and _has_media(result):
+                    return result
+
+            # Strategy 2: Scrape the Perma.cc page itself via Decodo and download media
+            html = await _get_html_via_decodo(url, session)
+            if html:
+                result = await to_multimodal_sequence(html, remove_urls=False, session=session, url=url)
+                if result and _has_media(result):
+                    return result
+
+            # Strategy 3: Use Playwright with media inlining (handles session-bound URLs)
+            html = await get_record_html(url)
+            if html:
+                return await to_multimodal_sequence(html, remove_urls=False, session=session, url=url)
+
+            # Strategy 4: Return Decodo text-only result if we got one earlier
+            if original_url:
+                result = await _scrape_original_via_decodo(original_url, session)
+                if result:
+                    return result
+
+            raise RuntimeError("Failed to retrieve Perma.cc record.")
+
+
+def _has_media(seq: MultimodalSequence) -> bool:
+    """Check if a MultimodalSequence contains any images or videos."""
+    return bool(seq.images or seq.videos)
+
+
+async def _scrape_original_via_decodo(
+        original_url: str, session: aiohttp.ClientSession
+) -> Optional[MultimodalSequence]:
+    """Scrape the original archived URL via Decodo. Media URLs on the original
+    site are normal HTTP URLs that can be downloaded directly."""
+    try:
+        result = await decodo.scrape(
+            original_url,
+            remove_urls=False,
+            session=session,
+            format="multimodal_sequence",
+            enable_js=True,
+            timeout=30,
+        )
+        if isinstance(result, MultimodalSequence):
+            return result
+    except Exception as e:
+        logger.debug(f"Decodo scrape of original URL failed: {e}")
+    return None
+
+
+async def _get_html_via_decodo(url: str, session: aiohttp.ClientSession) -> Optional[str]:
+    """Fetch Perma.cc page HTML using Decodo proxy service to bypass Cloudflare."""
+    try:
+        result = await decodo.scrape(
+            url,
+            remove_urls=False,
+            session=session,
+            format="html",
+            enable_js=True,
+            timeout=30,
+        )
+        if isinstance(result, str) and result.strip():
+            return result
+    except Exception as e:
+        logger.debug(f"Decodo scrape failed for Perma.cc: {e}")
+    return None
 
 
 async def _inline_media_in_frame(frame, image_limit: int = MAX_IMAGE_BYTES, video_limit: int = MAX_VIDEO_BYTES,
@@ -269,11 +424,17 @@ async def _inline_media_in_frame(frame, image_limit: int = MAX_IMAGE_BYTES, vide
 
 
 async def get_record_html(url: str) -> str | None:
-    """Retrieves the HTML of the record saved by Perma.cc. Loads the contents dynamically
-    and returns any contained media as data URIs (base64-encoded)."""
-    async with async_playwright() as p:
-        browser = await p.chromium.launch(headless=True)
-        context = await browser.new_context(accept_downloads=False)
+    """Fallback: Retrieves the HTML of the record saved by Perma.cc using Playwright.
+    Loads the contents dynamically and returns any contained media as data URIs (base64-encoded)."""
+    async with Stealth().use_async(async_playwright()) as p:
+        browser = await p.chromium.launch(
+            headless=False,
+            args=["--window-position=-10000,-10000"],
+        )
+        context = await browser.new_context(
+            accept_downloads=False,
+            user_agent=HEADERS["User-Agent"],
+        )
         page = await context.new_page()
 
         try:

{scrapemm-0.6.2 → scrapemm-0.6.4}/scrapemm/integrations/telegram.py

@@ -29,14 +29,14 @@ class Telegram(RetrievalIntegration):
         bot_token = get_secret("telegram_bot_token")
 
         if api_id and api_hash and bot_token:
-            self.client = TelegramClient(self.session_path, api_id, api_hash)
             try:
-                await self.client.start(bot_token=bot_token)  # Returns a coroutine b/c event loop exists already
+                self.client = TelegramClient(self.session_path, api_id, api_hash)
             except sqlite3.OperationalError:  # Database is locked from an interrupted previous session
                 # Remove the database file and try again
                 journal_path = Path(self.session_path + ".session-journal")
                 journal_path.unlink(missing_ok=True)
-                await self.client.start(bot_token=bot_token)  # Returns a coroutine b/c event loop exists already
+                self.client = TelegramClient(self.session_path, api_id, api_hash)
+            await self.client.start(bot_token=bot_token)  # Returns a coroutine b/c event loop exists already
             self.connected = True
             logger.info("✅ Successfully connected to Telegram.")
         else:

{scrapemm-0.6.2 → scrapemm-0.6.4}/scrapemm/integrations/x.py

@@ -162,7 +162,7 @@ Likes: {metrics['like_count']} - Retweets: {metrics['retweet_count']} - Replies:
                 properties_text += "\n- Protected"
             if user.withheld:
                 properties_text += "\n- Withheld"
-            if user.parody:
+            if hasattr(user, "parody") and user.parody:
                 properties_text += "\n- Marked as parody"
 
             text = f"""**Profile on X**

{scrapemm-0.6.2 → scrapemm-0.6.4}/scrapemm/retrieval.py

@@ -41,6 +41,8 @@ BEST_METHODS = {
     "perma.cc": ["integrations"],
     "mvau.lt": ["integrations"],
     "archive.org": ["integrations"],
+    "verafiles.org": ["decodo", "firecrawl"],
+    "youturn.in": ["decodo"],
     # Archive.today:
     "archive.today": ["integrations", "decodo"],
     "archive.is": ["integrations", "decodo"],
@@ -49,6 +51,8 @@ BEST_METHODS = {
     "archive.li": ["integrations", "decodo"],
     "archive.fo": ["integrations", "decodo"],
     "archive.md": ["integrations", "decodo"],
+    # More special cases
+    "washingtonpost.com": ["decodo"],
 }
 
 
@@ -185,7 +189,7 @@ async def _retrieve_single(
                                              session=session, format=format, actions=actions),
             "decodo": lambda: decodo.scrape(url, remove_urls, session,
                                             format=format,
-                                            timeout=10 if prioritize == "speed" else 30,
+                                            timeout=15 if prioritize == "speed" else 60,
                                             max_retries=1 if prioritize == "speed" else 5),
         }
 
@@ -220,6 +224,11 @@ async def _retrieve_single(
                 errors[method_name] = e
                 result = None
 
+        except TimeoutError as e:
+            logger.warning(f"Timeout while retrieving with method '{method_name}': {e}")
+            errors[method_name] = e
+            result = None
+
         except IPBannedError as e:
             logger.error(e)
             errors[method_name] = e
@@ -227,7 +236,7 @@ async def _retrieve_single(
 
         except Exception as e:
             logger.warning(f"Error while retrieving with method '{method_name}': {e}")
-            errors[method_name] = e
+            errors[method_name] = f"{type(e).__name__}: {e}"
             result = None
 
         if result is not None:

{scrapemm-0.6.2 → scrapemm-0.6.4}/scrapemm/secrets.py

@@ -9,8 +9,8 @@ from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
 
-from scrapemm.common import get_config_var, update_config, CONFIG_PATH, CONFIG_DIR
-from scrapemm.util import get_multiline_user_input
+from scrapemm.common import get_config_var, update_config, CONFIG_DIR
+from scrapemm.util import get_user_input
 
 logger = logging.getLogger("scrapeMM")
 
@@ -23,8 +23,7 @@ SECRETS = {
     "bluesky_password": "Bluesky password",
     "tiktok_client_key": "TikTok client key",
     "tiktok_client_secret": "TikTok client secret",
-    "decodo_username": "Decodo Web Scraping API username",
-    "decodo_password": "Decodo Web Scraping API password",
+    "decodo_token": "Decodo Web Scraping API basic authentication token",
     "youtube_cookie": "YouTube cookie string",
     "facebook_cookie": "Facebook cookie string",
 }
@@ -144,9 +143,9 @@ def override_secret(key_name: str):
     when nothing entered."""
     description = SECRETS[key_name]
     if key_name in ["youtube_cookie", "facebook_cookie"]:
-        user_input = get_multiline_user_input(f"Please enter the {description}. Hit Ctrl-D or Ctrl-Z to save it.")
+        user_input = get_user_input(f"Please enter the {description} (Alt+Enter to submit):", multiline=True)
     else:
-        user_input = getpass(f"Please enter the {description} (leave empty to skip): ", stream=sys.stdout)
+        user_input = get_user_input(f"Please enter the {description} (leave empty to skip):")
     if user_input:
         set_secret(key_name, user_input)
 
@@ -171,8 +170,7 @@ def configure_secrets(all_keys: bool = False):
 
     update_config(api_keys_configured=True)
 
-    logger.info("API keys configured successfully! If you want to change them, go to "
-                f"{CONFIG_PATH.as_posix()} and set 'api_keys_configured' to 'false' or "
+    logger.info("API keys configured successfully! If you want to change them, "
                 f"run scrapemm.secrets.configure_secrets().")
 
 

{scrapemm-0.6.2 → scrapemm-0.6.4}/scrapemm/util.py

@@ -38,7 +38,7 @@ def get_domain(url: str, keep_subdomain: bool = False) -> Optional[str]:
 async def run_with_semaphore(tasks: Iterable[Awaitable],
                              limit: int,
                              show_progress: bool = True,
-                             progress_description: str | None = None) -> list:
+                             progress_description: str | None = None) -> tuple:
     """
     Runs asynchronous tasks with a concurrency limit.
 
@@ -76,10 +76,32 @@ def read_urls_from_file(file_path):
         return f.read().splitlines()
 
 
-def get_multiline_user_input(prompt: str) -> str:
-    print(prompt)
-    lines = sys.stdin.readlines()
-    return "".join(lines)
+def get_user_input(prompt: str, multiline: bool = False) -> str:
+    """Prompts the user for input.
+    - Single-line: uses standard input(); Enter submits, empty input skips.
+    - Multiline: uses prompt_toolkit; Enter adds newline, Alt+Enter submits.
+    """
+    if multiline:
+        from prompt_toolkit import prompt as pt_prompt
+        from prompt_toolkit.key_binding import KeyBindings
+
+        print(prompt)
+
+        bindings = KeyBindings()
+
+        @bindings.add('escape', 'enter')
+        def _submit(event):
+            event.current_buffer.validate_and_handle()
+
+        result = pt_prompt(
+            ">>> ",
+            multiline=True,
+            key_bindings=bindings,
+            prompt_continuation="... ",
+        )
+        return result
+    else:
+        return input(f"{prompt} ")
 
 
 MAX_MEDIA_PER_PAGE = 32
@@ -127,7 +149,7 @@ async def resolve_media_hyperlinks(
 
     # Try to download media for each URL
     tasks = [download_medium(u, session=session, headers={"Referer": url}) for u in hrefs_urls.values()]
-    media: list[Item | None] = await run_with_semaphore(tasks, limit=100, show_progress=False)
+    media: tuple[Item | None] = await run_with_semaphore(tasks, limit=100, show_progress=False)
 
     href_media = dict(zip(hrefs_urls.keys(), media))
 

scrapemm-0.6.4/scripts/example.py

@@ -0,0 +1,11 @@
+from scrapemm import retrieve
+import asyncio
+
+if __name__ == "__main__":
+    url = "https://perma.cc/ZD7Z-B3U7?type=image"
+    result = asyncio.run(retrieve(url))
+    print(f"Retrieval time: {result.retrieval_time:.2f} seconds")
+    if result.errors:
+        print(result.errors)
+    else:
+        print(result.content)

{scrapemm-0.6.2 → scrapemm-0.6.4}/testing/test_retrieval.py

@@ -9,7 +9,6 @@ from scrapemm import retrieve
 @pytest.mark.parametrize("url", [
     "https://www.vishvasnews.com/viral/fact-check-upsc-has-not-reduced-the-maximum-age-limit-for-ias-and-ips-exams/",
     "https://health.medicaldialogues.in/fact-check/brain-health-fact-check/fact-check-is-sprite-the-best-remedy-for-headaches-in-the-world-140368",
-    "https://www.washingtonpost.com/politics/2024/05/15/bidens-false-claim-that-inflation-was-9-percent-when-he-took-office/",
     "https://assamese.factcrescendo.com/viral-claim-that-the-video-shows-the-incident-from-uttar-pradesh-and-the-youth-on-the-bike-and-the-youth-being-beaten-and-taken-away-by-the-police-are-the-same-youth-named-abdul-is-false/",
     # "https://factuel.afp.com/doc.afp.com.43ZN7NP",
     "https://leadstories.com/365cb414b83e29d26fecae374d55c743a3eac4c7.png",

scrapemm-0.6.2/pyproject.toml

@@ -1,44 +0,0 @@
-[project]
-name = "scrapeMM"
-version = "0.6.2"
-description = "Flexible multimodal scraper for social media and the open web."
-readme = "README.md"
-requires-python = ">=3.10"
-license = "Apache-2.0"
-authors = [{name = "Mark Rothermel", email = "mark.rothermel@tu-darmstadt.de"}]
-dependencies = [
-    "ezmm",
-    "telethon",
-    "tweepy",
-    "markdownify",
-    "platformdirs",
-    "PyYAML",
-    "atproto",
-    "TikTokResearchApi",
-    "yt-dlp",
-    "cryptography",
-    "firecrawl-py",
-    "playwright"
-]
-
-[project.urls]
-Homepage = "https://github.com/multimodal-ai-lab/scrapeMM"
-Issues = "https://github.com/multimodal-ai-lab/scrapeMM"
-
-[build-system]
-requires = ["setuptools"]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools.packages.find]
-namespaces = false  # Disable implicit namespaces
-
-[tool.setuptools.package-data]
-"*" = ["*.txt"]  # Include all .txt files contained in any (sub)package in the build
-
-[tool.pytest.ini_options]
-testpaths = [
-    "testing/",
-]
-asyncio_mode = "auto"
-asyncio_default_test_loop_scope = "session"
-asyncio_default_fixture_loop_scope = "session"

scrapemm-0.6.2/requirements.txt

@@ -1,25 +0,0 @@
-pytest~=8.4.1
-aiohttp~=3.13.5
-requests~=2.32.4
-ezMM~=0.5.1
-Telethon~=1.40.0
-PyYAML~=6.0.1
-tweepy~=4.16.0
-markdownify~=1.1.0
-tqdm~=4.66.3
-pytest-asyncio
-async_lru
-oauthlib
-platformdirs~=4.3.8
-atproto
-TikTokResearchApi~=1.0.4
-yt-dlp[default,curl-cffi]
-curl_cffi~=0.14.0b5  # yt-dlp doesn't support 0.15+ yet
-cryptography
-firecrawl-py~=4.10.4
-brotli  # Required to read brotli-encoded media
-playwright
-playwright-stealth
-beautifulsoup4
-pip-system-certs  # For better SSL certificate handling
-backports.zstd  # For better content parsing with Firecrawl

scrapemm-0.6.2/scrapeMM.egg-info/requires.txt

@@ -1,12 +0,0 @@
-ezmm
-telethon
-tweepy
-markdownify
-platformdirs
-PyYAML
-atproto
-TikTokResearchApi
-yt-dlp
-cryptography
-firecrawl-py
-playwright

scrapemm-0.6.2/scripts/example.py

@@ -1,10 +0,0 @@
-from scrapemm import retrieve
-import asyncio
-
-if __name__ == "__main__":
-    url = "https://www.snopes.com/fact-check/biden-bounty-maduro/"
-    result = asyncio.run(retrieve(url, methods=["firecrawl"]))
-    if result.errors:
-        print(result.errors)
-    else:
-        print(result.content)