scrapeMM 0.6.2__tar.gz → 0.6.3__tar.gz

{scrapemm-0.6.2/scrapeMM.egg-info → scrapemm-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.6.2
+Version: 0.6.3
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.6.2 → scrapemm-0.6.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.6.2"
+version = "0.6.3"
 description = "Flexible multimodal scraper for social media and the open web."
 readme = "README.md"
 requires-python = ">=3.10"

{scrapemm-0.6.2 → scrapemm-0.6.3}/requirements.txt

@@ -22,4 +22,5 @@ playwright
 playwright-stealth
 beautifulsoup4
 pip-system-certs  # For better SSL certificate handling
-backports.zstd  # For better content parsing with Firecrawl
+backports.zstd  # For better content parsing with Firecrawl
+prompt_toolkit  # For handling user terminal input

{scrapemm-0.6.2 → scrapemm-0.6.3/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.6.2
+Version: 0.6.3
 Summary: Flexible multimodal scraper for social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.6.2 → scrapemm-0.6.3}/scrapemm/integrations/decodo/decodo.py

@@ -26,23 +26,21 @@ class Decodo:
     DECODO_API_URL = "https://scraper-api.decodo.com/v2/scrape"
 
     def __init__(self):
-        self.username = None
-        self.password = None
+        self.basic_auth_token = None
         self.n_scrapes = 0
 
-    def _load_credentials(self):
+    def _load_token(self):
         """Loads Decodo credentials from the secrets manager."""
-        self.username = get_secret("decodo_username")
-        self.password = get_secret("decodo_password")
+        self.basic_auth_token = get_secret("decodo_token")
 
-        if self.username and self.password:
-            logger.info("✅ Decodo credentials loaded successfully.")
+        if self.basic_auth_token:
+            logger.info("✅ Decodo token set.")
         else:
-            logger.warning("⚠️ Decodo credentials not found. Please configure them in secrets.")
+            logger.warning("⚠️ Decodo auth token not found. Please configure it in secrets.")
 
-    def _has_credentials(self) -> bool:
+    def _has_token(self) -> bool:
         """Checks if Decodo credentials are available."""
-        return bool(self.username and self.password)
+        return bool(self.basic_auth_token)
 
     async def scrape(
             self, url: str,
@@ -66,10 +64,10 @@ class Decodo:
         Returns:
             MultimodalSequence containing the scraped content, or None if scraping failed
         """
-        if not self._has_credentials():
-            self._load_credentials()
+        if not self._has_token():
+            self._load_token()
 
-        if not self._has_credentials():
+        if not self._has_token():
             logger.warning("Cannot scrape with Decodo: credentials not configured.")
             return None
 
@@ -124,6 +122,7 @@ class Decodo:
         """
         headers = {
             'Content-Type': 'application/json',
+            'Authorization': f'Basic {self.basic_auth_token}',
         }
 
         # Build request payload
@@ -141,9 +140,6 @@ class Decodo:
         if use_premium_proxy:
             payload["proxy_pool"] = "premium"
 
-        # Create basic auth
-        auth = aiohttp.BasicAuth(self.username, self.password)
-
         # Retry loop with exponential backoff
         for attempt in range(max_retries + 1):
             try:
@@ -151,12 +147,11 @@ class Decodo:
                         self.DECODO_API_URL,
                         json=payload,
                         headers=headers,
-                        auth=auth,
                         timeout=aiohttp.ClientTimeout(total=timeout)
                 ) as response:
                     # Validate response health
                     if response.status != 200:
-                        logger.debug("Communication with Decodo API failed.")
+                        logger.debug(f"Communication with Decodo API failed. Status code: {response.status}")
 
                         if response.status == 429:  # Rate limit
                             if attempt >= max_retries:

{scrapemm-0.6.2 → scrapemm-0.6.3}/scrapemm/retrieval.py

@@ -41,6 +41,8 @@ BEST_METHODS = {
     "perma.cc": ["integrations"],
     "mvau.lt": ["integrations"],
     "archive.org": ["integrations"],
+    "verafiles.org": ["decodo", "firecrawl"],
+    "youturn.in": ["decodo"],
     # Archive.today:
     "archive.today": ["integrations", "decodo"],
     "archive.is": ["integrations", "decodo"],
@@ -185,7 +187,7 @@ async def _retrieve_single(
                                              session=session, format=format, actions=actions),
             "decodo": lambda: decodo.scrape(url, remove_urls, session,
                                             format=format,
-                                            timeout=10 if prioritize == "speed" else 30,
+                                            timeout=15 if prioritize == "speed" else 30,
                                             max_retries=1 if prioritize == "speed" else 5),
         }
 
@@ -227,7 +229,7 @@ async def _retrieve_single(
 
         except Exception as e:
             logger.warning(f"Error while retrieving with method '{method_name}': {e}")
-            errors[method_name] = e
+            errors[method_name] = f"{type(e).__name__}: {e}"
             result = None
 
         if result is not None:

{scrapemm-0.6.2 → scrapemm-0.6.3}/scrapemm/secrets.py

@@ -9,8 +9,8 @@ from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
 
-from scrapemm.common import get_config_var, update_config, CONFIG_PATH, CONFIG_DIR
-from scrapemm.util import get_multiline_user_input
+from scrapemm.common import get_config_var, update_config, CONFIG_DIR
+from scrapemm.util import get_user_input
 
 logger = logging.getLogger("scrapeMM")
 
@@ -23,8 +23,7 @@ SECRETS = {
     "bluesky_password": "Bluesky password",
     "tiktok_client_key": "TikTok client key",
     "tiktok_client_secret": "TikTok client secret",
-    "decodo_username": "Decodo Web Scraping API username",
-    "decodo_password": "Decodo Web Scraping API password",
+    "decodo_token": "Decodo Web Scraping API basic authentication token",
     "youtube_cookie": "YouTube cookie string",
     "facebook_cookie": "Facebook cookie string",
 }
@@ -144,9 +143,9 @@ def override_secret(key_name: str):
     when nothing entered."""
     description = SECRETS[key_name]
     if key_name in ["youtube_cookie", "facebook_cookie"]:
-        user_input = get_multiline_user_input(f"Please enter the {description}. Hit Ctrl-D or Ctrl-Z to save it.")
+        user_input = get_user_input(f"Please enter the {description} (Alt+Enter to submit):", multiline=True)
     else:
-        user_input = getpass(f"Please enter the {description} (leave empty to skip): ", stream=sys.stdout)
+        user_input = get_user_input(f"Please enter the {description} (leave empty to skip):")
     if user_input:
         set_secret(key_name, user_input)
 
@@ -171,8 +170,7 @@ def configure_secrets(all_keys: bool = False):
 
     update_config(api_keys_configured=True)
 
-    logger.info("API keys configured successfully! If you want to change them, go to "
-                f"{CONFIG_PATH.as_posix()} and set 'api_keys_configured' to 'false' or "
+    logger.info("API keys configured successfully! If you want to change them, "
                 f"run scrapemm.secrets.configure_secrets().")
 
 

{scrapemm-0.6.2 → scrapemm-0.6.3}/scrapemm/util.py

@@ -38,7 +38,7 @@ def get_domain(url: str, keep_subdomain: bool = False) -> Optional[str]:
 async def run_with_semaphore(tasks: Iterable[Awaitable],
                              limit: int,
                              show_progress: bool = True,
-                             progress_description: str | None = None) -> list:
+                             progress_description: str | None = None) -> tuple:
     """
     Runs asynchronous tasks with a concurrency limit.
 
@@ -76,10 +76,32 @@ def read_urls_from_file(file_path):
         return f.read().splitlines()
 
 
-def get_multiline_user_input(prompt: str) -> str:
-    print(prompt)
-    lines = sys.stdin.readlines()
-    return "".join(lines)
+def get_user_input(prompt: str, multiline: bool = False) -> str:
+    """Prompts the user for input.
+    - Single-line: uses standard input(); Enter submits, empty input skips.
+    - Multiline: uses prompt_toolkit; Enter adds newline, Alt+Enter submits.
+    """
+    if multiline:
+        from prompt_toolkit import prompt as pt_prompt
+        from prompt_toolkit.key_binding import KeyBindings
+
+        print(prompt)
+
+        bindings = KeyBindings()
+
+        @bindings.add('escape', 'enter')
+        def _submit(event):
+            event.current_buffer.validate_and_handle()
+
+        result = pt_prompt(
+            ">>> ",
+            multiline=True,
+            key_bindings=bindings,
+            prompt_continuation="... ",
+        )
+        return result
+    else:
+        return input(f"{prompt} ")
 
 
 MAX_MEDIA_PER_PAGE = 32
@@ -127,7 +149,7 @@ async def resolve_media_hyperlinks(
 
     # Try to download media for each URL
     tasks = [download_medium(u, session=session, headers={"Referer": url}) for u in hrefs_urls.values()]
-    media: list[Item | None] = await run_with_semaphore(tasks, limit=100, show_progress=False)
+    media: tuple[Item | None] = await run_with_semaphore(tasks, limit=100, show_progress=False)
 
     href_media = dict(zip(hrefs_urls.keys(), media))
 

scrapemm-0.6.3/scripts/example.py

@@ -0,0 +1,11 @@
+from scrapemm import retrieve
+import asyncio
+
+if __name__ == "__main__":
+    url = "https://youturn.in/factcheck/tamilnadu-india-alliance-final-list.html"
+    result = asyncio.run(retrieve(url, methods=["decodo", "firecrawl"]))
+    print(f"Retrieval time: {result.retrieval_time:.2f} seconds")
+    if result.errors:
+        print(result.errors)
+    else:
+        print(result.content)

scrapemm-0.6.2/scripts/example.py

@@ -1,10 +0,0 @@
-from scrapemm import retrieve
-import asyncio
-
-if __name__ == "__main__":
-    url = "https://www.snopes.com/fact-check/biden-bounty-maduro/"
-    result = asyncio.run(retrieve(url, methods=["firecrawl"]))
-    if result.errors:
-        print(result.errors)
-    else:
-        print(result.content)