scrapeMM 0.3.4__tar.gz → 0.3.6__tar.gz

{scrapemm-0.3.4/scrapeMM.egg-info → scrapemm-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.4
+Version: 0.3.6
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.3.4 → scrapemm-0.3.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.3.4"
+version = "0.3.6"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"

{scrapemm-0.3.4 → scrapemm-0.3.6}/requirements.txt

@@ -2,7 +2,7 @@ pytest~=8.4.1
 aiohttp~=3.12.13
 numpy~=1.26.4
 requests~=2.32.4
-ezMM~=0.3.4
+ezMM~=0.3.9
 Telethon~=1.40.0
 PyYAML~=6.0.1
 tweepy~=4.16.0

{scrapemm-0.3.4 → scrapemm-0.3.6/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.3.4
+Version: 0.3.6
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/integrations/__init__.py

@@ -18,9 +18,9 @@ DOMAIN_TO_INTEGRATION = {domain: integration
                          for domain in integration.domains}
 
 
-async def retrieve_via_integration(url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+async def retrieve_via_integration(url: str, **kwargs) -> Optional[MultimodalSequence]:
     domain = get_domain(url)
     if domain in DOMAIN_TO_INTEGRATION:
         integration = DOMAIN_TO_INTEGRATION[domain]
         if integration.connected or integration.connected is None:
-            return await integration.get(url, session)
+            return await integration.get(url, **kwargs)

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/integrations/base.py

@@ -1,7 +1,6 @@
 from abc import ABC, abstractmethod
 from typing import Optional
 
-import aiohttp
 from ezmm import MultimodalSequence
 
 from scrapemm.common import logger
@@ -22,7 +21,7 @@ class RetrievalIntegration(ABC):
         Must set self.connet = True if connection was successful, else False."""
         raise NotImplementedError
 
-    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         """Ensures connectivity before invoking the service for retrieval."""
         assert get_domain(url) in self.domains
         if self.connected is None:
@@ -31,9 +30,9 @@ class RetrievalIntegration(ABC):
                 logger.warning(f"❌ Connection to {self.name} service could not be established.")
         if self.connected:
             logger.debug(f"Calling {self.name} service for {url}")
-            return await self._get(url, session)
+            return await self._get(url, **kwargs)
 
     @abstractmethod
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         """Retrieves the contents present at the given URL."""
         raise NotImplementedError

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/integrations/bluesky.py

@@ -28,11 +28,12 @@ class Bluesky(RetrievalIntegration):
         self.client = Client()
         self._authenticate()
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Bluesky: {get_domain(url)}")
             return None
 
+        session = kwargs.get("session")
         if "post" in url:
             result = await self._retrieve_post(url, session)
         else:

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/integrations/fb.py

@@ -23,7 +23,7 @@ class Facebook(RetrievalIntegration):
         logger.info(f"✅ Facebook integration ready (yt-dlp only mode).")
         self.connected = True
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook post URL."""
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Facebook: {get_domain(url)}")
@@ -31,31 +31,29 @@ class Facebook(RetrievalIntegration):
 
         # Determine if this is a video or photo URL, act accordingly
         if self._is_video_url(url):
-            return await self._get_video(url, session)
+            return await self._get_video(url, **kwargs)
         elif self._is_photo_url(url):
-            return await self._get_photo(url, session)
+            return await self._get_photo(url, **kwargs)
 
         # The URL is not indicative, so try all methods
-        return (await self._get_video(url, session) or
-                await self._get_photo(url, session) or
-                await self._get_user_profile(url, session))
+        return (await self._get_video(url, **kwargs) or
+                await self._get_photo(url, **kwargs) or
+                await self._get_user_profile(url, **kwargs))
 
-    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get_video(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook video URL."""
         if self.api_available:
-            raise NotImplementedError
+            raise NotImplementedError("❌ Facebook video retrieval through API not yet supported.")
         else:
-            return await get_content_with_ytdlp(url, session, platform="Facebook")
+            return await get_content_with_ytdlp(url, platform="Facebook", **kwargs)
 
-    async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get_photo(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook photo URL."""
-        logger.error("❌ No available method to retrieve Facebook photo.")
-        return None
+        raise NotImplementedError("❌ No available method to retrieve Facebook photos.")
 
-    async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get_user_profile(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from a Facebook user profile URL."""
-        logger.error("❌ No available method to retrieve Facebook profiles.")
-        return None
+        raise NotImplementedError("❌ No available method to retrieve Facebook profiles.")
 
     def _is_video_url(self, url: str) -> bool:
         """Checks if the URL is a Facebook video URL."""

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/integrations/instagram.py

@@ -20,7 +20,7 @@ class Instagram(RetrievalIntegration):
         logger.info(f"✅ Instagram integration ready (yt-dlp only mode).")
         self.connected = True
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from an Instagram post URL."""
         if get_domain(url) not in self.domains:
             logger.error(f"❌ Invalid domain for Instagram: {get_domain(url)}")
@@ -28,26 +28,30 @@ class Instagram(RetrievalIntegration):
 
         # Determine if this is a video or profile URL
         if self._is_video_url(url):
-            return await self._get_video(url, session)
+            return await self._get_video(url, **kwargs)
         elif self._is_photo_url(url):
             # /p/ URLs can also be reels, so try both
-            return await self._get_video(url, session) or await self._get_photo(url, session)
+            content = await self._get_video(url, **kwargs)
+            if content and content.has_videos():
+                return content
+            else:
+                return await self._get_photo(url, **kwargs)
         else:
-            return await self._get_user_profile(url, session)
+            return await self._get_user_profile(url, **kwargs)
 
-    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get_video(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from an Instagram video URL."""
         if self.api_available:
             raise NotImplementedError
         else:
-            return await get_content_with_ytdlp(url, session, platform="Instagram")
+            return await get_content_with_ytdlp(url, platform="Instagram", **kwargs)
 
-    async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get_photo(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from an Instagram photo URL (can also be a reel)."""
-        logger.error("❌ No available method to retrieve Instagram photo.")
+        logger.warning("❌ Native Instagram photo download not yet supported. Use Decodo for that.")
         return None
 
-    async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get_user_profile(self, url: str, **kwargs) -> MultimodalSequence | None:
         """Retrieves content from an Instagram user profile URL."""
         username = self._extract_username(url)
         if username:

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/integrations/telegram.py

@@ -42,7 +42,7 @@ class Telegram(RetrievalIntegration):
             self.connected = False
             logger.warning("❌ Telegram integration not configured: Missing API keys.")
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
         """Retrieves content from a Telegram post URL."""
         assert get_domain(url) in self.domains
 

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/integrations/tiktok.py

@@ -44,6 +44,7 @@ class TikTok(RetrievalIntegration):
 
         if client_key and client_secret:
             try:
+                # TODO: Replace with async version
                 self.api = TikTokResearchAPI(
                     client_key=client_key,
                     client_secret=client_secret,
@@ -60,7 +61,9 @@ class TikTok(RetrievalIntegration):
         logger.info(f"✅ TikTok integration ready ({mode} mode).")
         self.connected = True
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get(self, url: str, **kwargs) -> MultimodalSequence | None:
+        session = kwargs.get('session')
+
         # Determine if this is a video or profile URL
         if self._is_video_url(url):
             return await self._get_video(url, session)
@@ -72,7 +75,7 @@ class TikTok(RetrievalIntegration):
 
         # Try API mode first if available
         if self.api_available:
-            result = await self._get_video_with_api(url, session)
+            result = await self._get_video_with_api(url)
             if result:
                 return result
             logger.warning("API method failed, falling back to yt-dlp...")
@@ -81,7 +84,7 @@ class TikTok(RetrievalIntegration):
         else:
             return await self._get_video_with_ytdlp(url, session)
 
-    async def _get_video_with_api(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+    async def _get_video_with_api(self, url: str) -> MultimodalSequence | None:
         """Retrieves video using TikTok Research API."""
         video_id = self._extract_video_id(url)
         if not video_id:

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/integrations/x.py

@@ -47,7 +47,8 @@ class X(RetrievalIntegration):
             self.connected = False
             logger.warning("❌ X (Twitter) integration not configured: Missing bearer token.")
 
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+    async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
+        session = kwargs.get("session")
         tweet_id = extract_tweet_id_from_url(url)
         if tweet_id:
             return await self._get_tweet(tweet_id, session)

scrapemm-0.3.6/scrapemm/integrations/youtube.py

@@ -0,0 +1,43 @@
+import logging
+from typing import Optional
+
+import aiohttp
+from ezmm import MultimodalSequence
+
+from scrapemm.scraping.ytdlp import get_content_with_ytdlp
+from .base import RetrievalIntegration
+from scrapemm.secrets import get_secret
+from ..common import CONFIG_DIR
+
+logger = logging.getLogger("scrapeMM")
+
+
+class YouTube(RetrievalIntegration):
+    """YouTube integration for downloading videos and shorts using yt-dlp.
+    YouTube is rate-limited to 333 videos per hour."""
+
+    name = "YouTube"
+    domains = [
+        "youtube.com",
+        "youtu.be",
+    ]
+    cookie_file = CONFIG_DIR / "youtube_cookie.txt"
+
+    async def _connect(self):
+        self.connected = True  # Connect always by default
+
+        cookie = get_secret("youtube_cookie")
+        if cookie:
+            # Save the cookie in a .txt file next to the secrets file
+            with open(self.cookie_file, "w") as f:
+                f.write(cookie)
+            logger.info(f"✅ Using cookie to connect to YouTube.")
+        else:
+            logger.warning(f"⚠️ Missing YouTube cookie. Won't be able to download videos, only thumbnails and metadata.")
+
+    async def _get(self, url: str, **kwargs) -> Optional[MultimodalSequence]:
+        """Downloads YouTube video or short using yt-dlp."""
+        return await get_content_with_ytdlp(url,
+                                            platform="YouTube",
+                                            cookie_file=self.cookie_file.as_posix(),
+                                            **kwargs)

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/retrieval.py

@@ -1,6 +1,7 @@
 import logging
-from typing import Optional, Collection
+import sqlite3
 from traceback import format_exc
+from typing import Optional, Collection
 
 import aiohttp
 from ezmm import MultimodalSequence, download_item
@@ -20,6 +21,8 @@ async def retrieve(
         actions: list[dict] = None,
         methods: list[str] = None,
         format: str = "multimodal_sequence",
+        max_video_size: int = None,
+        do_raise: bool = False,
 ) -> Optional[MultimodalSequence | str] | list[Optional[MultimodalSequence | str]]:
     """Main function of this repository. Downloads the contents present at the given URL(s).
     For each URL, returns a MultimodalSequence containing text, images, and videos.
@@ -41,6 +44,8 @@ async def retrieve(
     :param format: The format of the output. Available formats:
         - "multimodal_sequence" (MultimodalSequence containing parsed and downloaded media from the page)
         - "html" (string containing the raw HTML code of the page, not compatible with 'integrations' method)
+    :param max_video_size: Maximum size of videos to download, in MB. If None, no limit is applied.
+    :param do_raise: Whether to raise exceptions occurring during retrieval.
     """
     if methods is None:
         methods = METHODS
@@ -66,7 +71,9 @@ async def retrieve(
         urls_unique = set(urls_to_retrieve)
 
         # Retrieve URLs concurrently
-        tasks = [_retrieve_single(url, remove_urls, session, methods, actions, format) for url in urls_unique]
+        tasks = [_retrieve_single(url, remove_urls, session, methods, actions,
+                                  format, max_video_size, do_raise) for url in
+                 urls_unique]
         results = await run_with_semaphore(tasks, limit=20, show_progress=show_progress and len(urls_to_retrieve) > 1,
                                            progress_description="Retrieving URLs...")
 
@@ -85,6 +92,8 @@ async def _retrieve_single(
         methods: list[str],
         actions: list[dict] = None,
         format: str = "multimodal_sequence",
+        max_video_size: int = None,
+        do_raise: bool = False
 ) -> Optional[MultimodalSequence | str]:
     try:
         # Ensure URL is a string
@@ -101,9 +110,9 @@ async def _retrieve_single(
 
         # Define available retrieval methods
         method_map = {
-            "integrations": lambda: retrieve_via_integration(url, session),
+            "integrations": lambda: retrieve_via_integration(url, session=session, max_video_size=max_video_size),
             "firecrawl": lambda: fire.scrape(url, remove_urls=remove_urls,
-                                                  session=session, format=format, actions=actions),
+                                             session=session, format=format, actions=actions),
             "decodo": lambda: decodo.scrape(url, remove_urls, session, format=format),
         }
 
@@ -114,11 +123,26 @@ async def _retrieve_single(
                 continue
 
             logger.debug(f"Trying method: {method_name}")
+
             try:
                 result = await method_map[method_name]()
+
+            except sqlite3.OperationalError as e:
+                if str(e) == "attempt to write a readonly database":
+                    logger.error("ezMM database is read-only! Please check the database.")
+                else:
+                    logger.warning(f"Error while retrieving with method '{method_name}': {e}")
+                if do_raise:
+                    raise
+                else:
+                    result = None
+
             except Exception as e:
                 logger.warning(f"Error while retrieving with method '{method_name}': {e}")
-                result = None
+                if do_raise:
+                    raise
+                else:
+                    result = None
 
             if result is not None:
                 logger.debug(f"Successfully retrieved with method: {method_name}")

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/scraping/ytdlp.py

@@ -1,22 +1,31 @@
+import asyncio
 import logging
+import sys
 import tempfile
 import traceback
 from datetime import datetime
 from typing import Any, Optional
 
-import asyncio
 import aiohttp
 from ezmm import MultimodalSequence, download_image, Video, Image
 from yt_dlp import YoutubeDL
 
 logger = logging.getLogger("scrapeMM")
 
+# Add yt-dlp-specific logger to print warnings to console
+logger_yt_dlp = logging.getLogger("yt_dlp")
+logger_yt_dlp.setLevel(logging.WARNING)
+logger_yt_dlp.addHandler(logging.StreamHandler(sys.stdout))
+
 
 async def _download_with_ytdlp(
         url: str,
-        session: aiohttp.ClientSession
+        session: aiohttp.ClientSession,
+        max_video_size: int = None,
+        cookie_file: str = None
 ) -> tuple[Optional[Video], Optional[Image], Optional[dict[str, Any]]]:
-    """Downloads a video, its thumbnail, and the metadata using yt-dlp."""
+    """Downloads a video, its thumbnail, and the metadata using yt-dlp.
+    @param max_video_size: Maximum video size in bytes. If the video is larger, the download will be aborted."""
     try:
         with tempfile.NamedTemporaryFile() as temp_file:
             temp_path = temp_file.name
@@ -24,30 +33,34 @@ async def _download_with_ytdlp(
         ydl_opts: dict[str, Any] = dict(
             outtmpl=f'{temp_path}.%(ext)s',  # Output filename format
             format='best[ext=mp4]/best',  # Download the best video/audio quality
-            quiet=False,  # Show progress in terminal
+            max_filesize=max_video_size,
+            quiet=True,  # Silence logs in console
+            logger=logger_yt_dlp,  # Reroute logs to dedicated logger
             noplaylist=True,  # Disable playlist downloading
             retries=3,
-            cookies_from_browser='firefox',
-            cookies='cookies.txt',  # Use cookies from Firefox to bypass sign-in requirements
+            cookiefile=cookie_file,  # Use cookie to bypass sign-in requirements
         )
 
         if "youtube" in url or "youtu.be" in url:
             # YouTube delivers video and audio separately when downloaded above 720p.
             # This would require FFmpeg to merge them. Restrict to 720p to avoid that.
-            ydl_opts['format'] = 'best[height<=720][acodec!=none]'
+            ydl_opts['format'] = 'best[height<=720]'
             ydl_opts['extractor_args'] = dict(youtube=dict(player_client=["default"]))
 
         with YoutubeDL(ydl_opts) as ydl:
             metadata = ydl.extract_info(url, download=True)
 
-        ext = metadata.get("ext")
-
-        try:
-            video = Video(file_path=temp_path + f".{ext}", source_url=url)
-            video.relocate(move_not_copy=True)
-        except Exception as e:
-            logger.warning(f"Could not load downloaded video: {e}\n{traceback.format_exc()}")
-            video = None
+        video = None
+        if ext := metadata.get("ext"):
+            try:
+                video = Video(file_path=temp_path + f".{ext}", source_url=url)
+                video.relocate(move_not_copy=True)
+                if video.size > max_video_size:
+                    video = None  # Discard video
+            except FileNotFoundError:
+                pass
+            except Exception as e:
+                logger.warning(f"Could not load downloaded video: {e}\n{traceback.format_exc()}")
 
         thumbnail = None
         if thumbnail_url := metadata.get('thumbnail'):
@@ -56,7 +69,7 @@ async def _download_with_ytdlp(
         return video, thumbnail, metadata
 
     except Exception as e:
-        logger.warning(f"Could not download video with yt-dlp: {e}\n{traceback.format_exc()}")
+        logger.warning(f"Could not download video with yt-dlp: {e}")
         return None, None, None
 
 
@@ -102,13 +115,15 @@ Views: {fmt_count(view_count)} - Likes: {fmt_count(like_count)} - Comments: {fmt
     return MultimodalSequence(items)
 
 
-async def get_content_with_ytdlp(url: str, session: aiohttp.ClientSession, platform: str) -> MultimodalSequence | None:
+async def get_content_with_ytdlp(
+        url: str, session: aiohttp.ClientSession,
+        platform: str,
+        **kwargs
+) -> MultimodalSequence | None:
     """Retrieves video, thumbnail, and metadata using the powerful yt-dlp package."""
     # Run the download in a separate thread to avoid blocking the event loop
-    coroutine = await asyncio.to_thread(_download_with_ytdlp, url, session)
+    coroutine = await asyncio.to_thread(_download_with_ytdlp, url, session, **kwargs)
     video, thumbnail, metadata = await coroutine
-    if not video:
-        logger.warning(f"Video download failed for {url}")
     if metadata:
         return await compose_data_to_sequence(metadata, video, thumbnail, platform)
     return None

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/secrets.py

@@ -10,6 +10,7 @@ from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
 
 from scrapemm.common import get_config_var, update_config, CONFIG_PATH, CONFIG_DIR, logger
+from scrapemm.util import get_multiline_user_input
 
 SECRETS = {
     "x_bearer_token": "Bearer token of X (Twitter)",
@@ -22,6 +23,7 @@ SECRETS = {
     "tiktok_client_secret": "TikTok client secret",
     "decodo_username": "Decodo Web Scraping API username",
     "decodo_password": "Decodo Web Scraping API password",
+    "youtube_cookie": "YouTube cookie string"
 }
 
 SALT = b'\xa4\x93\xf1\x88\x13\x88'
@@ -125,40 +127,53 @@ def enter_password(pwd: str):
     _get_password(pwd=pwd)
 
 
-def configure_secrets(all_keys: bool = False):
-    """Gets the secrets from the user by running a CLI dialogue.
-    Saves them in an encrypted file. Deletes the existing secrets file if existing."""
-    logging.debug("Configuring new secrets...")
-
+def _set_new_password():
     # Delete existing secrets
-    if SECRETS_PATH.exists() and all_keys:
+    if SECRETS_PATH.exists():
         SECRETS_PATH.unlink()
 
-        # Set up a new password
-        _get_password("🔐 Enter a password to encrypt your secrets (you'll need it later to decrypt them): ")
+    # Set up a new password
+    _get_password("🔐 Enter a password to encrypt your secrets (you'll need it later to decrypt them): ")
+
+
+def override_secret(key_name: str):
+    """Prompts the user to enter a new value for the given secret key. Does nothing
+    when nothing entered."""
+    description = SECRETS[key_name]
+    if key_name == "youtube_cookie":
+        user_input = get_multiline_user_input(f"Please enter the {description}. Hit Ctrl-D or Ctrl-Z to save it.")
+    else:
+        user_input = getpass(f"Please enter the {description} (leave empty to skip): ", stream=sys.stdout)
+    if user_input:
+        set_secret(key_name, user_input)
+
 
-    prompted = False
-    for key_name, description in SECRETS.items():
+def configure_secrets(all_keys: bool = False):
+    """Gets the secrets from the user by running a CLI dialogue.
+    Saves them in an encrypted file. Only overrides an existing secret on non-empty input."""
+    logger.info("Starting secrets configuration...")
+
+    for key_name in SECRETS.keys():
         key_value = get_secret(key_name)
         if all_keys or not key_value:
             # Get and save the missing API key
-            user_input = getpass(f"Please enter the {description} (leave empty to skip): ", stream=sys.stdout)
-            prompted = True
-            if user_input:
-                set_secret(key_name, user_input)
+            override_secret(key_name)
 
     update_config(api_keys_configured=True)
 
-    if prompted:
-        print("API keys configured successfully! If you want to change them, go to "
-              f"{CONFIG_PATH.as_posix()} and set 'api_keys_configured' to 'false' or "
-              f"run scrapemm.api_keys.configure_api_keys().")
+    logger.info("API keys configured successfully! If you want to change them, go to "
+          f"{CONFIG_PATH.as_posix()} and set 'api_keys_configured' to 'false' or "
+          f"run scrapemm.api_keys.configure_api_keys().")
 
 
+# Ensure secrets file exists
+if not SECRETS_PATH.exists():
+    _set_new_password()
+
+# Ensure secrets are configured
 if not get_config_var("api_keys_configured"):
     configure_secrets()
 
-
 # Print secret summary
 logger.info("ScrapeMM secrets configuration:")
 for key_name, description in SECRETS.items():

{scrapemm-0.3.4 → scrapemm-0.3.6}/scrapemm/util.py

@@ -71,3 +71,9 @@ async def run_with_semaphore(tasks: Iterable[Awaitable],
 def read_urls_from_file(file_path):
     with open(file_path, 'r') as f:
         return f.read().splitlines()
+
+
+def get_multiline_user_input(prompt: str) -> str:
+    print(prompt)
+    lines = sys.stdin.readlines()
+    return "".join(lines)

{scrapemm-0.3.4 → scrapemm-0.3.6}/scripts/example.py

@@ -2,6 +2,6 @@ from scrapemm import retrieve
 import asyncio
 
 if __name__ == "__main__":
-    url = "https://www.youtube.com/shorts/cE0zgN6pYOc"
+    url = "https://www.instagram.com/reel/ClqTRryA6np/?utm_source=ig_embed&ig_rid=1ea336bf-737d-4526-8e12-233bf49f0488"
     result = asyncio.run(retrieve(url))
     print(result)

{scrapemm-0.3.4 → scrapemm-0.3.6}/testing/test_retrieval.py

@@ -133,3 +133,20 @@ async def test_youtube(url):
     print(result)
     assert result
     assert result.has_videos()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("url, max_video_size, download_expected", [
+    # ("https://www.facebook.com/reel/1089214926521000", None, True),
+    # ("https://www.facebook.com/reel/1089214926521000", 128_000_000, True),
+    ("https://www.facebook.com/reel/1089214926521000", 1_000_000, False),
+    # ("https://www.youtube.com/shorts/cE0zgN6pYOc", None, True),
+    # ("https://www.youtube.com/shorts/cE0zgN6pYOc", 4_000_000, True),
+    # ("https://www.youtube.com/shorts/cE0zgN6pYOc", 3_000_000, False),
+])
+async def test_max_video_size(url, max_video_size, download_expected):
+    result = await retrieve(url, max_video_size=max_video_size)
+    assert result.has_videos() == download_expected
+    if max_video_size and result.has_videos():
+        video = result.videos[0]
+        assert video.size <= max_video_size

scrapemm-0.3.4/scrapemm/integrations/youtube.py

@@ -1,28 +0,0 @@
-import logging
-from typing import Optional
-
-import aiohttp
-from ezmm import MultimodalSequence
-
-from scrapemm.scraping.ytdlp import get_content_with_ytdlp
-from .base import RetrievalIntegration
-
-logger = logging.getLogger("scrapeMM")
-
-
-class YouTube(RetrievalIntegration):
-    """YouTube integration for downloading videos and shorts using yt-dlp."""
-
-    name = "YouTube"
-    domains = [
-        "youtube.com",
-        "youtu.be",
-    ]
-
-    async def _connect(self):
-        self.connected = True
-
-    async def _get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
-        """Downloads YouTube video or short using yt-dlp."""
-        logger.debug(f"📺 Downloading YouTube content: {url}")
-        return await get_content_with_ytdlp(url, session, "YouTube")