scrapeMM 0.2.1__tar.gz → 0.2.2__tar.gz

{scrapemm-0.2.1/scrapeMM.egg-info → scrapemm-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.2.1
+Version: 0.2.2
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.2.1 → scrapemm-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.2.1"
+version = "0.2.2"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"
@@ -24,7 +24,7 @@ Homepage = "https://github.com/multimodal-ai-lab/scrapeMM"
 Issues = "https://github.com/multimodal-ai-lab/scrapeMM"
 
 [build-system]
-requires = ["setuptools>=60", "setuptools-scm>=8.0"]
+requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools.packages.find]

{scrapemm-0.2.1 → scrapemm-0.2.2/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.2.1
+Version: 0.2.2
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0

{scrapemm-0.2.1 → scrapemm-0.2.2}/scrapeMM.egg-info/SOURCES.txt

@@ -16,6 +16,8 @@ scrapemm/util.py
 scrapemm/integrations/__init__.py
 scrapemm/integrations/base.py
 scrapemm/integrations/bluesky.py
+scrapemm/integrations/fb.py
+scrapemm/integrations/instagram.py
 scrapemm/integrations/telegram.py
 scrapemm/integrations/tiktok.py
 scrapemm/integrations/x.py
@@ -23,6 +25,7 @@ scrapemm/scraping/__init__.py
 scrapemm/scraping/firecrawl.py
 scrapemm/scraping/no_bot_domains.txt
 scrapemm/scraping/util.py
+scrapemm/scraping/ytdlp.py
 scripts/example.py
 testing/test_retrieval.py
 testing/test_utils.py

{scrapemm-0.2.1 → scrapemm-0.2.2}/scrapemm/integrations/__init__.py

@@ -10,11 +10,13 @@ from ezmm import MultimodalSequence
 
 from scrapemm.util import get_domain
 from .bluesky import Bluesky
+from .fb import Facebook
+from .instagram import Instagram
 from .telegram import Telegram
 from .tiktok import TikTok
 from .x import X
 
-RETRIEVAL_INTEGRATIONS = [X(), Telegram(), Bluesky(), TikTok()]
+RETRIEVAL_INTEGRATIONS = [X(), Telegram(), Bluesky(), TikTok(), Instagram(), Facebook()]
 DOMAIN_TO_INTEGRATION = {domain: integration
                          for integration in RETRIEVAL_INTEGRATIONS
                          for domain in integration.domains

scrapemm-0.2.2/scrapemm/integrations/fb.py

@@ -0,0 +1,101 @@
+import logging
+from urllib.parse import urlparse
+import aiohttp
+from ezmm import MultimodalSequence, download_image
+
+from scrapemm.scraping.ytdlp import check_ytdlp_available,  get_video_with_ytdlp
+from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.util import get_domain
+
+logger = logging.getLogger("scrapeMM")
+
+
+class Facebook(RetrievalIntegration):
+
+    domains = ["facebook.com", "www.facebook.com"]
+
+    def __init__(self):
+        # Check if yt-dlp is available
+        self.ytdlp_available = check_ytdlp_available()
+
+        if self.ytdlp_available:
+            self.connected = True
+            mode = "yt-dlp only"
+            logger.info(f"✅ Facebook integration ready ({mode} mode).")
+        else:
+            self.connected = False
+            logger.warning("❌ Facebook integration not available: Neither API credentials nor yt-dlp found.")
+
+    async def get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook post URL."""
+        if not self.connected:
+            logger.error("❌ Facebook integration not connected. Cannot retrieve content.")
+            return None
+            
+        if get_domain(url) not in self.domains:
+            logger.error(f"❌ Invalid domain for Facebook: {get_domain(url)}")
+            return None
+        
+        # Determine if this is a video or profile URL
+        if self._is_video_url(url):
+            return await self._get_video(url, session)
+        elif self._is_photo_url(url):
+            return await self._get_photo(url, session)
+        else:
+            return await self._get_user_profile(url, session)
+        
+    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a TikTok video URL."""
+
+        # Fallback to yt-dlp mode
+        if self.ytdlp_available:
+            return await get_video_with_ytdlp(url, session, platform="Facebook")
+
+        logger.error("❌ No available method to retrieve Facebook video.")
+        return None
+    
+    async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook photo URL."""
+        logger.error("❌ No available method to retrieve Facebook photo.")
+        return None
+
+    async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook user profile URL."""
+        username = self._extract_username(url)
+        if username:
+            text = f"""**Facebook Profile**
+Username: @{username}
+URL: {url}
+
+Note: Profile details require Facebook API access.
+Configure API credentials for full profile information."""
+            return MultimodalSequence([text])
+        
+        return None
+
+    def _is_video_url(self, url: str) -> bool:
+        """Checks if the URL is a Facebook video URL."""
+        # video URLS are in the format: https://www.facebook.com/watch?v=VIDEO_ID
+        return "facebook.com/watch" in url
+    
+    def _extract_video_id(self, url: str) -> str:
+        """Extracts the video ID from a Facebook video URL."""
+        parsed_url = urlparse(url)
+        query_params = parsed_url.query
+        for param in query_params.split('&'):
+            if param.startswith('v='):
+                return param.split('=')[1]
+        return ""
+    
+    def _is_photo_url(self, url: str) -> bool:
+        """Checks if the URL is a Facebook photo URL."""
+        return "facebook.com/photo" in url or "facebook.com/photos" in url
+    
+    def _extract_username(self, url: str) -> str:
+        """Extracts the username from a Facebook profile URL."""
+        # url format: https://www.facebook.com/username<?...>
+        parsed_url = urlparse(url)
+        path_parts = parsed_url.path.strip('/').split('/')
+        if len(path_parts) > 0:
+            return path_parts[0]
+        return ""

scrapemm-0.2.2/scrapemm/integrations/instagram.py

@@ -0,0 +1,87 @@
+import logging
+from urllib.parse import urlparse
+import aiohttp
+from ezmm import MultimodalSequence, download_image
+
+from scrapemm.scraping.ytdlp import check_ytdlp_available,  get_video_with_ytdlp
+from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.util import get_domain
+
+logger = logging.getLogger("scrapeMM")
+
+
+class Instagram(RetrievalIntegration):
+    domains = ["instagram.com", "www.instagram.com"]
+
+    def __init__(self):
+        # Check if yt-dlp is available
+        self.ytdlp_available = check_ytdlp_available()
+
+        if self.ytdlp_available:
+            self.connected = True
+            mode = "yt-dlp only"
+            logger.info(f"✅ Instagram integration ready ({mode} mode).")
+        else:
+            self.connected = False
+            logger.warning("❌ Instagram integration not available: Neither API credentials nor yt-dlp found.")
+
+    async def get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from an Instagram post URL."""
+        if not self.connected:
+            logger.error("❌ Instagram integration not connected. Cannot retrieve content.")
+            return None
+            
+        if get_domain(url) not in self.domains:
+            logger.error(f"❌ Invalid domain for Instagram: {get_domain(url)}")
+            return None
+        
+        # Determine if this is a video or profile URL
+        if self._is_video_url(url):
+            return await self._get_video(url, session)
+        elif self._is_photo_url(url):
+            return await self._get_photo(url, session)
+        else:
+            return await self._get_user_profile(url, session)
+        
+    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from an Instagram video URL."""
+        if self.ytdlp_available:
+            return await get_video_with_ytdlp(url, session, platform="Instagram")
+
+        logger.error("❌ No available method to retrieve Instagram video.")
+        return None
+    
+    async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from an Instagram photo URL."""
+        logger.error("❌ No available method to retrieve Instagram photo.")
+        return None
+    
+    async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from an Instagram user profile URL."""
+        username = self._extract_username(url)
+        if username:
+            text = f"""**Instagram Profile**
+Username: @{username}
+URL: {url}
+
+Note: Profile details require Instagram API access.
+Configure API credentials for full profile information."""
+            return MultimodalSequence([text])
+
+        return None
+
+    def _is_video_url(self, url: str) -> bool:
+        """Checks if the URL is an Instagram video URL."""
+        return "instagram.com/reels" in url
+
+    def _is_photo_url(self, url: str) -> bool:
+        """Checks if the URL is an Instagram photo URL."""
+        return "instagram.com/p/" in url
+
+    def _extract_username(self, url: str) -> str:
+        """Extracts the username from an Instagram profile URL."""
+        parsed_url = urlparse(url)
+        path_parts = parsed_url.path.strip('/').split('/')
+        if len(path_parts) > 0:
+            return path_parts[0]
+        return ""

{scrapemm-0.2.1 → scrapemm-0.2.2}/scrapemm/integrations/tiktok.py

@@ -2,9 +2,9 @@ import asyncio
 import json
 import logging
 import re
+import sys
 import tempfile
 import os
-import subprocess
 from urllib.parse import urlparse
 from datetime import datetime
 from typing import Any
@@ -15,6 +15,7 @@ from ezmm.common.items import Video, Image
 
 from tiktok_research_api import TikTokResearchAPI, QueryVideoRequest, QueryUserInfoRequest, Criteria, Query
 
+from scrapemm.scraping.ytdlp import check_ytdlp_available
 from scrapemm.secrets import get_secret
 from scrapemm.integrations.base import RetrievalIntegration
 from scrapemm.util import get_domain
@@ -56,7 +57,7 @@ class TikTok(RetrievalIntegration):
                 logger.warning(f"❌ TikTok Research API connection failed: {e}. Using fallback mode.")
         
         # Check if yt-dlp is available
-        self.ytdlp_available = self._check_ytdlp_available()
+        self.ytdlp_available = check_ytdlp_available()
         
         if self.api_available or self.ytdlp_available:
             self.connected = True
@@ -66,15 +67,6 @@ class TikTok(RetrievalIntegration):
             self.connected = False
             logger.warning("❌ TikTok integration not available: Neither API credentials nor yt-dlp found.")
 
-    def _check_ytdlp_available(self) -> bool:
-        """Check if yt-dlp is available."""
-        try:
-            subprocess.run(['yt-dlp', '--version'], 
-                         capture_output=True, check=True, timeout=5)
-            return True
-        except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired):
-            return False
-
     async def get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
         """Retrieves content from a TikTok post URL."""
         if not self.connected:
@@ -217,7 +209,9 @@ Configure API credentials for full profile information."""
         """Extracts metadata using yt-dlp without downloading the video."""
         try:
             cmd = [
-                'yt-dlp',
+                sys.executable,  # Ensure to use the same Python interpreter as the calling process
+                '-m',
+                'yt_dlp',
                 '--no-download',
                 '--print-json',
                 '--no-warnings',
@@ -252,7 +246,9 @@ Configure API credentials for full profile information."""
                 temp_path = temp_file.name
 
             cmd = [
-                'yt-dlp',
+                sys.executable,  # Ensure to use the same Python interpreter as the calling process
+                '-m',
+                'yt_dlp',
                 '--no-playlist',
                 '--no-warnings',
                 '--quiet',

{scrapemm-0.2.1 → scrapemm-0.2.2}/scrapemm/integrations/x.py

@@ -40,7 +40,7 @@ class X(RetrievalIntegration):
     def __init__(self):
         bearer_token = get_secret("x_bearer_token")
         if bearer_token:
-            self.client = AsyncClient(bearer_token=bearer_token)
+            self.client = AsyncClient(bearer_token=bearer_token, wait_on_rate_limit=True)
             self.connected = True
             logger.info("✅ Successfully connected to X.")
         else:
@@ -80,18 +80,19 @@ class X(RetrievalIntegration):
 
             # Download the media
             media = []
-            for medium_raw in media_raw:
-                if medium_raw.type == "photo":
-                    url = medium_raw.url
-                    medium = await download_image(url, session=session)
-                elif medium_raw.type in ["video", "animated_gif"]:
-                    # Get the variant with the highest bitrate
-                    url = _get_best_quality_video_url(medium_raw.variants)
-                    medium = await download_video(url, session=session)
-                else:
-                    raise ValueError(f"Unsupported media type: {medium_raw.type}")
-                if medium:
-                    media.append(medium)
+            if media_raw:
+                for medium_raw in media_raw:
+                    if medium_raw.type == "photo":
+                        url = medium_raw.url
+                        medium = await download_image(url, session=session)
+                    elif medium_raw.type in ["video", "animated_gif"]:
+                        # Get the variant with the highest bitrate
+                        url = _get_best_quality_video_url(medium_raw.variants)
+                        medium = await download_video(url, session=session)
+                    else:
+                        raise ValueError(f"Unsupported media type: {medium_raw.type}")
+                    if medium:
+                        media.append(medium)
 
             tweet_str = f"""**Post on X**
 Author: {author.name}, @{author.username}

{scrapemm-0.2.1 → scrapemm-0.2.2}/scrapemm/scraping/util.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Optional
 
 import aiohttp
@@ -16,6 +17,7 @@ URL_REGEX = r"https?:\/\/(?:www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{
 DATA_URI_REGEX = r"data:([\w/+.-]+/[\w.+-]+);base64,([A-Za-z0-9+/=]+)"
 MD_HYPERLINK_REGEX = rf'(!?\[([^]^[]*)\]\((.*?)(?: "[^"]*")?\))'
 
+logger = logging.getLogger("scrapeMM")
 
 def find_firecrawl(urls):
     for url in urls:
@@ -153,5 +155,6 @@ def from_base64(b64_data: str, mime_type: str = "image/jpeg") -> Optional[Item]:
                 return Video(binary_data=binary_data)
             else:
                 raise ValueError(f"Unsupported media type: {mime_type}")
-    except ValueError as e:
-        print(f"Error decoding base64 data: {e}")
+    except Exception as e:
+        logger.warning(f"Error decoding base64 data!\n"
+                       f"{type(e).__name__}: {e}")

scrapemm-0.2.2/scrapemm/scraping/ytdlp.py

@@ -0,0 +1,190 @@
+import sys
+from typing import Any, cast
+
+import aiohttp
+import asyncio
+from datetime import datetime
+import json
+import logging
+import os
+import subprocess
+import tempfile
+
+from ezmm import MultimodalSequence, download_image
+from ezmm.common.items import Video, Image
+
+logger = logging.getLogger("scrapeMM")
+
+
+def check_ytdlp_available() -> bool:
+    """Returns True if yt-dlp is available, else False."""
+    try:
+        # Run yt-dlp --version to check if it's installed and working'
+        subprocess.run([sys.executable, '-m', 'yt_dlp', '--version'],
+                     capture_output=True, check=True, timeout=5)
+        return True
+    except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired) as e:
+        return False
+
+
+async def extract_metadata_with_ytdlp(url: str) -> dict[str, Any] | None:
+    """Extracts metadata using yt-dlp without downloading the video."""
+    try:
+        cmd = [
+            sys.executable,  # Ensure to use the same Python interpreter as the calling process
+            '-m',
+            'yt_dlp',
+            '--no-download',
+            '--print-json',
+            '--no-warnings',
+            '--quiet',
+            url
+        ]
+
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        
+        stdout, stderr = await process.communicate()
+        
+        if process.returncode != 0:
+            logger.error(f"❌ yt-dlp metadata extraction failed: {stderr.decode()}")
+            return None
+
+        # Parse JSON output
+        metadata = json.loads(stdout.decode())
+        with open('metadata.json', 'w') as f:
+            json.dump(metadata, f, indent=2)
+        return metadata
+
+    except Exception as e:
+        logger.error(f"❌ Error extracting metadata with yt-dlp: {e}")
+        return None
+    
+
+async def download_video_with_ytdlp(url: str) -> Video | None:
+    """Downloads a video using yt-dlp."""
+    try:
+        with tempfile.NamedTemporaryFile(suffix='.%(ext)s', delete=False) as temp_file:
+            temp_path = temp_file.name
+
+        cmd = [
+            sys.executable,  # Ensure to use the same Python interpreter as the calling process
+            '-m',
+            'yt_dlp',
+            '--no-playlist',
+            '--no-warnings',
+            '--quiet',
+            '--format', 'best[ext=mp4]/best',
+            '--output', temp_path,
+            url
+        ]
+
+        process = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        
+        _, stderr = await process.communicate()
+        
+        if process.returncode != 0:
+            logger.warning(f"yt-dlp video download failed: {stderr.decode()}")
+            return None
+
+        # Find the actual downloaded file (yt-dlp changes extension)
+        downloaded_file = None
+        for ext in ['.mp4', '.webm', '.mkv']:
+            potential_file = temp_path.replace('.%(ext)s', ext)
+            if os.path.exists(potential_file) and os.path.getsize(potential_file) > 0:
+                downloaded_file = potential_file
+                break
+
+        if downloaded_file and os.path.exists(downloaded_file):
+            with open(downloaded_file, 'rb') as f:
+                video_data = f.read()
+            
+            os.unlink(downloaded_file)
+            
+            video = Video(binary_data=video_data, source_url=url)
+            video.relocate(move_not_copy=True)
+            return cast(Video, video)
+
+        return None
+
+    except Exception as e:
+        logger.error(f"❌ Error downloading video with yt-dlp: {e}")
+        return None
+    
+
+async def download_thumbnail_with_ytdlp(url: str, session: aiohttp.ClientSession) -> Image | None:
+    """Downloads thumbnail using yt-dlp metadata."""
+    try:
+        metadata = await extract_metadata_with_ytdlp(url)
+        if not metadata:
+            return None
+
+        thumbnail_url = metadata.get('thumbnail')
+        if thumbnail_url:
+            return await download_image(thumbnail_url, session)
+
+    except Exception as e:
+        logger.error(f"❌ Error downloading thumbnail: {e}")
+    
+    return None
+
+
+async def create_video_sequence_from_ytdlp(metadata: dict, url: str, video: Video | None, thumbnail: Image | None, platform: str) -> MultimodalSequence:
+    """Creates MultimodalSequence from yt-dlp metadata."""
+    # title = metadata.get('title', '')
+    uploader = metadata.get('uploader', 'Unknown')
+    upload_date = metadata.get('upload_date', '')
+    duration = metadata.get('duration', 0)
+    view_count = metadata.get('view_count', 0)
+    like_count = metadata.get('like_count', 0)
+    comment_count = metadata.get('comment_count', 0)
+    description = metadata.get('description', '')
+    
+    # Format upload date
+    formatted_date = upload_date
+    if upload_date and len(upload_date) == 8:
+        try:
+            date_obj = datetime.strptime(upload_date, '%Y%m%d')
+            formatted_date = date_obj.strftime('%Y-%m-%d')
+        except ValueError:
+            pass
+
+    text = f"""**{platform} Video**
+Author: @{uploader}
+Posted: {formatted_date}
+Duration: {duration}s
+Views: {view_count:,} - Likes: {like_count:,} - Comments: {comment_count:,}
+
+{description}"""
+
+    items: list = [text]
+    if thumbnail:
+        items.append(thumbnail)
+    if video:
+        items.append(video)
+        
+    return MultimodalSequence(items)
+
+async def get_video_with_ytdlp(url: str, session: aiohttp.ClientSession, platform: str) -> MultimodalSequence | None:
+    """Retrieves video using only yt-dlp (no API required)."""
+    try:
+        # Get metadata and video using yt-dlp
+        metadata = await extract_metadata_with_ytdlp(url)
+        if not metadata:
+            return None
+
+        video = await download_video_with_ytdlp(url)
+        thumbnail = await download_thumbnail_with_ytdlp(url, session)
+        
+        return await create_video_sequence_from_ytdlp(metadata, url, video, thumbnail, platform)
+
+    except Exception as e:
+        logger.error(f"❌ Error retrieving video with yt-dlp: {e}")
+        return None

{scrapemm-0.2.1 → scrapemm-0.2.2}/scrapemm/secrets.py

@@ -118,9 +118,14 @@ def set_password(pwd: str):
 
 def configure_secrets(all_keys: bool = False):
     """Gets the secrets from the user by running a CLI dialogue.
-    Saves them in an encrypted file."""
+    Saves them in an encrypted file. Deletes the existing secrets file if existing."""
     logging.debug("Configuring new secrets...")
-    # Ensure a password is set
+
+    # Delete existing secrets
+    if SECRETS_PATH.exists():
+        SECRETS_PATH.unlink()
+
+    # Set up a new password
     _get_password("🔐 Enter a password to encrypt your secrets (you'll need it later to decrypt them): ")
 
     prompted = False