scrapeMM 0.1.3__tar.gz → 0.2.2__tar.gz

{scrapemm-0.1.3 → scrapemm-0.2.2}/.gitignore

@@ -2,4 +2,5 @@
 /.idea/
 /scrapeMM.egg-info/
 /dist/
-/config.yaml
+/config.yaml
+__pycache__/

{scrapemm-0.1.3/scrapeMM.egg-info → scrapemm-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.1.3
+Version: 0.2.2
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -13,9 +13,12 @@ Requires-Dist: ezmm
 Requires-Dist: telethon
 Requires-Dist: tweepy
 Requires-Dist: markdownify
-Requires-Dist: keyring
 Requires-Dist: platformdirs
 Requires-Dist: PyYAML
+Requires-Dist: atproto
+Requires-Dist: TikTokResearchApi
+Requires-Dist: yt-dlp
+Requires-Dist: cryptography
 Dynamic: license-file
 
 # scrapeMM: Multimodal Web Retrieval
@@ -26,11 +29,15 @@ This library aims to help developers and researchers to easily access multimodal
 ## Usage
 ```python
 from scrapemm import retrieve
+import asyncio
 
 url = "https://example.com"
-result = retrieve(url)
+loop = asyncio.get_event_loop()
+result = loop.run_until_complete(retrieve(url))
 result.render()
 ```
+`scrapeMM` will ask you for the **API keys** needed for the social media integrations. You may skip them if you don't need them. 
+You will also be prompted to choose a **password** that is used to secure the secrets in an encrypted file.
 
 ## How it works
 ```
@@ -44,7 +51,9 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
+- ✅ Bluesky
+- ✅ TikTok
+- ⏳ Threads
+- ⏳ Reddit
 - ⏳ Facebook
 - ⏳ Instagram
-- ⏳ Threads
-- ⏳ TikTok

{scrapemm-0.1.3 → scrapemm-0.2.2}/README.md

@@ -6,11 +6,15 @@ This library aims to help developers and researchers to easily access multimodal
 ## Usage
 ```python
 from scrapemm import retrieve
+import asyncio
 
 url = "https://example.com"
-result = retrieve(url)
+loop = asyncio.get_event_loop()
+result = loop.run_until_complete(retrieve(url))
 result.render()
 ```
+`scrapeMM` will ask you for the **API keys** needed for the social media integrations. You may skip them if you don't need them. 
+You will also be prompted to choose a **password** that is used to secure the secrets in an encrypted file.
 
 ## How it works
 ```
@@ -24,7 +28,9 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
+- ✅ Bluesky
+- ✅ TikTok
+- ⏳ Threads
+- ⏳ Reddit
 - ⏳ Facebook
 - ⏳ Instagram
-- ⏳ Threads
-- ⏳ TikTok

{scrapemm-0.1.3 → scrapemm-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "scrapeMM"
-version = "0.1.3"
+version = "0.2.2"
 description = "LLM-friendly scraper for media and text from social media and the open web."
 readme = "README.md"
 requires-python = ">=3.8"
@@ -11,9 +11,12 @@ dependencies = [
     "telethon",
     "tweepy",
     "markdownify",
-    "keyring",
     "platformdirs",
-    "PyYAML"
+    "PyYAML",
+    "atproto",
+    "TikTokResearchApi",
+    "yt-dlp",
+    "cryptography",
 ]
 
 [project.urls]
@@ -21,7 +24,7 @@ Homepage = "https://github.com/multimodal-ai-lab/scrapeMM"
 Issues = "https://github.com/multimodal-ai-lab/scrapeMM"
 
 [build-system]
-requires = ["setuptools>=60", "setuptools-scm>=8.0"]
+requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools.packages.find]
@@ -30,4 +33,4 @@ namespaces = false  # Disable implicit namespaces
 # exclude = ["*.egg-info", "tests*", "in*", "scripts*", "temp*", "secrets.yaml", "dist*"]
 
 [tool.setuptools.package-data]
-"*" = ["*.txt"]  # Include all .txt files in the build
+"*" = ["*.txt"]  # Include all .txt files contained in any (sub)package in the build

{scrapemm-0.1.3 → scrapemm-0.2.2}/requirements.txt

@@ -9,7 +9,10 @@ tweepy~=4.16.0
 markdownify~=1.1.0
 tqdm~=4.66.3
 pytest-asyncio
-keyring~=25.6.0
 async_lru
 oauthlib
-platformdirs~=4.3.8
+platformdirs~=4.3.8
+atproto
+TikTokResearchApi
+yt-dlp
+cryptography

{scrapemm-0.1.3 → scrapemm-0.2.2/scrapeMM.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.1.3
+Version: 0.2.2
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
@@ -13,9 +13,12 @@ Requires-Dist: ezmm
 Requires-Dist: telethon
 Requires-Dist: tweepy
 Requires-Dist: markdownify
-Requires-Dist: keyring
 Requires-Dist: platformdirs
 Requires-Dist: PyYAML
+Requires-Dist: atproto
+Requires-Dist: TikTokResearchApi
+Requires-Dist: yt-dlp
+Requires-Dist: cryptography
 Dynamic: license-file
 
 # scrapeMM: Multimodal Web Retrieval
@@ -26,11 +29,15 @@ This library aims to help developers and researchers to easily access multimodal
 ## Usage
 ```python
 from scrapemm import retrieve
+import asyncio
 
 url = "https://example.com"
-result = retrieve(url)
+loop = asyncio.get_event_loop()
+result = loop.run_until_complete(retrieve(url))
 result.render()
 ```
+`scrapeMM` will ask you for the **API keys** needed for the social media integrations. You may skip them if you don't need them. 
+You will also be prompted to choose a **password** that is used to secure the secrets in an encrypted file.
 
 ## How it works
 ```
@@ -44,7 +51,9 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
+- ✅ Bluesky
+- ✅ TikTok
+- ⏳ Threads
+- ⏳ Reddit
 - ⏳ Facebook
 - ⏳ Instagram
-- ⏳ Threads
-- ⏳ TikTok

{scrapemm-0.1.3 → scrapemm-0.2.2}/scrapeMM.egg-info/SOURCES.txt

@@ -9,18 +9,23 @@ scrapeMM.egg-info/dependency_links.txt
 scrapeMM.egg-info/requires.txt
 scrapeMM.egg-info/top_level.txt
 scrapemm/__init__.py
-scrapemm/api_keys.py
 scrapemm/common.py
 scrapemm/retrieval.py
+scrapemm/secrets.py
 scrapemm/util.py
 scrapemm/integrations/__init__.py
 scrapemm/integrations/base.py
+scrapemm/integrations/bluesky.py
+scrapemm/integrations/fb.py
+scrapemm/integrations/instagram.py
 scrapemm/integrations/telegram.py
+scrapemm/integrations/tiktok.py
 scrapemm/integrations/x.py
 scrapemm/scraping/__init__.py
 scrapemm/scraping/firecrawl.py
 scrapemm/scraping/no_bot_domains.txt
 scrapemm/scraping/util.py
+scrapemm/scraping/ytdlp.py
 scripts/example.py
 testing/test_retrieval.py
 testing/test_utils.py

{scrapemm-0.1.3 → scrapemm-0.2.2}/scrapeMM.egg-info/requires.txt

@@ -2,6 +2,9 @@ ezmm
 telethon
 tweepy
 markdownify
-keyring
 platformdirs
 PyYAML
+atproto
+TikTokResearchApi
+yt-dlp
+cryptography

scrapemm-0.2.2/scrapemm/__init__.py

@@ -0,0 +1,4 @@
+import scrapemm.common
+from .secrets import configure_secrets
+from .retrieval import retrieve
+from .integrations import Telegram, X

{scrapemm-0.1.3 → scrapemm-0.2.2}/scrapemm/common.py

@@ -18,6 +18,7 @@ if not logger.hasHandlers():
     formatter = logging.Formatter('[%(levelname)s]: %(message)s')
     handler.setFormatter(formatter)
     logger.addHandler(handler)
+    logger.propagate = False
 
 # Set up config directory
 CONFIG_DIR = Path(user_config_dir(APP_NAME))

{scrapemm-0.1.3 → scrapemm-0.2.2}/scrapemm/integrations/__init__.py

@@ -1,13 +1,22 @@
+import logging
+
+logger = logging.getLogger("scrapeMM")
+logger.debug("Starting integrations...")
+
 from typing import Optional
 
 import aiohttp
 from ezmm import MultimodalSequence
 
 from scrapemm.util import get_domain
+from .bluesky import Bluesky
+from .fb import Facebook
+from .instagram import Instagram
 from .telegram import Telegram
+from .tiktok import TikTok
 from .x import X
 
-RETRIEVAL_INTEGRATIONS = [X(), Telegram()]
+RETRIEVAL_INTEGRATIONS = [X(), Telegram(), Bluesky(), TikTok(), Instagram(), Facebook()]
 DOMAIN_TO_INTEGRATION = {domain: integration
                          for integration in RETRIEVAL_INTEGRATIONS
                          for domain in integration.domains

{scrapemm-0.1.3 → scrapemm-0.2.2}/scrapemm/integrations/base.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 import aiohttp
 from ezmm import MultimodalSequence
 
@@ -9,6 +11,6 @@ class RetrievalIntegration:
     domains: list[str]  # The domains supported by this integration
     connected: bool = False
 
-    async def get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence:
+    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
         """Retrieves the contents present at the given URL."""
         raise NotImplementedError

scrapemm-0.2.2/scrapemm/integrations/bluesky.py

@@ -0,0 +1,237 @@
+import logging
+from typing import Optional
+
+import aiohttp
+from atproto_client.exceptions import RequestErrorBase
+from atproto_client.models.common import XrpcError
+from ezmm import MultimodalSequence, download_image, download_video
+
+from scrapemm.secrets import get_secret
+from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.util import get_domain
+
+logger = logging.getLogger("scrapeMM")
+
+
+class Bluesky(RetrievalIntegration):
+    domains = ["bsky.app"]
+
+    def __init__(self):
+        self.username = get_secret("bluesky_username")
+        self.password = get_secret("bluesky_password")
+
+        if not (self.username and self.password):
+            logger.warning("❌ Bluesky integration not configured: Missing username or password.")
+            self.connected = False
+            return
+
+        from atproto import Client
+        self.client = Client()
+        self.authenticated = False
+        self._authenticate()
+
+    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        if not self.authenticated:
+            logger.error("❌ Not authenticated with Bluesky. Cannot retrieve content.")
+            return None
+
+        if get_domain(url) not in self.domains:
+            logger.error(f"❌ Invalid domain for Bluesky: {get_domain(url)}")
+            return None
+
+        if "post" in url:
+            result = await self._retrieve_post(url, session)
+        else:
+            result = await self._retrieve_profile(url, session)
+
+        return result
+
+    async def _retrieve_post(self, url: str, session: aiohttp.ClientSession = None) -> Optional[MultimodalSequence]:
+        """Retrieve a post from the given Bluesky URL."""
+        uri = self._construct_uri(url)
+        if not uri:
+            logger.error(f"❌ Could not construct URI for Bluesky post: {url}")
+            return None
+
+        try:
+            thread_response = self.client.get_post_thread(uri=uri, depth=0, parent_height=0)
+            thread = thread_response.thread
+
+            if hasattr(thread, 'py_type'):
+                thread_type = getattr(thread, 'py_type')
+                if thread_type == 'app.bsky.feed.defs#notFoundPost':
+                    raise Exception(f"Post not found for url {url}")
+                if thread_type == 'app.bsky.feed.defs#blockedPost':
+                    raise Exception(f"Post is blocked for url {url}")
+
+            # Extract post data
+            post_view = thread.post
+            record = post_view.record
+
+            # Basic post information
+            post_text = record.text if hasattr(record, 'text') else ''
+            created_at_str = record.created_at[:-1] if hasattr(record, 'created_at') else None
+
+            # Author information
+            author = post_view.author
+            author_username = author.handle if hasattr(author, 'handle') else ''
+            author_display_name = author.display_name if hasattr(author, 'display_name') else ''
+
+            # Engagement metrics
+            like_count = post_view.like_count if hasattr(post_view, 'like_count') else 0
+            comment_count = post_view.reply_count if hasattr(post_view, 'reply_count') else 0
+            share_count = post_view.repost_count if hasattr(post_view, 'repost_count') else 0
+
+            # Extract media (images)
+            media = []
+            # Check for embedded images in the post
+            if hasattr(post_view, 'embed'):
+                embed = post_view.embed
+
+                # For image embeds
+                if hasattr(embed, 'py_type') and getattr(embed, 'py_type') == 'app.bsky.embed.images#view':
+                    for img in embed.images:
+                        if hasattr(img, 'fullsize'):
+                            img_url = img.fullsize
+                            img = await download_image(img_url, session)
+                            media.append(img)
+                # For video embeds
+                elif hasattr(embed, 'py_type') and getattr(embed, 'py_type') == 'app.bsky.embed.video#view':
+                    video = await download_video(embed.playlist, session)
+                    if video:
+                        media.append(video)
+
+                        # Extract hashtags and mentions
+            hashtags, mentions, external_links = [], [], []
+            # Parse facets (rich text features like links, mentions, etc.)
+            if hasattr(record, 'facets') and record.facets:
+                for facet in record.facets:
+                    if hasattr(facet, 'features'):
+                        for feature in facet.features:
+                            if hasattr(feature, 'py_type'):
+                                feature_type = getattr(feature, 'py_type')
+                                if feature_type == 'app.bsky.richtext.facet#tag':
+                                    hashtags.append(feature.tag if hasattr(feature, 'tag') else '')
+                                elif feature_type == 'app.bsky.richtext.facet#mention':
+                                    mentions.append(feature.did if hasattr(feature, 'did') else '')
+                                elif feature_type == 'app.bsky.richtext.facet#link':
+                                    external_links.append(feature.uri)
+
+            # Check if this is a reply
+            is_reply, reply_to = False, None
+            if hasattr(record, 'reply'):
+                is_reply = True
+                # Get the parent post's author
+                if hasattr(record.reply, 'parent') and hasattr(record.reply.parent, 'uri'):
+                    parent_uri = record.reply.parent.uri
+                    post_id = parent_uri.split('/')[-1]
+                    reply_to_post = self.client.get_posts([parent_uri]).posts[0]
+                    reply_to_author = reply_to_post.author
+                    reply_to = f"https://bsky.app/profile/{reply_to_author.handle}/post/{post_id}"
+
+            text = f"""**Post on Bluesky**
+Author: {author_display_name} (@{author_username})
+Posted on: {created_at_str}
+Likes: {like_count} - Comments: {comment_count} - Shares: {share_count}
+{"Reply to: " + reply_to if is_reply and reply_to else ""}
+{post_text}"""
+            return MultimodalSequence([text, *media])
+
+        except Exception as e:
+            err_msg = error_to_string(e)
+            logger.error(f"❌ Error retrieving Bluesky post: {err_msg}")
+            return None
+
+    async def _retrieve_profile(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        """Retrieve a profile from the given Bluesky URL."""
+        profile = self.client.get_profile(url.split('/')[-1])
+
+        avatar = await download_image(profile.avatar, session) if profile.avatar else None
+        banner = await download_image(profile.banner, session) if profile.banner else None
+
+        text = f"""**Profile on Bluesky**
+User: {profile.display_name} (@{profile.handle})
+Created on: {profile.created_at}
+Profile image: {avatar.reference if avatar else 'None'}
+Profile banner: {banner.reference if banner else 'None'}
+
+URL: {url}
+Description: {profile.description or 'No description provided'}
+
+Metrics:
+- Follower count: {profile.followers_count}
+- Following count: {profile.follows_count}
+- Post count: {profile.posts_count}
+            """
+        return MultimodalSequence(text)
+
+    def _authenticate(self) -> bool:
+        """Authenticate with Bluesky using provided credentials."""
+        try:
+            self.client.login(self.username, self.password)
+            self.authenticated = True
+            logger.info(f"✅ Successfully authenticated with Bluesky as {self.username}")
+            return True
+        except Exception as e:
+            logger.error(f"❌ Error authenticating with Bluesky: {str(e)}")
+            return False
+
+    def _construct_uri(self, url: str) -> str:
+        # Extract post URI from the URL - Bluesky URLs typically look like:
+        # https://bsky.app/profile/username.bsky.social/post/abcdef123
+        try:
+            # Parse URL to extract components for building the AT URI
+            parts = url.split('/')
+            if len(parts) < 5 or "bsky.app" not in url:
+                raise Exception(f"Invalid Bluesky URL format for {url}.")
+
+            # Find the profile part of the URL
+            profile_idx = -1
+            for i, part in enumerate(parts):
+                if part == "profile":
+                    profile_idx = i
+                    break
+
+            if profile_idx < 0 or profile_idx + 3 >= len(parts):
+                raise Exception(f"Could not extract profile or post ID from {url}.")
+
+            handle = parts[profile_idx + 1]
+            post_id = parts[profile_idx + 3]
+
+            # Resolve the handle to a DID
+            did = self._resolve_handle(handle)
+
+            # Construct the AT URI
+            uri = f"at://{did}/app.bsky.feed.post/{post_id}"
+
+            return uri
+
+        except Exception as e:
+            err_msg = error_to_string(e)
+            logger.error(f"Error retrieving Bluesky post: {err_msg}")
+
+    def _resolve_handle(self, handle: str) -> str:
+        """Resolve a handle to a DID."""
+        try:
+            response = self.client.resolve_handle(handle)
+            return response.did
+        except Exception as e:
+            err_msg = error_to_string(e)
+            logger.error(f"Error resolving handle: {err_msg}")
+            return handle  # Return the handle itself as fallback
+
+
+def error_to_string(error: RequestErrorBase | Exception) -> str:
+    """Takes an Error object containing a response and prints the contents."""
+    if isinstance(error, RequestErrorBase):
+        response = error.response
+        code = response.status_code
+        content = response.content
+        if isinstance(content, XrpcError):
+            error_type = content.error
+            msg = content.message
+            return f"Error {code} ({error_type}): {msg}."
+        else:
+            return f"Error {code}: {content}."
+    else:
+        return str(error)

scrapemm-0.2.2/scrapemm/integrations/fb.py

@@ -0,0 +1,101 @@
+import logging
+from urllib.parse import urlparse
+import aiohttp
+from ezmm import MultimodalSequence, download_image
+
+from scrapemm.scraping.ytdlp import check_ytdlp_available,  get_video_with_ytdlp
+from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.util import get_domain
+
+logger = logging.getLogger("scrapeMM")
+
+
+class Facebook(RetrievalIntegration):
+
+    domains = ["facebook.com", "www.facebook.com"]
+
+    def __init__(self):
+        # Check if yt-dlp is available
+        self.ytdlp_available = check_ytdlp_available()
+
+        if self.ytdlp_available:
+            self.connected = True
+            mode = "yt-dlp only"
+            logger.info(f"✅ Facebook integration ready ({mode} mode).")
+        else:
+            self.connected = False
+            logger.warning("❌ Facebook integration not available: Neither API credentials nor yt-dlp found.")
+
+    async def get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook post URL."""
+        if not self.connected:
+            logger.error("❌ Facebook integration not connected. Cannot retrieve content.")
+            return None
+            
+        if get_domain(url) not in self.domains:
+            logger.error(f"❌ Invalid domain for Facebook: {get_domain(url)}")
+            return None
+        
+        # Determine if this is a video or profile URL
+        if self._is_video_url(url):
+            return await self._get_video(url, session)
+        elif self._is_photo_url(url):
+            return await self._get_photo(url, session)
+        else:
+            return await self._get_user_profile(url, session)
+        
+    async def _get_video(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a TikTok video URL."""
+
+        # Fallback to yt-dlp mode
+        if self.ytdlp_available:
+            return await get_video_with_ytdlp(url, session, platform="Facebook")
+
+        logger.error("❌ No available method to retrieve Facebook video.")
+        return None
+    
+    async def _get_photo(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook photo URL."""
+        logger.error("❌ No available method to retrieve Facebook photo.")
+        return None
+
+    async def _get_user_profile(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence | None:
+        """Retrieves content from a Facebook user profile URL."""
+        username = self._extract_username(url)
+        if username:
+            text = f"""**Facebook Profile**
+Username: @{username}
+URL: {url}
+
+Note: Profile details require Facebook API access.
+Configure API credentials for full profile information."""
+            return MultimodalSequence([text])
+        
+        return None
+
+    def _is_video_url(self, url: str) -> bool:
+        """Checks if the URL is a Facebook video URL."""
+        # video URLS are in the format: https://www.facebook.com/watch?v=VIDEO_ID
+        return "facebook.com/watch" in url
+    
+    def _extract_video_id(self, url: str) -> str:
+        """Extracts the video ID from a Facebook video URL."""
+        parsed_url = urlparse(url)
+        query_params = parsed_url.query
+        for param in query_params.split('&'):
+            if param.startswith('v='):
+                return param.split('=')[1]
+        return ""
+    
+    def _is_photo_url(self, url: str) -> bool:
+        """Checks if the URL is a Facebook photo URL."""
+        return "facebook.com/photo" in url or "facebook.com/photos" in url
+    
+    def _extract_username(self, url: str) -> str:
+        """Extracts the username from a Facebook profile URL."""
+        # url format: https://www.facebook.com/username<?...>
+        parsed_url = urlparse(url)
+        path_parts = parsed_url.path.strip('/').split('/')
+        if len(path_parts) > 0:
+            return path_parts[0]
+        return ""