scrapeMM 0.1.2__tar.gz → 0.2.1__tar.gz

scrapemm-0.2.1/.gitignore

@@ -0,0 +1,6 @@
+/temp/
+/.idea/
+/scrapeMM.egg-info/
+/dist/
+/config.yaml
+__pycache__/

{scrapemm-0.1.2/scrapeMM.egg-info → scrapemm-0.2.1}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.1.2
+Version: 0.2.1
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://github.com/multimodal-ai-lab/scrapeMM
-Project-URL: Bug Tracker, https://github.com/multimodal-ai-lab/scrapeMM
+Project-URL: Issues, https://github.com/multimodal-ai-lab/scrapeMM
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -13,7 +13,12 @@ Requires-Dist: ezmm
 Requires-Dist: telethon
 Requires-Dist: tweepy
 Requires-Dist: markdownify
-Requires-Dist: keyring
+Requires-Dist: platformdirs
+Requires-Dist: PyYAML
+Requires-Dist: atproto
+Requires-Dist: TikTokResearchApi
+Requires-Dist: yt-dlp
+Requires-Dist: cryptography
 Dynamic: license-file
 
 # scrapeMM: Multimodal Web Retrieval
@@ -24,11 +29,15 @@ This library aims to help developers and researchers to easily access multimodal
 ## Usage
 ```python
 from scrapemm import retrieve
+import asyncio
 
 url = "https://example.com"
-result = retrieve(url)
+loop = asyncio.get_event_loop()
+result = loop.run_until_complete(retrieve(url))
 result.render()
 ```
+`scrapeMM` will ask you for the **API keys** needed for the social media integrations. You may skip them if you don't need them. 
+You will also be prompted to choose a **password** that is used to secure the secrets in an encrypted file.
 
 ## How it works
 ```
@@ -42,7 +51,9 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
+- ✅ Bluesky
+- ✅ TikTok
+- ⏳ Threads
+- ⏳ Reddit
 - ⏳ Facebook
 - ⏳ Instagram
-- ⏳ Threads
-- ⏳ TikTok

{scrapemm-0.1.2 → scrapemm-0.2.1}/README.md

@@ -6,11 +6,15 @@ This library aims to help developers and researchers to easily access multimodal
 ## Usage
 ```python
 from scrapemm import retrieve
+import asyncio
 
 url = "https://example.com"
-result = retrieve(url)
+loop = asyncio.get_event_loop()
+result = loop.run_until_complete(retrieve(url))
 result.render()
 ```
+`scrapeMM` will ask you for the **API keys** needed for the social media integrations. You may skip them if you don't need them. 
+You will also be prompted to choose a **password** that is used to secure the secrets in an encrypted file.
 
 ## How it works
 ```
@@ -24,7 +28,9 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
+- ✅ Bluesky
+- ✅ TikTok
+- ⏳ Threads
+- ⏳ Reddit
 - ⏳ Facebook
 - ⏳ Instagram
-- ⏳ Threads
-- ⏳ TikTok

scrapemm-0.2.1/pyproject.toml

@@ -0,0 +1,36 @@
+[project]
+name = "scrapeMM"
+version = "0.2.1"
+description = "LLM-friendly scraper for media and text from social media and the open web."
+readme = "README.md"
+requires-python = ">=3.8"
+license = "Apache-2.0"
+authors = [{name = "Mark Rothermel", email = "mark.rothermel@tu-darmstadt.de"}]
+dependencies = [
+    "ezmm",
+    "telethon",
+    "tweepy",
+    "markdownify",
+    "platformdirs",
+    "PyYAML",
+    "atproto",
+    "TikTokResearchApi",
+    "yt-dlp",
+    "cryptography",
+]
+
+[project.urls]
+Homepage = "https://github.com/multimodal-ai-lab/scrapeMM"
+Issues = "https://github.com/multimodal-ai-lab/scrapeMM"
+
+[build-system]
+requires = ["setuptools>=60", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+namespaces = false  # Disable implicit namespaces
+# include = ["scrapemm/*"]
+# exclude = ["*.egg-info", "tests*", "in*", "scripts*", "temp*", "secrets.yaml", "dist*"]
+
+[tool.setuptools.package-data]
+"*" = ["*.txt"]  # Include all .txt files contained in any (sub)package in the build

scrapemm-0.2.1/requirements.txt

@@ -0,0 +1,18 @@
+pytest~=8.4.1
+aiohttp~=3.12.13
+numpy~=1.26.4
+requests~=2.32.4
+ezMM~=0.3.1
+Telethon~=1.40.0
+PyYAML~=6.0.1
+tweepy~=4.16.0
+markdownify~=1.1.0
+tqdm~=4.66.3
+pytest-asyncio
+async_lru
+oauthlib
+platformdirs~=4.3.8
+atproto
+TikTokResearchApi
+yt-dlp
+cryptography

{scrapemm-0.1.2 → scrapemm-0.2.1/scrapeMM.egg-info}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: scrapeMM
-Version: 0.1.2
+Version: 0.2.1
 Summary: LLM-friendly scraper for media and text from social media and the open web.
 Author-email: Mark Rothermel <mark.rothermel@tu-darmstadt.de>
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://github.com/multimodal-ai-lab/scrapeMM
-Project-URL: Bug Tracker, https://github.com/multimodal-ai-lab/scrapeMM
+Project-URL: Issues, https://github.com/multimodal-ai-lab/scrapeMM
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -13,7 +13,12 @@ Requires-Dist: ezmm
 Requires-Dist: telethon
 Requires-Dist: tweepy
 Requires-Dist: markdownify
-Requires-Dist: keyring
+Requires-Dist: platformdirs
+Requires-Dist: PyYAML
+Requires-Dist: atproto
+Requires-Dist: TikTokResearchApi
+Requires-Dist: yt-dlp
+Requires-Dist: cryptography
 Dynamic: license-file
 
 # scrapeMM: Multimodal Web Retrieval
@@ -24,11 +29,15 @@ This library aims to help developers and researchers to easily access multimodal
 ## Usage
 ```python
 from scrapemm import retrieve
+import asyncio
 
 url = "https://example.com"
-result = retrieve(url)
+loop = asyncio.get_event_loop()
+result = loop.run_until_complete(retrieve(url))
 result.render()
 ```
+`scrapeMM` will ask you for the **API keys** needed for the social media integrations. You may skip them if you don't need them. 
+You will also be prompted to choose a **password** that is used to secure the secrets in an encrypted file.
 
 ## How it works
 ```
@@ -42,7 +51,9 @@ Web scraping is done with [Firecrawl](https://github.com/mendableai/firecrawl).
 ## Supported Proprietary APIs
 - ✅ X/Twitter
 - ✅ Telegram
+- ✅ Bluesky
+- ✅ TikTok
+- ⏳ Threads
+- ⏳ Reddit
 - ⏳ Facebook
 - ⏳ Instagram
-- ⏳ Threads
-- ⏳ TikTok

{scrapemm-0.1.2 → scrapemm-0.2.1}/scrapeMM.egg-info/SOURCES.txt

@@ -1,20 +1,28 @@
+.gitignore
 LICENSE
 README.md
 pyproject.toml
+requirements.txt
 scrapeMM.egg-info/PKG-INFO
 scrapeMM.egg-info/SOURCES.txt
 scrapeMM.egg-info/dependency_links.txt
 scrapeMM.egg-info/requires.txt
 scrapeMM.egg-info/top_level.txt
 scrapemm/__init__.py
-scrapemm/api_keys.py
 scrapemm/common.py
 scrapemm/retrieval.py
+scrapemm/secrets.py
 scrapemm/util.py
 scrapemm/integrations/__init__.py
 scrapemm/integrations/base.py
+scrapemm/integrations/bluesky.py
 scrapemm/integrations/telegram.py
+scrapemm/integrations/tiktok.py
 scrapemm/integrations/x.py
 scrapemm/scraping/__init__.py
 scrapemm/scraping/firecrawl.py
-scrapemm/scraping/util.py
+scrapemm/scraping/no_bot_domains.txt
+scrapemm/scraping/util.py
+scripts/example.py
+testing/test_retrieval.py
+testing/test_utils.py

scrapemm-0.2.1/scrapeMM.egg-info/requires.txt

@@ -0,0 +1,10 @@
+ezmm
+telethon
+tweepy
+markdownify
+platformdirs
+PyYAML
+atproto
+TikTokResearchApi
+yt-dlp
+cryptography

scrapemm-0.2.1/scrapemm/__init__.py

@@ -0,0 +1,4 @@
+import scrapemm.common
+from .secrets import configure_secrets
+from .retrieval import retrieve
+from .integrations import Telegram, X

scrapemm-0.2.1/scrapemm/common.py

@@ -0,0 +1,47 @@
+import logging
+import os
+import sys
+from pathlib import Path
+from platformdirs import user_config_dir
+
+import yaml
+
+APP_NAME = "scrapeMM"
+
+# Set up logger
+logger = logging.getLogger(APP_NAME)
+logger.setLevel(logging.DEBUG)
+
+# Only add handler if none exists (avoid duplicate logs on rerun)
+if not logger.hasHandlers():
+    handler = logging.StreamHandler(sys.stdout)
+    formatter = logging.Formatter('[%(levelname)s]: %(message)s')
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    logger.propagate = False
+
+# Set up config directory
+CONFIG_DIR = Path(user_config_dir(APP_NAME))
+os.makedirs(CONFIG_DIR, exist_ok=True)
+CONFIG_PATH = CONFIG_DIR / "config.yaml"
+
+
+def load_config() -> dict:
+    if os.path.exists(CONFIG_PATH):
+        with open(CONFIG_PATH, "r") as f:
+            return yaml.safe_load(f) or {}
+    else:
+        return {}
+
+
+def update_config(**kwargs):
+    _config.update(kwargs)
+    yaml.dump(_config, open(CONFIG_PATH, "w"))
+
+
+def get_config_var(name: str, default=None) -> str:
+    return _config.get(name, default)
+
+
+# Load config
+_config = load_config()

{scrapemm-0.1.2 → scrapemm-0.2.1}/scrapemm/integrations/__init__.py

@@ -1,13 +1,20 @@
+import logging
+
+logger = logging.getLogger("scrapeMM")
+logger.debug("Starting integrations...")
+
 from typing import Optional
 
 import aiohttp
 from ezmm import MultimodalSequence
 
 from scrapemm.util import get_domain
+from .bluesky import Bluesky
 from .telegram import Telegram
+from .tiktok import TikTok
 from .x import X
 
-RETRIEVAL_INTEGRATIONS = [X(), Telegram()]
+RETRIEVAL_INTEGRATIONS = [X(), Telegram(), Bluesky(), TikTok()]
 DOMAIN_TO_INTEGRATION = {domain: integration
                          for integration in RETRIEVAL_INTEGRATIONS
                          for domain in integration.domains

{scrapemm-0.1.2 → scrapemm-0.2.1}/scrapemm/integrations/base.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 import aiohttp
 from ezmm import MultimodalSequence
 
@@ -9,6 +11,6 @@ class RetrievalIntegration:
     domains: list[str]  # The domains supported by this integration
     connected: bool = False
 
-    async def get(self, url: str, session: aiohttp.ClientSession) -> MultimodalSequence:
+    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
         """Retrieves the contents present at the given URL."""
         raise NotImplementedError

scrapemm-0.2.1/scrapemm/integrations/bluesky.py

@@ -0,0 +1,237 @@
+import logging
+from typing import Optional
+
+import aiohttp
+from atproto_client.exceptions import RequestErrorBase
+from atproto_client.models.common import XrpcError
+from ezmm import MultimodalSequence, download_image, download_video
+
+from scrapemm.secrets import get_secret
+from scrapemm.integrations.base import RetrievalIntegration
+from scrapemm.util import get_domain
+
+logger = logging.getLogger("scrapeMM")
+
+
+class Bluesky(RetrievalIntegration):
+    domains = ["bsky.app"]
+
+    def __init__(self):
+        self.username = get_secret("bluesky_username")
+        self.password = get_secret("bluesky_password")
+
+        if not (self.username and self.password):
+            logger.warning("❌ Bluesky integration not configured: Missing username or password.")
+            self.connected = False
+            return
+
+        from atproto import Client
+        self.client = Client()
+        self.authenticated = False
+        self._authenticate()
+
+    async def get(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        if not self.authenticated:
+            logger.error("❌ Not authenticated with Bluesky. Cannot retrieve content.")
+            return None
+
+        if get_domain(url) not in self.domains:
+            logger.error(f"❌ Invalid domain for Bluesky: {get_domain(url)}")
+            return None
+
+        if "post" in url:
+            result = await self._retrieve_post(url, session)
+        else:
+            result = await self._retrieve_profile(url, session)
+
+        return result
+
+    async def _retrieve_post(self, url: str, session: aiohttp.ClientSession = None) -> Optional[MultimodalSequence]:
+        """Retrieve a post from the given Bluesky URL."""
+        uri = self._construct_uri(url)
+        if not uri:
+            logger.error(f"❌ Could not construct URI for Bluesky post: {url}")
+            return None
+
+        try:
+            thread_response = self.client.get_post_thread(uri=uri, depth=0, parent_height=0)
+            thread = thread_response.thread
+
+            if hasattr(thread, 'py_type'):
+                thread_type = getattr(thread, 'py_type')
+                if thread_type == 'app.bsky.feed.defs#notFoundPost':
+                    raise Exception(f"Post not found for url {url}")
+                if thread_type == 'app.bsky.feed.defs#blockedPost':
+                    raise Exception(f"Post is blocked for url {url}")
+
+            # Extract post data
+            post_view = thread.post
+            record = post_view.record
+
+            # Basic post information
+            post_text = record.text if hasattr(record, 'text') else ''
+            created_at_str = record.created_at[:-1] if hasattr(record, 'created_at') else None
+
+            # Author information
+            author = post_view.author
+            author_username = author.handle if hasattr(author, 'handle') else ''
+            author_display_name = author.display_name if hasattr(author, 'display_name') else ''
+
+            # Engagement metrics
+            like_count = post_view.like_count if hasattr(post_view, 'like_count') else 0
+            comment_count = post_view.reply_count if hasattr(post_view, 'reply_count') else 0
+            share_count = post_view.repost_count if hasattr(post_view, 'repost_count') else 0
+
+            # Extract media (images)
+            media = []
+            # Check for embedded images in the post
+            if hasattr(post_view, 'embed'):
+                embed = post_view.embed
+
+                # For image embeds
+                if hasattr(embed, 'py_type') and getattr(embed, 'py_type') == 'app.bsky.embed.images#view':
+                    for img in embed.images:
+                        if hasattr(img, 'fullsize'):
+                            img_url = img.fullsize
+                            img = await download_image(img_url, session)
+                            media.append(img)
+                # For video embeds
+                elif hasattr(embed, 'py_type') and getattr(embed, 'py_type') == 'app.bsky.embed.video#view':
+                    video = await download_video(embed.playlist, session)
+                    if video:
+                        media.append(video)
+
+                        # Extract hashtags and mentions
+            hashtags, mentions, external_links = [], [], []
+            # Parse facets (rich text features like links, mentions, etc.)
+            if hasattr(record, 'facets') and record.facets:
+                for facet in record.facets:
+                    if hasattr(facet, 'features'):
+                        for feature in facet.features:
+                            if hasattr(feature, 'py_type'):
+                                feature_type = getattr(feature, 'py_type')
+                                if feature_type == 'app.bsky.richtext.facet#tag':
+                                    hashtags.append(feature.tag if hasattr(feature, 'tag') else '')
+                                elif feature_type == 'app.bsky.richtext.facet#mention':
+                                    mentions.append(feature.did if hasattr(feature, 'did') else '')
+                                elif feature_type == 'app.bsky.richtext.facet#link':
+                                    external_links.append(feature.uri)
+
+            # Check if this is a reply
+            is_reply, reply_to = False, None
+            if hasattr(record, 'reply'):
+                is_reply = True
+                # Get the parent post's author
+                if hasattr(record.reply, 'parent') and hasattr(record.reply.parent, 'uri'):
+                    parent_uri = record.reply.parent.uri
+                    post_id = parent_uri.split('/')[-1]
+                    reply_to_post = self.client.get_posts([parent_uri]).posts[0]
+                    reply_to_author = reply_to_post.author
+                    reply_to = f"https://bsky.app/profile/{reply_to_author.handle}/post/{post_id}"
+
+            text = f"""**Post on Bluesky**
+Author: {author_display_name} (@{author_username})
+Posted on: {created_at_str}
+Likes: {like_count} - Comments: {comment_count} - Shares: {share_count}
+{"Reply to: " + reply_to if is_reply and reply_to else ""}
+{post_text}"""
+            return MultimodalSequence([text, *media])
+
+        except Exception as e:
+            err_msg = error_to_string(e)
+            logger.error(f"❌ Error retrieving Bluesky post: {err_msg}")
+            return None
+
+    async def _retrieve_profile(self, url: str, session: aiohttp.ClientSession) -> Optional[MultimodalSequence]:
+        """Retrieve a profile from the given Bluesky URL."""
+        profile = self.client.get_profile(url.split('/')[-1])
+
+        avatar = await download_image(profile.avatar, session) if profile.avatar else None
+        banner = await download_image(profile.banner, session) if profile.banner else None
+
+        text = f"""**Profile on Bluesky**
+User: {profile.display_name} (@{profile.handle})
+Created on: {profile.created_at}
+Profile image: {avatar.reference if avatar else 'None'}
+Profile banner: {banner.reference if banner else 'None'}
+
+URL: {url}
+Description: {profile.description or 'No description provided'}
+
+Metrics:
+- Follower count: {profile.followers_count}
+- Following count: {profile.follows_count}
+- Post count: {profile.posts_count}
+            """
+        return MultimodalSequence(text)
+
+    def _authenticate(self) -> bool:
+        """Authenticate with Bluesky using provided credentials."""
+        try:
+            self.client.login(self.username, self.password)
+            self.authenticated = True
+            logger.info(f"✅ Successfully authenticated with Bluesky as {self.username}")
+            return True
+        except Exception as e:
+            logger.error(f"❌ Error authenticating with Bluesky: {str(e)}")
+            return False
+
+    def _construct_uri(self, url: str) -> str:
+        # Extract post URI from the URL - Bluesky URLs typically look like:
+        # https://bsky.app/profile/username.bsky.social/post/abcdef123
+        try:
+            # Parse URL to extract components for building the AT URI
+            parts = url.split('/')
+            if len(parts) < 5 or "bsky.app" not in url:
+                raise Exception(f"Invalid Bluesky URL format for {url}.")
+
+            # Find the profile part of the URL
+            profile_idx = -1
+            for i, part in enumerate(parts):
+                if part == "profile":
+                    profile_idx = i
+                    break
+
+            if profile_idx < 0 or profile_idx + 3 >= len(parts):
+                raise Exception(f"Could not extract profile or post ID from {url}.")
+
+            handle = parts[profile_idx + 1]
+            post_id = parts[profile_idx + 3]
+
+            # Resolve the handle to a DID
+            did = self._resolve_handle(handle)
+
+            # Construct the AT URI
+            uri = f"at://{did}/app.bsky.feed.post/{post_id}"
+
+            return uri
+
+        except Exception as e:
+            err_msg = error_to_string(e)
+            logger.error(f"Error retrieving Bluesky post: {err_msg}")
+
+    def _resolve_handle(self, handle: str) -> str:
+        """Resolve a handle to a DID."""
+        try:
+            response = self.client.resolve_handle(handle)
+            return response.did
+        except Exception as e:
+            err_msg = error_to_string(e)
+            logger.error(f"Error resolving handle: {err_msg}")
+            return handle  # Return the handle itself as fallback
+
+
+def error_to_string(error: RequestErrorBase | Exception) -> str:
+    """Takes an Error object containing a response and prints the contents."""
+    if isinstance(error, RequestErrorBase):
+        response = error.response
+        code = response.status_code
+        content = response.content
+        if isinstance(content, XrpcError):
+            error_type = content.error
+            msg = content.message
+            return f"Error {code} ({error_type}): {msg}."
+        else:
+            return f"Error {code}: {content}."
+    else:
+        return str(error)

{scrapemm-0.1.2 → scrapemm-0.2.1}/scrapemm/integrations/telegram.py

@@ -7,11 +7,11 @@ from ezmm import MultimodalSequence, Image, Item, Video
 from telethon import TelegramClient
 from telethon.tl.types import Channel, User
 
-from scrapemm.api_keys import get_api_key
+from scrapemm.secrets import get_secret
 from scrapemm.integrations.base import RetrievalIntegration
 from scrapemm.util import get_domain
 
-logger = logging.getLogger("Retriever")
+logger = logging.getLogger("scrapeMM")
 
 
 class Telegram(RetrievalIntegration):
@@ -21,9 +21,9 @@ class Telegram(RetrievalIntegration):
     session_path = "temp/telegram"
 
     def __init__(self):
-        api_id = int(get_api_key("telegram_api_id"))
-        api_hash = get_api_key("telegram_api_hash")
-        bot_token = get_api_key("telegram_bot_token")
+        api_id = int(get_secret("telegram_api_id")) if get_secret("telegram_api_id") else None
+        api_hash = get_secret("telegram_api_hash")
+        bot_token = get_secret("telegram_bot_token")
 
         if api_id and api_hash and bot_token:
             self.client = TelegramClient(self.session_path, api_id, api_hash)