pulse-engine 0.2.0__py3-none-any.whl

pulse_engine/adapters/twitter.py

@@ -0,0 +1,423 @@
+"""Twitter/X timeline adapter using twikit."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import re
+import tempfile
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from .models import TweetMetadata, TwitterUserResult
+
+logger = structlog.get_logger(__name__)
+
+
+def _patch_twikit_user() -> None:
+    """Override twikit.user.User.__init__ with safe .get() access.
+
+    twikit 2.3.3 uses hard bracket notation for optional API fields that
+    Twitter omits for some accounts (e.g. description.urls,
+    withheld_in_countries). This replaces the constructor in-process without
+    touching site-packages. Incorporates fixes from PR #418
+    (github.com/d60/twikit/pull/418).
+    """
+    try:
+        import twikit.user as _twikit_user
+    except ImportError:
+        return
+
+    def _safe_init(self: object, client: object, data: dict[str, Any]) -> None:
+        self._client = client  # type: ignore[attr-defined]
+        legacy = data.get("legacy", {})
+
+        self.id = data.get("rest_id", "")  # type: ignore[attr-defined]
+        self.created_at = legacy.get("created_at", "")  # type: ignore[attr-defined]
+        self.name = legacy.get("name", "")  # type: ignore[attr-defined]
+        self.screen_name = legacy.get("screen_name", "")  # type: ignore[attr-defined]
+        self.profile_image_url = legacy.get("profile_image_url_https", "")  # type: ignore[attr-defined]
+        self.profile_banner_url = legacy.get("profile_banner_url")  # type: ignore[attr-defined]
+        self.url = legacy.get("url")  # type: ignore[attr-defined]
+        self.location = legacy.get("location", "")  # type: ignore[attr-defined]
+        self.description = legacy.get("description", "")  # type: ignore[attr-defined]
+        self.description_urls = (  # type: ignore[attr-defined]
+            legacy.get("entities", {}).get("description", {}).get("urls", [])
+        )
+        self.urls = legacy.get("entities", {}).get("url", {}).get("urls")  # type: ignore[attr-defined]
+        self.pinned_tweet_ids = legacy.get("pinned_tweet_ids_str", [])  # type: ignore[attr-defined]
+        self.is_blue_verified = data.get("is_blue_verified", False)  # type: ignore[attr-defined]
+        self.verified = legacy.get("verified", False)  # type: ignore[attr-defined]
+        self.possibly_sensitive = legacy.get("possibly_sensitive", False)  # type: ignore[attr-defined]
+        self.can_dm = legacy.get("can_dm", False)  # type: ignore[attr-defined]
+        self.can_media_tag = legacy.get("can_media_tag", False)  # type: ignore[attr-defined]
+        self.want_retweets = legacy.get("want_retweets", False)  # type: ignore[attr-defined]
+        self.default_profile = legacy.get("default_profile", False)  # type: ignore[attr-defined]
+        self.default_profile_image = legacy.get("default_profile_image", False)  # type: ignore[attr-defined]
+        self.has_custom_timelines = legacy.get("has_custom_timelines", False)  # type: ignore[attr-defined]
+        self.followers_count = legacy.get("followers_count", 0)  # type: ignore[attr-defined]
+        self.fast_followers_count = legacy.get("fast_followers_count", 0)  # type: ignore[attr-defined]
+        self.normal_followers_count = legacy.get("normal_followers_count", 0)  # type: ignore[attr-defined]
+        self.following_count = legacy.get("friends_count", 0)  # type: ignore[attr-defined]
+        self.favourites_count = legacy.get("favourites_count", 0)  # type: ignore[attr-defined]
+        self.listed_count = legacy.get("listed_count", 0)  # type: ignore[attr-defined]
+        self.media_count = legacy.get("media_count", 0)  # type: ignore[attr-defined]
+        self.statuses_count = legacy.get("statuses_count", 0)  # type: ignore[attr-defined]
+        self.is_translator = legacy.get("is_translator", False)  # type: ignore[attr-defined]
+        self.translator_type = legacy.get("translator_type", "")  # type: ignore[attr-defined]
+        self.withheld_in_countries = legacy.get("withheld_in_countries", [])  # type: ignore[attr-defined]
+        self.protected = legacy.get("protected", False)  # type: ignore[attr-defined]
+
+    _twikit_user.User.__init__ = _safe_init
+    logger.debug("twikit_user_patched")
+
+
+_patch_twikit_user()
+
+_DEVANAGARI_RE = re.compile(r"[\u0900-\u097F]")
+_HASHTAG_RE = re.compile(r"#\w+")
+_URL_RE = re.compile(r"https?://[^\s]+")
+
+_TWEET_DATETIME_FORMATS = (
+    "%a %b %d %H:%M:%S +0000 %Y",
+    "%Y-%m-%dT%H:%M:%S+00:00",
+)
+
+
+def _detect_language(text: str) -> str:
+    deva = len(_DEVANAGARI_RE.findall(text))
+    total = len(text.replace(" ", ""))
+    if not total:
+        return "en"
+    ratio = deva / total
+    if ratio > 0.4:
+        return "hi"
+    if ratio > 0.1:
+        return "mixed"
+    return "en"
+
+
+def _parse_datetime(value: object) -> datetime:
+    if isinstance(value, datetime):
+        return value.replace(tzinfo=UTC) if value.tzinfo is None else value
+    s = str(value)
+    for fmt in _TWEET_DATETIME_FORMATS:
+        try:
+            return datetime.strptime(s, fmt).replace(tzinfo=UTC)
+        except ValueError:
+            continue
+    try:
+        dt = datetime.fromisoformat(s)
+        return dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt
+    except ValueError:
+        return datetime.now(UTC)
+
+
+def _to_model(tweet: object, screen_name: str) -> TweetMetadata | None:
+    """Convert a twikit tweet object to TweetMetadata. Returns None to skip."""
+    if getattr(tweet, "retweeted_tweet", None) is not None:
+        return None
+    if getattr(tweet, "in_reply_to", None) is not None:
+        return None
+
+    text: str = getattr(tweet, "text", "") or ""
+    note = getattr(tweet, "note_tweet", None)
+    if note:
+        text = getattr(note, "text", None) or text
+
+    if text.startswith("RT @"):
+        return None
+
+    has_images = has_videos = False
+    for item in getattr(tweet, "media", None) or []:
+        t = type(item).__name__.lower()
+        has_images = has_images or "photo" in t or "image" in t
+        has_videos = has_videos or "video" in t
+
+    views = 0
+    raw_views = getattr(tweet, "view_count", None)
+    if raw_views:
+        try:
+            views = int(str(raw_views).replace(",", ""))
+        except (ValueError, TypeError):
+            pass
+
+    tweet_id = str(getattr(tweet, "id", ""))
+    created_at = _parse_datetime(getattr(tweet, "created_at", datetime.now(UTC)))
+
+    return TweetMetadata(
+        tweet_id=tweet_id,
+        url=f"https://x.com/{screen_name}/status/{tweet_id}",
+        text=text,
+        screen_name=screen_name,
+        created_at=created_at,
+        language=_detect_language(text),
+        hashtags=_HASHTAG_RE.findall(text),
+        embedded_urls=_URL_RE.findall(text),
+        views=views,
+        retweet_count=getattr(tweet, "retweet_count", 0) or 0,
+        like_count=getattr(tweet, "favorite_count", 0) or 0,
+        reply_count=getattr(tweet, "reply_count", 0) or 0,
+        has_images=has_images,
+        has_videos=has_videos,
+    )
+
+
+class TwitterAdapter:
+    """Fetches tweets from one or more Twitter/X user timelines via twikit.
+
+    Mirrors the ``YouTubeMetadataAdapter`` interface: construct once with
+    credentials, call ``fetch()`` for a single user or ``fetch_many()`` for
+    a batch of users.
+
+    Cookies are loaded from AWS Secrets Manager (JSON string) or a local file:
+
+    Environment variables:
+        - ``TWITTER_COOKIES_SECRET_ID`` — Secrets Manager secret ID (JSON cookies)
+        - ``TWITTER_COOKIES_PATH``      — path to a local cookies.json file
+        - ``AWS_REGION``               — region for Secrets Manager (default: us-east-1)
+
+    Secrets Manager takes precedence over the local file.
+
+    Usage::
+
+        adapter = TwitterAdapter()
+        tweets = await adapter.fetch("srivatsayb", since=timedelta(days=1))
+
+        results = await adapter.fetch_many(
+            ["srivatsayb", "rahulgandhiinc"],
+            since=timedelta(days=7),
+        )
+    """
+
+    def __init__(self) -> None:
+        self._cookies_secret_id: str = os.environ.get(
+            "TWITTER_COOKIES_SECRET_ID", ""
+        ).strip()
+        self._cookies_path: str = os.environ.get("TWITTER_COOKIES_PATH", "").strip()
+        self._aws_region: str = os.environ.get("AWS_REGION", "us-east-1").strip()
+        self._cookies_cache: str | None = None  # lazily populated
+
+    async def fetch(
+        self,
+        screen_name: str,
+        since: timedelta | None = None,
+        max_tweets: int = 200,
+    ) -> list[TweetMetadata]:
+        """Fetch tweets from a single user's timeline.
+
+        Args:
+            screen_name: Twitter/X handle without @.
+            since: Only return tweets published within this window.
+            max_tweets: Upper bound on tweets returned (default: 200).
+
+        Returns:
+            List of ``TweetMetadata``, newest first.
+        """
+        cutoff = (datetime.now(UTC) - since) if since is not None else None
+        logger.info(
+            "twitter_fetch_started",
+            screen_name=screen_name,
+            since=str(since),
+            max_tweets=max_tweets,
+        )
+
+        client = await self._make_client()
+        tweets = await self._paginate(client, screen_name, cutoff, max_tweets)
+
+        logger.info(
+            "twitter_fetch_completed",
+            screen_name=screen_name,
+            count=len(tweets),
+        )
+        return tweets
+
+    async def fetch_many(
+        self,
+        screen_names: list[str],
+        since: timedelta | None = None,
+        max_tweets: int = 200,
+    ) -> list[TwitterUserResult]:
+        """Fetch tweets for multiple users sequentially (Twitter rate limits
+        make concurrency impractical here).
+
+        Returns one ``TwitterUserResult`` per user, preserving input order.
+        """
+        results: list[TwitterUserResult] = []
+        client = await self._make_client()
+
+        for i, screen_name in enumerate(screen_names):
+            cutoff = (datetime.now(UTC) - since) if since is not None else None
+            try:
+                tweets = await self._paginate(client, screen_name, cutoff, max_tweets)
+                results.append(
+                    TwitterUserResult(
+                        screen_name=screen_name, tweets=tweets, error=None
+                    )
+                )
+            except Exception as exc:
+                logger.warning(
+                    "twitter_fetch_failed",
+                    screen_name=screen_name,
+                    error=str(exc),
+                )
+                results.append(
+                    TwitterUserResult(
+                        screen_name=screen_name, tweets=[], error=str(exc)
+                    )
+                )
+            # Pause between users to avoid hammering the API.
+            if i < len(screen_names) - 1:
+                await asyncio.sleep(5)
+
+        return results
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    async def _resolve_cookies(self) -> str:
+        """Return cookies JSON string from Secrets Manager or local file."""
+        if self._cookies_cache is not None:
+            return self._cookies_cache
+
+        if self._cookies_secret_id:
+            from pulse_engine.secrets import fetch_secret
+
+            logger.debug(
+                "twitter_cookies_fetching_secret",
+                secret_id=self._cookies_secret_id,
+            )
+            raw = await fetch_secret(self._cookies_secret_id, self._aws_region)
+            # Secret may be stored as {"TWITTER_COOKIES_SECRET_ID": "<cookies json>"}
+            try:
+                parsed = json.loads(raw)
+                if isinstance(parsed, dict):
+                    # Extract the first value that looks like a cookies JSON string
+                    for v in parsed.values():
+                        if isinstance(v, str):
+                            raw = v
+                            break
+            except ValueError:
+                pass
+            cookies = raw
+            self._cookies_cache = cookies
+            logger.info(
+                "twitter_cookies_loaded_from_secrets_manager",
+                secret_id=self._cookies_secret_id,
+            )
+            return cookies
+
+        if self._cookies_path:
+            cookies = Path(self._cookies_path).read_text(encoding="utf-8")
+            self._cookies_cache = cookies
+            return cookies
+
+        raise RuntimeError(
+            "Twitter cookies not configured. Set TWITTER_COOKIES_SECRET_ID "
+            "or TWITTER_COOKIES_PATH."
+        )
+
+    async def _make_client(self) -> object:
+        """Build and return an authenticated twikit Client."""
+        try:
+            from twikit import Client
+        except ImportError:
+            raise RuntimeError(
+                "twikit is required for Twitter scraping: pip install twikit"
+            )
+
+        cookies_json = await self._resolve_cookies()
+
+        # twikit loads cookies from a file path, so materialise to a temp file.
+        with tempfile.NamedTemporaryFile(
+            mode="w", suffix=".json", delete=False, encoding="utf-8"
+        ) as f:
+            f.write(cookies_json)
+            tmp_path = f.name
+
+        client = Client()
+        client.load_cookies(tmp_path)
+        Path(tmp_path).unlink(missing_ok=True)
+        return client
+
+    async def _paginate(
+        self,
+        client: object,
+        screen_name: str,
+        cutoff: datetime | None,
+        max_tweets: int,
+    ) -> list[TweetMetadata]:
+        try:
+            from twikit import TooManyRequests
+        except ImportError:
+            raise RuntimeError("twikit is required: pip install twikit")
+
+        user = await client.get_user_by_screen_name(screen_name)  # type: ignore[attr-defined]
+        user_id = user.id
+
+        tweets: list[TweetMetadata] = []
+        cursor = None
+
+        while len(tweets) < max_tweets:
+            try:
+                page = await client.get_user_tweets(  # type: ignore[attr-defined]
+                    user_id, "Tweets", count=40, cursor=cursor
+                )
+            except TooManyRequests as exc:
+                reset = getattr(exc, "rate_limit_reset", None)
+                wait = (
+                    max(
+                        (
+                            datetime.fromtimestamp(reset) - datetime.now()
+                        ).total_seconds(),
+                        0,
+                    )
+                    if reset
+                    else 60
+                )
+                logger.warning("twitter_rate_limited", wait_seconds=wait)
+                await asyncio.sleep(wait)
+                continue
+            except Exception:
+                logger.warning(
+                    "twitter_page_fetch_failed",
+                    screen_name=screen_name,
+                    exc_info=True,
+                )
+                break
+
+            if not page:
+                break
+
+            stop = False
+            for tweet in page:
+                meta = _to_model(tweet, screen_name)
+                if meta is None:
+                    continue
+                if cutoff and meta.created_at < cutoff:
+                    stop = True
+                    break
+                tweets.append(meta)
+                if len(tweets) >= max_tweets:
+                    stop = True
+                    break
+
+            if stop:
+                break
+
+            try:
+                cursor = page.next_cursor
+            except AttributeError:
+                cursor = None
+
+            if not cursor:
+                break
+
+            await asyncio.sleep(3)
+
+        return tweets

pulse_engine/adapters/youtube_downloader.py

@@ -0,0 +1,186 @@
+"""YouTube audio downloader adapter (yt-dlp only)."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from pulse_engine.secrets import fetch_secret
+
+logger = structlog.get_logger(__name__)
+
+# Default yt-dlp player clients. "tv_embedded" returns audio-only streams for
+# age-restricted videos when cookies are provided. Override via YT_DLP_PLAYER_CLIENTS.
+_DEFAULT_PLAYER_CLIENTS = ["tv_embedded", "web"]
+
+
+def _build_yt_dlp_opts(
+    output_template: str,
+    extra: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build yt-dlp option dict, merging env defaults with caller overrides."""
+    raw_clients = os.environ.get("YT_DLP_PLAYER_CLIENTS", "").strip()
+    clients: list[str] = (
+        [c.strip() for c in raw_clients.split(",") if c.strip()]
+        if raw_clients
+        else _DEFAULT_PLAYER_CLIENTS
+    )
+
+    opts: dict[str, Any] = {
+        "format": "bestaudio/best",
+        "outtmpl": output_template,
+        "postprocessors": [
+            {
+                "key": "FFmpegExtractAudio",
+                "preferredcodec": "mp3",
+                "preferredquality": "64",
+            }
+        ],
+        "quiet": True,
+        "no_warnings": True,
+        # Fetch updated EJS challenge solver from GitHub when bundled version
+        # can't handle the current YouTube player's n-challenge.
+        # Must be a set — passing a string causes it to be iterated as chars.
+        "remote_components": {"ejs:github"},
+        "extractor_args": {
+            "youtube": {
+                "player_client": clients,
+            }
+        },
+    }
+
+    if not extra:
+        return opts
+
+    override = dict(extra)
+    # cookies_content is handled separately — materialised to a temp file.
+    override.pop("cookies_content", None)
+    ea = override.pop("extractor_args", None)
+    opts.update(override)
+    if isinstance(ea, dict):
+        existing = opts.get("extractor_args") or {}
+        merged: dict[str, Any] = {**existing}
+        for k, v in ea.items():
+            if isinstance(v, dict) and isinstance(merged.get(k), dict):
+                merged[k] = {**merged[k], **v}
+            else:
+                merged[k] = v
+        opts["extractor_args"] = merged
+
+    return opts
+
+
+class YouTubeDownloader:
+    """Downloads audio from YouTube via yt-dlp, returning a Path to an mp3 file.
+
+    Environment variables:
+        - ``YT_DLP_PLAYER_CLIENTS``    — comma-separated yt-dlp client list
+        - ``YT_DLP_COOKIES_SECRET_ID`` — Secrets Manager secret name/ARN for cookies.txt
+        - ``AWS_REGION``               — region for Secrets Manager (default: us-east-1)
+    """
+
+    def __init__(self, yt_dlp_opts: dict[str, Any] | None = None) -> None:
+        self._yt_dlp_opts = yt_dlp_opts or {}
+        self._cookies_secret_id: str = os.environ.get(
+            "YT_DLP_COOKIES_SECRET_ID", ""
+        ).strip()
+        self._aws_region: str = os.environ.get("AWS_REGION", "us-east-1").strip()
+        self._cookies_cache: str | None = None  # lazily populated
+
+    async def download(self, video_id: str, output_dir: str) -> Path | None:
+        """Download audio for *video_id* into *output_dir*, returning path to mp3.
+
+        Returns ``None`` if the download fails for any reason.
+        """
+        url = f"https://www.youtube.com/watch?v={video_id}"
+        logger.info("youtube_download_started", video_id=video_id)
+
+        result = await self._download_audio(url, output_dir)
+        if result is None:
+            logger.warning("youtube_download_failed", video_id=video_id)
+        else:
+            logger.info("youtube_download_completed", video_id=video_id, path=str(result))
+        return result
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    async def _resolve_cookies(self) -> str:
+        """Return cookies.txt content, fetching from Secrets Manager if configured.
+
+        Result is cached after the first successful fetch so subsequent videos
+        in the same adapter instance don't make redundant API calls.
+        """
+        if self._cookies_cache is not None:
+            return self._cookies_cache
+
+        if not self._cookies_secret_id:
+            self._cookies_cache = ""
+            return ""
+
+        logger.debug(
+            "youtube_cookies_fetching_secret",
+            secret_id=self._cookies_secret_id,
+            region=self._aws_region,
+        )
+        import json as _json
+
+        raw = await fetch_secret(self._cookies_secret_id, self._aws_region)
+        # Secret may be stored as {"YT_DLP_COOKIES_SECRET_ID": "<netscape txt>"}
+        try:
+            parsed = _json.loads(raw)
+            if isinstance(parsed, dict):
+                for v in parsed.values():
+                    if isinstance(v, str):
+                        raw = v
+                        break
+        except ValueError:
+            pass
+        cookies = raw
+        self._cookies_cache = cookies
+        logger.info(
+            "youtube_cookies_loaded_from_secrets_manager",
+            secret_id=self._cookies_secret_id,
+            chars=len(cookies),
+        )
+        return cookies
+
+    async def _download_audio(self, url: str, output_dir: str) -> Path | None:
+        """Download audio from a YouTube URL to *output_dir* via yt-dlp."""
+        try:
+            import yt_dlp
+        except ImportError:
+            logger.warning("yt_dlp_not_installed", url=url)
+            return None
+
+        output_template = str(Path(output_dir) / "%(id)s.%(ext)s")
+        ydl_opts = _build_yt_dlp_opts(output_template, extra=self._yt_dlp_opts)
+
+        # Resolve cookies (priority: Secrets Manager > env var).
+        cookies_content = await self._resolve_cookies()
+        if cookies_content and "cookiefile" not in ydl_opts:
+            cookies_path = Path(output_dir) / "cookies.txt"
+            cookies_path.write_text(cookies_content, encoding="utf-8")
+            try:
+                import os as _os
+                _os.chmod(cookies_path, 0o600)
+            except OSError:
+                pass
+            ydl_opts["cookiefile"] = str(cookies_path)
+
+        def _download() -> Path | None:
+            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                info = ydl.extract_info(url, download=True)
+                candidate = Path(output_dir) / f"{info.get('id', '')}.mp3"
+                return candidate if candidate.exists() else None
+
+        try:
+            return await asyncio.to_thread(_download)
+        except Exception:
+            logger.warning("youtube_audio_download_failed", url=url, exc_info=True)
+            return None