xtrm-tools 0.7.2 → 0.7.4

package/.xtrm/skills/default/last30days/scripts/lib/xiaohongshu_api.py

@@ -0,0 +1,162 @@
+"""Xiaohongshu HTTP API search client for last30days.
+
+Uses xpzouying/xiaohongshu-mcp REST endpoints:
+- GET/POST /api/v1/feeds/search
+- GET /api/v1/login/status
+"""
+
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from . import http
+
+
+def _to_int(value: Any) -> int:
+    """Convert Xiaohongshu count strings to int.
+
+    Supports plain ints and Chinese suffixes like 1.2万 / 3亿.
+    """
+    if value is None:
+        return 0
+    if isinstance(value, (int, float)):
+        return int(value)
+
+    text = str(value).strip().lower().replace(",", "")
+    if not text:
+        return 0
+
+    try:
+        if text.endswith("万"):
+            return int(float(text[:-1]) * 10000)
+        if text.endswith("亿"):
+            return int(float(text[:-1]) * 100000000)
+        return int(float(text))
+    except (TypeError, ValueError):
+        return 0
+
+
+def _timestamp_to_date_ms(ts: Any) -> Optional[str]:
+    """Convert millisecond timestamp to YYYY-MM-DD."""
+    try:
+        iv = int(ts)
+        if iv <= 0:
+            return None
+        # API examples use milliseconds.
+        dt = datetime.fromtimestamp(iv / 1000.0, tz=timezone.utc)
+        return dt.strftime("%Y-%m-%d")
+    except (TypeError, ValueError, OSError):
+        return None
+
+
+def _relevance_from_interactions(likes: int, comments: int, favorites: int) -> float:
+    """Heuristic relevance score from engagement metrics."""
+    # Weighted engagement with soft caps to [0, 1].
+    weighted = (likes * 1.0) + (comments * 2.5) + (favorites * 1.5)
+    # 5000 weighted engagement ~= strong relevance.
+    score = min(1.0, max(0.05, weighted / 5000.0))
+    return round(score, 3)
+
+
+def _build_note_url(feed_id: str, xsec_token: str) -> str:
+    """Build a stable Xiaohongshu note URL."""
+    if xsec_token:
+        return f"https://www.xiaohongshu.com/explore/{feed_id}?xsec_token={xsec_token}"
+    return f"https://www.xiaohongshu.com/explore/{feed_id}"
+
+
+def search_feeds(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    base_url: str,
+    depth: str = "default",
+) -> List[Dict[str, Any]]:
+    """Search Xiaohongshu feeds and normalize to web-item shape."""
+    base = (base_url or "").rstrip("/")
+    if not base:
+        raise ValueError("Missing Xiaohongshu API base URL")
+
+    # Quick login sanity check.
+    login = http.get(f"{base}/api/v1/login/status", timeout=8, retries=1)
+    is_logged_in = (
+        login.get("data", {}).get("is_logged_in")
+        if isinstance(login, dict) else False
+    )
+    if not is_logged_in:
+        raise http.HTTPError("Xiaohongshu API reachable but not logged in")
+
+    # API supports filters; use recency-oriented defaults.
+    publish_time = "一天内" if depth == "quick" else "一周内" if depth == "default" else "半年内"
+    payload = {
+        "keyword": topic,
+        "filters": {
+            "sort_by": "综合",
+            "note_type": "不限",
+            "publish_time": publish_time,
+            "search_scope": "不限",
+            "location": "不限",
+        },
+    }
+
+    resp = http.post(f"{base}/api/v1/feeds/search", payload, timeout=20, retries=1)
+    feeds = resp.get("data", {}).get("feeds", []) if isinstance(resp, dict) else []
+    if not isinstance(feeds, list):
+        feeds = []
+
+    # Cap source volume similarly to other web sources.
+    limit = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
+    items: List[Dict[str, Any]] = []
+
+    for i, feed in enumerate(feeds[:limit]):
+        if not isinstance(feed, dict):
+            continue
+        note = feed.get("noteCard") or {}
+        if not isinstance(note, dict):
+            note = {}
+        interact = note.get("interactInfo") or {}
+        if not isinstance(interact, dict):
+            interact = {}
+
+        feed_id = str(feed.get("id") or note.get("noteId") or "").strip()
+        if not feed_id:
+            continue
+
+        xsec_token = str(feed.get("xsecToken") or note.get("xsecToken") or "").strip()
+        title = str(
+            note.get("displayTitle")
+            or note.get("title")
+            or ""
+        ).strip()
+        snippet = str(
+            note.get("desc")
+            or note.get("displayDesc")
+            or title
+            or ""
+        ).strip()
+
+        likes = _to_int(interact.get("likedCount"))
+        comments = _to_int(interact.get("commentCount"))
+        favorites = _to_int(interact.get("collectedCount"))
+
+        date_value = _timestamp_to_date_ms(note.get("time"))
+        why = f"Xiaohongshu engagement: likes={likes}, comments={comments}, favorites={favorites}"
+
+        items.append({
+            "id": f"XHS{i+1}",
+            "title": title[:200] if title else f"Xiaohongshu note {feed_id}",
+            "url": _build_note_url(feed_id, xsec_token),
+            "source_domain": "xiaohongshu.com",
+            "snippet": snippet[:500],
+            "date": date_value,
+            "date_confidence": "high" if date_value else "low",
+            "relevance": _relevance_from_interactions(likes, comments, favorites),
+            "why_relevant": why,
+            # Keep raw engagement for debugging/possible future rendering.
+            "engagement": {
+                "likes": likes,
+                "comments": comments,
+                "favorites": favorites,
+            },
+        })
+
+    return items

package/.xtrm/skills/default/last30days/scripts/lib/youtube_yt.py

@@ -0,0 +1,538 @@
+"""YouTube search and transcript extraction via yt-dlp for /last30days v2.1.
+
+Uses yt-dlp (https://github.com/yt-dlp/yt-dlp) for both YouTube search and
+transcript extraction. No API keys needed — just have yt-dlp installed.
+
+Inspired by Peter Steinberger's toolchain approach (yt-dlp + summarize CLI).
+"""
+
+import json
+import math
+import os
+import re
+import signal
+import shutil
+import subprocess
+import sys
+import tempfile
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+# Depth configurations: how many videos to search / transcribe
+DEPTH_CONFIG = {
+    "quick": 10,
+    "default": 20,
+    "deep": 40,
+}
+
+TRANSCRIPT_LIMITS = {
+    "quick": 3,
+    "default": 5,
+    "deep": 8,
+}
+
+# Max words to keep from each transcript
+TRANSCRIPT_MAX_WORDS = 5000
+
+from .relevance import token_overlap_relevance as _compute_relevance
+
+
+def extract_transcript_highlights(transcript: str, topic: str, limit: int = 5) -> List[str]:
+    """Extract quotable highlights from a YouTube transcript.
+
+    Similar to reddit_enrich.extract_comment_insights() but for
+    continuous speech-to-text rather than threaded comments.
+    """
+    if not transcript:
+        return []
+
+    sentences = re.split(r'(?<=[.!?])\s+', transcript)
+
+    filler = [
+        r"^(hey |hi |what's up|welcome back|in today's video|don't forget to)",
+        r"(subscribe|like and comment|hit the bell|check out the link|down below)",
+        r"^(so |and |but |okay |alright |um |uh )",
+        r"(thanks for watching|see you (next|in the)|bye)",
+    ]
+
+    topic_words = [w.lower() for w in topic.lower().split() if len(w) > 2]
+
+    candidates = []
+    for sent in sentences:
+        sent = sent.strip()
+        words = sent.split()
+        if len(words) < 8 or len(words) > 50:
+            continue
+        if any(re.search(p, sent, re.IGNORECASE) for p in filler):
+            continue
+
+        score = 0
+        if re.search(r'\d', sent):
+            score += 2
+        if re.search(r'[A-Z][a-z]+', sent):
+            score += 1
+        if '?' in sent:
+            score += 1
+        sent_lower = sent.lower()
+        if any(w in sent_lower for w in topic_words):
+            score += 2
+
+        candidates.append((score, sent))
+
+    candidates.sort(key=lambda x: -x[0])
+    return [sent for _, sent in candidates[:limit]]
+
+
+def _log(msg: str):
+    """Log to stderr."""
+    sys.stderr.write(f"[YouTube] {msg}\n")
+    sys.stderr.flush()
+
+
+def is_ytdlp_installed() -> bool:
+    """Check if yt-dlp is available in PATH."""
+    return shutil.which("yt-dlp") is not None
+
+
+def _extract_core_subject(topic: str) -> str:
+    """Extract core subject from verbose query for YouTube search.
+
+    NOTE: 'tips', 'tricks', 'tutorial', 'guide', 'review', 'reviews'
+    are intentionally KEPT — they're YouTube content types that improve search.
+    """
+    from .query import extract_core_subject
+    # YouTube-specific noise set: smaller than default, keeps content-type words
+    _YT_NOISE = frozenset({
+        'best', 'top', 'good', 'great', 'awesome', 'killer',
+        'latest', 'new', 'news', 'update', 'updates',
+        'trending', 'hottest', 'popular', 'viral',
+        'practices', 'features',
+        'recommendations', 'advice',
+        'prompt', 'prompts', 'prompting',
+        'methods', 'strategies', 'approaches',
+    })
+    return extract_core_subject(topic, noise=_YT_NOISE)
+
+
+def search_youtube(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+) -> Dict[str, Any]:
+    """Search YouTube via yt-dlp. No API key needed.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: 'quick', 'default', or 'deep'
+
+    Returns:
+        Dict with 'items' list of video metadata dicts.
+    """
+    if not is_ytdlp_installed():
+        return {"items": [], "error": "yt-dlp not installed"}
+
+    count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    core_topic = _extract_core_subject(topic)
+
+    _log(f"Searching YouTube for '{core_topic}' (since {from_date}, count={count})")
+
+    # yt-dlp search with full metadata (no --flat-playlist so dates are real).
+    # NOTE: --dateafter intentionally omitted — YouTube search returns
+    # relevance-sorted results and strict date filtering returns 0 for
+    # evergreen topics. Python soft filter (below) handles date filtering.
+    cmd = [
+        "yt-dlp",
+        "--ignore-config",
+        "--no-cookies-from-browser",
+        f"ytsearch{count}:{core_topic}",
+        "--dump-json",
+        "--no-warnings",
+        "--no-download",
+    ]
+
+    preexec = os.setsid if hasattr(os, 'setsid') else None
+
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            preexec_fn=preexec,
+        )
+        try:
+            stdout, stderr = proc.communicate(timeout=120)
+        except subprocess.TimeoutExpired:
+            try:
+                os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+            except (ProcessLookupError, PermissionError, OSError):
+                proc.kill()
+            proc.wait(timeout=5)
+            _log("YouTube search timed out (120s)")
+            return {"items": [], "error": "Search timed out"}
+    except FileNotFoundError:
+        return {"items": [], "error": "yt-dlp not found"}
+
+    if not (stdout or "").strip():
+        _log("YouTube search returned 0 results")
+        return {"items": []}
+
+    # Parse JSON-per-line output
+    items = []
+    for line in stdout.strip().split("\n"):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            video = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+
+        video_id = video.get("id", "")
+        view_count = video.get("view_count") or 0
+        like_count = video.get("like_count") or 0
+        comment_count = video.get("comment_count") or 0
+        upload_date = video.get("upload_date", "")  # YYYYMMDD
+
+        # Convert YYYYMMDD to YYYY-MM-DD
+        date_str = None
+        if upload_date and len(upload_date) == 8:
+            date_str = f"{upload_date[:4]}-{upload_date[4:6]}-{upload_date[6:8]}"
+
+        items.append({
+            "video_id": video_id,
+            "title": video.get("title", ""),
+            "url": f"https://www.youtube.com/watch?v={video_id}",
+            "channel_name": video.get("channel", video.get("uploader", "")),
+            "date": date_str,
+            "engagement": {
+                "views": view_count,
+                "likes": like_count,
+                "comments": comment_count,
+            },
+            "duration": video.get("duration"),
+            "relevance": _compute_relevance(core_topic, video.get("title", "")),
+            "why_relevant": f"YouTube: {video.get('title', core_topic)[:60]}",
+        })
+
+    # Soft date filter: prefer recent items but fall back to all if too few
+    recent = [i for i in items if i["date"] and i["date"] >= from_date]
+    if len(recent) >= 3:
+        items = recent
+        _log(f"Found {len(items)} videos within date range")
+    else:
+        _log(f"Found {len(items)} videos ({len(recent)} within date range, keeping all)")
+
+    # Sort by views descending
+    items.sort(key=lambda x: x["engagement"]["views"], reverse=True)
+
+    return {"items": items}
+
+
+def _clean_vtt(vtt_text: str) -> str:
+    """Convert VTT subtitle format to clean plaintext."""
+    # Strip VTT header
+    text = re.sub(r'^WEBVTT.*?\n\n', '', vtt_text, flags=re.DOTALL)
+    # Strip timestamps
+    text = re.sub(r'\d{2}:\d{2}:\d{2}\.\d{3}\s*-->\s*\d{2}:\d{2}:\d{2}\.\d{3}.*\n', '', text)
+    # Strip position/alignment tags
+    text = re.sub(r'<[^>]+>', '', text)
+    # Strip cue numbers
+    text = re.sub(r'^\d+\s*$', '', text, flags=re.MULTILINE)
+    # Deduplicate overlapping lines
+    lines = text.strip().split('\n')
+    seen = set()
+    unique = []
+    for line in lines:
+        stripped = line.strip()
+        if stripped and stripped not in seen:
+            seen.add(stripped)
+            unique.append(stripped)
+    return re.sub(r'\s+', ' ', ' '.join(unique)).strip()
+
+
+_YT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+
+
+def _fetch_transcript_direct(video_id: str, timeout: int = 30) -> Optional[str]:
+    """Fetch YouTube transcript via direct HTTP without yt-dlp.
+
+    Scrapes the watch page HTML for the captions track URL in
+    ytInitialPlayerResponse, then fetches the VTT subtitle file.
+
+    Args:
+        video_id: YouTube video ID
+        timeout: HTTP request timeout in seconds
+
+    Returns:
+        Raw VTT text, or None if captions are unavailable.
+    """
+    watch_url = f"https://www.youtube.com/watch?v={video_id}"
+    headers = {
+        "User-Agent": _YT_USER_AGENT,
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+
+    # Step 1: Fetch the watch page HTML
+    req = urllib.request.Request(watch_url, headers=headers)
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            html = resp.read().decode("utf-8", errors="replace")
+    except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError) as exc:
+        _log(f"Direct transcript: failed to fetch watch page for {video_id}: {exc}")
+        return None
+
+    # Step 2: Extract captions URL from ytInitialPlayerResponse
+    # YouTube embeds this as a JS variable in the page HTML
+    match = re.search(
+        r'ytInitialPlayerResponse\s*=\s*(\{.+?\})\s*;(?:\s*var\s|\s*<\/script>)',
+        html,
+    )
+    if not match:
+        # Fallback: try the JSON embedded in the script tag
+        match = re.search(
+            r'var\s+ytInitialPlayerResponse\s*=\s*(\{.+?\})\s*;',
+            html,
+        )
+    if not match:
+        _log(f"Direct transcript: no ytInitialPlayerResponse found for {video_id}")
+        return None
+
+    try:
+        player_response = json.loads(match.group(1))
+    except json.JSONDecodeError:
+        _log(f"Direct transcript: failed to parse ytInitialPlayerResponse for {video_id}")
+        return None
+
+    # Navigate to caption tracks
+    captions = player_response.get("captions", {})
+    renderer = captions.get("playerCaptionsTracklistRenderer", {})
+    caption_tracks = renderer.get("captionTracks", [])
+
+    if not caption_tracks:
+        _log(f"Direct transcript: no caption tracks for {video_id}")
+        return None
+
+    # Find English track (prefer exact 'en', then any en variant, then first track)
+    base_url = None
+    for track in caption_tracks:
+        lang = track.get("languageCode", "")
+        if lang == "en":
+            base_url = track.get("baseUrl")
+            break
+    if not base_url:
+        for track in caption_tracks:
+            lang = track.get("languageCode", "")
+            if lang.startswith("en"):
+                base_url = track.get("baseUrl")
+                break
+    if not base_url:
+        # Fall back to first available track
+        base_url = caption_tracks[0].get("baseUrl")
+    if not base_url:
+        _log(f"Direct transcript: no baseUrl in caption tracks for {video_id}")
+        return None
+
+    # Step 3: Fetch the VTT subtitle file
+    sep = "&" if "?" in base_url else "?"
+    vtt_url = f"{base_url}{sep}fmt=vtt"
+    vtt_req = urllib.request.Request(vtt_url, headers=headers)
+    try:
+        with urllib.request.urlopen(vtt_req, timeout=timeout) as resp:
+            vtt_text = resp.read().decode("utf-8", errors="replace")
+    except (urllib.error.URLError, urllib.error.HTTPError, OSError, TimeoutError) as exc:
+        _log(f"Direct transcript: failed to fetch VTT for {video_id}: {exc}")
+        return None
+
+    if not vtt_text or not vtt_text.strip():
+        return None
+
+    return vtt_text
+
+
+def _fetch_transcript_ytdlp(video_id: str, temp_dir: str) -> Optional[str]:
+    """Fetch transcript using yt-dlp (original implementation).
+
+    Args:
+        video_id: YouTube video ID
+        temp_dir: Temporary directory for subtitle files
+
+    Returns:
+        Raw VTT text, or None if no captions available.
+    """
+    cmd = [
+        "yt-dlp",
+        "--ignore-config",
+        "--no-cookies-from-browser",
+        "--write-auto-subs",
+        "--sub-lang", "en",
+        "--sub-format", "vtt",
+        "--skip-download",
+        "--no-warnings",
+        "-o", f"{temp_dir}/%(id)s",
+        f"https://www.youtube.com/watch?v={video_id}",
+    ]
+
+    preexec = os.setsid if hasattr(os, 'setsid') else None
+
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            preexec_fn=preexec,
+        )
+        try:
+            proc.communicate(timeout=30)
+        except subprocess.TimeoutExpired:
+            try:
+                os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+            except (ProcessLookupError, PermissionError, OSError):
+                proc.kill()
+            proc.wait(timeout=5)
+            return None
+    except FileNotFoundError:
+        return None
+
+    # yt-dlp may save as .en.vtt or .en-orig.vtt
+    vtt_path = Path(temp_dir) / f"{video_id}.en.vtt"
+    if not vtt_path.exists():
+        # Try alternate naming
+        for p in Path(temp_dir).glob(f"{video_id}*.vtt"):
+            vtt_path = p
+            break
+        else:
+            return None
+
+    try:
+        return vtt_path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return None
+
+
+def fetch_transcript(video_id: str, temp_dir: str) -> Optional[str]:
+    """Fetch auto-generated transcript for a YouTube video.
+
+    Uses yt-dlp when available (preferred, more robust). Falls back to
+    direct HTTP transcript fetching when yt-dlp is not installed.
+
+    Args:
+        video_id: YouTube video ID
+        temp_dir: Temporary directory for subtitle files
+
+    Returns:
+        Plaintext transcript string, or None if no captions available.
+    """
+    raw_vtt = None
+    if is_ytdlp_installed():
+        raw_vtt = _fetch_transcript_ytdlp(video_id, temp_dir)
+    else:
+        _log("yt-dlp not installed, using direct HTTP transcript fetch")
+        raw_vtt = _fetch_transcript_direct(video_id)
+
+    if not raw_vtt:
+        return None
+
+    transcript = _clean_vtt(raw_vtt)
+
+    # Truncate to max words
+    words = transcript.split()
+    if len(words) > TRANSCRIPT_MAX_WORDS:
+        transcript = ' '.join(words[:TRANSCRIPT_MAX_WORDS]) + '...'
+
+    return transcript if transcript else None
+
+
+def fetch_transcripts_parallel(
+    video_ids: List[str],
+    max_workers: int = 5,
+) -> Dict[str, Optional[str]]:
+    """Fetch transcripts for multiple videos in parallel.
+
+    Args:
+        video_ids: List of YouTube video IDs
+        max_workers: Max parallel fetches
+
+    Returns:
+        Dict mapping video_id to transcript text (or None).
+    """
+    if not video_ids:
+        return {}
+
+    _log(f"Fetching transcripts for {len(video_ids)} videos")
+
+    results = {}
+    with tempfile.TemporaryDirectory() as temp_dir:
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            futures = {
+                executor.submit(fetch_transcript, vid, temp_dir): vid
+                for vid in video_ids
+            }
+            for future in as_completed(futures):
+                vid = futures[future]
+                try:
+                    results[vid] = future.result()
+                except Exception:
+                    results[vid] = None
+
+    got = sum(1 for v in results.values() if v)
+    _log(f"Got transcripts for {got}/{len(video_ids)} videos")
+    return results
+
+
+def search_and_transcribe(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+) -> Dict[str, Any]:
+    """Full YouTube search: find videos, then fetch transcripts for top results.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: 'quick', 'default', or 'deep'
+
+    Returns:
+        Dict with 'items' list. Each item has a 'transcript_snippet' field.
+    """
+    # Step 1: Search
+    search_result = search_youtube(topic, from_date, to_date, depth)
+    items = search_result.get("items", [])
+
+    if not items:
+        return search_result
+
+    # Step 2: Fetch transcripts for top N by views
+    transcript_limit = TRANSCRIPT_LIMITS.get(depth, TRANSCRIPT_LIMITS["default"])
+    top_ids = [item["video_id"] for item in items[:transcript_limit]]
+    transcripts = fetch_transcripts_parallel(top_ids)
+
+    # Step 3: Attach transcripts and extract highlights
+    core_topic = _extract_core_subject(topic)
+    for item in items:
+        vid = item["video_id"]
+        transcript = transcripts.get(vid)
+        item["transcript_snippet"] = transcript or ""
+        item["transcript_highlights"] = extract_transcript_highlights(
+            transcript or "", core_topic,
+        )
+
+    return {"items": items}
+
+
+def parse_youtube_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Parse YouTube search response to normalized format.
+
+    Returns:
+        List of item dicts ready for normalization.
+    """
+    return response.get("items", [])