xtrm-tools 0.7.3 → 0.7.7

package/.xtrm/skills/default/last30days/scripts/lib/__init__.py

@@ -0,0 +1 @@
+# last30days library modules

package/.xtrm/skills/default/last30days/scripts/lib/bird_x.py

@@ -0,0 +1,458 @@
+"""Bird X search client - vendored Twitter GraphQL search for /last30days v2.1.
+
+Uses a vendored subset of @steipete/bird v0.8.0 (MIT License) to search X
+via Twitter's GraphQL API. No external `bird` CLI binary needed - just Node.js 22+.
+"""
+
+import json
+import os
+import signal
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+from .relevance import token_overlap_relevance as _compute_relevance
+
+# Path to the vendored bird-search wrapper
+_BIRD_SEARCH_MJS = Path(__file__).parent / "vendor" / "bird-search" / "bird-search.mjs"
+
+# Depth configurations: number of results to request
+DEPTH_CONFIG = {
+    "quick": 12,
+    "default": 30,
+    "deep": 60,
+}
+
+# Module-level credentials injected from .env config
+_credentials: Dict[str, str] = {}
+
+
+def set_credentials(auth_token: Optional[str], ct0: Optional[str]):
+    """Inject AUTH_TOKEN/CT0 from .env config so Node subprocesses can use them."""
+    if auth_token:
+        _credentials['AUTH_TOKEN'] = auth_token
+    if ct0:
+        _credentials['CT0'] = ct0
+
+
+def _has_injected_credentials() -> bool:
+    """Return True when both X session cookies were injected from config."""
+    return bool(_credentials.get('AUTH_TOKEN') and _credentials.get('CT0'))
+
+
+def _subprocess_env() -> Dict[str, str]:
+    """Build env dict for Node subprocesses, merging injected credentials."""
+    env = os.environ.copy()
+    env.update(_credentials)
+    # When repo config already provides cookies, disable browser-cookie fallback
+    # so vendored Bird never hits Safari/Chrome keychain during automation.
+    if _has_injected_credentials():
+        env.setdefault("BIRD_DISABLE_BROWSER_COOKIES", "1")
+    return env
+
+
+def _log(msg: str):
+    """Log to stderr."""
+    sys.stderr.write(f"[Bird] {msg}\n")
+    sys.stderr.flush()
+
+
+def _extract_core_subject(topic: str) -> str:
+    """Extract core subject from verbose query for X search.
+
+    X search is literal keyword AND matching — all words must appear.
+    Aggressively strip question/meta/research words to keep only the
+    core product/concept name (max 5 words).
+    """
+    from .query import extract_core_subject
+    return extract_core_subject(topic, max_words=5, strip_suffixes=True)
+
+
+def is_bird_installed() -> bool:
+    """Check if vendored Bird search module is available.
+
+    Returns:
+        True if bird-search.mjs exists and Node.js 22+ is in PATH.
+    """
+    if not _BIRD_SEARCH_MJS.exists():
+        return False
+    return shutil.which("node") is not None
+
+
+def is_bird_authenticated() -> Optional[str]:
+    """Check if X credentials are available (env vars or browser cookies).
+
+    Returns:
+        Auth source string if authenticated, None otherwise.
+    """
+    if not is_bird_installed():
+        return None
+
+    if _has_injected_credentials():
+        return "env AUTH_TOKEN"
+
+    try:
+        result = subprocess.run(
+            ["node", str(_BIRD_SEARCH_MJS), "--whoami"],
+            capture_output=True,
+            text=True,
+            timeout=15,
+            env=_subprocess_env(),
+        )
+        if result.returncode == 0 and result.stdout.strip():
+            return result.stdout.strip().split('\n')[0]
+        return None
+    except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
+        return None
+
+
+def check_npm_available() -> bool:
+    """Check if npm is available (kept for API compatibility).
+
+    Returns:
+        True if 'npm' command is available in PATH, False otherwise.
+    """
+    return shutil.which("npm") is not None
+
+
+def install_bird() -> Tuple[bool, str]:
+    """No-op - Bird search is vendored in v2.1, no installation needed.
+
+    Returns:
+        Tuple of (success, message).
+    """
+    if is_bird_installed():
+        return True, "Bird search is bundled with /last30days v2.1 - no installation needed."
+    if not shutil.which("node"):
+        return False, "Node.js 22+ is required for X search. Install Node.js first."
+    return False, f"Vendored bird-search.mjs not found at {_BIRD_SEARCH_MJS}"
+
+
+def get_bird_status() -> Dict[str, Any]:
+    """Get comprehensive Bird search status.
+
+    Returns:
+        Dict with keys: installed, authenticated, username, can_install
+    """
+    installed = is_bird_installed()
+    auth_source = is_bird_authenticated() if installed else None
+
+    return {
+        "installed": installed,
+        "authenticated": auth_source is not None,
+        "username": auth_source,  # Now returns auth source (e.g., "Safari", "env AUTH_TOKEN")
+        "can_install": True,  # Always vendored in v2.1
+    }
+
+
+def _run_bird_search(query: str, count: int, timeout: int) -> Dict[str, Any]:
+    """Run a search using the vendored bird-search.mjs module.
+
+    Args:
+        query: Full search query string (including since: filter)
+        count: Number of results to request
+        timeout: Timeout in seconds
+
+    Returns:
+        Raw Bird JSON response or error dict.
+    """
+    cmd = [
+        "node", str(_BIRD_SEARCH_MJS),
+        query,
+        "--count", str(count),
+        "--json",
+    ]
+
+    # Use process groups for clean cleanup on timeout/kill
+    preexec = os.setsid if hasattr(os, 'setsid') else None
+
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            preexec_fn=preexec,
+            env=_subprocess_env(),
+        )
+
+        # Register for cleanup tracking (if available)
+        try:
+            from last30days import register_child_pid, unregister_child_pid
+            register_child_pid(proc.pid)
+        except ImportError:
+            pass
+
+        try:
+            stdout, stderr = proc.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            # Kill the entire process group
+            try:
+                os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+            except (ProcessLookupError, PermissionError, OSError):
+                proc.kill()
+            proc.wait(timeout=5)
+            return {"error": f"Search timed out after {timeout}s", "items": []}
+        finally:
+            try:
+                from last30days import unregister_child_pid
+                unregister_child_pid(proc.pid)
+            except (ImportError, Exception):
+                pass
+
+        if proc.returncode != 0:
+            error = stderr.strip() if stderr else "Bird search failed"
+            return {"error": error, "items": []}
+
+        output = stdout.strip() if stdout else ""
+        if not output:
+            return {"items": []}
+
+        return json.loads(output)
+
+    except json.JSONDecodeError as e:
+        return {"error": f"Invalid JSON response: {e}", "items": []}
+    except Exception as e:
+        return {"error": str(e), "items": []}
+
+
+def search_x(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+) -> Dict[str, Any]:
+    """Search X using Bird CLI with automatic retry on 0 results.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD) - unused but kept for API compatibility
+        depth: Research depth - "quick", "default", or "deep"
+
+    Returns:
+        Raw Bird JSON response or error dict.
+    """
+    count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    timeout = 30 if depth == "quick" else 45 if depth == "default" else 60
+
+    # Extract core subject - X search is literal, not semantic
+    core_topic = _extract_core_subject(topic)
+    query = f"{core_topic} since:{from_date}"
+
+    _log(f"Searching: {query}")
+    response = _run_bird_search(query, count, timeout)
+
+    # Check if we got results
+    items = parse_bird_response(response, query=core_topic)
+
+    # Retry with OR groups for multi-word queries (X supports OR operator)
+    core_words = core_topic.split()
+    if not items and len(core_words) >= 2:
+        from .query import extract_compound_terms
+        compounds = extract_compound_terms(topic)
+        if compounds:
+            # Build OR-group query: ("multi-agent" OR "agent simulation") since:DATE
+            or_parts = ' OR '.join(f'"{t}"' for t in compounds[:3])
+            _log(f"0 results for '{core_topic}', retrying with OR groups: {or_parts}")
+            query = f"({or_parts}) since:{from_date}"
+            response = _run_bird_search(query, count, timeout)
+            items = parse_bird_response(response, query=core_topic)
+
+    # Retry with fewer keywords if still 0 results and query has 3+ words
+    if not items and len(core_words) > 2:
+        shorter = ' '.join(core_words[:2])
+        _log(f"0 results for '{core_topic}', retrying with '{shorter}'")
+        query = f"{shorter} since:{from_date}"
+        response = _run_bird_search(query, count, timeout)
+        items = parse_bird_response(response, query=core_topic)
+
+    # Last-chance retry: use strongest remaining token (often the product name)
+    if not items and core_words:
+        low_signal = {
+            'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
+            'best', 'top', 'latest', 'new', 'plugin', 'plugins',
+            'skill', 'skills', 'tool', 'tools',
+        }
+        candidates = [w for w in core_words if w not in low_signal]
+        if candidates:
+            strongest = max(candidates, key=len)
+            _log(f"0 results for '{core_topic}', retrying with strongest token '{strongest}'")
+            query = f"{strongest} since:{from_date}"
+            response = _run_bird_search(query, count, timeout)
+
+    return response
+
+
+def search_handles(
+    handles: List[str],
+    topic: Optional[str],
+    from_date: str,
+    count_per: int = 5,
+) -> List[Dict[str, Any]]:
+    """Search specific X handles for topic-related content.
+
+    Runs targeted Bird searches using `from:handle topic` syntax.
+    Used in Phase 2 supplemental search after entity extraction.
+
+    Args:
+        handles: List of X handles to search (without @)
+        topic: Search topic (core subject), or None for unfiltered search
+        from_date: Start date (YYYY-MM-DD)
+        count_per: Results to request per handle
+
+    Returns:
+        List of raw item dicts (same format as parse_bird_response output).
+    """
+    all_items = []
+    core_topic = _extract_core_subject(topic) if topic else None
+
+    for handle in handles:
+        handle = handle.lstrip("@")
+        if core_topic:
+            query = f"from:{handle} {core_topic} since:{from_date}"
+        else:
+            query = f"from:{handle} since:{from_date}"
+
+        cmd = [
+            "node", str(_BIRD_SEARCH_MJS),
+            query,
+            "--count", str(count_per),
+            "--json",
+        ]
+
+        preexec = os.setsid if hasattr(os, 'setsid') else None
+
+        try:
+            proc = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                preexec_fn=preexec,
+                env=_subprocess_env(),
+            )
+
+            try:
+                stdout, stderr = proc.communicate(timeout=15)
+            except subprocess.TimeoutExpired:
+                try:
+                    os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+                except (ProcessLookupError, PermissionError, OSError):
+                    proc.kill()
+                proc.wait(timeout=5)
+                _log(f"Handle search timed out for @{handle}")
+                continue
+
+            if proc.returncode != 0:
+                _log(f"Handle search failed for @{handle}: {(stderr or '').strip()}")
+                continue
+
+            output = (stdout or "").strip()
+            if not output:
+                continue
+
+            response = json.loads(output)
+            items = parse_bird_response(response, query=core_topic)
+            all_items.extend(items)
+
+        except json.JSONDecodeError:
+            _log(f"Invalid JSON from handle search for @{handle}")
+        except Exception as e:
+            _log(f"Handle search error for @{handle}: {e}")
+
+    return all_items
+
+
+def parse_bird_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
+    """Parse Bird response to match xai_x output format.
+
+    Args:
+        response: Raw Bird JSON response
+        query: Original search query for relevance scoring
+
+    Returns:
+        List of normalized item dicts matching xai_x.parse_x_response() format.
+    """
+    items = []
+
+    # Check for errors
+    if "error" in response and response["error"]:
+        _log(f"Bird error: {response['error']}")
+        return items
+
+    # Bird returns a list of tweets directly or under a key
+    raw_items = response if isinstance(response, list) else response.get("items", response.get("tweets", []))
+
+    if not isinstance(raw_items, list):
+        return items
+
+    for i, tweet in enumerate(raw_items):
+        if not isinstance(tweet, dict):
+            continue
+
+        # Extract URL - Bird uses permanent_url or we construct from id
+        url = tweet.get("permanent_url") or tweet.get("url", "")
+        if not url and tweet.get("id"):
+            # Try different field structures Bird might use
+            author = tweet.get("author", {}) or tweet.get("user", {})
+            screen_name = author.get("username") or author.get("screen_name", "")
+            if screen_name:
+                url = f"https://x.com/{screen_name}/status/{tweet['id']}"
+
+        if not url:
+            continue
+
+        # Parse date from created_at/createdAt (e.g., "Wed Jan 15 14:30:00 +0000 2026")
+        date = None
+        created_at = tweet.get("createdAt") or tweet.get("created_at", "")
+        if created_at:
+            try:
+                # Try ISO format first (e.g., "2026-02-03T22:33:32Z")
+                # Check for ISO date separator, not just "T" (which appears in "Tue")
+                if len(created_at) > 10 and created_at[10] == "T":
+                    dt = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
+                else:
+                    # Twitter format: "Wed Jan 15 14:30:00 +0000 2026"
+                    dt = datetime.strptime(created_at, "%a %b %d %H:%M:%S %z %Y")
+                date = dt.strftime("%Y-%m-%d")
+            except (ValueError, TypeError):
+                pass
+
+        # Extract user info (Bird uses author.username, older format uses user.screen_name)
+        author = tweet.get("author", {}) or tweet.get("user", {})
+        author_handle = author.get("username") or author.get("screen_name", "") or tweet.get("author_handle", "")
+
+        # Build engagement dict (Bird uses camelCase: likeCount, retweetCount, etc.)
+        engagement = {
+            "likes": tweet.get("likeCount") or tweet.get("like_count") or tweet.get("favorite_count"),
+            "reposts": tweet.get("retweetCount") or tweet.get("retweet_count"),
+            "replies": tweet.get("replyCount") or tweet.get("reply_count"),
+            "quotes": tweet.get("quoteCount") or tweet.get("quote_count"),
+        }
+        # Convert to int where possible
+        for key in engagement:
+            if engagement[key] is not None:
+                try:
+                    engagement[key] = int(engagement[key])
+                except (ValueError, TypeError):
+                    engagement[key] = None
+
+        # Build normalized item
+        item = {
+            "id": f"X{i+1}",
+            "text": str(tweet.get("text", tweet.get("full_text", ""))).strip()[:500],
+            "url": url,
+            "author_handle": author_handle.lstrip("@"),
+            "date": date,
+            "engagement": engagement if any(v is not None for v in engagement.values()) else None,
+            "why_relevant": "",  # Bird doesn't provide relevance explanations
+            "relevance": _compute_relevance(query, str(tweet.get("text", ""))) if query else 0.7,
+        }
+
+        items.append(item)
+
+    return items

package/.xtrm/skills/default/last30days/scripts/lib/bluesky.py

@@ -0,0 +1,225 @@
+"""Bluesky search via AT Protocol (requires app password).
+
+Uses bsky.social for auth and public.api.bsky.app for post search.
+Requires BSKY_HANDLE and BSKY_APP_PASSWORD env vars.
+"""
+
+import math
+import re
+import sys
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from . import http
+
+BSKY_SESSION_URL = "https://bsky.social/xrpc/com.atproto.server.createSession"
+BSKY_SEARCH_URL = "https://public.api.bsky.app/xrpc/app.bsky.feed.searchPosts"
+
+DEPTH_CONFIG = {
+    "quick": 15,
+    "default": 30,
+    "deep": 60,
+}
+
+# Module-level token cache (valid for the lifetime of a single research run)
+_cached_token: Optional[str] = None
+_session_error: Optional[str] = None
+
+
+def _log(msg: str):
+    """Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
+    if sys.stderr.isatty():
+        sys.stderr.write(f"[Bluesky] {msg}\n")
+        sys.stderr.flush()
+
+
+def _create_session(handle: str, app_password: str) -> Optional[str]:
+    """Create an AT Protocol session and return the access token.
+
+    Args:
+        handle: Bluesky handle (e.g. user.bsky.social)
+        app_password: App password from bsky.app/settings/app-passwords
+
+    Returns:
+        Access JWT string, or None on failure. Sets _session_error on failure.
+    """
+    global _cached_token, _session_error
+    if _cached_token:
+        return _cached_token
+
+    try:
+        response = http.request(
+            "POST",
+            BSKY_SESSION_URL,
+            json_data={"identifier": handle, "password": app_password},
+            timeout=15,
+        )
+        token = response.get("accessJwt")
+        if token:
+            _cached_token = token
+            _session_error = None
+            _log("Session created successfully")
+            return token
+        _log("No accessJwt in session response")
+        _session_error = "No accessJwt in session response"
+        return None
+    except http.HTTPError as e:
+        if e.status_code == 403 and e.body and "cloudflare" in e.body.lower():
+            _session_error = "Cloudflare blocked the request (403 Forbidden). This is a network-level block, not an auth issue. Try a different network or VPN."
+        elif e.status_code == 401:
+            _session_error = "Invalid credentials (401 Unauthorized). Check BSKY_HANDLE and BSKY_APP_PASSWORD."
+        else:
+            _session_error = f"Session request failed: {e}"
+        _log(f"Session creation failed: {_session_error}")
+        return None
+    except Exception as e:
+        _session_error = f"Session request failed: {type(e).__name__}: {e}"
+        _log(f"Session creation failed: {_session_error}")
+        return None
+
+
+def _extract_core_subject(topic: str) -> str:
+    """Extract core subject from verbose query for Bluesky search."""
+    from .query import extract_core_subject
+    _BSKY_NOISE = frozenset({
+        'best', 'top', 'good', 'great', 'awesome',
+        'latest', 'new', 'news', 'update', 'updates',
+        'trending', 'hottest', 'popular', 'viral',
+        'practices', 'features', 'recommendations', 'advice',
+    })
+    return extract_core_subject(topic, noise=_BSKY_NOISE)
+
+
+def _parse_date(item: Dict[str, Any]) -> Optional[str]:
+    """Parse date from Bluesky post to YYYY-MM-DD.
+
+    AT Protocol uses ISO 8601 format in indexedAt and createdAt fields.
+    """
+    for key in ("indexedAt", "createdAt"):
+        val = item.get(key)
+        if val and isinstance(val, str):
+            try:
+                dt = datetime.fromisoformat(val.replace("Z", "+00:00"))
+                return dt.strftime("%Y-%m-%d")
+            except (ValueError, TypeError):
+                pass
+    return None
+
+
+def search_bluesky(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    config: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """Search Bluesky via AT Protocol API.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: 'quick', 'default', or 'deep'
+        config: Config dict with BSKY_HANDLE and BSKY_APP_PASSWORD
+
+    Returns:
+        Dict with 'posts' list from AT Protocol response.
+    """
+    config = config or {}
+    handle = config.get("BSKY_HANDLE", "")
+    app_password = config.get("BSKY_APP_PASSWORD", "")
+
+    if not handle or not app_password:
+        return {"posts": [], "error": "Bluesky credentials not configured"}
+
+    # Authenticate
+    token = _create_session(handle, app_password)
+    if not token:
+        error_msg = _session_error or "Bluesky session creation failed (unknown error)"
+        return {"posts": [], "error": error_msg}
+
+    count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    core_topic = _extract_core_subject(topic)
+
+    _log(f"Searching for '{core_topic}' (depth={depth}, limit={count})")
+
+    from urllib.parse import urlencode
+    params = {
+        "q": core_topic,
+        "limit": str(min(count, 100)),
+        "sort": "top",
+    }
+    url = f"{BSKY_SEARCH_URL}?{urlencode(params)}"
+
+    try:
+        response = http.request(
+            "GET", url,
+            headers={"Authorization": f"Bearer {token}"},
+            timeout=30,
+        )
+    except http.HTTPError as e:
+        _log(f"Search failed: {e}")
+        if e.status_code == 403 and e.body and "cloudflare" in e.body.lower():
+            return {"posts": [], "error": "Bluesky search blocked by Cloudflare (403). This is a network-level block - try a different network or VPN."}
+        return {"posts": [], "error": f"Bluesky search failed: {e}"}
+    except Exception as e:
+        _log(f"Search failed: {e}")
+        return {"posts": [], "error": f"Bluesky search failed: {type(e).__name__}: {e}"}
+
+    posts = response.get("posts", [])
+    _log(f"Found {len(posts)} posts")
+    return response
+
+
+def parse_bluesky_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Parse AT Protocol response into normalized item dicts.
+
+    Returns:
+        List of item dicts ready for normalization.
+    """
+    posts = response.get("posts", [])
+    items = []
+
+    for i, post in enumerate(posts):
+        record = post.get("record") or {}
+        text = record.get("text") or ""
+
+        author = post.get("author") or {}
+        handle = author.get("handle") or ""
+        display_name = author.get("displayName") or handle
+
+        # Post URI -> URL
+        # URI format: at://did:plc:xxx/app.bsky.feed.post/rkey
+        uri = post.get("uri") or ""
+        rkey = uri.rsplit("/", 1)[-1] if uri else ""
+        url = f"https://bsky.app/profile/{handle}/post/{rkey}" if handle and rkey else ""
+
+        likes = post.get("likeCount") or 0
+        reposts = post.get("repostCount") or 0
+        replies = post.get("replyCount") or 0
+        quotes = post.get("quoteCount") or 0
+
+        date_str = _parse_date(post) or _parse_date(record)
+
+        # Relevance: position-based (AT Protocol sorts by relevance with sort=top)
+        rank_score = max(0.3, 1.0 - (i * 0.02))
+        engagement_boost = min(0.2, math.log1p(likes + reposts) / 40)
+        relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
+
+        items.append({
+            "handle": handle,
+            "display_name": display_name,
+            "text": text,
+            "url": url,
+            "date": date_str,
+            "engagement": {
+                "likes": likes,
+                "reposts": reposts,
+                "replies": replies,
+                "quotes": quotes,
+            },
+            "relevance": round(relevance, 2),
+            "why_relevant": f"Bluesky: @{handle}: {text[:60]}" if text else f"Bluesky: {handle}",
+        })
+
+    return items