xtrm-tools 0.7.3 → 0.7.7

package/.xtrm/skills/default/last30days/scripts/lib/scrapecreators_x.py

@@ -0,0 +1,182 @@
+"""X/Twitter search via ScrapeCreators API for /last30days.
+
+Uses ScrapeCreators REST API to search Twitter/X by keyword.
+Same API key as Reddit, TikTok, and Instagram - one key covers all social sources.
+
+Requires SCRAPECREATORS_API_KEY in config.
+API docs: https://scrapecreators.com/docs
+"""
+
+import sys
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+try:
+    import requests as _requests
+except ImportError:
+    _requests = None
+
+SCRAPECREATORS_BASE = "https://api.scrapecreators.com/v1/twitter"
+
+DEPTH_CONFIG = {
+    "quick":   {"results_per_page": 10},
+    "default": {"results_per_page": 20},
+    "deep":    {"results_per_page": 40},
+}
+
+from .relevance import token_overlap_relevance as _compute_relevance
+
+
+def _extract_core_subject(topic: str) -> str:
+    """Extract core subject from verbose query for Twitter search."""
+    from .query import extract_core_subject
+    _SC_X_NOISE = frozenset({
+        'best', 'top', 'good', 'great', 'awesome',
+        'latest', 'new', 'news', 'update', 'updates',
+        'trending', 'hottest', 'popular', 'viral',
+        'practices', 'features', 'recommendations', 'advice',
+    })
+    return extract_core_subject(topic, noise=_SC_X_NOISE)
+
+
+def _log(msg: str):
+    if sys.stderr.isatty():
+        sys.stderr.write(f"[X/SC] {msg}\n")
+        sys.stderr.flush()
+
+
+def _sc_headers(token: str) -> Dict[str, str]:
+    return {
+        "x-api-key": token,
+        "Content-Type": "application/json",
+    }
+
+
+def _parse_date(item: Dict[str, Any]) -> Optional[str]:
+    """Parse date from ScrapeCreators Twitter item to YYYY-MM-DD."""
+    # Try created_at string (e.g. "Wed Oct 10 20:19:24 +0000 2018")
+    created_at = item.get("created_at")
+    if created_at and isinstance(created_at, str):
+        try:
+            dt = datetime.strptime(created_at, "%a %b %d %H:%M:%S %z %Y")
+            return dt.strftime("%Y-%m-%d")
+        except (ValueError, TypeError):
+            pass
+
+    # Try unix timestamp
+    ts = item.get("timestamp") or item.get("created_at_timestamp")
+    if ts:
+        try:
+            dt = datetime.fromtimestamp(int(ts), tz=timezone.utc)
+            return dt.strftime("%Y-%m-%d")
+        except (ValueError, TypeError, OSError):
+            pass
+
+    # Try ISO format
+    for key in ("created_at", "date"):
+        val = item.get(key)
+        if val and isinstance(val, str):
+            try:
+                dt = datetime.fromisoformat(val.replace("Z", "+00:00"))
+                return dt.strftime("%Y-%m-%d")
+            except (ValueError, TypeError):
+                pass
+
+    return None
+
+
+def search_x(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    token: str = None,
+) -> Dict[str, Any]:
+    """Search X/Twitter via ScrapeCreators API.
+
+    Returns:
+        Dict with 'items' list (in normalize_x_items format) and optional 'error'.
+    """
+    if not token:
+        return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
+
+    if not _requests:
+        return {"items": [], "error": "requests library not installed"}
+
+    config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    core_topic = _extract_core_subject(topic)
+
+    _log(f"Searching X for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
+
+    try:
+        resp = _requests.get(
+            f"{SCRAPECREATORS_BASE}/search/tweets",
+            params={"query": core_topic, "sort_by": "relevance"},
+            headers=_sc_headers(token),
+            timeout=30,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+    except Exception as e:
+        _log(f"ScrapeCreators error: {e}")
+        return {"items": [], "error": f"{type(e).__name__}: {e}"}
+
+    raw_items = data.get("tweets") or data.get("data") or data.get("results") or []
+    raw_items = raw_items[:config["results_per_page"]]
+
+    items = []
+    for i, raw in enumerate(raw_items):
+        tweet_id = str(raw.get("id") or raw.get("tweet_id") or raw.get("id_str") or f"sc-x-{i}")
+        text = raw.get("full_text") or raw.get("text") or ""
+        user = raw.get("user") or raw.get("author") or {}
+        author_handle = user.get("screen_name") or user.get("username") or ""
+
+        # Engagement metrics
+        likes = raw.get("favorite_count") or raw.get("likes") or 0
+        retweets = raw.get("retweet_count") or raw.get("retweets") or 0
+        replies = raw.get("reply_count") or raw.get("replies") or 0
+        quotes = raw.get("quote_count") or raw.get("quotes") or 0
+
+        date_str = _parse_date(raw)
+        relevance = _compute_relevance(core_topic, text)
+
+        url = ""
+        if author_handle and tweet_id and not tweet_id.startswith("sc-x-"):
+            url = f"https://x.com/{author_handle}/status/{tweet_id}"
+
+        items.append({
+            "id": tweet_id,
+            "text": text,
+            "url": url,
+            "author_handle": author_handle,
+            "date": date_str,
+            "engagement": {
+                "likes": likes,
+                "reposts": retweets,
+                "replies": replies,
+                "quotes": quotes,
+            },
+            "relevance": relevance,
+            "why_relevant": f"X: @{author_handle}: {text[:60]}" if text else f"X: {core_topic}",
+        })
+
+    # Date filter
+    in_range = [i for i in items if i["date"] and from_date <= i["date"] <= to_date]
+    out_of_range = len(items) - len(in_range)
+    if in_range:
+        items = in_range
+        if out_of_range:
+            _log(f"Filtered {out_of_range} tweets outside date range")
+    else:
+        _log(f"No tweets within date range, keeping all {len(items)}")
+
+    # Sort by engagement (likes + retweets)
+    items.sort(key=lambda x: (x["engagement"]["likes"] + x["engagement"]["reposts"]), reverse=True)
+
+    _log(f"Found {len(items)} tweets")
+    return {"items": items}
+
+
+def parse_x_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Parse search response to normalized format."""
+    return response.get("items", [])

package/.xtrm/skills/default/last30days/scripts/lib/setup_wizard.py

@@ -0,0 +1,186 @@
+"""First-run setup wizard for last30days.
+
+Detects first run, performs auto-setup (cookie extraction + yt-dlp check),
+and writes configuration. The actual wizard UI is SKILL.md-driven (the LLM
+presents it), but this module provides the detection and setup actions.
+"""
+
+import logging
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def is_first_run(config: Dict[str, Any]) -> bool:
+    """Return True if the setup wizard has not been completed.
+
+    Checks for SETUP_COMPLETE in the config dict. If it's not set
+    (None or empty string), the user hasn't gone through setup yet.
+    """
+    return not config.get("SETUP_COMPLETE")
+
+
+def run_auto_setup(config: Dict[str, Any]) -> Dict[str, Any]:
+    """Perform the auto-setup actions.
+
+    - Runs cookie extraction in auto mode for all registered domains
+    - Checks if yt-dlp is installed
+
+    Returns:
+        Dict with keys:
+          cookies_found: {source_name: browser_name} for each source where cookies were found
+          ytdlp_installed: bool
+          env_written: bool (always False here — caller writes config separately)
+    """
+    from . import cookie_extract
+    from .env import COOKIE_DOMAINS
+
+    cookies_found: Dict[str, str] = {}
+
+    for source_name, spec in COOKIE_DOMAINS.items():
+        domain = spec["domain"]
+        cookie_names = spec["cookies"]
+
+        try:
+            result = cookie_extract.extract_cookies_with_source("auto", domain, cookie_names)
+        except Exception as exc:
+            logger.debug("Cookie extraction failed for %s: %s", source_name, exc)
+            continue
+
+        if result is not None:
+            _cookies, browser_name = result
+            cookies_found[source_name] = browser_name
+
+    # Check yt-dlp availability and install via Homebrew if missing
+    ytdlp_action: str
+    if shutil.which("yt-dlp") is not None:
+        ytdlp_installed = True
+        ytdlp_action = "already_installed"
+    elif shutil.which("brew") is not None:
+        brew_stderr = ""
+        try:
+            proc = subprocess.run(
+                ["brew", "install", "yt-dlp"],
+                capture_output=True, text=True, timeout=120,
+            )
+            if proc.returncode == 0:
+                ytdlp_installed = True
+                ytdlp_action = "installed"
+            else:
+                ytdlp_installed = False
+                ytdlp_action = "install_failed"
+                brew_stderr = proc.stderr
+                logger.warning("brew install yt-dlp failed: %s", proc.stderr)
+        except Exception as exc:
+            ytdlp_installed = False
+            ytdlp_action = "install_failed"
+            brew_stderr = str(exc)
+            logger.warning("brew install yt-dlp exception: %s", exc)
+    else:
+        ytdlp_installed = False
+        ytdlp_action = "no_homebrew"
+
+    results: Dict[str, Any] = {
+        "cookies_found": cookies_found,
+        "ytdlp_installed": ytdlp_installed,
+        "ytdlp_action": ytdlp_action,
+        "env_written": False,
+    }
+    if ytdlp_action == "install_failed":
+        results["ytdlp_stderr"] = brew_stderr
+    return results
+
+
+def write_setup_config(env_path: Path, from_browser: str = "auto") -> bool:
+    """Write SETUP_COMPLETE and FROM_BROWSER to the .env file.
+
+    Creates the file and parent directories if needed.
+    Appends to existing file without overwriting existing keys.
+
+    Args:
+        env_path: Path to the .env file (e.g. ~/.config/last30days/.env)
+        from_browser: Browser extraction mode to write (default: "auto")
+
+    Returns:
+        True if config was written successfully, False on error.
+    """
+    try:
+        env_path = Path(env_path)
+        env_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Read existing content to avoid overwriting keys
+        existing_keys: set = set()
+        existing_content = ""
+        if env_path.exists():
+            existing_content = env_path.read_text(encoding="utf-8")
+            for line in existing_content.splitlines():
+                stripped = line.strip()
+                if stripped and not stripped.startswith("#") and "=" in stripped:
+                    key = stripped.split("=", 1)[0].strip()
+                    existing_keys.add(key)
+
+        lines_to_add = []
+        if "SETUP_COMPLETE" not in existing_keys:
+            lines_to_add.append("SETUP_COMPLETE=true")
+        if "FROM_BROWSER" not in existing_keys:
+            lines_to_add.append(f"FROM_BROWSER={from_browser}")
+
+        if not lines_to_add:
+            return True  # Nothing to write, already configured
+
+        # Ensure trailing newline before appending
+        with open(env_path, "a", encoding="utf-8") as f:
+            if existing_content and not existing_content.endswith("\n"):
+                f.write("\n")
+            f.write("\n".join(lines_to_add) + "\n")
+
+        return True
+
+    except OSError as exc:
+        logger.error("Failed to write setup config to %s: %s", env_path, exc)
+        return False
+
+
+def get_setup_status_text(results: Dict[str, Any]) -> str:
+    """Return a human-readable summary of auto-setup results.
+
+    Args:
+        results: Dict from run_auto_setup()
+
+    Returns:
+        Multi-line status text.
+    """
+    lines = []
+    lines.append("Setup complete! Here's what I found:")
+    lines.append("")
+
+    cookies_found = results.get("cookies_found", {})
+    if cookies_found:
+        for source, browser in cookies_found.items():
+            lines.append(f"  - {source.upper()} cookies found in {browser}")
+    else:
+        lines.append("  - No browser cookies found for X/Twitter")
+
+    ytdlp_action = results.get("ytdlp_action", "")
+    if ytdlp_action == "installed":
+        lines.append("  - Installed yt-dlp via Homebrew")
+    elif ytdlp_action == "install_failed":
+        lines.append("  - yt-dlp install failed \u2014 run `brew install yt-dlp` manually")
+    elif ytdlp_action == "no_homebrew":
+        lines.append("  - yt-dlp not found. Install Homebrew first, then: brew install yt-dlp")
+    elif ytdlp_action == "already_installed":
+        lines.append("  - yt-dlp already installed")
+    elif results.get("ytdlp_installed", False):
+        lines.append("  - yt-dlp is installed (YouTube search ready)")
+    else:
+        lines.append("  - yt-dlp not found (install with: brew install yt-dlp)")
+
+    env_written = results.get("env_written", False)
+    if env_written:
+        lines.append("")
+        lines.append("Configuration saved. Future runs will auto-detect your browsers.")
+
+    return "\n".join(lines)