xtrm-tools 0.7.3 → 0.7.4

package/.xtrm/skills/default/last30days/scripts/lib/instagram.py

@@ -0,0 +1,365 @@
+"""Instagram Reels search via ScrapeCreators API for /last30days.
+
+Uses ScrapeCreators REST API to search Instagram Reels by keyword, extract
+engagement metrics (views, likes, comments), and fetch video transcripts.
+
+Requires SCRAPECREATORS_API_KEY in config. 100 free API calls, then PAYG.
+API docs: https://scrapecreators.com/docs
+"""
+
+import re
+import sys
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional, Set
+
+try:
+    import requests as _requests
+except ImportError:
+    _requests = None
+
+from . import http
+
+SCRAPECREATORS_BASE = "https://api.scrapecreators.com"
+
+# Depth configurations: how many results to fetch / captions to extract
+DEPTH_CONFIG = {
+    "quick":   {"results_per_page": 10, "max_captions": 3},
+    "default": {"results_per_page": 20, "max_captions": 5},
+    "deep":    {"results_per_page": 40, "max_captions": 8},
+}
+
+# Max words to keep from each caption
+CAPTION_MAX_WORDS = 500
+
+from .relevance import token_overlap_relevance as _compute_relevance
+
+
+def _extract_core_subject(topic: str) -> str:
+    """Extract core subject from verbose query for Instagram search."""
+    from .query import extract_core_subject
+    _INSTAGRAM_NOISE = frozenset({
+        'best', 'top', 'good', 'great', 'awesome', 'killer',
+        'latest', 'new', 'news', 'update', 'updates',
+        'trending', 'hottest', 'popular', 'viral',
+        'practices', 'features',
+        'recommendations', 'advice',
+        'prompt', 'prompts', 'prompting',
+        'methods', 'strategies', 'approaches',
+    })
+    return extract_core_subject(topic, noise=_INSTAGRAM_NOISE)
+
+
+def _log(msg: str):
+    """Log to stderr (only in interactive terminals; spinner handles non-TTY)."""
+    if sys.stderr.isatty():
+        sys.stderr.write(f"[Instagram] {msg}\n")
+        sys.stderr.flush()
+
+
+def _sc_headers(token: str) -> Dict[str, str]:
+    """Build ScrapeCreators request headers."""
+    return {
+        "x-api-key": token,
+        "Content-Type": "application/json",
+    }
+
+
+def _parse_date(item: Dict[str, Any]) -> Optional[str]:
+    """Parse date from ScrapeCreators Instagram item to YYYY-MM-DD.
+
+    Handles taken_at as ISO string (e.g. "2026-02-26T16:00:00.000Z")
+    or unix timestamp.
+    """
+    ts = item.get("taken_at")
+    if not ts:
+        return None
+
+    # Try ISO string first (ScrapeCreators reels/search returns this)
+    if isinstance(ts, str):
+        try:
+            # Handle "2026-02-26T16:00:00.000Z" format
+            dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
+            return dt.strftime("%Y-%m-%d")
+        except (ValueError, TypeError):
+            pass
+        # Try just the date portion
+        if len(ts) >= 10:
+            return ts[:10]
+
+    # Fall back to unix timestamp
+    try:
+        dt = datetime.fromtimestamp(int(ts), tz=timezone.utc)
+        return dt.strftime("%Y-%m-%d")
+    except (ValueError, TypeError, OSError):
+        pass
+
+    return None
+
+
+def _extract_hashtags(caption_text: str) -> List[str]:
+    """Extract hashtags from Instagram caption text."""
+    if not caption_text:
+        return []
+    return re.findall(r'#(\w+)', caption_text)
+
+
+def search_instagram(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    token: str = None,
+) -> Dict[str, Any]:
+    """Search Instagram Reels via ScrapeCreators API.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: 'quick', 'default', or 'deep'
+        token: ScrapeCreators API key
+
+    Returns:
+        Dict with 'items' list and optional 'error'.
+    """
+    if not token:
+        return {"items": [], "error": "No SCRAPECREATORS_API_KEY configured"}
+
+    config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    core_topic = _extract_core_subject(topic)
+
+    _log(f"Searching Instagram for '{core_topic}' (depth={depth}, count={config['results_per_page']})")
+
+    if not _requests:
+        _log("requests library not installed, falling back to urllib")
+        try:
+            from urllib.parse import urlencode
+            params = urlencode({"query": core_topic})
+            url = f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search?{params}"
+            headers = _sc_headers(token)
+            headers["User-Agent"] = http.USER_AGENT
+            data = http.get(url, headers=headers, timeout=30, retries=2)
+        except Exception as e:
+            _log(f"ScrapeCreators error (urllib): {e}")
+            return {"items": [], "error": f"{type(e).__name__}: {e}"}
+    else:
+        try:
+            resp = _requests.get(
+                f"{SCRAPECREATORS_BASE}/v2/instagram/reels/search",
+                params={"query": core_topic},
+                headers=_sc_headers(token),
+                timeout=30,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except Exception as e:
+            _log(f"ScrapeCreators error: {e}")
+            return {"items": [], "error": f"{type(e).__name__}: {e}"}
+
+    # Items are in the 'reels' array (ScrapeCreators v2 response)
+    raw_items = data.get("reels") or data.get("items") or data.get("data") or []
+
+    # Limit to configured count
+    raw_items = raw_items[:config["results_per_page"]]
+
+    # Parse items
+    items = []
+    for raw in raw_items:
+        if not isinstance(raw, dict):
+            continue
+
+        # Extract reel ID and shortcode
+        reel_pk = str(raw.get("id", raw.get("pk", "")))
+        shortcode = raw.get("shortcode", raw.get("code", ""))
+
+        # Caption text — can be a string or dict depending on endpoint
+        caption_obj = raw.get("caption", "")
+        if isinstance(caption_obj, dict):
+            text = caption_obj.get("text", "")
+        elif isinstance(caption_obj, str):
+            text = caption_obj
+        else:
+            text = raw.get("desc", raw.get("text", ""))
+
+        # Engagement metrics
+        play_count = raw.get("video_play_count") or raw.get("video_view_count") or raw.get("play_count") or 0
+        like_count = raw.get("like_count") or 0
+        comment_count = raw.get("comment_count") or 0
+
+        # Author info — 'owner' in reels/search, 'user' in user/reels
+        owner = raw.get("owner") or raw.get("user") or {}
+        author_name = owner.get("username", "")
+
+        # Duration
+        duration = raw.get("video_duration")
+
+        # Date
+        date_str = _parse_date(raw)
+
+        # Hashtags from caption text
+        hashtags = _extract_hashtags(text)
+
+        # Compute relevance with hashtag boost
+        relevance = _compute_relevance(core_topic, text, hashtags)
+
+        # Build URL — prefer API-provided url, fallback to shortcode
+        url = raw.get("url", "")
+        if not url and shortcode:
+            url = f"https://www.instagram.com/reel/{shortcode}"
+
+        items.append({
+            "video_id": reel_pk,
+            "text": text,
+            "url": url,
+            "author_name": author_name,
+            "date": date_str,
+            "engagement": {
+                "views": play_count,
+                "likes": like_count,
+                "comments": comment_count,
+            },
+            "hashtags": hashtags,
+            "duration": duration,
+            "relevance": relevance,
+            "why_relevant": f"Instagram: {text[:60]}" if text else f"Instagram: {core_topic}",
+            "caption_snippet": "",  # populated by fetch_captions
+        })
+
+    # Hard date filter
+    in_range = [i for i in items if i["date"] and from_date <= i["date"] <= to_date]
+    out_of_range = len(items) - len(in_range)
+    if in_range:
+        items = in_range
+        if out_of_range:
+            _log(f"Filtered {out_of_range} reels outside date range")
+    else:
+        _log(f"No reels within date range, keeping all {len(items)}")
+
+    # Sort by views descending
+    items.sort(key=lambda x: x["engagement"]["views"], reverse=True)
+
+    _log(f"Found {len(items)} Instagram reels")
+    return {"items": items}
+
+
+def fetch_captions(
+    video_items: List[Dict[str, Any]],
+    token: str,
+    depth: str = "default",
+) -> Dict[str, str]:
+    """Fetch transcripts for top N Instagram reels via ScrapeCreators.
+
+    Strategy:
+    1. Use the 'text' field (caption) as baseline
+    2. For top N, call /v2/instagram/media/transcript for spoken-word captions
+
+    Args:
+        video_items: Items from search_instagram()
+        token: ScrapeCreators API key
+        depth: Depth level for caption limit
+
+    Returns:
+        Dict mapping video_id -> caption text (truncated to 500 words)
+    """
+    config = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    max_captions = config["max_captions"]
+
+    if not video_items or not token or not _requests:
+        return {}
+
+    top_items = video_items[:max_captions]
+    _log(f"Enriching captions for {len(top_items)} reels")
+
+    captions = {}
+
+    # First pass: use text field as caption (always available, free)
+    for item in top_items:
+        vid = item["video_id"]
+        text = item.get("text", "")
+        if text:
+            words = text.split()
+            if len(words) > CAPTION_MAX_WORDS:
+                text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
+            captions[vid] = text
+
+    # Second pass: try to get spoken-word transcripts (1 credit each)
+    for item in top_items:
+        vid = item["video_id"]
+        url = item.get("url", "")
+        if not url:
+            continue
+        try:
+            resp = _requests.get(
+                f"{SCRAPECREATORS_BASE}/v2/instagram/media/transcript",
+                params={"url": url},
+                headers=_sc_headers(token),
+                timeout=15,
+            )
+            if resp.status_code == 200:
+                data = resp.json()
+                transcripts = data.get("transcripts") or []
+                if transcripts and isinstance(transcripts, list):
+                    # Combine all transcript segments
+                    transcript_text = " ".join(
+                        t.get("text", "") for t in transcripts
+                        if isinstance(t, dict) and t.get("text")
+                    )
+                    if transcript_text:
+                        words = transcript_text.split()
+                        if len(words) > CAPTION_MAX_WORDS:
+                            transcript_text = ' '.join(words[:CAPTION_MAX_WORDS]) + '...'
+                        captions[vid] = transcript_text
+        except Exception as e:
+            _log(f"Transcript fetch failed for {vid}: {e}")
+
+    got = sum(1 for v in captions.values() if v)
+    _log(f"Got captions for {got}/{len(top_items)} reels")
+    return captions
+
+
+def search_and_enrich(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+    token: str = None,
+) -> Dict[str, Any]:
+    """Full Instagram search: find reels, then fetch captions for top results.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: 'quick', 'default', or 'deep'
+        token: ScrapeCreators API key
+
+    Returns:
+        Dict with 'items' list. Each item has a 'caption_snippet' field.
+    """
+    # Step 1: Search
+    search_result = search_instagram(topic, from_date, to_date, depth, token)
+    items = search_result.get("items", [])
+
+    if not items:
+        return search_result
+
+    # Step 2: Fetch captions for top N
+    captions = fetch_captions(items, token, depth)
+
+    # Step 3: Attach captions to items
+    for item in items:
+        vid = item["video_id"]
+        caption = captions.get(vid)
+        if caption:
+            item["caption_snippet"] = caption
+
+    return {"items": items, "error": search_result.get("error")}
+
+
+def parse_instagram_response(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Parse Instagram search response to normalized format.
+
+    Returns:
+        List of item dicts ready for normalization.
+    """
+    return response.get("items", [])

package/.xtrm/skills/default/last30days/scripts/lib/models.py

@@ -0,0 +1,221 @@
+"""Model auto-selection for last30days skill.
+
+Model selection philosophy: this tool uses LLM APIs exclusively for
+search tool invocation + structured JSON extraction. This is not
+reasoning-heavy or creative work — mini models handle it equally well
+at ~3-5x lower cost. We prefer the newest-generation mini model, falling
+back to mainline only when mini isn't available.
+
+xAI non-reasoning variant preferred: same pricing as reasoning, but
+faster (skips thinking phase, saves reasoning token output costs).
+"""
+
+import re
+import sys
+from typing import Dict, List, Optional, Tuple
+
+from . import cache, http, env
+
+# OpenAI API
+OPENAI_MODELS_URL = "https://api.openai.com/v1/models"
+# Ordered by cost-efficiency for web_search + JSON extraction tasks.
+# Mini models first: same structured extraction quality at ~3x lower cost.
+OPENAI_FALLBACK_MODELS = ["gpt-5-mini", "gpt-4.1-mini", "gpt-4.1", "gpt-4o"]
+CODEX_FALLBACK_MODELS = ["gpt-5.1-codex-mini", "gpt-5.2"]
+
+# xAI API - Agent Tools API requires grok-4 family
+# Non-reasoning: same price, faster, no unnecessary thinking tokens.
+# Both variants support function calling and structured outputs.
+XAI_MODELS_URL = "https://api.x.ai/v1/models"
+XAI_ALIASES = {
+    "latest": "grok-4-1-fast-non-reasoning",
+    "stable": "grok-4-1-fast-non-reasoning",
+}
+
+
+def parse_version(model_id: str) -> Optional[Tuple[int, ...]]:
+    """Parse semantic version from model ID.
+
+    Examples:
+        gpt-5 -> (5,)
+        gpt-5.2 -> (5, 2)
+        gpt-5.2.1 -> (5, 2, 1)
+    """
+    match = re.search(r'(\d+(?:\.\d+)*)', model_id)
+    if match:
+        return tuple(int(x) for x in match.group(1).split('.'))
+    return None
+
+
+def is_search_capable_model(model_id: str) -> bool:
+    """Check if model supports Responses API web_search with domain filtering.
+
+    Includes mini variants (same structured extraction quality, lower cost).
+    Excludes: nano (no web_search), gpt-4o-mini (no domain filtering),
+    chat/codex/pro/preview/turbo/search (specialized variants).
+
+    Note: gpt-5 with reasoning effort="minimal" does NOT support web_search
+    (per OpenAI docs). We never set reasoning params — our usage is pure
+    tool invocation + JSON extraction — so gpt-5 is safe to include here.
+    """
+    model_lower = model_id.lower()
+
+    # gpt-4o-mini does NOT support web_search with filters — exclude it
+    if model_lower.startswith("gpt-4o-mini"):
+        return False
+
+    # Must be gpt-4o, gpt-4.1[-mini], or gpt-5[-mini] series
+    if not re.match(r'^gpt-(?:4o|4\.1|5)(\.\d+)*(-mini)?$', model_lower):
+        return False
+
+    # Exclude unsupported variants
+    for exc in ['nano', 'chat', 'codex', 'pro', 'preview', 'turbo', 'search']:
+        if exc in model_lower:
+            return False
+
+    return True
+
+
+# Backward compat alias
+is_mainline_openai_model = is_search_capable_model
+
+
+def select_openai_model(
+    api_key: str,
+    policy: str = "auto",
+    pin: Optional[str] = None,
+    mock_models: Optional[List[Dict]] = None,
+) -> str:
+    """Select the most cost-efficient OpenAI model for web_search + JSON extraction.
+
+    Prefers mini models within the newest generation available, since the task
+    is structured extraction (not reasoning or creative work).
+
+    Args:
+        api_key: OpenAI API key
+        policy: 'auto' or 'pinned'
+        pin: Model to use if policy is 'pinned'
+        mock_models: Mock model list for testing
+
+    Returns:
+        Selected model ID
+    """
+    if policy == "pinned" and pin:
+        return pin
+
+    # Check cache first
+    cached = cache.get_cached_model("openai")
+    if cached:
+        return cached
+
+    # Fetch model list
+    if mock_models is not None:
+        models = mock_models
+    else:
+        try:
+            headers = {"Authorization": f"Bearer {api_key}"}
+            response = http.get(OPENAI_MODELS_URL, headers=headers)
+            models = response.get("data", [])
+        except http.HTTPError as e:
+            sys.stderr.write(f"[Models] Failed to fetch OpenAI models: {e}")
+            if hasattr(e, 'status_code') and e.status_code in (401, 403):
+                sys.stderr.write(" — API key may be invalid or lack permissions")
+            sys.stderr.write(f", using fallback {OPENAI_FALLBACK_MODELS[0]}\n")
+            return OPENAI_FALLBACK_MODELS[0]
+
+    candidates = [m for m in models if is_search_capable_model(m.get("id", ""))]
+
+    if not candidates:
+        return OPENAI_FALLBACK_MODELS[0]
+
+    # Sort: newest generation first, prefer mini within same generation
+    def sort_key(m):
+        model_id = m.get("id", "")
+        version = parse_version(model_id) or (0,)
+        major = version[0] if version else 0
+        is_mini = 1 if "mini" in model_id.lower() else 0
+        return (major, is_mini, version)
+
+    candidates.sort(key=sort_key, reverse=True)
+    selected = candidates[0]["id"]
+
+    cache.set_cached_model("openai", selected)
+
+    return selected
+
+
+def select_xai_model(
+    api_key: str,
+    policy: str = "latest",
+    pin: Optional[str] = None,
+    mock_models: Optional[List[Dict]] = None,
+) -> str:
+    """Select the best xAI model based on policy.
+
+    Args:
+        api_key: xAI API key
+        policy: 'latest', 'stable', or 'pinned'
+        pin: Model to use if policy is 'pinned'
+        mock_models: Mock model list for testing
+
+    Returns:
+        Selected model ID
+    """
+    if policy == "pinned" and pin:
+        return pin
+
+    # Use alias system
+    if policy in XAI_ALIASES:
+        alias = XAI_ALIASES[policy]
+
+        # Check cache first
+        cached = cache.get_cached_model("xai")
+        if cached:
+            return cached
+
+        # Cache the alias
+        cache.set_cached_model("xai", alias)
+        return alias
+
+    # Default to latest
+    return XAI_ALIASES["latest"]
+
+
+def get_models(
+    config: Dict,
+    mock_openai_models: Optional[List[Dict]] = None,
+    mock_xai_models: Optional[List[Dict]] = None,
+) -> Dict[str, Optional[str]]:
+    """Get selected models for both providers.
+
+    Returns:
+        Dict with 'openai' and 'xai' keys
+    """
+    result = {"openai": None, "xai": None}
+
+    if config.get("OPENAI_API_KEY"):
+        if config.get("OPENAI_AUTH_SOURCE") == env.AUTH_SOURCE_CODEX:
+            # Codex auth doesn't use the OpenAI models list endpoint
+            policy = config.get("OPENAI_MODEL_POLICY", "auto")
+            pin = config.get("OPENAI_MODEL_PIN")
+            if policy == "pinned" and pin:
+                result["openai"] = pin
+            else:
+                result["openai"] = CODEX_FALLBACK_MODELS[0]
+        else:
+            result["openai"] = select_openai_model(
+                config["OPENAI_API_KEY"],
+                config.get("OPENAI_MODEL_POLICY", "auto"),
+                config.get("OPENAI_MODEL_PIN"),
+                mock_openai_models,
+            )
+
+    if config.get("XAI_API_KEY"):
+        result["xai"] = select_xai_model(
+            config["XAI_API_KEY"],
+            config.get("XAI_MODEL_POLICY", "latest"),
+            config.get("XAI_MODEL_PIN"),
+            mock_xai_models,
+        )
+
+    return result