xtrm-tools 0.7.2 → 0.7.4

package/.xtrm/skills/default/last30days/scripts/lib/reddit_enrich.py

@@ -0,0 +1,325 @@
+"""Reddit thread enrichment with real engagement metrics.
+
+Supports two backends:
+1. ScrapeCreators API (preferred) - no rate limits, 1 credit/call
+2. reddit.com/.json (fallback) - free but 429-prone
+"""
+
+import re
+from typing import Any, Dict, List, Optional
+from urllib.parse import urlparse
+
+from . import http, dates
+
+
+def extract_reddit_path(url: str) -> Optional[str]:
+    """Extract the path from a Reddit URL.
+
+    Args:
+        url: Reddit URL
+
+    Returns:
+        Path component or None
+    """
+    try:
+        parsed = urlparse(url)
+        if "reddit.com" not in parsed.netloc:
+            return None
+        return parsed.path
+    except:
+        return None
+
+
+class RedditRateLimitError(Exception):
+    """Raised when Reddit returns HTTP 429 (rate limited)."""
+    pass
+
+
+def fetch_thread_data(
+    url: str,
+    mock_data: Optional[Dict] = None,
+    timeout: int = 30,
+    retries: int = 3,
+) -> Optional[Dict[str, Any]]:
+    """Fetch Reddit thread JSON data.
+
+    Args:
+        url: Reddit thread URL
+        mock_data: Mock data for testing
+        timeout: HTTP timeout per attempt in seconds
+        retries: Number of retries on failure
+
+    Returns:
+        Thread data dict or None on failure
+
+    Raises:
+        RedditRateLimitError: When Reddit returns 429 (caller should bail)
+    """
+    if mock_data is not None:
+        return mock_data
+
+    path = extract_reddit_path(url)
+    if not path:
+        return None
+
+    try:
+        data = http.get_reddit_json(path, timeout=timeout, retries=retries)
+        return data
+    except http.HTTPError as e:
+        if e.status_code == 429:
+            raise RedditRateLimitError(f"Reddit rate limited (429) fetching {url}") from e
+        return None
+
+
+def parse_thread_data(data: Any) -> Dict[str, Any]:
+    """Parse Reddit thread JSON into structured data.
+
+    Args:
+        data: Raw Reddit JSON response
+
+    Returns:
+        Dict with submission and comments data
+    """
+    result = {
+        "submission": None,
+        "comments": [],
+    }
+
+    if not isinstance(data, list) or len(data) < 1:
+        return result
+
+    # First element is submission listing
+    submission_listing = data[0]
+    if isinstance(submission_listing, dict):
+        children = submission_listing.get("data", {}).get("children", [])
+        if children:
+            sub_data = children[0].get("data", {})
+            result["submission"] = {
+                "score": sub_data.get("score"),
+                "num_comments": sub_data.get("num_comments"),
+                "upvote_ratio": sub_data.get("upvote_ratio"),
+                "created_utc": sub_data.get("created_utc"),
+                "permalink": sub_data.get("permalink"),
+                "title": sub_data.get("title"),
+                "selftext": sub_data.get("selftext", "")[:500],  # Truncate
+            }
+
+    # Second element is comments listing
+    if len(data) >= 2:
+        comments_listing = data[1]
+        if isinstance(comments_listing, dict):
+            children = comments_listing.get("data", {}).get("children", [])
+            for child in children:
+                if child.get("kind") != "t1":  # t1 = comment
+                    continue
+                c_data = child.get("data", {})
+                if not c_data.get("body"):
+                    continue
+
+                comment = {
+                    "score": c_data.get("score", 0),
+                    "created_utc": c_data.get("created_utc"),
+                    "author": c_data.get("author", "[deleted]"),
+                    "body": c_data.get("body", "")[:300],  # Truncate
+                    "permalink": c_data.get("permalink"),
+                }
+                result["comments"].append(comment)
+
+    return result
+
+
+def get_top_comments(comments: List[Dict], limit: int = 10) -> List[Dict[str, Any]]:
+    """Get top comments sorted by score.
+
+    Args:
+        comments: List of comment dicts
+        limit: Maximum number to return
+
+    Returns:
+        Top comments sorted by score
+    """
+    # Filter out deleted/removed
+    valid = [c for c in comments if c.get("author") not in ("[deleted]", "[removed]")]
+
+    # Sort by score descending
+    sorted_comments = sorted(valid, key=lambda c: c.get("score", 0), reverse=True)
+
+    return sorted_comments[:limit]
+
+
+def extract_comment_insights(comments: List[Dict], limit: int = 7) -> List[str]:
+    """Extract key insights from top comments.
+
+    Uses simple heuristics to identify valuable comments:
+    - Has substantive text
+    - Contains actionable information
+    - Not just agreement/disagreement
+
+    Args:
+        comments: Top comments
+        limit: Max insights to extract
+
+    Returns:
+        List of insight strings
+    """
+    insights = []
+
+    for comment in comments[:limit * 2]:  # Look at more comments than we need
+        body = comment.get("body", "").strip()
+        if not body or len(body) < 30:
+            continue
+
+        # Skip low-value patterns
+        skip_patterns = [
+            r'^(this|same|agreed|exactly|yep|nope|yes|no|thanks|thank you)\.?$',
+            r'^lol|lmao|haha',
+            r'^\[deleted\]',
+            r'^\[removed\]',
+        ]
+        if any(re.match(p, body.lower()) for p in skip_patterns):
+            continue
+
+        # Truncate to first meaningful sentence or ~150 chars
+        insight = body[:150]
+        if len(body) > 150:
+            # Try to find a sentence boundary
+            for i, char in enumerate(insight):
+                if char in '.!?' and i > 50:
+                    insight = insight[:i+1]
+                    break
+            else:
+                insight = insight.rstrip() + "..."
+
+        insights.append(insight)
+        if len(insights) >= limit:
+            break
+
+    return insights
+
+
+def enrich_reddit_item(
+    item: Dict[str, Any],
+    mock_thread_data: Optional[Dict] = None,
+    timeout: int = 10,
+    retries: int = 1,
+) -> Dict[str, Any]:
+    """Enrich a Reddit item with real engagement data.
+
+    Args:
+        item: Reddit item dict
+        mock_thread_data: Mock data for testing
+        timeout: HTTP timeout per attempt (default 10s for enrichment)
+        retries: Number of retries (default 1 — fail fast for enrichment)
+
+    Returns:
+        Enriched item dict
+
+    Raises:
+        RedditRateLimitError: Propagated so caller can bail on remaining items
+    """
+    url = item.get("url", "")
+
+    # Fetch thread data (RedditRateLimitError propagates to caller)
+    thread_data = fetch_thread_data(url, mock_thread_data, timeout=timeout, retries=retries)
+    if not thread_data:
+        return item
+
+    parsed = parse_thread_data(thread_data)
+    submission = parsed.get("submission")
+    comments = parsed.get("comments", [])
+
+    # Update engagement metrics
+    if submission:
+        item["engagement"] = {
+            "score": submission.get("score"),
+            "num_comments": submission.get("num_comments"),
+            "upvote_ratio": submission.get("upvote_ratio"),
+        }
+
+        # Update date from actual data
+        created_utc = submission.get("created_utc")
+        if created_utc:
+            item["date"] = dates.timestamp_to_date(created_utc)
+
+    # Get top comments
+    top_comments = get_top_comments(comments)
+    item["top_comments"] = []
+    for c in top_comments:
+        permalink = c.get("permalink", "")
+        comment_url = f"https://reddit.com{permalink}" if permalink else ""
+        item["top_comments"].append({
+            "score": c.get("score", 0),
+            "date": dates.timestamp_to_date(c.get("created_utc")),
+            "author": c.get("author", ""),
+            "excerpt": c.get("body", "")[:200],
+            "url": comment_url,
+        })
+
+    # Extract insights
+    item["comment_insights"] = extract_comment_insights(top_comments)
+
+    return item
+
+
+def enrich_reddit_item_sc(
+    item: Dict[str, Any],
+    token: str,
+    timeout: int = 30,
+) -> Dict[str, Any]:
+    """Enrich a Reddit item using ScrapeCreators comment API.
+
+    No rate limit risk. Uses 1 credit per call.
+
+    Args:
+        item: Reddit item dict (already has engagement from search)
+        token: ScrapeCreators API key
+        timeout: HTTP timeout
+
+    Returns:
+        Enriched item with top_comments and comment_insights
+    """
+    from . import reddit as reddit_mod
+
+    url = item.get("url", "")
+    if not url:
+        return item
+
+    raw_comments = reddit_mod.fetch_post_comments(url, token)
+    if not raw_comments:
+        return item
+
+    top_comments = []
+    for c in raw_comments[:10]:
+        body = c.get("body", "")
+        if not body or body in ("[deleted]", "[removed]"):
+            continue
+
+        score = c.get("ups") or c.get("score", 0)
+        author = c.get("author", "[deleted]")
+        permalink = c.get("permalink", "")
+        comment_url = f"https://reddit.com{permalink}" if permalink else ""
+
+        top_comments.append({
+            "score": score,
+            "date": dates.timestamp_to_date(c.get("created_utc")) if c.get("created_utc") else None,
+            "author": author,
+            "body": body[:300],
+            "excerpt": body[:200],
+            "url": comment_url,
+        })
+
+    top_comments.sort(key=lambda c: c.get("score", 0), reverse=True)
+
+    item["top_comments"] = []
+    for c in top_comments:
+        item["top_comments"].append({
+            "score": c.get("score", 0),
+            "date": c.get("date"),
+            "author": c.get("author", ""),
+            "excerpt": c.get("excerpt", ""),
+            "url": c.get("url", ""),
+        })
+
+    item["comment_insights"] = extract_comment_insights(top_comments)
+
+    return item

package/.xtrm/skills/default/last30days/scripts/lib/reddit_public.py

@@ -0,0 +1,259 @@
+"""Standalone Reddit public JSON search module.
+
+Searches Reddit using the free public JSON endpoints (no API key required).
+Promoted from last-resort fallback to robust primary free path.
+
+Endpoints:
+- Global: https://www.reddit.com/search.json?q={query}&sort=relevance&t=month&limit={limit}
+- Subreddit: https://www.reddit.com/r/{sub}/search.json?q={query}&restrict_sr=on&sort=relevance&t=month
+
+Handles 429 rate limits with exponential backoff, HTML anti-bot responses,
+network timeouts, and missing subreddits.
+"""
+
+import json
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any, Dict, List, Optional
+
+
+USER_AGENT = "last30days/3.0 (research tool)"
+
+# Depth-aware limits for thread counts
+DEPTH_LIMITS = {
+    "quick": 10,
+    "default": 25,
+    "deep": 50,
+}
+
+MAX_RETRIES = 3
+BASE_BACKOFF = 2.0  # seconds
+
+
+def _log(msg: str):
+    """Log to stderr."""
+    sys.stderr.write(f"[RedditPublic] {msg}\n")
+    sys.stderr.flush()
+
+
+def _url_encode(text: str) -> str:
+    """URL-encode a query string."""
+    return urllib.parse.quote_plus(text)
+
+
+def _fetch_json(url: str, timeout: int = 15) -> Optional[Dict[str, Any]]:
+    """Fetch JSON from a URL with retry on 429 and error handling.
+
+    Returns parsed JSON dict, or None on unrecoverable failure.
+    """
+    headers = {
+        "User-Agent": USER_AGENT,
+        "Accept": "application/json",
+    }
+    req = urllib.request.Request(url, headers=headers)
+
+    for attempt in range(MAX_RETRIES):
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                content_type = resp.headers.get("Content-Type", "")
+                if "json" not in content_type and "text/html" in content_type:
+                    _log(f"Anti-bot HTML response (Content-Type: {content_type})")
+                    return None
+
+                body = resp.read().decode("utf-8")
+                return json.loads(body)
+
+        except urllib.error.HTTPError as e:
+            if e.code == 429:
+                delay = BASE_BACKOFF * (2 ** attempt)
+                retry_after = None
+                if hasattr(e, "headers"):
+                    retry_after = e.headers.get("Retry-After")
+                if retry_after:
+                    try:
+                        delay = float(retry_after)
+                    except ValueError:
+                        pass
+                _log(f"429 rate limited, retry {attempt + 1}/{MAX_RETRIES} after {delay:.1f}s")
+                if attempt < MAX_RETRIES - 1:
+                    time.sleep(delay)
+                    continue
+                # Last attempt exhausted
+                _log("429 retries exhausted")
+                return None
+            elif e.code == 404:
+                _log(f"404 not found: {url}")
+                return None
+            elif e.code == 403:
+                _log(f"403 forbidden: {url}")
+                return None
+            else:
+                _log(f"HTTP {e.code}: {e.reason}")
+                return None
+
+        except (urllib.error.URLError, OSError, TimeoutError) as e:
+            _log(f"Network error: {e}")
+            return None
+
+        except json.JSONDecodeError as e:
+            _log(f"JSON decode error: {e}")
+            return None
+
+    return None
+
+
+def _parse_posts(data: Optional[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Parse Reddit listing JSON into normalized post dicts."""
+    if not data:
+        return []
+
+    children = data.get("data", {}).get("children", [])
+    posts = []
+
+    for child in children:
+        if child.get("kind") != "t3":
+            continue
+        post = child.get("data", {})
+        permalink = str(post.get("permalink", "")).strip()
+        if not permalink or "/comments/" not in permalink:
+            continue
+
+        score = int(post.get("score", 0) or 0)
+        num_comments = int(post.get("num_comments", 0) or 0)
+        selftext = str(post.get("selftext", ""))
+        author = str(post.get("author", "[deleted]"))
+        created_utc = post.get("created_utc")
+
+        # Parse date
+        date_str = None
+        if created_utc:
+            try:
+                from datetime import datetime, timezone
+                dt = datetime.fromtimestamp(float(created_utc), tz=timezone.utc)
+                date_str = dt.strftime("%Y-%m-%d")
+            except (ValueError, TypeError, OSError):
+                pass
+
+        posts.append({
+            "id": "",  # Will be assigned after dedup
+            "title": str(post.get("title", "")).strip(),
+            "url": f"https://www.reddit.com{permalink}",
+            "score": score,
+            "num_comments": num_comments,
+            "subreddit": str(post.get("subreddit", "")).strip(),
+            "created_utc": float(created_utc) if created_utc else None,
+            "author": author if author not in ("[deleted]", "[removed]") else "[deleted]",
+            "selftext": selftext[:500] if selftext else "",
+            # Normalized fields matching ScrapeCreators output
+            "date": date_str,
+            "engagement": {
+                "score": score,
+                "num_comments": num_comments,
+                "upvote_ratio": post.get("upvote_ratio"),
+            },
+            "relevance": _compute_relevance(score, num_comments),
+            "why_relevant": "Reddit public search",
+        })
+
+    return posts
+
+
+def _compute_relevance(score: int, num_comments: int) -> float:
+    """Estimate relevance from engagement signals."""
+    score_component = min(1.0, max(0.0, score / 500.0))
+    comments_component = min(1.0, max(0.0, num_comments / 200.0))
+    return round((score_component * 0.6) + (comments_component * 0.4), 3)
+
+
+def search(
+    query: str,
+    depth: str = "default",
+    subreddit: Optional[str] = None,
+    timeout: int = 15,
+) -> List[Dict[str, Any]]:
+    """Search Reddit via the public JSON endpoint.
+
+    Args:
+        query: Search query string
+        depth: 'quick', 'default', or 'deep' — controls result limit
+        subreddit: Optional subreddit name (without r/) for scoped search
+        timeout: HTTP timeout in seconds
+
+    Returns:
+        List of normalized post dicts. Empty list on any failure.
+    """
+    limit = DEPTH_LIMITS.get(depth, DEPTH_LIMITS["default"])
+    encoded_query = _url_encode(query)
+
+    if subreddit:
+        sub = subreddit.lstrip("r/").strip()
+        url = (
+            f"https://www.reddit.com/r/{sub}/search.json"
+            f"?q={encoded_query}&restrict_sr=on&sort=relevance&t=month&limit={limit}&raw_json=1"
+        )
+    else:
+        url = (
+            f"https://www.reddit.com/search.json"
+            f"?q={encoded_query}&sort=relevance&t=month&limit={limit}&raw_json=1"
+        )
+
+    data = _fetch_json(url, timeout=timeout)
+    posts = _parse_posts(data)
+
+    # Dedupe by URL and assign IDs
+    seen_urls = set()
+    unique = []
+    for post in posts:
+        if post["url"] not in seen_urls:
+            seen_urls.add(post["url"])
+            unique.append(post)
+
+    for i, post in enumerate(unique):
+        post["id"] = f"R{i + 1}"
+
+    return unique[:limit]
+
+
+def search_reddit_public(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+) -> List[Dict[str, Any]]:
+    """High-level Reddit public search matching the openai_reddit interface.
+
+    Runs global search, deduplicates, filters by date range, and sorts
+    by engagement. Compatible as a drop-in replacement in the fallback chain.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: 'quick', 'default', or 'deep'
+
+    Returns:
+        List of normalized item dicts matching ScrapeCreators output format.
+    """
+    results = search(topic, depth=depth)
+
+    # Date filter: keep posts in range or with unknown dates
+    filtered = []
+    for item in results:
+        d = item.get("date")
+        if d is None or (from_date <= d <= to_date):
+            filtered.append(item)
+
+    # Sort by engagement (score desc)
+    filtered.sort(
+        key=lambda x: x.get("engagement", {}).get("score", 0),
+        reverse=True,
+    )
+
+    # Re-index IDs
+    for i, item in enumerate(filtered):
+        item["id"] = f"R{i + 1}"
+
+    return filtered