xtrm-tools 0.7.3 → 0.7.4

package/.xtrm/skills/default/last30days/scripts/lib/exa_search.py

@@ -0,0 +1,176 @@
+"""Exa AI web search for last30days skill.
+
+Uses the Exa Search API as a free web search backend.
+Free tier: 1,000 searches/month, semantic search, no credit card required.
+
+API docs: https://docs.exa.ai/reference/search
+"""
+
+import sys
+from typing import Any, Dict, List, Optional
+from urllib.parse import urlparse
+
+from . import http
+
+ENDPOINT = "https://api.exa.ai/search"
+
+# Domains to exclude (handled by Reddit/X search)
+EXCLUDED_DOMAINS = {
+    "reddit.com", "www.reddit.com", "old.reddit.com",
+    "twitter.com", "www.twitter.com", "x.com", "www.x.com",
+}
+
+
+def search_web(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    api_key: str,
+    depth: str = "default",
+) -> List[Dict[str, Any]]:
+    """Search the web via Exa AI Search API.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        api_key: Exa API key
+        depth: 'quick', 'default', or 'deep'
+
+    Returns:
+        List of result dicts with keys: url, title, snippet, source_domain, date, relevance
+    """
+    num_results = {"quick": 8, "default": 15, "deep": 25}.get(depth, 15)
+    max_chars = {"quick": 1000, "default": 2000, "deep": 3000}.get(depth, 2000)
+
+    payload = {
+        "query": f"{topic} (from {from_date} to {to_date})",
+        "type": "auto",
+        "numResults": num_results,
+        "contents": {"text": {"maxCharacters": max_chars}},
+    }
+
+    # Add date filtering if dates are provided
+    if from_date:
+        payload["startPublishedDate"] = f"{from_date}T00:00:00.000Z"
+    if to_date:
+        payload["endPublishedDate"] = f"{to_date}T23:59:59.999Z"
+
+    sys.stderr.write(f"[Web] Searching Exa for: {topic}\n")
+    sys.stderr.flush()
+
+    try:
+        response = http.post(
+            ENDPOINT,
+            json_data=payload,
+            headers={
+                "x-api-key": api_key,
+            },
+            timeout=20,
+            retries=2,
+        )
+    except http.HTTPError as e:
+        status = e.status_code
+        if status == 401:
+            sys.stderr.write("[Web] Exa: invalid API key (401)\n")
+            sys.stderr.flush()
+            return []
+        if status == 429:
+            sys.stderr.write("[Web] Exa: rate limited (429)\n")
+            sys.stderr.flush()
+            return []
+        sys.stderr.write(f"[Web] Exa: HTTP error {status}: {e}\n")
+        sys.stderr.flush()
+        return []
+    except Exception as e:
+        sys.stderr.write(f"[Web] Exa: request failed: {e}\n")
+        sys.stderr.flush()
+        return []
+
+    return _normalize_results(response)
+
+
+def _normalize_results(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Convert Exa API response to websearch item schema.
+
+    Exa results have: title, url, text, publishedDate, score, author.
+    """
+    items = []
+
+    results = response.get("results", [])
+    if not isinstance(results, list):
+        return items
+
+    for i, result in enumerate(results):
+        if not isinstance(result, dict):
+            continue
+
+        url = result.get("url", "")
+        if not url:
+            continue
+
+        # Skip excluded domains
+        try:
+            domain = urlparse(url).netloc.lower()
+            if domain in EXCLUDED_DOMAINS:
+                continue
+            if domain.startswith("www."):
+                domain = domain[4:]
+        except Exception:
+            domain = ""
+
+        title = str(result.get("title", "")).strip()
+        # Exa returns page content in "text" field
+        snippet = str(result.get("text", "")).strip()
+
+        if not title and not snippet:
+            continue
+
+        # Parse publishedDate (ISO format from Exa: "2026-03-15T00:00:00.000Z")
+        date = _parse_exa_date(result.get("publishedDate"))
+        date_confidence = "med" if date else "low"
+
+        # Exa provides a relevance score
+        relevance = result.get("score", 0.6)
+        try:
+            relevance = min(1.0, max(0.0, float(relevance)))
+        except (TypeError, ValueError):
+            relevance = 0.6
+
+        items.append({
+            "id": f"W{i+1}",
+            "title": title[:200],
+            "url": url,
+            "source_domain": domain,
+            "snippet": snippet[:500],
+            "date": date,
+            "date_confidence": date_confidence,
+            "relevance": relevance,
+            "why_relevant": "",
+        })
+
+    sys.stderr.write(f"[Web] Exa: {len(items)} results\n")
+    sys.stderr.flush()
+
+    return items
+
+
+def _parse_exa_date(published_date: Optional[str]) -> Optional[str]:
+    """Parse Exa's publishedDate to YYYY-MM-DD.
+
+    Exa returns ISO format like "2026-03-15T00:00:00.000Z".
+    """
+    if not published_date:
+        return None
+
+    try:
+        # Extract YYYY-MM-DD from ISO datetime
+        if "T" in published_date:
+            return published_date.split("T")[0]
+        # Already YYYY-MM-DD
+        if len(published_date) >= 10:
+            return published_date[:10]
+    except Exception:
+        pass
+
+    return None

package/.xtrm/skills/default/last30days/scripts/lib/hackernews.py

@@ -0,0 +1,266 @@
+"""Hacker News search via Algolia API (free, no auth required).
+
+Uses hn.algolia.com/api/v1 for story discovery and comment enrichment.
+No API key needed - just HTTP calls via stdlib urllib.
+"""
+
+import html
+import math
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, Dict, List, Optional
+
+from . import http
+from .query import extract_core_subject
+from .relevance import token_overlap_relevance
+
+ALGOLIA_SEARCH_URL = "https://hn.algolia.com/api/v1/search"
+ALGOLIA_SEARCH_BY_DATE_URL = "https://hn.algolia.com/api/v1/search_by_date"
+ALGOLIA_ITEM_URL = "https://hn.algolia.com/api/v1/items"
+
+DEPTH_CONFIG = {
+    "quick": 15,
+    "default": 30,
+    "deep": 60,
+}
+
+ENRICH_LIMITS = {
+    "quick": 3,
+    "default": 5,
+    "deep": 10,
+}
+
+
+def _log(msg: str):
+    """Log to stderr (only in TTY mode to avoid cluttering Claude Code output)."""
+    if sys.stderr.isatty():
+        sys.stderr.write(f"[HN] {msg}\n")
+        sys.stderr.flush()
+
+
+def _date_to_unix(date_str: str) -> int:
+    """Convert YYYY-MM-DD to Unix timestamp (start of day UTC)."""
+    parts = date_str.split("-")
+    year, month, day = int(parts[0]), int(parts[1]), int(parts[2])
+    import calendar
+    import datetime
+    dt = datetime.datetime(year, month, day, tzinfo=datetime.timezone.utc)
+    return int(dt.timestamp())
+
+
+def _unix_to_date(ts: int) -> str:
+    """Convert Unix timestamp to YYYY-MM-DD."""
+    import datetime
+    dt = datetime.datetime.fromtimestamp(ts, tz=datetime.timezone.utc)
+    return dt.strftime("%Y-%m-%d")
+
+
+def _strip_html(text: str) -> str:
+    """Strip HTML tags and decode entities from HN comment text."""
+    import re
+    text = html.unescape(text)
+    text = re.sub(r'<p>', '\n', text)
+    text = re.sub(r'<[^>]+>', '', text)
+    return text.strip()
+
+
+def search_hackernews(
+    topic: str,
+    from_date: str,
+    to_date: str,
+    depth: str = "default",
+) -> Dict[str, Any]:
+    """Search Hacker News via Algolia API.
+
+    Args:
+        topic: Search topic
+        from_date: Start date (YYYY-MM-DD)
+        to_date: End date (YYYY-MM-DD)
+        depth: 'quick', 'default', or 'deep'
+
+    Returns:
+        Dict with Algolia response (contains 'hits' list).
+    """
+    count = DEPTH_CONFIG.get(depth, DEPTH_CONFIG["default"])
+    from_ts = _date_to_unix(from_date)
+    to_ts = _date_to_unix(to_date) + 86400  # Include the end date
+
+    # Use extracted core subject instead of raw topic for cleaner Algolia matching
+    core = extract_core_subject(topic)
+    _log(f"Searching for '{core}' (raw: '{topic}', since {from_date}, count={count})")
+
+    # Use relevance-sorted search with minimum engagement filter.
+    # NOTE: restrictSearchableAttributes=title omitted intentionally — it would
+    # miss Ask HN/Show HN threads where the topic appears in the body.
+    params = {
+        "query": core,
+        "tags": "story",
+        "numericFilters": f"created_at_i>{from_ts},created_at_i<{to_ts},points>2",
+        "hitsPerPage": str(count),
+    }
+
+    from urllib.parse import urlencode
+    url = f"{ALGOLIA_SEARCH_URL}?{urlencode(params)}"
+
+    try:
+        response = http.request("GET", url, timeout=30)
+    except http.HTTPError as e:
+        _log(f"Search failed: {e}")
+        return {"hits": [], "error": str(e)}
+    except Exception as e:
+        _log(f"Search failed: {e}")
+        return {"hits": [], "error": str(e)}
+
+    hits = response.get("hits", [])
+    _log(f"Found {len(hits)} stories")
+    return response
+
+
+def parse_hackernews_response(response: Dict[str, Any], query: str = "") -> List[Dict[str, Any]]:
+    """Parse Algolia response into normalized item dicts.
+
+    Args:
+        response: Algolia search response
+        query: Original search query for token-overlap relevance scoring
+
+    Returns:
+        List of item dicts ready for normalization.
+    """
+    hits = response.get("hits", [])
+    items = []
+
+    for i, hit in enumerate(hits):
+        object_id = hit.get("objectID", "")
+        points = hit.get("points") or 0
+        num_comments = hit.get("num_comments") or 0
+        created_at_i = hit.get("created_at_i")
+
+        date_str = None
+        if created_at_i:
+            date_str = _unix_to_date(created_at_i)
+
+        # Article URL vs HN discussion URL
+        article_url = hit.get("url") or ""
+        hn_url = f"https://news.ycombinator.com/item?id={object_id}"
+
+        # Relevance: blend Algolia rank with token-overlap content matching
+        rank_score = max(0.3, 1.0 - (i * 0.02))  # 1.0 -> 0.3 over 35 items
+        engagement_boost = min(0.2, math.log1p(points) / 40)
+        if query:
+            content_score = token_overlap_relevance(query, hit.get("title", ""))
+            relevance = min(1.0, 0.6 * rank_score + 0.4 * content_score + engagement_boost)
+        else:
+            relevance = min(1.0, rank_score * 0.7 + engagement_boost + 0.1)
+
+        items.append({
+            "object_id": object_id,
+            "title": hit.get("title", ""),
+            "url": article_url,
+            "hn_url": hn_url,
+            "author": hit.get("author", ""),
+            "date": date_str,
+            "engagement": {
+                "points": points,
+                "num_comments": num_comments,
+            },
+            "relevance": round(relevance, 2),
+            "why_relevant": f"HN story about {hit.get('title', 'topic')[:60]}",
+        })
+
+    return items
+
+
+def _fetch_item_comments(object_id: str, max_comments: int = 5) -> Dict[str, Any]:
+    """Fetch top-level comments for a story from Algolia items endpoint.
+
+    Args:
+        object_id: HN story ID
+        max_comments: Max comments to return
+
+    Returns:
+        Dict with 'comments' list and 'comment_insights' list.
+    """
+    url = f"{ALGOLIA_ITEM_URL}/{object_id}"
+
+    try:
+        data = http.request("GET", url, timeout=15)
+    except Exception as e:
+        _log(f"Failed to fetch comments for {object_id}: {e}")
+        return {"comments": [], "comment_insights": []}
+
+    children = data.get("children", [])
+
+    # Sort by points (highest first), filter to actual comments
+    real_comments = [
+        c for c in children
+        if c.get("text") and c.get("author")
+    ]
+    real_comments.sort(key=lambda c: c.get("points") or 0, reverse=True)
+
+    comments = []
+    insights = []
+    for c in real_comments[:max_comments]:
+        text = _strip_html(c.get("text", ""))
+        excerpt = text[:300] + "..." if len(text) > 300 else text
+        comments.append({
+            "author": c.get("author", ""),
+            "text": excerpt,
+            "points": c.get("points") or 0,
+        })
+        # First sentence as insight
+        first_sentence = text.split(". ")[0].split("\n")[0][:200]
+        if first_sentence:
+            insights.append(first_sentence)
+
+    return {"comments": comments, "comment_insights": insights}
+
+
+def enrich_top_stories(
+    items: List[Dict[str, Any]],
+    depth: str = "default",
+) -> List[Dict[str, Any]]:
+    """Fetch comments for top N stories by points.
+
+    Args:
+        items: Parsed HN items
+        depth: Research depth (controls how many to enrich)
+
+    Returns:
+        Items with top_comments and comment_insights added.
+    """
+    if not items:
+        return items
+
+    limit = ENRICH_LIMITS.get(depth, ENRICH_LIMITS["default"])
+
+    # Sort by points to enrich the most popular stories
+    by_points = sorted(
+        range(len(items)),
+        key=lambda i: items[i].get("engagement", {}).get("points", 0),
+        reverse=True,
+    )
+    to_enrich = by_points[:limit]
+
+    _log(f"Enriching top {len(to_enrich)} stories with comments")
+
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        futures = {
+            executor.submit(
+                _fetch_item_comments,
+                items[idx]["object_id"],
+            ): idx
+            for idx in to_enrich
+        }
+
+        for future in as_completed(futures):
+            idx = futures[future]
+            try:
+                result = future.result(timeout=15)
+                items[idx]["top_comments"] = result["comments"]
+                items[idx]["comment_insights"] = result["comment_insights"]
+            except Exception:
+                items[idx]["top_comments"] = []
+                items[idx]["comment_insights"] = []
+
+    return items

package/.xtrm/skills/default/last30days/scripts/lib/http.py

@@ -0,0 +1,174 @@
+"""HTTP utilities for last30days skill (stdlib only)."""
+
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+from typing import Any, Dict, Optional
+from urllib.parse import urlencode
+
+DEFAULT_TIMEOUT = 30
+DEBUG = os.environ.get("LAST30DAYS_DEBUG", "").lower() in ("1", "true", "yes")
+
+
+def log(msg: str):
+    """Log debug message to stderr."""
+    if DEBUG:
+        sys.stderr.write(f"[DEBUG] {msg}\n")
+        sys.stderr.flush()
+MAX_RETRIES = 5
+RETRY_DELAY = 2.0
+USER_AGENT = "last30days-skill/2.1 (Assistant Skill)"
+
+
+class HTTPError(Exception):
+    """HTTP request error with status code."""
+    def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
+        super().__init__(message)
+        self.status_code = status_code
+        self.body = body
+
+
+def request(
+    method: str,
+    url: str,
+    headers: Optional[Dict[str, str]] = None,
+    json_data: Optional[Dict[str, Any]] = None,
+    timeout: int = DEFAULT_TIMEOUT,
+    retries: int = MAX_RETRIES,
+    raw: bool = False,
+) -> Dict[str, Any]:
+    """Make an HTTP request and return JSON response.
+
+    Args:
+        method: HTTP method (GET, POST, etc.)
+        url: Request URL
+        headers: Optional headers dict
+        json_data: Optional JSON body (for POST)
+        timeout: Request timeout in seconds
+        retries: Number of retries on failure
+
+    Returns:
+        Parsed JSON response (or raw text if raw=True)
+
+    Raises:
+        HTTPError: On request failure
+    """
+    headers = headers or {}
+    headers.setdefault("User-Agent", USER_AGENT)
+
+    data = None
+    if json_data is not None:
+        data = json.dumps(json_data).encode('utf-8')
+        headers.setdefault("Content-Type", "application/json")
+
+    req = urllib.request.Request(url, data=data, headers=headers, method=method)
+
+    log(f"{method} {url}")
+
+    last_error = None
+    for attempt in range(retries):
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as response:
+                body = response.read().decode('utf-8')
+                log(f"Response: {response.status} ({len(body)} bytes)")
+                if raw:
+                    return body
+                return json.loads(body) if body else {}
+        except urllib.error.HTTPError as e:
+            body = None
+            try:
+                body = e.read().decode('utf-8')
+            except:
+                pass
+            log(f"HTTP Error {e.code}: {e.reason}")
+            if body:
+                snippet = " ".join(body.split())
+                log(f"Error body: {snippet[:200]}")
+            last_error = HTTPError(f"HTTP {e.code}: {e.reason}", e.code, body)
+
+            # Don't retry client errors (4xx) except rate limits
+            if 400 <= e.code < 500 and e.code != 429:
+                raise last_error
+
+            if attempt < retries - 1:
+                if e.code == 429:
+                    # Respect Retry-After header, fall back to exponential backoff
+                    retry_after = e.headers.get("Retry-After") if hasattr(e, 'headers') else None
+                    if retry_after:
+                        try:
+                            delay = float(retry_after)
+                        except ValueError:
+                            delay = RETRY_DELAY * (2 ** attempt) + 1
+                    else:
+                        delay = RETRY_DELAY * (2 ** attempt) + 1  # 2s, 5s, 9s...
+                    log(f"Rate limited (429). Waiting {delay:.1f}s before retry {attempt + 2}/{retries}")
+                else:
+                    delay = RETRY_DELAY * (2 ** attempt)
+                time.sleep(delay)
+        except urllib.error.URLError as e:
+            log(f"URL Error: {e.reason}")
+            last_error = HTTPError(f"URL Error: {e.reason}")
+            if attempt < retries - 1:
+                time.sleep(RETRY_DELAY * (attempt + 1))
+        except json.JSONDecodeError as e:
+            log(f"JSON decode error: {e}")
+            last_error = HTTPError(f"Invalid JSON response: {e}")
+            raise last_error
+        except (OSError, TimeoutError, ConnectionResetError) as e:
+            # Handle socket-level errors (connection reset, timeout, etc.)
+            log(f"Connection error: {type(e).__name__}: {e}")
+            last_error = HTTPError(f"Connection error: {type(e).__name__}: {e}")
+            if attempt < retries - 1:
+                time.sleep(RETRY_DELAY * (attempt + 1))
+
+    if last_error:
+        raise last_error
+    raise HTTPError("Request failed with no error details")
+
+
+def get(url: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]:
+    """Make a GET request."""
+    return request("GET", url, headers=headers, **kwargs)
+
+
+def post(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]:
+    """Make a POST request with JSON body."""
+    return request("POST", url, headers=headers, json_data=json_data, **kwargs)
+
+
+def post_raw(url: str, json_data: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> str:
+    """Make a POST request with JSON body and return raw text."""
+    return request("POST", url, headers=headers, json_data=json_data, raw=True, **kwargs)
+
+
+def get_reddit_json(path: str, timeout: int = DEFAULT_TIMEOUT, retries: int = MAX_RETRIES) -> Dict[str, Any]:
+    """Fetch Reddit thread JSON.
+
+    Args:
+        path: Reddit path (e.g., /r/subreddit/comments/id/title)
+        timeout: HTTP timeout per attempt in seconds
+        retries: Number of retries on failure
+
+    Returns:
+        Parsed JSON response
+    """
+    # Ensure path starts with /
+    if not path.startswith('/'):
+        path = '/' + path
+
+    # Remove trailing slash and add .json
+    path = path.rstrip('/')
+    if not path.endswith('.json'):
+        path = path + '.json'
+
+    url = f"https://www.reddit.com{path}?raw_json=1"
+
+    headers = {
+        "User-Agent": USER_AGENT,
+        "Accept": "application/json",
+    }
+
+    return get(url, headers=headers, timeout=timeout, retries=retries)