xtrm-tools 0.7.3 → 0.7.7

package/.xtrm/skills/default/last30days/scripts/lib/quality_nudge.py

@@ -0,0 +1,201 @@
+"""Post-research quality score and upgrade nudge.
+
+Computes a quality score based on 5 core sources and builds
+a nudge message describing what the user missed and how to fix it.
+"""
+
+from typing import Any, Dict, List, Optional
+
+
+# The 5 core sources
+CORE_SOURCES = ["hn", "polymarket", "x", "youtube", "reddit_comments"]
+
+# Labels for display
+SOURCE_LABELS = {
+    "hn": "Hacker News",
+    "polymarket": "Polymarket",
+    "x": "X/Twitter",
+    "youtube": "YouTube",
+    "reddit_comments": "Reddit with comments",
+}
+
+
+def _is_x_active(config: dict, research_results: dict) -> bool:
+    """Check if X source is active (has credentials AND didn't error)."""
+    has_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
+    if not has_creds:
+        return False
+    # If X errored this run, it's configured but broken
+    if research_results.get("x_error"):
+        return False
+    return True
+
+
+def _is_youtube_active(config: dict, research_results: dict) -> bool:
+    """Check if YouTube source is active (yt-dlp installed)."""
+    try:
+        from . import youtube_yt
+        has_ytdlp = youtube_yt.is_ytdlp_installed()
+    except Exception:
+        has_ytdlp = False
+    if not has_ytdlp:
+        return False
+    if research_results.get("youtube_error"):
+        return False
+    return True
+
+
+def _is_reddit_comments_active(config: dict, research_results: dict) -> bool:
+    """Check if Reddit with comments is active (ScrapeCreators)."""
+    has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
+    if not has_sc:
+        return False
+    if research_results.get("reddit_error"):
+        return False
+    return True
+
+
+def compute_quality_score(config: dict, research_results: dict) -> dict:
+    """Compute research quality score based on 5 core sources.
+
+    Args:
+        config: Configuration dict from env.get_config()
+        research_results: Dict with keys like x_error, youtube_error,
+            reddit_error reflecting what happened this run.
+
+    Returns:
+        {
+            "score_pct": 40-100,
+            "core_active": ["hn", "polymarket", ...],
+            "core_missing": ["x", "youtube", "reddit_comments"],
+            "core_errored": ["reddit_comments"],  # configured but errored
+            "nudge_text": "..." or None if 100%
+        }
+    """
+    core_active: List[str] = []
+    core_missing: List[str] = []
+    core_errored: List[str] = []
+
+    # HN and Polymarket are always active
+    core_active.append("hn")
+    core_active.append("polymarket")
+
+    # X
+    has_x_creds = bool(config.get("AUTH_TOKEN") or config.get("XAI_API_KEY"))
+    if _is_x_active(config, research_results):
+        core_active.append("x")
+    else:
+        core_missing.append("x")
+        if has_x_creds and research_results.get("x_error"):
+            core_errored.append("x")
+
+    # YouTube
+    try:
+        from . import youtube_yt
+        has_ytdlp = youtube_yt.is_ytdlp_installed()
+    except Exception:
+        has_ytdlp = False
+    if _is_youtube_active(config, research_results):
+        core_active.append("youtube")
+    else:
+        core_missing.append("youtube")
+        if has_ytdlp and research_results.get("youtube_error"):
+            core_errored.append("youtube")
+
+    # Reddit with comments (ScrapeCreators)
+    has_sc = bool(config.get("SCRAPECREATORS_API_KEY"))
+    if _is_reddit_comments_active(config, research_results):
+        core_active.append("reddit_comments")
+    else:
+        core_missing.append("reddit_comments")
+        if has_sc and research_results.get("reddit_error"):
+            core_errored.append("reddit_comments")
+
+    score_pct = int(len(core_active) / 5 * 100)
+
+    nudge_text = _build_nudge_text(core_missing, core_errored) if core_missing else None
+
+    return {
+        "score_pct": score_pct,
+        "core_active": core_active,
+        "core_missing": core_missing,
+        "core_errored": core_errored,
+        "nudge_text": nudge_text,
+    }
+
+
+def _build_nudge_text(core_missing: List[str], core_errored: List[str]) -> str:
+    """Build human-readable nudge text describing what was missed.
+
+    Prioritizes free suggestions before paid ones.
+    """
+    lines: List[str] = []
+
+    # Describe what was missed
+    missed_parts: List[str] = []
+    for src in core_missing:
+        label = SOURCE_LABELS[src]
+        if src in core_errored:
+            missed_parts.append(f"{label} (errored this run)")
+        else:
+            missed_parts.append(label)
+
+    active_count = 5 - len(core_missing)
+    lines.append(f"Research quality: {active_count}/5 core sources.")
+    lines.append(f"Missing: {', '.join(missed_parts)}.")
+    lines.append("")
+
+    # Free suggestions first
+    free_suggestions: List[str] = []
+    paid_suggestions: List[str] = []
+
+    if "x" in core_missing:
+        if "x" in core_errored:
+            free_suggestions.append(
+                "X errored — try refreshing your browser cookies "
+                "(log into x.com, then re-run)."
+            )
+        else:
+            free_suggestions.append(
+                "X/Twitter: scan browser cookies automatically — "
+                "just log into x.com in any browser and re-run."
+            )
+
+    if "youtube" in core_missing:
+        if "youtube" in core_errored:
+            free_suggestions.append(
+                "YouTube errored — check that yt-dlp is up to date: "
+                "brew upgrade yt-dlp"
+            )
+        else:
+            free_suggestions.append(
+                "YouTube: install yt-dlp — brew install yt-dlp"
+            )
+
+    if "reddit_comments" in core_missing:
+        if "reddit_comments" in core_errored:
+            paid_suggestions.append(
+                "Reddit comments errored — check your ScrapeCreators API key "
+                "at scrapecreators.com."
+            )
+        else:
+            paid_suggestions.append(
+                "Reddit with comments: add SCRAPECREATORS_API_KEY — "
+                "100 free API calls, no credit card — scrapecreators.com"
+            )
+
+    if free_suggestions:
+        lines.append("Free fixes:")
+        for s in free_suggestions:
+            lines.append(f"  - {s}")
+        lines.append("")
+
+    if paid_suggestions:
+        lines.append("Paid options:")
+        for s in paid_suggestions:
+            lines.append(f"  - {s}")
+        lines.append("")
+
+    lines.append("last30days has no affiliation with any API provider.")
+
+    return "\n".join(lines)

package/.xtrm/skills/default/last30days/scripts/lib/query.py

@@ -0,0 +1,117 @@
+"""Shared query preprocessing utilities: noise-word stripping, core subject
+extraction, and compound term detection. Used by all search modules."""
+
+import re
+from typing import FrozenSet, List, Optional, Set
+
+# Common multi-word prefixes stripped from all queries (identical across modules)
+PREFIXES = [
+    'what are the best', 'what is the best', 'what are the latest',
+    'what are people saying about', 'what do people think about',
+    'how do i use', 'how to use', 'how to',
+    'what are', 'what is', 'tips for', 'best practices for',
+]
+
+# Multi-word suffixes (used by bird_x)
+SUFFIXES = [
+    'best practices', 'use cases', 'prompt techniques',
+    'prompting techniques', 'prompting tips',
+]
+
+# Base noise words shared across most modules
+NOISE_WORDS = frozenset({
+    # Articles/prepositions/conjunctions
+    'a', 'an', 'the', 'is', 'are', 'was', 'were', 'and', 'or',
+    'of', 'in', 'on', 'for', 'with', 'about', 'to',
+    # Question words
+    'how', 'what', 'which', 'who', 'why', 'when', 'where',
+    'does', 'should', 'could', 'would',
+    # Research/meta descriptors
+    'best', 'top', 'good', 'great', 'awesome', 'killer',
+    'latest', 'new', 'news', 'update', 'updates',
+    'trendiest', 'trending', 'hottest', 'hot', 'popular', 'viral',
+    'practices', 'features', 'guide', 'tutorial',
+    'recommendations', 'advice', 'review', 'reviews',
+    'usecases', 'examples', 'comparison', 'versus', 'vs',
+    'plugin', 'plugins', 'skill', 'skills', 'tool', 'tools',
+    # Prompting meta words
+    'prompt', 'prompts', 'prompting', 'techniques', 'tips',
+    'tricks', 'methods', 'strategies', 'approaches',
+    # Action words
+    'using', 'uses', 'use',
+    # Misc filler
+    'people', 'saying', 'think', 'said', 'lately',
+})
+
+
+def extract_core_subject(
+    topic: str,
+    *,
+    noise: Optional[FrozenSet[str]] = None,
+    max_words: Optional[int] = None,
+    strip_suffixes: bool = False,
+) -> str:
+    """Extract core subject from a verbose search query.
+
+    Strips common question/meta prefixes and noise words to produce a
+    compact search-friendly query. Platforms customize via parameters.
+
+    Args:
+        topic: Raw user query
+        noise: Override noise word set (default: NOISE_WORDS)
+        max_words: Cap result to N words (default: no cap)
+        strip_suffixes: Also strip trailing multi-word suffixes (bird_x uses this)
+
+    Returns:
+        Cleaned query string
+    """
+    text = topic.lower().strip()
+    if not text:
+        return text
+
+    # Phase 1: Strip multi-word prefixes (longest first, stop after first match)
+    for p in PREFIXES:
+        if text.startswith(p + ' '):
+            text = text[len(p):].strip()
+            break
+
+    # Phase 2: Strip multi-word suffixes (opt-in)
+    if strip_suffixes:
+        for s in SUFFIXES:
+            if text.endswith(' ' + s):
+                text = text[:-len(s)].strip()
+                break
+
+    # Phase 3: Filter individual noise words
+    noise_set = noise if noise is not None else NOISE_WORDS
+    words = text.split()
+    filtered = [w for w in words if w not in noise_set]
+
+    # Apply word cap if requested
+    if max_words is not None and filtered:
+        filtered = filtered[:max_words]
+
+    result = ' '.join(filtered) if filtered else text
+    return result.rstrip('?!.') if not max_words else (result or topic.lower().strip())
+
+
+def extract_compound_terms(topic: str) -> List[str]:
+    """Detect multi-word terms that should be quoted in search queries.
+
+    Identifies:
+    - Hyphenated terms: "multi-agent", "vc-backed"
+    - Title-cased multi-word names: "Claude Code", "React Native"
+
+    Returns list of terms suitable for quoting (e.g., '"multi-agent"').
+    """
+    terms: List[str] = []
+
+    # Hyphenated terms
+    for match in re.finditer(r'\b\w+-\w+(?:-\w+)*\b', topic):
+        terms.append(match.group())
+
+    # Title-cased sequences (2+ capitalized words in a row)
+    for match in re.finditer(r'(?:[A-Z][a-z]+\s+){1,}[A-Z][a-z]+', topic):
+        terms.append(match.group())
+
+    return terms

package/.xtrm/skills/default/last30days/scripts/lib/query_type.py

@@ -0,0 +1,111 @@
+"""Query type detection for source selection and scoring adjustments."""
+
+import re
+from typing import Literal
+
+QueryType = Literal["product", "concept", "opinion", "how_to", "comparison", "breaking_news", "prediction"]
+
+# Pattern-based classification (no LLM, no external deps)
+_PRODUCT_PATTERNS = re.compile(
+    r"\b(price|pricing|cost|buy|purchase|deal|discount|subscription|plan|tier|free tier|alternative|prompt|prompts|prompting|template|templates)\b", re.I
+)
+_CONCEPT_PATTERNS = re.compile(
+    r"\b(what is|what are|explain|definition|how does|how do|overview|introduction|guide to|primer)\b", re.I
+)
+_OPINION_PATTERNS = re.compile(
+    r"\b(worth it|thoughts on|opinion|review|experience with|recommend|should i|pros and cons|good or bad)\b", re.I
+)
+_HOWTO_PATTERNS = re.compile(
+    r"\b(how to|tutorial|step by step|setup|install|configure|deploy|migrate|implement|build a|create a|prompting|prompts?|best practices|tips|examples|animation|animations|video workflow|render pipeline)\b",
+    re.I,
+)
+_COMPARISON_PATTERNS = re.compile(
+    r"\b(vs\.?|versus|compared to|comparison|better than|difference between|switch from)\b", re.I
+)
+_BREAKING_PATTERNS = re.compile(
+    r"\b(latest|breaking|just announced|launched|released|new|update|news|happened|today|this week)\b", re.I
+)
+_PREDICTION_PATTERNS = re.compile(
+    r"\b(predict|forecast|odds|chance|probability|election|outcome|bet on|market for)\b", re.I
+)
+
+
+def detect_query_type(topic: str) -> QueryType:
+    """Classify a query into a type using pattern matching.
+
+    Returns the first match in priority order:
+    comparison > how_to > product > opinion > prediction > concept > breaking_news.
+    """
+    # Most specific first
+    if _COMPARISON_PATTERNS.search(topic):
+        return "comparison"
+    if _HOWTO_PATTERNS.search(topic):
+        return "how_to"
+    if _PRODUCT_PATTERNS.search(topic):
+        return "product"
+    if _OPINION_PATTERNS.search(topic):
+        return "opinion"
+    if _PREDICTION_PATTERNS.search(topic):
+        return "prediction"
+    if _CONCEPT_PATTERNS.search(topic):
+        return "concept"
+    if _BREAKING_PATTERNS.search(topic):
+        return "breaking_news"
+
+    # Default: treat as breaking news (most common use case for "last 30 days")
+    return "breaking_news"
+
+
+# Source tiering by query type.
+# Tier 1: always run. Tier 2: run if available. Tier 3: opt-in only.
+# Sources not listed are implicitly tier 3 (opt-in).
+SOURCE_TIERS = {
+    "product":       {"tier1": {"reddit", "x", "youtube"}, "tier2": {"web", "tiktok"}},
+    "concept":       {"tier1": {"reddit", "hn", "web"},     "tier2": {"youtube", "x"}},
+    "opinion":       {"tier1": {"reddit", "x"},             "tier2": {"youtube", "bluesky"}},
+    "how_to":        {"tier1": {"youtube", "reddit", "hn"}, "tier2": {"web", "x"}},
+    "comparison":    {"tier1": {"reddit", "hn", "youtube"}, "tier2": {"x", "web"}},
+    "breaking_news": {"tier1": {"x", "reddit", "web"},      "tier2": {"hn", "bluesky", "youtube"}},
+    "prediction":    {"tier1": {"polymarket", "x", "reddit"}, "tier2": {"web", "hn", "youtube"}},
+}
+
+# WebSearch penalty adjustment by query type.
+# Points subtracted from websearch score (0-100 scale). 0 = no penalty, 15 = full penalty.
+# Concept/how_to queries benefit from authoritative web sources.
+WEBSEARCH_PENALTY_BY_TYPE = {
+    "product": 15,        # default: social discussion > blog posts
+    "concept": 0,         # web docs are the best source
+    "opinion": 15,        # social discussion > blog posts
+    "how_to": 5,          # tutorials on web are valuable
+    "comparison": 10,     # mix of social and web
+    "breaking_news": 10,  # news sites are valuable
+    "prediction": 15,     # social/market data > web articles
+}
+
+# Tiebreaker priority overrides by query type.
+# Maps source type name to priority (lower = higher priority).
+TIEBREAKER_BY_TYPE = {
+    "product":       {"reddit": 0, "x": 1, "youtube": 2, "tiktok": 3, "instagram": 4, "hn": 5, "web": 6, "polymarket": 7},
+    "concept":       {"hn": 0, "reddit": 1, "web": 2, "youtube": 3, "x": 4, "tiktok": 5, "instagram": 6, "polymarket": 7},
+    "opinion":       {"reddit": 0, "x": 1, "bluesky": 2, "youtube": 3, "hn": 4, "tiktok": 5, "web": 6, "polymarket": 7},
+    "how_to":        {"youtube": 0, "reddit": 1, "hn": 2, "web": 3, "x": 4, "tiktok": 5, "instagram": 6, "polymarket": 7},
+    "comparison":    {"reddit": 0, "hn": 1, "youtube": 2, "x": 3, "web": 4, "tiktok": 5, "instagram": 6, "polymarket": 7},
+    "breaking_news": {"x": 0, "reddit": 1, "web": 2, "hn": 3, "bluesky": 4, "tiktok": 5, "youtube": 6, "polymarket": 7},
+    "prediction":    {"polymarket": 0, "x": 1, "reddit": 2, "web": 3, "hn": 4, "bluesky": 5, "youtube": 6, "tiktok": 7},
+}
+
+
+def is_source_enabled(source: str, query_type: QueryType, explicitly_requested: bool = False) -> bool:
+    """Check if a source should run for a given query type.
+
+    Tier 1 and Tier 2 sources are enabled. Tier 3 (unlisted) sources only run
+    if explicitly requested via --search flag. Truth Social is always opt-in.
+    """
+    if source == "truthsocial":
+        return explicitly_requested
+
+    if explicitly_requested:
+        return True
+
+    tiers = SOURCE_TIERS.get(query_type, SOURCE_TIERS["breaking_news"])
+    return source in tiers["tier1"] or source in tiers["tier2"]