trendcite 0.1.0__py3-none-any.whl

trendcite/__init__.py

@@ -0,0 +1,4 @@
+"""TrendCite: evidence-first trend intelligence for founders."""
+
+__version__ = "0.1.0"
+__all__ = ["__version__"]

trendcite/__main__.py

@@ -0,0 +1,3 @@
+from trendcite.cli import main
+
+raise SystemExit(main())

trendcite/briefs.py

@@ -0,0 +1,234 @@
+"""Turn scored clusters into Content Opportunity Briefs (deterministic templates).
+
+Everything in a brief except the draft outline is derived from captured evidence
+and computed scores. The outline is a writing aid and is labelled as such.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from .models import Brief, EvidenceItem, TopicCluster
+from .scoring import HALF_LIFE_HOURS, WEIGHTS, age_hours, select_evidence
+from .signal import SignalBrief
+from .text import CRITICAL_RE
+
+SOURCE_NAMES = {
+    "hackernews": "Hacker News",
+    "github": "GitHub",
+    "rss": "RSS/Atom",
+    "reddit": "Reddit",
+    "x": "X",
+}
+
+
+def metric_summary(item: EvidenceItem) -> str:
+    m = item.metrics
+    if item.source == "hackernews":
+        return f"{int(m.get('points', 0))} points, {int(m.get('comments', 0))} comments"
+    if item.source == "github":
+        return f"{int(m.get('stars', 0))} stars, {int(m.get('forks', 0))} forks"
+    return "no engagement metrics available from this source"
+
+
+def _source_list(sources: list[str]) -> str:
+    return ", ".join(SOURCE_NAMES.get(s, s) for s in sources)
+
+
+def _angle(cluster: TopicCluster) -> str:
+    topic = cluster.label
+    assert cluster.score is not None
+    s = cluster.score
+    sources = set(s.sources)
+    if "github" in sources and sources & {"hackernews", "reddit"}:
+        return (
+            f"Builders are already shipping around {topic} while the discussion is still "
+            f"unsettled: share what actually works (and what doesn't) from first-hand use."
+        )
+    if "reddit" in sources and s.engagement < 0.6:
+        return (
+            f"Practitioners are asking open questions about {topic}; answer them with your "
+            f"own operating numbers and decisions rather than a summary of the debate."
+        )
+    if s.engagement >= 0.6 and s.corroboration >= 0.5:
+        return (
+            f"{topic} is drawing strong attention across communities; the useful angle is "
+            f"what it changes for a small team this quarter."
+        )
+    return (
+        f"{topic} is being covered by publishers; add a first-hand or contrarian take "
+        f"that the existing coverage lacks."
+    )
+
+
+def _why_now(
+    cluster: TopicCluster,
+    evidence: list[EvidenceItem],
+    now: datetime,
+    percentiles: dict[str, float],
+) -> list[str]:
+    assert cluster.score is not None
+    s = cluster.score
+    items = evidence
+    ages = [age_hours(i, now) for i in items]
+    lines = [
+        f"{len(items)} evidence item(s) shown and scored: {s.unique_urls} unique URL(s) from "
+        f"{len(s.sources)} source(s) ({_source_list(list(s.sources))}); newest "
+        f"{min(ages):.0f} h old, oldest {max(ages):.0f} h old."
+    ]
+    if len(cluster.items) > len(items):
+        lines.append(
+            f"{len(cluster.items)} items matched this topic in total; the rest are not "
+            f"shown and do not affect the score."
+        )
+    measured = [i for i in items if i.item_id in percentiles]
+    if measured:
+        top = max(measured, key=lambda i: (percentiles[i.item_id], i.item_id))
+        lines.append(
+            f'Strongest engagement signal: "{top.title}" '
+            f"({SOURCE_NAMES.get(top.source, top.source)}: "
+            f"{metric_summary(top)}; engagement percentile {percentiles[top.item_id] * 100:.0f} "
+            f"within that source this run)."
+        )
+    if cluster.niche_matches:
+        lines.append("Matches your niche terms: " + ", ".join(cluster.niche_matches) + ".")
+    if cluster.related_terms:
+        lines.append("Co-occurring terms: " + ", ".join(cluster.related_terms) + ".")
+    return lines
+
+
+def _score_explanation(cluster: TopicCluster) -> list[str]:
+    assert cluster.score is not None
+    s = cluster.score
+    w = WEIGHTS
+    rows = [
+        (
+            "recency",
+            w.recency,
+            s.recency,
+            f"{HALF_LIFE_HOURS:.0f} h half-life, averaged over items",
+        ),
+        ("engagement", w.engagement, s.engagement, "top-3 within-source percentiles"),
+        (
+            "corroboration",
+            w.corroboration,
+            s.corroboration,
+            f"{s.independent_sources} independent source(s): sources that each link a "
+            f"different URL",
+        ),
+        (
+            "relevance",
+            w.relevance,
+            s.relevance,
+            f"{len(cluster.niche_matches)} niche term(s) matched in the evidence",
+        ),
+        ("diversity", w.diversity, s.diversity, f"{s.publishers} distinct publisher(s)"),
+    ]
+    lines = [
+        f"{name}: {value:.2f} x {weight:.2f} = {100 * value * weight:.1f} pts ({why})"
+        for name, weight, value, why in rows
+    ]
+    lines.append(f"total: {s.total:.1f}/100, confidence {s.confidence}")
+    lines.append(
+        "all components are computed from the evidence items listed in this brief; high "
+        "confidence requires >= 3 independent sources, >= 4 unique URLs, >= 3 publishers, "
+        "a niche match (when a niche is set) and evidence that agrees on more than one word"
+    )
+    return lines
+
+
+def _counterpoints(cluster: TopicCluster, evidence: list[EvidenceItem], now: datetime) -> list[str]:
+    assert cluster.score is not None
+    s = cluster.score
+    notes: list[str] = []
+    if s.independent_sources <= 1:
+        where = _source_list(list(s.sources))
+        if len(s.sources) > 1:
+            notes.append(
+                f"Seen via {where}, but they link the same underlying URL; that is one "
+                f"story mirrored across channels, not independent corroboration."
+            )
+        else:
+            notes.append(
+                f"Only seen on {where}; this may be one community's echo chamber rather "
+                f"than a broad trend."
+            )
+    mirrored = len(evidence) - s.unique_urls
+    if mirrored > 0 and s.independent_sources > 1:
+        notes.append(
+            f"{mirrored} evidence item(s) repeat a URL already shown via another channel; "
+            f"repeats are counted once for corroboration."
+        )
+    if s.unique_urls <= 2:
+        notes.append(
+            f"Small sample ({s.unique_urls} unique URL(s)). Treat as an early signal, not a trend."
+        )
+    if not any(i.engagement() is not None for i in evidence):
+        notes.append("No engagement metrics are available for these sources; reach is unknown.")
+    stale = [i for i in evidence if age_hours(i, now) > 7 * 24]
+    if stale:
+        notes.append(f"{len(stale)} item(s) are over a week old; part of this signal is not new.")
+    critical = [i for i in evidence if CRITICAL_RE.search(i.title)]
+    if critical:
+        notes.append(
+            f'Part of the evidence is critical or cautionary (e.g. "{critical[0].title}"); '
+            f"address the downside explicitly instead of only the upside."
+        )
+    notes.append(
+        "Clustering is keyword-based: confirm the linked items really discuss the same thing "
+        "before relying on the corroboration count."
+    )
+    return notes
+
+
+def _questions(topic: str) -> list[str]:
+    return [
+        f"What have you personally shipped, broken or decided about {topic} in the last 90 days?",
+        "Where does this evidence disagree with what you hear from your own customers?",
+        f"For a 3-person team, is {topic} a 'do now', 'watch' or 'ignore' this quarter, and why?",
+        "Which number from your own business could you share to make the point concrete?",
+    ]
+
+
+def _outline(cluster: TopicCluster, evidence: list[EvidenceItem]) -> list[str]:
+    first = evidence[0]
+    return [
+        f'Hook: open with the concrete signal from evidence [1] ("{first.title}").',
+        "Context: summarise what the linked evidence shows, citing items by number.",
+        "Your POV: answer the first founder question with a specific story or decision.",
+        "Counterpoint: name the strongest objection listed above and respond to it honestly.",
+        f"Takeaway: one practical recommendation about {cluster.label} for small teams.",
+    ]
+
+
+def build_brief(
+    cluster: TopicCluster,
+    rank: int,
+    now: datetime,
+    percentiles: dict[str, float],
+    signal: SignalBrief | None = None,
+) -> Brief:
+    """Project one scored cluster (and, when available, its signal) into a public brief."""
+    assert cluster.score is not None
+    evidence = cluster.evidence or select_evidence(cluster.items, percentiles, now)
+    flags: list[str] = []
+    flagged = [i for i in evidence if i.flags]
+    if flagged:
+        flags.append(
+            f"{len(flagged)} evidence item(s) contain text that looks like instructions to an AI "
+            f"system (possible prompt injection). Shown as inert data only; nothing was executed."
+        )
+    return Brief(
+        rank=rank,
+        topic=cluster.label,
+        angle=_angle(cluster),
+        why_now=_why_now(cluster, evidence, now, percentiles),
+        evidence=evidence,
+        score=cluster.score,
+        score_explanation=_score_explanation(cluster),
+        counterpoints=_counterpoints(cluster, evidence, now),
+        founder_questions=_questions(cluster.label),
+        outline=_outline(cluster, evidence),
+        flags=flags,
+        signal=signal,
+    )

trendcite/cli.py

@@ -0,0 +1,128 @@
+"""Command-line interface: ``trendcite demo | live | sources``."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from . import __version__
+from .config import Config, ConfigError, load_config
+from .llm import LLMError, enhance_briefs, provider_from_env
+from .models import Report
+from .pipeline import make_adapters, run_demo, run_live
+from .render import to_json, to_markdown
+from .security import configure_logging, redact
+
+
+def _split(value: str | None) -> list[str] | None:
+    if value is None:
+        return None
+    return [v.strip() for v in value.split(",") if v.strip()]
+
+
+def _add_common(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--format", choices=["markdown", "json"], default="markdown")
+    p.add_argument("--out", type=Path, help="write the report to this file instead of stdout")
+    p.add_argument("--top", type=int, default=None, help="number of briefs, clamped to 3-5")
+    p.add_argument("--niche", help="comma-separated niche phrases, e.g. 'ai agents,pricing'")
+    p.add_argument(
+        "--llm",
+        action="store_true",
+        help="refine angles/outlines with the provider set in TRENDCITE_LLM_PROVIDER",
+    )
+    p.add_argument("-v", "--verbose", action="store_true")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="trendcite",
+        description="Evidence-first trend intelligence: traceable content opportunity briefs.",
+    )
+    parser.add_argument("--version", action="version", version=f"trendcite {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    demo = sub.add_parser("demo", help="offline demo on bundled synthetic fixtures (no network)")
+    _add_common(demo)
+
+    live = sub.add_parser("live", help="collect from public read-only sources and build briefs")
+    _add_common(live)
+    live.add_argument("--config", type=Path, help="path to trendcite.toml")
+    live.add_argument("--sources", help="comma-separated: hackernews,github,rss,reddit,x")
+    live.add_argument("--feeds", help="comma-separated RSS/Atom URLs (replaces configured feeds)")
+    live.add_argument("--subreddits", help="comma-separated subreddit names")
+    live.add_argument("--github-queries", help="comma-separated GitHub search queries")
+
+    sub.add_parser("sources", help="list source adapters and their status requirements")
+    return parser
+
+
+def _emit(text: str, out: Path | None) -> None:
+    if out is None:
+        sys.stdout.write(text)
+        return
+    out.write_text(text, encoding="utf-8")
+    sys.stderr.write(f"trendcite: wrote {out}\n")
+
+
+def _maybe_llm(report: Report, enabled: bool) -> None:
+    if not enabled:
+        return
+    try:
+        provider = provider_from_env()
+    except LLMError as exc:
+        report.notes.append(f"LLM synthesis disabled: {redact(str(exc))}")
+        return
+    if provider is None:
+        report.notes.append(
+            "--llm given but TRENDCITE_LLM_PROVIDER is not set; deterministic output only."
+        )
+        return
+    report.notes.extend(enhance_briefs(report.briefs, provider))
+
+
+def _live_config(args: argparse.Namespace) -> Config:
+    cfg = load_config(args.config)
+    for attr, value in (
+        ("sources", _split(args.sources)),
+        ("feeds", _split(args.feeds)),
+        ("subreddits", _split(args.subreddits)),
+        ("github_queries", _split(args.github_queries)),
+        ("niche", _split(args.niche)),
+    ):
+        if value is not None:
+            setattr(cfg, attr, value)
+    if args.top is not None:
+        cfg.top = args.top
+    return cfg
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)
+        if reconfigure is not None:
+            reconfigure(encoding="utf-8", errors="replace")
+    args = build_parser().parse_args(argv)
+    configure_logging(getattr(args, "verbose", False))
+    try:
+        if args.command == "sources":
+            for adapter in make_adapters(
+                Config(sources=["hackernews", "github", "rss", "reddit", "x"])
+            ):
+                print(f"{adapter.name:<11} {adapter.description}")
+            return 0
+        if args.command == "demo":
+            report = run_demo(top=args.top or 5, niche=_split(args.niche))
+        else:
+            report = run_live(_live_config(args))
+        _maybe_llm(report, args.llm)
+        rendered = to_json(report) if args.format == "json" else to_markdown(report)
+        _emit(rendered, args.out)
+        if args.command == "live" and not any(s.ok for s in report.source_status):
+            sys.stderr.write("trendcite: no source was reachable; see the source table.\n")
+            return 2
+        return 0
+    except (ConfigError, ValueError) as exc:
+        sys.stderr.write(f"trendcite: error: {redact(str(exc))}\n")
+        return 2

trendcite/cluster.py

@@ -0,0 +1,244 @@
+"""Deterministic topic clustering by shared key terms, with a coherence gate.
+
+Algorithm (greedy, fully deterministic):
+
+1. Extract unigram and bigram terms from each item's *feature view* (title + excerpt
+   head with URLs, domains and feed boilerplate removed; see ``text.item_text``).
+2. A term is a *seed candidate* if it is not generic or boilerplate and occurs in items
+   with >= ``min_items`` distinct underlying URLs (``EvidenceItem.story_key``).
+3. Coherence gate. The items a seed may claim depend on how distinctive the seed is:
+
+   - phrase (two words, at least one not generic): every item containing it. A shared
+     multi-word phrase is itself strong lexical agreement.
+   - specific word (not in the common-English reference lexicon ``lexicon.py``, e.g.
+     "mcp", "kubernetes"): every item containing it if they are cohesive
+     (``scoring.evidence_is_cohesive``: every URL shares a further specific term or
+     phrase with more than half of the other URLs).
+   - otherwise (a common word such as "decision", "agent", "memory", or a specific
+     word that failed the cohesion check): only the items that also share the seed's
+     best co-occurring *agreement terms* (``text.agreement_terms``). Uniform
+     principle: members must share terms worth >= 2 in total, where a phrase counts
+     2, a specific word 1 and a common word 0 (``text.specificity``). So "gpt" +
+     "astra" is enough, while "agent" needs a shared phrase or two more specific
+     words. The label names the extra terms. Inflected forms are checked against the
+     lexicon too ("testing" counts as "test").
+   - last resort, specific word from a single independent source: all its items form
+     a *candidate* cluster that claims no corroboration. The pipeline only publishes
+     clusters whose evidence is cohesive, so such a candidate is reported as
+     suppressed rather than turned into a brief.
+
+   The claimed set must still cover >= ``min_items`` distinct underlying URLs, so the
+   same link mirrored through two channels can never form a cluster by itself.
+4. Repeatedly pick the candidate whose claimed set has the most independent sources
+   (sources that each contribute a different underlying URL), then the most unique
+   URLs, then the most items, preferring phrases, then seeds that needed no extra
+   term, then alphabetical order.
+5. Stop when no candidate qualifies.
+
+Tradeoffs: the gate trades recall for precision. Items that share only a common word
+plus one other word are never clustered, even when they happen to be related; a
+specific word with two senses can still merge unrelated items when they also share a
+second specific term. Every brief keeps a counterpoint telling the reader to check
+the evidence.
+
+Items that never join a cluster are not turned into briefs: a single uncorroborated
+item is not treated as a trend.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from itertools import combinations
+
+from .models import EvidenceItem, TopicCluster
+from .scoring import evidence_is_cohesive, independent_sources
+from .text import (
+    STOPWORDS,
+    agreement_terms,
+    extract_terms,
+    is_seed_candidate,
+    is_specific,
+    item_text,
+    specificity,
+    tokenize,
+)
+
+DISPLAY_WORDS = {
+    "ai": "AI",
+    "mcp": "MCP",
+    "ci": "CI",
+    "crdt": "CRDT",
+    "sqlite": "SQLite",
+    "api": "API",
+    "llm": "LLM",
+    "saas": "SaaS",
+    "hn": "HN",
+    "github": "GitHub",
+    "gpt": "GPT",
+    "rag": "RAG",
+    "rl": "RL",
+    "ui": "UI",
+    "ux": "UX",
+    "css": "CSS",
+    "sql": "SQL",
+}
+
+
+def display_label(term: str) -> str:
+    label = " ".join(DISPLAY_WORDS.get(w, w) for w in term.split())
+    return label[:1].upper() + label[1:]
+
+
+def _best_label(seed: str, items: list[EvidenceItem]) -> str:
+    """Longest phrase (<= 3 words) containing the seed shared by >= 60% of items."""
+    seed_tokens = seed.split()
+    threshold = max(2, -(-len(items) * 3 // 5))
+    counts: Counter[str] = Counter()
+    for item in items:
+        toks = tokenize(item_text(item))
+        seen: set[str] = set()
+        for n in (3, 2):
+            for i in range(len(toks) - n + 1):
+                gram = toks[i : i + n]
+                if gram[0] in STOPWORDS or gram[-1] in STOPWORDS or len(set(gram)) < n:
+                    continue  # also skips tag runs like "mcp mcp-server"
+                for j in range(n - len(seed_tokens) + 1):
+                    if gram[j : j + len(seed_tokens)] == seed_tokens:
+                        seen.add(" ".join(gram))
+                        break
+        counts.update(seen)
+    best = seed
+    for phrase, count in sorted(
+        counts.items(), key=lambda kv: (-len(kv[0].split()), -kv[1], kv[0])
+    ):
+        if count >= threshold and is_seed_candidate(phrase):
+            best = phrase
+            break
+    return best
+
+
+def coherent_members(
+    seed: str, candidates: set[int], items: list[EvidenceItem], term_sets: list[set[str]]
+) -> tuple[list[int], str | None]:
+    """Indices (into ``items``) that ``seed`` may claim among ``candidates``.
+
+    Returns ``(members, qualifier)``; ``qualifier`` is the second term the members were
+    required to share, or ``None`` when the seed alone was sufficient.
+    """
+    ids = sorted(candidates)
+    if " " in seed:
+        return ids, None
+    specific = is_specific(seed)
+    if specific and evidence_is_cohesive(
+        seed, [items[i] for i in ids], [term_sets[i] for i in ids]
+    ):
+        return ids, None
+    # The seed alone is not enough: claim the items that also share the best
+    # co-occurring agreement term(s), so members share terms worth >= 2 in total
+    # (see ``text.specificity``). A specific seed needs one more term; a common seed
+    # needs a shared phrase or two shared specific words.
+    needed = 2 - specificity(seed)
+    by_term: dict[str, list[int]] = {}
+    for i in ids:
+        terms = sorted(agreement_terms(term_sets[i], seed))
+        for term in terms:
+            if specificity(term) >= needed:
+                by_term.setdefault(term, []).append(i)
+        if needed == 2:
+            words = [t for t in terms if specificity(t) == 1]
+            for a, b in combinations(words, 2):
+                by_term.setdefault(f"{a} + {b}", []).append(i)
+    best: tuple[tuple[int, int, int, int, str], list[int]] | None = None
+    for term, group in by_term.items():
+        members = [items[i] for i in group]
+        n_urls = len({m.story_key for m in members})
+        if n_urls < 2:
+            continue
+        rank = (
+            -independent_sources(members),
+            -n_urls,
+            -len(group),
+            0 if " " in term else 1,
+            term,
+        )
+        if best is None or rank < best[0]:
+            best = (rank, group)
+    if best is not None:
+        return best[1], best[0][4]
+    if specific and independent_sources([items[i] for i in ids]) <= 1:
+        # One community using one specific word: kept as a *candidate* (it claims no
+        # corroboration), but not published unless its evidence is cohesive.
+        return ids, None
+    return [], None
+
+
+def cluster_items(
+    items: list[EvidenceItem], min_items: int = 2, max_clusters: int = 25
+) -> list[TopicCluster]:
+    ordered = sorted(items, key=lambda i: (i.source, i.url, i.title))
+    term_sets = [extract_terms(item_text(i)) for i in ordered]
+    postings: dict[str, set[int]] = {}
+    for idx, terms in enumerate(term_sets):
+        for term in terms:
+            if is_seed_candidate(term):
+                postings.setdefault(term, set()).add(idx)
+    postings = {
+        t: ids
+        for t, ids in postings.items()
+        if len({ordered[i].story_key for i in ids}) >= min_items
+    }
+
+    assigned: set[int] = set()
+    clusters: list[TopicCluster] = []
+    while len(clusters) < max_clusters:
+        best: tuple[tuple[int, int, int, int, int, str], list[int], str | None] | None = None
+        for term, ids in postings.items():
+            free = ids - assigned
+            if len(free) < min_items:
+                continue
+            claimed, qualifier = coherent_members(term, free, ordered, term_sets)
+            claimed_items = [ordered[i] for i in claimed]
+            n_urls = len({i.story_key for i in claimed_items})
+            if n_urls < min_items:
+                continue
+            # More independent sources, then more unique URLs, then more items,
+            # then phrases, then seeds that needed no qualifier, then alphabetical.
+            rank = (
+                -independent_sources(claimed_items),
+                -n_urls,
+                -len(claimed),
+                0 if " " in term else 1,
+                0 if qualifier is None else 1,
+                term,
+            )
+            if best is None or rank < best[0]:
+                best = (rank, claimed, qualifier)
+        if best is None:
+            break
+        best_term = best[0][5]
+        members = best[1]
+        assigned.update(members)
+        cluster_items_ = [ordered[i] for i in members]
+        label = _best_label(best_term, cluster_items_)
+        qualifier = best[2]
+        if qualifier and not set(qualifier.split()) <= set(label.split()):
+            label = f"{label} + {qualifier}"
+        label_words = set(label.split())
+        related: Counter[str] = Counter()
+        for i in members:
+            related.update(
+                t
+                for t in term_sets[i]
+                if is_seed_candidate(t) and not set(t.split()) <= label_words
+            )
+        ranked = sorted(related.items(), key=lambda kv: (-kv[1], kv[0]))
+        related_terms = [t for t, c in ranked if c >= 2][:5]
+        clusters.append(
+            TopicCluster(
+                key=best_term,
+                label=display_label(label),
+                items=cluster_items_,
+                related_terms=related_terms,
+            )
+        )
+    return clusters

trendcite/config.py

@@ -0,0 +1,68 @@
+"""Configuration: defaults plus an optional TOML file (``trendcite.toml``).
+
+Configuration only names *what* to read (feeds, subreddits, queries, niche terms).
+It never contains credentials; optional LLM keys are read from the environment.
+"""
+
+from __future__ import annotations
+
+import tomllib
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+from typing import Any
+
+DEFAULT_FEEDS = [
+    "https://github.blog/feed/",
+    "https://simonwillison.net/atom/everything/",
+    "https://blog.pragmaticengineer.com/rss/",
+]
+DEFAULT_SUBREDDITS = ["startups", "SaaS", "programming"]
+DEFAULT_GITHUB_QUERIES = ["ai agent", "developer tools"]
+DEFAULT_NICHE = ["ai agents", "developer tools", "startups", "saas"]
+
+
+class ConfigError(ValueError):
+    pass
+
+
+@dataclass
+class Config:
+    niche: list[str] = field(default_factory=lambda: list(DEFAULT_NICHE))
+    feeds: list[str] = field(default_factory=lambda: list(DEFAULT_FEEDS))
+    subreddits: list[str] = field(default_factory=lambda: list(DEFAULT_SUBREDDITS))
+    github_queries: list[str] = field(default_factory=lambda: list(DEFAULT_GITHUB_QUERIES))
+    hn_limit: int = 30
+    sources: list[str] = field(default_factory=lambda: ["hackernews", "github", "rss", "reddit"])
+    top: int = 5
+
+
+def _str_list(value: Any, key: str, limit: int = 50) -> list[str]:
+    if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
+        raise ConfigError(f"'{key}' must be a list of strings")
+    return [v.strip() for v in value if v.strip()][:limit]
+
+
+def load_config(path: str | Path | None) -> Config:
+    cfg = Config()
+    if path is None:
+        return cfg
+    p = Path(path)
+    try:
+        data = tomllib.loads(p.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise ConfigError(f"config file not found: {p}") from exc
+    except tomllib.TOMLDecodeError as exc:
+        raise ConfigError(f"invalid TOML in {p}: {exc}") from exc
+    known = {f.name for f in fields(Config)}
+    unknown = sorted(set(data) - known)
+    if unknown:
+        raise ConfigError(f"unknown config keys: {', '.join(unknown)}")
+    for key in ("niche", "feeds", "subreddits", "github_queries", "sources"):
+        if key in data:
+            setattr(cfg, key, _str_list(data[key], key))
+    for key in ("hn_limit", "top"):
+        if key in data:
+            if not isinstance(data[key], int) or isinstance(data[key], bool):
+                raise ConfigError(f"'{key}' must be an integer")
+            setattr(cfg, key, data[key])
+    return cfg