gather-engine 2.4.0__tar.gz → 2.5.0__tar.gz

{gather_engine-2.4.0/src/gather_engine.egg-info → gather_engine-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gather-engine
-Version: 2.4.0
+Version: 2.5.0
 Summary: Collect web pages, papers, APIs, PDFs, scans, audio and video into a local corpus where every item records how it arrived.
 Author-email: Zain Dana Harper <zaindharper@gmail.com>
 License-Expression: AGPL-3.0-only
@@ -46,11 +46,11 @@ Gather is for researchers, journalists and developers who feed sources to a pers
 
 ## Install and first run
 
-Python 3.11 or newer. Version 2.4.0 is on PyPI as `gather-engine`.
+Python 3.11 or newer. Version 2.5.0 is on PyPI as `gather-engine`.
 
 ```bash
 pip install gather-engine
-gather --version                      # gather 2.4.0
+gather --version                      # gather 2.5.0
 ```
 
 Extract a saved page into hashed blocks, each tied to its place in the page:
@@ -302,7 +302,7 @@ The `web` adapter reads static HTML and does not run JavaScript; a client-render
 - [docs/WEB-ENGINE-UPLIFT.md](https://github.com/HarperZ9/gather/blob/main/docs/WEB-ENGINE-UPLIFT.md): the web-data engine roadmap and benchmarks.
 - [docs/ENTERPRISE-READINESS.md](https://github.com/HarperZ9/gather/blob/main/docs/ENTERPRISE-READINESS.md): context envelopes, action receipts, and host-neutral operation for unattended agents.
 - [docs/PILOT.md](https://github.com/HarperZ9/gather/blob/main/docs/PILOT.md): the accountable pilot evidence engine, its manifest boundary, and the private/shared evidence split.
-- [CHANGELOG.md](https://github.com/HarperZ9/gather/blob/main/CHANGELOG.md): version history. Current release: 2.4.0.
+- [CHANGELOG.md](https://github.com/HarperZ9/gather/blob/main/CHANGELOG.md): version history. Current release: 2.5.0.
 
 Peer projects: [crucible](https://github.com/HarperZ9/crucible) (judgment), [index](https://github.com/HarperZ9/index) (code maps), [forum](https://github.com/HarperZ9/forum) (orchestration), [telos](https://github.com/HarperZ9/telos) (the engine).
 

{gather_engine-2.4.0 → gather_engine-2.5.0}/README.md

@@ -11,11 +11,11 @@ Gather is for researchers, journalists and developers who feed sources to a pers
 
 ## Install and first run
 
-Python 3.11 or newer. Version 2.4.0 is on PyPI as `gather-engine`.
+Python 3.11 or newer. Version 2.5.0 is on PyPI as `gather-engine`.
 
 ```bash
 pip install gather-engine
-gather --version                      # gather 2.4.0
+gather --version                      # gather 2.5.0
 ```
 
 Extract a saved page into hashed blocks, each tied to its place in the page:
@@ -267,7 +267,7 @@ The `web` adapter reads static HTML and does not run JavaScript; a client-render
 - [docs/WEB-ENGINE-UPLIFT.md](https://github.com/HarperZ9/gather/blob/main/docs/WEB-ENGINE-UPLIFT.md): the web-data engine roadmap and benchmarks.
 - [docs/ENTERPRISE-READINESS.md](https://github.com/HarperZ9/gather/blob/main/docs/ENTERPRISE-READINESS.md): context envelopes, action receipts, and host-neutral operation for unattended agents.
 - [docs/PILOT.md](https://github.com/HarperZ9/gather/blob/main/docs/PILOT.md): the accountable pilot evidence engine, its manifest boundary, and the private/shared evidence split.
-- [CHANGELOG.md](https://github.com/HarperZ9/gather/blob/main/CHANGELOG.md): version history. Current release: 2.4.0.
+- [CHANGELOG.md](https://github.com/HarperZ9/gather/blob/main/CHANGELOG.md): version history. Current release: 2.5.0.
 
 Peer projects: [crucible](https://github.com/HarperZ9/crucible) (judgment), [index](https://github.com/HarperZ9/index) (code maps), [forum](https://github.com/HarperZ9/forum) (orchestration), [telos](https://github.com/HarperZ9/telos) (the engine).
 

{gather_engine-2.4.0 → gather_engine-2.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gather-engine"
-version = "2.4.0"
+version = "2.5.0"
 description = "Collect web pages, papers, APIs, PDFs, scans, audio and video into a local corpus where every item records how it arrived."
 readme = "README.md"
 requires-python = ">=3.11"

{gather_engine-2.4.0 → gather_engine-2.5.0}/src/gather/__init__.py

@@ -28,7 +28,7 @@ from gather.scope import filter_scope, in_scope
 from gather.source import Catalog, Source
 from gather.store import Corpus
 
-__version__ = "2.4.0"
+__version__ = "2.5.0"
 
 __all__ = [
     "Catalog", "Corpus", "CorpusRootDescriptor", "CorpusRootIdentity", "Digest", "Item", "NullProvenanceProvider", "NullSynthesizer",

{gather_engine-2.4.0 → gather_engine-2.5.0}/src/gather/cli_ext.py

@@ -9,11 +9,13 @@ def add_extension_parsers(sub, add_common: Callable) -> None:
     from gather.ledger_cmd import add_ledger_parser
     from gather.reddit_cmd import add_reddit_parser
     from gather.report_cmd import add_report_parsers
+    from gather.video_bundle_cmd import add_bundle_parsers
     from gather.video_cmd import add_channel_parser
     from gather.youtube_cmd import add_video_parsers
 
     add_video_parsers(sub, add_common)
     add_channel_parser(sub)
+    add_bundle_parsers(sub)
     add_reddit_parser(sub, add_common)
     add_report_parsers(sub, add_common)
     add_ledger_parser(sub)

gather_engine-2.5.0/src/gather/comment_balance.py

@@ -0,0 +1,234 @@
+"""Balanced comment synthesis: what a comment section adds, with critique and support side by side.
+
+``balance(comments)`` reads a video's comments (yt-dlp's records, or a bundle's thread file) and
+returns a ``gather.comment-balance/1`` record:
+
+- every comment in exactly one category (``gather.comment_signals``), with its evidence, so
+  nothing is dropped without a reason a reader can check;
+- substantive comments clustered into claims by shared vocabulary, ranked by comments and likes,
+  each with its critique, support and neutral counts and the comment ids behind it;
+- groups of likely coordinated promotion (two or more promotional comments that share signals or
+  wording) and near-duplicate text from several comments;
+- the balance: counts per category and stance, the critique-to-support ratio, and the share of
+  likes that went to promotion.
+
+A reply keeps its parent and thread ids. A reply that agrees with its parent takes the parent's
+stance toward the video, so agreement with a critique counts as critique.
+
+The record names no commenter: author fields are never read, @mentions are removed from excerpts,
+and each excerpt is under 15 words. It is deterministic and uses no model, so the item it becomes
+is ``compiled`` (``gather.derive``), never ``synthesized``. Pure.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from typing import Any
+
+from gather.comment_signals import AGREE_OPEN, INSULT, MENTION, URL, classify, tokens
+
+FORMAT = "gather.comment-balance/1"
+EXCERPT_WORDS = 14
+CLAIM_SIM = 0.2
+NEAR_DUP = 0.6
+
+
+def _likes(c: dict) -> int:
+    v = c.get("like_count", c.get("score"))
+    return v if isinstance(v, int) and v >= 0 else 0
+
+
+def excerpt(text: str, limit: int = EXCERPT_WORDS) -> str:
+    """The first sentence's opening words, at most ``limit``, with links and @mentions removed."""
+    plain = re.sub(r"\s+", " ", MENTION.sub("", URL.sub("", text or ""))).strip()
+    first = re.split(r"(?<=[.!?])\s", plain, maxsplit=1)[0]
+    words = first.split()
+    return " ".join(words[:limit]) + (" ..." if len(words) > limit else "")
+
+
+def flatten(tree: list[dict]) -> list[dict]:
+    """A thread file's nested comments as a flat list, each reply with its parent id."""
+    out: list[dict] = []
+    stack = [(c, None) for c in reversed(tree)]
+    while stack:
+        c, parent = stack.pop()
+        out.append({**{k: v for k, v in c.items() if k != "replies"}, "parent": c.get("parent") or parent})
+        stack.extend((r, c.get("id")) for r in reversed(c.get("replies") or []))
+    return out
+
+
+def _records(comments: list[dict]) -> list[dict]:
+    ids = {str(c.get("id")) for c in comments}
+    recs: list[dict[str, Any]] = []
+    for c in comments:
+        parent = c.get("parent")
+        parent = None if parent in (None, "", "root") else str(parent)
+        s = classify(str(c.get("text") or ""), from_channel=bool(c.get("author_is_uploader")))
+        recs.append({"id": str(c.get("id")), "parent": parent, "parent_missing": bool(parent and parent not in ids),
+                     "category": s.category, "stance": s.stance, "evidence": s.evidence, "promo": s.promo,
+                     "likes": _likes(c), "words": s.words, "question": s.question, "pointer": s.pointer,
+                     "_text": str(c.get("text") or ""), "_tokens": tokens(str(c.get("text") or ""))})
+    by = {r["id"]: r for r in recs}
+    for r in recs:
+        root, seen = r, set()
+        while root["parent"] and root["parent"] in by and root["id"] not in seen:
+            seen.add(root["id"])
+            root = by[root["parent"]]
+        r["thread"] = root["id"]
+        p = by.get(r["parent"] or "")
+        opens = r["stance"] == "support" or bool(AGREE_OPEN.match(r["_text"]))
+        agrees = p is not None and opens and p["category"] == "substantive" and p["stance"] in ("critique", "support")
+        r["toward_video"] = p["stance"] if agrees and p else r["stance"]
+        r["agrees_with_parent"] = agrees
+    return recs
+
+
+def _shingles(toks: list[str]) -> set[tuple[str, ...]]:
+    return {tuple(toks[i:i + 3]) for i in range(max(0, len(toks) - 2))}
+
+
+def _jaccard(a: set, b: set) -> float:
+    return len(a & b) / len(a | b) if a and b else 0.0
+
+
+def _groups(recs: list[dict], linked) -> list[list[dict]]:
+    parent = list(range(len(recs)))
+
+    def find(i: int) -> int:
+        while parent[i] != i:
+            parent[i] = parent[parent[i]]
+            i = parent[i]
+        return i
+
+    for i in range(len(recs)):
+        for j in range(i + 1, len(recs)):
+            if linked(recs[i], recs[j]):
+                parent[find(i)] = find(j)
+    out: dict[int, list[dict]] = {}
+    for i, r in enumerate(recs):
+        out.setdefault(find(i), []).append(r)
+    return [g for g in out.values() if len(g) > 1]
+
+
+def mark_coordination(recs: list[dict]) -> list[dict]:
+    """Groups of likely coordinated comments, marked ``coordinated_spam`` in place: promotional
+    comments that share two signals or a third of their wording, and three or more comments of
+    eight words or more with near-identical text."""
+    promo = [r for r in recs if r["category"] == "promotional"]
+    groups: list[dict[str, Any]] = [{"kind": "promotion with shared signals", "members": g} for g in _groups(
+        promo, lambda a, b: len(set(a["promo"]) & set(b["promo"])) >= 2
+        or _jaccard(_shingles(a["_tokens"]), _shingles(b["_tokens"])) >= 0.33)]
+    long = [r for r in recs if r["words"] >= 8 and r["category"] not in ("channel", "promotional")]
+    for dup in _groups(long, lambda a, b: _jaccard(_shingles(a["_tokens"]), _shingles(b["_tokens"])) >= NEAR_DUP):
+        if len(dup) >= 3:
+            groups.append({"kind": "near-identical text", "members": dup})
+    out = []
+    for n, g in enumerate(groups, 1):
+        shared = sorted(set.intersection(*(set(m["promo"]) for m in g["members"]))) if g["kind"].startswith("promotion") else []
+        for m in g["members"]:
+            m["category"], m["group"] = "coordinated_spam", n
+        out.append({"group": n, "kind": g["kind"], "comments": [m["id"] for m in g["members"]], "shared_signals": shared,
+                    "likes": sum(m["likes"] for m in g["members"])})
+    return out
+
+
+def _vectors(recs: list[dict]) -> list[dict[str, float]]:
+    """tf-idf vectors of unit length. A word in more than a third of the comments (the video's own
+    topic word) and a word only one comment uses carry no grouping weight."""
+    df = Counter(t for r in recs for t in set(r["_tokens"]))
+    n = len(recs)
+    vecs = []
+    for r in recs:
+        tf = Counter(t for t in r["_tokens"] if 1 < df[t] <= max(2, n / 3))
+        v = {t: (1 + math.log(c)) * math.log(1 + n / df[t]) for t, c in tf.items()}
+        norm = math.sqrt(sum(x * x for x in v.values())) or 1.0
+        vecs.append({t: x / norm for t, x in v.items()})
+    return vecs
+
+
+def _cos(a: dict[str, float], b: dict[str, float]) -> float:
+    if len(a) > len(b):
+        a, b = b, a
+    return sum(x * b.get(t, 0.0) for t, x in a.items())
+
+
+def cluster_claims(recs: list[dict]) -> list[dict]:
+    """Substantive comments grouped into claims. Most-liked first, each comment joins the closest
+    claim at or above ``CLAIM_SIM``; a reply below that joins its parent's claim, so a conversation
+    stays together. Claims are ranked by comments, then likes."""
+    sub = sorted((r for r in recs if r["category"] == "substantive"), key=lambda r: (-r["likes"], r["id"]))
+    vecs = dict(zip((r["id"] for r in sub), _vectors(sub)))
+    claims: list[dict] = []
+    home: dict[str, dict] = {}
+    for r in sub:
+        v = vecs[r["id"]]
+        best, score = None, 0.0
+        for c in claims:
+            s = _cos(v, c["centroid"]) / (math.sqrt(sum(x * x for x in c["centroid"].values())) or 1.0)
+            if s > score:
+                best, score = c, s
+        if score < CLAIM_SIM:
+            best = home.get(r["parent"] or "") or None
+        if best is None:
+            best = {"centroid": {}, "members": []}
+            claims.append(best)
+        best["members"].append(r)
+        for t, x in v.items():
+            best["centroid"][t] = best["centroid"].get(t, 0.0) + x
+        home[r["id"]] = best
+    return [_claim(c) for c in sorted(claims, key=lambda c: (-len(c["members"]), -sum(m["likes"] for m in c["members"])))]
+
+
+def _claim(c: dict) -> dict:
+    members = c["members"]
+    # the excerpt comes from the most-liked member that attacks no one, when there is one
+    civil = [m for m in members if not INSULT.search(m["_text"])] or members
+    top = max(civil, key=lambda m: (m["likes"], m["words"]))
+    stances = Counter(m["toward_video"] for m in members)
+    terms = [t for t, _ in sorted(c["centroid"].items(), key=lambda kv: -kv[1])[:4]]
+    return {"terms": terms, "comments": [m["id"] for m in members], "top_level": sum(1 for m in members if not m["parent"]),
+            "replies": sum(1 for m in members if m["parent"]), "critique": stances["critique"], "support": stances["support"],
+            "mixed": stances["mixed"], "neutral": stances["neutral"], "likes": sum(m["likes"] for m in members),
+            "excerpt": {"comment": top["id"], "text": excerpt(top["_text"])}, "threads": sorted({m["thread"] for m in members})}
+
+
+CATEGORIES = ("substantive", "reaction", "ad_hominem", "dismissal", "promotional", "coordinated_spam", "channel")
+
+
+def _balance_line(crit: int, supp: int) -> str:
+    if not crit and not supp:
+        return "No substantive comment takes a side, so there is no balance to report."
+    ratio = min(crit, supp) / max(crit, supp)
+    if ratio >= 0.5:
+        return f"Critique and support are both well represented: {crit} and {supp} substantive comments."
+    side, other = ("critique", "support") if crit > supp else ("support", "critique")
+    return f"Weighted toward {side}: {max(crit, supp)} {side} against {min(crit, supp)} {other} among substantive comments."
+
+
+def balance(comments: list[dict], *, title: str = "", url: str = "", max_claims: int = 20) -> dict:
+    """The balanced synthesis of one video's comments. ``comments`` are flat records with ``id``,
+    ``parent`` (None, ``"root"`` or a comment id), ``text``, ``like_count`` and optionally
+    ``author_is_uploader``; any author field is ignored. Pure and deterministic."""
+    recs = _records(comments)
+    groups = mark_coordination(recs)
+    claims = cluster_claims(recs)
+    cats = Counter(r["category"] for r in recs)
+    sub = [r for r in recs if r["category"] == "substantive"]
+    toward = Counter(r["toward_video"] for r in sub)
+    likes = sum(r["likes"] for r in recs)
+    spam_likes = sum(r["likes"] for r in recs if r["category"] in ("promotional", "coordinated_spam"))
+    top10 = sorted(recs, key=lambda r: -r["likes"])[:10]
+    public = [{k: v for k, v in r.items() if not k.startswith("_") and k != "promo"} for r in recs]
+    return {
+        "format": FORMAT, "title": title, "url": url, "method": "compiled",
+        "counts": {"comments": len(recs), "top_level": sum(1 for r in recs if not r["parent"]), "replies": sum(1 for r in recs if r["parent"]),
+                   "replies_parent_missing": sum(1 for r in recs if r["parent_missing"]), **{c: cats[c] for c in CATEGORIES}},
+        "balance": {"critique": toward["critique"], "support": toward["support"], "mixed": toward["mixed"], "neutral": toward["neutral"],
+                    "questions": sum(1 for r in sub if r["question"]), "pointers": sum(1 for r in sub if r["pointer"]),
+                    "sentence": _balance_line(toward["critique"], toward["support"]),
+                    "likes_total": likes, "likes_to_promotion": spam_likes,
+                    "top10_by_likes": dict(Counter(r["category"] if r["category"] != "substantive" else f"substantive:{r['toward_video']}" for r in top10))},
+        "claims": claims[:max_claims], "claims_total": len(claims), "coordinated": groups, "comments": public,
+    }

gather_engine-2.5.0/src/gather/comment_report.py

@@ -0,0 +1,78 @@
+"""The balanced comment synthesis as a page a person reads, and as a receipted Gather item.
+
+``report_markdown`` renders a ``gather.comment-balance/1`` record: the balance first, then the
+substantive claims ranked with their counts and one short excerpt each, then what was set aside and
+why, every group with the comment ids behind it. It names no commenter. ``balance_item`` wraps the
+page as a ``compiled`` derived item over the comment items it read (``gather.derive``), so the
+receipt says it was assembled by rules and points back to the exact comments.
+"""
+
+from __future__ import annotations
+
+from gather.derive import COMPILED, derive
+from gather.item import Item
+
+SET_ASIDE = (
+    ("coordinated_spam", "Likely coordinated spam"),
+    ("promotional", "Promotion from a single comment"),
+    ("ad_hominem", "About the person, not the argument"),
+    ("dismissal", "Dismissal with no reason given"),
+    ("reaction", "Short reactions"),
+    ("channel", "From the channel itself"),
+)
+
+
+def _ids(ids: list[str], limit: int = 12) -> str:
+    shown = ", ".join(ids[:limit])
+    return shown + (f" and {len(ids) - limit} more" if len(ids) > limit else "")
+
+
+def _claim_lines(n: int, c: dict) -> list[str]:
+    sides = ", ".join(f"{c[k]} {k}" for k in ("critique", "support", "mixed", "neutral") if c[k])
+    return [f"{n}. **{', '.join(c['terms']) or 'untitled'}**: {len(c['comments'])} comments ({c['top_level']} top-level, "
+            f"{c['replies']} replies; {sides}), {c['likes']} likes.",
+            f"   Excerpt: \"{c['excerpt']['text']}\" (comment {c['excerpt']['comment']})",
+            f"   Comments: {_ids(c['comments'])}", ""]
+
+
+def report_markdown(rec: dict) -> str:
+    """The record as Markdown: balance, ranked claims, what was set aside, and how it was made."""
+    k, b = rec["counts"], rec["balance"]
+    out = [f"# Comment balance{': ' + rec['title'] if rec.get('title') else ''}", ""]
+    if rec.get("url"):
+        out += [f"Source: {rec['url']}", ""]
+    missing = k["replies_parent_missing"]
+    orphans = f", {missing} of them with a parent that was not loaded" if missing else ""
+    out += [f"{k['comments']} comments read: {k['top_level']} top-level and {k['replies']} replies{orphans}.",
+            "Commenters are not named. Each excerpt is under 15 words.", "", "## Balance", "", f"- {b['sentence']}",
+            f"- Substantive comments: {k['substantive']} ({b['critique']} critique, {b['support']} support, {b['mixed']} mixed, "
+            f"{b['neutral']} neutral; {b['questions']} ask a question, {b['pointers']} point to a source).",
+            f"- Set aside: {k['coordinated_spam']} likely coordinated spam, {k['promotional']} other promotion, {k['ad_hominem']} about the "
+            f"person, {k['dismissal']} dismissals, {k['reaction']} short reactions, {k['channel']} from the channel.",
+            f"- Likes: {b['likes_total']} in all, {b['likes_to_promotion']} of them on promotion.",
+            f"- The 10 most-liked comments: {', '.join(f'{v} {n}' for n, v in sorted(b['top10_by_likes'].items(), key=lambda kv: -kv[1]))}.", "",
+            "## Substantive claims, ranked", ""]
+    for n, c in enumerate(rec["claims"], 1):
+        out += _claim_lines(n, c)
+    if rec["claims_total"] > len(rec["claims"]):
+        out += [f"{rec['claims_total'] - len(rec['claims'])} smaller claims are in the JSON record.", ""]
+    out += ["## Set aside, with the reason", ""]
+    for g in rec["coordinated"]:
+        why = f"shared signals: {', '.join(g['shared_signals'])}" if g["shared_signals"] else g["kind"]
+        out.append(f"- Group {g['group']}, {g['kind']}: {len(g['comments'])} comments, {g['likes']} likes ({why}). Comments: {_ids(g['comments'])}")
+    for cat, label in SET_ASIDE[1:]:
+        ids = [r["id"] for r in rec["comments"] if r["category"] == cat]
+        if ids:
+            out.append(f"- {label}: {len(ids)}. Comments: {_ids(ids, 8)}")
+    out += ["", "## How this was made", "",
+            "Deterministic rules, no model: the method is `compiled`. Categories come from word patterns, and an argument "
+            "marker outweighs an insult, so a harsh critique that gives a reason stays substantive. Claims group comments by "
+            "shared vocabulary. Sarcasm and irony can be misread. A reply that agrees with its parent counts toward the "
+            "parent's side.", ""]
+    return "\n".join(out)
+
+
+def balance_item(rec: dict, inputs: list[Item], *, fetched_at: float, ref: str) -> Item:
+    """The page as a ``compiled`` item derived from the comment items it read."""
+    return derive(inputs, report_markdown(rec), fetched_at=fetched_at, ref=ref, method=COMPILED,
+                  title=f"comment balance: {rec.get('title') or ref}", kind="synthesis", source="synthesis")

gather_engine-2.5.0/src/gather/comment_signals.py

@@ -0,0 +1,141 @@
+"""Per-comment signals for the balanced comment synthesis (``gather.comment_balance``).
+
+Each comment gets one category and the evidence for it: ``substantive`` (an argument, a question
+or a pointer about the topic), ``reaction`` (short, no argument), ``ad_hominem`` (an attack on a
+person, or on a person's associations, with no argument), ``dismissal`` (rejects the work with no
+argument), ``promotional`` (testimony that points to a product), and ``channel`` (the uploader's
+own comment). Substantive comments also get a stance toward the video: ``critique``, ``support``
+or ``neutral``.
+
+The rules are lexical and deterministic, and stance-neutral: the same words count the same way
+whoever they are aimed at. An argument marker always wins over an insult, so a harsh critique that
+gives a reason stays substantive. Pure: no network, no model.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+URL = re.compile(r"https?://\S+|www\.\S+", re.I)
+MENTION = re.compile(r"(?<!\w)@[\w.\-]+")
+WORD = re.compile(r"[a-z][a-z'\-]*")
+
+STOPWORDS = frozenset("""
+a an the and or but if then than so to of in on at by for with from into onto about as is are was were be been being am
+do does did done doing have has had having it its it's this that these those there here he she him her his hers they them
+their theirs we us our you your yours i me my mine what which who whom whose when where why how not no nor just only also
+very really more most much many some any all each every both few other such own same can could would should will shall may
+might must let lets let's get got gets like one two way thing things something anything nothing everything even still yet
+because however therefore thus though although while whereas since until up down out over under again further once too
+s t don doesn didn isn aren wasn weren won wouldn couldn shouldn can't don't doesn't isn't i'm i've i'd you're he's she's
+that's what's there's they're we're it'll gonna wanna yeah oh ok okay lol video talk guy guys
+""".split())
+
+# An argument: a reason, a contrast, a definition, a test, a mechanism.
+ARGUMENT = re.compile(
+    r"\b(because|therefore|thus|hence|so that|which means|implies|if\b.{1,60}\bthen|however|but\b|whereas|although|"
+    r"explain\w*|distinguish\w*|define\w*|definition|argu\w+|evidence|assum\w+|claim\w*|mechanism|predict\w*|"
+    r"test\w*|falsif\w+|counter\w*|conflat\w+|confus\w+|differen\w+|operation\w*|model\w*|hypothes\w+)\b", re.I)
+
+CRITIQUE = re.compile(
+    r"\b(however|disagree\w*|wrong|mistak\w+|flaw\w*|fails?\b|failed|misses|problem with|the problem|"
+    r"not convinc\w+|unconvinc\w+|doesn'?t (explain|follow|address|answer|work)|does not (explain|follow|address|answer)|"
+    r"not the same|isn'?t (true|right|how)|conflat\w+|circular|begs the question|unfalsifiable|no evidence|"
+    r"category error|overstat\w+|hand-?wav\w+|ignores?|skeptic\w*|sceptic\w*|doubt\w*|objection\w*|opposite)\b", re.I)
+
+SUPPORT = re.compile(
+    r"\b(agree\w*|great|brilliant|love[ds]?|loving|fantastic|thank\w*|excellent|insightful|best|amazing|awesome|"
+    r"exactly|well said|spot on|makes sense|wonderful|superb|enjoy\w*|fascinating|mind.?blow\w*|right on|beautiful\w*|"
+    r"inspir\w+|incredible|favou?rite|genius|legend\w*|good (one|point|interview|episode|talk|conversation)|so good|"
+    r"more of this|keep (it|them) coming|fair point|nailed it)\b", re.I)
+
+INSULT = re.compile(
+    r"\b(grifter\w*|charlatan\w*|fraud\w*|clown\w*|idiot\w*|moron\w*|stupid|dumb|quack\w*|huckster\w*|con ?man|"
+    r"snake ?oil|creep\w*|pseudo-?intellectual\w*|hack\b|loser\w*|imbecile\w*|delusional|narcissis\w+|crank\w*|"
+    r"is a joke|bullshitter\w*)\b", re.I)
+
+DISMISS = re.compile(r"\b(nonsense|pseudoscien\w+|word salad|gibberish|garbage|rubbish|drivel|bs\b|bullshit|waste of time|meaningless|babble)\b", re.I)
+
+# An attack on a person's associations instead of their claim (circumstantial ad hominem).
+ASSOCIATION = re.compile(r"\b(epstein|associat\w+ with|friends? with|funded by|ties to|paid by|shill\w*)\b", re.I)
+
+PERSON = re.compile(r"\b(he|him|his|she|her|they|them|this guy|the guest|the host|you|your)\b", re.I)
+
+# Promotion: testimony, a product named with its author, praise for the product, a call to act.
+PROMO_SIGNALS: dict[str, re.Pattern[str]] = {
+    "product named with its author": re.compile(
+        r"\b(book|novel|guide|course|program|app|channel)\b[^.!?\n]{0,40}\b(called|titled|named|by|from)\b\s+[A-Z]"
+        r"|\bread(ing)?\s+(the\s+)?[A-Z][\w']+(\s+[A-Z][\w']+){1,5}\s*(\(|,?\s*by\b|\s+from\b)\s*[A-Z]\w+\s+[A-Z]\w+", re.S),
+    "personal testimony": re.compile(
+        r"\b(changed my life|my (only )?savio(u)?r|my (anxiety|depression|stress)\b|i (finally )?found (all )?the answers|"
+        r"luckily,? (i|someone)|i feel (such|so much)|so much more energy|i was stuck|i used to (feel|get) (so|stuck)|"
+        r"shifted my perspective|helped me (break|get) free)\b", re.I),
+    "praise for the product": re.compile(
+        r"\b(genius|genial|must.read|should be taught in schools|nobody talks about|secret info|life.?changing|"
+        r"my favou?rite one|eye.?opening read)\b", re.I),
+    "call to action": re.compile(r"\b(check out|look up|dm me|message me|whatsapp|telegram|link in (my )?bio|sign up)\b", re.I),
+}
+
+
+@dataclass(slots=True)
+class Signals:
+    """What the rules saw in one comment."""
+
+    category: str
+    stance: str = "neutral"
+    evidence: list[str] = field(default_factory=list)
+    promo: list[str] = field(default_factory=list)
+    words: int = 0
+    question: bool = False
+    pointer: bool = False
+
+
+def tokens(text: str) -> list[str]:
+    """Content words, lower case, with links, mentions and stopwords removed."""
+    plain = MENTION.sub(" ", URL.sub(" ", text or "")).lower()
+    return [w.strip("'-") for w in WORD.findall(plain) if len(w) > 2 and w.strip("'-") not in STOPWORDS]
+
+
+def promo_signals(text: str) -> list[str]:
+    """The promotion signals a comment carries, by name."""
+    return [name for name, rx in PROMO_SIGNALS.items() if rx.search(text or "")]
+
+
+def _stance(text: str) -> tuple[str, list[str]]:
+    c = [m.group(0).lower() for m in CRITIQUE.finditer(text)]
+    s = [m.group(0).lower() for m in SUPPORT.finditer(text)]
+    if len(c) > len(s):
+        return "critique", c[:4]
+    if len(s) > len(c):
+        return "support", s[:4]
+    return ("mixed", (c + s)[:4]) if c else ("neutral", [])
+
+
+def classify(text: str, *, from_channel: bool = False) -> Signals:
+    """One comment's category, stance and evidence. Pure and deterministic."""
+    raw = text or ""
+    words = len(MENTION.sub(" ", URL.sub(" ", raw)).split())
+    question = "?" in raw
+    pointer = bool(URL.search(raw)) or bool(re.search(r"\b(paper|preprint|study|see also|read)\b", raw, re.I))
+    if from_channel:
+        return Signals("channel", words=words, question=question, pointer=pointer, evidence=["the uploader's own comment"])
+    promo = promo_signals(raw)
+    if len(promo) >= 2:
+        return Signals("promotional", words=words, promo=promo, evidence=promo)
+    argued = bool(ARGUMENT.search(raw)) and words >= 8
+    insult, dismiss, assoc = INSULT.search(raw), DISMISS.search(raw), ASSOCIATION.search(raw)
+    person = PERSON.search(raw)
+    if not argued and (insult or assoc) and (person or words < 25):
+        hit = (insult or assoc).group(0).lower()  # type: ignore[union-attr]
+        return Signals("ad_hominem", words=words, evidence=[f"about the person: {hit}"])
+    if not argued and dismiss and words < 30:
+        return Signals("dismissal", words=words, evidence=[f"rejects without a reason: {dismiss.group(0).lower()}"])
+    if argued or words >= 15 or (question and words >= 6) or (pointer and words >= 6):
+        stance, ev = _stance(raw)
+        return Signals("substantive", stance=stance, evidence=ev, words=words, question=question, pointer=pointer)
+    return Signals("reaction", words=words, question=question, evidence=["short, with no argument"])
+
+
+# A reply that opens by agreeing ("Exactly, ...", "Agreed.") sides with the comment it answers.
+AGREE_OPEN = re.compile(r"^\W*(exactly|agreed?|yes|yep|true|right|indeed|well said|spot on|this\W*$|100%)\b", re.I)