scamgate 0.3.3__py3-none-any.whl

scamgate/refresh.py

@@ -0,0 +1,233 @@
+"""Refreshing the known-bad-link feeds without extra packages.
+
+The Hermes plugin's `/scamgate update`, its `scamgate_update_feeds` tool, `hermes scamgate
+update-feeds` and the background refresh all come through `refresh()`. None of them takes a feed
+name, a URL or a folder from the caller: it always downloads the default feeds from their fixed
+sources into the configured cache. An agent can ask for a refresh, but can't choose what lands in
+the cache.
+
+A lock file in the cache folder makes sure only one refresh runs at a time, however many Hermes
+processes (gateway, chats, cron jobs) have the plugin loaded.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import threading
+import time
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Callable, Iterator
+
+from . import feeds
+
+AUTO_REFRESH_HOURS = 6.0  # the background refresh downloads again once the lists are this old
+TOOL_MIN_GAP_HOURS = 1.0  # the agent tool doesn't download again within this time
+LOCK_NAME = ".update.lock"
+LOCK_STALE_SECONDS = 30 * 60  # a lock older than this was left by a refresh that died
+OFF_VALUES = ("0", "false", "off", "no")
+ON_VALUES = ("1", "true", "on", "yes")
+_log = logging.getLogger("scamgate")
+
+
+def env_flag(name: str, default: bool) -> bool:
+    """A yes/no environment setting. Unset, empty or unrecognised values (such as an unfilled
+    `${user_config.KEY}` placeholder) mean `default`."""
+    value = (os.environ.get(name) or "").strip().lower()
+    if value in OFF_VALUES:
+        return False
+    if value in ON_VALUES:
+        return True
+    return default
+
+
+@dataclass
+class Outcome:
+    updated: list[str] = field(default_factory=list)
+    failed: dict[str, str] = field(default_factory=dict)
+    not_configured: dict[str, str] = field(default_factory=dict)
+    counts: dict[str, dict[str, int]] = field(default_factory=dict)
+    skipped: str = ""  # "fresh" | "busy" when nothing was downloaded on purpose
+
+    @property
+    def ok(self) -> bool:
+        return bool(self.updated) or self.skipped in ("fresh", "busy")
+
+    def as_dict(self) -> dict:
+        return {"updated": self.updated, "failed": self.failed, "not_configured": self.not_configured,
+                "counts": self.counts, "skipped": self.skipped, "cache": str(feeds.cache_dir())}
+
+    def text(self) -> str:
+        if self.skipped == "fresh":
+            return f"scamgate: the phishing lists were refreshed less than {TOOL_MIN_GAP_HOURS:g} hour ago."
+        if self.skipped == "busy":
+            return "scamgate: another refresh is already running; the lists will update when it finishes."
+        lines = []
+        for name in self.updated:
+            c = self.counts.get(name, {})
+            lines.append(f"{name}: updated ({c.get('urls', 0)} links, {c.get('hosts', 0)} hosts)")
+        for name, why in self.failed.items():
+            lines.append(f"{name}: not updated, kept the old list ({why})")
+        for name, why in self.not_configured.items():
+            lines.append(f"{name}: {why}")
+        head = "scamgate: phishing lists refreshed." if self.updated else "scamgate: no list could be refreshed."
+        return head + "\n" + "\n".join(lines)
+
+
+def _configured_defaults() -> list[str]:
+    """Default feeds that can be downloaded here (URLhaus only counts once its key is set)."""
+    return [n for n, f in feeds.FEEDS.items() if f.default and f.url() is not None]
+
+
+def oldest_age_hours(now: float | None = None) -> float:
+    """Age of the oldest default feed in hours; infinity when one was never downloaded."""
+    now = now or time.time()
+    meta = feeds.read_meta()
+    oldest = 0.0
+    for name in _configured_defaults():
+        ts = meta.get(name, {}).get("fetched_at")
+        try:
+            age = (now - datetime.fromisoformat(ts).timestamp()) / 3600 if ts else float("inf")
+        except ValueError:
+            age = float("inf")
+        oldest = max(oldest, age)
+    return oldest
+
+
+@contextmanager
+def _lock() -> Iterator[bool]:
+    d = feeds.cache_dir()
+    d.mkdir(parents=True, exist_ok=True)
+    path = d / LOCK_NAME
+    try:
+        if time.time() - path.stat().st_mtime > LOCK_STALE_SECONDS:
+            path.unlink(missing_ok=True)
+    except OSError:
+        pass
+    try:
+        fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    except FileExistsError:
+        yield False
+        return
+    with os.fdopen(fd, "w") as f:
+        f.write(str(os.getpid()))
+    try:
+        yield True
+    finally:
+        path.unlink(missing_ok=True)
+
+
+def refresh(*, if_older_than_hours: float | None = None) -> Outcome:
+    """Download the default feeds into the cache. Never raises for network or parse errors."""
+    if if_older_than_hours is not None and oldest_age_hours() < if_older_than_hours:
+        return Outcome(skipped="fresh")
+    with _lock() as got:
+        if not got:
+            return Outcome(skipped="busy")
+        results = feeds.update_feeds()
+    out = Outcome()
+    for r in results:
+        if r.ok:
+            out.updated.append(r.name)
+            out.counts[r.name] = {"urls": r.urls, "hosts": r.hosts}
+        elif r.message.startswith("skipped"):
+            out.not_configured[r.name] = r.message
+        else:
+            out.failed[r.name] = r.message
+    return out
+
+
+def auto_pass() -> Outcome:
+    """One pass of the background refresh: download only when the lists are 6 hours old."""
+    return refresh(if_older_than_hours=AUTO_REFRESH_HOURS)
+
+
+# ---------------------------------------------------------------- background refresh thread
+
+_BG_LOCK = threading.Lock()
+_BG: tuple[threading.Thread, threading.Event] | None = None
+
+
+def _background_loop(stop: threading.Event, first_delay: float, every: float) -> None:
+    if stop.wait(first_delay):
+        return
+    while True:
+        try:
+            out = auto_pass()
+            if out.failed:
+                _log.warning("scamgate: background feed refresh failed for %s", ", ".join(out.failed))
+        except Exception:
+            _log.warning("scamgate: background feed refresh failed", exc_info=True)
+        if stop.wait(every):
+            return
+
+
+def start_background(*, first_delay: float, every: float,
+                     on_unload: Callable[[Callable[[], None]], object] | None = None) -> threading.Thread:
+    """One daemon refresh thread per process (the Hermes plugin, or `scamgate serve` with
+    SCAMGATE_AUTO_UPDATE on). Processes share the lock file, so only one downloads at a time."""
+    global _BG
+    with _BG_LOCK:
+        if _BG is not None and _BG[0].is_alive() and not _BG[1].is_set():
+            return _BG[0]
+        stop = threading.Event()
+        thread = threading.Thread(target=_background_loop, args=(stop, first_delay, every),
+                                  name="scamgate-feed-refresh", daemon=True)
+        _BG = (thread, stop)
+        thread.start()
+    if callable(on_unload):
+        try:
+            on_unload(stop.set)
+        except Exception:
+            pass
+    return thread
+
+
+def status() -> tuple[bool, list[str]]:
+    """(any feeds cached, one line per cached feed)."""
+    meta = {k: v for k, v in feeds.read_meta().items() if k in feeds.FEEDS}
+    if not meta:
+        return False, []
+    idx = feeds.FeedIndex(fetched_at={k: v.get("fetched_at", "") for k, v in meta.items()})
+    stale = set(idx.stale_feeds())
+    lines = [f"{name}: {m.get('urls', 0)} links, {m.get('hosts', 0)} hosts, fetched {m.get('fetched_at')}"
+             + (" (stale)" if name in stale else "") for name, m in meta.items()]
+    return True, lines
+
+
+# ---------------------------------------------------------------- `hermes scamgate ...`
+
+def setup_cli(parser) -> None:
+    """Builds `hermes scamgate ...`. Hermes sets `func=cli_main` on this parser."""
+    sub = parser.add_subparsers(dest="scamgate_command", metavar="COMMAND")
+    sub.required = True
+    up = sub.add_parser("update-feeds", help="Download the default phishing lists into the cache now")
+    up.set_defaults(scamgate_run=_cli_update)
+    st = sub.add_parser("feeds-status", help="Show the cached phishing lists and their age")
+    st.set_defaults(scamgate_run=_cli_status)
+
+
+def _cli_update(args) -> int:
+    out = refresh()
+    print(out.text())
+    print(f"cache: {feeds.cache_dir()}")
+    return 0 if out.ok else 1
+
+
+def _cli_status(args) -> int:
+    any_cached, lines = status()
+    if not any_cached:
+        print("No phishing lists cached yet. Run `hermes scamgate update-feeds`, or `/scamgate update` in a chat.")
+        return 1
+    print("\n".join(lines))
+    return 0
+
+
+def cli_main(args) -> int:
+    run = getattr(args, "scamgate_run", None)
+    if run is None:
+        print("Usage: hermes scamgate update-feeds | feeds-status")
+        return 2
+    return run(args)

scamgate/registry.py

@@ -0,0 +1,66 @@
+"""A tiny rule registry.
+
+Add a rule by decorating a function:
+
+    @register("pay.my_rule", kind="payment")
+    def my_rule(check: PaymentCheck, ctx: RuleContext) -> Hit | None:
+        ...
+
+Then give it an entry under `rules:` in config/rules.yaml (enabled + weight).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Literal
+
+Kind = Literal["url", "payment"]
+
+
+@dataclass
+class Hit:
+    explanation: str
+    weight: int | None = None  # None -> use the rule's configured weight
+
+
+@dataclass
+class RuleContext:
+    config: dict[str, Any]
+    rule_config: dict[str, Any]
+    blocklist: frozenset[str] = field(default_factory=frozenset)
+    feeds: Any = None  # scamgate.feeds.FeedIndex | None
+
+
+RuleFn = Callable[[Any, RuleContext], "Hit | list[Hit] | None"]
+
+
+@dataclass(frozen=True)
+class Rule:
+    rule_id: str
+    kind: Kind
+    fn: RuleFn
+    description: str
+
+
+_REGISTRY: dict[str, Rule] = {}
+
+
+def register(rule_id: str, kind: Kind) -> Callable[[RuleFn], RuleFn]:
+    def deco(fn: RuleFn) -> RuleFn:
+        if rule_id in _REGISTRY:
+            raise ValueError(f"duplicate rule id: {rule_id}")
+        _REGISTRY[rule_id] = Rule(rule_id, kind, fn, (fn.__doc__ or "").strip())
+        return fn
+
+    return deco
+
+
+def rules_for(kind: Kind) -> list[Rule]:
+    # Importing the rule modules populates the registry.
+    from .rules import payment, url  # noqa: F401
+
+    return [r for r in _REGISTRY.values() if r.kind == kind]
+
+
+def all_rules() -> list[Rule]:
+    return rules_for("url") + rules_for("payment")

scamgate/rules/__init__.py

@@ -0,0 +1 @@
+"""Rule modules. Each module registers its rules on import."""

scamgate/rules/payment.py

@@ -0,0 +1,127 @@
+"""Payment rules, based on common push-payment and purchase-scam typologies."""
+
+from __future__ import annotations
+
+import re
+
+from ..config import lst
+from ..models import PaymentCheck, PaymentMethod, VopResult
+from ..registry import Hit, RuleContext, register
+
+
+def _text(check: PaymentCheck) -> str:
+    return re.sub(r"\s+", " ", (check.context_text or "").lower().replace("’", "'"))
+
+
+def _find(text: str, phrases: list[str]) -> list[str]:
+    return [p for p in phrases if re.search(rf"(?<![a-z]){re.escape(p)}(?![a-z])", text)]
+
+
+def _quote(found: list[str], n: int = 3) -> str:
+    return ", ".join(f"'{f}'" for f in found[:n])
+
+
+@register("pay.new_payee_urgency", kind="payment")
+def new_payee_urgency(check: PaymentCheck, ctx: RuleContext) -> Hit | None:
+    """First payment to this payee, and the request is urgent."""
+    if not check.is_new_payee:
+        return None
+    found = _find(_text(check), lst(ctx.config, "urgency_terms"))
+    if found:
+        return Hit(
+            f"This is a new payee and the request is urgent ({_quote(found)}). "
+            "Scammers rush people so they skip checks."
+        )
+    return None
+
+
+@register("pay.bank_details_changed", kind="payment")
+def bank_details_changed(check: PaymentCheck, ctx: RuleContext) -> Hit | None:
+    """Payee's bank details changed recently (invoice-redirection pattern)."""
+    found = _find(_text(check), lst(ctx.config, "bank_change_phrases"))
+    if check.bank_details_changed or found:
+        why = f" (message says {_quote(found, 1)})" if found else ""
+        return Hit(
+            f"The payee's bank details have changed{why}. Invoice-redirection fraud works "
+            "exactly like this; confirm the new account by phone using a number you already had."
+        )
+    return None
+
+
+@register("pay.irreversible_method", kind="payment")
+def irreversible_method(check: PaymentCheck, ctx: RuleContext) -> Hit | None:
+    """Payment method is hard or impossible to reverse."""
+    if check.method in (PaymentMethod.crypto, PaymentMethod.gift_card):
+        label = "crypto" if check.method is PaymentMethod.crypto else "gift cards"
+        return Hit(f"Paying by {label} is effectively irreversible, and it's the method scammers ask for most.")
+    if check.method is PaymentMethod.bank_transfer and check.is_new_payee:
+        if check.vop_result is not None and check.vop_result.value == "match":
+            return Hit("A bank transfer to a new payee is hard to recall once sent. Your bank confirmed the "
+                       "name matches the account, which helps.",
+                       weight=int(ctx.rule_config.get("new_payee_transfer_vop_match_weight", 10)))
+        return Hit(
+            "A bank transfer to a new payee is hard to recall once sent. Check the name matches the account "
+            "with your bank's Verification of Payee, or confirm the details by phone, before sending.",
+            weight=int(ctx.rule_config.get("new_payee_transfer_weight", 25)),
+        )
+    return None
+
+
+@register("pay.method_restricted", kind="payment")
+def method_restricted(check: PaymentCheck, ctx: RuleContext) -> Hit | None:
+    """Seller insists on a specific irreversible payment method."""
+    found = _find(_text(check), lst(ctx.config, "method_restriction_phrases"))
+    if found:
+        return Hit(
+            f"The seller restricts how you can pay ({_quote(found)}). Legit sellers rarely "
+            "refuse protected methods like cards."
+        )
+    return None
+
+
+@register("pay.over_budget", kind="payment")
+def over_budget(check: PaymentCheck, ctx: RuleContext) -> Hit | None:
+    """Amount exceeds the budget the human gave the agent."""
+    if check.budget is not None and check.amount > check.budget:
+        return Hit(
+            f"{check.amount:,.2f} {check.currency} is over the budget of "
+            f"{check.budget:,.2f} {check.currency}."
+        )
+    return None
+
+
+@register("pay.pressure_phrases", kind="payment")
+def pressure_phrases(check: PaymentCheck, ctx: RuleContext) -> Hit | None:
+    """Messages use pressure, secrecy, or too-good-to-be-true language."""
+    found = _find(_text(check), lst(ctx.config, "pressure_phrases"))
+    if found:
+        return Hit(
+            f"The messages use pressure or secrecy language ({_quote(found)}), "
+            "typical of CEO-fraud, impersonation and investment scams."
+        )
+    return None
+
+
+@register("pay.vop_mismatch", kind="payment")
+def vop_mismatch(check: PaymentCheck, ctx: RuleContext) -> Hit | None:
+    """Verification of Payee says the name doesn't match the account."""
+    if check.vop_result is None:
+        return None
+    weights = ctx.rule_config.get("weights", {})
+    w = int(weights.get(check.vop_result.value, 0))
+    msg = {
+        VopResult.no_match: f"Verification of Payee says '{check.payee_name}' does NOT match the account holder.",
+        VopResult.close_match: f"Verification of Payee only partly matches '{check.payee_name}'; check the exact name.",
+        VopResult.unavailable: "Verification of Payee wasn't available, so the payee name is unconfirmed.",
+        VopResult.match: "Verification of Payee matched.",
+    }[check.vop_result]
+    return Hit(msg, weight=w) if w > 0 else None
+
+
+@register("pay.unbacked_known_claim", kind="payment")
+def unbacked_known_claim(check: PaymentCheck, ctx: RuleContext) -> Hit | None:
+    """The agent said this is a known payee, but there's no record of paying them."""
+    if check.agent_claim_unbacked:
+        return Hit("The agent described this as a payee you know, but you've never paid them from here. "
+                   "Scam messages often claim familiarity to skip checks.")
+    return None