otpguard 0.1.0__py3-none-any.whl

otpguard/__init__.py

@@ -0,0 +1,9 @@
+"""otpguard: decide whether an OTP SMS should be sent, before you pay for it."""
+from .model import Action, Decision, OtpRequest, Signal
+from .policy import Policy
+from .sentinel import AsyncSentinel, Sentinel
+from .store import MemoryStore, StoreError
+
+__version__ = "0.1.0"
+__all__ = ["Action", "Decision", "OtpRequest", "Signal", "Policy", "Sentinel", "AsyncSentinel", "MemoryStore",
+           "StoreError", "__version__"]

otpguard/baselines.py

@@ -0,0 +1,81 @@
+"""Hourly baselines (EWMA mean and variance) that the population detectors compare against.
+
+Each key keeps an overall estimate and, once there is enough history, one per hour of
+the week. Hours with no traffic count as zeros; for sparse keys such as prefixes the
+zeros are applied lazily when the key is next touched."""
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Any
+
+from .store.base import Store
+
+
+def hour_of_week(hour_index: int) -> int:
+    # hour_index is hours since the epoch; 1970-01-01 was a Thursday
+    return (hour_index + 3 * 24) % 168
+
+
+@dataclass
+class Estimate:
+    mean: float
+    std: float
+    n: int            # hours of history behind the key
+    weekly: bool = False
+
+
+class Baselines:
+    def __init__(self, store: Store, alpha: float = 0.05, how_min_samples: int = 3):
+        self.store = store
+        self.alpha = alpha
+        self.how_min = how_min_samples
+
+    def _key(self, dim: str, key: str) -> str:
+        return f"base:{dim}:{key}"
+
+    def update(self, dim: str, key: str, hour_index: int, value: float, alpha: float | None = None,
+               by_hour_of_week: bool = True) -> None:
+        a = alpha or self.alpha
+        k = self._key(dim, key)
+        rec: dict[str, Any] = self.store.get_json(k) or {"m": None, "v": 0.0, "n": 0, "h": None, "w": {}}
+        # decay for the hours we did not see (sparse keys)
+        if rec["m"] is not None and rec["h"] is not None and hour_index - rec["h"] > 1:
+            gap = hour_index - rec["h"] - 1
+            f = (1 - a) ** gap
+            rec["v"] = rec["v"] * f + (1 - f) * rec["m"] ** 2 * a   # rough: zeros add spread
+            rec["m"] *= f
+            rec["n"] += gap
+        rec["m"], rec["v"] = _ewma(rec["m"], rec["v"], value, a)
+        rec["n"] += 1
+        rec["h"] = hour_index
+        if by_hour_of_week:
+            how = str(hour_of_week(hour_index))
+            w = rec["w"].get(how) or [None, 0.0, 0]
+            # weekly series are short, so they adapt faster
+            w[0], w[1] = _ewma(w[0], w[1], value, min(0.5, a * 6))
+            w[2] += 1
+            rec["w"][how] = w
+        self.store.set_json(k, rec, ttl_s=60 * 86400)
+
+    def get(self, dim: str, key: str, hour_index: int) -> Estimate | None:
+        rec = self.store.get_json(self._key(dim, key))
+        if not rec or rec["m"] is None:
+            return None
+        w = rec["w"].get(str(hour_of_week(hour_index)))
+        if w and w[2] >= self.how_min and w[0] is not None:
+            return Estimate(w[0], math.sqrt(max(w[1], 0.0)), rec["n"], weekly=True)
+        m, v = rec["m"], rec["v"]
+        if rec["h"] is not None and hour_index - rec["h"] > 1:
+            f = (1 - self.alpha) ** (hour_index - rec["h"] - 1)
+            m *= f
+        return Estimate(m, math.sqrt(max(v, 0.0)), rec["n"])
+
+
+def _ewma(mean: float | None, var: float, x: float, a: float) -> tuple[float, float]:
+    if mean is None:
+        return x, 0.0
+    d = x - mean
+    mean = mean + a * d
+    var = (1 - a) * (var + a * d * d)
+    return mean, var

otpguard/cli.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from . import __version__
+from . import simulate as sim
+from .model import OtpRequest
+from .policy import Policy, _ts
+
+
+def _policy(path: str | None) -> Policy:
+    return Policy.load(path) if path else Policy()
+
+
+def cmd_check(a) -> int:
+    from .sentinel import Sentinel
+    g = Sentinel(_policy(a.policy))
+    d = g.check(OtpRequest(phone=a.phone, purpose=a.purpose, ip=a.ip))
+    print(json.dumps(d.to_dict(), indent=2))
+    return 0
+
+
+def cmd_validate(a) -> int:
+    p = Policy.load(a.policy)
+    print(f"ok: version {p.version}, mode {p.mode}, {len(p.purposes)} purposes, "
+          f"{len(p.countries)} country rules, {len(p.campaigns)} campaigns")
+    return 0
+
+
+def cmd_replay(a) -> int:
+    from .replay import Mapping, load_incidents, replay, write_report
+    mapping = Mapping.load(a.mapping)
+    rep = replay(a.log, _policy(a.policy), mapping,
+                 load_incidents(a.incidents, mapping.tz) if a.incidents else None,
+                 as_enforced=not a.shadow, decisions_out=a.decisions,
+                 progress_every=a.progress)
+    j, m = write_report(rep, a.out)
+    print(m.read_text(encoding="utf-8"))
+    print(f"wrote {j} and {m}")
+    return 0
+
+
+def cmd_sim_baseline(a) -> int:
+    surge = (_ts(a.surge[0]), _ts(a.surge[1], end=True), float(a.surge[2])) if a.surge else None
+    n = sim.write(sim.baseline(a.days, _ts(a.start), a.rate, a.seed, surge), a.out)
+    print(f"wrote {n:,} rows to {a.out}")
+    return 0
+
+
+def cmd_sim_inject(a) -> int:
+    d = sim.scenario_defaults(a.scenario)
+    hours = a.hours if a.hours is not None else d["hours"]
+    rate = a.rate if a.rate is not None else d["rate"]
+    otp_types = a.otp_types.split(",") if a.otp_types else None
+    if a.volume is not None:
+        rate = a.volume * sim.otp_rate_per_hour(a.base, otp_types)
+    ctypes = a.customer_types.split(",") if a.customer_types else None
+    existing = sim.customers_in(a.base, customer_types=ctypes) if a.scenario == "A5" else None
+    attack = sim.attack(a.scenario, _ts(a.start), hours, rate, a.seed, existing)
+    n = sim.write(sim.merge(sim.read(a.base), attack), a.out)
+    print(f"wrote {n:,} rows to {a.out} ({a.scenario}: {sim.SCENARIOS[a.scenario]}, "
+          f"{rate:.0f} attack requests/h)")
+    return 0
+
+
+def cmd_sim_assign_ips(a) -> int:
+    mix = None
+    if a.mix:
+        import yaml
+        mix = [(m["asn"], float(m["share"]), int(m["networks"])) for m in yaml.safe_load(open(a.mix))]
+    n = sim.write(sim.assign_ips(sim.read(a.base), mix, a.seed), a.out)
+    print(f"wrote {n:,} rows to {a.out}")
+    return 0
+
+
+def cmd_sim_import(a) -> int:
+    import yaml
+    rows, companion = sim.import_log(a.log, a.mapping)
+    n = sim.write(rows, a.out)
+    mpath = a.mapping_out or str(a.out).rsplit(".", 1)[0] + ".mapping.yaml"
+    with open(mpath, "w", encoding="utf-8") as fh:
+        yaml.safe_dump(companion, fh, sort_keys=False)
+    print(f"wrote {n:,} rows to {a.out} and its replay mapping to {mpath}")
+    return 0
+
+
+def cmd_prune(a) -> int:
+    from .sql import SqlAudit
+    n = SqlAudit(a.sql, create=False).prune(a.days)
+    print(f"deleted {n['otp_events']:,} decisions and {n['alerts']:,} alerts older than {a.days} days")
+    return 0
+
+
+def cmd_serve(a) -> int:
+    import uvicorn
+
+    from .sentinel import Sentinel
+    from .server import create_app
+    g = Sentinel.from_config(a.policy, redis_url=a.redis, sql_url=a.sql, asn_tsv=a.asn) if a.policy \
+        else Sentinel(Policy())
+    uvicorn.run(create_app(g, show_reasons=a.show_reasons), host=a.host, port=a.port)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(prog="otpguard")
+    ap.add_argument("--version", action="version", version=f"otpguard {__version__}")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    c = sub.add_parser("check", help="run one request through a policy")
+    c.add_argument("--policy")
+    c.add_argument("--phone", required=True)
+    c.add_argument("--purpose", default="login")
+    c.add_argument("--ip")
+    c.set_defaults(fn=cmd_check)
+
+    v = sub.add_parser("validate", help="load a policy file and report problems")
+    v.add_argument("policy")
+    v.set_defaults(fn=cmd_validate)
+
+    r = sub.add_parser("replay", help="replay a historical log")
+    r.add_argument("log")
+    r.add_argument("--mapping", required=True)
+    r.add_argument("--policy")
+    r.add_argument("--incidents")
+    r.add_argument("--out", default="replay-report")
+    r.add_argument("--decisions", help="also write one line per decision to this CSV")
+    r.add_argument("--shadow", action="store_true",
+                   help="count every request as sent (default: blocked ones are not sent)")
+    r.add_argument("--progress", type=int, default=0, metavar="N")
+    r.set_defaults(fn=cmd_replay)
+
+    s = sub.add_parser("simulate", help="synthetic traffic and attacks")
+    ss = s.add_subparsers(dest="what", required=True)
+    b = ss.add_parser("baseline")
+    b.add_argument("--days", type=int, default=28)
+    b.add_argument("--start", default="2026-01-05")
+    b.add_argument("--rate", type=float, default=400, help="average OTPs per hour")
+    b.add_argument("--surge", nargs=3, metavar=("START", "END", "FACTOR"))
+    b.add_argument("--seed", type=int, default=1)
+    b.add_argument("--out", required=True)
+    b.set_defaults(fn=cmd_sim_baseline)
+    i = ss.add_parser("inject")
+    i.add_argument("--base", required=True)
+    i.add_argument("--scenario", required=True, choices=sorted(sim.SCENARIOS))
+    i.add_argument("--start", required=True)
+    i.add_argument("--hours", type=float)
+    i.add_argument("--rate", type=float, help="attack requests per hour")
+    i.add_argument("--volume", type=float,
+                   help="attack rate as a fraction of the log's mean OTP rate (overrides --rate)")
+    i.add_argument("--otp-types", help="comma-separated types that are OTPs in --base (default otp:*)")
+    i.add_argument("--customer-types",
+                   help="comma-separated types that mark customers, for A5 (default event:*)")
+    i.add_argument("--seed", type=int, default=7)
+    i.add_argument("--out", required=True)
+    i.set_defaults(fn=cmd_sim_inject)
+    im = ss.add_parser("import", help="convert a real log to the simulator schema, to inject attacks into it")
+    im.add_argument("log")
+    im.add_argument("--mapping", required=True)
+    im.add_argument("--out", required=True)
+    im.add_argument("--mapping-out", help="where to write the replay mapping (default: next to --out)")
+    im.set_defaults(fn=cmd_sim_import)
+    ai = ss.add_parser("assign-ips", help="give a log without addresses synthetic ones (for A2 / D8 / D9)")
+    ai.add_argument("--base", required=True)
+    ai.add_argument("--mix", help="YAML list of {asn, share, networks}")
+    ai.add_argument("--seed", type=int, default=11)
+    ai.add_argument("--out", required=True)
+    ai.set_defaults(fn=cmd_sim_assign_ips)
+
+    pr = sub.add_parser("prune", help="delete audit rows older than the retention period")
+    pr.add_argument("--sql", required=True)
+    pr.add_argument("--days", type=int, default=90)
+    pr.set_defaults(fn=cmd_prune)
+
+    sv = sub.add_parser("serve", help="run the HTTP service")
+    sv.add_argument("--policy")
+    sv.add_argument("--redis")
+    sv.add_argument("--sql")
+    sv.add_argument("--asn", help="iptoasn.com TSV file")
+    sv.add_argument("--host", default="127.0.0.1")
+    sv.add_argument("--port", type=int, default=8080)
+    sv.add_argument("--show-reasons", action="store_true")
+    sv.set_defaults(fn=cmd_serve)
+
+    a = ap.parse_args(argv)
+    return a.fn(a)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

otpguard/detectors/__init__.py

@@ -0,0 +1,15 @@
+from .base import Context, Detector
+from .budget import SpendBudget
+from .history import ResendWithoutVerify, VerificationRatio
+from .population import (ClientIntegrity, ConversionMismatch, CountryShare, IpRotation,
+                         NetworkConcentration, Pacing, PrefixBurst)
+from .request import DestinationRisk, PurposeCap
+
+ALL = [PurposeCap, DestinationRisk, ResendWithoutVerify, VerificationRatio, CountryShare,
+       PrefixBurst, Pacing, NetworkConcentration, IpRotation, ClientIntegrity,
+       ConversionMismatch, SpendBudget]
+
+# detectors a solved CAPTCHA says something about; destination detectors keep their weight
+CLIENT_SIDE = {"D7", "D8", "D9", "D10"}
+
+__all__ = ["Context", "Detector", "ALL", "CLIENT_SIDE"] + [d.__name__ for d in ALL]

otpguard/detectors/base.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from ..enrich import Network, Phone
+from ..model import OtpRequest, Signal
+
+if TYPE_CHECKING:
+    from ..baselines import Baselines
+    from ..policy import Policy
+    from ..store.base import Store
+
+
+@dataclass
+class Context:
+    req: OtpRequest
+    ts: float
+    hour_index: int
+    phone: Phone
+    net: Network | None
+    policy: "Policy"
+    store: "Store"
+    baselines: "Baselines"
+    is_new_number: bool = False
+    is_new_ip: bool = False
+    in_campaign: bool = False
+    notes: dict = field(default_factory=dict)
+
+    @property
+    def purpose(self) -> str:
+        return self.req.purpose
+
+    @property
+    def cc(self) -> str:
+        return self.phone.region
+
+
+class Detector:
+    id = "D0"
+    layer = "L0"
+    name = "detector"
+    needs_verification = False
+    needs_events = False
+    needs_ip = False
+
+    def __init__(self, policy: "Policy"):
+        self.cfg = policy.det(self.id)
+
+    def active(self, ctx: Context) -> bool:
+        p = ctx.policy
+        if not p.enabled(self.id):
+            return False
+        if self.needs_verification and not p.verification_feedback:
+            return False
+        if self.needs_events and not p.event_feedback:
+            return False
+        if self.needs_ip and ctx.net is None:
+            return False
+        return True
+
+    def record(self, ctx: Context) -> None:
+        """Update state for this request before any detector evaluates it."""
+
+    def evaluate(self, ctx: Context) -> Signal | None:
+        raise NotImplementedError
+
+    def sent(self, ctx: Context) -> None:
+        """Called when the OTP is going out (allowed, or shadow mode)."""
+
+    def not_sent(self, ctx: Context) -> None:
+        """Called when the OTP is not going out, so reservations can be given back."""
+
+    def signal(self, score: float, detail: str, scope: str = "", **kw) -> Signal:
+        return Signal(detector=self.id, score=max(0.0, min(1.0, score)), detail=detail, scope=scope, **kw)
+
+
+def clamp(x: float, lo: float = 0.0, hi: float = 1.0) -> float:
+    return max(lo, min(hi, x))

otpguard/detectors/budget.py

@@ -0,0 +1,57 @@
+"""Layer 4: money. The last line of defence when nothing else fires."""
+from __future__ import annotations
+
+from ..model import Action, Signal
+from .base import Context, Detector
+
+_HOUR, _DAY = 60, 1440
+
+
+class SpendBudget(Detector):
+    """D12. Spend per country and in total, per hour and per day.
+
+    Each budget has a soft level (challenge above it) and a hard level
+    (``block_factor`` × budget, block above it). The hard levels are taken atomically
+    with ``Store.reserve`` so concurrent requests cannot overspend; the soft level is
+    read from the sums the reservation returns."""
+
+    id, layer, name = "D12", "L4", "spend budget"
+
+    def evaluate(self, ctx: Context) -> Signal | None:
+        p = ctx.policy
+        rule = p.country(ctx.cc)
+        price = float(rule.price)
+        if price <= 0:
+            return None
+        bf = float(self.cfg["block_factor"])
+        scopes = [(f"spend:{ctx.cc}", f"{ctx.cc} spend", rule.budget_per_hour, rule.budget_per_day),
+                  ("spend:all", "total spend", p.budget_per_hour, p.budget_per_day)]
+        reserved: list[str] = []
+        worst: Signal | None = None
+        for key, label, per_hour, per_day in scopes:
+            windows = [(w, float(b) * bf) for w, b in ((_HOUR, per_hour), (_DAY, per_day)) if b]
+            if not windows:
+                continue
+            ok, sums = ctx.store.reserve(key, ctx.ts, windows, price)
+            if not ok:
+                for k in reserved:                      # give back what we took so far
+                    ctx.store.incr(k, ctx.ts, -price)
+                w, limit, s = next((w, b, s) for (w, b), s in zip(windows, sums) if s + price > b)
+                per = "hourly" if w == _HOUR else "daily"
+                return self.signal(1.0, f"{label} {s:.2f} is far over the {per} budget {limit / bf:.2f}",
+                                   scope=f"country:{ctx.cc}" if key != "spend:all" else "budget:all",
+                                   force=Action.BLOCK, retry_after=3600 if w == _HOUR else 6 * 3600)
+            reserved.append(key)
+            for (w, limit), s in zip(windows, sums):
+                if s + price > limit / bf and worst is None:
+                    per = "hourly" if w == _HOUR else "daily"
+                    worst = self.signal(0.6, f"{label} {s:.2f} is over the {per} budget {limit / bf:.2f}",
+                                        scope=f"country:{ctx.cc}" if key != "spend:all" else "budget:all",
+                                        force=Action.CHALLENGE, retry_after=600)
+        ctx.notes["D12_reserved"] = (reserved, price)
+        return worst
+
+    def not_sent(self, ctx: Context) -> None:
+        keys, price = ctx.notes.pop("D12_reserved", ([], 0.0))
+        for k in keys:
+            ctx.store.incr(k, ctx.ts, -price)

otpguard/detectors/history.py

@@ -0,0 +1,59 @@
+"""Layer 2: what happened to earlier codes for the same number or country."""
+from __future__ import annotations
+
+from ..model import Signal
+from .base import Context, Detector, clamp
+
+
+class ResendWithoutVerify(Detector):
+    """D3. Several codes to the same number and none of them entered."""
+
+    id, layer, name = "D3", "L2", "resend without verify"
+    needs_verification = True
+
+    def evaluate(self, ctx: Context) -> Signal | None:
+        w, k = int(self.cfg["window_min"]), int(self.cfg["k"])
+        h = ctx.phone.hash
+        sends = ctx.store.window_sum(f"snd:{h}", ctx.ts, w)
+        if sends < k:
+            return None
+        if ctx.store.window_sum(f"ver:{h}", ctx.ts, w) > 0:
+            return None
+        return self.signal(clamp((sends - k + 1) / k + 0.3),
+                           f"{int(sends)} codes to this number in {w} min, none entered",
+                           scope=f"phone:{h}")
+
+    def sent(self, ctx: Context) -> None:
+        ctx.store.incr(f"snd:{ctx.phone.hash}", ctx.ts)
+
+
+class VerificationRatio(Detector):
+    """D4. Share of codes that get entered, per country and purpose."""
+
+    id, layer, name = "D4", "L2", "verification ratio"
+    needs_verification = True
+    LAG_S = 300   # codes sent in the last few minutes may not be typed in yet
+
+    def evaluate(self, ctx: Context) -> Signal | None:
+        c = self.cfg
+        w = int(c["window_min"])
+        key = f"{ctx.purpose}:cc:{ctx.cc}"
+        sent = ctx.store.window_sum(f"snd:p:{key}", ctx.ts - self.LAG_S, max(1, w - 5))
+        if sent < c["min_sent"]:
+            return None
+        ver = ctx.store.window_sum(f"ver:p:{key}", ctx.ts, w)
+        base = ctx.baselines.get("vr", key, ctx.hour_index)
+        r0 = base.mean if base and base.n >= 6 else float(c["prior_ratio"])
+        r0 = max(r0, 0.05)
+        k = float(c["prior_strength"])
+        mean = (ver + r0 * k) / (sent + k)
+        deficit = 1 - mean / r0
+        drop = float(c["drop"])
+        if deficit <= drop:
+            return None
+        return self.signal(0.4 + 0.6 * (deficit - drop) / (1 - drop),
+                           f"only {ver:.0f} of {sent:.0f} {ctx.purpose} codes to {ctx.cc} entered "
+                           f"(usually {r0:.0%})", scope=f"country:{ctx.cc}")
+
+    def sent(self, ctx: Context) -> None:
+        ctx.store.incr(f"snd:p:{ctx.purpose}:cc:{ctx.cc}", ctx.ts)