answer-engine-benchmark 0.1.0__py3-none-any.whl

answer_engine_benchmark/__init__.py

@@ -0,0 +1,3 @@
+"""answer-engine-benchmark: ask AI answer engines your buyers' questions and count who they cite."""
+
+__version__ = "0.1.0"

answer_engine_benchmark/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

answer_engine_benchmark/cli.py

@@ -0,0 +1,168 @@
+"""aeb: ask AI answer engines your buyers' questions and count who they cite.
+
+Exit codes: 0 done, 1 a run finished but every call failed, 2 bad input.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__
+from .engines import ENGINES, available
+from .questions import QuestionError, count, load
+from .runner import read_jsonl, rescore, run
+from .summary import compare, pick_questions, render, render_compare
+
+DOCS = "https://synapsereality.io/open-source/answer-engine-benchmark/"
+
+
+def _kv(raw: str) -> tuple[str, str]:
+    k, sep, v = raw.partition("=")
+    if not sep or not k.strip():
+        raise argparse.ArgumentTypeError(f"expected name=value, got {raw!r}")
+    return k.strip(), v.strip()
+
+
+def _common(p: argparse.ArgumentParser) -> None:
+    p.add_argument("questions", help="question file (YAML)")
+    p.add_argument("--set", action="append", type=_kv, default=[], metavar="NAME=VALUE",
+                   help="fill a {placeholder}, e.g. --set brand='Acme Ltd' (repeatable)")
+
+
+def _engine_flags(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--engine", action="append", choices=sorted(ENGINES), help="only these engines (repeatable)")
+    p.add_argument("--model", action="append", type=_kv, default=[], metavar="ENGINE=MODEL",
+                   help="override a model, e.g. --model claude=claude-opus-5")
+    p.add_argument("--runs", type=int, default=3, help="times to ask each question on each engine (default 3)")
+    p.add_argument("--max-calls", type=int, default=500,
+                   help="refuse to start a run that needs more API calls than this (default 500)")
+    p.add_argument("--sleep", type=float, default=1.0, help="seconds between calls (default 1)")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    ap = argparse.ArgumentParser(prog="aeb", description=__doc__, epilog=f"Docs: {DOCS}",
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--version", action="version", version=f"aeb {__version__}")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    p = sub.add_parser("check", help="validate a question file and show what a run would cost in calls")
+    _common(p)
+    _engine_flags(p)
+    p.add_argument("--allow-unfilled", action="store_true", help="accept the template's example values")
+
+    p = sub.add_parser("run", help="ask every question on every engine and write answers.jsonl + report.md")
+    _common(p)
+    _engine_flags(p)
+    p.add_argument("--out", required=True, type=Path, help="output folder")
+    p.add_argument("--group", action="append", help="only these question groups (repeatable)")
+    p.add_argument("--dry-run", action="store_true", help="first question of each group, 1 run: a cheap smoke test")
+    p.add_argument("--anonymise", action="store_true", help="label other domains Source A, B, ... in report.md")
+
+    p = sub.add_parser("report", help="rebuild report.md from answers.jsonl (spends nothing)")
+    p.add_argument("answers", type=Path, help="answers.jsonl")
+    p.add_argument("questions", nargs="?", help="question file: re-score with its ours/brand_terms first")
+    p.add_argument("--set", action="append", type=_kv, default=[], metavar="NAME=VALUE")
+    p.add_argument("--anonymise", action="store_true")
+    p.add_argument("--out", type=Path, help="write here instead of stdout")
+
+    p = sub.add_parser("noise", help="re-ask a sample of questions and compare with the baseline")
+    _common(p)
+    _engine_flags(p)
+    p.add_argument("--baseline", required=True, type=Path, help="answers.jsonl from the first run")
+    p.add_argument("--out", required=True, type=Path, help="output folder for the repeat run")
+    p.add_argument("--sample", type=int, default=5, help="questions to re-ask, spread across groups (default 5)")
+    p.add_argument("--tolerance", type=int, default=1, help="allowed difference in citations (default 1)")
+    return ap
+
+
+def _plan(q: dict, engines: dict, runs: int, max_calls: int) -> int | None:
+    calls = count(q) * len(engines) * runs
+    names = ", ".join(f"{n} ({e.model})" for n, e in engines.items()) or "none"
+    print(f"{count(q)} questions x {len(engines)} engines x {runs} runs = {calls} calls. Engines: {names}")
+    if calls > max_calls:
+        print(f"aeb: {calls} calls is over --max-calls {max_calls}. Raise it if you mean it.", file=sys.stderr)
+        return None
+    return calls
+
+
+def _progress(rec: dict) -> None:
+    flag = "ERR " if rec["error"] else "CITE" if rec["cited"] else "ment" if rec["mentioned"] else "  - "
+    cost = rec["cost_usd"] or 0
+    print(f"{flag} {rec['engine']:<10} {rec['group']:<11} pos={rec['position'] or '-':<3} ${cost:.4f} "
+          f"{rec['question'][:70]} {rec['error'][:80]}", flush=True)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        if args.cmd == "report":
+            recs = read_jsonl(args.answers)
+            ours = []
+            if args.questions:
+                q = load(args.questions, dict(args.set), allow_unfilled=True)
+                recs, ours = rescore(recs, q), q["ours"]
+            text = render(recs, ours, anonymise=args.anonymise)
+            if args.out:
+                args.out.write_text(text, encoding="utf-8")
+            else:
+                print(text, end="")
+            return 0
+
+        q = load(args.questions, dict(args.set), allow_unfilled=getattr(args, "allow_unfilled", False))
+        engines = available(only=args.engine, models=dict(args.model))
+        if args.cmd == "check":
+            print(f"ok: {count(q)} questions in {len(q['groups'])} groups; ours = {q['ours']}")
+            missing = [f"{n} ({e.env_key})" for n, e in ENGINES.items()
+                       if n not in engines and (not args.engine or n in args.engine)]
+            if missing:
+                print("no key set for: " + ", ".join(missing))
+            _plan(q, engines, args.runs, args.max_calls)
+            return 0
+
+        if not engines:
+            keys = ", ".join(e.env_key for e in ENGINES.values())
+            print(f"aeb: no engine has a key. Set one or more of: {keys}", file=sys.stderr)
+            return 2
+
+        if args.cmd == "run":
+            if args.group:
+                q["groups"] = {g: v for g, v in q["groups"].items() if g in args.group}
+            if args.dry_run:
+                q["groups"] = {g: v[:1] for g, v in q["groups"].items()}
+                args.runs = 1
+            if _plan(q, engines, args.runs, args.max_calls) is None:
+                return 2
+            answers = args.out / "answers.jsonl"
+            recs = run(q, engines, runs=args.runs, out=answers, on_result=_progress, sleep=args.sleep)
+            everything = read_jsonl(answers)
+            (args.out / "report.md").write_text(render(everything, q["ours"], anonymise=args.anonymise),
+                                                encoding="utf-8")
+            cost = sum(r.get("cost_usd") or 0 for r in recs)
+            print(f"wrote {answers} (+{len(recs)} answers) and report.md. This run cost about ${cost:.3f}")
+            return 1 if recs and all(r["error"] for r in recs) else 0
+
+        if args.cmd == "noise":
+            baseline = read_jsonl(args.baseline)
+            picked = pick_questions(q, args.sample)
+            groups: dict[str, list[str]] = {}
+            for g, question in picked:
+                groups.setdefault(g, []).append(question)
+            q["groups"] = groups
+            if _plan(q, engines, args.runs, args.max_calls) is None:
+                return 2
+            answers = args.out / "repeat.jsonl"
+            run(q, engines, runs=args.runs, out=answers, on_result=_progress, sleep=args.sleep, label="repeat")
+            rows = compare(baseline, read_jsonl(answers), tolerance=args.tolerance)
+            text = render_compare(rows, args.tolerance)
+            (args.out / "noise.md").write_text(text, encoding="utf-8")
+            print(text, end="")
+            return 0
+    except QuestionError as e:
+        print(f"aeb: {e}", file=sys.stderr)
+        return 2
+    except (OSError, ValueError) as e:
+        print(f"aeb: {e}", file=sys.stderr)
+        return 2
+    return 2

answer_engine_benchmark/engines.py

@@ -0,0 +1,240 @@
+"""One adapter per answer engine, over plain HTTPS so no vendor SDK is needed.
+
+Every adapter takes a question and returns:
+
+    {"text": str, "urls": [str], "model": str, "usage": dict, "cost_usd": float | None}
+
+`urls` holds every source the engine attached to the answer, in the order it gave
+them, because the position of your first citation is scored. Keys are read from
+the environment only and are never written to a log or a results file.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+
+import requests
+
+TIMEOUT = 180
+SYSTEM_HINT = "Answer as you would for any user. Cite your sources with links."
+
+# Public list prices in USD per 1M tokens (input, output), checked 2026-09. Vendors change
+# these, so a run's cost is an estimate unless the API reports it (Perplexity does).
+PRICES: dict[str, tuple[float, float]] = {
+    "gpt-5-mini": (0.25, 2.00),
+    "gemini-3.5-flash": (0.30, 2.50),
+    "claude-opus-5": (5.00, 25.00),
+    "claude-sonnet-5": (2.00, 10.00),
+    "perplexity/sonar": (1.00, 1.00),
+}
+# Claude's web search tool is billed per search on top of tokens: $10 per 1,000.
+ANTHROPIC_SEARCH_USD = 0.01
+
+
+def token_cost(model: str, inp: int, out: int) -> float | None:
+    p = PRICES.get(model)
+    if p is None:
+        # Dated snapshots such as gpt-5-mini-2025-08-07 price like their base name.
+        p = next((v for k, v in PRICES.items() if model.startswith(k + "-")), None)
+    return round((inp * p[0] + out * p[1]) / 1e6, 6) if p else None
+
+
+# ----------------------------------------------------------------------------- OpenAI ----
+def openai_answer(question: str, *, key: str, model: str = "gpt-5-mini") -> dict:
+    body = {
+        "model": model,
+        "tools": [{"type": "web_search"}],
+        "tool_choice": "auto",
+        "input": question,
+        "instructions": SYSTEM_HINT,
+    }
+    r = requests.post("https://api.openai.com/v1/responses",
+                      headers={"Authorization": f"Bearer {key}"}, json=body, timeout=TIMEOUT)
+    r.raise_for_status()
+    d = r.json()
+    text, urls = [], []
+    for item in d.get("output", []) or []:
+        if item.get("type") != "message":
+            continue
+        for c in item.get("content", []) or []:
+            if c.get("type") == "output_text":
+                text.append(c.get("text", ""))
+                urls += [a["url"] for a in c.get("annotations", []) or []
+                         if a.get("type") == "url_citation" and a.get("url")]
+    u = d.get("usage", {}) or {}
+    return {
+        "text": "\n".join(text),
+        "urls": urls,
+        "model": d.get("model", model),
+        "usage": u,
+        "cost_usd": token_cost(d.get("model", model), u.get("input_tokens", 0), u.get("output_tokens", 0)),
+    }
+
+
+# ----------------------------------------------------------------------------- Gemini ----
+def gemini_answer(question: str, *, key: str, model: str = "gemini-3.5-flash") -> dict:
+    body = {"contents": [{"parts": [{"text": question}]}], "tools": [{"google_search": {}}]}
+    # The key goes in a header, not the URL. In the query string it ends up in the text of
+    # any HTTPError, and from there in the results file.
+    r = requests.post(
+        f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent",
+        headers={"x-goog-api-key": key}, json=body, timeout=TIMEOUT,
+    )
+    r.raise_for_status()
+    d = r.json()
+    cand = (d.get("candidates") or [{}])[0]
+    text = "\n".join(p.get("text", "") for p in (cand.get("content") or {}).get("parts", []) or [])
+    gm = cand.get("groundingMetadata") or {}
+    urls = [(ch.get("web") or {}).get("uri") for ch in gm.get("groundingChunks", []) or []]
+    u = d.get("usageMetadata", {}) or {}
+    return {
+        "text": text,
+        # These are vertexaisearch.cloud.google.com redirect URLs; resolve.py turns them
+        # into the real sources before scoring.
+        "urls": [x for x in urls if x],
+        "model": model,
+        "usage": u,
+        "cost_usd": token_cost(model, u.get("promptTokenCount", 0), u.get("candidatesTokenCount", 0)),
+        "grounding_queries": gm.get("webSearchQueries", []),
+    }
+
+
+# ----------------------------------------------------------------------------- Claude ----
+def anthropic_answer(question: str, *, key: str, model: str = "claude-sonnet-5") -> dict:
+    """Claude through the Messages API with the server-side web search tool.
+
+    No model fallback is configured on purpose: an answer from a different model would be
+    scored as this one. A refusal is recorded as an error instead.
+    """
+    messages: list[dict] = [{"role": "user", "content": question}]
+    headers = {"x-api-key": key, "anthropic-version": "2023-06-01"}
+    text: list[str] = []
+    urls: list[str] = []
+    usage = {"input_tokens": 0, "output_tokens": 0, "web_search_requests": 0}
+    served = model
+    # A long server-tool turn can stop with `pause_turn`. Sending the partial turn back
+    # lets it continue. Three rounds is plenty for a single question.
+    for _ in range(3):
+        body = {
+            "model": model,
+            "max_tokens": 16000,
+            "system": SYSTEM_HINT,
+            "messages": messages,
+            "tools": [{"type": "web_search_20260209", "name": "web_search", "max_uses": 5}],
+        }
+        r = requests.post("https://api.anthropic.com/v1/messages", headers=headers, json=body,
+                          timeout=TIMEOUT)
+        r.raise_for_status()
+        d = r.json()
+        served = d.get("model", served)
+        u = d.get("usage", {}) or {}
+        usage["input_tokens"] += u.get("input_tokens", 0)
+        usage["output_tokens"] += u.get("output_tokens", 0)
+        usage["web_search_requests"] += (u.get("server_tool_use") or {}).get("web_search_requests", 0)
+        for b in d.get("content", []) or []:
+            if b.get("type") == "text":
+                text.append(b.get("text", ""))
+                urls += [c["url"] for c in b.get("citations", []) or [] if c.get("url")]
+            elif b.get("type") == "web_search_tool_result" and isinstance(b.get("content"), list):
+                urls += [x["url"] for x in b["content"] if isinstance(x, dict) and x.get("url")]
+        stop = d.get("stop_reason")
+        if stop == "refusal":
+            raise RuntimeError(f"refusal: {(d.get('stop_details') or {}).get('category')}")
+        if stop != "pause_turn":
+            break
+        messages = [messages[0], {"role": "assistant", "content": d.get("content", [])}]
+    cost = token_cost(served, usage["input_tokens"], usage["output_tokens"])
+    if cost is not None:
+        cost = round(cost + usage["web_search_requests"] * ANTHROPIC_SEARCH_USD, 6)
+    return {"text": "".join(text), "urls": urls, "model": served, "usage": usage, "cost_usd": cost}
+
+
+# ------------------------------------------------------------------------- Perplexity ----
+def perplexity_answer(question: str, *, key: str, model: str = "perplexity/sonar") -> dict:
+    """Perplexity's Responses API. Web search is OFF unless the tool is passed.
+
+    Without `tools=[{"type": "web_search"}]` the call still succeeds and returns a fluent
+    answer naming real companies, with zero citations. For a benchmark scored on
+    citations that is the worst failure there is: no error, and no data.
+    """
+    body = {"model": model, "input": question, "tools": [{"type": "web_search"}]}
+    r = requests.post("https://api.perplexity.ai/v1/responses",
+                      headers={"Authorization": f"Bearer {key}"}, json=body, timeout=TIMEOUT)
+    r.raise_for_status()
+    d = r.json()
+    text, urls = "", []
+    for item in d.get("output", []) or []:
+        if item.get("type") == "search_results":
+            urls += [x.get("url") for x in item.get("results") or [] if x.get("url")]
+        for c in item.get("content", []) or []:
+            text += c.get("text", "")
+            urls += [a.get("url") for a in c.get("annotations") or [] if a.get("url")]
+    u = d.get("usage", {}) or {}
+    reported = (u.get("cost") or {}).get("total_cost")
+    return {
+        "text": text,
+        "urls": list(dict.fromkeys(urls)),
+        "model": model,
+        "usage": u,
+        "cost_usd": round(float(reported), 6) if reported is not None
+        else token_cost(model, u.get("input_tokens", 0), u.get("output_tokens", 0)),
+    }
+
+
+@dataclass(frozen=True)
+class Engine:
+    name: str
+    env_key: str
+    fn: Callable[..., dict]
+    default_model: str
+
+
+ENGINES: dict[str, Engine] = {
+    "openai": Engine("openai", "OPENAI_API_KEY", openai_answer, "gpt-5-mini"),
+    "gemini": Engine("gemini", "GEMINI_API_KEY", gemini_answer, "gemini-3.5-flash"),
+    "claude": Engine("claude", "ANTHROPIC_API_KEY", anthropic_answer, "claude-sonnet-5"),
+    "perplexity": Engine("perplexity", "PERPLEXITY_API_KEY", perplexity_answer, "perplexity/sonar"),
+}
+
+
+@dataclass(frozen=True)
+class Bound:
+    """An engine with its key and model resolved. `key` is never printed or stored."""
+
+    name: str
+    model: str
+    key: str
+    fn: Callable[..., dict]
+
+    def ask(self, question: str) -> dict:
+        return self.fn(question, key=self.key, model=self.model)
+
+    def __repr__(self) -> str:  # keep the key out of tracebacks and debug output
+        return f"Bound(name={self.name!r}, model={self.model!r})"
+
+
+def available(env: dict | None = None, *, only: list[str] | None = None,
+              models: dict[str, str] | None = None) -> dict[str, Bound]:
+    """Engines whose key is set. `models` overrides a default model per engine."""
+    env = os.environ if env is None else env
+    models = models or {}
+    out = {}
+    for name, e in ENGINES.items():
+        if only and name not in only:
+            continue
+        key = env.get(e.env_key, "")
+        if key:
+            out[name] = Bound(name, models.get(name, e.default_model), key, e.fn)
+    return out
+
+
+def redact(message: str, secrets: list[str]) -> str:
+    """Remove every key from an error message before it is logged."""
+    for s in secrets:
+        if s and len(s) >= 6:
+            message = message.replace(s, "[redacted]")
+    # Belt and braces for key-shaped strings that were not in the list.
+    return re.sub(r"(key=)[^&\s]+", r"\1[redacted]", message)

answer_engine_benchmark/questions.py

@@ -0,0 +1,66 @@
+"""Question files: YAML with placeholders you fill with your own brand."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import yaml
+
+PLACEHOLDER = re.compile(r"\{([a-z_]+)\}")
+# Values the shipped template uses so it parses. A run refuses them.
+UNFILLED = {"your brand", "example.com", "your category", "your audience", "your city"}
+
+
+class QuestionError(ValueError):
+    pass
+
+
+def _fill(value, vars_: dict[str, str]):
+    if isinstance(value, str):
+        def sub(m: re.Match) -> str:
+            k = m.group(1)
+            if k not in vars_:
+                raise QuestionError(f"placeholder {{{k}}} has no value; add it under vars: or pass --set {k}=...")
+            return str(vars_[k])
+        return PLACEHOLDER.sub(sub, value)
+    if isinstance(value, list):
+        return [_fill(v, vars_) for v in value]
+    if isinstance(value, dict):
+        return {k: _fill(v, vars_) for k, v in value.items()}
+    return value
+
+
+def load(path: str | Path, overrides: dict[str, str] | None = None, *, allow_unfilled: bool = False) -> dict:
+    """Read a question file and fill its placeholders.
+
+    Values come from the file's `vars:` block, then from `overrides` (the --set flags).
+    Returns {"ours", "brand_terms", "stale_markers", "groups": {name: [question]}}.
+    """
+    raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) or {}
+    vars_ = {str(k): str(v) for k, v in (raw.get("vars") or {}).items()}
+    vars_.update(overrides or {})
+    if not allow_unfilled:
+        left = sorted(k for k, v in vars_.items() if v.strip().lower() in UNFILLED)
+        if left:
+            raise QuestionError(
+                "these vars still hold the template's example values: " + ", ".join(left)
+                + ". Set them in the file or with --set name=value.")
+    q = _fill({k: raw.get(k) for k in ("ours", "brand_terms", "stale_markers", "groups")}, vars_)
+    if not q.get("ours"):
+        raise QuestionError("`ours` is empty: list the domains (or domain/path) that count as your citation")
+    if not q.get("brand_terms"):
+        raise QuestionError("`brand_terms` is empty: list the names that count as a mention")
+    groups = q.get("groups") or {}
+    if not isinstance(groups, dict) or not any(groups.values()):
+        raise QuestionError("`groups` has no questions")
+    for g, qs in groups.items():
+        if not isinstance(qs, list) or not all(isinstance(x, str) and x.strip() for x in qs):
+            raise QuestionError(f"group {g!r} must be a list of question strings")
+    q["stale_markers"] = q.get("stale_markers") or []
+    q["groups"] = {g: list(qs) for g, qs in groups.items()}
+    return q
+
+
+def count(q: dict) -> int:
+    return sum(len(v) for v in q["groups"].values())

answer_engine_benchmark/runner.py

@@ -0,0 +1,76 @@
+"""Ask every question on every engine, N times, and append each answer to a JSONL file."""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from .engines import Bound, redact
+from .scoring import Resolver, score
+
+
+def ask_once(engine: Bound, question: str) -> tuple[dict, str]:
+    try:
+        return engine.ask(question), ""
+    except Exception as e:  # noqa: BLE001 - one failed call is a row, not a crash
+        msg = redact(f"{type(e).__name__}: {e}", [engine.key])[:300]
+        return {"text": "", "urls": [], "model": engine.model, "usage": {}, "cost_usd": 0.0}, msg
+
+
+def run(q: dict, engines: dict[str, Bound], *, runs: int = 3, out: Path,
+        on_result: Callable[[dict], None] | None = None, sleep: float = 1.0,
+        resolver: Resolver | None = None, label: str = "baseline") -> list[dict]:
+    """Every question x engine x run. Each record is written and flushed as it lands, so
+    an interrupted run keeps what it paid for."""
+    resolver = resolver or Resolver(out.parent / "redirect_cache.json")
+    records = []
+    out.parent.mkdir(parents=True, exist_ok=True)
+    with out.open("a", encoding="utf-8") as f:
+        for group, qs in q["groups"].items():
+            for question in qs:
+                for name, eng in engines.items():
+                    for i in range(runs):
+                        t0 = time.time()
+                        ans, err = ask_once(eng, question)
+                        resolver.warm(ans.get("urls", []))
+                        ans["urls"] = [resolver(u) for u in ans.get("urls", [])]
+                        rec = {
+                            "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+                            "label": label,
+                            "group": group,
+                            "question": question,
+                            "engine": name,
+                            "model": ans.get("model") or eng.model,
+                            "run": i + 1,
+                            "seconds": round(time.time() - t0, 1),
+                            "error": err,
+                            "answer": ans.get("text", ""),
+                            "urls": ans.get("urls", []),
+                            "usage": ans.get("usage", {}),
+                            "cost_usd": ans.get("cost_usd"),
+                            **score(ans, q["ours"], q["brand_terms"], stale_markers=q["stale_markers"]),
+                        }
+                        f.write(json.dumps(rec, ensure_ascii=False) + "\n")
+                        f.flush()
+                        records.append(rec)
+                        if on_result:
+                            on_result(rec)
+                        if sleep:
+                            time.sleep(sleep)
+    resolver.save()
+    return records
+
+
+def read_jsonl(path: Path) -> list[dict]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def rescore(records: list[dict], q: dict) -> list[dict]:
+    """Score stored answers again, for example after changing `ours` or `stale_markers`.
+    Spends nothing."""
+    for r in records:
+        r.update(score({"text": r.get("answer", ""), "urls": r.get("urls", [])},
+                       q["ours"], q["brand_terms"], stale_markers=q["stale_markers"]))
+    return records

answer_engine_benchmark/scoring.py

@@ -0,0 +1,131 @@
+"""The citation rule and the rest of the scoring.
+
+An answer CITES you when one of the URLs it gives, in the text or in its source
+list, is on one of your properties (see `is_ours`). It MENTIONS you when one of
+your brand terms appears in the text. Only aggregates over repeated runs mean
+anything: the same question asked twice can come back with different sources.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import requests
+
+URL_RX = re.compile(r"https?://[^\s\)\]\"'>]+")
+# Only matches within this many characters of a brand term count as being about you.
+PROXIMITY = 200
+
+# Gemini's grounding sources are proxy URLs on this host that redirect to the real page.
+# Scored as they are, every Gemini citation looks like a citation of Google.
+REDIRECT_HOSTS = ("vertexaisearch.cloud.google.com",)
+
+
+class Resolver:
+    """Follows citation-proxy redirects to the real URL, with a JSON cache on disk."""
+
+    def __init__(self, cache_file: Path | None = None, timeout: int = 20) -> None:
+        self.cache_file = cache_file
+        self.timeout = timeout
+        self.cache: dict[str, str] = {}
+        if cache_file and cache_file.exists():
+            try:
+                self.cache = json.loads(cache_file.read_text())
+            except (OSError, ValueError):
+                self.cache = {}
+
+    def __call__(self, url: str) -> str:
+        if not any(h in url for h in REDIRECT_HOSTS):
+            return url
+        if url not in self.cache:
+            try:
+                self.cache[url] = requests.head(url, allow_redirects=True, timeout=self.timeout).url or url
+            except requests.RequestException:
+                return url  # not cached, so a later run tries again
+        return self.cache[url]
+
+    def warm(self, urls: list[str], workers: int = 16) -> None:
+        from concurrent.futures import ThreadPoolExecutor
+
+        todo = [u for u in dict.fromkeys(urls) if any(h in u for h in REDIRECT_HOSTS) and u not in self.cache]
+        if todo:
+            with ThreadPoolExecutor(max_workers=workers) as ex:
+                list(ex.map(self, todo))
+
+    def save(self) -> None:
+        if self.cache_file:
+            self.cache_file.parent.mkdir(parents=True, exist_ok=True)
+            self.cache_file.write_text(json.dumps(self.cache))
+
+
+def domain(url: str) -> str:
+    host = (urlsplit(url).hostname or "").lower()
+    return host.removeprefix("www.")
+
+
+def is_ours(url: str, ours: list[str]) -> bool:
+    """Is this URL one of your properties?
+
+    A bare domain (`example.com`) matches that host and its subdomains. A
+    path-qualified entry (`github.com/example`) matches only up to a path
+    boundary, so `github.com/example/repo` is yours and `github.com/example-fan`
+    is not. Reducing entries to their host would make every GitHub citation yours.
+    """
+    host = domain(url)
+    rest = url.split("://", 1)[-1].lower()
+    rest = rest.removeprefix("www.")
+    for o in ours:
+        o = o.lower().strip().rstrip("/")
+        if o.startswith(("http://", "https://")):
+            o = o.split("://", 1)[1]
+        o = o.removeprefix("www.")
+        if "/" in o:
+            if rest.startswith(o) and (len(rest) == len(o) or rest[len(o)] in "/?#"):
+                return True
+        elif host == o or host.endswith("." + o):
+            return True
+    return False
+
+
+def _near(text: str, terms: list[str], needles: list[str]) -> list[str]:
+    """Needles that appear within PROXIMITY characters of any term."""
+    low = text.lower()
+    spans = [m.start() for t in terms if t for m in re.finditer(re.escape(t.lower()), low)]
+    hits = []
+    for n in needles:
+        for m in re.finditer(re.escape(n.lower()), low):
+            if any(abs(m.start() - s) <= PROXIMITY for s in spans):
+                hits.append(n)
+                break
+    return hits
+
+
+def score(answer: dict, ours: list[str], brand_terms: list[str], *,
+          stale_markers: list[str] | None = None, resolve=None) -> dict:
+    """Score one answer. `resolve` maps a URL to its real destination (see Resolver)."""
+    resolve = resolve or (lambda u: u)
+    text = answer.get("text", "") or ""
+    cited = list(dict.fromkeys((answer.get("urls") or []) + URL_RX.findall(text)))
+    cited = [resolve(u.rstrip(".,;")) for u in cited]
+    domains = list(dict.fromkeys(d for d in (domain(u) for u in cited) if d))
+    our_urls = [u for u in cited if is_ours(u, ours)]
+    # Position: rank of your first source among the distinct sources cited. A source is
+    # a domain, except that your URLs on a shared host (github.com/you) count as a
+    # source of their own, so someone else's repo cited first still ranks ahead of you.
+    sources = list(dict.fromkeys(
+        ("ours:" if is_ours(u, ours) else "") + domain(u) for u in cited if domain(u)))
+    position = next((i + 1 for i, s in enumerate(sources) if s.startswith("ours:")), None)
+    mentioned = any(t and t.lower() in text.lower() for t in brand_terms)
+    stale = _near(text, brand_terms, stale_markers or []) if mentioned else []
+    return {
+        "mentioned": mentioned,
+        "cited": bool(our_urls),
+        "position": position,
+        "our_urls": our_urls[:5],
+        "domains_cited": domains,
+        "n_domains": len(domains),
+        "stale": stale,
+    }

answer_engine_benchmark/summary.py

@@ -0,0 +1,170 @@
+"""Aggregates, share of voice, the Markdown report, and the noise check."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+
+from .scoring import is_ours
+
+
+def _bucket() -> dict:
+    return {"runs": 0, "errors": 0, "mentioned": 0, "cited": 0, "stale": 0, "positions": [], "cost": 0.0}
+
+
+def summarise(records: list[dict]) -> dict:
+    """Rates are over answered runs: a failed call is counted as an error, never as a miss."""
+    slices: dict[str, dict] = defaultdict(_bucket)
+    for r in records:
+        for key in ("all", f"engine:{r['engine']}", f"group:{r['group']}", f"{r['group']} x {r['engine']}"):
+            b = slices[key]
+            b["runs"] += 1
+            b["cost"] += r.get("cost_usd") or 0
+            if r.get("error"):
+                b["errors"] += 1
+                continue
+            b["mentioned"] += bool(r.get("mentioned"))
+            b["cited"] += bool(r.get("cited"))
+            b["stale"] += bool(r.get("stale"))
+            if r.get("position"):
+                b["positions"].append(r["position"])
+    out = {}
+    for key, b in slices.items():
+        n = b["runs"] - b["errors"]
+        pos = b.pop("positions")
+        out[key] = {
+            **b,
+            "answered": n,
+            "mention_rate": round(b["mentioned"] / n, 4) if n else None,
+            "citation_rate": round(b["cited"] / n, 4) if n else None,
+            "avg_position": round(sum(pos) / len(pos), 2) if pos else None,
+            "cost": round(b["cost"], 4),
+        }
+    return out
+
+
+def share_of_voice(records: list[dict], *, groups: list[str] | None = None) -> list[tuple[str, int]]:
+    """Domains by the number of answers citing them at least once."""
+    counts: dict[str, int] = defaultdict(int)
+    for r in records:
+        if r.get("error") or (groups and r["group"] not in groups):
+            continue
+        for d in set(r.get("domains_cited") or []):
+            counts[d] += 1
+    return sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
+
+
+def _label(i: int) -> str:
+    s = ""
+    i += 1
+    while i:
+        i, rem = divmod(i - 1, 26)
+        s = chr(65 + rem) + s
+    return s
+
+
+def _pct(x: float | None) -> str:
+    return "n/a" if x is None else f"{x:.0%}"
+
+
+def render(records: list[dict], ours: list[str], *, anonymise: bool = False, top: int = 15) -> str:
+    """A Markdown report. With `anonymise`, every domain that is not yours becomes
+    "Source A", "Source B" and so on, which is what you want before sharing it."""
+    s = summarise(records)
+    questions = {r["question"] for r in records}
+    lines = [
+        f"# Answer engine benchmark: {len(records)} answers, {len(questions)} questions",
+        "",
+        "Cited = the answer links to one of your domains. Rates leave out failed calls.",
+        "",
+        "| slice | answers | errors | mentioned | cited | avg position when cited | cost USD |",
+        "|---|---|---|---|---|---|---|",
+    ]
+    order = ["all"] + sorted(k for k in s if k.startswith("engine:")) + sorted(k for k in s if k.startswith("group:"))
+    for k in order:
+        b = s[k]
+        lines.append(f"| {k} | {b['answered']} | {b['errors']} | {_pct(b['mention_rate'])} | "
+                     f"{b['cited']} of {b['answered']} ({_pct(b['citation_rate'])}) | "
+                     f"{b['avg_position'] or '-'} | {b['cost']:.4f} |")
+
+    engines = sorted({r["engine"] for r in records})
+    groups = list(dict.fromkeys(r["group"] for r in records))
+    lines += ["", "## Cited, by group and engine", "", "| group | " + " | ".join(engines) + " |",
+              "|---|" + "---|" * len(engines)]
+    for g in groups:
+        cells = []
+        for e in engines:
+            b = s.get(f"{g} x {e}")
+            cells.append(f"{b['cited']} of {b['answered']}" if b else "-")
+        lines.append(f"| {g} | " + " | ".join(cells) + " |")
+
+    sov = share_of_voice(records)
+    names: dict[str, str] = {}
+    lines += ["", "## Sources cited most", "", "| source | answers citing it |", "|---|---|"]
+    for i, (d, n) in enumerate(sov[:top]):
+        mine = is_ours("https://" + d, ours)
+        if mine:
+            label = f"**{d}** (yours)"
+        elif anonymise:
+            label = names.setdefault(d, f"Source {_label(len(names))}")
+        else:
+            label = d
+        lines.append(f"| {label} | {n} |")
+
+    never = sorted(questions - {r["question"] for r in records if r.get("cited")})
+    lines += ["", "## Questions where you were never cited", ""] + [f"- {x}" for x in never or ["none"]]
+    stale = [(r["engine"], r["question"], r["stale"]) for r in records if r.get("stale")]
+    lines += ["", "## Answers with a stale or wrong marker near your name", ""]
+    lines += [f"- {e}: {q} ({', '.join(m)})" for e, q, m in stale[:30]] or ["- none"]
+    return "\n".join(lines) + "\n"
+
+
+# ------------------------------------------------------------------------ noise check ----
+def pick_questions(q: dict, n: int) -> list[tuple[str, str]]:
+    """n questions spread across groups, round robin, in file order. Deterministic, so a
+    re-run of the noise check asks the same ones."""
+    pools = {g: list(qs) for g, qs in q["groups"].items()}
+    picked: list[tuple[str, str]] = []
+    while len(picked) < n and any(pools.values()):
+        for g in list(pools):
+            if pools[g] and len(picked) < n:
+                picked.append((g, pools[g].pop(0)))
+    return picked
+
+
+def compare(baseline: list[dict], repeat: list[dict], *, tolerance: int = 1) -> list[dict]:
+    """Per question and engine: citations in the repeat against the baseline.
+
+    When the two have a different number of answered runs, the baseline count is scaled
+    to the repeat's run count. A pair is `within` when the difference is at most
+    `tolerance` citations.
+    """
+    def tally(recs: list[dict]) -> dict[tuple[str, str], tuple[int, int]]:
+        t: dict[tuple[str, str], list[int]] = defaultdict(lambda: [0, 0])
+        for r in recs:
+            if r.get("error"):
+                continue
+            k = (r["question"], r["engine"])
+            t[k][0] += bool(r.get("cited"))
+            t[k][1] += 1
+        return {k: (v[0], v[1]) for k, v in t.items()}
+
+    base, rep = tally(baseline), tally(repeat)
+    rows = []
+    for k in sorted(rep):
+        if k not in base:
+            continue
+        (bc, bn), (rc, rn) = base[k], rep[k]
+        expected = bc * rn / bn if bn else 0
+        delta = rc - expected
+        rows.append({"question": k[0], "engine": k[1], "baseline": f"{bc} of {bn}", "repeat": f"{rc} of {rn}",
+                     "delta": round(delta, 2), "within": abs(delta) <= tolerance})
+    return rows
+
+
+def render_compare(rows: list[dict], tolerance: int) -> str:
+    ok = sum(r["within"] for r in rows)
+    lines = [f"# Noise check: {ok} of {len(rows)} question-engine pairs within +/-{tolerance} citation", "",
+             "| question | engine | baseline | repeat | delta | within |", "|---|---|---|---|---|---|"]
+    lines += [f"| {r['question']} | {r['engine']} | {r['baseline']} | {r['repeat']} | {r['delta']:+g} | "
+              f"{'yes' if r['within'] else 'NO'} |" for r in rows]
+    return "\n".join(lines) + "\n"

answer_engine_benchmark-0.1.0.dist-info/METADATA

@@ -0,0 +1,180 @@
+Metadata-Version: 2.4
+Name: answer-engine-benchmark
+Version: 0.1.0
+Summary: Ask ChatGPT, Gemini, Perplexity and Claude your buyers' questions, several times each, and count who they cite.
+Author: Synapse Research Ltd
+License-Expression: MIT
+Project-URL: Homepage, https://synapsereality.io/open-source/answer-engine-benchmark/
+Project-URL: Documentation, https://synapsereality.io/open-source/answer-engine-benchmark/
+Project-URL: Repository, https://github.com/synapsereality/answer-engine-benchmark
+Project-URL: Issues, https://github.com/synapsereality/answer-engine-benchmark/issues
+Keywords: aeo,geo,answer-engine-optimization,llm,citations,benchmark,seo
+Classifier: Environment :: Console
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.28
+Requires-Dist: PyYAML>=6
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == "test"
+Dynamic: license-file
+
+# answer-engine-benchmark
+
+Asks ChatGPT, Gemini, Perplexity and Claude the questions your buyers ask, with
+web search on, several times each. Then it counts how often each answer cites
+your site, what it cites instead, and what it says about you. Every answer is
+kept in a JSONL file, so you can check any number in the report against the
+text behind it.
+
+Docs: https://synapsereality.io/open-source/answer-engine-benchmark/
+
+```bash
+pip install answer-engine-benchmark
+export OPENAI_API_KEY=... GEMINI_API_KEY=... PERPLEXITY_API_KEY=... ANTHROPIC_API_KEY=...
+aeb run questions/template.yaml --out runs/first \
+  --set brand="Acme Analytics" --set domain=acme.example \
+  --set category="invoice software" --set audience="small accounting firms"
+```
+
+## Why repeat every question
+
+The same question can come back with different sources a minute later. One
+answer is one sample. The default is 3 runs per question per engine, and every
+rate in the report is over those runs. `aeb noise` re-asks a sample of questions
+later and tells you whether a change you see is bigger than the noise.
+
+## The question set
+
+`questions/template.yaml` holds 24 questions in four groups:
+
+| group | the buyer is | your name in the question |
+|---|---|---|
+| discovery | looking for a provider | no |
+| comparison | learning how to choose | no |
+| brand | asking about you | yes |
+| trust | checking you are safe to buy from | yes |
+
+Discovery and comparison tell you whether engines find you when nobody asked
+for you. Brand and trust tell you what they say when somebody does.
+
+Fill the four `vars` in the file, or pass them with `--set`. A run won't start
+while any of them still holds its example value. Add your own questions under
+any group, or new groups. Write them the way a buyer types, and never make a
+competitor the subject of a question.
+
+The file also sets:
+
+- `ours`: domains that count as your citation. `acme.example` covers its
+  subdomains. `github.com/acme` covers only paths under it, so a citation of
+  someone else's GitHub repo is not yours.
+- `brand_terms`: names that count as a mention.
+- `stale_markers`: phrases that describe you wrongly or out of date, such as an
+  old product or a wrong founding year. One is flagged only within 200
+  characters of a brand term, so another company "founded in 2015" doesn't count
+  against you.
+
+## How an answer is scored
+
+**Cited** means one of the answer's URLs is yours. The URLs are the sources the
+engine attached plus any link in the text. **Mentioned** means a brand term
+appears in the text. An answer can mention you without citing you, and that
+difference is worth watching. **Position** is the rank of your first source
+among the distinct sites the answer cited.
+
+Failed calls are recorded as errors and left out of every rate. A rate limit
+never shows up as "not cited".
+
+Gemini returns its sources as Google redirect links. They are resolved to the
+real pages before scoring, with a cache next to the results. Otherwise every
+Gemini answer would look like it cited Google.
+
+## Engines
+
+| engine | key | default model | how it searches |
+|---|---|---|---|
+| `openai` | `OPENAI_API_KEY` | `gpt-5-mini` | Responses API `web_search` tool |
+| `gemini` | `GEMINI_API_KEY` | `gemini-3.5-flash` | Google Search grounding |
+| `perplexity` | `PERPLEXITY_API_KEY` | `perplexity/sonar` | Responses API `web_search` tool |
+| `claude` | `ANTHROPIC_API_KEY` | `claude-sonnet-5` | Messages API web search tool |
+
+Keys are read from the environment only. They are sent in request headers and
+removed from any error message before it is written, so they never reach the
+results file. An engine with no key is skipped.
+
+Change a model with `--model claude=claude-opus-5`. Pick the models your
+buyers actually use in the apps, or say in the report which ones you used.
+
+Perplexity's API doesn't search unless you ask it to. Without the search tool
+it still answers, fluently, with no citations. This adapter always turns search
+on.
+
+## Commands
+
+```bash
+aeb check QUESTIONS [--set ...]           # validate, show which keys are set and how many calls a run needs
+aeb run QUESTIONS --out DIR [--set ...]   # ask everything, write DIR/answers.jsonl and DIR/report.md
+aeb run QUESTIONS --out DIR --dry-run     # first question of each group, once: a cheap smoke test
+aeb report DIR/answers.jsonl [QUESTIONS]  # rebuild the report, re-scoring if you changed the question file
+aeb noise QUESTIONS --baseline DIR/answers.jsonl --out DIR2 --sample 5
+```
+
+`--anonymise` on `run` and `report` replaces every domain that isn't yours with
+"Source A", "Source B" and so on. Use it before you share a report outside your
+company. `--max-calls` (default 500) stops a run that would make more calls
+than you expected.
+
+`answers.jsonl` gets one line per answer, appended as it arrives, so a stopped
+run keeps what it already paid for. Each line holds the question, engine, model,
+run number, the full answer text, every URL, token usage, cost, and the score
+fields.
+
+## What it costs
+
+A full run of the template is 24 questions x 4 engines x 3 runs = 288 calls.
+Our own run of 360 answers cost $1.39 in API fees without Claude (details in
+`examples/`). Adding Claude costs more, because web search on the Claude API is
+billed per search on top of tokens. Each
+row carries its cost. Perplexity reports the real cost. The others are
+estimated from list prices in `engines.py`, so check them against your bills.
+
+## Measure as a stranger
+
+Run the benchmark from an account and machine that has never been told who you
+are. We learned this from a run through a coding assistant's CLI instead of an
+API. The CLI passed the operator's own git identity to the model, and many brand
+answers then told the reader the company was probably their own. The API
+adapters here send only the question and a one-line instruction to cite sources.
+
+## Example
+
+`examples/synapse-launch-week-2026-09.md` is a real run on one company's own
+site, with the published figures only. 105 of 156 answers that named the
+company cited its site. Of the 204 that didn't name it, 1 did. It is an example
+of the output, not part of the question set.
+
+## Install from source
+
+```bash
+git clone https://github.com/synapsereality/answer-engine-benchmark
+cd answer-engine-benchmark
+pip install .
+aeb --help
+```
+
+Python 3.10 or later. Needs `requests` and `PyYAML`.
+
+## Tests
+
+```bash
+pip install -e ".[test]"
+pytest
+```
+
+33 tests. They mock every API, so they spend nothing and need no keys.
+
+## Licence
+
+MIT. Made by [Synapse](https://synapsereality.io/open-source/answer-engine-benchmark/).

answer_engine_benchmark-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+answer_engine_benchmark/__init__.py,sha256=NSAqFwcQhCATH4ZFhWJZn8w3qxys2NYBtxFn0ZbCAm4,124
+answer_engine_benchmark/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
+answer_engine_benchmark/cli.py,sha256=UAhDHi9PzFQwM7b7HzAwI2hzYsH5K1IWzwuIIDehH6U,8238
+answer_engine_benchmark/engines.py,sha256=45NtCAcsl1UmFH6yQrLsSRCw4CQFzC804Ctk0zWNKCY,10272
+answer_engine_benchmark/questions.py,sha256=8f6d7KGMflNS-_Zqz0x-H2vcoVH_fNMEV2FLXiazaLE,2740
+answer_engine_benchmark/runner.py,sha256=R6df-hgpw600b6xpJyYessT3-pYwX8vGyJ5LkfsLJzc,3389
+answer_engine_benchmark/scoring.py,sha256=Z-3iItdQCdh8v4zOIKVNuoL34N0RYvK7uZsIRcRx4IQ,5328
+answer_engine_benchmark/summary.py,sha256=5S3BajlbW99gsihtYOZBlkrlYMjkv6xuLkmHgWJnefo,7157
+answer_engine_benchmark-0.1.0.dist-info/licenses/LICENSE,sha256=zcvyWWK63eEsOTtAaaeqm_Ng-i9qAwO2I-7sHNcti6c,1077
+answer_engine_benchmark-0.1.0.dist-info/METADATA,sha256=icY0EjsqCGwlCUNyhRf84RaYDx-Cg7hCqn_HEha0-2k,7613
+answer_engine_benchmark-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+answer_engine_benchmark-0.1.0.dist-info/entry_points.txt,sha256=47Vg0cYT6JAvUlhxVtqFECcerh2vtsvS_rr3QIR0Kh4,57
+answer_engine_benchmark-0.1.0.dist-info/top_level.txt,sha256=09-xkykfQfaUL3HnAexMMjl2eXLRULX2C-2KESjzCMY,24
+answer_engine_benchmark-0.1.0.dist-info/RECORD,,

answer_engine_benchmark-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

answer_engine_benchmark-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+aeb = answer_engine_benchmark.cli:main

answer_engine_benchmark-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Synapse Research Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

answer_engine_benchmark-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+answer_engine_benchmark