jevqa 0.1.0__py3-none-any.whl

jevqa/__init__.py

@@ -0,0 +1,2 @@
+"""jevqa: Jev-guided autonomous monkey tester."""
+__version__ = "0.1.0"

jevqa/absence.py

@@ -0,0 +1,129 @@
+"""Bounded UI crawl and spec-grounded absence probes; --self-test runs offline checks."""
+import argparse, json, os, re
+from .llm import claude_json
+from urllib.parse import urlparse
+import requests
+from playwright.sync_api import Error, sync_playwright
+
+JEV_URL = "https://api.typesafe.ai/v1/systemone"
+CONTROLS = 'a,button,[role=button],[role=tab],[role=switch],[role=combobox],[role=menuitem]'
+
+def _snapshot(page):
+    return page.evaluate("""sel => {
+      const visible = e => !!(e.getClientRects().length && getComputedStyle(e).visibility !== 'hidden');
+      const label = e => ((e.labels?.[0]?.innerText) || e.getAttribute('aria-label') || e.innerText || e.placeholder || e.title || e.name || '').trim();
+      return {url:location.href, text:document.body.innerText,
+        controls:[...document.querySelectorAll(sel)].filter(visible).map(label).filter(Boolean),
+        fields:[...document.querySelectorAll('input:not([type=hidden]),select,textarea')].filter(visible).map(label).filter(Boolean)};
+    }""", CONTROLS)
+
+def crawl(page, base, max_pages=8):
+    """Visit same-origin nav links, first detail, and role/mode states (including same URL)."""
+    origin = urlparse(base)[:2]
+    queue, visited, pages = [base], set(), []
+    def capture():
+        snap = _snapshot(page)
+        if snap not in pages and len(pages) < max_pages: pages.append(snap)
+        links = page.locator('nav a[href],header a[href],[role=navigation] a[href],main a[href]').evaluate_all('(els) => els.map(e => e.href)')
+        for link in links:
+            link = link.split('#')[0]
+            if urlparse(link)[:2] == origin and link not in visited and link not in queue: queue.append(link)
+    while queue and len(pages) < max_pages:
+        url = queue.pop(0)
+        if url in visited: continue
+        visited.add(url)
+        page.goto(url, wait_until='domcontentloaded', timeout=15000); page.wait_for_timeout(400)
+        capture()
+        actions = page.locator(CONTROLS).filter(has_text=re.compile(r'^(switch|view as|.*mode\b)|^(owner|traveler|organizer|visitor|employer|seeker|admin)$', re.I))
+        names = actions.all_text_contents() + page.get_by_role('tab').all_text_contents()
+        detail = page.locator('article a[href],[class*=card] a[href],a[href]').filter(has_text=re.compile(r'details|view|read more', re.I)).first
+        if not detail.count(): detail = page.get_by_role('button', name=re.compile(r'^\d+ comments?$|view details|read more', re.I)).first
+        for name in [None] + list(dict.fromkeys(n.strip() for n in names if n.strip())):
+            if len(pages) >= max_pages: break
+            try:
+                target = detail if name is None else page.locator(CONTROLS).filter(has_text=re.compile('^' + re.escape(name) + '$')).first
+                if not target.count(): continue
+                href = target.get_attribute('href')
+                if href and not href.startswith(('/', '#')) and urlparse(href)[:2] != origin: continue
+                target.click(timeout=1500); page.wait_for_timeout(300); capture()
+            except Error:
+                pass  # ponytail: one attempt per target; failed interactions need a deeper crawler.
+            finally:
+                page.goto(url, wait_until='domcontentloaded', timeout=15000); page.wait_for_timeout(200)
+    return pages
+
+def _plan(spec, pages):
+    prompt = f'''Derive absence probes from this specification, NOT from controls already present.
+SPEC: {spec}
+PAGES: {json.dumps(pages, ensure_ascii=False)}
+Return ONLY a JSON list of {{page, verb, expected_control, kind, match_terms, source_quote}}.
+page is the zero-based PAGES index (states can share URLs). kind is control or content.
+source_quote MUST be an exact spec excerpt requiring the feature. No inferred CRUD or extra filters.
+Include missing features. Split alternative statuses and filter dimensions into separate probes.
+A reopened state does not prove a different status exists. An authenticated page need not show signup.
+Control match_terms are literal synonyms, excluding generic words like button or filter.
+For concrete content use kind=content and match_terms with a narrowly scoped Python regex proving
+that content (e.g. currency plus numeric amount for an actual price, not a dollar tier alone).
+Only target relevant states. Skip requirements needing an action result not available in this crawl.'''
+    plan = claude_json(prompt, model='sonnet')
+    if not isinstance(plan, list): raise ValueError('Sonnet must return a list')
+    for p in plan:
+        if not isinstance(p, dict) or not all(k in p for k in ('page','verb','expected_control','kind','match_terms','source_quote')): raise ValueError('Invalid probe schema')
+        if type(p['page']) is not int or not 0 <= p['page'] < len(pages): raise ValueError('Invalid page index')
+        if p['kind'] not in ('control', 'content') or not isinstance(p['match_terms'], list) or not p['match_terms']: raise ValueError('Invalid probe kind/terms')
+        if not all(isinstance(x, str) and x.strip() for x in [p['verb'], p['expected_control'], p['source_quote'], *p['match_terms']]): raise ValueError('Empty probe text')
+    return [p for p in plan if p['source_quote'] in spec]
+
+def _missing(p, page):
+    if p['kind'] == 'content': return not any(re.search(t, page['text'], re.I) for t in p['match_terms'])
+    return not any(re.search(r'(?<!\w)' + re.escape(t) + r'(?!\w)', label, re.I)
+                   for t in p['match_terms'] for label in page['controls'] + page['fields'])
+
+def probe(spec, pages):
+    """One Sonnet plan, deterministic absence checks, one Jev question per candidate."""
+    if not pages: raise ValueError('No pages crawled; absence cannot be established')
+    plan, findings, seen = _plan(spec, pages), [], set()
+    for p in plan:
+        page = pages[p['page']]
+        key = (p['page'], p['expected_control'].casefold())
+        if key in seen or not _missing(p, page): continue
+        seen.add(key)
+        response = requests.post(JEV_URL, headers={'Authorization': f"Bearer {os.environ['TYPESAFE_API_KEY']}"},
+            json={'model': 'jev-latest', 'state': {'requirement': p['source_quote'], 'expected_control': p['expected_control'], **page},
+                  'questions': {'feature_missing': {'type':'noul', 'instructions':
+                    'Is the required control/content absent in this captured state? Allow equivalent labels. '
+                    'Do not infer absence from an unperformed action, unavailable role or closed dialog.'}}}, timeout=30)
+        response.raise_for_status()
+        score = float(response.json()['answers']['feature_missing']['noul'])
+        if not 0 <= score <= 1: raise ValueError('Invalid Jev confidence')
+        if score >= .6:   # ponytail: 0.61 = gold 07-16 (No Longer Relevant); recalibrate if fp rises
+            findings.append({'url':page['url'], 'verb':p['verb'], 'expected_control':p['expected_control'],
+                'evidence':json.dumps({'page':p['page'], 'requirement':p['source_quote'], 'missing_terms':p['match_terms'],
+                                       'controls':page['controls'], 'fields':page['fields'], 'text':page['text']}, ensure_ascii=False),
+                'confidence':round(score, 2)})
+    return findings
+
+def _self_test():
+    page = {'text':'Price $$$', 'controls':['Reopen', 'Credit'], 'fields':['Search posts...']}
+    p = {'kind':'control', 'match_terms':['Edit']}
+    assert _missing(p, page)  # Credit must not match Edit.
+    assert not _missing({**p, 'match_terms':['Search']}, page)
+    assert _missing({**p, 'match_terms':['No Longer Relevant']}, page)
+    p = {'kind':'content', 'match_terms':[r'\$\s*\d+']}
+    assert _missing(p, page)
+    assert not _missing(p, {**page, 'text':'Price $120'})
+    print('absence self-test: PASS')
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--base'); parser.add_argument('--goal'); parser.add_argument('--self-test', action='store_true')
+    args = parser.parse_args()
+    if args.self_test: return _self_test()
+    if not args.base or not args.goal: parser.error('--base and --goal are required')
+    with sync_playwright() as pw:
+        browser = pw.chromium.launch(headless=True)
+        try: findings = probe(args.goal, crawl(browser.new_page(), args.base))
+        finally: browser.close()
+    print(json.dumps(findings, ensure_ascii=False))
+
+

jevqa/checklist_prompt.json

@@ -0,0 +1,40 @@
+{
+ "instructions": "You are writing the QA checklist for a spec\u2192checklist generator being trained on synthetic CRUD-app data. Given a one-paragraph product spec, output 15-30 checklist items \u2014 the exact list a meticulous manual QA engineer would run against a live build of this spec.\n\nRules for each item:\n- One observable behavior per item: a concrete user action + the exact, visible result that must follow (state changed, value displayed, error shown, nothing else affected).\n- Never invent a feature, field, role, or workflow the spec doesn't state or clearly imply. If the spec doesn't mention permissions, don't test permissions.\n- Write like a human QA engineer filing a ticket, not like a requirements doc: plain, specific, occasionally blunt (\"X does not happen\" rather than \"X should occur\").\n\nCover the full surface, in roughly this order:\n1. **Entity lifecycle** \u2014 for every entity the spec implies (not just the primary one): create with valid data \u2192 appears immediately in the right list/view with the right fields; edit \u2192 updates in place, no duplicate; delete \u2192 disappears everywhere, including from any total/summary/report that referenced it.\n2. **Validation & boundaries** \u2014 required fields, invalid values (empty, negative, zero, duplicate), and what visible error appears; don't just assert \"an error occurs,\" assert the record is NOT saved.\n3. **Relationships between entities** \u2014 attaching/detaching one entity to another, cascade behavior on delete (orphaned data), and whether a newly created sub-entity becomes selectable elsewhere immediately.\n4. **List operations** \u2014 filter (single and combined), sort, search, clear filters, and counts (item count matches rows shown).\n5. **Derived data integrity** \u2014 sums, totals, charts, forecasts, reports: verify they match the underlying records and update live when a source record changes.\n6. **Cross-view consistency & navigation** \u2014 moving between list/detail/report views without losing filters or data, detail view shows the full record.\n7. **Roles/permissions** \u2014 only if the spec explicitly names more than one role; test that the restricted action is blocked/hidden for the wrong role and available for the right one.\n8. **Feedback** \u2014 confirmations on save/delete, and that failed actions show specific (not generic) errors.\n\nPrioritize items that exercise an edit operation, a delete-cascade, a confirmation message, or a derived total \u2014 these are the categories most likely to be wrong in a real implementation, so a checklist that skips them isn't doing its job. Vary phrasing and structure across items so the list doesn't read as a template filled in per-entity.",
+ "demos": [
+  {
+   "spec": "I want a community forum where users can register, create profiles with a short bio and interests, start and reply to discussion threads in different categories, and upvote or like helpful posts. People should be able to search for threads by keywords, filter by category, and see a list of trending or most active discussions over a recent period.",
+   "expectations": [
+    "New user can register with email/password and the account is created successfully",
+    "Registering with an already-used email shows a clear error and does not create a duplicate account",
+    "Registering with missing required fields (email/password) is blocked with a validation error",
+    "User can log in with valid credentials and reach their authenticated view",
+    "User can log out and is returned to a logged-out state",
+    "User can create/edit a profile with a short bio and a list of interests, and the saved values persist after page reload",
+    "Profile bio has an enforced length limit and rejects/trims text beyond it",
+    "User can start a new discussion thread by selecting a category, entering a title and body",
+    "A newly created thread appears in the correct category's thread list with correct title, author, and timestamp",
+    "Attempting to create a thread without a title, body, or category is blocked with a validation error",
+    "User can reply to an existing thread and the reply appears in the thread in correct chronological order",
+    "Reply count/timestamp on the thread list updates after a new reply is posted",
+    "User can upvote/like a post or reply once, and the like count increments by exactly one",
+    "User cannot upvote the same post multiple times (duplicate vote is prevented or toggles off)",
+    "Upvote counts are visible on both the thread list and the thread detail view",
+    "User can search threads by keyword and results include threads whose title or body match the keyword",
+    "Searching with a keyword that matches nothing returns an empty state, not an error",
+    "User can filter the thread list by category and only threads in that category are shown",
+    "Clearing the category filter returns the full unfiltered thread list",
+    "Trending/most-active list shows threads ranked by activity (replies/upvotes) within the stated recent period",
+    "Threads outside the trending period's window do not appear in the trending list even if highly active historically",
+    "Clicking a thread in the trending list navigates to that thread's full detail view",
+    "Navigating from the thread list to a thread detail page shows the full original post plus all replies",
+    "Navigating from a user's profile to their created threads (if listed) shows only threads authored by that user",
+    "Editing a thread or reply the user authored saves the updated content and displays it correctly afterward",
+    "Deleting a thread or reply the user authored removes it from the thread list/detail view",
+    "A logged-out user attempting to create a thread, reply, or upvote is prompted to register/log in instead",
+    "Category list/navigation shows all defined categories and selecting one shows only threads in that category",
+    "Submitting a thread or reply shows a confirmation (visual feedback) that the action succeeded",
+    "Invalid actions (e.g., empty search, malformed input) show a clear inline error rather than a silent failure or crash"
+   ]
+  }
+ ]
+}

jevqa/cli.py

@@ -0,0 +1,51 @@
+"""jevqa run <url> --spec <file>: autonomous pre-QA bug hunt, report.md + screenshots in --out."""
+import argparse, os, sys, time
+from pathlib import Path
+
+GOAL = ("Explore this app like a manual tester hunting for bugs; try to complete every feature end to end "
+        "(fill whole forms and submit, open details, use filters). The app was built from this spec: {spec}")
+
+
+def ensure_browser():
+    """First run: fetch Chromium for Playwright (~150 MB, once per machine)."""
+    import subprocess
+    from playwright.sync_api import sync_playwright
+    with sync_playwright() as p:
+        if Path(p.chromium.executable_path).exists(): return
+    print("jevqa: downloading Chromium for Playwright (once)...")
+    subprocess.run([sys.executable, "-m", "playwright", "install", "chromium"], check=True)
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="jevqa", description="Jev-guided monkey tester: finds missing features, validation gaps and dead controls before QA does.")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    r = sub.add_parser("run", help="test one deployed app")
+    r.add_argument("url")
+    r.add_argument("--spec", required=True, help="spec / README / PRD text file the app was built from")
+    r.add_argument("--steps", type=int, default=50, help="action budget (default 50 ≈ 5 min, ~$0.35)")
+    r.add_argument("--out", default="jevqa-runs")
+    r.add_argument("--headed", action="store_true")
+    c = sub.add_parser("config", help="set or show API keys (stored in ~/.config/jevqa/config.json)")
+    c.add_argument("--reset", action="store_true", help="forget saved keys and ask again")
+    a = ap.parse_args(argv)
+    from . import config
+    if a.cmd == "config":
+        if a.reset and config.PATH.exists(): config.PATH.unlink()
+        config.ensure(); config.show(); return
+    config.ensure()
+    ensure_browser()
+    from . import tester, report, telemetry
+    telemetry.track("run_started", steps=a.steps, backend="cli" if not os.environ.get("ANTHROPIC_API_KEY") else "api")
+    spec = Path(a.spec).read_text()
+    argv = ["--base", a.url, "--goal", GOAL.format(spec=spec), "--steps", str(a.steps), "--out", a.out] + (["--headed"] if a.headed else [])
+    t0 = time.time()
+    run = tester.main(argv)
+    path, n, usd = report.write_markdown(run, a.url)
+    print(f"\n{n} findings -> {path}  ({time.time() - t0:.0f}s, ${usd:.2f})")
+    telemetry.track("run_finished", steps=a.steps, findings=n, secs=round(time.time() - t0), usd=round(usd, 2))
+    if os.environ.get("GITHUB_OUTPUT"):
+        with open(os.environ["GITHUB_OUTPUT"], "a") as fh: fh.write(f"report={path}\nfindings={n}\nrun_dir={run}\n")
+
+
+if __name__ == "__main__":
+    main()

jevqa/config.py

@@ -0,0 +1,58 @@
+"""Keys live in ~/.config/jevqa/config.json (env vars override). `ensure()` runs before every `jevqa run`:
+all set -> continue silently; something missing -> ask interactively (or fail with instructions when there is no TTY, e.g. CI)."""
+import getpass, json, os, shutil, sys
+from pathlib import Path
+
+PATH = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config")) / "jevqa" / "config.json"
+KEYS = {"TYPESAFE_API_KEY": "TypeSafe (Jev) API key — https://console.typesafe.ai",
+        "ANTHROPIC_API_KEY": "Anthropic API key — https://console.anthropic.com (or leave empty to use the Claude Code CLI)"}
+
+
+def load():
+    try: return json.loads(PATH.read_text())
+    except (OSError, ValueError): return {}
+
+
+def save(cfg):
+    PATH.parent.mkdir(parents=True, exist_ok=True)
+    PATH.write_text(json.dumps(cfg, indent=1)); PATH.chmod(0o600)
+
+
+def _ask(key, cfg):
+    print(f"\n{key} is not set. {KEYS[key]}")
+    if key == "ANTHROPIC_API_KEY" and shutil.which("claude"):
+        print("  1) enter an Anthropic API key (recommended, ~$0.25 per app)\n  2) use the logged-in Claude Code CLI (billed to your subscription)\n  3) quit")
+        c = input("choice [1]: ").strip() or "1"
+        if c == "2": cfg["CLAUDE_BACKEND"] = "cli"; return True
+        if c != "1": return False
+    v = getpass.getpass(f"{key}: ").strip()
+    if not v: return False
+    cfg[key] = v; return True
+
+
+def ensure():
+    cfg = load()
+    for k, v in cfg.items():
+        if k in KEYS and v: os.environ.setdefault(k, v)
+    missing = [k for k in KEYS if not os.environ.get(k)]
+    if "ANTHROPIC_API_KEY" in missing and cfg.get("CLAUDE_BACKEND") == "cli" and shutil.which("claude"): missing.remove("ANTHROPIC_API_KEY")
+    if not missing: return
+    if not sys.stdin.isatty():
+        sys.exit("missing: " + ", ".join(missing) + f". Set them as env vars / secrets, or run `jevqa config` once on a machine with a terminal (stored in {PATH}).")
+    from . import telemetry; telemetry.track("keys_prompted", missing=missing)
+    changed = False
+    for k in missing:
+        if not _ask(k, cfg): sys.exit("cannot run without it")
+        changed = True
+    if changed and (input(f"save to {PATH}? [Y/n]: ").strip().lower() or "y") == "y": save(cfg)
+    for k in KEYS:
+        if cfg.get(k): os.environ.setdefault(k, cfg[k])
+
+
+def show():
+    cfg = load()
+    for k in KEYS:
+        src = "env" if os.environ.get(k) else ("config" if cfg.get(k) else "-")
+        print(f"{k:20s} {src}")
+    print(f"{'claude backend':20s} {'cli' if cfg.get('CLAUDE_BACKEND') == 'cli' and not os.environ.get('ANTHROPIC_API_KEY') else 'api'}")
+    print(f"file: {PATH}")

jevqa/llm.py

@@ -0,0 +1,38 @@
+"""Claude calls via the Anthropic SDK (replaces `claude -p` CLI): JSON in, parsed JSON out, cost tracked in USAGE."""
+import json, os, re, subprocess
+
+MODELS = {"sonnet": "claude-sonnet-5", "haiku": "claude-haiku-4-5-20251001"}
+PRICE = {"sonnet": (3.0, 15.0), "haiku": (1.0, 5.0)}      # $/MTok in, out
+USAGE = {"claude_usd": 0.0, "claude_calls": 0}
+_client = None
+
+
+def _parse(body):
+    fence = re.search(r"```(?:json)?\s*(.*?)```", body, re.S)
+    for cand in ([fence.group(1)] if fence else []) + [body] + [m.group() for m in re.finditer(r"[\[{].*[\]}]", body, re.S)]:
+        try: return json.loads(cand)
+        except ValueError: pass
+    return None
+
+
+def _cli(prompt, model):
+    """Fallback when ANTHROPIC_API_KEY is absent: the user's logged-in Claude Code CLI (subscription billing)."""
+    out = subprocess.run(["claude", "-p", "--output-format", "json", "--model", model], input=prompt, text=True, capture_output=True).stdout
+    try: res = json.loads(out)
+    except ValueError: return None
+    if not isinstance(res, dict): return None
+    USAGE["claude_usd"] += res.get("total_cost_usd", 0); USAGE["claude_calls"] += 1
+    return _parse(res.get("result", ""))
+
+
+def claude_json(prompt, model="haiku", max_tokens=8000):
+    global _client
+    if not os.environ.get("ANTHROPIC_API_KEY"): return _cli(prompt, model)
+    if _client is None:
+        import anthropic
+        _client = anthropic.Anthropic()                 # ANTHROPIC_API_KEY from env
+    r = _client.messages.create(model=os.environ.get(f"JEVQA_{model.upper()}", MODELS[model]), max_tokens=max_tokens,
+                                messages=[{"role": "user", "content": prompt}])
+    pi, po = PRICE[model]
+    USAGE["claude_usd"] += r.usage.input_tokens / 1e6 * pi + r.usage.output_tokens / 1e6 * po; USAGE["claude_calls"] += 1
+    return _parse("".join(b.text for b in r.content if b.type == "text"))

jevqa/oracles.json

@@ -0,0 +1,7 @@
+{
+ "error_shown": "Does `after.text` show an error, stack trace, or failure notice NOT in `before.text` that is NOT a normal client-side validation message correctly describing the user's own incomplete/invalid input (e.g. 'Last Name is required' after submitting with that field empty is expected, not a bug)?",
+ "action_ignored": "Given `action` and the change from `before.text` to `after.text`, did the UI fail to react as the action's label promises, EXCLUDING: actions on external/social links (Facebook, LinkedIn, X/Twitter) opening a new tab; 'Add to cart'/'Remove' clicks whose only unmet expectation is cart badge count (already tracked); AND text-input actions where `after.text`'s field value snapshot cannot be trusted to reflect live typed state (only flag if a DIFFERENT, unexpected value appears in that field, not merely an unchanged/empty one)?",
+ "inconsistent_state": "Does `after.text` contain data that contradicts itself or the action (e.g. wrong item shown, counts that do not match the visible list, wrong page for the action)?",
+ "broken_content": "Does `after.text` show placeholder text, missing labels, untranslated keys, or broken layout that is NOT simply the tester's own typed input being echoed back, and NOT normal menu/nav items (e.g. 'Dynamic Catalog', 'Lazy Load', 'Spinner', 'Slider') that are legitimate labels rather than garbled output?",
+ "dead_end": "Is the user now stuck: no visible way to continue or go back to the main flow from `after.text`?"
+}

jevqa/report.py

@@ -0,0 +1,84 @@
+"""Deterministic bug-report sentences per finding (the benchmark's REPORTS=template stage) + markdown writer."""
+import hashlib, json, os, requests
+from pathlib import Path
+
+WHAT = {"zeros": "a chart, total or summary shows zero/empty values although data exists", "count_mismatch": "a counter or badge does not match the number of visible items",
+        "wrong_page": "the app shows the wrong page or view for the action", "wrong_item": "a different item than the one acted on is shown or changed", "empty_list": "a list that should contain items is empty",
+        "placeholder": "placeholder, garbled or untranslated text is visible", "duplicate": "a duplicate entry or record appeared", "stale": "old values remain after the update/delete",
+        "missing_feedback": "no confirmation or feedback appeared after the action", "missing_control": "a control or content the spec requires on this page is absent", "other": "something else"}
+VAGUE = {"initial_broken", "inconsistent_state", "broken_content", "expectation_violated", "page_incomplete"}
+
+def what_is_wrong(f, cache):
+    """Post-hoc Jev choice on the stored evidence: name the concrete defect for vague flags."""
+    ev = f.get("evidence") or {}
+    if not (set(f.get("flags") or []) & VAGUE) or not (ev.get("after") or {}).get("text"): return None
+    key = hashlib.md5((f.get("action", "") + ev["after"]["text"][:1500]).encode()).hexdigest()
+    if key not in cache:
+        st = {"action": f.get("action"), "flags": f.get("flags"), "expectation": f.get("expectation"), "before": (ev.get("before") or {}).get("text", "")[:1500], "after": ev["after"]["text"][:2500], "diff": f.get("diff")}
+        r = requests.post("https://api.typesafe.ai/v1/systemone", headers={"Authorization": f"Bearer {os.environ['TYPESAFE_API_KEY']}"}, timeout=30,
+                          json={"model": "jev-latest", "state": st, "questions": {"what": {"type": "choice", "criteria": WHAT,
+                                "instructions": "An oracle flagged this state with `flags`. Judging `after` (and `diff`, `expectation`), which single description names what is CONCRETELY wrong on the page?"}}})
+        r.raise_for_status(); a = r.json()["answers"]["what"]
+        cache[key] = a["choice"] if a["confidence"] >= 0.3 and a["choice"] != "other" else None
+    return cache[key]
+
+def template_report(f, cache):
+    ev, d = f.get("evidence") or {}, f.get("diff") or {}
+    PHRASE = {"action_ignored": "clicking/using it had no visible effect (the UI did not react as the control's label promises)", "invalid_accepted": "an INVALID value was accepted without any validation error", "valid_rejected": "a valid submission was rejected or ignored", "form_values_mangled": "after submitting, the form shows values that are neither the defaults nor what was entered (silently normalized instead of validated)", "detail_mismatch": "the opened detail view shows a value contradicting the one in the clicked card/link", "cap_below_capacity": "the +/- control stops at a limit lower than the availability/capacity the page states", "permission_leak": "a control/content that the spec reserves for another role is visible on this screen", "duplicate_banner": "the success/confirmation message appears twice (the effect fired more than once)",
+              "error_shown": "an error message or failure notice appeared", "inconsistent_state": "the page then showed self-contradictory data (counts, items or page not matching the action)", "broken_content": "the page showed placeholder/garbled/missing content",
+              "dead_end": "the user was left with no way to continue", "expectation_violated": "the spec expectation for this step was NOT met", "initial_broken": "the initial page renders wrong seed data/summary", "page_incomplete": "the page lacks content/controls the spec requires here",
+              "role_leak": "a control reserved for another role is visible", "filter_unchanged": "the list did not change after the filter/search/sort", "filter_inconsistent": "items inconsistent with the selected filter remained", "value_not_propagated": "the submitted record did not appear where it should",
+              "effect_magnitude": "one click changed a counter by 2 or more (double-fire)", "dead_link": "the link led nowhere (URL and page unchanged)", "blank_page": "the page went blank", "js_exception": "a JavaScript exception was thrown", "http_error": "an HTTP error response occurred", "expectation_unreachable": "the feature the spec requires could not be found anywhere"}
+    flags = f.get("flags") or []
+    if "expectation_unreachable" in flags:
+        return f'BUG (missing feature): the spec requires "{f.get("expectation")}", but no control, page or content implementing it exists in the app — the tester searched for it over several steps (last page {f.get("page")}) and found nothing to click, fill or read for it.'
+    what = what_is_wrong(f, cache)
+    out = f'BUG: on {f.get("page")}, after "{f.get("action")}", ' + "; ".join(PHRASE.get(k, k) for k in flags) + (f' — concretely: {WHAT[what]}' if what else "") + "."
+    if f.get("repeated"): out += f' The control was pressed {f["repeated"]["presses"]} times in a row and the value stopped changing (see counters below).'
+    if f.get("expectation"): out += f' Expected (spec): {f["expectation"]}.'
+    d = f.get("diff") or {}
+    if d: out += f' Actual: url_changed={d.get("url_changed")}; text that appeared: {json.dumps(d.get("added_lines", [])[:6], ensure_ascii=False)[:400]}; counters: {d.get("counters_changed")}.'
+    out += f' Details: the tester did: {f.get("action")}.'
+    if f.get("values"): out += f' Submitted values: {json.dumps(f["values"], ensure_ascii=False)[:400]}.'
+    if f.get("rule"): out += f' The payload deliberately violated the rule: {f["rule"]}.'
+    if f.get("expectation"): out += f' Spec expectation for this step: {f["expectation"]}.'
+    out += f' Oracle verdict: {", ".join(f.get("flags") or [])}.'
+    if d: out += f' Observed: url_changed={d.get("url_changed")}; added text: {json.dumps(d.get("added_lines", [])[:6], ensure_ascii=False)[:400]}; counters: {d.get("counters_changed")}.'
+    else: out += f' Visible text after: {(ev.get("after") or {}).get("text", "")[:300]!r}.'
+    for k in ("console_errors", "js_exceptions", "http_errors"):
+        if ev.get(k): out += f' {k}: {json.dumps(ev[k])[:200]}.'
+    return out
+
+
+
+
+def reports_for(rep):
+    """[(finding, sentence)] for a finished run: replay filter, absence/page errors verbatim, template for the rest."""
+    cache, out = {}, []
+    for f in rep["findings"]:
+        if f.get("reproduced") is False and not f.get("deterministic"): continue      # verify-by-replay: drop non-reproducible
+        kind = f.get("kind")
+        if (f.get("report") and f.get("how") == "absence") or kind in ("pageerror", "blank_page"):
+            out.append((f, f.get("report") or f.get("message") or f.get("action") or kind))
+        else:
+            out.append((f, template_report(f, cache)))
+    return out
+
+
+def write_markdown(run, url):
+    rep = json.load(open(run / "findings.json"))
+    items = reports_for(rep)
+    usd = rep["jev"]["input_tokens"] / 1e6 * 0.042 + rep["jev"].get("claude_usd", 0)
+    lines = [f"# jevqa report: {url}", "",
+             f"{len(items)} findings · {rep['steps']} steps · {rep['states']} screens · {rep['secs']:.0f}s · ${usd:.2f} (Jev {rep['jev']['calls']} calls, Claude {rep['jev'].get('claude_calls', 0)})", "",
+             f"Expectations exercised without violation: {len(rep.get('subgoals_done', []))}/{len(rep.get('expectations', []))}. "
+             "Expect roughly 1 in 8 findings to be a confirmed defect; the rest are noise or defects outside the spec. Triage top to bottom.", ""]
+    order = {"absence": 0, "init": 1, "probe": 2, "step": 3}
+    items.sort(key=lambda t: order.get((t[0].get("how") or "").split("(")[0], 9))
+    for i, (f, text) in enumerate(items, 1):
+        flags = ", ".join(f.get("flags") or [])
+        lines += [f"## {i}. {flags or f.get('kind') or 'finding'}", "", text, ""]
+        shot = run / "flagged" / f"{f.get('step', -1):03d}.png"
+        if shot.exists(): lines += [f"![step {f.get('step')}](flagged/{shot.name})", ""]
+    (run / "report.md").write_text("\n".join(lines))
+    return run / "report.md", len(items), usd

jevqa/telemetry.py

@@ -0,0 +1,20 @@
+"""Anonymous usage events to PostHog. Off with JEVQA_TELEMETRY=0. Never sends URLs, specs or report text."""
+import hashlib, json, os, platform, uuid
+from . import __version__
+
+KEY = os.environ.get("JEVQA_POSTHOG_KEY", "phc_pNFJdSzEDFo7XZSom3bvsxLtivVzJqQRNPzVT2qs5ree")   # public write-only project key
+HOST = "https://eu.i.posthog.com"
+
+
+def _id():
+    try: return hashlib.sha256(f"{uuid.getnode()}{platform.node()}".encode()).hexdigest()[:16]
+    except Exception: return "anon"
+
+
+def track(event, **props):
+    if not KEY or os.environ.get("JEVQA_TELEMETRY", "1") == "0" or os.environ.get("CI") and os.environ.get("JEVQA_TELEMETRY") != "1": return
+    try:
+        import requests
+        requests.post(f"{HOST}/capture/", timeout=3, json={"api_key": KEY, "event": event, "distinct_id": _id(),
+                      "properties": {"version": __version__, "os": platform.system(), "ci": bool(os.environ.get("GITHUB_ACTIONS")), **props}})
+    except Exception: pass

jevqa/tester.py

@@ -0,0 +1,623 @@
+"""Jev-guided monkey tester, v8.
+
+Layers: code (abstract state graph, cheap oracles, recording, replay) -> Jev (action choice, sub-goal check, oracles)
+        -> Claude, sparse (Sonnet checklist of expectations once per app, Haiku payload pair once per form) -> Claude report (bench/run.py)
+Run: uv run python monkey.py --steps 50 --base http://localhost:6100 --goal "<spec>"
+"""
+import argparse, hashlib, json, os, random, re, time
+from pathlib import Path
+
+import requests
+from playwright.sync_api import sync_playwright
+
+JEV_URL = "https://api.typesafe.ai/v1/systemone"
+KEY = os.environ.get("TYPESAFE_API_KEY", "")
+SAUCE = "https://www.saucedemo.com"
+BASE = SAUCE           # overridden by --base
+MAX_CANDIDATES = 40
+EPSILON = 0.15         # share of random side-steps
+FLAG_THRESHOLD = 0.7   # noul above this = suspicious state
+MAX_RULES = 8          # ponytail: boundary submits per form; raise if forms with >8 typed fields matter
+URL_CAP = 8            # consecutive steps on one path before forcing an exit to an unvisited path
+SEL = ('a[href], button, input, select, textarea, [role=button], [role=option], [role=menuitem], [role=tab], '
+       '[role=checkbox], [role=switch], [role=slider], [onclick]')
+USAGE = {"input_tokens": 0, "calls": 0}
+ORACLES = json.load(open(Path(__file__).parent / "oracles.json"))
+from . import absence
+FORM_CACHE = {}
+
+
+# ---------------- models ----------------
+def jev(state, questions):
+    for attempt in range(3):
+        try:
+            r = requests.post(JEV_URL, headers={"Authorization": f"Bearer {KEY}"},
+                              json={"model": "jev-latest", "state": state, "questions": questions}, timeout=30)
+            r.raise_for_status(); break
+        except requests.RequestException:
+            if attempt == 2: raise
+            time.sleep(2 * (attempt + 1))
+    d = r.json(); USAGE["input_tokens"] += d.get("usage", {}).get("input_tokens", 0); USAGE["calls"] += 1
+    return d["answers"]
+
+
+from .llm import claude_json, USAGE as CLAUDE_USAGE
+
+
+# ---------------- page model ----------------
+def snapshot(page):
+    text = page.inner_text("body")[:6000]
+    fields = page.evaluate("""() => [...document.querySelectorAll('input:not([type=hidden]), select, textarea')]
+        .map(e => `[field ${e.placeholder || e.name || e.id || e.type}='${e.value}']`).join(' ')""")
+    text += "\n" + fields
+    data = page.evaluate("""(sel) => {
+      const fieldTag = t => ['input', 'textarea', 'select'].includes(t);
+      const DATE = /select (a )?date|pick a date|choose (a )?date|dd[/]mm|mm[/]dd|yyyy|calendar/i;
+      const widget = e => e.getAttribute('role') === 'combobox' ? 'combobox' : e.getAttribute('role') === 'slider' ? 'slider'
+        : ['checkbox', 'switch'].includes(e.getAttribute('role')) ? 'checkbox'
+        : (e.tagName === 'BUTTON' && (DATE.test(e.innerText) || e.querySelector('svg.lucide-calendar, svg.lucide-calendar-days'))) ? 'date' : null;
+      const SUBMIT = /save|creat|submit|add|updat|sign|log ?in|regist|confirm|apply|send|book|place|pay/i;
+      const nFields = el => el.querySelectorAll('input:not([type=hidden]):not([type=checkbox]):not([type=radio]), select, textarea, [role=combobox], [role=slider]').length;
+      const hasSubmit = el => [...el.querySelectorAll('button')].some(b => b.type === 'submit' || SUBMIT.test(b.innerText));
+      // smallest ancestor holding >=2 fields and a submit-like button = the form, whatever markup the app used
+      const formOf = e => { let c = e.parentElement; while (c && c !== document.body && !(nFields(c) >= 2 && hasSubmit(c))) c = c.parentElement; return c && c !== document.body ? c : null; };
+      const containers = new Map();
+      const els = [...document.querySelectorAll(sel)].map((e, i) => [e, i]).filter(([e]) => {
+        const r = e.getBoundingClientRect();
+        if (!(r.width > 0 && r.height > 0) || e.disabled) return false;
+        const cx = r.left + r.width / 2, cy = r.top + r.height / 2;
+        if (cy < 0 || cy > innerHeight) return true;   // off-screen centre: Playwright scrolls to it; overlay check impossible
+        const hit = document.elementFromPoint(cx, cy);
+        return hit && (hit === e || e.contains(hit));
+      }).map(([e, i]) => {
+        const tag = e.tagName.toLowerCase();
+        const box = e.closest('[class*=item], li, tr, form, section, article');
+        const name = box && box !== e ? box.querySelector('[class*=name], [class*=title], h1, h2, h3, h4') : null;
+        const near = e.closest('div, li, td');
+        const label = (e.labels && e.labels[0] ? e.labels[0].innerText : '') || (near && near.querySelector('label') ? near.querySelector('label').innerText : '');
+        const w = widget(e);
+        const cont = fieldTag(tag) || tag === 'button' || w ? formOf(e) : null;
+        let cid = null;
+        if (cont) { if (!containers.has(cont)) containers.set(cont, containers.size); cid = containers.get(cont); }
+        const svg = e.querySelector('svg');
+        return {
+          i, tag, type: w || e.type || '', widget: w || undefined, cid, href: tag === 'a' ? e.getAttribute('href') : null,
+          text: (fieldTag(tag) || w ? (label || e.placeholder || e.getAttribute('aria-label') || e.name || e.id || e.value)
+                               : (e.innerText || e.value || e.getAttribute('aria-label') || e.title || e.name || e.id
+                                  || (svg ? svg.getAttribute('class') : '') || '')).trim().slice(0, 60),
+          ctx: name ? name.innerText.trim().slice(0, 40) : '',
+          attrs: fieldTag(tag) ? {required: e.required || undefined, min: e.min || undefined, max: e.max || undefined,
+                                  placeholder: e.placeholder || undefined,
+                                  options: tag === 'select' ? [...e.options].map(o => o.text).slice(0, 12) : undefined} : undefined
+        };
+      });
+      const forms = {};
+      for (const [cont, cid] of containers) {
+        const f = els.filter(x => x.cid === cid && (fieldTag(x.tag) || x.widget) && !['checkbox', 'radio', 'submit', 'button', 'file'].includes(x.type));
+        const btns = els.filter(x => x.cid === cid && x.tag === 'button' && !x.widget);
+        const btn = btns.find(x => x.type === 'submit' && !/cancel|close/i.test(x.text)) || btns.find(x => SUBMIT.test(x.text));
+        if (f.length >= 2) {
+          const h = cont.querySelector('h1, h2, h3, h4, [class*=title]');
+          forms[cid] = {cid, name: (h ? h.innerText : (btn ? btn.text : 'form')).trim().slice(0, 50), fields: f.map(x => x.i), submit: btn ? btn.i : null};
+        }
+      }
+      return {els, forms};
+    }""", SEL)
+    return {"url": page.url.replace(BASE, ""), "title": page.title(), "text": text, "elements": data["els"], "forms": data["forms"]}
+
+
+def norm(s): return re.sub(r"\d+", "#", s or "")
+
+
+def counters(text): return set(re.findall(r"[A-Za-z][A-Za-z ]{2,30}[:#]?\s*\(?\d+\)?", text))   # "Total Events 3", "Upcoming (0)"
+
+
+def counter_map(text): return dict(re.findall(r"([A-Za-z][A-Za-z ]{2,30}?)\s*[:#]?\s*\(?\$?(-?[\d,]+(?:\.\d+)?)k?\)?(?=\s|$)", text))
+
+
+def diff(before, after):
+    """Code-computed change between two snapshots: what the oracle points at instead of two text blobs (v9)."""
+    b, a = before["text"].splitlines(), after["text"].splitlines()
+    bs, as_ = set(b), set(a); cb, ca = counter_map(before["text"]), counter_map(after["text"])
+    d = {"url_changed": before["url"] != after["url"],
+         "added_lines": [x for x in a if x.strip() and x not in bs][:30], "removed_lines": [x for x in b if x.strip() and x not in as_][:30],
+         "counters_changed": {k: [cb[k], ca[k]] for k in cb if k in ca and cb[k] != ca[k]}}
+    d["changed"] = d["url_changed"] or bool(d["added_lines"]) or bool(d["removed_lines"]) or bool(d["counters_changed"])
+    return d
+
+
+def state_sig(snap):
+    """Abstract state: normalized path + set of interactive elements; free text ignored."""
+    path = norm(snap["url"].split("?")[0])
+    els = sorted({f'{e["tag"]}:{e["type"]}:{norm(e["text"])[:30]}' for e in snap["elements"]})[:80]
+    return hashlib.md5((path + "|" + "|".join(els)).encode()).hexdigest()[:10]
+
+
+def locator(page, idx): return page.locator(SEL).nth(idx)
+
+
+def describe(el):
+    ctx = f' ({el["ctx"]})' if el["ctx"] and el["ctx"] not in el["text"] else ""
+    return f'{el["tag"]}{"[" + el["type"] + "]" if el["type"] else ""} "{el["text"]}"{ctx}'
+
+
+def login(page, user):
+    page.goto(BASE); page.fill("#user-name", user); page.fill("#password", "secret_sauce"); page.click("#login-button")
+    page.wait_for_load_state("networkidle")
+
+
+# ---------------- actions ----------------
+def drive(page, loc, kind, value=None, past=False):
+    """Radix/shadcn widgets: combobox -> option (by text, else first enabled); date trigger -> enabled day after today (before, when past);
+    slider -> ArrowRight x3; role checkbox/switch -> click. Returns the option/day text picked."""
+    if kind == "combobox":
+        loc.click(timeout=3000); page.wait_for_timeout(300)
+        opts = page.locator("[role=option]:not([aria-disabled=true]):not([data-disabled])")
+        pick = opts.filter(has_text=str(value)) if value else opts
+        pick = pick.first if pick.count() else opts.first
+        text = pick.inner_text(timeout=1500); pick.click(timeout=1500); return text
+    if kind == "date":
+        loc.click(timeout=3000); page.wait_for_timeout(300)
+        if past:
+            page.locator("button[name=previous-month]").first.click(timeout=1500); page.wait_for_timeout(200)
+        days = page.locator("[role=grid] [role=gridcell]:not([disabled]):not([aria-disabled=true])")
+        n = days.count(); text = None
+        if n:
+            day = days.nth(0 if past else min(n - 1, 3))    # earliest enabled day when past is wanted; a few days after the first enabled otherwise
+            text = day.inner_text(timeout=1500); day.click(timeout=1500); page.wait_for_timeout(200)
+        if page.locator("[role=grid]").count(): loc.click(timeout=1500)   # popover still open: toggle it shut
+        return text
+    if kind == "slider":
+        loc.focus(timeout=3000)
+        for _ in range(3): page.keyboard.press("ArrowRight")
+        return "+3"
+    loc.click(timeout=3000); return "toggled"       # role checkbox / switch
+
+
+def act(page, el, value=None):
+    """One primitive action; returns the value typed (for replay)."""
+    loc = locator(page, el["i"])
+    if el.get("widget"): value = drive(page, loc, el["widget"], value)
+    elif el["tag"] == "textarea" or (el["tag"] == "input" and el["type"] in ("text", "email", "password", "search", "url", "tel", "number", "date", "")):
+        if value is None:
+            value = {"number": "12", "date": "2026-10-01", "email": "qa@example.com"}.get(el["type"]) or random.choice(["Test", "", "12345", "<script>x</script>", "Іван"])
+        if el["type"] == "number": value = re.sub(r"[^\d.-]", "", value) or "1"
+        loc.fill(value, timeout=3000)
+    elif el["tag"] == "select":
+        loc.select_option(index=1)
+    else:
+        loc.click(timeout=3000)
+    page.wait_for_timeout(600)
+    return value
+
+
+def form_payloads(snap, form, spec):
+    fields = [e for e in snap["elements"] if e["i"] in form["fields"]]
+    key = hashlib.md5((norm(snap["url"]) + form["name"] + "|".join(f["text"] for f in fields)).encode()).hexdigest()[:10]
+    if key not in FORM_CACHE:
+        meta = [{"label": f["text"], "type": f["type"] or f["tag"], **{k: v for k, v in (f["attrs"] or {}).items() if v}} for f in fields]
+        FORM_CACHE[key] = claude_json(f"""A web app was built from this spec:
+{spec}
+
+Today is {time.strftime("%Y-%m-%d")}. A tester is on page {snap["url"]} at the form "{form["name"]}" with these fields:
+{json.dumps(meta, ensure_ascii=False)}
+
+Produce test payloads. Values must be strings; for select fields use one of the listed options verbatim; dates as YYYY-MM-DD; numbers as plain digits.
+Return ONLY JSON: {{"valid": {{"<label>": "<realistic valid value>", ...all fields...}},
+ "invalid": {{"field": "<label>", "value": "<value violating a business rule or boundary implied by the spec/field (e.g. 0 or -1 for a price/quantity/count/budget, rating 7 on a 0-5 scale, a past date where the future is required or a far-future date where the past is required, a range whose start is after its end, malformed email, 300-char name)>", "rule": "<the rule it violates, one sentence>"}}}}""") or {}
+    if not isinstance(FORM_CACHE[key], dict) or not isinstance(FORM_CACHE[key].get("valid"), dict): FORM_CACHE[key] = {}
+    return fields, FORM_CACHE[key]
+
+
+def boundary_rules(fields, valid):
+    """Rule-based boundary matrix per field type, no LLM: [(label, value, rule, past)]."""
+    def kind(f):
+        typ, lab = (f["type"] or "").lower(), f["text"].lower()
+        if typ == "number" or re.search(r"budget|guest|price|quantity|qty|amount|count|salary|capacity|party|size|rating|age", lab): return "number"
+        if typ == "date" or f.get("widget") == "date" or "date" in lab: return "date"
+        if typ == "email" or "email" in lab: return "email"
+        if typ == "tel" or "phone" in lab: return "phone"
+        return None
+    today = time.time(); rules = []
+    for f in fields:
+        lab, k = f["text"], kind(f)
+        if k == "number": rules += [(lab, "0", f"{lab} must be a positive number, 0 is not allowed", False), (lab, "-1", f"{lab} cannot be negative", False)]
+        elif k == "date":
+            rules.append((lab, time.strftime("%Y-%m-%d", time.localtime(today - 86400)), f"{lab} cannot be in the past (yesterday)", True))
+            if not f.get("widget"): rules.append((lab, time.strftime("%Y-%m-%d", time.localtime(today + 50 * 365 * 86400)), f"{lab} must be realistic, 50 years ahead is invalid", False))
+        elif k == "email": rules.append((lab, "notanemail", f"{lab} must be a valid email address", False))
+        elif k == "phone": rules.append((lab, "abc", f"{lab} must be a valid phone number", False))
+    labs = [f["text"] for f in fields]
+    for s in labs:                                               # range: start > end, first matching partner
+        e = next((x for x in labs if x != s and re.search(r"end|\bto\b|max", x, re.I)), None) if re.search(r"start|from|min", s, re.I) else None
+        if e: rules.append(((s, e), ("2099-12-31", "2000-01-01") if "date" in s.lower() else ("999999", "1"), f"{s} must not be greater than {e}", False)); break
+    return rules[:MAX_RULES]
+
+
+def run_form(page, snap, form, spec, opener=None):
+    """Fill the form once per boundary rule (a validated form stays open), then all-valid, then the same valid payload again (duplicate). Returns [(phase, values, rule, after)]."""
+    fields, pay = form_payloads(snap, form, spec)
+    if not pay.get("valid"): return []
+    by_label = {f["text"]: f for f in fields}
+
+    def fill(values, past=False):
+        for label, v in values.items():
+            f = by_label.get(label)
+            if not f: continue
+            try:
+                loc = locator(page, f["i"])
+                if f.get("widget"): drive(page, loc, f["widget"], v, past=past and f["widget"] == "date")
+                elif f["tag"] == "select":
+                    try: loc.select_option(label=str(v), timeout=1500)
+                    except Exception: loc.select_option(index=1, timeout=1500)
+                else: loc.fill(str(v), timeout=1500)
+            except Exception: pass
+        for cb in [e for e in snap["elements"] if e["cid"] == form.get("cid") and e["type"] == "checkbox"]:
+            try: locator(page, cb["i"]).check(timeout=1000)
+            except Exception: pass
+
+    def complete_required():
+        """Native validation blocks submit silently: fill every EMPTY :invalid field (the deliberately invalid one has a value and is left alone)."""
+        inv = page.locator(":invalid"); n = 0
+        for k in range(min(inv.count(), 12)):
+            e = inv.nth(k)
+            try:
+                tag, typ, val = e.evaluate("e => [e.tagName.toLowerCase(), e.type || '', e.value]")
+                if val: continue
+                if tag == "select": e.select_option(index=1, timeout=1000)
+                elif typ == "checkbox": e.check(timeout=1000)
+                else: e.fill({"number": "1", "date": time.strftime("%Y-%m-%d", time.localtime(time.time() + 30 * 86400)), "email": "qa@example.com", "tel": "+14155550123", "url": "https://example.com"}.get(typ, "Test value"), timeout=1000)
+                n += 1
+            except Exception: pass
+        return n
+
+    def submit():
+        for _ in range(2):
+            pre = snapshot(page)
+            try:
+                if form["submit"] is not None: locator(page, form["submit"]).click(timeout=3000)
+                else: page.keyboard.press("Enter")
+            except Exception: pass
+            page.wait_for_timeout(900)
+            if page.url.replace(BASE, "") != snap["url"] or not complete_required(): break   # navigated, or nothing left to complete
+        after = snapshot(page)
+        after["blocked"] = after["url"] == pre["url"] and after["text"] == pre["text"] and counters(after["text"]) == counters(pre["text"])
+        return after
+
+    phases = []
+    def closed(): return len({f["text"] for f in fields} & {e["text"] for e in phases[-1][3]["elements"]}) < 2   # form gone (dialog closed / navigated)
+    def reopen():
+        """Click the element that made the form appear (dialog forms close on accept); rebind to the fresh snapshot."""
+        nonlocal snap, form, fields, by_label
+        cur = snapshot(page)
+        names = ([opener["text"]] if opener and opener.get("text") else []) + \
+                [e["text"] for e in cur["elements"] if e["tag"] == "button" and re.search(r"edit|update|modify|details|add|new|create|start", e["text"], re.I)][:3]
+        for name in names:
+            try: page.get_by_text(name, exact=True).first.click(timeout=2000); page.wait_for_timeout(700)
+            except Exception: continue
+            s2 = snapshot(page); f2 = next((f for f in s2["forms"].values() if f["name"] == form["name"]), None)
+            if f2:
+                snap, form = s2, f2; fields = [e for e in s2["elements"] if e["i"] in f2["fields"]]; by_label = {f["text"]: f for f in fields}
+                return True
+        return False
+    for label, value, rule, past in boundary_rules(fields, pay["valid"]):
+        values = {**pay["valid"], **(dict(zip(label, value)) if isinstance(label, tuple) else {label: value})}
+        fill(values, past=past); phases.append(("invalid", values, rule, submit()))
+        if closed() and not reopen(): return phases
+    fill(pay["valid"]); phases.append(("valid", pay["valid"], "", submit()))
+    if not phases[-1][3]["blocked"] and (not closed() or reopen()):
+        fill(pay["valid"]); phases.append(("invalid", pay["valid"], "duplicate: an identical record was just created, a second identical submit must be rejected", submit()))
+    return phases
+
+
+# ---------------- oracles ----------------
+def cheap_oracles(el, before, after, http_err, js_err):
+    hits = {}
+    if http_err: hits["http_error"] = 1.0
+    if js_err: hits["js_exception"] = 1.0
+    if len(after["text"].strip()) < 20: hits["blank_page"] = 1.0
+    href = el.get("href") or ""
+    if el["tag"] == "a" and href.startswith("/") and href.split("?")[0] != before["url"].split("?")[0] \
+            and after["url"] == before["url"] and after["text"] == before["text"]:
+        hits["dead_link"] = 1.0
+    return hits
+
+
+def jev_oracles(oracle_state, extra=None):
+    qs = {k: {"type": "noul", "instructions": v} for k, v in ORACLES.items()} | (extra or {})
+    return {k: round(v["noul"], 2) for k, v in jev(oracle_state, qs).items()}
+
+
+# ---------------- expectations (checklist generated once per app; drives both the chooser and the oracle) ----------------
+TUNED = Path(__file__).parent / "checklist_prompt.json"   # DSPy/MIPROv2-optimized instruction (+demo), see bench/dspy_checklist.py
+
+
+def checklist(spec):
+    """Once per app: expectations + concrete UI steps to exercise each (Sonnet). Steps are matched to real elements by Jev at run time."""
+    head = ("Write the test checklist a meticulous manual QA engineer would verify: 15-30 items. Cover every feature end to end (create -> appears "
+            "in the right place with the right data -> edit -> delete), business rules/boundaries implied by the spec, navigation, and feedback. Do not invent features.")
+    if TUNED.exists():
+        t = json.load(open(TUNED)); head = t["instructions"]
+        if t["demos"]: head += "\n\nExample of the expected quality for another spec:\nSpec: " + t["demos"][0]["spec"] + "\nChecklist: " + json.dumps(t["demos"][0]["expectations"], ensure_ascii=False)
+    res = claude_json(f"""{head}
+
+Now the spec under test:
+{spec}
+
+Each item = ONE observable expected behaviour plus the concrete UI steps to exercise it, starting from the app's home page.
+Cover EVERY user role or mode the spec names (the customer/visitor side AND the admin/owner/organizer side); when the app switches roles via a toggle or link, include that switch in the steps.
+For every feature the spec names, include one item whose expectation is that the control for it EXISTS on the relevant page (delete/edit buttons, filters, history views), so missing features are caught.
+Steps are short imperative UI actions with concrete values, e.g. "click 'Add Product'", "fill 'Price' with '-5'", "select 'Fitness' in 'Category'",
+"fill the form and submit", "open the first product". 2-6 steps each; the LAST step is the action whose result proves or breaks the expectation.
+Return ONLY JSON: [{{"expect": "<what must be visible after the last step>", "steps": ["...", "..."]}}, ...]""", model="sonnet")
+    if isinstance(res, dict): res = next((v for v in res.values() if isinstance(v, list)), [])
+    out = []
+    for x in res or []:
+        if isinstance(x, dict) and isinstance(x.get("expect"), str):
+            out.append({"expect": x["expect"], "steps": [t for t in (x.get("steps") or []) if isinstance(t, str)][:6]})
+        elif isinstance(x, str): out.append({"expect": x, "steps": []})
+    return out[:30]
+
+
+def step_value(step):
+    m = re.search(r"""(?:with|to|=|:)\s*['"“]([^'"”]+)['"”]""", step)
+    return m.group(1) if m else None
+
+
+# ---------------- main ----------------
+def main(argv=None):
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--steps", type=int, default=30)
+    ap.add_argument("--user", default="problem_user")  # saucedemo: standard_user | problem_user | error_user
+    ap.add_argument("--headed", action="store_true")
+    ap.add_argument("--base", default=SAUCE)
+    ap.add_argument("--goal", default="Explore this e-commerce app like a manual tester hunting for bugs: prefer actions that progress a purchase flow, touch untested features, or submit forms. Avoid logout.")
+    ap.add_argument("--out", default="runs")
+    a = ap.parse_args(argv)
+    global BASE; BASE = a.base.rstrip("/")
+    random.seed(hashlib.md5(a.goal.encode()).hexdigest())   # same app -> same random side-steps; re-runs differ only where the app or Jev does
+    home = BASE + ("/inventory.html" if BASE == SAUCE else "/")
+
+    t0 = time.time()
+    run = Path(a.out) / time.strftime("%Y%m%d-%H%M%S")
+    (run / "flagged").mkdir(parents=True)
+    log = open(run / "steps.jsonl", "w")
+    G = {}                                            # abstract state -> action -> {"n", "ineff"}
+    visited, flagged_pairs, findings_raw = {}, set(), []
+    scen = checklist(a.goal)                      # [{expect, steps}]
+    if scen:                                      # triage: exercise the expectations most likely to be broken first (Jev score, one call)
+        risk = jev({"spec": a.goal, "expectations": {f"E{i}": x["expect"] for i, x in enumerate(scen)}},
+                   {f"E{i}": {"type": "score", "instructions": f"How likely is expectation `expectations.E{i}` to be VIOLATED in an AI-generated (vibe-coded) implementation of `spec`? Multi-step flows, edits/deletes propagating to lists and totals, business-rule validation and cross-page consistency break most often; static display and simple navigation rarely do.",
+                               "criteria": ["almost certainly works", "probably works", "coin flip", "probably broken", "almost certainly broken"]} for i in range(len(scen))})
+        scen = [x for _, x in sorted(zip([-risk[f"E{i}"]["score"] for i in range(len(scen))], scen), key=lambda t: t[0])]
+    subgoals, done, checked = [x["expect"] for x in scen], [], set()   # done = exercised w/o violation; checked = judged either way
+    steps_of = {x["expect"]: x["steps"] for x in scen}
+    NEG = [t for t in subgoals if re.search(r"not (be )?(shown|visible|available|usable|displayed|offered|allowed|permitted)|cannot|must not|hidden|is refused|or is refused|only the (poster|author|owner|creator|admin)|only (an? )?(admin|owner|author)", t, re.I) and not re.search(r"\bexists?\b", t, re.I)]   # negative items: absence there is a pass (v14 strict regex; \bno\b matched "No Longer Relevant")
+    seen_lines = set()                                  # every screen line seen in the run: a control named by a stalled scenario that never appeared anywhere = missing feature (v14 absence path)
+    print(f"    checklist: {len(subgoals)} expectations, {sum(len(x['steps']) for x in scen)} steps")
+    E = {f"E{i}": t for i, t in enumerate(subgoals)}
+    sg_steps = sg_step_i = sg_budget = streak = 0; streak_path = None; seen_paths = set(); stalled = []; done_tuples = set(); opener = None
+    console, failed, http_err, js_err = [], [], [], []
+
+    def record(step, how, action, before, after, probs, hits, extra, page):
+        key = (state_sig(before), action)
+        if hits and key in flagged_pairs: hits = {}
+        if hits: flagged_pairs.add(key)
+        rec = {"step": step, "state": key[0], "how": how, "action": action, "from": before["url"], "to": after["url"],
+               "oracles": probs, "flags": list(hits), **extra}
+        log.write(json.dumps(rec, ensure_ascii=False) + "\n"); log.flush()
+        mark = " <-- FLAG " + ",".join(hits) if hits else ""
+        print(f'{step:03d} {how:11s} {action[:45]:45s} {before["url"][-22:]:>22s} -> {after["url"][-22:]:<22s}{mark}')
+        if hits:
+            page.screenshot(path=run / "flagged" / f"{step:03d}.png")
+            findings_raw.append({**rec, "evidence": {"action": action, "before": {"url": before["url"], "text": before["text"][:2000]},
+                                                     "after": {"url": after["url"], "text": after["text"][:3000]},
+                                                     "console_errors": console[:5], "failed_requests": failed[:5], "http_errors": http_err[:5],
+                                                     "js_exceptions": js_err[:3], **extra}})
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=not a.headed)
+        page = browser.new_page()
+        page.on("console", lambda m: console.append(m.text[:200]) if m.type == "error" else None)
+        page.on("pageerror", lambda e: js_err.append(str(e)[:200]))
+        page.on("requestfailed", lambda r: failed.append(r.url[:200]))
+        page.on("response", lambda r: http_err.append(f"{r.status} {r.url[:150]}")
+                if r.status >= 400 and r.request.resource_type in ("xhr", "fetch", "document") else None)
+        if BASE == SAUCE: login(page, a.user)
+        else: page.goto(home); page.wait_for_load_state("networkidle")
+
+        step, absent = 0, []
+        if absence and BASE != SAUCE:                     # crawl nav/detail/role states once, one step charged per page; absences become deterministic findings
+            try:
+                pages = absence.crawl(page, home); absent = absence.probe(a.goal + "\n" + "\n".join(subgoals), pages)   # checklist expectations are a second quote source (spec rarely names delete/edit)
+                for pg in pages:
+                    log.write(json.dumps({"step": step, "state": "", "how": "crawl", "action": f'crawl {pg["url"].replace(BASE, "")}', "from": pg["url"].replace(BASE, ""), "to": pg["url"].replace(BASE, ""), "oracles": {}, "flags": []}) + "\n"); step += 1
+                print(f'    absence: {len(pages)} pages crawled ({step} steps charged), {len(absent)} absent: {[x["expected_control"] for x in absent]}')
+            except Exception as e: print(f"    absence failed: {str(e)[:120]}")
+            page.goto(home); page.wait_for_load_state("networkidle")
+        init = snapshot(page); blank = {"url": "", "text": "", "elements": []}
+        probs = jev_oracles({"action": "open the app's initial page (fresh load, no interaction yet)", "before": blank, "after": {"url": init["url"], "text": init["text"][:3000]}, "spec": a.goal},
+                            {"initial_broken": {"type": "noul", "instructions": "Judging `after.text` (the first page of an app described by `spec`): does it show clearly wrong seed data or rendering, e.g. a chart/summary whose values are all 0 or $0 while lists on the same page hold items, NaN/undefined/null, or totals contradicting listed rows? Answer no for a merely empty app with no data anywhere."}})
+        hits = {k: v for k, v in probs.items() if k in ("initial_broken", "broken_content", "inconsistent_state") and v >= FLAG_THRESHOLD}
+        record(step, "init", "initial page", blank | {"url": init["url"]}, init, probs, hits, {"value": None, "expectation": None}, page); step += 1
+        while step < a.steps:
+            if not page.url.startswith(BASE): page.goto(home)
+            if BASE == SAUCE and "user-name" in page.content() and page.url.rstrip("/") == BASE: login(page, a.user)
+            before = snapshot(page); seen_lines.update(before["text"].splitlines())
+            sig = state_sig(before)
+            visited[sig] = visited.get(sig, 0) + 1
+            mem = G.setdefault(sig, {})
+
+            # --- current target: next expectation not yet exercised, round-robin with patience ---
+            pending = [t for t in subgoals if t not in checked]
+            if pending and sg_steps >= sg_budget:                       # overran its share: stalled, never retried
+                if sg_steps: stalled.append(pending[0]); checked.add(pending[0]); pending.pop(0)
+                sg_steps = sg_step_i = 0
+                if pending: sg_budget = max(len(steps_of.get(pending[0], [])), (a.steps - step) // len(pending))   # ponytail: floor(remaining/pending) alone is 1 step for 30 items at budget 50; an item at least gets its own scenario length
+            subgoal = pending[0] if pending else None
+            steps = steps_of.get(subgoal, []) if subgoal else []
+            scen_step = steps[sg_step_i] if sg_step_i < len(steps) else None
+            final_step = scen_step is not None and sg_step_i == len(steps) - 1
+
+            # --- candidates: primitives + whole-form macros, ranked by graph memory (fresh > useful > any) ---
+            cands = [(describe(e), ("el", e)) for e in before["elements"][:MAX_CANDIDATES]]
+            cands += [(f'form "{f["name"]}" (fill all {len(f["fields"])} fields and submit)', ("form", f)) for f in before["forms"].values()]
+            if not cands:
+                page.keyboard.press("Escape"); page.wait_for_timeout(300)
+                if not snapshot(page)["elements"]: page.goto(home)
+                continue
+            path = norm(before["url"].split("?")[0])
+            cands = [c for c in cands if (path, c[0]) not in done_tuples or c[1][1].get("tag") in ("input", "textarea")] or cands   # never redo an identical (route, action) unless typed values differ
+            fresh = [c for c in cands if c[0] not in mem]
+            useful = [c for c in cands if c[0] in mem and not mem[c[0]]["ineff"]]
+            pool = fresh or useful or cands
+            picked = None
+            seen_paths.add(path)
+            streak = streak + 1 if path == streak_path else 1; streak_path = path
+            if streak > URL_CAP:                                   # trapped on one page: leave via a link to a path never visited
+                exits = [c for c in cands if c[1][0] == "el" and (c[1][1].get("href") or "").startswith("/") and norm(c[1][1]["href"].split("?")[0]) not in seen_paths]
+                if exits: picked = random.choice(exits); how = "escape"
+                else: page.keyboard.press("Escape"); page.goto(home); streak = 0; continue
+            if scen_step and not picked:
+                wide = [(describe(e), ("el", e)) for e in before["elements"][:120]] + cands[len(cands) - len(before["forms"]):]
+                crit = {f"e{i}": c[0] for i, c in enumerate(wide)} | {"none": "no candidate performs this step"}
+                ans = jev({"page": before["url"], "visible_text": before["text"][:2500], "scenario_step": scen_step, "expectation": subgoal},
+                          {"el": {"type": "choice", "criteria": crit, "instructions": "Which candidate element performs `scenario_step` on this page? Choose none if no candidate does."}})["el"]
+                sg_step_i += 1                                  # advance either way: a missing step is skipped, not retried
+                best, p_best = max(((k, v) for k, v in ans["probabilities"].items() if k != "none"), key=lambda kv: kv[1], default=("none", 0))
+                if best != "none" and p_best >= 0.25:
+                    picked = wide[int(best[1:])]; how = f'step({p_best:.2f})'
+                elif os.environ.get("MONKEY_DEBUG"):
+                    print(f'    step miss: "{scen_step[:60]}" ({ans["confidence"]:.2f}) on {before["url"]}')
+            if picked:
+                desc, (kind, obj) = picked
+            elif not pending and random.random() < EPSILON:              # side-steps only once the queue is empty
+                desc, (kind, obj) = random.choice(pool); how = "random"
+            else:
+                ans = jev({"page": before["url"], "visible_text": before["text"][:2500], "visits_to_this_state": visited[sig],
+                           "goal": a.goal, "current_subgoal": subgoal or "explore untested features"},
+                          {"next": {"type": "choice", "criteria": {f"e{i}": c[0] for i, c in enumerate(pool)},
+                                    "instructions": "Which element should the tester interact with next to make progress toward `current_subgoal` (and the overall `goal`)? Prefer a whole-form action when a form is the way forward."}})["next"]
+                desc, (kind, obj) = pool[int(ans["choice"][1:])]; how = f'jev({ans["confidence"]:.2f})'
+            mem.setdefault(desc, {"n": 0, "ineff": False}); mem[desc]["n"] += 1
+            console.clear(); failed.clear(); http_err.clear(); js_err.clear()
+            def judge(st, extra_q, dead_click=False):
+                """Which expectation does this step exercise (Jev choice, or the scenario's own on its final step), is it violated (Jev noul), plus generic + extra oracles."""
+                exp = subgoal if (final_step and picked) else None
+                if E and not exp:
+                    rel = jev({k: v for k, v in st.items() if k != "current_subgoal"},
+                              {"rel": {"type": "choice", "criteria": E | {"none": "no listed expectation is exercised or checkable from this action and its result"},
+                                       "instructions": "Which expectation does this action and its visible result exercise (make checkable), if any?"}})["rel"]
+                    if rel["choice"] != "none" and rel["confidence"] >= 0.4: exp = E[rel["choice"]]
+                q = dict(extra_q)
+                if exp:
+                    st = st | {"expectation": exp}
+                    q["expectation_violated"] = {"type": "noul", "instructions": "`expectation` states what must be visible after this kind of action. Judging only `before` -> `after` for `action`, is the expectation clearly VIOLATED (the required result did not appear, or something contradicting it appeared)? Answer no if the action does not fully exercise it yet or the result matches."}
+                    q["expectation_verified"] = {"type": "noul", "instructions": "Judging only `before` -> `after` for `action`, did the result required by `expectation` clearly APPEAR, so the expectation is now positively confirmed? Answer no if the action merely navigated, opened a form, or changed nothing relevant."}
+                probs = jev_oracles(st, q)
+                strong = {k for k in probs if k in extra_q or k == "expectation_violated"} | ({"action_ignored"} if dead_click else set())
+                hits = {k: v for k, v in probs.items() if k != "expectation_verified" and v >= FLAG_THRESHOLD and (k in strong or probs.get("expectation_violated", 0) >= FLAG_THRESHOLD)}
+                if exp:
+                    verified = probs.get("expectation_verified", 0) >= 0.6 and st.get("diff", {}).get("changed", True)
+                    if verified or "expectation_violated" in hits: checked.add(exp)      # resolved either way; otherwise it stays pending
+                    if verified and "expectation_violated" not in hits and exp not in done: done.append(exp)
+                return probs, hits, exp
+
+            if kind == "form":
+                phases = run_form(page, before, obj, a.goal, opener)
+                mem[desc]["ineff"] = True                       # one-shot: same payloads would be replayed
+                if not phases: step += 1; continue
+                prev = before
+                for phase, values, rule, after in phases:
+                    st = {"action": f"submitted form {obj['name']} with {phase} values", "submitted_values": values, "violated_rule": rule or None,
+                          "before": {"url": prev["url"], "text": prev["text"][:2000]}, "after": {"url": after["url"], "text": after["text"][:3000]}, "diff": diff(prev, after),
+                          "console_errors": console[:5], "http_errors": http_err[:5], "current_subgoal": subgoal}
+                    q = {"invalid_accepted": {"type": "noul", "instructions": "The submission deliberately violated `violated_rule`. Did the app ACCEPT it anyway (success message, item created/updated, form closed, new data visible) instead of showing a validation error and keeping the form open?"}} if phase == "invalid" else \
+                        {"valid_rejected": {"type": "noul", "instructions": "All `submitted_values` were valid and realistic. Did the app REJECT or ignore the submission (validation error shown, form still open unchanged, no success/confirmation, data not appearing) instead of accepting it?"}}
+                    if after.get("blocked"):                         # nothing changed at all: browser/app validation swallowed the submit
+                        probs, hits, exp = {}, {}, None; st["form_blocked"] = True
+                    else:
+                        probs, hits, exp = judge(st, q)
+                    hits |= cheap_oracles({"tag": "form"}, prev, after, http_err, js_err)
+                    record(step, how, f"{desc} [{phase}]", prev, after, probs, hits, {"values": values, "rule": rule, "expectation": exp, "form_blocked": after.get("blocked", False), "diff": st["diff"]}, page)
+                    prev = after; step += 1; sg_steps += 1
+                done_tuples.add((path, desc))
+                continue
+
+            el = obj
+            try:
+                value = act(page, el, step_value(scen_step) if picked and scen_step else None)
+            except Exception as e:
+                log.write(json.dumps({"step": step, "action": desc, "error": str(e)[:200]}) + "\n")
+                print(f'{step:03d} {how:11s} {desc[:45]:45s} ERR {str(e).splitlines()[0][:50]}'); mem[desc]["ineff"] = True; step += 1; continue
+            after = snapshot(page); seen_lines.update(after["text"].splitlines())
+            if {f["name"] for f in after["forms"].values()} - {f["name"] for f in before["forms"].values()}: opener = el   # this click opened a form
+            if state_sig(after) == sig and after["text"] == before["text"]: mem[desc]["ineff"] = True
+            st = {"action": desc, "before": {"url": before["url"], "text": before["text"][:2000]}, "after": {"url": after["url"], "text": after["text"][:3000]}, "diff": diff(before, after),
+                  "console_errors": console[:5], "failed_requests": failed[:5], "http_errors": http_err[:5], "current_subgoal": subgoal}
+            dead_click = mem[desc]["ineff"] and el["tag"] in ("button", "a")   # code-observed: click changed nothing at all
+            probs, hits, exp = judge(st, {}, dead_click)
+            hits |= cheap_oracles(el, before, after, http_err, js_err)
+            record(step, how, desc, before, after, probs, hits, {"value": value, "expectation": exp, "diff": st["diff"]}, page)
+            done_tuples.add((path, desc, value)); step += 1; sg_steps += 1
+
+        # --- absence (v14): a stalled scenario whose click step names a control that never appeared on any screen = the feature is missing (deterministic, no judge) ---
+        seen = "\n".join(seen_lines); last = snapshot(page)
+        def present(name): return all(re.search(rf"(?<![A-Za-z]){re.escape(w)}(?![A-Za-z])", seen, re.I) for w in re.findall(r"[A-Za-z][A-Za-z-]+", name)) if re.findall(r"[A-Za-z][A-Za-z-]+", name) else True
+        reported = {x["expected_control"].casefold() for x in absent}                          # dedupe: one report per control name, none if v9's probe already reported it
+        for item in stalled:
+            if item in NEG or any(json.loads(x["evidence"])["requirement"] in item for x in absent): continue   # negative items: absence is a pass; v9 probe already fired for this expectation
+            names = [n for st_ in steps_of.get(item, []) if re.match(r"\s*(click|press|tap|open)\b", st_, re.I) for n in re.findall(r"""['"‘“]([^'"’”]{1,40})['"’”]""", st_)]
+            missing = [n for n in names if not present(n) and n.casefold() not in reported]
+            if missing:
+                reported.add(missing[0].casefold())
+                record(step, "absence", f"searched every screen for a '{missing[0]}' control", last, last, {}, {"expectation_unreachable": 1.0}, {"value": None, "expectation": item, "deterministic": True, "missing_controls": missing}, page); step += 1
+
+        # --- verify by replay: redo each flagged primitive action from its page; forms are stateful, left as reproduced=None ---
+        findings, seen = [], set()
+        for f in findings_raw:
+            key = (f["action"].split(" (")[0].split(" [")[0], f["from"], ",".join(f["flags"]))
+            if key in seen: continue
+            seen.add(key)
+            f = {**f, "id": len(findings), "page": f["from"], "reproduced": None,
+                 "count": sum(1 for g in findings_raw if (g["action"].split(" (")[0].split(" [")[0], g["from"], ",".join(g["flags"])) == key)}
+            if not f["action"].startswith("form ") and f.get("how") != "absence" and len(findings) < 12:
+                try:
+                    page.goto(BASE + f["from"]); page.wait_for_load_state("networkidle"); page.wait_for_timeout(400)
+                    snap = snapshot(page)
+                    el = next(e for e in snap["elements"] if describe(e) == f["action"])
+                    console.clear(); failed.clear(); http_err.clear(); js_err.clear()
+                    act(page, el, f.get("value")); after = snapshot(page)
+                    st = {"action": f["action"], "before": {"url": snap["url"], "text": snap["text"][:2000]}, "after": {"url": after["url"], "text": after["text"][:3000]},
+                          "console_errors": console[:5], "failed_requests": failed[:5], "http_errors": http_err[:5]}
+                    if f.get("expectation"):
+                        st["expectation"] = f["expectation"]
+                        probs = jev_oracles(st, {"expectation_violated": {"type": "noul", "instructions": "`expectation` states what must be visible after this kind of action. Judging only `before` -> `after` for `action`, is the expectation clearly VIOLATED?"}})
+                    else: probs = jev_oracles(st)
+                    again = {k for k, v in probs.items() if v >= FLAG_THRESHOLD} | set(cheap_oracles(el, snap, after, http_err, js_err))
+                    f["reproduced"] = bool(again & set(f["flags"]))
+                except Exception as e:
+                    f["replay_error"] = str(e)[:120]
+            f["action"] = key[0]
+            findings.append(f)
+        absent = list({x["expected_control"].casefold(): x for x in reversed(absent)}.values())[::-1]   # one feature_missing report per control name across pages (first page wins)
+        for x in absent:                                  # deterministic: no replay, no Sonnet needed (`report` is the sentence to pass through)
+            ev = json.loads(x["evidence"]); url = x["url"].replace(BASE, "")
+            findings.append({"id": len(findings), "step": -1, "state": "", "how": "absence", "action": f'absence probe: {x["verb"]}', "page": url, "from": url, "to": url,
+                             "oracles": {"feature_missing": x["confidence"]}, "flags": ["feature_missing"], "reproduced": True, "count": 1, "expectation": ev["requirement"],
+                             "report": f'On {url} the spec requires "{ev["requirement"]}" but no {x["expected_control"]} control or content exists (looked for {", ".join(ev["missing_terms"])} among {len(ev["controls"])} controls and {len(ev["fields"])} fields).',
+                             "evidence": {"action": x["verb"], "before": {"url": url, "text": ""}, "after": {"url": url, "text": ev["text"][:3000]}, "console_errors": [], "failed_requests": [], "http_errors": [], "js_exceptions": [],
+                                          "controls": ev["controls"], "fields": ev["fields"], "missing_terms": ev["missing_terms"]}})
+        browser.close()
+
+    print(f"\n{a.steps} steps, {len(visited)} distinct states, {len(findings_raw)} flagged, {len(done)}/{len(subgoals)} expectations exercised -> {run}/flagged/")
+    for f in findings:
+        print(f'  {f["action"][:40]:40s} @ {f["page"][-24:]:24s} -> {",".join(f["flags"])[:35]:35s} x{f["count"]} reproduced={f["reproduced"]}')
+    json.dump({"run": str(run), "steps": a.steps, "user": a.user, "states": len(visited), "flagged_steps": len(findings_raw),
+               "subgoals_done": done, "stalled": stalled, "expectations": subgoals, "checked": sorted(checked), "secs": round(time.time() - t0, 1), "jev": USAGE | CLAUDE_USAGE, "findings": findings},
+              open(run / "findings.json", "w"), indent=1, ensure_ascii=False)
+    return run
+
+
+if __name__ == "__main__":
+    main()

jevqa-0.1.0.dist-info/METADATA

@@ -0,0 +1,113 @@
+Metadata-Version: 2.5
+Name: jevqa
+Version: 0.1.0
+Summary: Autonomous pre-QA bug hunter for web apps: Jev (TypeSafe) picks actions and judges screens, Claude reads the spec once.
+Author: Dmytro Tolok
+License-Expression: MIT
+License-File: LICENSE
+Keywords: jev,monkey-testing,playwright,qa,testing,typesafe
+Requires-Python: >=3.12
+Requires-Dist: anthropic>=0.40
+Requires-Dist: playwright>=1.63
+Requires-Dist: requests>=2.32
+Description-Content-Type: text/markdown
+
+# jevqa
+
+**Five minutes and 35 cents before you show anyone your app.**
+
+Point it at a deployed web app and the README it was built from. It explores the app on its own, then writes `report.md`: the missing delete button, the date field that accepts 1823, the Save that saves nothing, each with a screenshot.
+
+```bash
+uvx jevqa run http://localhost:3000 --spec README.md
+```
+
+![jevqa run: checklist, exploration, flags, report.md](docs/demo.gif)
+
+No selectors. No test scripts. Nothing to maintain. The first run asks for two API keys and remembers them.
+
+→ **[See a real report](examples/blog-app/report.md)** from a blog app: 7 findings in 269 seconds, two of them confirmed bugs (no way to edit posts, no way to publish).
+
+## Why it is cheap
+
+Every decision on every screen is a typed judgment from [Jev](https://typesafe.ai), TypeSafe's System One model: a probability or a choice, not generated text, at $0.042 per million tokens. Jev picks the next action and answers a fixed set of oracle questions about each screen. Claude reads your spec once to write the checklist. A full run is 5–6 minutes and $0.29–0.41.
+
+The same job done by a Claude Opus 5 agent with Playwright MCP: $3.20 per app, and it found fewer known bugs. [Benchmark →](BENCHMARK.md)
+
+## What it finds
+
+Measured on 20 [WebTestBench](https://github.com/friedrichor/WebTestBench) apps with 107 known bugs: 22% of them, at 5–6 minutes and $0.29–0.41 per app. In order of how often it finds them:
+
+1. **Missing or unreachable features** — no edit/delete/search/sort control, a role that can only view. Half of everything it finds, and the most reliable half.
+2. **Validation gaps** — past dates accepted, bad phone numbers, duplicates, start after end.
+3. **Dead controls** — a click that does nothing, "Save" with no effect, a list that does not update after submit.
+4. **Data lost after reload.**
+5. **Broken first render** — charts of zeros, page errors on entry.
+
+## What it will NOT find
+
+- **Interaction quality**: sliders, drag and drop, animation, keyboard flows. 2 of 15 known interaction bugs.
+- **Wrong numbers and wrong content**: a total that is off by one, a stat that lies. It sees text, not truth.
+- **Permissions and ownership**: user A editing user B's data. It rarely switches roles deep enough.
+- **Long multi-step flows**: checkout, onboarding, anything past 5–6 dependent steps in a 50-action budget.
+- **Anything your spec does not mention.** The checklist comes from the spec; a thin README gives a thin run.
+
+**About 1 in 8 findings is a confirmed defect.** The rest is noise or defects the spec does not name. We publish that number because a tool that hides it is lying to you. Read the report top to bottom: missing features come first and are the most reliable; skim the rest in ninety seconds. This is a pre-QA smoke bot. It clears the obvious defects before QA starts; it does not replace QA on UX or business logic.
+
+## GitHub Action
+
+Run it on every preview deployment:
+
+```yaml
+- uses: Todmy/jevqa@main
+  with:
+    url: ${{ steps.deploy.outputs.preview_url }}
+    spec: docs/PRD.md
+  env:
+    TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }}
+    ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+```
+
+The report lands in the job summary and as a workflow artifact. Outputs: `report` (path), `findings` (count).
+
+## Setup
+
+Two keys, asked for on first run and stored in `~/.config/jevqa/config.json` (env vars override, which is what CI uses):
+
+- **TypeSafe** — [console.typesafe.ai](https://console.typesafe.ai). Jev does the exploring and judging: ~$0.05–0.10 per app.
+- **Claude** — an [Anthropic API key](https://console.anthropic.com), or pick "use the Claude Code CLI" if you have it logged in. Three calls per app: ~$0.25.
+
+Chromium for Playwright is downloaded automatically on the first run (~150 MB, once per machine). `jevqa config` shows or resets the keys.
+
+| flag | default | |
+|---|---|---|
+| `--spec FILE` | required | spec, PRD or README the app was built from |
+| `--steps N` | 50 | action budget; time and cost scale linearly |
+| `--out DIR` | `jevqa-runs` | where runs are written |
+| `--headed` | off | watch the browser |
+
+Models: Claude Sonnet 5 for the checklist, Haiku 4.5 for form payloads; override with `JEVQA_SONNET` / `JEVQA_HAIKU`.
+
+## How it works
+
+1. Claude turns the spec into 15–30 expectations, each with concrete UI steps.
+2. A bounded crawl visits nav, detail and role states. Spec-quoted controls that appear on no screen become **missing feature** findings.
+3. The action loop follows scenario steps where Jev can match them to a real element, otherwise explores, and submits boundary payloads (negative numbers, past dates, reversed ranges, duplicates) on every form.
+4. After each action Jev answers the oracle questions on the before/after text. Flags are verified by replay; non-reproducible ones are dropped.
+5. Each finding becomes one deterministic sentence with Expected / Actual and a screenshot.
+
+## Hosted version?
+
+Today jevqa is a local CLI and an Action, bring your own keys. If you would pay ~$29/month for a GitHub App that comments on pull requests, keeps run history and needs no keys, **[👍 this issue](../../issues/1)**. We build it when enough people do.
+
+## Telemetry
+
+Anonymous counts only (version, step budget, number of findings, seconds, cost estimate). Never the URL, the spec or the report. Off with `JEVQA_TELEMETRY=0`; off by default in CI.
+
+## Add your tester to the benchmark
+
+[BENCHMARK.md](BENCHMARK.md) has the methodology, the leaderboard and how to submit. Nobody else in this category publishes recall. Prove us wrong.
+
+## License
+
+MIT

jevqa-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+jevqa/__init__.py,sha256=rMj39g8wcLsxFHmg2hVfeuXwVHBGOCCJKrQ81hVEOJo,72
+jevqa/absence.py,sha256=gn7t3wy56bPQF9chlKiqdmkOT_ZITvTUr7j98fqdgUE,8369
+jevqa/checklist_prompt.json,sha256=VZcTtY896nP70KQnocKdLTlTqLzc-dmXPcPkMPm8068,6183
+jevqa/cli.py,sha256=iZ1sboNSFAldTspk6RnsxU_RtvujnCw5n2m0NrnT2qg,2697
+jevqa/config.py,sha256=wlDZG_y0anSskuOOEiwmj3ntWHy8IuFMeQw3eX6K_Rk,2634
+jevqa/llm.py,sha256=RUqf4hLSQoUsmCoAnqCgaOgQPv3qYzzNXdI6P_oWmGo,1884
+jevqa/oracles.json,sha256=1VRLeLBL0JN5Vf0lBVz88Uk8rwmUdwIYlhB--d2T5zY,1494
+jevqa/report.py,sha256=uM414_1nVivn0R_4siZbU22PwAXhYOI09UZq-PlsKRA,8304
+jevqa/telemetry.py,sha256=M-S4A-3hH6v3l7nHtH5qI0Wnpg_hw1c6g5i0e-4RvbU,969
+jevqa/tester.py,sha256=CxR1HF3_CJPq7IyBxUBjAybHZwRZIiUhWd9o_8TNvgM,46646
+jevqa-0.1.0.dist-info/METADATA,sha256=CRrtE4Jjl0jWtLkO1VexiGdyxXQ1jjGleLOzrml5aD8,6116
+jevqa-0.1.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
+jevqa-0.1.0.dist-info/entry_points.txt,sha256=aEzqWF1eBitBomA9notQ9fI8M5rqeh-HvpUuSP4xJ98,41
+jevqa-0.1.0.dist-info/licenses/LICENSE,sha256=vIBbMpdaIFywhOoy5y_2C_RErodO_NMfhNI1vwaVrvo,1069
+jevqa-0.1.0.dist-info/RECORD,,

jevqa-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.3
+Root-Is-Purelib: true
+Tag: py3-none-any

jevqa-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+jevqa = jevqa.cli:main

jevqa-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2026 Dmytro Tolok
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.