plm-software 0.1.0__tar.gz

plm_software-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+*.pyc
+.DS_Store
+logs/
+.worktrees/
+.benchmarks/

plm_software-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ro-rbl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

plm_software-0.1.0/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.5
+Name: plm-software
+Version: 0.1.0
+Summary: plm pack: Code tasks, checked by running the examples in a sandbox
+Project-URL: Homepage, https://github.com/ro-rbl/plm-orchestrator
+Project-URL: Source, https://github.com/ro-rbl/plm-orchestrator
+Project-URL: Issues, https://github.com/ro-rbl/plm-orchestrator/issues
+Author: ro-rbl
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.13
+Requires-Dist: plm-core<0.2,>=0.1
+Description-Content-Type: text/markdown
+
+# plm-software
+
+Plm pack: Code tasks, checked by running the examples in a sandbox.
+
+A domain pack for [plm-core](https://github.com/ro-rbl/plm-orchestrator): install it with `plm packs install software`, which also pulls the
+models it needs into Ollama. Documentation, benchmark results and caveats: https://github.com/ro-rbl/plm-orchestrator

plm_software-0.1.0/README.md

@@ -0,0 +1,6 @@
+# plm-software
+
+Plm pack: Code tasks, checked by running the examples in a sandbox.
+
+A domain pack for [plm-core](https://github.com/ro-rbl/plm-orchestrator): install it with `plm packs install software`, which also pulls the
+models it needs into Ollama. Documentation, benchmark results and caveats: https://github.com/ro-rbl/plm-orchestrator

plm_software-0.1.0/pyproject.toml

@@ -0,0 +1,24 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "plm-software"
+version = "0.1.0"
+description = "plm pack: Code tasks, checked by running the examples in a sandbox"
+readme = "README.md"
+authors = [{ name = "ro-rbl" }]
+requires-python = ">=3.13"
+license = "MIT"
+dependencies = ["plm-core>=0.1,<0.2"]
+
+[project.entry-points."plm.packs"]
+software = "plm_software.manifest:PACK"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/plm_software"]
+
+[project.urls]
+Homepage = "https://github.com/ro-rbl/plm-orchestrator"
+Source = "https://github.com/ro-rbl/plm-orchestrator"
+Issues = "https://github.com/ro-rbl/plm-orchestrator/issues"

plm_software-0.1.0/src/plm_software/__init__.py

@@ -0,0 +1 @@
+"""Software pack: code tasks, checked by running the examples (and optionally generated tests)."""

plm_software-0.1.0/src/plm_software/claims.py

@@ -0,0 +1,12 @@
+"""Which tasks the software pack claims: a request to write code, or a def signature."""
+
+import math
+import re
+
+SOFTWARE_TASK = re.compile(r"\b(?:write|implement|create|code)\b.{0,40}\b(?:function|class|method|program)\b|\bdef \w+\s*\(",
+                           re.IGNORECASE | re.DOTALL)
+
+
+def claims(text):
+    """Certain whenever the text asks for code, so software wins over every numeric pack."""
+    return math.inf if SOFTWARE_TASK.search(text) else 0

plm_software-0.1.0/src/plm_software/codegen.py

@@ -0,0 +1,37 @@
+"""Prompting for code tasks and pulling the code back out of the reply."""
+
+import re
+
+from plm_software.sandbox import ALLOWED_IMPORTS
+
+# v1 said "including any imports" and listed typing; small models then wrote
+# "from typing import int", which crashes (13 of the 0.6B and 4B errors).
+PROMPT_VERSION = 2
+CODE_FORMAT = (
+    "Reply with one ```python code block containing the complete implementation. Import a module "
+    "only if your code uses it; built-in types such as int, str, list and dict need no import. "
+    "You may only import these standard-library modules: "
+    + ", ".join(sorted(ALLOWED_IMPORTS))
+    + ". Do not include tests, example usage or input()."
+)
+
+CODE_RE = re.compile(r"```(?:python|py)?[ \t]*\n(.*?)```", re.DOTALL | re.IGNORECASE)
+
+
+def build_prompt(task):
+    if not task["examples"]:
+        return f"{task['prompt']}\n\n{CODE_FORMAT}"
+    examples = "\n".join(f"    {e}" for e in task["examples"])
+    return f"{task['prompt']}\n\nExamples (each expression must be True):\n{examples}\n\n{CODE_FORMAT}"
+
+
+def extract_code(text, entry):
+    """The last fenced block that defines `entry`; else the last block; else unfenced code."""
+    defines = re.compile(rf"^\s*(?:def|class)\s+{re.escape(entry)}\b", re.MULTILINE)
+    blocks = CODE_RE.findall(text or "")
+    for block in reversed(blocks):
+        if defines.search(block):
+            return block
+    if blocks:
+        return blocks[-1]
+    return text if defines.search(text or "") else None

plm_software-0.1.0/src/plm_software/controller.py

@@ -0,0 +1,153 @@
+"""Software-pack controller: pick a level, have a specialist write the code, check it, escalate.
+
+The check runs the code in the sandbox: it must load, define the requested
+function or class, and pass every example given in the task.
+
+Until 2026-10-24 every level used a cheap-first cascade (4B -> 8B -> coder),
+picked on our 45-task eval: 40/45 at 56% of always-coder compute (coder alone
+39/45). The public benchmarks (evals/bench_code.py, CODE_RULE fixed before
+them, accuracy first as the user chose) disagreed: HumanEval+ and MBPP+ pooled,
+540 tasks:
+
+    always coder 14B                       410/540 (76%) at 100% of always-coder compute
+    cascade 4B -> 8B -> coder (old)        387/540 (72%) at 64%
+    cascade 8B -> coder                    394/540 (73%) at 77%
+    always 8B                              353/540 (65%) at 55%
+
+The coder beats the cascade by +6 to +40 tasks (paired 95%). The cascade lost
+because the examples check rarely sees a wrong HumanEval answer (its docstrings
+show no parseable examples: 14 of 112 wrong answers caught). So every level goes
+to the coder; the level only sets the token budget.
+
+thorough=True adds tests written by the large model from the task alone
+(plm/software/testgen.py). In the eval that reached 42/45 but cost 228% of
+always-coder compute: about 1 in 8 generated tests is itself wrong, which flags
+correct code and forces escalation. So it is opt-in, not the default. When no
+answer passes everything, thorough mode returns the one that passed the examples
+and the most generated tests.
+
+Model names can be overridden with PLM_SMALL_MODEL, PLM_LARGE_MODEL and PLM_CODER_MODEL.
+"""
+
+import os
+import re
+import time
+
+from plm.ollama import ask
+
+from plm_software.codegen import build_prompt, extract_code
+from plm_software.levels import classify
+from plm_software.sandbox import run_tests
+from plm_software.testgen import failing_tests, generate_tests
+
+MODELS = {
+    "small": os.environ.get("PLM_SMALL_MODEL", "hf.co/bartowski/Qwen_Qwen3-4B-GGUF:Q4_K_M"),
+    "large": os.environ.get("PLM_LARGE_MODEL", "qwen3:8b"),
+    "coder": os.environ.get("PLM_CODER_MODEL", "qwen2.5-coder:14b"),
+}
+TEST_WRITER = "large"
+
+# Per level: token budget and the escalation chain of model keys. Budgets are
+# >2.5x the longest passing answer at that level in the eval sweep; the coder hit none on the benchmarks.
+CHAIN = ["coder"]
+LEVELS = {
+    "calm": {"num_predict": 512, "chain": CHAIN},
+    "alert": {"num_predict": 1024, "chain": CHAIN},
+    "intensive": {"num_predict": 1536, "chain": CHAIN},
+}
+
+ENTRY_RE = re.compile(r"`(\w+)\s*\(|\bdef\s+(\w+)\s*\(|`?\bclass\s+`?(\w+)")
+EXAMPLE_RE = re.compile(r"^\s*(?:assert\s+|>>>\s*)(.+?)\s*$", re.MULTILINE)
+
+
+def parse_task(text):
+    """Split pasted task text into a description, the entry name and example expressions."""
+    m = ENTRY_RE.search(text)
+    entry = next((g for g in m.groups() if g), None) if m else None
+    examples = [e for e in EXAMPLE_RE.findall(text) if re.search(r"==|\bis\b|\bin\b|raises\(", e)]
+    description = EXAMPLE_RE.sub("", text).strip()
+    return {"prompt": description, "entry": entry, "examples": examples}
+
+
+def attempt(task, model, num_predict, timeout):
+    """One model writes the code; it is checked against the task's own examples."""
+    start = time.time()
+    r = ask(model, build_prompt(task), False, num_predict, timeout)
+    code = extract_code(r["response"], task["entry"] or "")
+    step = {"model": model, "tokens": r["eval_tokens"], "prompt_tokens": r["prompt_tokens"], "model_text": r["response"], "code": code,
+            "capped": r["done_reason"] == "length", "issues": [], "generated_failures": 0}
+    if code is None:
+        step["issues"].append("no code block")
+    else:
+        must_define = [f"callable({task['entry']})"] if task["entry"] else []
+        result = run_tests(code, must_define + task["examples"])
+        step["examples"] = result
+        if result["error"]:
+            step["issues"].append(result["error"])
+        else:
+            step["issues"] += [f"failed: {t}" + (f" ({why})" if why != "evaluated to False" else "")
+                               for t, why in result["failures"]]
+    step["seconds"] = time.time() - start
+    return step
+
+
+def check_generated(step, task, tests):
+    """Run the generated tests on code that already passed the examples."""
+    result = run_tests(step["code"], tests)
+    fails = [(t, "error") for t in tests] if result["error"] else failing_tests(result, task["entry"])
+    step["generated_failures"] = len(fails)
+    step["issues"] += [f"failed generated test: {t}" for t, _ in fails]
+
+
+def solve(text, level=None, timeout=600, thorough=False):
+    """Write code for one task. Returns the code plus a record of every model call."""
+    start = time.time()
+    task = parse_task(text)
+    if level:
+        reason = "set by caller"
+    else:
+        level, reason = classify(task["prompt"])
+    plan = LEVELS[level]
+
+    steps, writer, tests = [], None, None
+    for model_key in plan["chain"]:
+        step = attempt(task, MODELS[model_key], plan["num_predict"], timeout)
+        steps.append(step)
+        if thorough and not step["issues"] and task["entry"]:
+            if tests is None:  # only pay for test writing once some code passes the examples
+                tests, writer = generate_tests(task, MODELS[TEST_WRITER], timeout=timeout)
+                writer["tests"] = tests
+            check_generated(step, task, tests)
+        if not step["issues"]:
+            break
+
+    final = steps[-1]
+    if thorough and final["issues"]:
+        passed_examples = [s for s in steps if s["code"] and not any(
+            not i.startswith("failed generated test") for i in s["issues"])]
+        if passed_examples:
+            # Fewest generated-test failures; ties go to the later, stronger model,
+            # which is the rule evals/simulate_code_routing.py measured.
+            final = min(passed_examples, key=lambda s: (s["generated_failures"], -steps.index(s)))
+    n_examples = len(task["examples"])
+    reason += (f"; {n_examples} example{'s' if n_examples != 1 else ''} to check" if n_examples
+               else "; no examples given, so only loading is checked")
+    if writer:
+        reason += f"; {len(tests)} generated tests"
+    # Loading and defining the entry point alone don't show the code works.
+    checked = bool(n_examples or tests)
+    return {
+        "pack": "software",
+        "task": text,
+        "level": level,
+        "reason": reason,
+        "answer": {"code": final["code"]} if final["code"] else None,
+        "checked": checked,
+        "unchecked_reason": None if checked else "no examples given, so only loading was checked",
+        "trusted": checked and not final["issues"],
+        "final_step": steps.index(final) + 1,
+        "steps": steps,
+        "test_writer": writer,
+        "tokens": sum(s["tokens"] for s in steps) + (writer["tokens"] if writer else 0),
+        "seconds": time.time() - start,
+    }

plm_software-0.1.0/src/plm_software/levels.py

@@ -0,0 +1,35 @@
+"""Adrenaline level for a coding task, from its description: rules only, no model.
+
+    calm      one transformation: a loop, a comprehension, a builtin
+    alert     one standard algorithm, or input-validation rules to get exactly right
+    intensive designing a data structure, search, optimization/DP, or writing a parser
+
+The cue families are written from those definitions, not fitted per task.
+Evaluate them with:  .venv/bin/python evals/eval_levels.py --pack software
+"""
+
+import re
+
+INTENSIVE_CUES = {
+    "design": r"write a class\b",
+    "search": r"shortest|\bpaths?\b|graph|backtrack|queens|ladder|island|sudoku",
+    "optimize": r"\b(?:minimum|maximum|fewest|largest number|longest|lexicographically smallest)\b",
+    "dp": r"dynamic programming|subsequence|knapsack|levenshtein|edit distance",
+    "parser": r"\bparses?\b|\bevaluates?\b|precedence|(?:without|do not use) (?:using )?(?:eval|the \w+ module)",
+}
+ALERT_CUES = {
+    "rules": r"\bvalid\b|raise \w*error|leading zeros|at most once|in that order|unbalanced|balanced",
+    "algorithm": r"binary search|o\(log n\)|o\(n\)|linear time|merge|frequen|anagram|nested|matrix|rotate"
+                 r"|roman|encode|cipher|prefix|triangle|bracket|interval|range strings",
+}
+
+
+def classify(text):
+    """Return (level, reason)."""
+    hits = lambda cues: sorted(n for n, p in cues.items() if re.search(p, text, re.IGNORECASE))  # noqa: E731
+    intensive, alert = hits(INTENSIVE_CUES), hits(ALERT_CUES)
+    if intensive:
+        return "intensive", "cues: " + ", ".join(intensive)
+    if alert:
+        return "alert", "cues: " + ", ".join(alert)
+    return "calm", "no algorithm cues"

plm_software-0.1.0/src/plm_software/manifest.py

@@ -0,0 +1,9 @@
+"""The software pack's manifest: what the router needs to know about it."""
+
+from plm.pack import Pack
+
+from plm_software import controller
+from plm_software.claims import claims
+
+PACK = Pack(name="software", version="0.1.0", claims=claims, priority=100, solve=controller.solve,
+            levels=controller.LEVELS, models=controller.MODELS, eval_set="evals/software/tasks.jsonl")

plm_software-0.1.0/src/plm_software/sandbox.py

@@ -0,0 +1,127 @@
+"""Run model-written code against test expressions in an isolated process.
+
+Each test is a Python expression that must evaluate to True. The code runs in a
+separate `python -I -S` process in an empty temp directory, with a time limit
+per test and for the whole run. Imports are limited to a standard-library
+allowlist. Like plm/numeric/calc.py, the screen guards against accidents; it is not a
+security sandbox.
+"""
+
+import json
+import re
+import subprocess
+import sys
+import tempfile
+
+ALLOWED_IMPORTS = {
+    "math", "collections", "heapq", "itertools", "functools", "re", "string", "bisect",
+    "typing", "dataclasses", "operator", "fractions", "decimal", "copy", "statistics", "enum",
+}
+# Unlike calc.py, dunders are allowed in general (classes need __init__); only the escape hatches are blocked.
+# The builtins are matched only as bare names, so methods like re.compile( stay allowed.
+BLOCKED = re.compile(
+    r"(?<![.\w])(?:open|exec|eval|compile|input|breakpoint|globals|locals|vars|__import__)\s*\("
+    r"|__(?:builtins|subclasses|globals|code|loader|spec|bases|mro)__"
+)
+
+HARNESS = r"""
+import json, signal, sys
+
+payload = json.loads(sys.stdin.read())
+
+def raises(exc, fn, *args, **kwargs):
+    try:
+        fn(*args, **kwargs)
+    except exc:
+        return True
+    except Exception:
+        return False
+    return False
+
+def on_alarm(signum, frame):
+    raise TimeoutError("test took too long")
+
+signal.signal(signal.SIGALRM, on_alarm)
+ns = {"raises": raises, "__name__": "solution"}
+try:
+    signal.setitimer(signal.ITIMER_REAL, payload["per_test"])
+    exec(compile(payload["code"], "<solution>", "exec"), ns)
+    signal.setitimer(signal.ITIMER_REAL, 0)
+except BaseException as e:
+    print("\n" + json.dumps({"loaded": False, "error": f"{type(e).__name__}: {e}"[:300], "results": []}))
+    sys.exit(0)
+
+results = []
+for t in payload["tests"]:
+    try:
+        signal.setitimer(signal.ITIMER_REAL, payload["per_test"])
+        ok = eval(t, ns) is True
+        signal.setitimer(signal.ITIMER_REAL, 0)
+        results.append([ok, None if ok else "evaluated to False"])
+    except BaseException as e:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+        results.append([False, f"{type(e).__name__}: {e}"[:200]])
+print("\n" + json.dumps({"loaded": True, "error": None, "results": results}))
+"""
+
+
+def screen(code):
+    """Return a reason the code is refused, or None if it may run."""
+    for line in code.splitlines():
+        m = re.match(r"\s*(?:from\s+(\S+)\s+import|import\s+(.+))", line)
+        if m:
+            names = [m.group(1)] if m.group(1) else [n.split(" as ")[0].strip() for n in m.group(2).split(",")]
+            bad = [n for n in names if n.split(".")[0] not in ALLOWED_IMPORTS]
+            if bad:
+                return f"import not allowed: {', '.join(bad)}"
+    hit = BLOCKED.search(code)
+    return f"blocked construct: {hit.group().strip()}" if hit else None
+
+
+def run_tests(code, tests, per_test=5, total=60):
+    """Run `code`, then each test. Returns {loaded, error, passed, total, failures}."""
+    result = {"loaded": False, "error": None, "passed": 0, "total": len(tests), "failures": []}
+    refused = screen(code)
+    if refused:
+        result["error"] = f"refused: {refused}"
+        return result
+    payload = json.dumps({"code": code, "tests": tests, "per_test": per_test})
+    with tempfile.TemporaryDirectory() as tmp:
+        try:
+            p = subprocess.run([sys.executable, "-I", "-S", "-c", HARNESS], input=payload, cwd=tmp,
+                               timeout=total, capture_output=True, text=True)
+        except subprocess.TimeoutExpired:
+            result["error"] = f"timed out after {total}s"
+            return result
+    try:
+        out = json.loads(p.stdout.strip().splitlines()[-1])  # the code's own prints come first
+    except (IndexError, json.JSONDecodeError):
+        result["error"] = f"harness crashed: {(p.stderr.strip().splitlines() or ['?'])[-1][:200]}"
+        return result
+    result["loaded"], result["error"] = out["loaded"], out["error"]
+    result["passed"] = sum(ok for ok, _ in out["results"])
+    result["failures"] = [(t, why) for t, (ok, why) in zip(tests, out["results"]) if not ok]
+    return result
+
+
+if __name__ == "__main__":
+    good = "def add(a, b):\n    return a + b\n"
+    r = run_tests(good, ["add(1, 2) == 3", "add(-1, 1) == 0"])
+    assert (r["passed"], r["total"], r["error"]) == (2, 2, None), r
+    r = run_tests(good, ["add(1, 2) == 4", "add(1) == 1"])
+    assert r["passed"] == 0 and "TypeError" in r["failures"][1][1], r
+    r = run_tests("def f():\n    while True: pass\n", ["f() is None", "1 == 1"], per_test=1)
+    assert r["passed"] == 1 and "TimeoutError" in r["failures"][0][1], r  # one hang fails one test
+    assert run_tests("def f(:\n", ["f()"])["error"].startswith("SyntaxError"), run_tests("def f(:\n", ["f()"])
+    assert "not allowed" in run_tests("import os\n", [])["error"]
+    assert "blocked" in run_tests("x = open('f')\n", [])["error"]
+    assert "blocked" in run_tests("x = ().__class__.__bases__\n", [])["error"]
+    assert "blocked" in run_tests("x = compile('1', 'f', 'eval')\n", [])["error"]
+    r = run_tests("import re\nP = re.compile(r'\\d+')\n", ["P.fullmatch('12') is not None"])
+    assert r["passed"] == 1, r  # re.compile is a method call, not the compile builtin
+    r = run_tests("import collections\nclass C:\n    def __init__(self):\n        self.d = collections.deque()\n", ["C().d is not None"])
+    assert r["passed"] == 1, r  # classes with __init__ and allowed imports run
+    r = run_tests("print('noise')\ndef g():\n    print('more')\n    return 1\n", ["g() == 1"])
+    assert r["passed"] == 1, r  # the solution's own prints don't break result parsing
+    assert run_tests("raise SystemExit(3)\n", ["1 == 1"])["error"].startswith("SystemExit")
+    print("sandbox self-test: ok")

plm_software-0.1.0/src/plm_software/testgen.py

@@ -0,0 +1,97 @@
+"""Stronger code checks: a second model writes extra test cases from the task alone.
+
+The test writer never sees the candidate code, so its tests are independent of
+it. Its expected values can still be wrong, and a wrong test flags correct code;
+evals/gen_tests.py measures how often that happens against reference solutions.
+"""
+
+import ast
+import re
+
+from plm.ollama import ask
+
+from plm_software.codegen import CODE_RE
+
+TESTGEN_PROMPT_VERSION = 1
+N_TESTS = 8
+
+
+def build_testgen_prompt(task):
+    examples = "\n".join(f"    {e}" for e in task["examples"])
+    shown = f"\n\nExamples (each expression must be True):\n{examples}" if task["examples"] else ""
+    return (
+        f"{task['prompt']}{shown}\n\n"
+        f"Do not write the implementation. Instead, write {N_TESTS} more test cases for it as Python "
+        "expressions that must evaluate to True, one per line, in a ```python code block. Each one must "
+        f"call `{task['entry']}` and compare the result with the exact expected value, in the same style "
+        "as the examples. Cover cases the examples miss, such as empty input, a single element, "
+        "duplicates, negative numbers, unsorted input and boundary values, but only where they make sense "
+        "for this task and its stated rules. Work out every expected value carefully from the task "
+        "description: a wrong expected value is worse than a missing test."
+    )
+
+
+def parse_tests(text, entry, examples=()):
+    """Valid, distinct test expressions that call `entry` and aren't copies of the examples."""
+    blocks = CODE_RE.findall(text or "") or [text or ""]
+    seen = {e.replace(" ", "") for e in examples}
+    tests = []
+    for line in "\n".join(blocks).splitlines():
+        expr = re.sub(r"^\s*(?:assert\s+|>>>\s*)", "", line).split("  #")[0].strip()
+        if not expr or expr.startswith("#") or not re.search(rf"\b{re.escape(entry)}\b", expr):
+            continue
+        try:
+            ast.parse(expr, mode="eval")
+        except SyntaxError:
+            continue
+        key = expr.replace(" ", "")
+        if key not in seen:
+            seen.add(key)
+            tests.append(expr)
+    return tests[:N_TESTS]
+
+
+def is_broken_test(why, entry):
+    """True if a test failed because the test itself is malformed, whatever the code does.
+
+    A NameError for anything other than the entry point means the test uses a
+    name it never defined (e.g. a misplaced parenthesis in a lambda), so it
+    says nothing about the candidate and is ignored.
+    """
+    m = re.match(r"NameError: name '(\w+)' is not defined", why or "")
+    return bool(m) and m.group(1) != entry
+
+
+def failing_tests(result, entry):
+    """Failures from run_tests() that count against the code (broken tests removed)."""
+    return [(t, why) for t, why in result["failures"] if not is_broken_test(why, entry)]
+
+
+def generate_tests(task, model, num_predict=1024, timeout=600):
+    """Ask `model` for extra tests. Returns (tests, call record)."""
+    r = ask(model, build_testgen_prompt(task), False, num_predict, timeout)
+    tests = parse_tests(r["response"], task["entry"], task["examples"])
+    return tests, {"model": model, "tokens": r["eval_tokens"], "prompt_tokens": r["prompt_tokens"], "seconds": r["wall_s"], "raw": r["response"],
+                   "capped": r["done_reason"] == "length"}
+
+
+if __name__ == "__main__":
+    reply = """Here are tests:
+```python
+assert f([]) == []
+f([1]) == [1]   # single
+f([3, 1]) == [1, 3]
+# a comment
+f([2, 2] == [2, 2]
+g(1) == 2
+f([1]) == [1]
+```"""
+    got = parse_tests(reply, "f", examples=["f([3,1]) == [1,3]"])
+    assert got == ["f([]) == []", "f([1]) == [1]"], got  # strips assert/comments; drops bad syntax, other names, dupes, examples
+    assert parse_tests("no code", "f") == []
+    assert is_broken_test("NameError: name 'c' is not defined", "LRUCache")
+    assert not is_broken_test("NameError: name 'LRUCache' is not defined", "LRUCache")  # code didn't define it
+    assert not is_broken_test("evaluated to False", "f")
+    res = {"failures": [("t1", "NameError: name 'c' is not defined"), ("t2", "evaluated to False")]}
+    assert failing_tests(res, "f") == [("t2", "evaluated to False")]
+    print("testgen self-test: ok")