commitarena 0.1.1__py3-none-any.whl

commitarena/__init__.py

@@ -0,0 +1,40 @@
+"""commitarena: replay real bug fixes from git history against any model, in a sandbox with no network."""
+
+__version__ = "0.1.1"
+
+from .agent import SYSTEM_PROMPT, AgentConfig  # noqa: E402
+from .errors import (  # noqa: E402
+    BudgetExceeded,
+    CommitArenaError,
+    DockerUnavailable,
+    MissingAPIKey,
+    ModelUnavailable,
+    TaskError,
+)
+from .llm import Budget, ModelSpec  # noqa: E402
+from .runner import ContextFile, RunResult, run_suite, run_task  # noqa: E402
+from .sandbox import EnvSpec, SandboxConfig, ensure_docker  # noqa: E402
+from .tasks import Task, load_tasks  # noqa: E402
+
+__all__ = [
+    "SYSTEM_PROMPT",
+    "AgentConfig",
+    "Budget",
+    "BudgetExceeded",
+    "CommitArenaError",
+    "ContextFile",
+    "DockerUnavailable",
+    "EnvSpec",
+    "MissingAPIKey",
+    "ModelSpec",
+    "ModelUnavailable",
+    "RunResult",
+    "SandboxConfig",
+    "Task",
+    "TaskError",
+    "__version__",
+    "ensure_docker",
+    "load_tasks",
+    "run_suite",
+    "run_task",
+]

commitarena/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

commitarena/agent.py

@@ -0,0 +1,205 @@
+"""The minimal agent harness: one `bash` tool, same loop and prompt for every model."""
+
+from __future__ import annotations
+
+import json
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+from .errors import BudgetExceeded
+from .llm import ChatClient, Usage
+from .sandbox import Sandbox
+
+SYSTEM_PROMPT = """\
+You are a software engineer fixing a bug in a Python repository. The repository is at /work inside a \
+Linux container with no network access (pip install will not work; the dependencies are already \
+installed). Use the `bash` tool to inspect files, edit them (for example with python, sed or a heredoc) \
+and run tests. Each call runs in a fresh shell starting in /work; long outputs are truncated.
+
+Work in small steps: read the failing tests, find the cause in the source code, make a minimal fix, \
+and run the tests. Do not modify test files: they are restored before scoring. When the tests pass, \
+reply with a one-line summary and no tool call to finish."""
+
+NUDGE = (
+    "Your last reply was empty or cut off at the token limit. Keep your reasoning short and call the bash "
+    "tool, or reply with a one-line summary if you are done."
+)
+MAX_NUDGES = 2  # consecutive empty or truncated replies before giving up
+
+BASH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "bash",
+        "description": "Run a bash command in /work and return its combined stdout/stderr and exit code.",
+        "parameters": {
+            "type": "object",
+            "properties": {"command": {"type": "string", "description": "The command to run."}},
+            "required": ["command"],
+        },
+    },
+}
+
+
+@dataclass
+class AgentConfig:
+    max_steps: int = 30  # model calls
+    max_wall_time: int = 1200  # seconds for the whole attempt
+    max_output_chars: int = 6000  # per tool result, head + tail kept
+    max_tokens: int = 4096  # per model reply
+    temperature: float = 0.0
+    system_prompt: str = SYSTEM_PROMPT
+    # Older tool outputs are elided once the conversation passes this size, so a 16K-token local
+    # model keeps the task text and the latest outputs (about 3.5 characters per token).
+    max_context_chars: int = 40000
+
+
+@dataclass
+class AgentOutcome:
+    steps: int = 0
+    tool_calls: int = 0
+    usage: Usage = field(default_factory=Usage)
+    stop_reason: str = ""  # finished | max_steps | time_limit | no_action | budget_exceeded | model_error
+    error: str = ""
+    final_message: str = ""
+    trajectory: list[dict] = field(default_factory=list)
+
+
+def truncate(text: str, limit: int) -> str:
+    if len(text) <= limit:
+        return text
+    half = limit // 2
+    return (
+        f"{text[:half]}\n... [{len(text) - limit} characters truncated. Use grep -n or sed -n 'START,ENDp' "
+        f"FILE to see specific lines] ...\n{text[-half:]}"
+    )
+
+
+ELIDED = "[older output elided to fit the context window]"
+REPEAT_NOTE = "[note: you already ran this exact command and got the same output. Try a different approach.]"
+
+
+def fit_context(messages: list[dict], max_chars: int, keep_last: int = 4) -> list[dict]:
+    """Copy of `messages` with the oldest tool outputs elided until the total fits in `max_chars`.
+
+    The system prompt, the task and the last `keep_last` messages are never elided."""
+    size = sum(len(m.get("content") or "") for m in messages)
+    if size <= max_chars:
+        return messages
+    out = [dict(m) for m in messages]
+    for m in out[2 : max(2, len(out) - keep_last)]:
+        if size <= max_chars:
+            break
+        if m["role"] == "tool" and m["content"] != ELIDED:
+            size -= len(m["content"]) - len(ELIDED)
+            m["content"] = ELIDED
+    return out
+
+
+def message_text(content) -> str:
+    """Reply text. Some providers return a list of blocks (e.g. thinking + text); keep the text ones."""
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    parts = []
+    for block in content:
+        kind = block.get("type") if isinstance(block, dict) else getattr(block, "type", None)
+        text = block.get("text") if isinstance(block, dict) else getattr(block, "text", None)
+        if kind in (None, "text") and isinstance(text, str):
+            parts.append(text)
+    return "".join(parts)
+
+
+def _assistant_dict(msg) -> dict:
+    d: dict = {"role": "assistant", "content": message_text(msg.content)}
+    if msg.tool_calls:
+        d["tool_calls"] = [
+            {
+                "id": tc.id,
+                "type": "function",
+                "function": {"name": tc.function.name, "arguments": tc.function.arguments},
+            }
+            for tc in msg.tool_calls
+        ]
+    return d
+
+
+def run_agent(
+    client: ChatClient,
+    sandbox: Sandbox,
+    prompt: str,
+    config: AgentConfig | None = None,
+    system_extra: str = "",
+    on_step: Callable[[int, str], None] | None = None,
+) -> AgentOutcome:
+    cfg = config or AgentConfig()
+    system = cfg.system_prompt + (f"\n\n{system_extra}" if system_extra else "")
+    messages: list[dict] = [{"role": "system", "content": system}, {"role": "user", "content": prompt}]
+    out = AgentOutcome()
+    out.trajectory = messages
+    start = time.monotonic()
+    nudges = 0
+    seen: dict[str, str] = {}  # command -> last output, to flag exact repeats
+    while True:
+        if out.steps >= cfg.max_steps:
+            out.stop_reason = "max_steps"
+            break
+        if time.monotonic() - start > cfg.max_wall_time:
+            out.stop_reason = "time_limit"
+            break
+        try:
+            resp, usage = client.chat(
+                fit_context(messages, cfg.max_context_chars),
+                tools=[BASH_TOOL],
+                temperature=cfg.temperature,
+                top_p=1.0,
+                max_tokens=cfg.max_tokens,
+            )
+        except BudgetExceeded as e:
+            out.stop_reason, out.error = "budget_exceeded", str(e)
+            break
+        out.steps += 1
+        out.usage.add(usage)
+        if not resp.choices:
+            out.stop_reason, out.error = "model_error", "empty response"
+            break
+        msg = resp.choices[0].message
+        finish = getattr(resp.choices[0], "finish_reason", None)
+        text = message_text(msg.content)
+        if not msg.tool_calls and (finish == "length" or not text.strip()):
+            # Thinking models can spend the whole reply budget reasoning. Ask once more instead of
+            # treating an empty or truncated reply as "done". Same rule for every model.
+            if nudges < MAX_NUDGES:
+                nudges += 1
+                messages.append({"role": "assistant", "content": text.strip() or "(no reply)"})
+                messages.append({"role": "user", "content": NUDGE})
+                continue
+            out.stop_reason = "no_action"
+            break
+        messages.append(_assistant_dict(msg))
+        if not msg.tool_calls:
+            out.stop_reason = "finished"
+            out.final_message = text
+            break
+        nudges = 0
+        for tc in msg.tool_calls:
+            out.tool_calls += 1
+            try:
+                args = json.loads(tc.function.arguments or "{}")
+                command = args["command"] if tc.function.name == "bash" else None
+            except (json.JSONDecodeError, KeyError, TypeError):
+                command = None
+            if not isinstance(command, str):
+                result = 'error: call the bash tool with a JSON object {"command": "..."}'
+            else:
+                if on_step:
+                    on_step(out.steps, command)
+                r = sandbox.exec(command)
+                result = truncate(r.output, cfg.max_output_chars) + f"\n[exit code {r.exit_code}]"
+                if seen.get(command) == result:
+                    result += "\n" + REPEAT_NOTE
+                else:
+                    seen[command] = result
+            messages.append({"role": "tool", "tool_call_id": tc.id, "content": result})
+    return out

commitarena/cli.py

@@ -0,0 +1,313 @@
+"""Command line interface: mine, run, report, demo."""
+
+from __future__ import annotations
+
+import sys
+import webbrowser
+from datetime import datetime
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from . import __version__
+from .errors import CommitArenaError
+
+app = typer.Typer(
+    add_completion=False,
+    no_args_is_help=True,
+    rich_markup_mode="rich",
+    help="Your git history is the benchmark. Replay real bug fixes against any model, in a sandbox with "
+    "no network.",
+)
+# Wide enough for the leaderboard when output is piped (Rich falls back to 80 columns).
+console = Console(width=None if sys.stdout.isatty() else 120, highlight=False)
+err = Console(stderr=True)
+
+
+def _version(value: bool) -> None:
+    if value:
+        console.print(f"commitarena {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def _root(
+    version: Annotated[
+        bool, typer.Option("--version", callback=_version, is_eager=True, help="Show the version.")
+    ] = False,
+) -> None:
+    pass
+
+
+def _parse_pairs(values: list[str] | None, what: str) -> dict[str, str]:
+    out: dict[str, str] = {}
+    for v in values or []:
+        if "=" not in v:
+            raise CommitArenaError(f"{what} must look like MODEL=VALUE, got '{v}'.")
+        k, val = v.rsplit("=", 1)
+        out[k.strip()] = val.strip()
+    return out
+
+
+def _csv(value: str | None) -> list[str]:
+    return [x.strip() for x in (value or "").split(",") if x.strip()]
+
+
+@app.command()
+def mine(
+    repo: Annotated[str, typer.Argument(help="Git URL or local path of a Python repo that uses pytest.")],
+    out: Annotated[Path | None, typer.Option(help="Where to write tasks.json.")] = None,
+    rev: Annotated[str, typer.Option(help="Branch or commit to walk back from.")] = "HEAD",
+    since: Annotated[str | None, typer.Option(help="Only commits after this date (git --since).")] = None,
+    until: Annotated[str | None, typer.Option(help="Only commits before this date (git --until).")] = None,
+    scan: Annotated[int, typer.Option(help="How many recent commits to scan.")] = 200,
+    max_tasks: Annotated[int, typer.Option(help="Stop after this many validated tasks.")] = 10,
+    python: Annotated[str, typer.Option(help="Python version of the test image.")] = "3.12",
+    setup: Annotated[
+        str, typer.Option(help="Install command run once at image build time (with network).")
+    ] = "pip install --no-cache-dir -e .",
+    test_timeout: Annotated[int, typer.Option(help="Seconds allowed for one test-suite run.")] = 600,
+) -> None:
+    """Find commits that fix a bug with a test, and keep those whose new tests fail without the fix."""
+    from .mine import mine as do_mine
+    from .sandbox import EnvSpec
+    from .tasks import repo_slug
+
+    out = out or Path("tasks") / repo_slug(repo) / "tasks.json"
+
+    def on_event(kind: str, d: dict) -> None:
+        if kind == "build":
+            console.print(f"[dim]building test image {d['image']} (network on, only for this step)[/dim]")
+        elif kind == "candidates":
+            console.print(f"{d['count']} candidate commits touch both source and tests")
+        elif kind == "task":
+            console.print(f"  [green]valid[/green]  {d['id']}  {d['f2p']} fail->pass, {d['p2p']} pass->pass")
+        elif kind == "reject":
+            console.print(f"  [dim]skip   {d['sha']}  {d['reason']}[/dim]")
+
+    tasks, meta = do_mine(
+        repo, out=out, rev=rev, since=since, until=until, scan=scan, max_tasks=max_tasks,
+        env=EnvSpec(python=python, setup=setup), test_timeout=test_timeout, on_event=on_event,
+    )  # fmt: skip
+    console.print(f"\n[bold]{len(tasks)} validated tasks[/bold] written to {out}")
+    if tasks:
+        console.print(f"Next: commitarena run --tasks {out} --models ollama/qwen3:4b")
+
+
+def results_table(results, cutoffs: dict[str, str] | None = None) -> Table:
+    from .report import leaderboard
+
+    rows = leaderboard([r.to_dict() if hasattr(r, "to_dict") else r for r in results], cutoffs or {})
+    t = Table(title="Leaderboard", title_justify="left")
+    t.add_column("model", no_wrap=True)
+    for col in ("pass", "rate", "before cutoff", "after cutoff", "cost $", "tokens", "avg s", "avg steps"):
+        t.add_column(col, justify="right")
+
+    def split(s):
+        return f"{s['passed']}/{s['total']}" if s["total"] else "-"
+
+    for r in rows:
+        t.add_row(
+            r["model"], f"{r['passed']}/{r['total']}", f"{r['rate'] * 100:.0f}%", split(r["before"]),
+            split(r["after"]), f"{r['cost']:.4f}", f"{r['tokens']:,}", f"{r['avg_time']:.0f}",
+            f"{r['avg_steps']:.1f}",
+        )  # fmt: skip
+    return t
+
+
+@app.command()
+def run(
+    tasks: Annotated[Path, typer.Option(help="tasks.json from `commitarena mine`.")],
+    models: Annotated[str, typer.Option(help="Comma-separated provider/model, e.g. ollama/qwen3:4b.")],
+    out: Annotated[Path | None, typer.Option(help="Run directory (resumable).")] = None,
+    max_cost: Annotated[float, typer.Option(help="Stop before any call once this many USD are spent.")] = 1.0,
+    task_ids: Annotated[str | None, typer.Option(help="Comma-separated task ids to run.")] = None,
+    limit: Annotated[int | None, typer.Option(help="Run only the first N tasks.")] = None,
+    max_steps: Annotated[int, typer.Option(help="Model calls per attempt.")] = 30,
+    max_time: Annotated[int, typer.Option(help="Seconds per attempt.")] = 1200,
+    context_file: Annotated[
+        Path | None, typer.Option(help="AGENTS.md/CLAUDE.md to inject into the attempt.")
+    ] = None,
+    context_mode: Annotated[str, typer.Option(help="both | repo | prompt")] = "both",
+    strip_context_files: Annotated[
+        bool, typer.Option(help="Remove the repo's own AGENTS.md / CLAUDE.md first.")
+    ] = False,
+    cutoff: Annotated[
+        list[str] | None, typer.Option(help="Training cutoff, MODEL=YYYY-MM-DD (repeatable).")
+    ] = None,
+    price: Annotated[
+        list[str] | None, typer.Option(help="Price override, MODEL=IN,OUT in USD per 1M tokens (repeatable).")
+    ] = None,
+    cpus: Annotated[float, typer.Option(help="CPU limit of the sandbox.")] = 2.0,
+    memory: Annotated[str, typer.Option(help="Memory limit of the sandbox.")] = "2g",
+    command_timeout: Annotated[int, typer.Option(help="Seconds per agent command.")] = 120,
+) -> None:
+    """Run every task against every model with the same minimal agent, then write report.html."""
+    from .agent import AgentConfig
+    from .llm import Budget, ChatClient, ModelSpec
+    from .report import write_report
+    from .runner import ContextFile, run_suite
+    from .sandbox import SandboxConfig, ensure_docker
+    from .tasks import load_tasks
+
+    prices = {}
+    for k, v in _parse_pairs(price, "--price").items():
+        try:
+            pin, pout = (float(x) for x in v.split(","))
+        except ValueError as e:
+            raise CommitArenaError(f"--price for {k} must be IN,OUT (numbers), got '{v}'.") from e
+        prices[k] = (pin, pout)
+    specs = [ModelSpec.parse(m, prices.get(m)) for m in _csv(models)]
+    if not specs:
+        raise CommitArenaError("--models is empty.")
+    for s in specs:  # validate keys and prices before any work
+        ChatClient(s)._sdk()
+        if s.is_local:
+            _warn_ollama_context(s.name)
+    task_list = load_tasks(tasks, _csv(task_ids) or None)
+    if limit:
+        task_list = task_list[:limit]
+    ensure_docker()
+    out = out or Path("runs") / datetime.now().strftime("%Y%m%d-%H%M%S")
+    ctx = ContextFile.from_path(context_file, context_mode) if context_file else None
+    budget = Budget(max_cost=max_cost)
+    cutoffs = _parse_pairs(cutoff, "--cutoff")
+    console.print(
+        f"{len(task_list)} tasks x {len(specs)} models, sandbox --network none, {cpus} CPU, {memory}, "
+        f"budget ${max_cost:.2f}. Results in {out}"
+    )
+
+    def on_start(task, model):
+        console.print(f"[bold]{model.id}[/bold]  {task.id}  [dim]({task.date[:10]})[/dim]")
+
+    def on_step(step, command):
+        first = command.strip().splitlines()[0] if command.strip() else ""
+        console.print(f"  [dim]{step:>2} $ {first[:100]}[/dim]")
+
+    def on_result(r):
+        mark = "[green]PASS[/green]" if r.passed else "[red]FAIL[/red]"
+        console.print(
+            f"  {mark}  {r.fail_to_pass_passed}/{r.fail_to_pass_total} fail->pass, "
+            f"{r.pass_to_pass_failed} broken, {r.steps} steps, {r.wall_time_s:.0f}s, "
+            f"${r.cost_usd:.4f} ({r.stop_reason})  total ${budget.spent:.4f}"
+        )
+
+    results, abort = run_suite(
+        task_list, specs, out, budget=budget, context=ctx, strip_context_files=strip_context_files,
+        agent_config=AgentConfig(max_steps=max_steps, max_wall_time=max_time),
+        sandbox_config=SandboxConfig(cpus=cpus, memory=memory, command_timeout=command_timeout),
+        cutoffs=cutoffs, on_result=on_result, on_start=on_start, on_step=on_step,
+    )  # fmt: skip
+    report = write_report(out)
+    console.print(results_table(results, _load_cutoffs(out)))
+    console.print(f"Spent ${budget.spent:.4f}. Report: {report}")
+    if abort:
+        err.print(f"[red]{abort}[/red]")
+        raise typer.Exit(2)
+
+
+def _warn_ollama_context(name: str) -> None:
+    """Ollama's OpenAI endpoint ignores num_ctx; a small default window silently drops the task text."""
+    import json
+    import os
+    import urllib.request
+
+    from .llm import _ollama_base
+
+    if os.environ.get("OLLAMA_CONTEXT_LENGTH"):
+        return
+    try:
+        req = urllib.request.Request(
+            _ollama_base().removesuffix("/v1") + "/api/show", data=json.dumps({"model": name}).encode()
+        )
+        with urllib.request.urlopen(req, timeout=10) as r:
+            info = json.load(r)
+    except Exception:  # noqa: BLE001 - the real call reports connection problems
+        return
+    if "num_ctx" not in (info.get("parameters") or ""):
+        err.print(
+            f"[yellow]warning:[/yellow] ollama/{name} has no num_ctx set, so Ollama may serve it with a small "
+            f"default context (often 4096 tokens) and drop the task text. Create a variant:\n"
+            f"  printf 'FROM {name}\\nPARAMETER num_ctx 16384\\n' > Modelfile && "
+            f"ollama create {name.split(':')[0]}:16k -f Modelfile"
+        )
+
+
+def _load_cutoffs(run_dir: Path) -> dict[str, str]:
+    import json
+
+    from .report import DEFAULT_CUTOFFS
+
+    meta = json.loads((run_dir / "run.json").read_text()) if (run_dir / "run.json").exists() else {}
+    return {**DEFAULT_CUTOFFS, **meta.get("cutoffs", {})}
+
+
+@app.command()
+def report(
+    run_dir: Annotated[Path, typer.Argument(help="Run directory written by `commitarena run`.")],
+    out: Annotated[Path | None, typer.Option(help="Output HTML (default: <run_dir>/report.html).")] = None,
+    cutoff: Annotated[
+        list[str] | None, typer.Option(help="Override a training cutoff, MODEL=YYYY-MM-DD.")
+    ] = None,
+    open_browser: Annotated[bool, typer.Option("--open", help="Open the report in a browser.")] = False,
+) -> None:
+    """Render the static HTML report: leaderboard, task x model matrix, cutoff split, diffs."""
+    from .report import load_run, write_report
+
+    if not (run_dir / "results.jsonl").exists():
+        raise CommitArenaError(f"No results.jsonl in {run_dir}. Run `commitarena run` first.")
+    path = write_report(run_dir, out, _parse_pairs(cutoff, "--cutoff"))
+    data = load_run(run_dir)
+    console.print(
+        results_table(data["results"], {**_load_cutoffs(run_dir), **_parse_pairs(cutoff, "--cutoff")})
+    )
+    console.print(f"Report: {path}")
+    if open_browser:
+        webbrowser.open(path.resolve().as_uri())
+
+
+@app.command()
+def demo(
+    out: Annotated[Path, typer.Option(help="Where to write the demo report.")] = Path(
+        "commitarena-demo.html"
+    ),
+    open_browser: Annotated[bool, typer.Option("--open/--no-open", help="Open it in a browser.")] = True,
+) -> None:
+    """Show a RECORDED real run (no keys, no Docker): leaderboard in the terminal plus the HTML report."""
+    from .report import demo_dir, load_run, write_report
+
+    d = demo_dir()
+    data = load_run(d)
+    m = data["meta"]
+    console.print(
+        f"[bold]Recorded result[/bold] (not a live run): {m.get('updated_at', '')[:10]}, commitarena "
+        f"{m.get('version')}, models {', '.join(m.get('models', []))}, on {m.get('hardware', '?')}."
+    )
+    console.print(f"Repo: {m.get('repo', '')} ({m.get('repo_license', '')})")
+    console.print(results_table(data["results"], data["cutoffs"]))
+    path = write_report(d, out)
+    console.print(f"Report: {path}")
+    console.print("Run it yourself: commitarena mine <repo> && commitarena run --tasks ... --models ...")
+    if open_browser:
+        webbrowser.open(path.resolve().as_uri())
+
+
+def _sigterm(signum, frame):  # noqa: ARG001
+    raise KeyboardInterrupt
+
+
+def main() -> None:
+    import signal
+
+    signal.signal(signal.SIGTERM, _sigterm)  # so sandboxes are removed on kill
+    try:
+        app()
+    except CommitArenaError as e:
+        err.print(f"[red]error:[/red] {e}")
+        sys.exit(1)
+    except KeyboardInterrupt:
+        err.print("[yellow]interrupted[/yellow]")
+        sys.exit(130)

commitarena/demo/LICENSE-sqlparse

@@ -0,0 +1,25 @@
+Copyright (c) 2016, Andi Albrecht <albrecht.andi@gmail.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the authors nor the names of its contributors may be
+      used to endorse or promote products derived from this software without
+      specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.