outturn 0.1.0__py3-none-any.whl

outturn/__init__.py

@@ -0,0 +1,5 @@
+"""outturn: did the agent produce the right outcome? As a number."""
+from .models import Discrepancy, Entry, Kind, Outcome, Scenario, Turn
+
+__version__ = "0.1.0"
+__all__ = ["Outcome", "Entry", "Scenario", "Turn", "Discrepancy", "Kind", "__version__"]

outturn/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

outturn/adapters/__init__.py

@@ -0,0 +1,68 @@
+"""Adapter lookup.
+
+Two ways to name an adapter:
+
+    --adapter demo-booking
+        one of the demo agents that ship with outturn
+
+    --adapter myproject.agents:MyAgent
+        anything importable. Your agent lives in your repo, not in this one.
+
+The second form is the point. You should never have to edit a file inside
+this package to test your own agent.
+"""
+from __future__ import annotations
+
+import importlib
+
+from .base import Agent
+from .demo_booking import DemoBookingAgent
+from .demo_order import DemoOrderAgent
+
+BUILTIN: dict[str, type] = {
+    DemoOrderAgent.name: DemoOrderAgent,
+    DemoBookingAgent.name: DemoBookingAgent,
+}
+
+
+class AdapterError(ValueError):
+    pass
+
+
+def _load_path(spec: str) -> type:
+    """Import 'package.module:ClassName' or 'package.module.ClassName'."""
+    if ":" in spec:
+        module_name, _, attr = spec.partition(":")
+    else:
+        module_name, _, attr = spec.rpartition(".")
+    if not module_name or not attr:
+        raise AdapterError(
+            f"{spec!r} is not a known adapter and does not look like an import path. "
+            f"Use 'module:ClassName', or one of: {', '.join(sorted(BUILTIN))}"
+        )
+    try:
+        module = importlib.import_module(module_name)
+    except ImportError as e:
+        raise AdapterError(f"could not import {module_name!r}: {e}") from e
+    try:
+        return getattr(module, attr)
+    except AttributeError as e:
+        raise AdapterError(f"{module_name!r} has no attribute {attr!r}") from e
+
+
+def get(spec: str) -> Agent:
+    """Resolve an adapter name or import path into an instance."""
+    cls = BUILTIN.get(spec) or _load_path(spec)
+
+    agent = cls() if isinstance(cls, type) else cls   # a factory function is fine too
+
+    for method in ("reset", "send", "outcome"):
+        if not callable(getattr(agent, method, None)):
+            raise AdapterError(
+                f"{spec!r} is missing {method}(). An adapter needs reset(), send(text) "
+                f"and outcome()."
+            )
+    return agent
+
+
+__all__ = ["Agent", "AdapterError", "BUILTIN", "get", "DemoOrderAgent", "DemoBookingAgent"]

outturn/adapters/base.py

@@ -0,0 +1,25 @@
+"""The adapter contract. Three methods.
+
+    reset()      start a fresh conversation
+    send(text)   one caller turn in, the agent's reply out
+    outcome()    the structured result, right now
+
+If your agent cannot hand back a structured outcome, outturn cannot help you,
+and that is itself the finding. State that lives only inside the model's
+context is not inspectable, and what is not inspectable is not testable.
+Move the outcome into code and the tool works.
+"""
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+from ..models import Outcome
+
+
+@runtime_checkable
+class Agent(Protocol):
+    name: str
+
+    def reset(self) -> None: ...
+    def send(self, text: str) -> str: ...
+    def outcome(self) -> Outcome: ...

outturn/adapters/demo_booking.py

@@ -0,0 +1,75 @@
+"""A deliberately imperfect appointment booking agent.
+
+This one exists to prove the point of the whole rewrite: a booking has no
+line items at all. It is a handful of scalar fields. The same engine handles
+it without a single special case.
+
+Its planted flaw is the one real booking agents actually have: it hears the
+service and the day, and forgets to carry the duration across when the
+customer changes their mind mid call.
+"""
+from __future__ import annotations
+
+import random
+import re
+
+from ..models import Outcome
+
+SERVICES = {
+    "deep tissue massage": 60,
+    "swedish massage": 60,
+    "sports massage": 90,
+    "facial": 45,
+}
+_DAYS = ("monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday")
+
+
+class DemoBookingAgent:
+    name = "demo-booking"
+
+    def __init__(self, forget_duration_rate: float = 0.4) -> None:
+        self.forget_duration_rate = forget_duration_rate
+        self._state: dict = {}
+
+    def reset(self) -> None:
+        self._state = {"confirmed": False}
+
+    def send(self, text: str) -> str:
+        t = text.lower().strip()
+        changed = False
+
+        for service, minutes in SERVICES.items():
+            if service in t or service.split()[0] in t:
+                switching = "service" in self._state and self._state["service"] != service
+                self._state["service"] = service
+                # the planted flaw: on a switch it sometimes keeps the old duration
+                if not switching or random.random() > self.forget_duration_rate:
+                    self._state["duration_minutes"] = minutes
+                changed = True
+                break
+
+        m = re.search(r"\b(\d{1,2})[:.](\d{2})\s*(am|pm)?\b", t)
+        if m:
+            hour, minute, ampm = int(m.group(1)), m.group(2), m.group(3)
+            if ampm == "pm" and hour < 12:
+                hour += 12
+            self._state["time"] = f"{hour:02d}:{minute}"
+            changed = True
+
+        for day in _DAYS:
+            if day in t:
+                self._state["day"] = day
+                changed = True
+                break
+
+        if any(w in t for w in ("yes", "confirm", "book it", "that works", "go ahead")):
+            self._state["confirmed"] = True
+            return "Booked. You will get a confirmation by text."
+
+        if changed:
+            bits = [str(self._state.get(k)) for k in ("service", "day", "time") if self._state.get(k)]
+            return "Got it, " + ", ".join(bits) + ". Shall I book that?"
+        return "Sorry, I did not catch that."
+
+    def outcome(self) -> Outcome:
+        return Outcome(entries=(), fields=dict(self._state))

outturn/adapters/demo_order.py

@@ -0,0 +1,109 @@
+"""A deliberately imperfect restaurant ordering agent.
+
+It ships so you can see real output before writing any integration, and it
+fails in two ways on purpose, because an eval you cannot fail is not an eval.
+
+  1. It drops a modifier at random under load. That is what FLAKY looks like.
+  2. Its scope guard is tuned too tight, so "do you sell garlic bread" is
+     treated as an off topic question rather than a customer trying to order.
+     That is not a strawman. It is behaviour observed on a shipped agent.
+"""
+from __future__ import annotations
+
+import random
+import re
+
+from ..models import Entry, Outcome
+
+MENU = {
+    "pepperoni pizza": 12.00,
+    "margherita pizza": 10.50,
+    "garlic bread": 4.50,
+    "coke": 2.00,
+    "sparkling water": 2.50,
+}
+MODIFIERS = {"thin crust", "extra cheese", "no cheese", "well done", "sliced"}
+_NUM = {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "a": 1, "an": 1}
+
+
+class DemoOrderAgent:
+    name = "demo-order"
+
+    def __init__(self, drop_modifier_rate: float = 0.35) -> None:
+        self.drop_modifier_rate = drop_modifier_rate
+        self._lines: list[dict] = []
+
+    def reset(self) -> None:
+        self._lines = []
+
+    # ------------------------------------------------------------------ #
+
+    def _quantity(self, text: str) -> int:
+        m = re.search(r"\b(\d+)\b", text)
+        if m:
+            return int(m.group(1))
+        for word, n in _NUM.items():
+            if re.search(rf"\b{word}\b", text):
+                return n
+        return 1
+
+    def _find_item(self, text: str) -> str | None:
+        for item in sorted(MENU, key=len, reverse=True):
+            if item in text:
+                return item
+        if "pizza" in text:
+            return "pepperoni pizza"
+        return None
+
+    def send(self, text: str) -> str:
+        t = text.lower().strip()
+
+        # the scope guard, deliberately too tight
+        if t.startswith(("do you", "have you got", "is there")) and "?" in text:
+            return "I can only take orders from the menu, I am afraid."
+
+        if any(w in t for w in ("cancel", "remove", "take off", "drop the")):
+            target = self._find_item(t)
+            for i in range(len(self._lines) - 1, -1, -1):
+                if target is None or self._lines[i]["name"] == target:
+                    removed = self._lines.pop(i)
+                    return f"Removed the {removed['name']}."
+            return "There is nothing like that on your order."
+
+        mods = [m for m in MODIFIERS if m in t]
+        item = self._find_item(t)
+
+        # modifiers with no item named apply to the most recent line
+        if item is None and mods and self._lines:
+            line = self._lines[-1]
+            for m in mods:
+                if random.random() > self.drop_modifier_rate:
+                    if m not in line["tags"]:
+                        line["tags"].append(m)
+            return f"Noted on the {line['name']}."
+
+        if item is None:
+            if re.search(r"\bmake (it|that|those|them)\b", t):
+                if self._lines:
+                    self._lines[-1]["quantity"] = self._quantity(t)
+                    return f"Updated to {self._lines[-1]['quantity']}."
+            return "Sorry, I did not catch an item."
+
+        qty = self._quantity(t)
+        kept = [m for m in mods if random.random() > self.drop_modifier_rate]
+        self._lines.append({"name": item, "quantity": qty, "unit_price": MENU[item], "tags": kept})
+        return f"Added {qty} {item}."
+
+    def outcome(self) -> Outcome:
+        total = round(sum(l["quantity"] * l["unit_price"] for l in self._lines), 2)
+        return Outcome(
+            entries=tuple(
+                Entry(
+                    name=l["name"],
+                    numbers={"quantity": float(l["quantity"]), "unit_price": float(l["unit_price"])},
+                    tags=tuple(l["tags"]),
+                )
+                for l in self._lines
+            ),
+            fields={"total": total},
+        )

outturn/adapters/livekit.py

@@ -0,0 +1,175 @@
+"""LiveKit Agents driver.
+
+Not imported by default. Needs `pip install outturn[livekit]`.
+
+LiveKit ships its own roomless testing path, and this driver uses it rather
+than inventing one:
+
+    async with AgentSession(llm=llm) as session:
+        await session.start(MyAgent())
+        result = await session.run(user_input="two pepperoni pizzas")
+
+No room, no audio, no telephony. Fast enough to run in CI on every commit.
+
+THE ONE DESIGN DECISION WORTH KNOWING
+
+The outcome does not live in LiveKit. It lives in whatever your own function
+tools wrote it to, usually the session userdata, and only you know its shape.
+So this driver does the session work and you pass one callable that reads
+your state and returns an Outcome. The driver stays thin and never breaks
+because someone structured their state differently.
+
+USAGE
+
+    from dataclasses import dataclass, field
+    from livekit.agents import AgentSession, inference
+    from outturn.adapters.livekit import LiveKitAgent
+    from outturn.models import Outcome
+
+    @dataclass
+    class Booking:
+        service: str | None = None
+        day: str | None = None
+        confirmed: bool = False
+
+    async def make_session():
+        llm = inference.LLM(model="openai/gpt-4o-mini")
+        state = Booking()
+        session = AgentSession(llm=llm, userdata=state)
+        return session, MyAgent(), state
+
+    def read_outcome(state: Booking) -> Outcome:
+        return Outcome(fields={
+            "service": state.service,
+            "day": state.day,
+            "confirmed": state.confirmed,
+        })
+
+    agent = LiveKitAgent(make_session, read_outcome)
+
+    # then:  outturn scenarios/ --adapter myproject.evals:agent
+
+WHAT THIS DOES NOT TEST
+
+Speech to text, endpointing, turn taking. It measures whether your agent
+understands correctly, not whether it hears correctly. Both matter.
+"""
+from __future__ import annotations
+
+import asyncio
+import inspect
+from typing import Any, Callable
+
+from ..models import Outcome
+
+
+
+
+class LiveKitAgent:
+    """Drives a LiveKit AgentSession through text turns, one scenario at a time.
+
+    session_factory   async callable returning (session, agent, state).
+                      Called fresh on every reset, so each run starts clean.
+    read_outcome      takes that state, returns an Outcome.
+    timeout           seconds to wait for a single turn before the run fails.
+    """
+
+    name = "livekit"
+
+    def __init__(
+        self,
+        session_factory: Callable[[], Any],
+        read_outcome: Callable[[Any], Outcome],
+        *,
+        timeout: float = 60.0,
+    ) -> None:
+        self._factory = session_factory
+        self._read = read_outcome
+        self._timeout = timeout
+        self._session: Any = None
+        self._state: Any = None
+        self._loop: asyncio.AbstractEventLoop | None = None
+
+    # -- event loop plumbing, because outturn's contract is sync ----------- #
+
+    def _get_loop(self) -> asyncio.AbstractEventLoop:
+        if self._loop is None or self._loop.is_closed():
+            self._loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self._loop)
+        return self._loop
+
+    def _run(self, coro):
+        return self._get_loop().run_until_complete(coro)
+
+    # -- the three method contract ---------------------------------------- #
+
+    def reset(self) -> None:
+        """Fresh session for every run. No state leaks between runs."""
+        self.close()
+
+        async def _start():
+            made = self._factory()
+            if inspect.isawaitable(made):
+                made = await made
+            if not isinstance(made, tuple) or len(made) != 3:
+                raise TypeError(
+                    "session_factory must return (session, agent, state), "
+                    f"got {type(made).__name__}"
+                )
+            session, agent, state = made
+            await session.__aenter__()
+            await session.start(agent)
+            return session, state
+
+        self._session, self._state = self._run(_start())
+
+    def send(self, text: str) -> str:
+        if self._session is None:
+            raise RuntimeError("call reset() before send()")
+
+        async def _turn() -> str:
+            result = await asyncio.wait_for(
+                self._session.run(user_input=text), timeout=self._timeout
+            )
+            return _extract_reply(result)
+
+        return self._run(_turn())
+
+    def outcome(self) -> Outcome:
+        if self._state is None:
+            raise RuntimeError("call reset() before outcome()")
+        return self._read(self._state)
+
+    def close(self) -> None:
+        if self._session is not None and self._loop and not self._loop.is_closed():
+            try:
+                self._run(self._session.__aexit__(None, None, None))
+            except Exception:
+                pass
+        self._session = None
+        self._state = None
+
+
+def _extract_reply(result: Any) -> str:
+    """Extract the assistant's text from a RunResult.
+
+    Verified against livekit-agents 1.8.2: the reply is a ChatMessageEvent
+    in result.events with item.role == 'assistant' and item.content as a list
+    of strings. Returns the last such message, or "" if none is found.
+
+    Returns "" rather than raising so the outcome assertion can still run.
+    expect_reply_contains is meant to be used sparingly anyway.
+    """
+    events = getattr(result, "events", None) or []
+    for event in reversed(list(events)):
+        item = getattr(event, "item", None)
+        if item is None:
+            continue
+        if getattr(item, "role", None) != "assistant":
+            continue
+        content = getattr(item, "content", None)
+        if isinstance(content, list) and content:
+            first = content[0]
+            if isinstance(first, str) and first:
+                return first
+    return ""

outturn/assertions.py

@@ -0,0 +1,176 @@
+"""The diff engine.
+
+One rule governs every line here: numbers are compared exactly, words are
+compared fuzzily. A quantity of 3 is not "close to" 2 and a duration of 90
+minutes is not "close to" 60. But "deep tissue massage" and "Deep Tissue
+Massage (60min)" are the same service.
+
+Everywhere being clever would have been possible, being clever would have
+hidden a real failure behind a generous match. That is the one thing a test
+harness must never do.
+"""
+from __future__ import annotations
+
+import re
+from difflib import SequenceMatcher
+from typing import Any
+
+from .models import Discrepancy, Entry, Kind, Outcome
+
+_PUNCT = re.compile(r"[^a-z0-9 ]+")
+_SPACE = re.compile(r"\s+")
+# words carrying no identity, dropped before comparison
+_NOISE = {"a", "an", "the", "of", "with", "and", "please", "order", "for"}
+
+DEFAULT_NAME_THRESHOLD = 0.82
+DEFAULT_TAG_THRESHOLD = 0.80
+DEFAULT_TOLERANCE = 0.005
+
+
+def normalize(text: str) -> str:
+    """Lowercase, strip punctuation, drop noise words, crudely singularize.
+
+    Deliberately simple. If you need a stemmer here, your names are the
+    problem, not the matcher.
+    """
+    t = _PUNCT.sub(" ", str(text).lower())
+    words = [w for w in _SPACE.sub(" ", t).strip().split(" ") if w and w not in _NOISE]
+    out = []
+    for w in words:
+        if len(w) > 3 and w.endswith("es") and not w.endswith("ses"):
+            w = w[:-2]
+        elif len(w) > 3 and w.endswith("s") and not w.endswith("ss"):
+            w = w[:-1]
+        out.append(w)
+    return " ".join(sorted(out))
+
+
+def similarity(a: str, b: str) -> float:
+    return SequenceMatcher(None, normalize(a), normalize(b)).ratio()
+
+
+def _numbers_equal(a: float, b: float, tolerance: float) -> bool:
+    return abs(float(a) - float(b)) <= tolerance
+
+
+def _number_agreement(want: Entry, have: Entry, tolerance: float) -> int:
+    """How many of the expected numbers this candidate also matches.
+
+    Used only to break ties between candidates whose names score identically.
+    Two cart lines called "coffee" are indistinguishable by name, so the one
+    whose quantity also matches is the intended pairing. vaevals picked
+    whichever came first, which made a real bug look like two.
+    """
+    return sum(
+        1
+        for k, v in want.numbers.items()
+        if k in have.numbers and _numbers_equal(v, have.numbers[k], tolerance)
+    )
+
+
+def _match_tags(
+    expected: tuple[str, ...],
+    actual: tuple[str, ...],
+    where: str,
+    threshold: float,
+) -> list[Discrepancy]:
+    out: list[Discrepancy] = []
+    remaining = list(actual)
+    for want in expected:
+        best_i, best_score = -1, 0.0
+        for i, have in enumerate(remaining):
+            s = similarity(want, have)
+            if s > best_score:
+                best_i, best_score = i, s
+        if best_score >= threshold:
+            remaining.pop(best_i)
+        else:
+            out.append(Discrepancy(Kind.MISSING_TAG, where, expected=want, actual=None))
+    for leftover in remaining:
+        out.append(Discrepancy(Kind.EXTRA_TAG, where, expected=None, actual=leftover))
+    return out
+
+
+def _compare_fields(
+    expected: dict[str, Any],
+    actual: dict[str, Any],
+    *,
+    tag_threshold: float,
+    tolerance: float,
+) -> list[Discrepancy]:
+    """Scalars. Numbers and booleans exact, strings fuzzy.
+
+    Extra fields in the actual outcome are ignored on purpose. The scenario
+    states what must be true, not everything that may be present.
+    """
+    out: list[Discrepancy] = []
+    for key, want in expected.items():
+        if key not in actual or actual[key] is None:
+            out.append(Discrepancy(Kind.MISSING_FIELD, key, expected=want, actual=None))
+            continue
+        have = actual[key]
+
+        if isinstance(want, bool) or isinstance(have, bool):
+            if bool(want) != bool(have):
+                out.append(Discrepancy(Kind.WRONG_FIELD, key, expected=want, actual=have))
+        elif isinstance(want, (int, float)) and isinstance(have, (int, float)):
+            if not _numbers_equal(want, have, tolerance):
+                out.append(Discrepancy(Kind.WRONG_FIELD, key, expected=want, actual=have))
+        else:
+            if similarity(str(want), str(have)) < tag_threshold:
+                out.append(Discrepancy(Kind.WRONG_FIELD, key, expected=want, actual=have))
+    return out
+
+
+def compare_outcomes(
+    expected: Outcome,
+    actual: Outcome,
+    *,
+    name_threshold: float = DEFAULT_NAME_THRESHOLD,
+    tag_threshold: float = DEFAULT_TAG_THRESHOLD,
+    tolerance: float = DEFAULT_TOLERANCE,
+) -> list[Discrepancy]:
+    """Best-match on entry names, exact on every number, fuzzy on every word."""
+    out: list[Discrepancy] = []
+    unmatched: list[Entry] = list(actual.entries)
+
+    for want in expected.entries:
+        best_i, best_key = -1, (0.0, -1)
+        for i, have in enumerate(unmatched):
+            score = similarity(want.name, have.name)
+            key = (score, _number_agreement(want, have, tolerance))
+            if key > best_key:
+                best_i, best_key = i, key
+
+        if best_key[0] < name_threshold:
+            out.append(
+                Discrepancy(Kind.MISSING_ENTRY, "not found", expected=want.name, actual=None)
+            )
+            continue
+
+        have = unmatched.pop(best_i)
+
+        for key, wanted in want.numbers.items():
+            if key not in have.numbers:
+                out.append(
+                    Discrepancy(Kind.MISSING_NUMBER, f"{key} on {want.name!r}",
+                                expected=wanted, actual=None)
+                )
+            elif not _numbers_equal(wanted, have.numbers[key], tolerance):
+                out.append(
+                    Discrepancy(Kind.WRONG_NUMBER, f"{key} on {want.name!r}",
+                                expected=wanted, actual=have.numbers[key])
+                )
+
+        out.extend(_match_tags(want.tags, have.tags, f"on {want.name!r}", tag_threshold))
+
+    for leftover in unmatched:
+        out.append(
+            Discrepancy(Kind.EXTRA_ENTRY, "not expected", expected=None, actual=leftover.name)
+        )
+
+    out.extend(
+        _compare_fields(expected.fields, actual.fields,
+                        tag_threshold=tag_threshold, tolerance=tolerance)
+    )
+    return out

outturn/cli.py

@@ -0,0 +1,58 @@
+"""Command line entry point."""
+from __future__ import annotations
+
+import argparse
+import random
+import sys
+from pathlib import Path
+
+from . import adapters, report, runner, scenario
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="outturn",
+        description="Did the agent produce the right outcome? As a number.",
+    )
+    p.add_argument("path", type=Path, help="a scenario file, or a directory of them")
+    p.add_argument("--adapter", default="demo-order",
+                   help="a built in name, or an import path like myproject.agents:MyAgent")
+    p.add_argument("--runs", type=int, default=5, help="runs per scenario (default 5)")
+    p.add_argument("--seed", type=int, default=None, help="seed the RNG for a repeatable run")
+    p.add_argument("--threshold", type=float, default=None,
+                   help="exit 1 if overall accuracy falls below this (0 to 1)")
+    p.add_argument("--json", action="store_true", help="machine readable output")
+    p.add_argument("--list-adapters", action="store_true", help="show registered adapters and exit")
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    if args.list_adapters:
+        for name in sorted(adapters.BUILTIN):
+            print(name)
+        print("\nor any import path, for example  myproject.agents:MyAgent")
+        return 0
+
+    if args.runs < 1:
+        print("--runs must be at least 1", file=sys.stderr)
+        return 2
+    if args.seed is not None:
+        random.seed(args.seed)
+
+    try:
+        scenarios = scenario.load(args.path)
+        agent = adapters.get(args.adapter)
+    except (scenario.ScenarioError, adapters.AdapterError, OSError) as e:
+        print(str(e).strip("'"), file=sys.stderr)
+        return 2
+
+    reports = runner.run_all(agent, scenarios, args.runs)
+    print(report.as_json(reports) if args.json else report.render(reports))
+
+    if args.threshold is not None:
+        passes, runs = report.overall(reports)
+        if runs and (passes / runs) < args.threshold:
+            return 1
+    return 0

outturn/models.py

@@ -0,0 +1,158 @@
+"""Core data types.
+
+One idea runs through all of it: an agent conversation ends in a structured
+outcome, and a structured outcome can be compared exactly. A restaurant order
+is one kind of outcome. So is a booking, a triage, a qualified lead.
+
+Two comparison rules, applied everywhere:
+  numbers are compared exactly     a quantity of 3 is not close to 2
+  words are compared fuzzily       "deep tissue" and "Deep Tissue Massage" match
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Entry:
+    """One repeated thing in an outcome.
+
+    A cart line. A passenger. A prescription. Anything the agent can produce
+    more than one of.
+
+    name    matched fuzzily, it is how the entry is identified
+    numbers matched exactly, quantity, price, duration, dosage
+    tags    matched fuzzily as a set, modifiers, options, flags
+    """
+
+    name: str
+    numbers: dict[str, float] = field(default_factory=dict)
+    tags: tuple[str, ...] = ()
+
+    @staticmethod
+    def from_dict(d: dict[str, Any]) -> "Entry":
+        if "name" not in d:
+            raise ValueError("an entry needs a 'name'")
+        numbers = {
+            str(k): float(v)
+            for k, v in d.items()
+            if k not in ("name", "tags") and isinstance(v, (int, float)) and not isinstance(v, bool)
+        }
+        tags = d.get("tags", ())
+        if isinstance(tags, str):
+            tags = (tags,)
+        return Entry(name=str(d["name"]), numbers=numbers, tags=tuple(str(t) for t in tags))
+
+
+@dataclass(frozen=True)
+class Outcome:
+    """What the conversation actually produced.
+
+    entries  the repeated things, may be empty
+    fields   the scalars. numbers and booleans exact, strings fuzzy
+
+    A restaurant order is entries plus a total field.
+    A booking is no entries at all, just fields.
+    Both work.
+    """
+
+    entries: tuple[Entry, ...] = ()
+    fields: dict[str, Any] = field(default_factory=dict)
+
+    @staticmethod
+    def from_dict(d: dict[str, Any] | None) -> "Outcome":
+        d = dict(d or {})
+        raw_entries = d.pop("entries", ())
+        return Outcome(
+            entries=tuple(Entry.from_dict(e) for e in raw_entries),
+            fields={str(k): v for k, v in d.items()},
+        )
+
+    def is_empty(self) -> bool:
+        return not self.entries and not self.fields
+
+
+@dataclass(frozen=True)
+class Turn:
+    """One thing the caller says."""
+
+    user: str
+    # substrings the reply must contain. use sparingly: asserting on phrasing
+    # is brittle by nature, which is the problem this tool exists to avoid.
+    expect_reply_contains: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class Scenario:
+    id: str
+    turns: tuple[Turn, ...]
+    expect: Outcome
+    description: str = ""
+    tags: tuple[str, ...] = ()
+    runs: int | None = None   # overrides the global run count
+
+
+class Kind(str, Enum):
+    MISSING_ENTRY = "missing_entry"
+    EXTRA_ENTRY = "extra_entry"
+    WRONG_NUMBER = "wrong_number"
+    MISSING_NUMBER = "missing_number"
+    MISSING_TAG = "missing_tag"
+    EXTRA_TAG = "extra_tag"
+    MISSING_FIELD = "missing_field"
+    WRONG_FIELD = "wrong_field"
+    REPLY_MISSING_TEXT = "reply_missing_text"
+
+
+@dataclass(frozen=True)
+class Discrepancy:
+    kind: Kind
+    detail: str
+    expected: Any = None
+    actual: Any = None
+
+    def __str__(self) -> str:
+        if self.expected is None and self.actual is None:
+            return f"{self.kind.value}: {self.detail}"
+        return f"{self.kind.value}: {self.detail} (expected {self.expected!r}, got {self.actual!r})"
+
+
+@dataclass
+class RunResult:
+    scenario_id: str
+    run_index: int
+    discrepancies: list[Discrepancy] = field(default_factory=list)
+    transcript: list[tuple[str, str]] = field(default_factory=list)
+    error: str | None = None
+
+    @property
+    def passed(self) -> bool:
+        return self.error is None and not self.discrepancies
+
+
+@dataclass
+class ScenarioReport:
+    scenario_id: str
+    results: list[RunResult] = field(default_factory=list)
+
+    @property
+    def runs(self) -> int:
+        return len(self.results)
+
+    @property
+    def passes(self) -> int:
+        return sum(1 for r in self.results if r.passed)
+
+    @property
+    def pass_rate(self) -> float:
+        return (self.passes / self.runs) if self.runs else 0.0
+
+    @property
+    def is_flaky(self) -> bool:
+        """Passed sometimes and failed sometimes.
+
+        Worse than always failing, because it ships.
+        """
+        return 0 < self.passes < self.runs

outturn/report.py

@@ -0,0 +1,72 @@
+"""Turns reports into something a person reads in five seconds."""
+from __future__ import annotations
+
+import json
+from collections import Counter
+
+from .models import ScenarioReport
+
+
+def _failure_lines(rep: ScenarioReport, limit: int = 3) -> list[str]:
+    counts: Counter[str] = Counter()
+    for r in rep.results:
+        if r.error:
+            counts[f"error: {r.error}"] += 1
+        for d in r.discrepancies:
+            counts[str(d)] += 1
+    return [f"        {n}x {text}" for text, n in counts.most_common(limit)]
+
+
+def overall(reports: list[ScenarioReport]) -> tuple[int, int]:
+    passes = sum(r.passes for r in reports)
+    runs = sum(r.runs for r in reports)
+    return passes, runs
+
+
+def render(reports: list[ScenarioReport]) -> str:
+    lines: list[str] = []
+    for rep in reports:
+        flag = "  FLAKY" if rep.is_flaky else ""
+        lines.append(
+            f"{rep.pass_rate * 100:5.0f}%  {rep.scenario_id:<34} {rep.passes}/{rep.runs}{flag}"
+        )
+        if rep.passes < rep.runs:
+            lines.extend(_failure_lines(rep))
+
+    passes, runs = overall(reports)
+    rate = (passes / runs * 100) if runs else 0.0
+    lines.append("")
+    lines.append(f"outcome accuracy: {rate:.1f}%  ({passes}/{runs} runs)")
+
+    flaky = [r.scenario_id for r in reports if r.is_flaky]
+    if flaky:
+        lines.append(f"flaky scenarios ({len(flaky)}): {', '.join(flaky)}")
+        lines.append("a scenario that passes sometimes is a bug that ships sometimes.")
+    return "\n".join(lines)
+
+
+def as_json(reports: list[ScenarioReport]) -> str:
+    passes, runs = overall(reports)
+    return json.dumps(
+        {
+            "accuracy": (passes / runs) if runs else 0.0,
+            "passes": passes,
+            "runs": runs,
+            "flaky": [r.scenario_id for r in reports if r.is_flaky],
+            "scenarios": [
+                {
+                    "id": r.scenario_id,
+                    "passes": r.passes,
+                    "runs": r.runs,
+                    "pass_rate": r.pass_rate,
+                    "flaky": r.is_flaky,
+                    "failures": sorted(
+                        {str(d) for res in r.results for d in res.discrepancies}
+                    ),
+                    "errors": sorted({res.error for res in r.results if res.error}),
+                }
+                for r in reports
+            ],
+        },
+        indent=2,
+    )

outturn/runner.py

@@ -0,0 +1,42 @@
+"""Runs each scenario N times.
+
+N, not once. Agents are nondeterministic, so one green run is not evidence.
+A bug that appears one run in five is invisible to a boolean test and obvious
+in a percentage.
+"""
+from __future__ import annotations
+
+from .adapters.base import Agent
+from .assertions import compare_outcomes
+from .models import Discrepancy, Kind, RunResult, Scenario, ScenarioReport
+
+
+def run_once(agent: Agent, scenario: Scenario, run_index: int) -> RunResult:
+    result = RunResult(scenario_id=scenario.id, run_index=run_index)
+    try:
+        agent.reset()
+        for turn in scenario.turns:
+            reply = agent.send(turn.user)
+            result.transcript.append((turn.user, reply))
+            for needle in turn.expect_reply_contains:
+                if needle.lower() not in (reply or "").lower():
+                    result.discrepancies.append(
+                        Discrepancy(Kind.REPLY_MISSING_TEXT, f"after {turn.user!r}",
+                                    expected=needle, actual=reply)
+                    )
+        result.discrepancies.extend(compare_outcomes(scenario.expect, agent.outcome()))
+    except Exception as e:  # an agent that crashes is a failing run, not a crashing suite
+        result.error = f"{type(e).__name__}: {e}"
+    return result
+
+
+def run_scenario(agent: Agent, scenario: Scenario, runs: int) -> ScenarioReport:
+    n = scenario.runs if scenario.runs is not None else runs
+    report = ScenarioReport(scenario_id=scenario.id)
+    for i in range(n):
+        report.results.append(run_once(agent, scenario, i))
+    return report
+
+
+def run_all(agent: Agent, scenarios: list[Scenario], runs: int) -> list[ScenarioReport]:
+    return [run_scenario(agent, s, runs) for s in scenarios]

outturn/scenario.py

@@ -0,0 +1,67 @@
+"""Load scenarios from YAML. Fail loudly on a bad file, never guess."""
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from .models import Outcome, Scenario, Turn
+
+
+class ScenarioError(ValueError):
+    pass
+
+
+def _turn(raw: object, ctx: str) -> Turn:
+    if isinstance(raw, str):
+        return Turn(user=raw)
+    if isinstance(raw, dict):
+        if "user" not in raw:
+            raise ScenarioError(f"{ctx}: a turn needs a 'user' key")
+        contains = raw.get("expect_reply_contains", ())
+        if isinstance(contains, str):
+            contains = (contains,)
+        return Turn(user=str(raw["user"]), expect_reply_contains=tuple(str(c) for c in contains))
+    raise ScenarioError(f"{ctx}: a turn must be a string or a mapping, got {type(raw).__name__}")
+
+
+def load_file(path: Path) -> Scenario:
+    raw = yaml.safe_load(path.read_text())
+    if not isinstance(raw, dict):
+        raise ScenarioError(f"{path}: top level must be a mapping")
+    for key in ("id", "turns", "expect"):
+        if key not in raw:
+            raise ScenarioError(f"{path}: missing required key {key!r}")
+    turns = raw["turns"]
+    if not isinstance(turns, list) or not turns:
+        raise ScenarioError(f"{path}: 'turns' must be a non empty list")
+    try:
+        expect = Outcome.from_dict(raw["expect"])
+    except ValueError as e:
+        raise ScenarioError(f"{path}: bad 'expect' block, {e}") from e
+    if expect.is_empty():
+        raise ScenarioError(f"{path}: 'expect' asserts nothing. A scenario that cannot fail is not a test.")
+    return Scenario(
+        id=str(raw["id"]),
+        description=str(raw.get("description", "")),
+        turns=tuple(_turn(t, str(path)) for t in turns),
+        expect=expect,
+        tags=tuple(str(t) for t in raw.get("tags", ())),
+        runs=(int(raw["runs"]) if raw.get("runs") is not None else None),
+    )
+
+
+def load(target: Path) -> list[Scenario]:
+    """A single file, or every .yaml under a directory, recursively."""
+    if target.is_file():
+        return [load_file(target)]
+    files = sorted(p for p in target.rglob("*.y*ml") if p.is_file())
+    if not files:
+        raise ScenarioError(f"{target}: no .yaml scenario files found")
+    scenarios = [load_file(p) for p in files]
+    seen: set[str] = set()
+    for s in scenarios:
+        if s.id in seen:
+            raise ScenarioError(f"duplicate scenario id {s.id!r}")
+        seen.add(s.id)
+    return scenarios

outturn-0.1.0.dist-info/METADATA

@@ -0,0 +1,240 @@
+Metadata-Version: 2.4
+Name: outturn
+Version: 0.1.0
+Summary: Did the agent produce the right outcome? As a number.
+Author: Wisdom Omons
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/OsasDTEch/outturn
+Project-URL: Source, https://github.com/OsasDTEch/outturn
+Keywords: voice-agents,llm,evaluation,testing,livekit,agents
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: PyYAML>=6.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4; extra == "dev"
+Provides-Extra: livekit
+Requires-Dist: livekit-agents==1.8.2; extra == "livekit"
+Dynamic: license-file
+
+# outturn
+
+**Did the agent produce the right outcome? As a number.**
+
+A conversation does not end in words. It ends in an order, a booking, a routed ticket, a qualified lead. That is structured data, and structured data can be compared exactly. So whether your agent got it right does not have to be something you learn from a refund. It can be a percentage that runs on every commit.
+
+```
+  100%  simple_booking                     8/8
+   50%  service_switch_duration            4/8  FLAKY
+        4x wrong_field: duration_minutes (expected 90, got 60)
+  100%  never_confirmed                    8/8
+
+outcome accuracy: 83.3%  (20/24 runs)
+flaky scenarios (1): service_switch_duration
+a scenario that passes sometimes is a bug that ships sometimes.
+```
+
+---
+
+## Why this exists
+
+Most voice agent testing is a person calling the number and listening. That finds the bug in front of you and none of the others, it cannot run in CI, and it produces an opinion instead of a number.
+
+The alternatives are not much better. Asserting on the transcript is asserting on phrasing, which changes every time you touch the prompt. Using a model to grade the conversation means one nondeterministic system judging another.
+
+The outcome is the way out. It is the thing the customer actually receives, it is already structured, and it can be compared exactly.
+
+## Three design decisions
+
+**Numbers are exact, words are fuzzy.** A quantity of 3 is not close to 2, and a 90 minute appointment is not close to a 60 minute one. But "deep tissue massage" and "Deep Tissue Massage (60min)" are the same service. So every number is compared exactly and every word by normalized similarity.
+
+**Every scenario runs N times and reports a pass rate.** Agents are nondeterministic. One green run is not evidence. A bug that appears one run in five is invisible to a boolean test and obvious in a percentage, so outturn reports rates and flags anything that passed sometimes and failed sometimes as FLAKY. Intermittent is worse than broken, because intermittent hides.
+
+**An outcome is repeated things plus scalars.** That is all. An order is entries plus a total. A booking is no entries at all, just fields. A triage is fields. One model, no special cases per domain, which is why the same engine handles an ordering agent and a scheduling agent without a line of new code.
+
+## Install
+
+```bash
+pip install outturn
+```
+
+Or from source, if you want the demo scenarios to play with:
+
+```bash
+git clone https://github.com/OsasDTEch/outturn
+cd outturn
+pip install -e ".[dev]"
+```
+
+## Run it right now
+
+Two deliberately imperfect demo agents ship with the repo, so you can see real output before writing any integration.
+
+```bash
+outturn scenarios/order   --adapter demo-order   --runs 8 --seed 42
+outturn scenarios/booking --adapter demo-booking --runs 8 --seed 42
+```
+
+Each fails on purpose, because an eval you cannot fail is not an eval.
+
+The ordering agent drops a modifier at random under load, which is what FLAKY looks like, and its scope guard is tuned too tight so "do you sell garlic bread" is treated as an off topic question rather than a customer trying to order. That second one is not a strawman, it is behaviour observed on a shipped restaurant agent.
+
+The booking agent forgets to carry the duration across when the customer switches service mid call. A 90 minute massage in a 60 minute slot double books the therapist, and nobody finds out until the day.
+
+## Writing a scenario
+
+Turns in, expected outcome out.
+
+**An order**, which has line items:
+
+```yaml
+id: modifier_stacking
+description: >
+  Two modifiers on one item, added in a separate turn from the item itself.
+tags: [order, modifiers]
+
+turns:
+  - "Hi, can I get two large pepperoni pizzas"
+  - "Actually make those thin crust"
+  - "And extra cheese on them please"
+
+expect:
+  entries:
+    - name: pepperoni pizza
+      quantity: 2
+      unit_price: 12.00
+      tags: [thin crust, extra cheese]
+  total: 24.00
+```
+
+**A booking**, which has none:
+
+```yaml
+id: service_switch_duration
+turns:
+  - "Can I book a deep tissue massage for Friday"
+  - "Actually make it a sports massage instead"
+  - "4:00 pm, and yes please book it"
+
+expect:
+  service: sports massage
+  day: friday
+  time: "16:00"
+  duration_minutes: 90
+  confirmed: true
+```
+
+Same engine. Anything numeric under an entry becomes an exact comparison. Anything at the top level is a field: numbers and booleans exact, strings fuzzy.
+
+You can also assert on a reply, though use it sparingly since phrasing is the brittle part:
+
+```yaml
+turns:
+  - user: "Do you sell garlic bread?"
+    expect_reply_contains: ["garlic bread"]
+```
+
+## Connecting your own agent
+
+Implement three methods.
+
+```python
+from outturn.models import Outcome, Entry
+
+class MyAgent:
+    name = "my-agent"
+
+    def reset(self) -> None:
+        """Fresh conversation. Called before every run."""
+        self.session = start_session()
+
+    def send(self, text: str) -> str:
+        """One caller turn in, the agent's reply out."""
+        return self.session.turn(text)
+
+    def outcome(self) -> Outcome:
+        """The structured result, right now."""
+        return Outcome(
+            entries=tuple(
+                Entry(name=l.name,
+                      numbers={"quantity": l.qty, "unit_price": l.price},
+                      tags=tuple(l.modifiers))
+                for l in self.session.order.lines
+            ),
+            fields={"total": self.session.order.total},
+        )
+```
+
+Then point outturn at it. **Your agent lives in your repo, not in this one.**
+
+```bash
+outturn scenarios/ --adapter myproject.agents:MyAgent --runs 10
+```
+
+Anything importable works. If the object is missing one of the three methods,
+outturn says so before the run starts rather than failing with an
+AttributeError halfway through.
+
+If your agent cannot hand back a structured outcome, outturn cannot help you, and that is itself the finding. State that lives only in the model's context is not inspectable, and what is not inspectable is not testable. Move the outcome into code and the tool works.
+
+## LiveKit
+
+There is a driver at `outturn/adapters/livekit.py`. It takes a session factory and one callable that reads your state and returns an Outcome, because the outcome does not live in LiveKit, it lives in whatever your own function tools wrote it to.
+
+**Verified against livekit-agents 1.8.2.** Tested with the ollama.com cloud API (model `gemma4:31b`, OpenAI-compatible endpoint). Confirmed:
+
+- A session starts with no room and no audio.
+- State written by function tools persists across multiple `session.run()` calls on the same session, so multi-turn scenarios work as written.
+- The assistant reply is a `ChatMessageEvent` in `result.events` with `item.content[0]` as the text.
+
+Pin your own install to `livekit-agents==1.8.2` until you have tested against a newer version. This API is young and the `RunResult` shape has changed between releases.
+
+## In CI
+
+```bash
+outturn scenarios/booking --adapter demo-booking --runs 10 --threshold 0.95
+```
+
+Exits non zero if overall accuracy falls below the threshold. Every bug you fix becomes a scenario, so it can never come back silently. `--json` gives machine readable output for tracking accuracy over time.
+
+## The scenarios worth writing
+
+The ones that break agents, roughly in order of how often they do:
+
+- **Corrections mid call.** The customer changes their mind after the agent has already recorded something. This is where most outcomes go wrong, and the failure is usually a field that did not get updated alongside the one that did.
+- **Reference without naming.** "Make it three" with no item named.
+- **Cancellation.** Added, then removed. Do the derived values follow?
+- **Confirmation.** Did the agent act on intent rather than on an actual yes? Booking a customer who was still thinking is a real and expensive failure.
+- **Name collisions.** Two items or two services that sound alike over a phone line.
+- **Illegal combinations.** Extra cheese on a coke. Should be rejected by schema, not accepted politely and discovered later.
+- **Out of scope questions that are really orders.** "Do you sell X" is a customer trying to buy X.
+
+## What this does not test
+
+outturn drives your agent through text. That is fast enough to run in CI on every commit, and it isolates the reasoning and outcome layer from the audio layer.
+
+It does not test speech to text, endpointing or turn taking, and those are real sources of failure, particularly on telephony where audio is narrowband and degrades worst on exactly what matters here: names, numbers and proper nouns.
+
+**So this measures whether your agent understands correctly, not whether it hears correctly.** Both matter. For the timing half of the picture, see [voice-latency-profiler](https://github.com/OsasDTEch/voice-latency-profiler).
+
+## Status
+
+v0.1. The core works and is tested, 31 tests. Roadmap, roughly in order:
+
+- [x] LiveKit driver verified against livekit-agents 1.8.2 (ollama.com cloud, `gemma4:31b`)
+- [ ] Path assertions: which tools were called, with which arguments
+- [ ] Pipecat driver
+- [ ] Audio mode, TTS in and STT out, for end to end runs
+- [ ] Accuracy tracked over time, so regressions show as a trend
+
+`PRD.md` and `TRD.md` in this repo cover the reasoning and the internals.
+
+This project supersedes [voice-agent-evals](https://github.com/OsasDTEch/voice-agent-evals), which asserted on carts only. An order is one kind of outcome among many, and the narrower version was useful to about a tenth of the agents worth testing.
+
+## Licence
+
+MIT. Built by [Wisdom Omons](https://linkedin.com/in/omons-wisdom).

outturn-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+outturn/__init__.py,sha256=7tECRjTfloJxUAFQMhiyl8Oj3DotJuN9z3lQOrWJFn8,251
+outturn/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
+outturn/assertions.py,sha256=tTx1Sv3S1xxfKUOTZe02fIN38NQhSM_9iqVKTAzUP8E,6243
+outturn/cli.py,sha256=127Hwi3too_Rz1h1DlWKXuU3YYwufZFfGz2C9D2jMOI,2115
+outturn/models.py,sha256=o6a-_z8YCtE4iL5-hoAru0X2toPuft8L_3-Bow-6Ct0,4564
+outturn/report.py,sha256=Ev4in-SAjtPQIPrRsLZxBydyoLhljXwyXiirpm1MeIw,2391
+outturn/runner.py,sha256=RACFiJ4ZD2G0GSCpZd4FayA8pyqGIqM1iUlOB6fzxqs,1744
+outturn/scenario.py,sha256=XmJ5GQ4DzUYmcTV3el4NXYwjAE9CypzFxGcxIWMICH8,2446
+outturn/adapters/__init__.py,sha256=F4wvsQAcf-fgERHkQXYmXvhdqnjnBlqAiY-Ln7pty-M,2114
+outturn/adapters/base.py,sha256=Xsk0ymoZbr7donfiTIzpEgq3bM7zI9-TC8trbtXgllU,755
+outturn/adapters/demo_booking.py,sha256=PCkjsFxab9yoODGHU78Ua87xu0sgC6wxveANP6AXcCQ,2622
+outturn/adapters/demo_order.py,sha256=w1UqGXfRzshCyBL0Vv11qYqkKmly0cxZWaDA-x11ICQ,3990
+outturn/adapters/livekit.py,sha256=eA1_NTvbpzQoVVmzTfu64H9ZtfqsAglf21s4LOzG6Rg,5815
+outturn-0.1.0.dist-info/licenses/LICENSE,sha256=kM3jvrV9E_rQdz7LH8CfBIuunYVt5FIXd2kiVoRQu08,1069
+outturn-0.1.0.dist-info/METADATA,sha256=D1a-Jhq33C_MhXE9pOVEzvXKwE7fgj-aRzYyU_zVWqU,10673
+outturn-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+outturn-0.1.0.dist-info/entry_points.txt,sha256=Q1mlhUgk2gC4AlyKx2A2JJGi5nuZd5uZctaGgM1p2E8,45
+outturn-0.1.0.dist-info/top_level.txt,sha256=2kGQyWMtSL4ys27HOT8DpQ28PZg_VqKC-W6s96hz9bo,8
+outturn-0.1.0.dist-info/RECORD,,

outturn-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

outturn-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+outturn = outturn.cli:main

outturn-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Wisdom Omons
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

outturn-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+outturn