stated-confidence 0.1.0__py3-none-any.whl

stated_confidence/__init__.py

@@ -0,0 +1,41 @@
+"""stated-confidence: make agents say how sure they are, and on what basis.
+
+An LLM asked to attach a confidence *level* and an evidential *basis* to
+each claim it makes tends to fabricate less: naming the kind of evidence
+forces a provenance check the model would otherwise skip. This package is
+that schema, the vocabulary behind it, prompt guidance for filling it, and
+the consistency checks that send hedging-without-meaning-it back to the
+model.
+"""
+
+from .basis import (
+    BasisEnum,
+    DataAnalysisBasis,
+    DecisionBasis,
+    GeneralKnowledgeBasis,
+)
+from .checks import check_all, check_consistency, iter_confidences
+from .guidance import confidence_instructions
+from .levels import ConfidenceLevel, ConfidenceType
+from .model import Confidence
+from .presentation import SCALE_EXPLANATION, Tone, describe, tone
+from .terms import Term
+
+__all__ = [
+    "SCALE_EXPLANATION",
+    "BasisEnum",
+    "Confidence",
+    "ConfidenceLevel",
+    "ConfidenceType",
+    "DataAnalysisBasis",
+    "DecisionBasis",
+    "GeneralKnowledgeBasis",
+    "Term",
+    "Tone",
+    "check_all",
+    "check_consistency",
+    "confidence_instructions",
+    "describe",
+    "iter_confidences",
+    "tone",
+]

stated_confidence/basis.py

@@ -0,0 +1,260 @@
+"""Basis vocabularies: *what kind of evidence* stands behind a claim.
+
+The level scale is fixed; the basis set is the extension point. A domain
+declares its own by subclassing ``BasisEnum`` and filling ``__terms__``
+and ``__fallback__``. Three sets ship: general knowledge, data analysis,
+and decisions.
+"""
+
+import logging
+from enum import StrEnum
+from typing import ClassVar
+
+from .levels import normalize_token
+from .terms import Term
+
+logger = logging.getLogger("stated_confidence")
+
+
+class BasisEnum(StrEnum):
+    """Base class for basis vocabularies.
+
+    ``__terms__`` maps each value to its ``Term``; ``__fallback__`` is the
+    value substituted for anything unrecognized. Models occasionally invent
+    plausible values (one emitted ``INDUSTRY_STANDARD`` before it existed),
+    and one bad string must not fail a whole response. The substitution is
+    logged at WARNING on the ``stated_confidence`` logger so a basis set
+    can be grown deliberately from what models actually say.
+    """
+
+    __terms__: ClassVar[dict[str, Term]] = {}
+    __fallback__: ClassVar[str] = ""
+    __registry__: ClassVar[list[type["BasisEnum"]]] = []
+
+    def __init_subclass__(cls, **kwargs: object) -> None:
+        super().__init_subclass__(**kwargs)
+        BasisEnum.__registry__.append(cls)
+
+    @classmethod
+    def registered(cls) -> "list[type[BasisEnum]]":
+        """Every basis set defined so far, shipped and user-defined."""
+        return list(cls.__registry__)
+
+    @classmethod
+    def _missing_(cls, value: object) -> "BasisEnum | None":
+        if not isinstance(value, str):
+            return None
+        token = normalize_token(value)
+        for member in cls:
+            if member.value == token:
+                return member
+        if not cls.__fallback__:
+            return None
+        logger.warning(
+            "Unrecognized %s %r; using %s",
+            cls.__name__,
+            value,
+            cls.__fallback__,
+        )
+        return cls(cls.__fallback__)
+
+    @classmethod
+    def fallback(cls) -> "BasisEnum":
+        return cls(cls.__fallback__)
+
+    @property
+    def term(self) -> Term:
+        return type(self).__terms__[self.value]
+
+    @property
+    def strength(self) -> str | None:
+        return self.term.strength
+
+    @classmethod
+    def weakest(cls) -> list["BasisEnum"]:
+        return [member for member in cls if member.strength == "weak"]
+
+
+class GeneralKnowledgeBasis(BasisEnum):
+    """For claims about the world: facts, history, science, fiction."""
+
+    DOCUMENTED_FACT = "DOCUMENTED_FACT"
+    SCIENTIFIC_CONSENSUS = "SCIENTIFIC_CONSENSUS"
+    HISTORICAL_RECORD = "HISTORICAL_RECORD"
+    INDUSTRY_STANDARD = "INDUSTRY_STANDARD"
+    EXPERT_OPINION = "EXPERT_OPINION"
+    THEORETICAL_FRAMEWORK = "THEORETICAL_FRAMEWORK"
+    FICTIONAL_UNIVERSE = "FICTIONAL_UNIVERSE"
+    SPECULATION = "SPECULATION"
+    EDUCATED_GUESS = "EDUCATED_GUESS"
+
+    __fallback__ = "EXPERT_OPINION"
+    __terms__ = {
+        "DOCUMENTED_FACT": Term(
+            label="Documented fact",
+            description="Stated directly in recorded, citable sources.",
+            guidance="recorded evidence you could cite",
+            strength="strong",
+        ),
+        "SCIENTIFIC_CONSENSUS": Term(
+            label="Scientific consensus",
+            description="Supported by peer-reviewed scientific agreement.",
+            guidance="peer-reviewed agreement",
+            strength="strong",
+        ),
+        "HISTORICAL_RECORD": Term(
+            label="Historical record",
+            description="Part of the documented historical record.",
+            guidance="documented history",
+            strength="strong",
+        ),
+        "INDUSTRY_STANDARD": Term(
+            label="Industry standard",
+            description=(
+                "Established standards or widespread industry practice."
+            ),
+            guidance=(
+                "established standards and widespread engineering practice"
+            ),
+            strength="strong",
+        ),
+        "EXPERT_OPINION": Term(
+            label="Expert opinion",
+            description="Based on the judgment of subject-matter experts.",
+            guidance="the judgment of subject-matter experts",
+            strength="moderate",
+        ),
+        "THEORETICAL_FRAMEWORK": Term(
+            label="Theoretical framework",
+            description="Follows from an established theory or model.",
+            guidance="an established theory or model",
+            strength="moderate",
+        ),
+        "FICTIONAL_UNIVERSE": Term(
+            label="Fictional universe",
+            description="True within a fictional or creative work.",
+            guidance="creative content that is true within its fiction",
+            strength="moderate",
+        ),
+        "SPECULATION": Term(
+            label="Speculation",
+            description="An unverified theory or possibility.",
+            guidance="unverified theories",
+            strength="weak",
+        ),
+        "EDUCATED_GUESS": Term(
+            label="Educated guess",
+            description="A plausible inference made without direct evidence.",
+            guidance="plausible inference with no direct evidence",
+            strength="weak",
+        ),
+    }
+
+
+class DataAnalysisBasis(BasisEnum):
+    """For claims an agent makes about data it retrieved or computed."""
+
+    QUERY_RESULT = "QUERY_RESULT"
+    COMPUTED = "COMPUTED"
+    SCHEMA_INFERENCE = "SCHEMA_INFERENCE"
+    GENERAL_KNOWLEDGE = "GENERAL_KNOWLEDGE"
+    ASSUMPTION = "ASSUMPTION"
+
+    __fallback__ = "ASSUMPTION"
+    __terms__ = {
+        "QUERY_RESULT": Term(
+            label="Query result",
+            description="Read directly from the rows the query returned.",
+            guidance="a value read directly from returned rows",
+            strength="strong",
+        ),
+        "COMPUTED": Term(
+            label="Computed",
+            description=(
+                "Arithmetic over the returned rows, such as a ratio, "
+                "difference, or trend."
+            ),
+            guidance=(
+                "arithmetic you did over returned rows "
+                "(a ratio, difference, or trend)"
+            ),
+            strength="strong",
+        ),
+        "SCHEMA_INFERENCE": Term(
+            label="Schema inference",
+            description=(
+                "Inferred from table or column names and the schema, "
+                "not verified against data."
+            ),
+            guidance=(
+                "something inferred from table or column names "
+                "rather than checked against data"
+            ),
+            strength="moderate",
+        ),
+        "GENERAL_KNOWLEDGE": Term(
+            label="General knowledge",
+            description=(
+                "Background knowledge about the domain, not from this data."
+            ),
+            guidance="background knowledge about the domain, not this data",
+            strength="moderate",
+        ),
+        "ASSUMPTION": Term(
+            label="Assumption",
+            description="A premise that was not checked against the data.",
+            guidance="a premise you did not check against the data",
+            strength="weak",
+        ),
+    }
+
+
+class DecisionBasis(BasisEnum):
+    """For recommendations and decisions."""
+
+    POLICY = "POLICY"
+    MEASURED_OUTCOME = "MEASURED_OUTCOME"
+    PRECEDENT = "PRECEDENT"
+    TRADEOFF_ANALYSIS = "TRADEOFF_ANALYSIS"
+    EXPERT_JUDGMENT = "EXPERT_JUDGMENT"
+    ASSUMPTION = "ASSUMPTION"
+
+    __fallback__ = "ASSUMPTION"
+    __terms__ = {
+        "POLICY": Term(
+            label="Policy",
+            description="Required by a stated policy, rule, or constraint.",
+            guidance="a stated policy, rule, or hard constraint",
+            strength="strong",
+        ),
+        "MEASURED_OUTCOME": Term(
+            label="Measured outcome",
+            description="Observed results from measurement or experiment.",
+            guidance="results actually measured or observed",
+            strength="strong",
+        ),
+        "PRECEDENT": Term(
+            label="Precedent",
+            description="What was done in comparable past cases.",
+            guidance="what comparable past cases did",
+            strength="moderate",
+        ),
+        "TRADEOFF_ANALYSIS": Term(
+            label="Trade-off analysis",
+            description="Weighing the options against stated criteria.",
+            guidance="weighing the options against stated criteria",
+            strength="moderate",
+        ),
+        "EXPERT_JUDGMENT": Term(
+            label="Expert judgment",
+            description="Based on the judgment of subject-matter experts.",
+            guidance="the judgment of subject-matter experts",
+            strength="moderate",
+        ),
+        "ASSUMPTION": Term(
+            label="Assumption",
+            description="A premise that was taken as given, not verified.",
+            guidance="a premise taken as given without verification",
+            strength="weak",
+        ),
+    }

stated_confidence/checks.py

@@ -0,0 +1,84 @@
+"""Consistency rules between what a ``Confidence`` says about itself.
+
+Each rule catches a way of hedging without meaning it: a HIGH level on a
+basis that admits there is no evidence, a SPECULATIVE level on a basis
+that says the claim is documented, or a non-HIGH level with no reason
+given. Violations are plain sentences a
+model can act on when they come back as a retry.
+"""
+
+from collections.abc import Iterator
+from typing import Any
+
+from pydantic import BaseModel
+
+from .levels import ConfidenceLevel, ConfidenceType
+from .model import Confidence
+
+
+def check_consistency(confidence: Confidence) -> list[str]:
+    """Return every violated rule, or an empty list when consistent."""
+    violations: list[str] = []
+    level = confidence.level
+    basis = confidence.basis
+    if level is ConfidenceLevel.SPECULATIVE and basis.strength == "strong":
+        violations.append(
+            f"level SPECULATIVE contradicts basis {basis.value} "
+            f"({basis.term.description.rstrip('.').lower()}); a claim that "
+            "rests on that is asserted, not speculated. Raise the level or "
+            "choose the basis the idea actually rests on."
+        )
+    if level is ConfidenceLevel.HIGH and basis.strength == "weak":
+        violations.append(
+            f"level HIGH contradicts basis {basis.value} "
+            f"({basis.term.description.rstrip('.').lower()}); lower the "
+            "level or choose the basis that actually supports the claim."
+        )
+    if (
+        level is ConfidenceLevel.HIGH
+        and confidence.type is ConfidenceType.SPECULATIVE
+    ):
+        violations.append(
+            "level HIGH contradicts type SPECULATIVE; lower the level or "
+            "change the type."
+        )
+    if level is not ConfidenceLevel.HIGH and not confidence.reasoning.strip():
+        violations.append(
+            f"reasoning is empty; say in a sentence why the confidence is "
+            f"{level.value}."
+        )
+    return violations
+
+
+def iter_confidences(
+    value: Any, path: str = ""
+) -> Iterator[tuple[str, Confidence]]:
+    """Yield every ``Confidence`` reachable from ``value`` with its path.
+
+    Walks Pydantic models, lists, tuples, and dict values, so an output
+    model holding ``findings: list[Finding]`` needs no adapter.
+    """
+    if isinstance(value, Confidence):
+        yield path, value
+        return
+    if isinstance(value, BaseModel):
+        for name in type(value).model_fields:
+            child = getattr(value, name)
+            yield from iter_confidences(
+                child, f"{path}.{name}" if path else name
+            )
+    elif isinstance(value, list | tuple):
+        for index, item in enumerate(value):
+            yield from iter_confidences(item, f"{path}[{index}]")
+    elif isinstance(value, dict):
+        for key, item in value.items():
+            yield from iter_confidences(item, f"{path}[{key!r}]")
+
+
+def check_all(value: Any) -> list[str]:
+    """Violations for every ``Confidence`` in ``value``, prefixed by path."""
+    return [
+        f"{path}: {violation}" if path else violation
+        for path, confidence in iter_confidences(value)
+        for violation in check_consistency(confidence)
+    ]

stated_confidence/guidance.py

@@ -0,0 +1,52 @@
+"""Prompt text that teaches a model to fill ``Confidence`` honestly.
+
+The field descriptions in the JSON schema carry the same vocabulary; this
+block belongs in the system prompt or instructions, where models weight
+it more than schema descriptions and where the rule about numbers lands
+before the model starts writing.
+"""
+
+from .basis import BasisEnum
+from .levels import LEVEL_TERMS, ConfidenceLevel
+
+
+def confidence_instructions(
+    basis_set: type[BasisEnum],
+    *,
+    field_name: str = "confidence",
+    localized: bool = False,
+) -> str:
+    """Instructions for filling every ``field_name`` in the output.
+
+    ``localized`` adds the rule that free text follows the user's language
+    while enum values stay uppercase English.
+    """
+    levels = "; ".join(
+        f"{level.value} for {LEVEL_TERMS[level].guidance}"
+        for level in ConfidenceLevel
+    )
+    bases = "; ".join(
+        f"{member.value} for {member.term.guidance}" for member in basis_set
+    )
+    weakest = " or ".join(member.value for member in basis_set.weakest())
+    lines = [
+        f"For every `{field_name}`, state how sure you are and what that "
+        "certainty rests on. Before writing a claim, ask what kind of "
+        "evidence you actually have for it; if the honest answer is "
+        f"{weakest}, say so rather than presenting the claim as established.",
+        f"`{field_name}.level` must be exactly one of: {levels}. Never use "
+        "numbers or percentages for the level. SPECULATIVE is the level for "
+        "an idea you are offering, not a claim you are making; use it "
+        "freely, and say in `reasoning` what the idea extends.",
+        f"`{field_name}.basis` must be exactly one of: {bases}.",
+        f"`{field_name}.reasoning` is one or two sentences on why that "
+        "level and basis apply. `evidence` lists what concretely backs the "
+        "claim; leave it empty when nothing concrete does.",
+    ]
+    if localized:
+        lines.append(
+            "Write `reasoning` and `label` in the language of the user's "
+            "input; `level`, `basis`, and `type` values stay in uppercase "
+            "English."
+        )
+    return "\n".join(lines)

stated_confidence/levels.py

@@ -0,0 +1,152 @@
+"""The confidence level scale and the kind-of-information type."""
+
+import logging
+from enum import StrEnum
+
+from .terms import Term
+
+logger = logging.getLogger("stated_confidence")
+
+NUMERIC_HIGH = 0.8
+NUMERIC_MEDIUM = 0.5
+
+
+def normalize_token(value: str) -> str:
+    return value.strip().upper().replace(" ", "_").replace("-", "_")
+
+
+class ConfidenceLevel(StrEnum):
+    """How sure the model is that a claim is true.
+
+    ``SPECULATIVE`` is not a low rung on the scale: it is the level for
+    ideas the model offers rather than asserts (an extension a source
+    suggests, a hypothesis, a what-if). Giving the model that room is part
+    of what keeps it from dressing ideas up as facts.
+    """
+
+    HIGH = "HIGH"
+    MEDIUM = "MEDIUM"
+    LOW = "LOW"
+    SPECULATIVE = "SPECULATIVE"
+
+    @classmethod
+    def _missing_(cls, value: object) -> "ConfidenceLevel | None":
+        # Models ignore "no numbers" often enough that mapping a score is
+        # cheaper than failing the whole response; case drift ("High")
+        # is the historical storage format.
+        if isinstance(value, bool):
+            return None
+        if isinstance(value, int | float):
+            return cls.from_score(float(value))
+        if not isinstance(value, str):
+            return None
+        token = normalize_token(value)
+        for member in cls:
+            if member.value == token:
+                return member
+        try:
+            return cls.from_score(float(token))
+        except ValueError:
+            logger.warning(
+                "Unrecognized confidence level %r; using MEDIUM", value
+            )
+            return cls.MEDIUM
+
+    @classmethod
+    def from_score(cls, score: float) -> "ConfidenceLevel":
+        """Map a 0-1 (or 0-100) probability-style score onto the scale."""
+        if score > 1:
+            score /= 100
+        if score >= NUMERIC_HIGH:
+            return cls.HIGH
+        if score >= NUMERIC_MEDIUM:
+            return cls.MEDIUM
+        return cls.LOW
+
+
+LEVEL_TERMS: dict[ConfidenceLevel, Term] = {
+    ConfidenceLevel.HIGH: Term(
+        label="High confidence",
+        description="Well-documented information that is firmly established.",
+        guidance="well-documented claims you could verify directly",
+        strength="strong",
+    ),
+    ConfidenceLevel.MEDIUM: Term(
+        label="Medium confidence",
+        description=(
+            "Generally accepted information that involves some interpretation."
+        ),
+        guidance="generally accepted claims that involve some interpretation",
+        strength="moderate",
+    ),
+    ConfidenceLevel.LOW: Term(
+        label="Low confidence",
+        description=(
+            "Uncertain or disputed content with limited supporting evidence."
+        ),
+        guidance="uncertain, disputed, or speculative claims",
+        strength="weak",
+    ),
+    ConfidenceLevel.SPECULATIVE: Term(
+        label="Speculative",
+        description=(
+            "Offered as a possibility rather than asserted: hypothetical, "
+            "fictional, or an idea that goes beyond the sources."
+        ),
+        guidance=(
+            "ideas you are proposing rather than asserting: extensions, "
+            "hypotheses, what-ifs that go beyond the evidence"
+        ),
+        strength="weak",
+    ),
+}
+
+
+class ConfidenceType(StrEnum):
+    """What kind of information the claim is."""
+
+    FACTUAL = "FACTUAL"
+    CONSENSUS = "CONSENSUS"
+    THEORETICAL = "THEORETICAL"
+    SPECULATIVE = "SPECULATIVE"
+
+    @classmethod
+    def _missing_(cls, value: object) -> "ConfidenceType | None":
+        if not isinstance(value, str):
+            return None
+        token = normalize_token(value)
+        for member in cls:
+            if member.value == token:
+                return member
+        logger.warning(
+            "Unrecognized confidence type %r; using CONSENSUS", value
+        )
+        return cls.CONSENSUS
+
+
+TYPE_TERMS: dict[ConfidenceType, Term] = {
+    ConfidenceType.FACTUAL: Term(
+        label="Factual",
+        description="Based on verifiable facts.",
+        guidance="real-world verifiable facts such as recorded events",
+        strength="strong",
+    ),
+    ConfidenceType.CONSENSUS: Term(
+        label="Consensus",
+        description="Reflects broad agreement among sources.",
+        guidance="generally accepted information",
+        strength="moderate",
+    ),
+    ConfidenceType.THEORETICAL: Term(
+        label="Theoretical",
+        description="Follows from a theory or framework.",
+        guidance="scientific theories and scholarly frameworks",
+        strength="moderate",
+    ),
+    ConfidenceType.SPECULATIVE: Term(
+        label="Speculative",
+        description="A hypothesis or conjecture.",
+        guidance="fiction, hypothetical scenarios, or unverified claims",
+        strength="weak",
+    ),
+}

stated_confidence/model.py

@@ -0,0 +1,107 @@
+"""The ``Confidence`` model attached to each claim an agent makes."""
+
+from typing import Any, get_args
+
+from pydantic import BaseModel, Field, model_validator
+
+from .basis import BasisEnum
+from .levels import (
+    LEVEL_TERMS,
+    TYPE_TERMS,
+    ConfidenceLevel,
+    ConfidenceType,
+)
+
+
+def level_guidance() -> str:
+    options = ", ".join(
+        f"{level.value} for {LEVEL_TERMS[level].guidance}"
+        for level in ConfidenceLevel
+    )
+    return (
+        f"Confidence level, exactly one of {options}. "
+        "Never use numbers or percentages."
+    )
+
+
+def type_guidance() -> str:
+    options = ", ".join(
+        f"{kind.value} for {term.guidance}"
+        for kind, term in TYPE_TERMS.items()
+    )
+    return f"Kind of information being assessed: {options}."
+
+
+class Confidence[B: BasisEnum](BaseModel):
+    """A stated confidence level and the evidential basis behind it.
+
+    Parametrize with a basis set: ``Confidence[DataAnalysisBasis]``. The
+    generic form is what an agent's output model embeds; presentation and
+    consistency checks work on any parametrization.
+    """
+
+    level: ConfidenceLevel = Field(
+        default=ConfidenceLevel.MEDIUM, description=level_guidance()
+    )
+    basis: B = Field(
+        description=(
+            "The kind of evidence behind this claim. Choose the value that "
+            "most honestly describes what the claim rests on."
+        )
+    )
+    type: ConfidenceType | None = Field(
+        default=None, description=type_guidance()
+    )
+    label: str = Field(
+        default="",
+        description=(
+            "Human-readable confidence label in the language of the user's "
+            "input, e.g. 'Haute confiance'. Leave empty for English."
+        ),
+    )
+    reasoning: str = Field(
+        default="",
+        description=(
+            "One or two sentences on why this level and basis apply, in the "
+            "language of the user's input."
+        ),
+    )
+    evidence: list[str] = Field(
+        default_factory=list,
+        description=(
+            "What concretely backs the claim: source titles or URLs, "
+            "document ids, column names, tool results. Empty when nothing "
+            "concrete does."
+        ),
+    )
+
+    @classmethod
+    def basis_set(cls) -> "type[BasisEnum] | None":
+        """The basis enum this class is parametrized with, if any."""
+        annotation = cls.model_fields["basis"].annotation
+        if isinstance(annotation, type) and issubclass(annotation, BasisEnum):
+            return annotation
+        for arg in get_args(annotation):
+            if isinstance(arg, type) and issubclass(arg, BasisEnum):
+                return arg
+        return None
+
+    @model_validator(mode="before")
+    @classmethod
+    def _default_missing_basis(cls, data: Any) -> Any:
+        # A model that omits the basis entirely gets the set's fallback,
+        # for the same reason an invented value does: one thin claim must
+        # not fail the whole response.
+        if isinstance(data, dict) and data.get("basis") in (None, ""):
+            basis_set = cls.basis_set()
+            if basis_set is not None and basis_set.__fallback__:
+                data = {**data, "basis": basis_set.__fallback__}
+        return data
+
+    @property
+    def is_weak(self) -> bool:
+        """True when the level or the basis says this is not to be relied on."""
+        return (
+            LEVEL_TERMS[self.level].strength == "weak"
+            or self.basis.strength == "weak"
+        )

stated_confidence/presentation.py

@@ -0,0 +1,70 @@
+"""Display vocabulary: labels, descriptions, and a framework-neutral tone.
+
+Kept free of any CSS or component library so a web UI, a terminal, and a
+report can all read the same words.
+"""
+
+from typing import Literal
+
+from .basis import BasisEnum
+from .levels import LEVEL_TERMS, TYPE_TERMS, ConfidenceLevel, ConfidenceType
+from .terms import Term
+
+Tone = Literal["positive", "neutral", "caution", "warning"]
+
+_TONES: dict[ConfidenceLevel, Tone] = {
+    ConfidenceLevel.HIGH: "positive",
+    ConfidenceLevel.MEDIUM: "neutral",
+    ConfidenceLevel.LOW: "caution",
+    ConfidenceLevel.SPECULATIVE: "warning",
+}
+
+SCALE_EXPLANATION = (
+    "The model's own assessment of how well-supported this content is. "
+    + " · ".join(
+        f"{level.value.title()}: {LEVEL_TERMS[level].description.rstrip('.').lower()}"
+        for level in ConfidenceLevel
+    )
+)
+
+
+def _prettify(value: str) -> str:
+    words = value.replace("_", " ").lower().strip()
+    return words[:1].upper() + words[1:]
+
+
+def describe(
+    value: ConfidenceLevel | ConfidenceType | BasisEnum | str,
+) -> Term:
+    """The ``Term`` for an enum member, or a prettified stand-in for a string.
+
+    Strings are matched against the levels and every registered basis
+    set; anything else (a value from a set this process has not defined,
+    an old spelling) gets a readable label and no description, never an
+    error.
+    """
+    if isinstance(value, ConfidenceLevel):
+        return LEVEL_TERMS[value]
+    if isinstance(value, ConfidenceType):
+        return TYPE_TERMS[value]
+    if isinstance(value, BasisEnum):
+        return value.term
+    token = value.strip().upper()
+    if token in ConfidenceLevel.__members__:
+        return LEVEL_TERMS[ConfidenceLevel[token]]
+    for basis_set in BasisEnum.registered():
+        if token in basis_set.__terms__:
+            return basis_set.__terms__[token]
+    return Term(label=_prettify(value), description="")
+
+
+def tone(level: ConfidenceLevel | str | None) -> Tone:
+    """A four-way tone for badges and highlights; unknown values are neutral."""
+    if level is None:
+        return "neutral"
+    if not isinstance(level, ConfidenceLevel):
+        try:
+            level = ConfidenceLevel(level)
+        except ValueError:
+            return "neutral"
+    return _TONES[level]

stated_confidence/pydantic_ai.py

@@ -0,0 +1,47 @@
+"""Pydantic AI integration: an output validator that sends inconsistent
+confidence assessments back to the model as a retry.
+
+Usage::
+
+    from stated_confidence.pydantic_ai import confidence_output_validator
+
+    agent = Agent(..., output_type=Report)
+    agent.output_validator(confidence_output_validator())
+
+Requires the ``pydantic-ai`` extra.
+"""
+
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+from pydantic_ai import ModelRetry, RunContext
+
+from .checks import check_all
+
+OutputT = TypeVar("OutputT")
+ExtraCheck = Callable[[RunContext[Any], Any], list[str]]
+
+
+def confidence_output_validator(
+    *extra_checks: ExtraCheck,
+) -> Callable[[RunContext[Any], OutputT], OutputT]:
+    """Build a validator raising ``ModelRetry`` for every violation found.
+
+    ``extra_checks`` are ``(ctx, output) -> list[str]`` callables for rules
+    that need the run's context, such as checking that evidence names a
+    column the agent's query actually returned. Their messages are joined
+    with the built-in consistency rules into one retry prompt.
+    """
+
+    def validate(ctx: RunContext[Any], output: OutputT) -> OutputT:
+        violations = check_all(output)
+        for check in extra_checks:
+            violations.extend(check(ctx, output))
+        if violations:
+            raise ModelRetry(
+                "Fix these confidence assessments and answer again:\n"
+                + "\n".join(f"- {v}" for v in violations)
+            )
+        return output
+
+    return validate

stated_confidence/taxonomy.py

@@ -0,0 +1,57 @@
+"""Export the vocabulary as JSON, the interchange format other language
+packages (and the checked-in ``taxonomy.json``) are built from.
+
+Run ``python -m stated_confidence.taxonomy`` to print it.
+"""
+
+import json
+import sys
+from dataclasses import asdict
+from typing import Any
+
+from .basis import (
+    BasisEnum,
+    DataAnalysisBasis,
+    DecisionBasis,
+    GeneralKnowledgeBasis,
+)
+from .levels import LEVEL_TERMS, TYPE_TERMS
+from .presentation import _TONES
+
+BASIS_SETS: tuple[type[BasisEnum], ...] = (
+    GeneralKnowledgeBasis,
+    DataAnalysisBasis,
+    DecisionBasis,
+)
+
+
+def _terms(table: dict[Any, Any]) -> dict[str, dict[str, Any]]:
+    return {str(key.value): asdict(term) for key, term in table.items()}
+
+
+def taxonomy() -> dict[str, Any]:
+    return {
+        "version": 1,
+        "levels": {
+            level.value: {**asdict(term), "tone": _TONES[level]}
+            for level, term in LEVEL_TERMS.items()
+        },
+        "types": _terms(TYPE_TERMS),
+        "basis_sets": {
+            basis_set.__name__: {
+                "fallback": basis_set.__fallback__,
+                "terms": {
+                    member.value: asdict(member.term) for member in basis_set
+                },
+            }
+            for basis_set in BASIS_SETS
+        },
+    }
+
+
+def dumps() -> str:
+    return json.dumps(taxonomy(), indent=2, ensure_ascii=False) + "\n"
+
+
+if __name__ == "__main__":
+    sys.stdout.write(dumps())

stated_confidence/terms.py

@@ -0,0 +1,19 @@
+"""Vocabulary entries shared by every enum in the taxonomy.
+
+A ``Term`` carries three texts for one enum value: what a UI shows
+(``label``, ``description``) and what a model is told when asked to pick
+the value (``guidance``). ``strength`` is what the consistency checks use.
+"""
+
+from dataclasses import dataclass
+from typing import Literal
+
+Strength = Literal["strong", "moderate", "weak"]
+
+
+@dataclass(frozen=True, slots=True)
+class Term:
+    label: str
+    description: str
+    guidance: str = ""
+    strength: Strength | None = None

stated_confidence-0.1.0.dist-info/METADATA

@@ -0,0 +1,129 @@
+Metadata-Version: 2.5
+Name: stated-confidence
+Version: 0.1.0
+Summary: Make LLM agents state the confidence level and the evidential basis behind each claim.
+Project-URL: Homepage, https://github.com/johnwlockwood/stated-confidence
+Author-email: John Lockwood <johnwlockwood@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,confidence,grounding,hallucination,llm,pydantic,pydantic-ai
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: pydantic>=2.5
+Provides-Extra: pydantic-ai
+Requires-Dist: pydantic-ai-slim>=1.0; extra == 'pydantic-ai'
+Description-Content-Type: text/markdown
+
+# stated-confidence
+
+Make LLM agents say how sure they are, and what that certainty rests on.
+
+Ask a model to attach a confidence **level** and an evidential **basis** to
+each claim it produces and it fabricates less. The level alone is common
+practice; the basis is what does the work. Choosing between "documented
+fact", "expert opinion", and "educated guess" forces a provenance check the
+model would otherwise skip, and a claim it cannot back gets labelled as a
+guess or quietly dropped instead of dressed up as a figure.
+
+This package is that schema, the vocabulary behind it, prompt guidance for
+filling it, and the consistency checks that send hedging-without-meaning-it
+back to the model. It was extracted from a production knowledge-graph
+generator where it was observed to sharply reduce invented facts and
+figures. The effect is observed, not yet measured; an evaluation harness is
+planned.
+
+## Install
+
+```bash
+pip install stated-confidence            # pydantic only
+pip install "stated-confidence[pydantic-ai]"
+```
+
+## The shape
+
+```python
+from stated_confidence import Confidence, DataAnalysisBasis
+
+
+class Finding(BaseModel):
+    text: str
+    confidence: Confidence[DataAnalysisBasis]
+```
+
+A `Confidence` has:
+
+| field       | meaning                                                                   |
+| ----------- | ------------------------------------------------------------------------- |
+| `level`     | `HIGH`, `MEDIUM`, `LOW`, or `SPECULATIVE`. Never a number.                |
+| `basis`     | The kind of evidence, from a basis set chosen for the domain.             |
+| `reasoning` | One or two sentences on why that level and basis apply.                   |
+| `evidence`  | What concretely backs the claim: sources, ids, column names, tool results. |
+| `type`      | Optional: `FACTUAL`, `CONSENSUS`, `THEORETICAL`, `SPECULATIVE`.           |
+| `label`     | Optional localized label for display.                                     |
+
+`SPECULATIVE` is not the bottom of the scale. It is the level for an idea
+the model is offering rather than a claim it is making: an extension a
+source suggests, a hypothesis, a what-if. A generator that has somewhere
+honest to put an idea does not dress it up as a fact; take that room away
+and the ideas come back as fabricated "facts" or disappear altogether.
+
+The level scale is fixed. The **basis set** is the extension point. Three
+ship:
+
+- `GeneralKnowledgeBasis` for claims about the world: documented fact,
+  scientific consensus, historical record, industry standard, expert
+  opinion, theoretical framework, fictional universe, speculation, educated
+  guess.
+- `DataAnalysisBasis` for agents reasoning over data they retrieved: query
+  result, computed, schema inference, general knowledge, assumption.
+- `DecisionBasis` for recommendations: policy, measured outcome, precedent,
+  trade-off analysis, expert judgment, assumption.
+
+Define your own by subclassing `BasisEnum` with a `__terms__` table and a
+`__fallback__` value.
+
+## Prompting
+
+```python
+from stated_confidence import confidence_instructions
+
+system_prompt += confidence_instructions(DataAnalysisBasis, localized=True)
+```
+
+## Pydantic AI
+
+```python
+from pydantic_ai import Agent
+from stated_confidence.pydantic_ai import confidence_output_validator
+
+agent = Agent("openai:gpt-5", output_type=Report)
+agent.output_validator(confidence_output_validator())
+```
+
+Every `Confidence` anywhere in the output is checked. A `HIGH` level on a
+basis of `ASSUMPTION`, a `SPECULATIVE` level on a basis of `DOCUMENTED_FACT`,
+or a non-`HIGH` level with empty reasoning comes back to the model as a
+retry listing what to fix. Pass extra `(ctx, output) -> list[str]` callables for rules that
+need run context, such as requiring that evidence names a column the
+agent's query actually returned.
+
+## Lenient parsing, strict prompting
+
+Models drift. `"High"` parses. `0.9` parses (as `HIGH`). An invented basis
+value parses as the set's fallback and is logged at `WARNING` on the
+`stated_confidence` logger so the set can grow from what models actually
+say.
+
+## Display
+
+`describe(value)` returns a label and description for any level, type, or
+basis value; `tone(level)` maps to `positive`, `neutral`, `caution`, or
+`warning` with no CSS framework attached. `taxonomy.json` at the package
+root is the same vocabulary as data, for other languages to consume.

stated_confidence-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+stated_confidence/__init__.py,sha256=dPe7yjNwuK_546rgGYnM7SKQ9FJBxqv3ZIk8UxoXHsY,1192
+stated_confidence/basis.py,sha256=goO44qynjvztxzWY_c3wP0YbbYj_uWPaZAz_rsrZHTs,9033
+stated_confidence/checks.py,sha256=nQSej_cU-u2ChjYwpeMSD--fdABeY1uGmwhA7L-8ByE,3205
+stated_confidence/guidance.py,sha256=knNSHHhqrHdUjxz7faiY2I9a5wu2dx45jzIRaNha5Jw,2190
+stated_confidence/levels.py,sha256=aJ4pNg_LpMnK21G0EXj9dLYEsfILuTKsT6MESslc0nA,4836
+stated_confidence/model.py,sha256=3qakjnU-QME8jQbIpgtMzahNGBlfAmqV4FuiGqvWPEQ,3498
+stated_confidence/presentation.py,sha256=71sYU6IN26bshWxsH6a6CAwozLLuZzPqsrI6sVlRenA,2291
+stated_confidence/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+stated_confidence/pydantic_ai.py,sha256=qRPUCPjZ88i6jFSkJBPRv1KDb1h9AwF9IbqdpOB6Pks,1507
+stated_confidence/taxonomy.py,sha256=ZjeI2lVq8VJo4g4ojc_9XmPq2Q6yjFh6fH5-4KrqFVA,1429
+stated_confidence/terms.py,sha256=t66DaqsSVJbZjwacQMGyDSR2aWaxNfLa7Jkd0vxvv-I,538
+stated_confidence-0.1.0.dist-info/METADATA,sha256=Kv-X6k1tOxtmYW3P5y6JA11Qs3ts_eO0WuPqmqkwdWQ,5360
+stated_confidence-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+stated_confidence-0.1.0.dist-info/licenses/LICENSE,sha256=MxI5z_i6TBS_xUk8SVZxCPi069K8Da586gjyrCWpNus,1070
+stated_confidence-0.1.0.dist-info/RECORD,,

stated_confidence-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

stated_confidence-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 John Lockwood
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.