agentloss 0.0.1__tar.gz

agentloss-0.0.1/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

agentloss-0.0.1/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: agentloss
+Version: 0.0.1
+Summary: Measure the real-world error rate and dollar cost of an AI agent's decisions. OpenTelemetry-native.
+License: Apache-2.0
+Project-URL: Homepage, https://agentloss.com
+Project-URL: Repository, https://github.com/ADMT-ai/agentloss
+Project-URL: Organization, https://admt.ai
+Keywords: ai-agents,agent-evals,production-evals,llm,reliability,error-rate,cost-of-errors,dollar-loss,ground-truth,outcomes,opentelemetry,openinference,agent-observability
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Monitoring
+Classifier: Development Status :: 3 - Alpha
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: claude
+Requires-Dist: anthropic>=0.40; extra == "claude"
+Provides-Extra: mcp
+Requires-Dist: mcp>=1.0; extra == "mcp"
+Dynamic: license-file
+
+# agentloss
+
+**Your eval tool tells you your AI agent's hallucination rate. `agentloss` tells you what it
+costs.** An OpenTelemetry-native SDK that measures the real-world **error rate and dollar
+loss** of an AI agent's decisions — by capturing its consequential actions in-process and
+joining them to ground truth (real resolved outcomes, not an offline labeled set).
+
+Every eval/observability tool scores *quality proxies* — LLM-judge, hallucination rate, task
+completion. `agentloss` answers the question the market keeps asking and no tool measures:
+*what are my agent's mistakes costing, and is it safe to trust with more autonomy?*
+
+> Part of **ADMT** (Automated Decision-Making Technology) — [admt.ai](https://admt.ai).
+
+## Install
+
+```bash
+pip install agentloss
+```
+
+## Quickstart
+
+Instrument only the **consequential action** — the tool call that moves money or commits the
+business — not every LLM call.
+
+```python
+from agentloss import decision, report_outcome, Decision
+
+@decision
+def approve_payment(invoice):
+    action = run_matching(invoice)            # "approve" | "hold" | "reject"
+    return Decision(action=action, value_at_risk_usd=invoice.total,
+                    business_key=invoice.number, use_case="ap_3way_match")
+
+# when the outcome resolves (correction, dispute, audit, human review):
+report_outcome(business_key="INV-1", ground_truth="duplicate-should-block",
+               source="recovery_audit", realized_loss_usd=14200)
+```
+
+It computes the error rate by segment (with confidence intervals), **realized + expected dollar
+loss**, and the agent's incremental risk vs. a baseline. Raw prompts/records stay in your
+boundary; only derived metrics leave.
+
+## How it works
+
+- **Instrument consequential actions, not the whole agent.** The costly events are the handful
+  of tool calls that move money or commit state.
+- **Ground truth arrives late, from outside the agent** — a correction, dispute, audit result,
+  or human review. Capture it via `report_outcome`, the human-review queue, and active sampling
+  + a verification agent. This is *real resolved outcomes*, not an offline dataset.
+- **Honest statistics.** Monetary-unit sampling with a target verifier budget; two-phase
+  calibration corrects a fallible verifier's bias back to truth (with confidence intervals).
+
+See [`docs/SDK-SPEC.md`](docs/SDK-SPEC.md) for the full API, `agentloss.*` semantic conventions,
+and the pack/adapter model.
+
+## Try the demo
+
+An oracle-validated harness that seeds an accounts-payable environment with *known* errors and
+checks that `agentloss` recovers the true error rate and dollar loss:
+
+```bash
+python -m dogfood.run                                  # deterministic mock, no deps
+AGENTLOSS_VERIFIER_LLM=claude ANTHROPIC_API_KEY=... python -m dogfood.run
+```
+
+## For AI coding agents
+
+`agentloss` is built to be discovered and wired by coding agents:
+[`llms.txt`](llms.txt), the [`instrument-agent-reliability`](skills/instrument-agent-reliability/SKILL.md)
+skill, the [`AGENTS.md`](AGENTS.md) rule, and an [MCP server](mcp/agentloss_mcp.py)
+(`how_to_instrument`, `explain_attribute`, `validate_integration`).
+
+## License
+
+Apache-2.0.

agentloss-0.0.1/README.md

@@ -0,0 +1,75 @@
+# agentloss
+
+**Your eval tool tells you your AI agent's hallucination rate. `agentloss` tells you what it
+costs.** An OpenTelemetry-native SDK that measures the real-world **error rate and dollar
+loss** of an AI agent's decisions — by capturing its consequential actions in-process and
+joining them to ground truth (real resolved outcomes, not an offline labeled set).
+
+Every eval/observability tool scores *quality proxies* — LLM-judge, hallucination rate, task
+completion. `agentloss` answers the question the market keeps asking and no tool measures:
+*what are my agent's mistakes costing, and is it safe to trust with more autonomy?*
+
+> Part of **ADMT** (Automated Decision-Making Technology) — [admt.ai](https://admt.ai).
+
+## Install
+
+```bash
+pip install agentloss
+```
+
+## Quickstart
+
+Instrument only the **consequential action** — the tool call that moves money or commits the
+business — not every LLM call.
+
+```python
+from agentloss import decision, report_outcome, Decision
+
+@decision
+def approve_payment(invoice):
+    action = run_matching(invoice)            # "approve" | "hold" | "reject"
+    return Decision(action=action, value_at_risk_usd=invoice.total,
+                    business_key=invoice.number, use_case="ap_3way_match")
+
+# when the outcome resolves (correction, dispute, audit, human review):
+report_outcome(business_key="INV-1", ground_truth="duplicate-should-block",
+               source="recovery_audit", realized_loss_usd=14200)
+```
+
+It computes the error rate by segment (with confidence intervals), **realized + expected dollar
+loss**, and the agent's incremental risk vs. a baseline. Raw prompts/records stay in your
+boundary; only derived metrics leave.
+
+## How it works
+
+- **Instrument consequential actions, not the whole agent.** The costly events are the handful
+  of tool calls that move money or commit state.
+- **Ground truth arrives late, from outside the agent** — a correction, dispute, audit result,
+  or human review. Capture it via `report_outcome`, the human-review queue, and active sampling
+  + a verification agent. This is *real resolved outcomes*, not an offline dataset.
+- **Honest statistics.** Monetary-unit sampling with a target verifier budget; two-phase
+  calibration corrects a fallible verifier's bias back to truth (with confidence intervals).
+
+See [`docs/SDK-SPEC.md`](docs/SDK-SPEC.md) for the full API, `agentloss.*` semantic conventions,
+and the pack/adapter model.
+
+## Try the demo
+
+An oracle-validated harness that seeds an accounts-payable environment with *known* errors and
+checks that `agentloss` recovers the true error rate and dollar loss:
+
+```bash
+python -m dogfood.run                                  # deterministic mock, no deps
+AGENTLOSS_VERIFIER_LLM=claude ANTHROPIC_API_KEY=... python -m dogfood.run
+```
+
+## For AI coding agents
+
+`agentloss` is built to be discovered and wired by coding agents:
+[`llms.txt`](llms.txt), the [`instrument-agent-reliability`](skills/instrument-agent-reliability/SKILL.md)
+skill, the [`AGENTS.md`](AGENTS.md) rule, and an [MCP server](mcp/agentloss_mcp.py)
+(`how_to_instrument`, `explain_attribute`, `validate_integration`).
+
+## License
+
+Apache-2.0.

agentloss-0.0.1/agentloss/__init__.py

@@ -0,0 +1,3 @@
+from .core import Decision, STORE, decision, report_outcome
+
+__all__ = ["Decision", "STORE", "decision", "report_outcome"]

agentloss-0.0.1/agentloss/calibration.py

@@ -0,0 +1,108 @@
+"""Calibration harness — makes a FALLIBLE verifier trustworthy.
+
+A real verification agent (e.g. Claude) has false alarms and misses, so its silver labels
+are biased. We spend a small GOLD budget (human review, or the oracle in the dogfood) to
+correct that bias with a two-phase, screen-and-confirm design:
+
+  Phase 1 — confirm EVERY flag. Verifier-positives are rare, so gold-label all of them.
+            This yields exact precision and the true-error weight among flags.
+  Phase 2 — spot-check a fraction q of the (many) verifier-negatives to estimate the
+            miss rate, and reweight the misses back up by 1/q.
+
+Corrected error total = (confirmed true flags) + (estimated missed errors), each
+Horvitz-Thompson-weighted by the original 1/pi. Losses are taken from GOLD (not the
+verifier's estimate), so dollars are corrected too. A bootstrap gives CIs that fold in
+both the PPS sampling and the calibration uncertainty.
+"""
+from .core import STORE
+
+
+def _pct(xs, p):
+    if not xs:
+        return 0.0
+    xs = sorted(xs)
+    i = min(len(xs) - 1, max(0, int(round(p * (len(xs) - 1)))))
+    return xs[i]
+
+
+def _resample_sum(contribs, rng):
+    n = len(contribs)
+    if n == 0:
+        return 0.0
+    return sum(contribs[rng.randrange(n)] for _ in range(n))
+
+
+def calibrate(gold_action, gold_loss, cfg, rng, B=400):
+    approved = [d for d in STORE.decisions.values() if d.action == "approve"]
+    N = len(approved)
+    q = cfg.cal_negative_sample_rate
+
+    # Sampled approved decisions split by label source:
+    #   * gold (audit-caught) — already truth, count directly (no new gold spent)
+    #   * silver (verifier)   — run the two-phase confirm/spot-check
+    gold_err_c, gold_loss_c = [], []
+    silver = []
+    for d in approved:
+        o = STORE.outcomes.get(d.business_key)
+        if not o or not o.sampled:
+            continue
+        if o.source == "verification_agent":
+            silver.append((d, o))
+        else:                                   # gold (recovery_audit): audit only labels errors
+            err = o.ground_truth != "approve"
+            gold_err_c.append((1.0 / o.pi) if err else 0.0)
+            gold_loss_c.append(((o.realized_loss_usd or 0.0) / o.pi) if err else 0.0)
+
+    positives = [(d, o) for d, o in silver if o.ground_truth != "approve"]
+    negatives = [(d, o) for d, o in silver if o.ground_truth == "approve"]
+
+    # Phase 1 — gold-confirm every flag
+    tp = fp = 0
+    pos_err_c, pos_loss_c = [], []
+    for d, o in positives:
+        if gold_action(d.business_key) != "approve":
+            tp += 1
+            pos_err_c.append(1.0 / o.pi)
+            pos_loss_c.append(gold_loss(d.business_key) / o.pi)
+        else:
+            fp += 1
+            pos_err_c.append(0.0)
+            pos_loss_c.append(0.0)
+
+    # Phase 2 — spot-check q of the approvals to catch misses
+    checked = miss = 0
+    neg_err_c, neg_loss_c = [], []
+    for d, o in negatives:
+        if rng.random() >= q:
+            continue
+        checked += 1
+        if gold_action(d.business_key) != "approve":
+            miss += 1
+            neg_err_c.append(1.0 / (o.pi * q))
+            neg_loss_c.append(gold_loss(d.business_key) / (o.pi * q))
+        else:
+            neg_err_c.append(0.0)
+            neg_loss_c.append(0.0)
+
+    corrected_rate = (sum(gold_err_c) + sum(pos_err_c) + sum(neg_err_c)) / N if N else 0.0
+    corrected_loss = sum(gold_loss_c) + sum(pos_loss_c) + sum(neg_loss_c)
+    precision = tp / (tp + fp) if (tp + fp) else float("nan")   # verifier's own precision
+    w_pos, w_miss = sum(pos_err_c), sum(neg_err_c)
+    recall = w_pos / (w_pos + w_miss) if (w_pos + w_miss) else float("nan")  # verifier's own recall
+
+    rates, losses = [], []
+    for _ in range(B):
+        e = _resample_sum(gold_err_c, rng) + _resample_sum(pos_err_c, rng) + _resample_sum(neg_err_c, rng)
+        l = _resample_sum(gold_loss_c, rng) + _resample_sum(pos_loss_c, rng) + _resample_sum(neg_loss_c, rng)
+        rates.append(e / N if N else 0.0)
+        losses.append(l)
+
+    return {
+        "corrected_rate": corrected_rate,
+        "rate_lo": _pct(rates, 0.025), "rate_hi": _pct(rates, 0.975),
+        "corrected_loss": corrected_loss,
+        "loss_lo": _pct(losses, 0.025), "loss_hi": _pct(losses, 0.975),
+        "precision": precision, "recall": recall,
+        "tp": tp, "fp": fp, "missed_in_sample": miss, "neg_checked": checked,
+        "gold_budget": len(positives) + checked,
+    }

agentloss-0.0.1/agentloss/core.py

@@ -0,0 +1,67 @@
+"""Core: decision capture + outcome store. Mirrors the `agentloss.*` decision/outcome
+shapes from docs/SDK-SPEC.md (here as plain objects instead of OTel spans)."""
+import itertools
+from dataclasses import dataclass, field
+
+_counter = itertools.count(1)
+
+
+@dataclass
+class Decision:
+    action: str                 # approve | hold | reject
+    value_at_risk_usd: float
+    business_key: str           # invoice_no — the join key for delayed outcomes
+    use_case: str = "ap_3way_match"
+    model: str = "mock"
+    in_envelope: bool = True
+    decision_id: str = ""
+
+
+@dataclass
+class Outcome:
+    ground_truth: str
+    source: str                 # human_queue | verification_agent | recovery_audit
+    fidelity: str               # gold | silver
+    confidence: float = 1.0
+    realized_loss_usd: float = None
+    recovery_usd: float = None
+    estimated_loss_usd: float = None
+    sampled: bool = False       # included in the random rate-estimation sample?
+    pi: float = None            # inclusion probability (for Horvitz-Thompson reweighting)
+
+
+class Store:
+    def __init__(self):
+        self.decisions = {}     # business_key -> Decision
+        self.outcomes = {}      # business_key -> Outcome
+
+    def record(self, d: Decision) -> Decision:
+        d.decision_id = f"d_{next(_counter)}"
+        self.decisions[d.business_key] = d
+        return d
+
+    def add_outcome(self, business_key, **kw):
+        self.outcomes[business_key] = Outcome(**kw)
+
+    def has_gold(self, business_key):
+        o = self.outcomes.get(business_key)
+        return o is not None and o.fidelity == "gold"
+
+
+STORE = Store()
+
+
+def decision(fn):
+    """Wrap a function that returns a Decision; record it. The SDK's `@decision`."""
+    def wrap(*a, **k):
+        return STORE.record(fn(*a, **k))
+    return wrap
+
+
+def report_outcome(business_key, ground_truth, source, fidelity="gold",
+                   confidence=1.0, realized_loss_usd=None, recovery_usd=None,
+                   estimated_loss_usd=None, sampled=False, pi=None):
+    STORE.add_outcome(business_key, ground_truth=ground_truth, source=source,
+                      fidelity=fidelity, confidence=confidence,
+                      realized_loss_usd=realized_loss_usd, recovery_usd=recovery_usd,
+                      estimated_loss_usd=estimated_loss_usd, sampled=sampled, pi=pi)

agentloss-0.0.1/agentloss/metrics.py

@@ -0,0 +1,70 @@
+"""Metrics: false-approve rate (Wilson CI + Horvitz-Thompson point estimate) and loss.
+
+We estimate the *false-approve* rate on the auto-approved population using the sampler's
+silver labels, reweighted by inclusion probability. Realized dollars come only from gold
+recovery-audit outcomes; expected dollars come from the reweighted silver estimates.
+"""
+from math import sqrt
+from .core import STORE
+
+
+def wilson(k, n, z=1.96):
+    if n == 0:
+        return 0.0, 0.0, 0.0
+    p = k / n
+    denom = 1 + z * z / n
+    center = (p + z * z / (2 * n)) / denom
+    half = z * sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
+    return p, max(0.0, center - half), min(1.0, center + half)
+
+
+def false_approve(cfg):
+    approved = [d for d in STORE.decisions.values() if d.action == "approve"]
+    N = len(approved)
+    k, n = 0, 0
+    ht_num = 0.0            # Horvitz-Thompson numerator: sum(err / pi)
+    exp_loss = 0.0          # reweighted expected loss
+    var_loss = 0.0         # HT variance of the loss total: sum (1-pi)/pi^2 * y^2
+    for d in approved:
+        o = STORE.outcomes.get(d.business_key)
+        if o is None or not o.sampled:          # only the random rate sample (gold OR silver)
+            continue
+        err = 1 if o.ground_truth != "approve" else 0
+        n += 1
+        k += err
+        ht_num += err / o.pi
+        if err:
+            loss = o.estimated_loss_usd if o.estimated_loss_usd is not None else o.realized_loss_usd
+            if loss:
+                exp_loss += loss / o.pi
+                var_loss += (1 - o.pi) / (o.pi * o.pi) * loss * loss   # take-all (pi=1) → 0
+    p_sample, lo, hi = wilson(k, n)
+    return {
+        "N_approved": N,
+        "n_sampled": n,
+        "k_errors": k,
+        "rate_sampled": p_sample,       # unweighted sample proportion
+        "rate_ht": ht_num / N if N else 0.0,   # importance-reweighted population estimate
+        "ci_lo": lo,
+        "ci_hi": hi,
+        "expected_loss_usd": exp_loss,
+        "expected_loss_se": sqrt(var_loss),
+    }
+
+
+def realized_loss():
+    total, recovered = 0.0, 0.0
+    for o in STORE.outcomes.values():
+        if o.source == "recovery_audit" and o.realized_loss_usd:
+            total += o.realized_loss_usd
+            recovered += o.recovery_usd or 0.0
+    return {"realized_loss_usd": total, "recovered_usd": recovered,
+            "net_realized_usd": total - recovered}
+
+
+def gt_resolvable_rate():
+    """Fraction of decisions with any reachable ground-truth source (the SDK's early-warning)."""
+    if not STORE.decisions:
+        return 0.0
+    resolved = sum(1 for k in STORE.decisions if k in STORE.outcomes)
+    return resolved / len(STORE.decisions)

agentloss-0.0.1/agentloss/sampler.py

@@ -0,0 +1,70 @@
+"""Active sampling + verification, with a target-n PPS budget.
+
+Inclusion probability is proportional to value-at-risk (the variance-minimizing choice for
+a loss total), scaled so the EXPECTED sample size equals `sample_target_n`, plus a floor so
+small-value items still get sampled for the rate. Big-ticket items saturate to pi=1 (a
+certainty stratum falls out naturally). Each decision's pi is recorded for Horvitz-Thompson
+reweighting in metrics.
+
+Decisions that already carry a GOLD label (audit/human) are kept, not skipped — skipping
+them would bias the rate down.
+"""
+from .core import STORE, report_outcome
+
+
+def inclusion_probs(decisions, cfg):
+    """pi_i = min(1, floor + beta * value_i), with beta chosen so sum(pi) == target_n."""
+    sizes = {d.business_key: max(d.value_at_risk_usd, 1.0) for d in decisions}
+    keys = list(sizes)
+    N = len(keys)
+    if N == 0:
+        return {}
+    target = min(cfg.sample_target_n, N)
+    # reserve at most half the budget for the floor so PPS has room to work
+    floor = min(cfg.sample_floor, 0.5 * target / N)
+
+    def total(beta):
+        return sum(min(1.0, floor + beta * sizes[k]) for k in keys)
+
+    lo, hi = 0.0, 1.0
+    while total(hi) < target and hi < 1e15:
+        hi *= 10.0
+    for _ in range(80):                       # bisection: total() is increasing in beta
+        mid = (lo + hi) / 2.0
+        if total(mid) < target:
+            lo = mid
+        else:
+            hi = mid
+    beta = (lo + hi) / 2.0
+    return {k: min(1.0, floor + beta * sizes[k]) for k in keys}
+
+
+def run(invoices_by_no, erp, cfg, rng, verify_fn):
+    """verify_fn(invoice, erp, cfg) -> {should_have_been, confidence, failed_check, estimated_loss}"""
+    probs = inclusion_probs(list(STORE.decisions.values()), cfg)
+    n_sampled = n_verified = 0
+    for key, d in list(STORE.decisions.items()):
+        pi = probs[key]
+        if rng.random() >= pi:
+            continue
+        n_sampled += 1
+        existing = STORE.outcomes.get(key)
+        if existing is not None:                 # keep the gold label; just mark it sampled
+            existing.sampled = True
+            existing.pi = pi
+            continue
+        v = verify_fn(invoices_by_no[key], erp, cfg)
+        n_verified += 1
+        if n_verified % 25 == 0:
+            print(f"[verify] {n_verified} verified", flush=True)
+        report_outcome(
+            key,
+            ground_truth=v["should_have_been"],
+            source="verification_agent",
+            fidelity="silver",
+            confidence=v["confidence"],
+            estimated_loss_usd=v["estimated_loss"],
+            sampled=True,
+            pi=pi,
+        )
+    return n_sampled

agentloss-0.0.1/agentloss/verifier.py

@@ -0,0 +1,68 @@
+"""Verification agent: gathers THOROUGH evidence from the ERP verifier surface, then asks
+the LLM to re-adjudicate. This is the engine behind Tier-A ground truth — legitimate
+because it uses more data/time than the production agent did."""
+
+
+def gather_evidence(inv, erp, cfg):
+    vr = erp.vendor_risk(inv["vendor_id"])
+    fuzzy, overlap = erp.find_fuzzy_duplicate(inv)
+    fuzzy_full = fuzzy is not None and overlap >= 0.99
+    fuzzy_partial = fuzzy is not None and 0.3 <= overlap < 0.99
+
+    contract_overbill, overbill_amt = False, 0.0
+    for l in inv["lines"]:
+        cp = erp.contract_price(inv["vendor_id"], l["item"])
+        if cp and l["unit_price"] > cp * (1 + cfg.price_tolerance):
+            contract_overbill = True
+            overbill_amt += (l["unit_price"] - cp) * l["qty"]
+
+    qty_over, qty_amt = False, 0.0
+    po = erp.get_po(inv["po_no"])
+    if po:
+        ordered = {l["item"]: l["qty"] for l in po["lines"]}
+        price = {l["item"]: l["unit_price"] for l in po["lines"]}
+        for l in inv["lines"]:
+            if l["qty"] > ordered.get(l["item"], 10**12):
+                qty_over = True
+                qty_amt += (l["qty"] - ordered.get(l["item"], 0)) * price.get(l["item"], l["unit_price"])
+
+    return {
+        "vr": vr,
+        "bank_changed": inv.get("bank_changed", False),
+        "fuzzy_full": fuzzy_full,
+        "fuzzy_partial": fuzzy_partial,
+        "contract_overbill": contract_overbill,
+        "overbill_amt": round(overbill_amt, 2),
+        "qty_over": qty_over,
+        "qty_amt": round(qty_amt, 2),
+        "amount": inv["amount"],
+    }
+
+
+def make_verifier(llm):
+    def verify(inv, erp, cfg):
+        return llm.verify(gather_evidence(inv, erp, cfg))
+    return verify
+
+
+def make_fallible(base_verify, cfg):
+    """Perturb a verifier to simulate real-world fallibility (false alarms / misses).
+    A no-op when both knobs are 0 (e.g. real Claude, whose errors are already inherent)."""
+    from random import Random
+    fp, fn = cfg.verifier_fp_rate, cfg.verifier_fn_rate
+    if fp <= 0 and fn <= 0:
+        return base_verify
+
+    def verify(inv, erp, cfg_):
+        v = base_verify(inv, erp, cfg_)
+        r = Random(f"{cfg.seed}:{inv['invoice_no']}")    # deterministic per decision
+        if v["should_have_been"] == "approve":
+            if r.random() < fp:                          # false alarm
+                return {"should_have_been": "hold", "confidence": 0.5,
+                        "failed_check": "spurious", "estimated_loss": round(inv["amount"] * 0.5, 2)}
+        else:
+            if r.random() < fn:                          # miss a real error
+                return {"should_have_been": "approve", "confidence": 0.5,
+                        "failed_check": None, "estimated_loss": 0.0}
+        return v
+    return verify

agentloss-0.0.1/agentloss.egg-info/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: agentloss
+Version: 0.0.1
+Summary: Measure the real-world error rate and dollar cost of an AI agent's decisions. OpenTelemetry-native.
+License: Apache-2.0
+Project-URL: Homepage, https://agentloss.com
+Project-URL: Repository, https://github.com/ADMT-ai/agentloss
+Project-URL: Organization, https://admt.ai
+Keywords: ai-agents,agent-evals,production-evals,llm,reliability,error-rate,cost-of-errors,dollar-loss,ground-truth,outcomes,opentelemetry,openinference,agent-observability
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Monitoring
+Classifier: Development Status :: 3 - Alpha
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: claude
+Requires-Dist: anthropic>=0.40; extra == "claude"
+Provides-Extra: mcp
+Requires-Dist: mcp>=1.0; extra == "mcp"
+Dynamic: license-file
+
+# agentloss
+
+**Your eval tool tells you your AI agent's hallucination rate. `agentloss` tells you what it
+costs.** An OpenTelemetry-native SDK that measures the real-world **error rate and dollar
+loss** of an AI agent's decisions — by capturing its consequential actions in-process and
+joining them to ground truth (real resolved outcomes, not an offline labeled set).
+
+Every eval/observability tool scores *quality proxies* — LLM-judge, hallucination rate, task
+completion. `agentloss` answers the question the market keeps asking and no tool measures:
+*what are my agent's mistakes costing, and is it safe to trust with more autonomy?*
+
+> Part of **ADMT** (Automated Decision-Making Technology) — [admt.ai](https://admt.ai).
+
+## Install
+
+```bash
+pip install agentloss
+```
+
+## Quickstart
+
+Instrument only the **consequential action** — the tool call that moves money or commits the
+business — not every LLM call.
+
+```python
+from agentloss import decision, report_outcome, Decision
+
+@decision
+def approve_payment(invoice):
+    action = run_matching(invoice)            # "approve" | "hold" | "reject"
+    return Decision(action=action, value_at_risk_usd=invoice.total,
+                    business_key=invoice.number, use_case="ap_3way_match")
+
+# when the outcome resolves (correction, dispute, audit, human review):
+report_outcome(business_key="INV-1", ground_truth="duplicate-should-block",
+               source="recovery_audit", realized_loss_usd=14200)
+```
+
+It computes the error rate by segment (with confidence intervals), **realized + expected dollar
+loss**, and the agent's incremental risk vs. a baseline. Raw prompts/records stay in your
+boundary; only derived metrics leave.
+
+## How it works
+
+- **Instrument consequential actions, not the whole agent.** The costly events are the handful
+  of tool calls that move money or commit state.
+- **Ground truth arrives late, from outside the agent** — a correction, dispute, audit result,
+  or human review. Capture it via `report_outcome`, the human-review queue, and active sampling
+  + a verification agent. This is *real resolved outcomes*, not an offline dataset.
+- **Honest statistics.** Monetary-unit sampling with a target verifier budget; two-phase
+  calibration corrects a fallible verifier's bias back to truth (with confidence intervals).
+
+See [`docs/SDK-SPEC.md`](docs/SDK-SPEC.md) for the full API, `agentloss.*` semantic conventions,
+and the pack/adapter model.
+
+## Try the demo
+
+An oracle-validated harness that seeds an accounts-payable environment with *known* errors and
+checks that `agentloss` recovers the true error rate and dollar loss:
+
+```bash
+python -m dogfood.run                                  # deterministic mock, no deps
+AGENTLOSS_VERIFIER_LLM=claude ANTHROPIC_API_KEY=... python -m dogfood.run
+```
+
+## For AI coding agents
+
+`agentloss` is built to be discovered and wired by coding agents:
+[`llms.txt`](llms.txt), the [`instrument-agent-reliability`](skills/instrument-agent-reliability/SKILL.md)
+skill, the [`AGENTS.md`](AGENTS.md) rule, and an [MCP server](mcp/agentloss_mcp.py)
+(`how_to_instrument`, `explain_attribute`, `validate_integration`).
+
+## License
+
+Apache-2.0.

agentloss-0.0.1/agentloss.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+LICENSE
+README.md
+pyproject.toml
+agentloss/__init__.py
+agentloss/calibration.py
+agentloss/core.py
+agentloss/metrics.py
+agentloss/sampler.py
+agentloss/verifier.py
+agentloss.egg-info/PKG-INFO
+agentloss.egg-info/SOURCES.txt
+agentloss.egg-info/dependency_links.txt
+agentloss.egg-info/requires.txt
+agentloss.egg-info/top_level.txt

agentloss-0.0.1/agentloss.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

agentloss-0.0.1/agentloss.egg-info/requires.txt

@@ -0,0 +1,6 @@
+
+[claude]
+anthropic>=0.40
+
+[mcp]
+mcp>=1.0

agentloss-0.0.1/agentloss.egg-info/top_level.txt

@@ -0,0 +1 @@
+agentloss

agentloss-0.0.1/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "agentloss"
+version = "0.0.1"
+description = "Measure the real-world error rate and dollar cost of an AI agent's decisions. OpenTelemetry-native."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "Apache-2.0" }
+keywords = [
+    "ai-agents", "agent-evals", "production-evals", "llm", "reliability",
+    "error-rate", "cost-of-errors", "dollar-loss", "ground-truth", "outcomes",
+    "opentelemetry", "openinference", "agent-observability",
+]
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: System :: Monitoring",
+    "Development Status :: 3 - Alpha",
+]
+dependencies = []
+
+[project.optional-dependencies]
+claude = ["anthropic>=0.40"]
+mcp = ["mcp>=1.0"]
+
+[project.urls]
+Homepage = "https://agentloss.com"
+Repository = "https://github.com/ADMT-ai/agentloss"
+Organization = "https://admt.ai"
+
+[tool.setuptools]
+packages = ["agentloss"]

agentloss-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+