agentfox 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentfox/__init__.py +45 -0
- agentfox/agent_loop.py +222 -0
- agentfox/agent_messaging.py +90 -0
- agentfox/answerability.py +605 -0
- agentfox/arbitration.py +309 -0
- agentfox/attribution.py +538 -0
- agentfox/audit/__init__.py +47 -0
- agentfox/audit/chain.py +420 -0
- agentfox/audit/evidence.py +560 -0
- agentfox/audit/otel.py +160 -0
- agentfox/audit/siem.py +233 -0
- agentfox/audit/trace.py +337 -0
- agentfox/autoguard.py +1261 -0
- agentfox/availability.py +702 -0
- agentfox/business/__init__.py +43 -0
- agentfox/business/catalogue.py +583 -0
- agentfox/business/compile.py +1010 -0
- agentfox/business/graph.py +314 -0
- agentfox/business/ladder.py +396 -0
- agentfox/business/store.py +159 -0
- agentfox/cli/__init__.py +5 -0
- agentfox/cli/_style.py +40 -0
- agentfox/cli/auth_cli.py +222 -0
- agentfox/cli/business_cli.py +403 -0
- agentfox/cli/capability_cli.py +475 -0
- agentfox/cli/controls_cli.py +516 -0
- agentfox/cli/demo.py +662 -0
- agentfox/cli/main.py +2065 -0
- agentfox/cli/mcp_cli.py +39 -0
- agentfox/cli/onboarding.py +682 -0
- agentfox/cli/quickscan.py +187 -0
- agentfox/cli/submit.py +120 -0
- agentfox/commitments.py +496 -0
- agentfox/compliance/__init__.py +43 -0
- agentfox/compliance/catalog.py +250 -0
- agentfox/compliance/risk.py +380 -0
- agentfox/compliance/status.py +524 -0
- agentfox/compliance_data/controls.yaml +944 -0
- agentfox/compliance_data/obligations.yaml +91 -0
- agentfox/config.py +502 -0
- agentfox/context_integrity.py +609 -0
- agentfox/control_flow.py +164 -0
- agentfox/crypto.py +45 -0
- agentfox/data_access.py +384 -0
- agentfox/db.py +233 -0
- agentfox/discovery.py +847 -0
- agentfox/discovery_openapi.py +113 -0
- agentfox/effects.py +461 -0
- agentfox/enforcement.py +3161 -0
- agentfox/entitlement.py +464 -0
- agentfox/escalation.py +826 -0
- agentfox/evaluation/__init__.py +98 -0
- agentfox/evaluation/adapters.py +189 -0
- agentfox/evaluation/adaptive.py +1098 -0
- agentfox/evaluation/drift.py +303 -0
- agentfox/evaluation/gating.py +309 -0
- agentfox/evaluation/model_groundedness.py +127 -0
- agentfox/evaluation/ragas_adapter.py +241 -0
- agentfox/evaluation/redteam.py +1280 -0
- agentfox/evaluation/runner.py +366 -0
- agentfox/evaluation/scorers.py +350 -0
- agentfox/evaluation/silent_failure.py +438 -0
- agentfox/finding.py +40 -0
- agentfox/findings.py +377 -0
- agentfox/gateway/__init__.py +5 -0
- agentfox/gateway/app.py +555 -0
- agentfox/gateway/auth.py +307 -0
- agentfox/gateway/deps.py +126 -0
- agentfox/gateway/playground_sessions.py +414 -0
- agentfox/gateway/routes/__init__.py +1 -0
- agentfox/gateway/routes/answerability.py +165 -0
- agentfox/gateway/routes/discovery.py +99 -0
- agentfox/gateway/routes/entitlement.py +213 -0
- agentfox/gateway/routes/escalation.py +326 -0
- agentfox/gateway/routes/evaluation.py +798 -0
- agentfox/gateway/routes/governance.py +675 -0
- agentfox/gateway/routes/inline.py +964 -0
- agentfox/gateway/routes/integrations.py +608 -0
- agentfox/gateway/routes/jobs.py +145 -0
- agentfox/gateway/routes/memory.py +101 -0
- agentfox/gateway/routes/messaging.py +82 -0
- agentfox/gateway/routes/onboarding.py +293 -0
- agentfox/gateway/routes/playground.py +272 -0
- agentfox/gateway/routes/playground_deps.py +34 -0
- agentfox/gateway/routes/policy.py +512 -0
- agentfox/gateway/routes/proposals.py +158 -0
- agentfox/gateway/routes/provenance.py +345 -0
- agentfox/gateway/routes/registry.py +1034 -0
- agentfox/gateway/routes/tuning.py +263 -0
- agentfox/gateway/routes/waitlist.py +183 -0
- agentfox/gateway/verdicts.py +70 -0
- agentfox/guardrails/__init__.py +107 -0
- agentfox/guardrails/actions.py +865 -0
- agentfox/guardrails/adapters/__init__.py +0 -0
- agentfox/guardrails/adapters/classifiers.py +362 -0
- agentfox/guardrails/adapters/embeddings.py +158 -0
- agentfox/guardrails/adapters/hub.py +356 -0
- agentfox/guardrails/adapters/presidio.py +121 -0
- agentfox/guardrails/adapters/rails.py +119 -0
- agentfox/guardrails/base.py +258 -0
- agentfox/guardrails/composition.py +150 -0
- agentfox/guardrails/data/injection_corpus.json +239 -0
- agentfox/guardrails/detectors/__init__.py +0 -0
- agentfox/guardrails/detectors/injection.py +627 -0
- agentfox/guardrails/detectors/pii.py +162 -0
- agentfox/guardrails/detectors/safety.py +96 -0
- agentfox/guardrails/detectors/schema.py +145 -0
- agentfox/guardrails/detectors/secrets.py +128 -0
- agentfox/guardrails/normalize.py +756 -0
- agentfox/guardrails/pipeline.py +262 -0
- agentfox/guardrails/taint.py +183 -0
- agentfox/guardrails/tuning.py +907 -0
- agentfox/identity/__init__.py +41 -0
- agentfox/identity/service.py +620 -0
- agentfox/ids.py +57 -0
- agentfox/improvement/__init__.py +7 -0
- agentfox/improvement/appliers.py +445 -0
- agentfox/improvement/contract.py +121 -0
- agentfox/improvement/loops.py +235 -0
- agentfox/improvement/proposals.py +781 -0
- agentfox/integrations/__init__.py +39 -0
- agentfox/integrations/correlation.py +360 -0
- agentfox/integrations/fastapi.py +235 -0
- agentfox/integrations/langgraph.py +393 -0
- agentfox/integrations/mcp.py +440 -0
- agentfox/integrations/prometheus.py +226 -0
- agentfox/integrity.py +1024 -0
- agentfox/job_handlers.py +278 -0
- agentfox/jobs.py +178 -0
- agentfox/jobs_db.py +312 -0
- agentfox/mcp_server.py +863 -0
- agentfox/models.py +1802 -0
- agentfox/operator_log.py +235 -0
- agentfox/policies_data/baseline.yaml +165 -0
- agentfox/policies_data/eu-ai-act-high-risk.yaml +113 -0
- agentfox/policies_data/tool-containment.yaml +197 -0
- agentfox/policy/__init__.py +101 -0
- agentfox/policy/canary.py +401 -0
- agentfox/policy/engine.py +315 -0
- agentfox/policy/hierarchy.py +354 -0
- agentfox/policy/model.py +262 -0
- agentfox/policy/opa.py +248 -0
- agentfox/policy/simulate.py +185 -0
- agentfox/policy/store.py +287 -0
- agentfox/provenance.py +560 -0
- agentfox/providers/__init__.py +43 -0
- agentfox/providers/base.py +192 -0
- agentfox/providers/echo.py +146 -0
- agentfox/providers/enterprise.py +366 -0
- agentfox/providers/remote.py +299 -0
- agentfox/register.py +281 -0
- agentfox/registry/__init__.py +43 -0
- agentfox/registry/control.py +161 -0
- agentfox/registry/service.py +838 -0
- agentfox/reliability.py +343 -0
- agentfox/scheduler.py +224 -0
- agentfox/sdk/__init__.py +481 -0
- agentfox/seed.py +660 -0
- agentfox/session_scan.py +124 -0
- agentfox/sycophancy.py +181 -0
- agentfox/system_log.py +132 -0
- agentfox/tenancy.py +298 -0
- agentfox/tool_contract.py +264 -0
- agentfox/trajectory.py +457 -0
- agentfox/webhooks.py +390 -0
- agentfox-0.3.1.dist-info/METADATA +542 -0
- agentfox-0.3.1.dist-info/RECORD +171 -0
- agentfox-0.3.1.dist-info/WHEEL +4 -0
- agentfox-0.3.1.dist-info/entry_points.txt +3 -0
- agentfox-0.3.1.dist-info/licenses/LICENSE +202 -0
- nometria/__init__.py +62 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""P4-3 — model-based groundedness, alongside the lexical scorer.
|
|
2
|
+
|
|
3
|
+
`silent_failure.py::groundedness` is deliberately lexical — numeric agreement plus
|
|
4
|
+
token overlap, cheap enough to run inline. Its own docstring names exactly what that
|
|
5
|
+
buys and what it costs: "this is grounding, not entailment: it will not catch a
|
|
6
|
+
fabricated claim phrased entirely in the context's own words and numbers." A claim
|
|
7
|
+
that reuses the context's vocabulary while changing what it asserts scores as fully
|
|
8
|
+
supported, because token overlap cannot see the substitution.
|
|
9
|
+
|
|
10
|
+
This is the second opinion that gap was always going to need, and it is built the
|
|
11
|
+
same way every other model call in this codebase already is: through
|
|
12
|
+
`ModelProvider.judge()` — the exact mechanism `evaluation/scorers.py::LlmJudgeScorer`
|
|
13
|
+
already uses for rubric-based judging, not a new prompt-and-parse scheme invented for
|
|
14
|
+
this one scorer. The rubric is the retrieved context plus an instruction to judge
|
|
15
|
+
strictly against it; everything about *how* that gets scored (provider selection,
|
|
16
|
+
model pinning for reproducibility, the deterministic offline fallback) is
|
|
17
|
+
`LlmJudgeScorer`'s own established contract, reused rather than duplicated.
|
|
18
|
+
|
|
19
|
+
Offline, the pinned judge is `echo:deterministic`, same as `LlmJudgeScorer` — honest
|
|
20
|
+
about what that is (rubric-keyword coverage, `providers/echo.py::judge`'s own words:
|
|
21
|
+
"not a substitute for a model judge"), not a claim of model-level judgment. A team
|
|
22
|
+
that wants the real thing configures a real provider, exactly as they already do for
|
|
23
|
+
`llm_judge` and for Ragas's model-based path in `ragas_adapter.py`.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import logging
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
from ..providers import get_provider
|
|
32
|
+
from .scorers import BaseScorer, ScoreContext, ScoreResult, register_scorer
|
|
33
|
+
from .silent_failure import groundedness
|
|
34
|
+
|
|
35
|
+
log = logging.getLogger(__name__)
|
|
36
|
+
|
|
37
|
+
_RUBRIC_TEMPLATE = (
|
|
38
|
+
"Score, from 0.0 to 1.0, how well the output is supported by the reference "
|
|
39
|
+
"context below. 1.0 means every factual claim, number and named entity in the "
|
|
40
|
+
"output is verifiable against the context; 0.0 means the output introduces "
|
|
41
|
+
"facts the context does not state. Judge strictly against the context given — "
|
|
42
|
+
"outside knowledge, even if true, does not count as support.\n\n"
|
|
43
|
+
"Reference context:\n{context}"
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _context_text(ctx: ScoreContext) -> str:
|
|
48
|
+
"""Retrieved context for grounding, from wherever the harness put it — the same
|
|
49
|
+
keys `silent_failure.py`'s lexical scorer reads, reimplemented locally rather
|
|
50
|
+
than importing that module's private helper (the same choice
|
|
51
|
+
`ragas_adapter.py::_context_list` already made for the identical question)."""
|
|
52
|
+
for key in ("retrieved", "context", "documents", "sources"):
|
|
53
|
+
value = ctx.context.get(key)
|
|
54
|
+
if isinstance(value, str) and value.strip():
|
|
55
|
+
return value
|
|
56
|
+
if isinstance(value, list):
|
|
57
|
+
parts = [
|
|
58
|
+
v if isinstance(v, str) else str(v.get("text") or v.get("content") or "")
|
|
59
|
+
for v in value
|
|
60
|
+
]
|
|
61
|
+
joined = "\n".join(p for p in parts if p)
|
|
62
|
+
if joined.strip():
|
|
63
|
+
return joined
|
|
64
|
+
return ""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def model_groundedness(
|
|
68
|
+
output: str, context_text: str, *, judge_model: str = "echo:deterministic"
|
|
69
|
+
) -> dict[str, Any]:
|
|
70
|
+
"""The judge verdict, with a lexical fallback if the call itself fails.
|
|
71
|
+
|
|
72
|
+
A scoring signal that can crash an eval run is worse than a weaker one, so a
|
|
73
|
+
provider error falls back to the lexical scorer rather than propagating —
|
|
74
|
+
`implementation`/`judge_model` on the result says which one actually ran, the
|
|
75
|
+
same disclosure `ragas_adapter.py` makes when its own delegation fails.
|
|
76
|
+
"""
|
|
77
|
+
if not context_text.strip():
|
|
78
|
+
# Nothing to be grounded against, and nothing a judge could check either —
|
|
79
|
+
# the lexical scorer's own reading of an empty context ("not a failure").
|
|
80
|
+
return {
|
|
81
|
+
"score": 1.0,
|
|
82
|
+
"rationale": "no retrieved context; nothing to judge",
|
|
83
|
+
"judge_model": judge_model,
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
provider_key, _, model = judge_model.partition(":")
|
|
87
|
+
try:
|
|
88
|
+
provider = get_provider(provider_key)
|
|
89
|
+
verdict = provider.judge(
|
|
90
|
+
output=output, rubric=_RUBRIC_TEMPLATE.format(context=context_text[:6000]),
|
|
91
|
+
model=model or "default",
|
|
92
|
+
)
|
|
93
|
+
return {
|
|
94
|
+
"score": float(verdict["score"]),
|
|
95
|
+
"rationale": verdict.get("rationale", ""),
|
|
96
|
+
"judge_model": judge_model,
|
|
97
|
+
}
|
|
98
|
+
except Exception as exc: # a judge call must not be able to fail the eval run
|
|
99
|
+
log.warning("model groundedness judge call failed, falling back to lexical: %s", exc)
|
|
100
|
+
report = groundedness(output, context_text)
|
|
101
|
+
return {
|
|
102
|
+
"score": report.score,
|
|
103
|
+
"rationale": f"judge call failed ({exc}); fell back to lexical scoring",
|
|
104
|
+
"judge_model": "native-lexical",
|
|
105
|
+
"unsupported_claims": report.unsupported[:10],
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class ModelGroundednessScorer(BaseScorer):
|
|
110
|
+
"""Selectable alongside `groundedness` (`silent_failure.py`), not a replacement
|
|
111
|
+
for it — an eval suite can run both and see where they disagree, which is itself
|
|
112
|
+
a signal: the lexical scorer full-marking something the judge flags is exactly
|
|
113
|
+
the paraphrased-fabrication case neither one alone is enough to catch."""
|
|
114
|
+
|
|
115
|
+
key, kind, threshold = "model_groundedness", "groundedness", 0.7
|
|
116
|
+
|
|
117
|
+
def __init__(self, judge_model: str = "echo:deterministic") -> None:
|
|
118
|
+
self.judge_model = judge_model
|
|
119
|
+
|
|
120
|
+
def score(self, output: str, ctx: ScoreContext) -> ScoreResult:
|
|
121
|
+
result = model_groundedness(output, _context_text(ctx), judge_model=self.judge_model)
|
|
122
|
+
return self._result(result["score"], **{k: v for k, v in result.items() if k != "score"})
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
register_scorer(ModelGroundednessScorer())
|
|
126
|
+
|
|
127
|
+
__all__ = ["model_groundedness", "ModelGroundednessScorer"]
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
"""I-8 / P4-9 — Ragas scorer adapter. 3 of 11 use it for retrieval evaluation.
|
|
2
|
+
|
|
3
|
+
This is a vocabulary adapter, not a capability import. Teams evaluating RAG describe
|
|
4
|
+
quality in Ragas terms — faithfulness, answer relevancy, context precision, context
|
|
5
|
+
recall — and asking them to re-learn our scorer names to adopt governance is friction
|
|
6
|
+
for no benefit. The names are theirs; the numbers are computed here, so an offline
|
|
7
|
+
install still produces them.
|
|
8
|
+
|
|
9
|
+
Where Ragas itself is installed and egress is allowed, `run_ragas` delegates to it:
|
|
10
|
+
their model-based implementations are better than our lexical ones, and pretending
|
|
11
|
+
otherwise would be the same overstatement the earlier PRD drafts made about
|
|
12
|
+
groundedness. The native path exists so the metrics are *available* offline, not so
|
|
13
|
+
we can claim parity — `implementation` is reported on every result for exactly that
|
|
14
|
+
reason.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import logging
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from .scorers import BaseScorer, ScoreContext, ScoreResult, content_tokens, register_scorer, sentences
|
|
24
|
+
|
|
25
|
+
log = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
#: Their names, in their order. A team reading a AgentFox report should recognise it.
|
|
28
|
+
RAGAS_METRICS = ("faithfulness", "answer_relevancy", "context_precision", "context_recall")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def ragas_available() -> bool:
|
|
32
|
+
try: # pragma: no cover - exercised only where ragas is installed
|
|
33
|
+
import ragas # noqa: F401
|
|
34
|
+
|
|
35
|
+
return True
|
|
36
|
+
except Exception:
|
|
37
|
+
return False
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class RagasSample:
|
|
42
|
+
question: str
|
|
43
|
+
answer: str
|
|
44
|
+
contexts: list[str] = field(default_factory=list)
|
|
45
|
+
ground_truth: str = ""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class RagasScores:
|
|
50
|
+
faithfulness: float = 0.0
|
|
51
|
+
answer_relevancy: float = 0.0
|
|
52
|
+
context_precision: float = 0.0
|
|
53
|
+
context_recall: float = 0.0
|
|
54
|
+
implementation: str = "native-lexical"
|
|
55
|
+
|
|
56
|
+
def to_json(self) -> dict[str, Any]:
|
|
57
|
+
return {
|
|
58
|
+
"faithfulness": round(self.faithfulness, 4),
|
|
59
|
+
"answer_relevancy": round(self.answer_relevancy, 4),
|
|
60
|
+
"context_precision": round(self.context_precision, 4),
|
|
61
|
+
"context_recall": round(self.context_recall, 4),
|
|
62
|
+
# Named on every result: our lexical implementation is weaker than theirs
|
|
63
|
+
# and a consumer of these numbers is entitled to know which produced them.
|
|
64
|
+
"implementation": self.implementation,
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _overlap(claim: str, corpus: str) -> float:
|
|
69
|
+
claim_tokens = content_tokens(claim)
|
|
70
|
+
if not claim_tokens:
|
|
71
|
+
return 1.0
|
|
72
|
+
corpus_tokens = content_tokens(corpus)
|
|
73
|
+
return len(claim_tokens & corpus_tokens) / len(claim_tokens)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def native_scores(sample: RagasSample) -> RagasScores:
|
|
77
|
+
"""Lexical approximations, computed offline.
|
|
78
|
+
|
|
79
|
+
Deliberately the same mechanics as our groundedness scorer, so a team comparing
|
|
80
|
+
the two does not get two different numbers for the same underlying judgement.
|
|
81
|
+
"""
|
|
82
|
+
context = " ".join(sample.contexts)
|
|
83
|
+
|
|
84
|
+
supported = [s for s in sentences(sample.answer) if _overlap(s, context) >= 0.6]
|
|
85
|
+
total = max(1, len(sentences(sample.answer)))
|
|
86
|
+
faithfulness = len(supported) / total if sample.contexts else 0.0
|
|
87
|
+
|
|
88
|
+
answer_relevancy = _overlap(sample.question, sample.answer) if sample.question else 0.0
|
|
89
|
+
|
|
90
|
+
useful = [c for c in sample.contexts if _overlap(sample.answer, c) >= 0.2]
|
|
91
|
+
context_precision = len(useful) / len(sample.contexts) if sample.contexts else 0.0
|
|
92
|
+
|
|
93
|
+
context_recall = (
|
|
94
|
+
_overlap(sample.ground_truth, context) if sample.ground_truth and sample.contexts else 0.0
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
return RagasScores(
|
|
98
|
+
faithfulness=faithfulness,
|
|
99
|
+
answer_relevancy=answer_relevancy,
|
|
100
|
+
context_precision=context_precision,
|
|
101
|
+
context_recall=context_recall,
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def score_sample(sample: RagasSample, *, prefer_ragas: bool = True) -> RagasScores:
|
|
106
|
+
"""One sample, scored in Ragas vocabulary.
|
|
107
|
+
|
|
108
|
+
Delegates to Ragas when it is installed, because their model-based metrics are
|
|
109
|
+
better than our lexical ones. The native path keeps the vocabulary available
|
|
110
|
+
offline rather than claiming equivalence.
|
|
111
|
+
"""
|
|
112
|
+
if prefer_ragas and ragas_available(): # pragma: no cover - needs ragas + a model
|
|
113
|
+
try:
|
|
114
|
+
return _delegate(sample)
|
|
115
|
+
except Exception as exc:
|
|
116
|
+
log.warning("ragas delegation failed, falling back to native: %s", exc)
|
|
117
|
+
return native_scores(sample)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _delegate(sample: RagasSample) -> RagasScores: # pragma: no cover - needs ragas
|
|
121
|
+
from datasets import Dataset
|
|
122
|
+
from ragas import evaluate
|
|
123
|
+
from ragas.metrics import (
|
|
124
|
+
answer_relevancy,
|
|
125
|
+
context_precision,
|
|
126
|
+
context_recall,
|
|
127
|
+
faithfulness,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
dataset = Dataset.from_dict(
|
|
131
|
+
{
|
|
132
|
+
"question": [sample.question],
|
|
133
|
+
"answer": [sample.answer],
|
|
134
|
+
"contexts": [sample.contexts],
|
|
135
|
+
"ground_truth": [sample.ground_truth],
|
|
136
|
+
}
|
|
137
|
+
)
|
|
138
|
+
result = evaluate(
|
|
139
|
+
dataset, metrics=[faithfulness, answer_relevancy, context_precision, context_recall]
|
|
140
|
+
)
|
|
141
|
+
scores = result.to_pandas().iloc[0]
|
|
142
|
+
return RagasScores(
|
|
143
|
+
faithfulness=float(scores.get("faithfulness", 0.0)),
|
|
144
|
+
answer_relevancy=float(scores.get("answer_relevancy", 0.0)),
|
|
145
|
+
context_precision=float(scores.get("context_precision", 0.0)),
|
|
146
|
+
context_recall=float(scores.get("context_recall", 0.0)),
|
|
147
|
+
implementation="ragas",
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def score_dataset(samples: list[RagasSample], *, prefer_ragas: bool = True) -> dict[str, Any]:
|
|
152
|
+
"""Aggregate over a dataset, reporting the per-sample floor alongside the mean.
|
|
153
|
+
|
|
154
|
+
The mean is what Ragas reports and what teams expect. The minimum is what matters
|
|
155
|
+
for governance: one unfaithful answer in a hundred is not an acceptable average,
|
|
156
|
+
it is an incident waiting for a user to find it.
|
|
157
|
+
"""
|
|
158
|
+
if not samples:
|
|
159
|
+
return {"samples": 0, "metrics": {}, "implementation": "none"}
|
|
160
|
+
scored = [score_sample(s, prefer_ragas=prefer_ragas) for s in samples]
|
|
161
|
+
metrics: dict[str, Any] = {}
|
|
162
|
+
for name in RAGAS_METRICS:
|
|
163
|
+
values = [getattr(s, name) for s in scored]
|
|
164
|
+
metrics[name] = {
|
|
165
|
+
"mean": round(sum(values) / len(values), 4),
|
|
166
|
+
"min": round(min(values), 4),
|
|
167
|
+
"below_0_7": sum(1 for v in values if v < 0.7),
|
|
168
|
+
}
|
|
169
|
+
return {
|
|
170
|
+
"samples": len(samples),
|
|
171
|
+
"metrics": metrics,
|
|
172
|
+
"implementation": scored[0].implementation,
|
|
173
|
+
"per_sample": [s.to_json() for s in scored],
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
# ---------------------------------------------------------------------------
|
|
178
|
+
# P4-9 wiring — Ragas metrics as selectable scorers (not just a standalone report)
|
|
179
|
+
# ---------------------------------------------------------------------------
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _context_list(ctx: ScoreContext) -> list[str]:
|
|
183
|
+
"""Retrieved context, from wherever the harness put it — same keys the
|
|
184
|
+
groundedness scorer already checks, so a case only has to supply this once."""
|
|
185
|
+
for key in ("retrieved", "context", "documents", "sources"):
|
|
186
|
+
value = ctx.context.get(key)
|
|
187
|
+
if isinstance(value, list) and value:
|
|
188
|
+
return [v if isinstance(v, str) else str(v.get("text") or v.get("content") or "") for v in value]
|
|
189
|
+
if isinstance(value, str) and value.strip():
|
|
190
|
+
return [value]
|
|
191
|
+
return []
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _ragas_sample(output: str, ctx: ScoreContext) -> RagasSample:
|
|
195
|
+
return RagasSample(
|
|
196
|
+
question=str(ctx.case_input.get("input") or ctx.case_input.get("question") or ""),
|
|
197
|
+
answer=output,
|
|
198
|
+
contexts=_context_list(ctx),
|
|
199
|
+
ground_truth=str(ctx.expected.get("output") or ctx.expected.get("ground_truth") or ""),
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class _RagasMetricScorer(BaseScorer):
|
|
204
|
+
"""One selectable scorer per Ragas metric, named exactly as Ragas names it —
|
|
205
|
+
a team already fluent in Ragas vocabulary shouldn't have to relearn ours.
|
|
206
|
+
Falls back to the native lexical approximation where the real `ragas` package
|
|
207
|
+
isn't installed; `detail.implementation` on every result says which ran."""
|
|
208
|
+
|
|
209
|
+
metric: str = ""
|
|
210
|
+
kind = "ragas"
|
|
211
|
+
threshold = 0.7
|
|
212
|
+
|
|
213
|
+
def score(self, output: str, ctx: ScoreContext) -> ScoreResult:
|
|
214
|
+
scores = score_sample(_ragas_sample(output, ctx))
|
|
215
|
+
value = getattr(scores, self.metric)
|
|
216
|
+
return self._result(value, implementation=scores.implementation)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
class RagasFaithfulnessScorer(_RagasMetricScorer):
|
|
220
|
+
key, metric = "ragas_faithfulness", "faithfulness"
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class RagasAnswerRelevancyScorer(_RagasMetricScorer):
|
|
224
|
+
key, metric = "ragas_answer_relevancy", "answer_relevancy"
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class RagasContextPrecisionScorer(_RagasMetricScorer):
|
|
228
|
+
key, metric = "ragas_context_precision", "context_precision"
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
class RagasContextRecallScorer(_RagasMetricScorer):
|
|
232
|
+
key, metric = "ragas_context_recall", "context_recall"
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
for _rs in (
|
|
236
|
+
RagasFaithfulnessScorer(),
|
|
237
|
+
RagasAnswerRelevancyScorer(),
|
|
238
|
+
RagasContextPrecisionScorer(),
|
|
239
|
+
RagasContextRecallScorer(),
|
|
240
|
+
):
|
|
241
|
+
register_scorer(_rs)
|