sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/_rag_diagnostics_importers.py

@@ -0,0 +1,242 @@
+"""Offline, qrel-only BEIR/TREC import. No downloads, inference, or SDK imports.
+
+This imports caller-held files and records exact byte receipts. Neither qrels
+nor a caller's cluster mapping establish independent source answerability.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from pathlib import Path
+import re
+
+from ._rag_diagnostics_common import digest
+from ._rag_diagnostics_contracts import ContractError, validate_dataset
+
+MAX_INPUT_BYTES = 20_000_000
+MAX_OUTPUT_BYTES = 20_000_000
+INPUT_KEYS = ('corpus', 'queries', 'qrels', 'baseline_run', 'candidate_run')
+ORDER_POLICY = 'ascending_explicit_rank; equal_rank_retains_input_file_order; score_never_reorders_rows'
+
+
+def _require(condition, message):
+    if not condition:
+        raise ContractError(message)
+
+
+def _id(value, context):
+    _require(type(value) is str and 0 < len(value) <= 180 and not any(c.isspace() for c in value),
+             context + ': expected a nonempty ID without whitespace (maximum 180 characters)')
+    return value
+
+
+def _pairs(items):
+    result = {}
+    for key, value in items:
+        _require(key not in result, 'Duplicate JSON object key')
+        result[key] = value
+    return result
+
+
+def _json(raw):
+    def bad_constant(_):
+        raise ContractError('Nonfinite JSON number')
+    def finite_float(value):
+        number = float(value)
+        _require(math.isfinite(number), 'Nonfinite JSON number')
+        return number
+    try:
+        return json.loads(raw, object_pairs_hook=_pairs, parse_constant=bad_constant, parse_float=finite_float)
+    except (ValueError, UnicodeDecodeError) as exc:
+        if isinstance(exc, ContractError):
+            raise
+        raise ContractError('Invalid UTF-8 JSON input') from None
+
+
+def _read(path, key, expected):
+    path = Path(path)
+    _require(path.is_file() and not path.is_symlink(), key + ': expected a regular local file')
+    _require(0 < path.stat().st_size <= MAX_INPUT_BYTES, key + ': empty or oversized input')
+    raw = path.read_bytes()
+    _require(0 < len(raw) <= MAX_INPUT_BYTES, key + ': empty or oversized input')
+    sha = hashlib.sha256(raw).hexdigest()
+    if expected is not None:
+        _require(expected == sha, key + ': input SHA256 differs from caller pin')
+    try:
+        text = raw.decode('utf-8')
+    except UnicodeDecodeError:
+        raise ContractError(key + ': invalid UTF-8') from None
+    return text, {'file_name': path.name, 'sha256': sha, 'size_bytes': len(raw)}
+
+
+def _jsonl(text, *, corpus):
+    rows = {}
+    for line_number, line in enumerate(text.splitlines(), 1):
+        if not line.strip():
+            continue
+        row = _json(line)
+        _require(type(row) is dict, 'JSONL row must be an object')
+        identity = _id(row.get('_id'), 'JSONL _id')
+        _require(identity not in rows, 'Duplicate corpus or query ID')
+        _require(type(row.get('text')) is str and (corpus or bool(row['text'].strip())), 'JSONL text is required')
+        if corpus:
+            title = row.get('title', '')
+            _require(type(title) is str, 'Corpus title must be a string when supplied')
+            # Preserve exact original title/text in this immutable scoring unit.
+            rows[identity] = {'id': identity, 'text': (title + '\n' if title else '') + row['text']}
+        else:
+            rows[identity] = row['text']
+    _require(bool(rows), 'Corpus and queries must each contain rows')
+    _require(len(rows) <= (10000 if corpus else 5000), 'Input exceeds local prototype row limit')
+    return rows
+
+
+def _qrels(text, corpus_ids, query_ids):
+    rows = {}
+    saw_content = False
+    for line in text.splitlines():
+        if not line.strip():
+            continue
+        fields = [part.strip() for part in line.split('\t')]
+        if not saw_content and fields == ['query-id', 'corpus-id', 'score']:
+            saw_content = True
+            continue
+        saw_content = True
+        _require(len(fields) == 3, 'Qrels require three tab-separated fields')
+        qid, did, value = fields
+        _id(qid, 'Qrel query ID'); _id(did, 'Qrel document ID')
+        _require(qid in query_ids, 'Qrels contain an unknown query ID')
+        _require(did in corpus_ids, 'Qrels contain an out-of-corpus document ID')
+        _require(re.fullmatch(r'[0-9]+', value) is not None and 0 <= int(value) <= 100,
+                 'Qrel grade must be an integer between 0 and 100')
+        _require(did not in rows.setdefault(qid, {}), 'Duplicate qrel query/document pair')
+        rows[qid][did] = int(value)
+    _require(set(rows) == set(query_ids), 'Qrels must cover exactly the supplied query population')
+    return rows
+
+
+def _run(text, corpus_ids, query_ids):
+    rows, names, seen = {}, set(), set()
+    for line_number, line in enumerate(text.splitlines(), 1):
+        if not line.strip():
+            continue
+        fields = line.split()
+        _require(len(fields) == 6, 'TREC run requires qid Q0 docid rank score runname')
+        qid, marker, did, rank, score, run_name = fields
+        _id(qid, 'Run query ID'); _id(did, 'Run document ID'); _id(run_name, 'Run name')
+        _require(marker == 'Q0', 'TREC run second column must be Q0')
+        _require(qid in query_ids, 'TREC run contains an unknown query ID')
+        _require(did in corpus_ids, 'TREC run contains an out-of-corpus document ID')
+        _require((qid, did) not in seen, 'Duplicate TREC query/document pair')
+        seen.add((qid, did))
+        _require(re.fullmatch(r'[0-9]+', rank) is not None and 1 <= int(rank) <= 1000000,
+                 'TREC rank must be a positive integer')
+        try:
+            score_number = float(score)
+        except ValueError:
+            raise ContractError('TREC score must be finite') from None
+        _require(math.isfinite(score_number), 'TREC score must be finite')
+        names.add(run_name)
+        rows.setdefault(qid, []).append({'document_id': did, 'rank': int(rank),
+                                       'score': score_number, 'input_line': line_number})
+    _require(set(rows) == set(query_ids), 'Each TREC run must cover exactly the supplied query population')
+    _require(len(names) == 1, 'Each TREC file must identify exactly one run')
+    for ranking in rows.values():
+        ranking.sort(key=lambda row: (row['rank'], row['input_line']))
+    return rows, next(iter(names))
+
+
+def import_beir(corpus_path, queries_path, qrels_path, baseline_run_path,
+                candidate_run_path, *, release='external-beir-local.v1',
+                cluster_mapping_path=None, expected_sha256=None):
+    """Import paired caller-held rankings into a retrieval-only local dataset.
+
+    ``expected_sha256`` optionally pins all five inputs (plus ``clusters`` when
+    used). Without a mapping, every query shares one conservative cluster and
+    independence is explicitly false. A complete mapping is a caller assertion;
+    this offline importer cannot authenticate its independence or label review.
+    """
+    _id(release, 'Release')
+    paths = dict(zip(INPUT_KEYS, (corpus_path, queries_path, qrels_path, baseline_run_path, candidate_run_path)))
+    if cluster_mapping_path is not None:
+        paths['clusters'] = cluster_mapping_path
+    if expected_sha256 is not None:
+        _require(type(expected_sha256) is dict and set(expected_sha256) == set(paths),
+                 'Expected SHA256 pins must cover exactly all supplied inputs')
+        _require(all(type(value) is str and re.fullmatch(r'[0-9a-f]{64}', value)
+                     for value in expected_sha256.values()), 'Expected SHA256 pins must be lowercase 64-character hex')
+    contents, receipts = {}, {}
+    for key, path in paths.items():
+        contents[key], receipts[key] = _read(path, key, expected_sha256.get(key) if expected_sha256 else None)
+    corpus = _jsonl(contents['corpus'], corpus=True)
+    queries = _jsonl(contents['queries'], corpus=False)
+    qrels = _qrels(contents['qrels'], corpus, queries)
+    runs, run_names = {}, {}
+    for variant in ('baseline', 'candidate'):
+        runs[variant], run_names[variant] = _run(contents[variant + '_run'], corpus, queries)
+    mapping = None
+    if cluster_mapping_path is not None:
+        mapping = _json(contents['clusters'])
+        _require(type(mapping) is dict and set(mapping) == set(queries),
+                 'Cluster mapping must cover exactly the supplied query population')
+        for qid, cluster in mapping.items():
+            _id(qid, 'Cluster query ID'); _id(cluster, 'Cluster ID')
+    corpus_bytes = len(json.dumps(list(corpus.values()), ensure_ascii=False, allow_nan=False).encode('utf-8'))
+    _require(corpus_bytes * len(queries) <= MAX_OUTPUT_BYTES,
+             'Repeated corpus exceeds 20MB local limit; supply an explicit smaller cohort/corpus')
+    suite_id = 'external-beir-retrieval'
+    cases = []
+    for qid, question in queries.items():
+        case_id = 'beir-' + digest(qid)[:24]
+        traces = {}
+        for variant in ('baseline', 'candidate'):
+            traces[variant] = {'id': case_id + '/' + variant, 'execution_mode': 'recorded_replay',
+                'stages': {'indexed': list(corpus), 'retrieved': [row['document_id'] for row in runs[variant][qid]],
+                           'reranked': None, 'selected': None, 'prompt': None},
+                'response': {'status': 'ok', 'text': '', 'abstained': None},
+                'operational': {'origin': 'component_reported', 'latency_ms': None, 'cost_usd': None, 'tokens': None},
+                'source_run': {'name': run_names[variant], 'ranking': runs[variant][qid],
+                               'input_sha256': receipts[variant + '_run']['sha256']}}
+        cases.append({'id': case_id, 'suite_id': suite_id,
+            'cluster_id': mapping[qid] if mapping is not None else 'unverified-all-imported-queries',
+            'question': question, 'documents': list(corpus.values()), 'source_query_id': qid,
+            'references': {'qrels': qrels[qid], 'corpus_answerable': None, 'required_atoms': None,
+                'atom_support': None, 'required_relations': None, 'relation_support': None,
+                'reviews': {'baseline': None, 'candidate': None}}, 'traces': traces})
+    data = {'schema_version': 'sphynx.rag-diagnostics.dataset.v1', 'release': release,
+        'suites': [{'id': suite_id, 'title': 'Imported retrieval rankings',
+                    'description': 'Caller-held corpus, qrels and paired TREC rankings. Retrieval-only recorded data; no generated answer or semantic audit.',
+                    'inspiration': 'BEIR-compatible input conventions; no official BEIR evaluator parity claimed'}],
+        'cases': cases,
+        'provenance': {'kind': 'external_qrel_only_import', 'importer_version': 'sphynx.rag-diagnostics.beir-import.v1',
+            'input_receipts': receipts, 'caller_sha256_pins_verified': expected_sha256 is not None,
+            'ordering_policy': ORDER_POLICY, 'run_names': run_names,
+            'independent_clusters_verified': mapping is not None,
+            'cluster_mapping_basis': 'caller_supplied_not_authenticated' if mapping is not None else 'unknown; one conservative shared cluster',
+            'semantic_review': 'absent', 'corpus_answerability': 'not_inferred_from_positive_qrels',
+            'unjudged_policy': 'absent_qrel_is_unknown; never_insert_zero',
+            'source_rights': 'caller_responsibility; not_verified_by_importer',
+            'population': 'caller_selected_external_queries; representativeness_not_verified'}}
+    validate_dataset(data)
+    # Current wire contract repeats corpus documents per case; fail rather than
+    # silently sampling rows or constructing an oversized preview bundle.
+    _require(len(json.dumps(data, ensure_ascii=False, allow_nan=False).encode('utf-8')) <= MAX_OUTPUT_BYTES,
+             'Imported dataset exceeds 20MB local limit; supply an explicit smaller cohort/corpus')
+    return data
+
+
+def write_dataset(data, output_path):
+    """Write a new immutable review artifact; refuse replacement or truncation."""
+    validate_dataset(data)
+    raw = (json.dumps(data, indent=2, ensure_ascii=False, allow_nan=False) + '\n').encode('utf-8')
+    _require(len(raw) <= MAX_OUTPUT_BYTES, 'Pretty JSON output exceeds 20MB local limit')
+    output = Path(output_path)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        with output.open('xb') as handle:
+            handle.write(raw)
+    except FileExistsError:
+        raise ContractError('Output already exists; select a new release file') from None
+    return {'path': str(output.resolve()), 'sha256': hashlib.sha256(raw).hexdigest(),
+            'cases': len(data['cases']), 'suites': len(data['suites'])}

sphynx/_rag_diagnostics_ranking.py

@@ -0,0 +1,81 @@
+"""Known-qrel retrieval metrics using trec_eval linear-gain conventions.
+
+https://github.com/usnistgov/trec_eval/blob/master/m_ndcg_cut.c
+https://github.com/usnistgov/trec_eval/blob/master/m_map_cut.c
+Tie resolution is upstream: the supplied unique ordered IDs are authoritative.
+No trec_eval binary parity is claimed for unspecified ties or unlabeled queries.
+"""
+from __future__ import annotations
+import math
+from ._rag_diagnostics_common import metric, STAGES
+
+
+def ranking_metrics(case,trace,cutoff=3):
+    if type(cutoff) is not int or not 1<=cutoff<=1000:
+        raise ValueError('cutoff must be an integer from 1 to 1000')
+    refs=case['references'];qrels=refs.get('qrels');rows=[]
+    for stage in ('retrieved','reranked','selected','prompt'):
+        ranked=trace['stages'].get(stage)
+        names=('recall','precision','average_precision','ndcg_linear','reciprocal_rank','unjudged_fraction')
+        for name in names:
+            mid='retrieval.'+name+'@'+str(cutoff)
+            if ranked is None or qrels is None:
+                rows.append(metric(mid,case['id'],stage,reason='missing stage or qrels',direction='lower' if name=='unjudged_fraction' else 'higher'));continue
+            top=ranked[:cutoff];positive={key for key,grade in qrels.items() if grade>0}
+            hits=[int(qrels.get(key,0)>0) for key in top]
+            total=len(positive);found=sum(hits)
+            kwargs={'evidence_refs':top,'population':'fixed full case qrels; positives > 0; unjudged retained at original ranks','direction':'higher'}
+            if name=='unjudged_fraction':
+                unknown=sum(key not in qrels for key in top)
+                rows.append(metric(mid,case['id'],stage,unknown/len(top) if top else None,numerator=unknown,denominator=len(top),status='available' if top else 'unavailable_zero_population',reason='No returned items' if not top else 'Unjudged contributes zero gain only to the known-qrel projection; it is not a negative label',direction='lower',evidence_refs=top));continue
+            if name=='precision':
+                rows.append(metric(mid,case['id'],stage,found/cutoff,numerator=found,denominator=cutoff,**kwargs));continue
+            if not total:
+                rows.append(metric(mid,case['id'],stage,status='unavailable_zero_population',reason='No known positive qrels; local undefined policy differs from trec_eval zero convention',**kwargs));continue
+            if name=='recall':
+                value=found/total;numerator=found;denominator=total
+            elif name=='average_precision':
+                cumulative=0;numerator=0.0
+                for rank,hit in enumerate(hits,1):
+                    cumulative+=hit
+                    if hit:numerator+=cumulative/rank
+                denominator=total;value=numerator/denominator
+            elif name=='reciprocal_rank':
+                first=next((i for i,hit in enumerate(hits,1) if hit),None)
+                value=1/first if first else 0.0;numerator=1 if first else 0;denominator=first if first else 1
+                if first is None:
+                    kwargs['reason']='A valid ranking returned no known relevant item within the cutoff; observed reciprocal rank is 0/1, not missing data.'
+            else:
+                numerator=sum(qrels.get(key,0)/math.log2(rank+1) for rank,key in enumerate(top,1))
+                denominator=sum(g/math.log2(rank+1) for rank,g in enumerate(sorted(qrels.values(),reverse=True)[:cutoff],1))
+                value=numerator/denominator
+            rows.append(metric(mid,case['id'],stage,value,numerator=numerator,denominator=denominator,**kwargs))
+    return rows
+
+
+def evidence_metrics(case,trace):
+    refs=case['references'];rows=[]
+    for stage in STAGES:
+        ids=trace['stages'].get(stage)
+        for key,mapping,label in [('required_atoms','atom_support','atom'),('required_relations','relation_support','relation')]:
+            required=refs.get(key);support=refs.get(mapping)
+            reason=None;status=None;count=None;denominator=None
+            if ids is None or required is None or support is None or not set(required)<=set(support):
+                reason='Missing stage or protected evidence mapping'
+            elif not required:
+                reason='No required '+label+' population';status='unavailable_zero_population'
+            else:
+                denominator=len(required)
+                if label=='relation':
+                    # Every listed document is part of the required relation.
+                    # Empty support cannot satisfy a relation by vacuous truth.
+                    count=sum(bool(support[item]) and set(support[item])<=set(ids) for item in required)
+                    reason='A relation is covered only when every document in its nonempty required support set is present.'
+                else:
+                    count=sum(bool(set(support[item])&set(ids)) for item in required)
+                    reason='An atom is covered when any declared alternative supporting document is present.'
+            value=count/denominator if denominator else None
+            rows.append(metric('evidence.'+label+'_recall',case['id'],stage,value,numerator=count,denominator=denominator,status=status,reason=reason,evidence_refs=ids or []))
+            if label=='atom':
+                rows.append(metric('evidence.complete_required_atoms',case['id'],stage,float(count==denominator) if denominator else None,numerator=int(count==denominator) if denominator else None,denominator=1 if denominator else None,status=status,reason=reason,evidence_refs=ids or []))
+    return rows

sphynx/_rag_diagnostics_semantic.py

@@ -0,0 +1,325 @@
+"""Local arithmetic over explicit, trace-bound semantic reviews.
+
+No text matching, claim extraction, model inference, or provider calls occur here.
+Synthetic reviews exercise definitions; they do not demonstrate judge accuracy.
+
+Sources: RAGAS faithfulness (supported response claims / response claims):
+https://docs.ragas.io/en/stable/concepts/metrics/available_metrics/faithfulness/
+Sufficient Context is a question/context judgment, independent of correctness:
+https://arxiv.org/html/2411.06037v3#S3
+Citation arithmetic is ALCE-inspired over reviewed claims, not paper parity:
+https://github.com/princeton-nlp/ALCE/blob/main/eval.py#L276
+Claim-level separation of support and correctness follows RAGChecker's diagnosis:
+https://arxiv.org/html/2408.08067v2#A2
+
+The caller validates the review's trace digest. Missing and unresolved judgments
+remain unavailable. Empty claim populations never become perfect faithfulness.
+"""
+from __future__ import annotations
+
+from ._rag_diagnostics_common import metric
+
+
+_GROUNDED_REQUIREMENTS = (
+    'Requires a correct, relevant answer, exhaustive claim support, supporting '
+    'citations for citation-required claims, useful emitted citations, and full required-fact coverage '
+    'when required atoms are configured; an explicitly expected abstention '
+    'also succeeds.'
+)
+
+
+def _bool(value):
+    return value if type(value) is bool else None
+
+
+def semantic_metrics(case, trace, review):
+    """Return semantic metric records without deriving semantic labels from text.
+
+    Required-fact coverage counts gold atoms linked to *correct* reviewed claims.
+    It is not context sufficiency. Overall answer correctness and response claim
+    correctness remain independent judgments. Success gates are conjunctive;
+    scores cannot compensate for a failed requirement.
+    """
+    case_id = case['id']
+    refs = case.get('references') or {}
+    response = trace.get('response') or {}
+    stages = trace.get('stages') or {}
+    review = review if isinstance(review, dict) else None
+    provenance = (review or {}).get('provenance') or {}
+    grader = 'synthetic_label' if provenance.get('kind') == 'synthetic' else 'reviewed_label'
+    records = []
+
+    def emit(metric_id, stage, value=None, *, numerator=None, denominator=None,
+             status=None, reason=None, evidence=None, population='this case',
+             direction='higher', unit='ratio', grader_type=None):
+        result = metric(metric_id, case_id, stage, value, numerator=numerator,
+                        denominator=denominator, status=status, reason=reason,
+                        grader_type=grader_type or grader, evidence_refs=evidence,
+                        population=population, direction=direction, unit=unit)
+        records.append(result)
+        return result
+
+    def unavailable(metric_id, stage, reason, *, status='unavailable_missing_input',
+                    numerator=None, denominator=None, evidence=None,
+                    population='this case', direction='higher'):
+        return emit(metric_id, stage, status=status, reason=reason,
+                    numerator=numerator, denominator=denominator,
+                    evidence=evidence, population=population, direction=direction)
+
+    def judged_bool(metric_id, stage, label, field, *, missing_stage=False):
+        if missing_stage:
+            return unavailable(metric_id, stage, 'The context stage was not recorded.',
+                               status='unavailable_missing_stage')
+        if review is None:
+            return unavailable(metric_id, stage, 'A trace-bound semantic review is required.')
+        value = _bool(label)
+        if value is None:
+            return unavailable(metric_id, stage, f'{field} has no resolved boolean judgment.',
+                               status='unavailable_unknown_judgment', evidence=[f'review:{field}'])
+        return emit(metric_id, stage, int(value), numerator=int(value), denominator=1,
+                    evidence=[f'review:{field}'], unit='boolean')
+
+    sufficiency = (review or {}).get('context_sufficiency') or {}
+    for stage in ('retrieved', 'selected', 'prompt'):
+        judged_bool(f'context.sufficient.{stage}', stage, sufficiency.get(stage),
+                    f'context_sufficiency.{stage}', missing_stage=stages.get(stage) is None)
+
+    execution_ok = response.get('status') == 'ok'
+    execution_failed = response.get('status') in ('error', 'timeout')
+    answer_labels = {}
+    for metric_id, field in (
+        ('generation.answer_correctness', 'answer_correct'),
+        ('generation.response_relevance', 'answer_relevant'),
+    ):
+        if not execution_ok:
+            answer_labels[metric_id] = unavailable(
+                metric_id, 'generation', 'No successful answer execution was recorded.',
+                status='unavailable_failed_execution' if execution_failed else 'unavailable_missing_input')
+        else:
+            answer_labels[metric_id] = judged_bool(metric_id, 'generation', (review or {}).get(field), field)
+
+    claims = (review or {}).get('claims')
+    claim_gate = None
+    if not execution_ok:
+        claim_gate = ('unavailable_failed_execution' if execution_failed else 'unavailable_missing_input',
+                      'No successful answer execution was recorded.')
+    elif review is None or not isinstance(claims, list):
+        claim_gate = ('unavailable_missing_input', 'A trace-bound claim audit is required.')
+    elif review.get('claims_exhaustive') is not True:
+        claim_gate = ('unavailable_non_exhaustive_review',
+                      'The claim audit is not exhaustive; omitted claims could change the score.')
+    elif any(not isinstance(claim, dict) or not claim.get('id') for claim in claims):
+        claim_gate = ('unavailable_invalid_input', 'Every reviewed claim needs an identity.')
+    elif len({claim['id'] for claim in claims}) != len(claims):
+        claim_gate = ('unavailable_invalid_input', 'Claim identities must be unique.')
+    claim_list = claims if isinstance(claims, list) else []
+    claim_refs = [f"claim:{claim.get('id')}" for claim in claim_list if isinstance(claim, dict)]
+
+    def claim_fraction(metric_id, field, *, invert=False, direction='higher'):
+        population = 'all response claims in an exhaustive audit'
+        if claim_gate:
+            return unavailable(metric_id, 'generation', claim_gate[1], status=claim_gate[0],
+                               evidence=claim_refs, population=population, direction=direction)
+        if field == 'supported_by_context' and stages.get('prompt') is None:
+            return unavailable(metric_id, 'generation', 'The delivered prompt context was not recorded.',
+                               status='unavailable_missing_stage', evidence=claim_refs,
+                               population=population, direction=direction)
+        if not claim_list:
+            return unavailable(metric_id, 'generation', 'The response has no audited claims.',
+                               status='unavailable_zero_population', numerator=0, denominator=0,
+                               evidence=claim_refs, population=population, direction=direction)
+        labels = [_bool(claim.get(field)) for claim in claim_list]
+        known = sum(value is (False if invert else True) for value in labels)
+        if any(value is None for value in labels):
+            return unavailable(metric_id, 'generation', f'{field} is unresolved for one or more claims.',
+                               status='unavailable_unknown_judgment', numerator=known,
+                               denominator=len(labels), evidence=claim_refs,
+                               population=population, direction=direction)
+        return emit(metric_id, 'generation', known / len(labels), numerator=known,
+                    denominator=len(labels), evidence=claim_refs, population=population,
+                    direction=direction)
+
+    faithfulness = claim_fraction('generation.faithfulness', 'supported_by_context')
+    claim_fraction('generation.unsupported_claim_fraction', 'supported_by_context', invert=True,
+                   direction='lower')
+    claim_fraction('generation.claim_correctness', 'correct')
+
+    atoms = refs.get('required_atoms')
+    fact_id = 'generation.required_fact_coverage'
+    if not isinstance(atoms, list):
+        fact_coverage = unavailable(fact_id, 'generation', 'Required reference atoms were not supplied.')
+    elif not atoms:
+        fact_coverage = unavailable(fact_id, 'generation', 'No required reference atoms were declared.',
+                                    status='unavailable_zero_population', numerator=0, denominator=0)
+    elif claim_gate:
+        fact_coverage = unavailable(fact_id, 'generation', claim_gate[1], status=claim_gate[0])
+    else:
+        required = set(atoms)
+        covered, unresolved = set(), set()
+        missing_links = False
+        for claim in claim_list:
+            links = claim.get('reference_atoms')
+            if not isinstance(links, list):
+                missing_links = True
+                continue
+            matching = required.intersection(links)
+            if _bool(claim.get('correct')) is True:
+                covered.update(matching)
+            elif _bool(claim.get('correct')) is None:
+                unresolved.update(matching)
+        evidence = claim_refs + [f'reference_atom:{atom}' for atom in sorted(required)]
+        if missing_links or unresolved.difference(covered):
+            fact_coverage = unavailable(fact_id, 'generation',
+                'Claim-to-reference links or correctness judgments remain unresolved.',
+                status='unavailable_unknown_judgment', numerator=len(covered),
+                denominator=len(required), evidence=evidence, population='declared required reference atoms')
+        else:
+            fact_coverage = emit(fact_id, 'generation', len(covered) / len(required),
+                numerator=len(covered), denominator=len(required), evidence=evidence,
+                population='declared required reference atoms',
+                reason='Counts unique required atoms expressed in reviewed correct response claims; not context sufficiency.')
+
+    citation_ids = ('citation.reference_validity', 'citation.necessity_precision', 'citation.completeness')
+    citation_results = {}
+    if claim_gate:
+        for metric_id in citation_ids:
+            citation_results[metric_id] = unavailable(metric_id, 'citation', claim_gate[1], status=claim_gate[0])
+    else:
+        documents = {document['id'] for document in case.get('documents', [])}
+        malformed = any(not isinstance(claim.get('cited_ids'), list)
+                        or len(set(claim.get('cited_ids', []))) != len(claim.get('cited_ids', []))
+                        for claim in claim_list)
+        if malformed:
+            for metric_id in citation_ids:
+                citation_results[metric_id] = unavailable(metric_id, 'citation',
+                    'Cited document identities must be supplied as a list without duplicates.',
+                    status='unavailable_invalid_input')
+        else:
+            total_citations = sum(len(claim['cited_ids']) for claim in claim_list)
+            valid_citations = sum(doc_id in documents for claim in claim_list for doc_id in claim['cited_ids'])
+            useful = 0
+            unknown_usefulness = 0
+            required_count = 0
+            complete = 0
+            unknown_completeness = 0
+            missing_requirement = False
+            for claim in claim_list:
+                cited = claim['cited_ids']
+                valid = bool(cited) and all(doc_id in documents for doc_id in cited)
+                joint = _bool(claim.get('joint_citation_support')) if valid else False
+                required_citation = _bool(claim.get('requires_citation'))
+                if required_citation is None:
+                    missing_requirement = True
+                elif required_citation:
+                    required_count += 1
+                    if joint is True:
+                        complete += 1
+                    elif joint is None:
+                        unknown_completeness += 1
+                individual = claim.get('individual_support') or {}
+                leave_one_out = claim.get('leave_one_out_support') or {}
+                for doc_id in cited:
+                    if joint is False:
+                        continue
+                    if joint is None:
+                        unknown_usefulness += 1
+                        continue
+                    if len(cited) == 1:
+                        useful += 1
+                        continue
+                    alone = _bool(individual.get(doc_id))
+                    remainder = _bool(leave_one_out.get(doc_id))
+                    if alone is True or remainder is False:
+                        useful += 1
+                    elif alone is None or remainder is None:
+                        unknown_usefulness += 1
+            for metric_id, numerator, denominator, unknown, population in (
+                ('citation.reference_validity', valid_citations, total_citations, 0, 'all emitted citation references'),
+                ('citation.necessity_precision', useful, total_citations, unknown_usefulness, 'all emitted citation references'),
+                ('citation.completeness', complete, required_count,
+                 unknown_completeness + int(missing_requirement), 'all citation-required response claims'),
+            ):
+                evidence = claim_refs + [f'document:{doc_id}' for claim in claim_list for doc_id in claim['cited_ids']]
+                if unknown:
+                    result = unavailable(metric_id, 'citation',
+                        'Citation requirements or support judgments remain unresolved.',
+                        status='unavailable_unknown_judgment', numerator=numerator,
+                        denominator=denominator, evidence=evidence, population=population)
+                elif denominator == 0:
+                    result = unavailable(metric_id, 'citation', 'The citation metric has no eligible population.',
+                        status='unavailable_zero_population', numerator=0, denominator=0,
+                        evidence=evidence, population=population)
+                else:
+                    result = emit(metric_id, 'citation', numerator / denominator,
+                        numerator=numerator, denominator=denominator, evidence=evidence,
+                        population=population, reason='ALCE-inspired arithmetic over explicit claim reviews; not official ALCE parity.')
+                citation_results[metric_id] = result
+
+    abstained = _bool(response.get('abstained'))
+    expected = _bool((review or {}).get('abstention_expected'))
+    if not execution_ok or review is None or abstained is None or expected is None:
+        abstention = unavailable('outcome.abstention_correctness', 'outcome',
+            'A successful response and explicit actual/expected abstention labels are required.',
+            status='unavailable_failed_execution' if execution_failed else 'unavailable_missing_input')
+    else:
+        abstention = emit('outcome.abstention_correctness', 'outcome', int(abstained == expected),
+            numerator=int(abstained == expected), denominator=1, unit='boolean',
+            evidence=['response:abstained', 'review:abstention_expected'],
+            reason='Checks the declared abstention policy, including unnecessary abstention.')
+
+    if execution_failed:
+        task = emit('outcome.task_success', 'outcome', 0, numerator=0, denominator=1,
+                    reason=f"Execution ended with {response['status']}.", unit='boolean', grader_type='deterministic')
+    elif abstention['value'] is None:
+        task = unavailable('outcome.task_success', 'outcome', abstention['reason'], status=abstention['status'])
+    elif abstention['value'] == 0:
+        task = emit('outcome.task_success', 'outcome', 0, numerator=0, denominator=1,
+                    reason='The response did not follow the explicit abstention policy.', unit='boolean')
+    elif abstained:
+        task = emit('outcome.task_success', 'outcome', 1, numerator=1, denominator=1,
+                    reason='The response abstained when the review explicitly required abstention.', unit='boolean')
+    else:
+        values = [answer_labels[key]['value'] for key in ('generation.answer_correctness', 'generation.response_relevance')]
+        if 0 in values:
+            task = emit('outcome.task_success', 'outcome', 0, numerator=0, denominator=1,
+                        reason='The reviewed answer is incorrect or irrelevant.', unit='boolean')
+        elif None in values:
+            task = unavailable('outcome.task_success', 'outcome',
+                'Answer correctness and relevance must both be resolved.', status='unavailable_unknown_judgment')
+        else:
+            task = emit('outcome.task_success', 'outcome', 1, numerator=1, denominator=1,
+                        reason='The non-abstaining answer is reviewed as correct and relevant.', unit='boolean')
+
+    grounded_id = 'outcome.grounded_task_success'
+    if task['value'] is None:
+        unavailable(grounded_id, 'outcome', task['reason'], status=task['status'])
+    elif task['value'] == 0:
+        emit(grounded_id, 'outcome', task['value'], numerator=task['value'], denominator=1,
+             reason=f"{task['reason']} {_GROUNDED_REQUIREMENTS}", unit='boolean')
+    elif abstained is True and claim_gate:
+        unavailable(grounded_id, 'outcome',
+                    'An exhaustive claim audit is required to confirm the abstention contains no unsupported assertions.',
+                    status=claim_gate[0])
+    elif abstained is True and not claim_list:
+        emit(grounded_id, 'outcome', 1, numerator=1, denominator=1,
+             reason='The review required abstention and an exhaustive audit found no asserted claims.', unit='boolean')
+    else:
+        requirements = [faithfulness]
+        if not abstained and isinstance(atoms, list) and atoms:
+            requirements.append(fact_coverage)
+        requires_citation = any(claim.get('requires_citation') is True for claim in claim_list)
+        missing_citation_requirement = any(_bool(claim.get('requires_citation')) is None for claim in claim_list)
+        if requires_citation or missing_citation_requirement:
+            requirements.append(citation_results['citation.completeness'])
+        if any(isinstance(claim.get('cited_ids'), list) and claim['cited_ids'] for claim in claim_list):
+            requirements.append(citation_results['citation.necessity_precision'])
+        if any(result['value'] is not None and result['value'] < 1 for result in requirements):
+            emit(grounded_id, 'outcome', 0, numerator=0, denominator=1,
+                 reason=f'A grounding, citation, or required-fact condition failed. {_GROUNDED_REQUIREMENTS}', unit='boolean')
+        elif any(result['value'] is None for result in requirements):
+            unavailable(grounded_id, 'outcome', f'One or more grounding conditions are unavailable. {_GROUNDED_REQUIREMENTS}',
+                        status='unavailable_unknown_judgment')
+        else:
+            emit(grounded_id, 'outcome', 1, numerator=1, denominator=1,
+                 reason=_GROUNDED_REQUIREMENTS, unit='boolean')
+    return records