sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/rag_papers/semantic.py

@@ -0,0 +1,288 @@
+"""Paper-specific semantic arithmetic over exact-input captured judgments.
+
+Model extraction and entailment are performed by the separate judge execution
+module. These functions never call a model or turn model labels into human gold.
+"""
+from __future__ import annotations
+
+import math
+import re
+
+from .contracts import require
+from .scoring import labels_for, measure, normalize_answer, ratio, result
+
+
+def booleans(values):
+    require(type(values) is list and all(type(v) is bool for v in values), 'Expected resolved boolean judgments')
+    return values
+
+
+def evaluate_sufficient_context(case, trace, corpus, *, judgments=()):
+    """Sufficiency and conditional generation outcomes; no qrel substitution."""
+    labels = labels_for(judgments, case, trace, 'sufficient_context')
+    sufficient = None if labels is None else labels.get('sufficient')
+    correct = None if labels is None else labels.get('answer_correct')
+    require(sufficient is None or type(sufficient) is bool, 'Invalid sufficiency verdict')
+    require(correct is None or type(correct) is bool, 'Invalid correctness verdict')
+    source = case['references']['answerable']
+    metrics = {'context_sufficient': measure(None if sufficient is None else float(sufficient)),
+        'retrieval_sufficiency_recall': measure(None if source is not True or sufficient is None else float(sufficient),
+            denominator=1 if source is True else None, reason='Conditioned on independently source-answerable cases'),
+        'success_given_sufficient_context': measure(None if sufficient is not True or correct is None else
+            float(correct and trace['response']['status'] == 'ok' and not trace['response']['abstained']),
+            denominator=1 if sufficient is True else None),
+        'incorrect_response_given_insufficient_context': measure(None if sufficient is not False or correct is None else
+            float(not correct and not trace['response']['abstained']), denominator=1 if sufficient is False else None)}
+    return result('sufficient_context', case, trace, corpus, metrics, judgments=judgments,
+                  notes=['Sufficiency is a question/context judgment, not proof of correct generation.',
+                         'Conditional exclusions and missing judgments must be aggregated explicitly.'])
+
+
+def evaluate_ragas(case, trace, corpus, *, judgments=()):
+    """RAGAs claim support, reference-claim recall and ranked context precision.
+
+    Response relevancy consumes retained reverse-question embedding vectors;
+    no embedding similarity is fabricated from text overlap.
+    """
+    labels = labels_for(judgments, case, trace, 'ragas')
+    metrics = {}
+    for key, name in [('claim_supported', 'faithfulness'), ('reference_claim_supported', 'context_recall')]:
+        if labels is None or labels.get(key) is None:
+            metrics[name] = measure(reason='Required claim entailment judgments unavailable')
+        else:
+            values = booleans(labels[key]); metrics[name] = ratio(sum(values), len(values))
+    relevant = None if labels is None else labels.get('context_relevant')
+    if relevant is None:
+        metrics['context_precision'] = measure(reason='Ranked context relevance judgments unavailable')
+    else:
+        values = booleans(relevant)
+        require(trace['stages']['prompt'] is not None and len(values) == len(trace['stages']['prompt']),
+                'Context judgments do not cover the exact ranked prompt')
+        found = 0; precision_sum = 0.0
+        for rank, value in enumerate(values, 1):
+            found += value
+            if value:
+                precision_sum += found/rank
+        # RAGAs uses an epsilon denominator, yielding zero when no contexts are
+        # relevant. Record that convention instead of changing it to missing.
+        metrics['context_precision'] = measure(precision_sum/found if found else 0.0,
+            numerator=precision_sum, denominator=found)
+    vectors = None if labels is None else labels.get('question_vectors')
+    query = None if labels is None else labels.get('query_vector')
+    if vectors is None or query is None:
+        metrics['response_relevancy'] = measure(reason='Reverse-question embeddings unavailable')
+    else:
+        require(type(query) is list and query and type(vectors) is list and vectors, 'Invalid embedding population')
+        sims = []
+        for vector in vectors:
+            require(type(vector) is list and len(vector) == len(query)
+                    and all(type(x) in (int, float) and math.isfinite(x) for x in query+vector),
+                    'Invalid retained embedding vector')
+            norms = math.sqrt(sum(x*x for x in query)*sum(x*x for x in vector))
+            require(norms > 0, 'Zero-norm embedding has no cosine similarity')
+            sims.append(sum(x*y for x, y in zip(query, vector))/norms)
+        noncommittal = labels.get('noncommittal')
+        require(type(noncommittal) is bool, 'Reverse-question noncommittal judgment required')
+        metrics['response_relevancy'] = measure(0.0 if noncommittal else sum(sims)/len(sims), unit='cosine')
+    return result('ragas', case, trace, corpus, metrics, judgments=judgments,
+                  notes=['Named Sphynx protocol with explicit retained claim/context/vector inputs; not all metrics in the evolving Ragas package.'])
+
+
+def evaluate_alce(case, trace, corpus, *, judgments=(), citation_cap=3):
+    """ALCE joint citation entailment and useful/necessary citation precision.
+
+    Sentence segmentation is a retained input to the judgment. The complete
+    ordered sentence texts must reconstruct the response modulo whitespace.
+    NLI labels bind joint, individual and leave-one-out cited document sets.
+    """
+    require(type(citation_cap) is int and citation_cap >= 1, 'Invalid citation cap')
+    labels = labels_for(judgments, case, trace, 'alce')
+    metrics = {}; rows = None if labels is None else labels.get('sentences')
+    if rows is None:
+        metrics = {name: measure(reason='Sentence-bound NLI judgments unavailable')
+                   for name in ('citation_recall', 'citation_precision')}
+    else:
+        require(labels.get('citation_cap',3)==citation_cap,'ALCE citation-cap judgment differs from scoring policy')
+        require(type(rows) is list and all(type(r) is dict and type(r.get('text')) is str for r in rows),
+                'Invalid ALCE sentence population')
+        require(' '.join(' '.join(r['text'] for r in rows).split()) == ' '.join(trace['response']['text'].split()),
+                'ALCE sentences do not cover the complete response')
+        prompt = trace['stages']['prompt']
+        require(prompt is not None, 'Delivered context is required for ALCE')
+        supported = useful = citations = 0
+        for row in rows:
+            cited = row.get('cited_ids')
+            require(type(cited) is list and all(type(d) is str for d in cited), 'Invalid citation list')
+            numbers = [int(n) for n in re.findall(r'\[(\d+)\]', row['text'])]
+            expected = [prompt[n-1] if 1 <= n <= len(prompt) else 'invalid:'+str(n) for n in numbers]
+            require(cited == expected, 'Citations differ from emitted numeric references')
+            # Unlike the old diagnostics contract, repeated emitted references
+            # retain their occurrence count in the native ALCE denominator.
+            valid = bool(cited) and all(d in prompt for d in cited)
+            if not valid:
+                continue
+            cited = cited[:citation_cap]; citations += len(cited)
+            joint = row.get('joint_support')
+            require(type(joint) is bool, 'Joint citation entailment unresolved')
+            supported += joint
+            if not joint:
+                continue
+            if len(cited) == 1:
+                useful += 1; continue
+            alone, without = row.get('individual_support'), row.get('leave_one_out_support')
+            require(type(alone) is dict and type(without) is dict, 'Citation ablations unavailable')
+            for did in cited:
+                require(type(alone.get(did)) is bool and type(without.get(did)) is bool,
+                        'Citation ablation judgment unresolved')
+                useful += alone[did] or not without[did]
+        metrics['citation_recall'] = ratio(supported, len(rows))
+        metrics['citation_precision'] = measure(useful/citations if citations else 0.0,
+            numerator=useful, denominator=citations, reason='ALCE zero-citation convention' if not citations else None)
+    native = case['references']['native']
+    groups = native.get('asqa_short_answers')
+    if groups is not None:
+        text = normalize_answer(re.sub(r'\[\d+\]', '', trace['response']['text']))
+        require(type(groups) is list and groups and all(type(g) is list and g for g in groups), 'Invalid ASQA answer groups')
+        hits = [any(normalize_answer(a) in text for a in aliases) for aliases in groups]
+        metrics['asqa_str_em'] = ratio(sum(hits), len(hits))
+        metrics['asqa_str_em_hit'] = measure(float(all(hits)))
+    groups = native.get('qampari_answers')
+    if groups is not None:
+        require(type(groups) is list and groups and all(type(g) is list and g for g in groups), 'Invalid QAMPARI answers')
+        text = re.sub(r'\[\d+\]', '', trace['response']['text']).rstrip().rstrip('.').rstrip(',')
+        predictions = [normalize_answer(p.strip()) for p in text.split(',')]
+        predictions = [p for p in predictions if p]
+        gold = [[normalize_answer(a) for a in aliases] for aliases in groups]
+        precision = sum(p in {a for aliases in gold for a in aliases} for p in predictions)/len(predictions) if predictions else 0.0
+        recovered = sum(any(a in predictions for a in aliases) for aliases in gold)
+        recall = recovered/len(gold); top5 = min(5, recovered)/min(5, len(gold))
+        metrics.update(qampari_precision=measure(precision), qampari_recall=measure(recall),
+            qampari_recall_top5=measure(top5), qampari_f1=measure(2*precision*recall/(precision+recall) if precision+recall else 0.0),
+            qampari_f1_top5=measure(2*precision*top5/(precision+top5) if precision+top5 else 0.0))
+    return result('alce', case, trace, corpus, metrics, judgments=judgments,parameters=dict(citation_cap=citation_cap),
+        notes=['Citation cap is explicit; sentence/NLI model identity is in the retained judgment.',
+               'ROUGE, MAUVE and QA-model auxiliary metrics are not silently approximated.'])
+
+
+def evaluate_factscore(case, trace, corpus, *, judgments=(), gamma=10):
+    """FActScore atomic precision and exp(1-gamma/n) short-response penalty."""
+    require(type(gamma) in (int, float) and math.isfinite(gamma) and gamma >= 0, 'Invalid FActScore gamma')
+    labels = labels_for(judgments, case, trace, 'factscore')
+    supported = None if labels is None else labels.get('fact_supported')
+    metrics = {key:measure(reason='Atomic facts and knowledge-source verification unavailable')
+               for key in ('atomic_precision','factscore','response_eligible','length_penalty','fact_count')}
+    if supported is None:
+        pass
+    else:
+        supported = booleans(supported)
+        if not supported:
+            metrics.update(atomic_precision=measure(reason='No extracted facts'),
+                           factscore=measure(reason='No extracted facts'), response_eligible=measure(0.0),
+                           fact_count=measure(0,unit='facts'))
+        else:
+            count = len(supported); raw = sum(supported)/count
+            penalty = 1.0 if not gamma or count >= gamma else math.exp(1-gamma/count)
+            metrics = {'atomic_precision': ratio(sum(supported), count),
+                       'factscore': measure(raw*penalty), 'response_eligible': measure(1.0),
+                       'length_penalty': measure(penalty), 'fact_count': measure(count, unit='facts')}
+    return result('factscore', case, trace, corpus, metrics, judgments=judgments,parameters=dict(gamma=gamma),
+                  notes=['Knowledge-source verification is distinct from support in the delivered RAG context.',
+                         'Facts/extraction and knowledge source must be captured by the judge workflow.'])
+
+
+def evaluate_ragchecker(case, trace, corpus, *, judgments=()):
+    """RAGChecker's eleven diagnostic rates from captured entailment matrices."""
+    labels = labels_for(judgments, case, trace, 'ragchecker')
+    names = ('precision','recall','f1','claim_recall','context_precision','context_utilization',
+             'noise_sensitivity_in_relevant','noise_sensitivity_in_irrelevant','hallucination','self_knowledge','faithfulness')
+    if labels is None:
+        metrics = {name: measure(reason='Claim entailment matrices unavailable') for name in names}
+    else:
+        a2r = None if labels.get('answer2response') is None else booleans(labels['answer2response'])
+        r2a = None if labels.get('response2answer') is None else booleans(labels['response2answer'])
+        d2a, d2r = labels.get('retrieved2answer'), labels.get('retrieved2response')
+        prompt = trace['stages']['prompt']
+        metrics={name:measure(reason='Required directional entailment inputs unavailable') for name in names}
+        def known_ratio(n,d):
+            return measure(n/d if d else 0.0,numerator=n,denominator=d,
+                reason='RAGChecker resolved-empty zero convention' if not d else None)
+        for values,name in [(a2r,'precision'),(r2a,'recall')]:
+            if values is not None:metrics[name]=known_ratio(sum(values),len(values))
+        if a2r is not None and r2a is not None:
+            p,r=metrics['precision']['value'],metrics['recall']['value']
+            metrics['f1']=measure(2*p*r/(p+r) if p+r else 0.0)
+        def matrix(value,known_claims,claim_texts):
+            if value is None:return None
+            require(type(value) is list and prompt is not None and len(value)==len(prompt),'Context matrices differ from delivered context')
+            count=len(known_claims) if known_claims is not None else len(claim_texts) if claim_texts is not None else len(value[0]) if value else 0
+            for row in value:require(len(booleans(row))==count,'Entailment matrix shape differs')
+            return [any(row[i] for row in value) for i in range(count)]
+        retrieved=matrix(d2a,r2a,labels.get('reference_claims'))
+        grounded=matrix(d2r,a2r,labels.get('response_claims'))
+        relevant=None if d2a is None else [any(row) for row in d2a]
+        if retrieved is not None:
+            metrics['claim_recall']=known_ratio(sum(retrieved),len(retrieved))
+            metrics['context_precision']=known_ratio(sum(relevant),len(relevant))
+            if r2a is not None:
+                metrics['context_utilization']=known_ratio(sum(x and y for x,y in zip(retrieved,r2a)),sum(retrieved))
+        if grounded is not None:metrics['faithfulness']=known_ratio(sum(grounded),len(grounded))
+        if grounded is not None and a2r is not None:
+            metrics['hallucination']=known_ratio(sum(not c and not s for c,s in zip(a2r,grounded)),len(a2r))
+            metrics['self_knowledge']=known_ratio(sum(c and not s for c,s in zip(a2r,grounded)),len(a2r))
+        if relevant is not None and d2r is not None and a2r is not None:
+            relevant_support=[any(row[i] for row,flag in zip(d2r,relevant) if flag) for i in range(len(a2r))]
+            irrelevant_support=[not relevant_support[i] and any(row[i] for row,flag in zip(d2r,relevant) if not flag) for i in range(len(a2r))]
+            for supports,name in [(relevant_support,'noise_sensitivity_in_relevant'),(irrelevant_support,'noise_sensitivity_in_irrelevant')]:
+                metrics[name]=known_ratio(sum(not c and s for c,s in zip(a2r,supports)),len(a2r))
+    return result('ragchecker', case, trace, corpus, metrics, judgments=judgments,
+                  notes=['Claims, chunk boundaries and directional entailments are retained inputs, not inferred from token overlap.'])
+
+
+def evaluate_ares(predictions, audit, *, declarations=None, confidence=0.95):
+    """ARES-style held human calibration and explicitly gated PPI arithmetic.
+
+    Training/inference belong to the captured judge adapter. This entry point
+    deliberately retains the independent-sample estimator's unvalidated-coverage
+    status; benchmark arithmetic does not establish Sable population coverage.
+    """
+    from ..rag_diagnostics import estimate_rag_ppi_mean
+    from .._rag_diagnostics_calibration import judge_audit
+    from .registry import paper_spec
+    estimate = estimate_rag_ppi_mean(predictions, audit, declarations=declarations, confidence=confidence)
+    audit_rows = [dict(id=r['id'], human_label=r.get('human_label'), judge_label=(r['prediction'] >= 0.5)
+                      if type(r.get('prediction')) in (int,float) and math.isfinite(r['prediction'])
+                      and 0<=r['prediction']<=1 else None) for r in audit]
+    return dict(protocol=paper_spec('ares'), estimate=estimate,
+        audit=judge_audit(audit_rows, provenance_kind='caller-supplied-independent-audit'),
+        training_performed=False, validated_population_coverage=False)
+
+
+def evaluate_trec_rag_2025(case, trace, corpus, *, judgments=(), cutoff=10):
+    """TREC RAG retrieval and nugget/attribution dimensions, never one score."""
+    from .scoring import evaluate_beir
+    metrics = {'retrieval.'+k:v for k,v in evaluate_beir(case,trace,corpus,cutoff=cutoff)['metrics'].items()}
+    labels = labels_for(judgments, case, trace, 'trec_rag_2025')
+    for field, metric in [('nuggets_covered','nugget_coverage'),('sentences_supported','sentence_attribution')]:
+        values = None if labels is None else labels.get(field)
+        native = case['references']['native'].get('nuggets') if field=='nuggets_covered' else trace['native'].get('sentence_citations')
+        if values is None or native is None:
+            metrics[metric]=measure(reason='Complete native annotation units and their judgments are required')
+        else:
+            require(type(native) is list and len(booleans(values))==len(native),'TREC judgment census differs from native units')
+            metrics[metric]=ratio(sum(values),len(values))
+    return result('trec_rag_2025', case, trace, corpus, metrics, judgments=judgments,parameters=dict(cutoff=cutoff),
+        notes=['Native narrative/nugget annotations stay protected. This is not a TREC submission or a claim of official leaderboard equivalence.'])
+
+
+def evaluate_lost_in_the_middle(case, trace, corpus, *, judgments=()):
+    """Score one frozen context-position trial; sweep orchestration is separate."""
+    from .scoring import answer_scores
+    refs = case['references']['answers']
+    metrics = {'answer_accuracy': measure(reason='Reference answer aliases unavailable')}
+    if refs is not None:
+        text = normalize_answer(trace['response']['text'])
+        metrics['answer_accuracy'] = measure(float(trace['response']['status']=='ok'
+            and any(normalize_answer(a) in text for a in refs)))
+    return result('lost_in_the_middle', case, trace, corpus, metrics,
+        notes=['Normalized answer-containing accuracy follows the QA sweep boundary; key-value retrieval is a distinct experiment.',
+               'Position, token budget, trial order and source content are bound by the sweep plan.'])

sphynx/rag_papers/trec.py

@@ -0,0 +1,80 @@
+"""TREC 2025 RAG native topic/segment and sentence-citation boundaries.
+
+See the organizer's Aug 13 2025 v2 guidelines. These format adapters do not
+reproduce AutoNuggetizer's model prompts or claim official leaderboard scores.
+"""
+from __future__ import annotations
+import copy
+
+from .contracts import digest, make_case, make_trace, parse, require, validate_trace
+from .datasets import _document, _read, _write_bundle
+
+
+def prepare_trec_rag_2025_dataset(topics_path, segments_path, output_dir, *,
+        topics_sha256, segments_sha256, qrels_path=None, qrels_sha256=None,
+        annotations=None, collection_scope):
+    """Load organizer id/title topic JSONL and docid/segment corpus JSONL.
+
+    Pin an explicit local corpus/shard. The 120M+ collection is never fetched
+    implicitly or represented as a complete collection when only a shard exists.
+    Supplied annotations are a protected id->native-object mapping, e.g. nuggets.
+    """
+    require(type(collection_scope) is str and collection_scope.strip(),'Declare the exact local collection scope')
+    topics=[parse(l) for l in _read(topics_path,topics_sha256).splitlines() if l.strip()]
+    segments=[parse(l) for l in _read(segments_path,segments_sha256).splitlines() if l.strip()]
+    corpus={}
+    for row in segments:
+        require(row['docid'] not in corpus,'Duplicate native segment ID')
+        _document(corpus,row['segment'],title=row['title'],key=row['docid'])
+    qrels={}
+    require((qrels_path is None)==(qrels_sha256 is None),'Qrels path and pin must be supplied together')
+    if qrels_path is not None:
+        for line in _read(qrels_path,qrels_sha256).decode('utf-8').splitlines():
+            if not line.strip():continue
+            q,iteration,d,g=line.split()
+            require(d in corpus and d not in qrels.setdefault(q,{}),'Qrel is duplicate or outside the declared collection')
+            qrels[q][d]=int(g)
+    annotations=annotations or {};require(type(annotations) is dict,'Native annotations must be keyed by topic ID')
+    seen=set();cases=[]
+    for row in topics:
+        qid=row['id'];require(qid not in seen,'Duplicate narrative ID');seen.add(qid)
+        cases.append(make_case(qid,row['title'],list(corpus),origin='public',snapshot=segments_sha256,
+            family='trec_rag_2025',source_id='trec_rag_2025',split='test',
+            references=dict(qrels=qrels.get(qid),native=dict(topic=row,collection_scope=collection_scope,
+                **copy.deepcopy(annotations.get(qid,{}))))))
+    require(set(qrels)<=seen and set(annotations)<=seen,'Native annotations reference absent topics')
+    return _write_bundle(corpus,cases,output_dir,dict(kind='trec-rag-2025-native',topics_sha256=topics_sha256,
+        segments_sha256=segments_sha256,qrels_sha256=qrels_sha256,annotations_sha256=digest(annotations),
+        collection_scope=collection_scope,official_autonuggetizer_executed=False))
+
+
+def trace_from_trec_rag_2025(case, corpus, record, *, candidate_id, source_sha256):
+    """Read native narrative/references/answer rows, preserving citation units.
+
+    Numeric citations are zero-based indices into references; string citations
+    are native segment IDs. The original output is retained unchanged.
+    """
+    require(record['metadata']['narrative_id']==case['id'] and record['metadata']['narrative']==case['question'],
+            'Native submission narrative differs')
+    references=record['references']
+    require(type(references) is list and len(references)<=100 and len(set(references))==len(references)
+            and set(references)<=set(case['document_ids']),'Invalid TREC reference inventory')
+    rows=record['answer'];require(type(rows) is list,'Native sentence array required')
+    units=[]
+    for row in rows:
+        require(type(row) is dict and set(row)=={'text','citations'} and type(row['text']) is str
+                and type(row['citations']) is list,'Invalid native TREC sentence')
+        ids=[]
+        for cite in row['citations']:
+            if type(cite) is int:
+                require(0<=cite<len(references),'Citation index outside references');did=references[cite]
+            else:
+                require(type(cite) is str and cite in references,'Citation segment outside references');did=cite
+            ids.append(did)
+        units.append(dict(text=row['text'],cited_ids=ids))
+    text=' '.join(r['text'] for r in units)
+    require(len(text.split())<=400,'Native response exceeds 400-word boundary')
+    trace=make_trace(case,corpus,text=text,ranked=references,prompt=references,
+                     candidate_id=candidate_id,source_digest=source_sha256)
+    trace['native']=dict(trec_rag_2025=copy.deepcopy(record),sentence_citations=units)
+    return validate_trace(trace,case,corpus)