sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/rag_papers/sable.py

@@ -0,0 +1,135 @@
+"""Explicit Sable/Reflex bridges. No question-text matching or invented gold."""
+from __future__ import annotations
+import copy
+
+from .contracts import (digest, make_case, make_trace, parse, require, validate_case,
+                        validate_trace)
+from .datasets import _document, _read, _write_bundle
+
+
+def prepare_sable_paper_dataset(source_path, output_dir, *, expected_sha256,
+                                 approval_receipt=None, synthetic=False):
+    """Preserve DatasetV3 history, grades, evidence groups and authority metadata.
+
+    Sable qrel grades stay 0..3. BEIR's >0 and Sable's configured relevance
+    threshold remain distinct: use evaluate_sable_evidence for Sable coverage.
+    Non-synthetic input needs an explicit owner-held usage receipt; possession
+    of a repository is not customer-data authorization.
+    """
+    from ..dataset_v3 import validate_dataset_v3
+    raw=_read(source_path,expected_sha256); native=parse(raw);validate_dataset_v3(native)
+    require(type(synthetic) is bool,'Explicit synthetic status required')
+    if not synthetic:
+        require(type(approval_receipt) is dict and set(approval_receipt)=={'owner','purpose','source_sha256'}
+                and approval_receipt['source_sha256']==expected_sha256
+                and all(type(v) is str and v.strip() for v in approval_receipt.values()),'Owner-approved source usage receipt required')
+    corpus={};cases=[]
+    for task in native['tasks']:
+        docs=[_document(corpus,c['input'],key=task['task_id']+':'+c['candidate_id']) for c in task['candidates']]
+        names={c['candidate_id']:d for c,d in zip(task['candidates'],docs)}
+        qrels={names[c['candidate_id']]:c['qrel'] for c in task['candidates'] if c['judgment_status']=='judged'}
+        required=[g for g in task['evidence_groups'] if g['requirement']=='required']
+        c=make_case(task['task_id'],task['query'],docs,
+            history=[dict(role=t['role'],content=t['text']) for t in task['conversation_context']],
+            snapshot=expected_sha256,origin='synthetic' if synthetic else 'owner-held',
+            family='sable.'+task['customer_id'],split=task['partition'],cluster_id=task['cluster_id'],source_id=native['dataset_id'],
+            context={'as_of':task['evaluation_time']},references=dict(qrels=qrels,
+                answerable=task['answerability']=='answerable',expected_action=task['expected_behavior'],
+                required_atoms=[g['evidence_group_id'] for g in required],
+                atom_support={g['evidence_group_id']:[names[i] for i in g['acceptable_candidate_ids']] for g in required},
+                native=dict(dataset_v3_task=task,candidate_id_map=names,sable_relevance_threshold=native['default_relevance_threshold'])))
+        cases.append(c)
+    return _write_bundle(corpus,cases,output_dir,dict(kind='sable-dataset-v3',source_sha256=expected_sha256,
+        synthetic=synthetic,approval_receipt=copy.deepcopy(approval_receipt),customer_validation_completed=False),source=raw)
+
+
+def evaluate_sable_evidence(case, trace, corpus):
+    """Sable adaptation: required evidence-group recall at each observed stage."""
+    from .scoring import ratio, measure
+    validate_case(case,corpus);validate_trace(trace,case,corpus)
+    required=case['references']['required_atoms'];support=case['references']['atom_support']
+    metrics={}
+    for stage, ranking in trace['stages'].items():
+        metrics[stage+'.required_group_recall']=measure(reason='Required evidence or observed stage unavailable')
+        if ranking is not None and required is not None and support is not None:
+            metrics[stage+'.required_group_recall']=ratio(sum(bool(set(support[a])&set(ranking)) for a in required),len(required))
+    return dict(protocol='sphynx.sable.evidence.v1',paper_parity=False,case_id=case['id'],
+        case_sha256=digest(case),trace_sha256=digest(trace),metrics=metrics)
+
+
+def bind_reflex_case_history(diagnostics_dataset, case_bank_path, *, expected_sha256, case_map):
+    """Join an already validated Reflex pairing to explicit original case IDs.
+
+    Every search and every bank case must be accounted for. A bank may have
+    multiple searches; no original question/history is replaced by a rewritten
+    query. Gold and generation stay unknown when only retrieval was observed.
+    """
+    from .._rag_diagnostics_contracts import validate_dataset
+    validated=validate_dataset(diagnostics_dataset)
+    rows=[parse(line) for line in _read(case_bank_path,expected_sha256).splitlines() if line.strip()]
+    bank={r['id']:r for r in rows};require(len(bank)==len(rows),'Duplicate Reflex case bank ID')
+    require(type(case_map) is dict and set(case_map)=={c['id'] for c in validated['cases']}
+            and set(case_map.values())==set(bank),'Explicit complete Reflex-to-bank mapping required')
+    corpus={}; cases=[]; traces={'baseline':[],'candidate':[]}
+    for old in validated['cases']:
+        row=bank[case_map[old['id']]]
+        require(old['question'] in row['rag_queries'],'Recorded retrieval query differs from its explicit case bank')
+        docs=[_document(corpus,d['text'],key=d['id']) for d in old['documents']]
+        c=make_case(old['id'],row['raw_user_question'],docs,history=row['conversation_history'],query=old['question'],
+            snapshot=validated['provenance']['baseline_sha256'],origin='recorded',family='sable.reflex',
+            cluster_id=old['cluster_id'],source_id='reflex-case-bank',native_id=row['id'],
+            references=dict(native={'case_bank_sha256':expected_sha256,'bank_row':row}))
+        cases.append(c)
+        for variant in traces:
+            oldtrace=old['traces'][variant]
+            t=make_trace(c,corpus,status='missing',attempt_id=oldtrace['id'],candidate_id=variant,
+                source_digest=digest(validated['provenance']['execution_identities'][variant]))
+            t['stages']=copy.deepcopy(oldtrace['stages'])
+            t['native']={'reflex':copy.deepcopy(oldtrace),'generation_observed':False}
+            traces[variant].append(validate_trace(t,c,corpus))
+    return dict(corpus=corpus,cases=cases,traces=traces,source_sha256=digest(validated),
+        answer_generation_observed=False,human_review_status='KIV')
+
+
+def trace_from_level3(case, corpus, response, *, candidate_id, source_sha256, candidate_id_map):
+    """Map an exact level-3 full-RAG response; reject query and evidence drift."""
+    require(response.get('schema_version')=='sable.rag.level3-trace.v1'
+            and response.get('task_id')==case['id'] and response.get('as_uttered_query')==case['question'],
+            'Full-RAG trace does not bind the original case')
+    from .._full_rag_trace import validate_level3_trace
+    raw=copy.deepcopy(response)
+    if 'expected_invocation' not in raw:raw['expected_invocation']=case['references']['expected_action']!='no_rag'
+    if raw.get('latency_ms')=={}:raw['latency_ms']={k:0.0 for k in ('routing','rewrite','retrieval','generation','total')}
+    parsed=validate_level3_trace(raw)
+    def mapped(values):
+        require(all(v in candidate_id_map for v in values),'Level-3 evidence is absent from the explicit ID map')
+        return [candidate_id_map[v] for v in values]
+    trace=make_trace(case,corpus,text=parsed.answer or '',ranked=mapped(parsed.retrieved_candidate_ids),
+        prompt=mapped(parsed.handed_context_ids),status='error' if response['call_outcome']=='error' else 'ok',
+        abstained=parsed.abstained,candidate_id=candidate_id,source_digest=source_sha256)
+    trace['rewritten_query']=parsed.rewritten_query
+    trace['operational']['tokens']=parsed.prompt_tokens+parsed.completion_tokens
+    trace['native']={'level3':copy.deepcopy(response)}
+    return validate_trace(trace,case,corpus)
+
+
+def trace_from_full_rag_capture(case, corpus, request, response, *, candidate_id, source_sha256, candidate_id_map):
+    """Verify retained full-RAG request/response digests and original history."""
+    from ..full_rag import validate_full_rag_response
+    from ..integration_contracts import value_digest
+    require(request['input_digest']==value_digest({k:v for k,v in request.items() if k!='input_digest'}),
+            'Retained full-RAG request digest differs')
+    require(request['case_id']==case['id'] and request['query']==case['question']
+            and request['conversation_context']==[dict(role=t['role'],text=t['content']) for t in case['conversation_history']],
+            'Retained full-RAG original question or history differs')
+    require(set(candidate_id_map.values())==set(case['document_ids']) and len(candidate_id_map)==len(case['document_ids']),
+            'Full-RAG evidence map differs from case inventory')
+    require({r['candidate_id'] for r in request['candidates']}==set(candidate_id_map)
+            and all(r['input']==corpus[candidate_id_map[r['candidate_id']]]['text'] for r in request['candidates']),
+            'Retained full-RAG document bytes differ')
+    validate_full_rag_response(response,request,expected_invocation=case['references']['expected_action']!='no_rag')
+    trace=trace_from_level3(case,corpus,response['trace'],candidate_id=candidate_id,
+                           source_sha256=source_sha256,candidate_id_map=candidate_id_map)
+    trace['native']['full_rag_request_sha256']=digest(request)
+    trace['native']['full_rag_response_sha256']=digest(response)
+    return validate_trace(trace,case,corpus)

sphynx/rag_papers/schemas.py

@@ -0,0 +1,12 @@
+"""Portable JSON Schema documents, independent of Python dataclasses."""
+import copy
+import json
+from importlib.resources import files
+from .contracts import require
+
+
+def get_paper_schema(kind):
+    require(kind in ('case','trace','judgment','metric_result','result'),'Unknown paper wire schema')
+    schema=json.loads(files('sphynx').joinpath('data/rag-paper-schemas.v1.json').read_text(encoding='utf-8'))
+    schema['$ref']='#/$defs/'+kind
+    return copy.deepcopy(schema)

sphynx/rag_papers/scoring.py

@@ -0,0 +1,249 @@
+"""Independent implementations of named papers' deterministic scoring rules."""
+from __future__ import annotations
+
+from collections import Counter
+import math
+import re
+import string
+
+from .contracts import (RESULT_SCHEMA, digest, require, validate_case,
+                        validate_judgment, validate_trace)
+from .registry import paper_spec
+
+
+def normalize_answer(text):
+    """SQuAD-family English normalization; deliberately not Unicode WER policy."""
+    require(type(text) is str, 'Answer must be text')
+    text = text.lower().translate(str.maketrans('', '', string.punctuation))
+    return ' '.join(re.sub(r'\b(a|an|the)\b', ' ', text).split())
+
+
+def answer_scores(prediction, references, *, categorical=False):
+    require(type(references) is list and all(type(x) is str for x in references), 'Invalid answer aliases')
+    if not references:
+        return dict(em=None, f1=None, precision=None, recall=None)
+    predicted = normalize_answer(prediction)
+    rows = []
+    for reference in references:
+        gold = normalize_answer(reference)
+        em = int(predicted == gold)
+        if categorical and (predicted in ('yes', 'no', 'noanswer') or gold in ('yes', 'no', 'noanswer')) and not em:
+            precision = recall = f1 = 0.0
+        else:
+            p, g = predicted.split(), gold.split()
+            overlap = sum((Counter(p) & Counter(g)).values())
+            precision = overlap / len(p) if p else 0.0
+            recall = overlap / len(g) if g else 0.0
+            f1 = 2 * precision * recall / (precision + recall) if overlap else 0.0
+        rows.append(dict(em=em, f1=f1, precision=precision, recall=recall))
+    # Official answer F1 chooses a complete reference comparison, not separately
+    # maximized precision and recall from incompatible references.
+    best = max(rows, key=lambda row: row['f1'])
+    return {**best, 'em': max(row['em'] for row in rows)}
+
+
+def set_scores(predicted, gold, *, empty_f1=False):
+    p, g = set(predicted), set(gold)
+    overlap = len(p & g)
+    precision = overlap / len(p) if p else 0.0
+    recall = overlap / len(g) if g else 0.0
+    return dict(em=float(p == g), precision=precision, recall=recall,
+                f1=2*precision*recall/(precision+recall) if overlap else float(empty_f1 and p == g))
+
+
+def measure(value=None, *, numerator=None, denominator=None, reason=None, unit='ratio'):
+    return dict(value=value, numerator=numerator, denominator=denominator, unit=unit,
+                status='available' if value is not None else 'unavailable', reason=reason)
+
+
+def ratio(numerator, denominator, reason='Empty eligible population'):
+    return measure(numerator/denominator if denominator else None,
+                   numerator=numerator, denominator=denominator,
+                   reason=None if denominator else reason)
+
+
+def result(paper, case, trace, corpus, metrics, *, judgments=(), notes=(), parameters=None):
+    validate_case(case, corpus); validate_trace(trace, case, corpus)
+    for row in judgments:
+        validate_judgment(row, case, trace)
+    return dict(schema_version=RESULT_SCHEMA, protocol=paper_spec(paper),
+                case_id=case['id'], case_sha256=digest(case), trace_sha256=digest(trace),
+                input_sha256=trace['input_sha256'], response_status=trace['response']['status'],
+                metrics=metrics, judgment_sha256=[digest(j) for j in judgments],
+                parameters={} if parameters is None else parameters,
+                judgment_origins=sorted({j['provenance']['origin'] for j in judgments}),
+                notes=list(notes), hosted_admitted=False)
+
+
+def labels_for(judgments, case, trace, task):
+    matches = [j for j in judgments if j['task'] == task]
+    require(len(matches) <= 1, 'Duplicate judgment for task')
+    if not matches:
+        return None
+    j = validate_judgment(matches[0], case, trace, task)
+    return j['labels'] if j['status'] == 'judged' else None
+
+
+def evaluate_beir(case, trace, corpus, *, cutoff=10, stage='retrieved'):
+    """BEIR/TREC linear nDCG, AP, recall, precision and MRR on native qrels.
+
+    Supplied unique order resolves ties. No-positive topics retain trec_eval's
+    zero convention, with their eligibility count separately available.
+    """
+    require(type(cutoff) is int and 1 <= cutoff <= 1_000_000, 'Invalid ranking cutoff')
+    require(stage in trace['stages'], 'Unknown ranking stage')
+    qrels, ranking = case['references']['qrels'], trace['stages'][stage]
+    names = ['ndcg', 'map', 'recall', 'precision', 'mrr', 'unjudged_fraction']
+    if qrels is None or ranking is None:
+        scores = {name+'@'+str(cutoff): measure(reason='Native qrels or recorded ranking unavailable') for name in names}
+    else:
+        top = ranking[:cutoff]
+        relevant = {d for d, grade in qrels.items() if grade > 0}
+        hits = [d in relevant for d in top]
+        found, total = sum(hits), len(relevant)
+        ideal = sum(grade/math.log2(rank+1) for rank, grade in enumerate(sorted(qrels.values(), reverse=True)[:cutoff], 1))
+        dcg = sum(qrels.get(d, 0)/math.log2(rank+1) for rank, d in enumerate(top, 1))
+        seen = 0; ap = 0.0
+        for rank, hit in enumerate(hits, 1):
+            seen += hit
+            if hit:
+                ap += seen/rank
+        first = next((rank for rank, hit in enumerate(hits, 1) if hit), None)
+        scores = {'ndcg': measure(dcg/ideal if ideal else 0.0, numerator=dcg, denominator=ideal),
+                  'map': measure(ap/total if total else 0.0, numerator=ap, denominator=total),
+                  'recall': measure(found/total if total else 0.0, numerator=found, denominator=total),
+                  'precision': ratio(found, cutoff), 'mrr': measure(1/first if first else 0.0),
+                  'unjudged_fraction': ratio(sum(d not in qrels for d in top), len(top))}
+        scores = {name+'@'+str(cutoff): row for name, row in scores.items()}
+    scores['known_positive_count'] = measure(None if qrels is None else sum(g>0 for g in qrels.values()),unit='documents')
+    return result('beir', case, trace, corpus, scores, parameters=dict(cutoff=cutoff,stage=stage),
+                  notes=['Absent qrels remain unknown; zero gain is a ranking projection, not a new negative judgment.'])
+
+
+def _qa(case, trace, *, hotpot=False):
+    answer = case['references']['answers']
+    if answer is None:
+        return {key: measure(reason='Reference answers unavailable') for key in ('answer_em','answer_f1','answer_precision','answer_recall')}
+    raw = answer_scores(trace['response']['text'], answer, categorical=hotpot)
+    if trace['response']['status'] != 'ok':
+        raw = dict.fromkeys(raw, 0.0)
+    return {'answer_'+key: measure(value) for key, value in raw.items()}
+
+
+def evaluate_hotpotqa(case, trace, corpus):
+    """Official answer/support/joint arithmetic on title+sentence-index units."""
+    metrics = _qa(case, trace, hotpot=True)
+    gold, pred = case['references']['support_facts'], trace['response']['support_facts']
+    if gold is not None and pred is not None:
+        def facts(rows):
+            require(all(type(r) is list and len(r) == 2 and type(r[0]) is str
+                        and type(r[1]) is int and r[1] >= 0 for r in rows), 'Invalid supporting fact unit')
+            return [tuple(r) for r in rows]
+        support = set_scores(facts(pred), facts(gold))
+        if trace['response']['status'] != 'ok':
+            support = dict.fromkeys(support, 0.0)
+        metrics.update({'support_'+k: measure(v) for k, v in support.items()})
+        p = metrics.get('answer_precision', {}).get('value')
+        r = metrics.get('answer_recall', {}).get('value')
+        jp = None if p is None else p * support['precision']
+        jr = None if r is None else r * support['recall']
+        metrics['joint_em'] = measure(None if p is None else metrics['answer_em']['value'] * support['em'])
+        metrics['joint_f1'] = measure(None if jp is None else 2*jp*jr/(jp+jr) if jp+jr else 0.0)
+        metrics['joint_precision']=measure(jp);metrics['joint_recall']=measure(jr)
+    else:
+        metrics.update({name: measure(reason='Sentence support annotation/prediction unavailable')
+                        for name in ('support_em','support_f1','support_precision','support_recall',
+                                     'joint_em','joint_f1','joint_precision','joint_recall')})
+    return result('hotpotqa', case, trace, corpus, metrics)
+
+
+def evaluate_musique(case, trace, corpus):
+    """MuSiQue answer aliases and supporting-paragraph EM/F1; retain hop graph."""
+    metrics = _qa(case, trace)
+    gold, pred = case['references']['support_ids'], trace['response']['support_ids']
+    if gold is not None and pred is not None:
+        values = set_scores(pred, gold, empty_f1=True)
+        if trace['response']['status'] != 'ok':
+            values = dict.fromkeys(values, 0.0)
+        metrics.update({'support_'+k: measure(v) for k, v in values.items()})
+    else:
+        metrics.update({k: measure(reason='Paragraph support annotation/prediction unavailable') for k in ('support_em','support_f1','support_precision','support_recall')})
+    expected = case['references']['answerable']
+    metrics['answerability_accuracy'] = measure(None if expected is None else
+        float(trace['response']['status'] == 'ok' and (not trace['response']['abstained']) == expected))
+    required=case['references']['required_atoms'];supports=case['references']['atom_support'];prompt=trace['stages']['prompt']
+    metrics['adaptation.required_hop_context_recall']=measure(reason='Complete hop annotations or prompt unavailable')
+    if required and supports is not None and prompt is not None:
+        metrics['adaptation.required_hop_context_recall']=ratio(sum(bool(set(supports[a])&set(prompt)) for a in required),len(required))
+    return result('musique', case, trace, corpus, metrics,
+                  notes=['Native question decomposition is retained; supporting two unrelated facts does not establish connected reasoning.',
+                         'The adaptation-prefixed metric records hop evidence availability. Use plan_musique for actual leave-one-hop-out execution.'])
+
+
+def evaluate_qasper(case, trace, corpus):
+    """Best human-reference answer/evidence F1, with independent denominators."""
+    annotations = case['references']['annotation_sets']
+    metrics = {}
+    if annotations is None or not annotations:
+        metrics = {k: measure(reason='Human QASPER annotation sets unavailable') for k in ('answer_f1','evidence_f1')}
+    else:
+        require(all(type(a) is dict and set(a) == {'answer', 'evidence_ids'}
+                    and type(a['answer']) is str and (a['evidence_ids'] is None or type(a['evidence_ids']) is list) for a in annotations),
+                'Invalid QASPER reference set')
+        metrics['answer_f1'] = measure(max(answer_scores(trace['response']['text'], [a['answer']])['f1']
+            if normalize_answer(trace['response']['text']) and normalize_answer(a['answer']) else 0.0 for a in annotations))
+        pred = trace['response']['support_ids']
+        known = pred is not None and all(a['evidence_ids'] is not None for a in annotations)
+        def evidence_f1(gold):
+            # QASPER identifies evidence by paragraph text, retaining list lengths
+            # in its denominator even when identical paragraphs occur twice.
+            p = [corpus[d]['text'] for d in pred]
+            g = [corpus[d]['text'] for d in gold]
+            overlap = len(set(p) & set(g))
+            return 2*overlap/(len(p)+len(g)) if p or g else 1.0
+        metrics['evidence_f1'] = measure(max(evidence_f1(a['evidence_ids']) for a in annotations) if known else None,
+            reason=None if known else 'Predicted evidence or a complete native reference mapping unavailable')
+        if trace['response']['status'] != 'ok':
+            metrics = {k: measure(0.0) for k in metrics}
+    return result('qasper', case, trace, corpus, metrics,
+                  notes=['Answer and evidence can maximize over different human annotations, matching the reference evaluator.'])
+
+
+def evaluate_rgb(case, trace, corpus, *, judgments=()):
+    """RGB's all-required answer-group substring match, separately per testbed."""
+    native = case['references']['native']; groups = native.get('answer_groups')
+    metrics = {}
+    if groups is None:
+        metrics['accuracy'] = measure(reason='Native RGB answer groups unavailable')
+    else:
+        require(type(groups) is list and groups and all(type(g) is list and g
+                and all(type(a) is str and a for a in g) for g in groups), 'Invalid RGB answer groups')
+        text = trace['response']['text'].lower()
+        success = all(any(alias.lower() in text for alias in group) for group in groups)
+        metrics['accuracy'] = measure(float(success and trace['response']['status'] == 'ok'))
+    labels = labels_for(judgments, case, trace, 'rgb')
+    for name in ('rejection','error_detected','error_corrected'):
+        value = None if labels is None else labels.get(name)
+        require(value is None or type(value) is bool, 'Invalid RGB semantic verdict')
+        metrics[name] = measure(None if value is None else float(value), reason='Captured testbed-specific judgment required' if value is None else None)
+    return result('rgb', case, trace, corpus, metrics, judgments=judgments,
+        notes=['Substring matching is RGB compatibility, not an independent factuality judgment.',
+               'Scenario testbeds are never silently pooled into one robustness score.'])
+
+
+def evaluate_crag(case, trace, corpus, *, judgments=()):
+    """CRAG utility over exact retained verdicts: correct=1, abstain=0, wrong=-1."""
+    labels = labels_for(judgments, case, trace, 'crag.correctness')
+    if trace['response']['status'] != 'ok':
+        verdict = 'incorrect'
+    elif trace['response']['abstained']:
+        verdict = 'abstain'
+    elif labels is None:
+        verdict = None
+    else:
+        verdict = labels.get('verdict')
+        require(verdict in ('correct', 'incorrect', 'abstain'), 'Invalid CRAG verdict')
+    score = None if verdict is None else {'correct': 1.0, 'abstain': 0.0, 'incorrect': -1.0}[verdict]
+    return result('crag', case, trace, corpus, {'utility': measure(score, unit='score',
+        reason='A captured correctness verdict is required' if score is None else None)}, judgments=judgments,
+        notes=['This implements the scoring rule, not an assertion that a supplied/model verdict is true.'])