sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/rag_papers/__init__.py

@@ -0,0 +1,29 @@
+"""Named research protocols for the separate, local rag.papers.v1 suite.
+
+Import specific paper names to make a call's evaluation approach visible.
+These SDK 0.5.0 submodule APIs preserve the existing root exports and do not imply
+official-framework parity, human calibration, or hosted-suite admission.
+"""
+from .contracts import (PaperEvaluationError, make_case, make_trace, judgment,
+                        candidate_input, validate_case, validate_trace, validate_judgment)
+from .datasets import (PaperDataset, prepare_paper_dataset, prepare_official_paper_dataset,
+                       prepare_canonical_paper_dataset, load_paper_dataset)
+from .registry import describe_paper_methods
+from .scoring import (evaluate_beir, evaluate_hotpotqa, evaluate_musique,
+                      evaluate_qasper, evaluate_rgb, evaluate_crag)
+from .semantic import (evaluate_sufficient_context, evaluate_ragas, evaluate_alce,
+                       evaluate_ares, evaluate_factscore, evaluate_ragchecker,
+                       evaluate_lost_in_the_middle, evaluate_trec_rag_2025)
+from .judges import JsonJudgeAdapter, JudgeSession, run_paper_judge, prepare_ares_training, train_ares_judge
+from .candidates import candidate_trace
+from .runner import (run_paper_suite, load_paper_run, compare_paper_runs, capture_paper_candidate)
+from .experiments import plan_lost_in_the_middle, plan_rgb, plan_musique, prepare_intervention_dataset
+from .sable import (prepare_sable_paper_dataset, evaluate_sable_evidence,
+                    bind_reflex_case_history, trace_from_level3, trace_from_full_rag_capture)
+from .trec import prepare_trec_rag_2025_dataset, trace_from_trec_rag_2025
+from .schemas import get_paper_schema
+
+__all__ = [name for name in globals() if name.startswith(('evaluate_', 'prepare_', 'validate_', 'make_', 'plan_'))] + [
+    'PaperEvaluationError','PaperDataset','judgment','candidate_input','describe_paper_methods','load_paper_dataset',
+    'JsonJudgeAdapter','JudgeSession','run_paper_judge','run_paper_suite','load_paper_run','candidate_trace','train_ares_judge',
+    'compare_paper_runs','capture_paper_candidate','bind_reflex_case_history','trace_from_level3','trace_from_full_rag_capture','trace_from_trec_rag_2025','get_paper_schema']

sphynx/rag_papers/__main__.py

@@ -0,0 +1,45 @@
+"""A distinct invocation: python -m sphynx.rag_papers --help."""
+from __future__ import annotations
+import argparse
+import json
+from pathlib import Path
+
+from .contracts import PaperEvaluationError, parse
+from .datasets import prepare_paper_dataset, prepare_official_paper_dataset, load_paper_dataset
+from .registry import describe_paper_methods
+from .runner import run_paper_suite
+
+
+def main(argv=None):
+    parser=argparse.ArgumentParser(description='Source-only rag.papers.v1; offline intake and grading, no implicit model execution')
+    sub=parser.add_subparsers(dest='command',required=True)
+    sub.add_parser('methods')
+    native=sub.add_parser('prepare')
+    native.add_argument('paper', choices=('beir','hotpotqa','musique','qasper','alce','rgb','crag','lost_in_the_middle'))
+    native.add_argument('source');native.add_argument('output');native.add_argument('--sha256',required=True)
+    native.add_argument('--source-url');native.add_argument('--split',default='development');native.add_argument('--scenario')
+    bridge=sub.add_parser('bridge-official');bridge.add_argument('source');bridge.add_argument('output');bridge.add_argument('--manifest-sha256',required=True)
+    grade=sub.add_parser('grade');grade.add_argument('dataset');grade.add_argument('traces');grade.add_argument('output')
+    grade.add_argument('--manifest-sha256',required=True);grade.add_argument('--methods',nargs='+',required=True)
+    grade.add_argument('--candidate',required=True);grade.add_argument('--source-sha256',required=True)
+    grade.add_argument('--judgments')
+    args=parser.parse_args(argv)
+    def lines(path):
+        file=Path(path)
+        if file.stat().st_size>200_000_000:raise PaperEvaluationError('Evidence file exceeds limit')
+        return [parse(line) for line in file.read_bytes().splitlines() if line.strip()]
+    try:
+        if args.command=='methods':out=describe_paper_methods()
+        elif args.command=='prepare':out=prepare_paper_dataset(args.paper,args.source,args.output,expected_sha256=args.sha256,
+            source_url=args.source_url,split=args.split,scenario=args.scenario)
+        elif args.command=='bridge-official':out=prepare_official_paper_dataset(args.source,args.output,expected_manifest_sha256=args.manifest_sha256)
+        else:
+            dataset=load_paper_dataset(args.dataset,expected_manifest_sha256=args.manifest_sha256)
+            out=run_paper_suite(dataset,lines(args.traces),args.output,methods=args.methods,
+                judgments=[] if not args.judgments else lines(args.judgments),candidate_id=args.candidate,source_sha256=args.source_sha256)
+        print(json.dumps(out,indent=2,allow_nan=False));return 0
+    except (ValueError,OSError,KeyError,TypeError):
+        parser.exit(2,'Paper evaluation failed: invalid inputs, identities or output path; inspect the pinned input contract.\n')
+
+
+if __name__=='__main__':raise SystemExit(main())

sphynx/rag_papers/candidates.py

@@ -0,0 +1,19 @@
+"""Candidate-side trace construction using only the projected public input."""
+from .contracts import STAGES, TRACE_SCHEMA, digest, exact, require, validate_trace
+
+
+def candidate_trace(payload, *, candidate_id, source_sha256, text='', ranked=None,
+                    prompt=None, status='ok', abstained=False, attempt_id='attempt-1'):
+    """Build a return value inside a trusted candidate without receiving gold."""
+    exact(payload,('case_id','question','conversation_history','retrieval_query','context','corpus_snapshot_id','documents'), 'candidate input')
+    permitted={d['id'] for d in payload['documents']}
+    trace=dict(schema_version=TRACE_SCHEMA,case_id=payload['case_id'],attempt_id=attempt_id,
+        input_sha256=digest(payload),candidate=dict(id=candidate_id,source_sha256=source_sha256),
+        rewritten_query=payload['retrieval_query'],native={},stages=dict.fromkeys(STAGES),
+        response=dict(status=status,text=text,abstained=abstained,support_ids=None,support_facts=None),
+        operational=dict(tokens=None,cost_usd=None))
+    trace['stages'].update(retrieved=ranked,prompt=prompt)
+    validate_trace(trace)
+    for stage in trace['stages'].values():
+        require(stage is None or set(stage)<=permitted,'Candidate trace uses unavailable documents')
+    return trace

sphynx/rag_papers/contracts.py

@@ -0,0 +1,282 @@
+"""Paper evaluation wire contracts. Native gold remains separate from inputs."""
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import math
+from pathlib import Path
+
+CASE_SCHEMA = 'sphynx.rag.case.v1'
+TRACE_SCHEMA = 'sphynx.rag.trace.v1'
+JUDGMENT_SCHEMA = 'sphynx.rag.judgment.v1'
+RESULT_SCHEMA = 'sphynx.rag.paper-result.v1'
+STAGES = ('indexed', 'retrieved', 'reranked', 'selected', 'prompt')
+REFERENCE_FIELDS = ('answers', 'qrels', 'support_ids', 'support_facts', 'answerable',
+                    'expected_action', 'required_atoms', 'atom_support',
+                    'required_relations', 'relation_support', 'annotation_sets', 'native')
+MAX_BYTES = 200_000_000
+
+
+class PaperEvaluationError(ValueError):
+    """A paper protocol's data, identity or execution boundary is invalid."""
+
+
+def require(condition, message):
+    if not condition:
+        raise PaperEvaluationError(message)
+
+
+def canonical(value):
+    try:
+        return json.dumps(value, ensure_ascii=False, sort_keys=True,
+                          separators=(',', ':'), allow_nan=False).encode('utf-8')
+    except (ValueError, TypeError, RecursionError):
+        raise PaperEvaluationError('Expected bounded finite JSON') from None
+
+
+def digest(value):
+    return hashlib.sha256(canonical(value)).hexdigest()
+
+
+def parse(raw):
+    require(len(raw) <= MAX_BYTES, 'JSON exceeds the local byte limit')
+    def pairs(items):
+        out = {}
+        for key, value in items:
+            require(key not in out, 'Duplicate JSON object key')
+            out[key] = value
+        return out
+    try:
+        result = json.loads(raw, object_pairs_hook=pairs,
+                            parse_constant=lambda _: require(False, 'Nonfinite JSON value'))
+        canonical(result)
+        return result
+    except (ValueError, UnicodeError, RecursionError):
+        raise PaperEvaluationError('Invalid finite UTF-8 JSON') from None
+
+
+def exact(value, fields, where):
+    require(type(value) is dict and set(value) == set(fields), where + ' fields differ')
+
+
+def identity(value):
+    require(type(value) is str and 0 < len(value) <= 256
+            and not any(ord(c) < 32 for c in value), 'Invalid record identity')
+    return value
+
+
+def checksum(value):
+    require(type(value) is str and len(value) == 64
+            and all(c in '0123456789abcdef' for c in value), 'Expected lowercase SHA256')
+    return value
+
+
+def ids(values):
+    require(type(values) is list and len(values) <= 1_000_000, 'Expected bounded ID list')
+    for value in values:
+        identity(value)
+    require(len(set(values)) == len(values), 'Duplicate record identity')
+    return set(values)
+
+
+def facts(values):
+    require(values is None or type(values) is list and all(type(row) is list and len(row)==2
+        and type(row[0]) is str and type(row[1]) is int and row[1]>=0 for row in values), 'Invalid title/sentence-index supporting facts')
+
+
+def make_case(case_id, question, document_ids, *, references=None, history=(), query=None,
+              snapshot='unspecified', origin='synthetic', family='unspecified', split='development',
+              cluster_id=None, native_id=None, source_id='caller-supplied', context=None):
+    refs = dict.fromkeys(REFERENCE_FIELDS)
+    refs['native'] = {}
+    if references:
+        require(set(references) <= set(refs), 'Unknown reference fields')
+        refs.update(copy.deepcopy(references))
+    value = dict(schema_version=CASE_SCHEMA, id=case_id, question=question,
+                 conversation_history=copy.deepcopy(list(history)), retrieval_query=query,
+                 context=copy.deepcopy(context or {}),
+                 corpus_snapshot_id=snapshot, document_ids=list(document_ids), references=refs,
+                 metadata=dict(origin=origin, scenario_family=family, split=split,
+                               cluster_id=cluster_id or case_id, source_id=source_id,
+                               native_id=native_id or case_id))
+    return validate_case(value)
+
+
+def validate_case(value, corpus=None):
+    exact(value, ('schema_version', 'id', 'question', 'conversation_history', 'retrieval_query',
+                  'context', 'corpus_snapshot_id', 'document_ids', 'references', 'metadata'), 'case')
+    require(value['schema_version'] == CASE_SCHEMA, 'Unsupported RAG case schema')
+    identity(value['id']); identity(value['corpus_snapshot_id'])
+    require(type(value['question']) is str and 0 < len(value['question']) <= 100_000,
+            'Original question is required')
+    require(value['retrieval_query'] is None or type(value['retrieval_query']) is str,
+            'Retrieval query must be text or unknown')
+    require(type(value['context']) is dict and set(value['context']) <= {'as_of', 'locale'}
+            and all(type(v) is str for v in value['context'].values()), 'Invalid public task context')
+    require(type(value['conversation_history']) is list and len(value['conversation_history']) <= 1000,
+            'Invalid conversation history')
+    for turn in value['conversation_history']:
+        exact(turn, ('role', 'content'), 'history turn')
+        require(turn['role'] in ('system', 'user', 'assistant', 'tool')
+                and type(turn['content']) is str, 'Invalid history turn')
+    documents = ids(value['document_ids'])
+    if corpus is not None:
+        require(all(d in corpus for d in documents), 'Case references absent corpus documents')
+    meta = value['metadata']
+    exact(meta, ('origin', 'scenario_family', 'split', 'cluster_id', 'source_id', 'native_id'), 'metadata')
+    require(meta['origin'] in ('synthetic', 'public', 'owner-held', 'recorded'), 'Invalid origin')
+    for field in meta:
+        identity(meta[field])
+    refs = value['references']; exact(refs, REFERENCE_FIELDS, 'references')
+    # A known source may be absent from a particular candidate's context. Gold
+    # identities bind the shared corpus, not only documents delivered to it.
+    reference_universe = corpus
+    for field in ('answerable',):
+        require(refs[field] is None or type(refs[field]) is bool, 'Invalid answerability label')
+    require(refs['expected_action'] in (None, 'answer', 'abstain', 'clarify', 'no_rag'),
+            'Invalid expected action')
+    for field in ('answers', 'support_ids', 'required_atoms', 'required_relations'):
+        require(refs[field] is None or type(refs[field]) is list
+                and all(type(x) is str for x in refs[field]), 'Invalid reference text/ID list')
+    for field in ('support_ids',):
+        if refs[field] is not None:
+            support = ids(refs[field])
+            require(reference_universe is None or all(d in reference_universe for d in support), 'Gold support outside frozen corpus')
+    if refs['qrels'] is not None:
+        require(type(refs['qrels']) is dict
+                and all(type(v) is int and 0 <= v <= 100 for v in refs['qrels'].values()),
+                'Invalid native qrels')
+        keys=ids(list(refs['qrels']))
+        require(reference_universe is None or all(d in reference_universe for d in keys), 'Qrel outside frozen corpus')
+    require(type(refs['native']) is dict, 'Native annotations must be an object')
+    for field in ('atom_support', 'relation_support'):
+        if refs[field] is not None:
+            require(type(refs[field]) is dict, 'Invalid evidence mapping')
+            for required, support in refs[field].items():
+                identity(required)
+                support_ids=ids(support)
+                require(reference_universe is None or all(d in reference_universe for d in support_ids), 'Evidence support outside frozen corpus')
+    for field in ('support_facts', 'annotation_sets'):
+        require(refs[field] is None or type(refs[field]) is list, 'Invalid annotation list')
+    facts(refs['support_facts'])
+    if refs['annotation_sets'] is not None:
+        for annotation in refs['annotation_sets']:
+            exact(annotation,('answer','evidence_ids'),'answer annotation')
+            require(type(annotation['answer']) is str,'Invalid annotated answer')
+            if annotation['evidence_ids'] is not None:
+                evidence=ids(annotation['evidence_ids'])
+                require(reference_universe is None or all(d in reference_universe for d in evidence),'Annotated evidence outside corpus')
+    canonical(value)
+    return value
+
+
+def candidate_input(case, corpus):
+    validate_case(case, corpus)
+    # This projection is deliberately built field by field, including corpus
+    # rows. Native annotations and accidental extra corpus fields never enter it.
+    return copy.deepcopy(dict(case_id=case['id'], question=case['question'],
+        conversation_history=case['conversation_history'], retrieval_query=case['retrieval_query'],
+        context=case['context'],
+        corpus_snapshot_id=case['corpus_snapshot_id'], documents=[
+            dict(id=did, text=corpus[did]['text'], title=corpus[did].get('title', ''))
+            for did in case['document_ids']]))
+
+
+def make_trace(case, corpus, *, text='', ranked=None, prompt=None, status='ok', abstained=False,
+               attempt_id='attempt-1', candidate_id='fixture', source_digest=None,
+               support_ids=None, support_facts=None):
+    value = dict(schema_version=TRACE_SCHEMA, case_id=case['id'], attempt_id=attempt_id,
+        rewritten_query=case['retrieval_query'], native={},
+        input_sha256=digest(candidate_input(case, corpus)), candidate=dict(id=candidate_id,
+        source_sha256=source_digest or digest({'fixture': candidate_id})),
+        stages=dict.fromkeys(STAGES), response=dict(status=status, text=text, abstained=abstained,
+        support_ids=support_ids, support_facts=support_facts), operational=dict(tokens=None, cost_usd=None))
+    value['stages']['retrieved'] = ranked
+    value['stages']['prompt'] = prompt
+    return validate_trace(value, case, corpus)
+
+
+def validate_trace(value, case=None, corpus=None):
+    exact(value, ('schema_version', 'case_id', 'attempt_id', 'input_sha256', 'candidate', 'rewritten_query', 'native',
+                  'stages', 'response', 'operational'), 'trace')
+    require(value['schema_version'] == TRACE_SCHEMA, 'Unsupported RAG trace schema')
+    identity(value['case_id']); identity(value['attempt_id']); checksum(value['input_sha256'])
+    require(value['rewritten_query'] is None or type(value['rewritten_query']) is str, 'Invalid captured rewritten query')
+    require(type(value['native']) is dict, 'Native trace capture must be an object')
+    exact(value['candidate'], ('id', 'source_sha256'), 'candidate')
+    identity(value['candidate']['id']); checksum(value['candidate']['source_sha256'])
+    exact(value['stages'], STAGES, 'trace stages')
+    previous = None
+    for stage in STAGES:
+        current = value['stages'][stage]
+        if current is None:
+            continue
+        current = ids(current)
+        if previous is not None:
+            require(current <= previous, 'Later stage invents documents')
+        if case is not None:
+            require(current <= set(case['document_ids']), 'Trace uses unavailable documents')
+        previous = current
+    response = value['response']
+    exact(response, ('status', 'text', 'abstained', 'support_ids', 'support_facts'), 'response')
+    require(response['status'] in ('ok', 'error', 'timeout', 'missing'), 'Unknown response status')
+    require(type(response['text']) is str and type(response['abstained']) is bool, 'Invalid response')
+    if response['support_ids'] is not None:
+        support = ids(response['support_ids'])
+        if case is not None:
+            require(support <= set(case['document_ids']), 'Predicted support outside case')
+    require(response['support_facts'] is None or type(response['support_facts']) is list,
+            'Invalid predicted supporting facts')
+    facts(response['support_facts'])
+    exact(value['operational'], ('tokens', 'cost_usd'), 'operations')
+    for item in value['operational'].values():
+        require(item is None or type(item) in (int, float) and math.isfinite(item) and item >= 0,
+                'Invalid recorded usage')
+    if case is not None:
+        require(value['case_id'] == case['id'], 'Trace case identity differs')
+        if corpus is not None:
+            require(value['input_sha256'] == digest(candidate_input(case, corpus)), 'Trace inputs changed')
+    canonical(value)
+    return value
+
+
+def judgment(case, trace, task, labels, *, origin='fixture', model='fixture', rubric='v1'):
+    value = dict(schema_version=JUDGMENT_SCHEMA, case_id=case['id'], trace_sha256=digest(trace),
+        input_sha256=trace['input_sha256'], task=task, labels=copy.deepcopy(labels),
+        provenance=dict(origin=origin, model=model, rubric=rubric), status='judged',
+        usage=dict(tokens=None, cost_usd=None))
+    validate_judgment(value, case, trace, task)
+    return value
+
+
+def validate_judgment(value, case=None, trace=None, task=None):
+    exact(value, ('schema_version', 'case_id', 'trace_sha256', 'input_sha256', 'task',
+                  'labels', 'provenance', 'status', 'usage'), 'judgment')
+    require(value['schema_version'] == JUDGMENT_SCHEMA, 'Unknown judgment schema')
+    identity(value['case_id']); checksum(value['trace_sha256']); checksum(value['input_sha256'])
+    require((case is None) == (trace is None), 'Supply both case and trace to bind a review')
+    if case is not None:
+        require(value['case_id'] == case['id'] and value['trace_sha256'] == digest(trace)
+                and value['input_sha256'] == trace['input_sha256'], 'Judgment does not bind the exact case/trace inputs')
+    require(task is None or value['task'] == task, 'Judgment task differs')
+    identity(value['task']); require(type(value['labels']) is dict, 'Judgment labels must be an object')
+    exact(value['provenance'], ('origin', 'model', 'rubric'), 'judgment provenance')
+    require(value['provenance']['origin'] in ('fixture', 'model', 'human'), 'Invalid review origin')
+    for field in ('model', 'rubric'):
+        identity(value['provenance'][field])
+    require(value['status'] in ('judged', 'abstained', 'error', 'timeout'), 'Invalid judgment status')
+    exact(value['usage'], ('tokens', 'cost_usd'), 'judgment usage')
+    for item in value['usage'].values():
+        require(item is None or type(item) in (int, float) and math.isfinite(item) and item >= 0,
+                'Invalid judgment usage')
+    canonical(value)
+    return value
+
+
+def save_json(value, path):
+    target = Path(path)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    with target.open('xb') as handle:
+        handle.write(canonical(value) + b'\n')
+    return hashlib.sha256(target.read_bytes()).hexdigest()