apsimo 1.3.0__py3-none-any.whl

apsimo/qualification/memory_cases.py

@@ -0,0 +1,209 @@
+"""Finite cases using actual memory formation and scoped lexical recollection.
+
+The candidate serves extraction. The configured judging role is a supporting
+consumer dependency, recorded by the runner; no role is silently rebound here.
+These are not native conversation or base-Hermes comparison results.
+"""
+from contextlib import closing
+from datetime import datetime
+import json
+
+from .records import CaseSpec
+
+
+def _recollect(projection, query, *, contact, session, now, max_chars):
+    from apsimo.beliefs.source_time import interpret_time_query
+    from apsimo.intelligence.graph.recall import pack_memory_context
+    from apsimo.turns.source_annotations import expand, current_candidates
+    ledger = projection.ledger
+    hits = ledger.search_sources(query, contact_id=contact, session_id=session, limit=10)
+    _, rows = projection.prepare_context([], hits, contact_id=contact, session_id=session,
+        time_query=interpret_time_query(query, now=datetime.fromisoformat(now), timezone_name='UTC'))
+    rows = expand(ledger, rows, contact_id=contact, session_id=session)
+    rows = current_candidates(ledger, rows, contact_id=contact, session_id=session)
+    selected, body = pack_memory_context(rows, limit=5, max_chars=max_chars)
+    refs = {ref['source_id']: ref for row in selected for ref in row.get('_annotation_source_refs', [])}
+    return {'body': body, 'selected': selected, 'citations': list(refs.values())}
+
+
+async def source_memory(inputs, context):
+    """One projection attempt per input, then reopen the same isolated ledger."""
+    from apsimo.beliefs.source_projection import SourceClaimProjection
+    from apsimo.turns import TurnIdempotencyLedger
+    turns = inputs['turns']
+    if not 1 <= len(turns) <= 8 or len({t['id'] for t in turns}) != len(turns):
+        raise ValueError('Memory qualification needs1..8 distinct declared sources')
+    ledger = TurnIdempotencyLedger(context.state_dir/'turn-idempotency.db')
+    projection = SourceClaimProjection(ledger)
+    recorded, process_calls = [], 0
+    for turn in turns:
+        if not isinstance(turn['text'], str) or not 1 <= len(turn['text']) <= 2000:
+            raise ValueError('Source input must be bounded text')
+        assert ledger.record_source(turn['id'], contact_id=inputs['contact_id'],
+            session_id=turn['session_id'], occurred_at=turn['occurred_at'],
+            messages=[{'role': 'user', 'content': turn['text']}])
+        recorded.append(turn['id'])
+        process_calls += 1
+        await projection.process_one(context.router)
+        job = next(row for row in projection.status(inputs['contact_id']) if row['turn_id'] == turn['id'])
+        if job['status'] != 'complete':
+            break  # Retain the failed first attempt; never wait/retry its queue.
+    refs_before = ledger.source_references(recorded, contact_id=inputs['contact_id'],
+                                           session_id=inputs['recall_session'])
+    reopened = SourceClaimProjection(TurnIdempotencyLedger(ledger.db_path))
+    refs_after = reopened.ledger.source_references(recorded, contact_id=inputs['contact_id'],
+                                                  session_id=inputs['recall_session'])
+    with closing(reopened.ledger._connect()) as db:
+        claims = [dict(json.loads(row['data_json']), **{k: row[k] for k in (
+            'id', 'turn_id', 'message_hash', 'retracted_by', 'superseded_by')})
+            for row in db.execute('SELECT * FROM source_claims ORDER BY turn_id,id')]
+        sources = [{'id': row['turn_id'], 'contact_id': row['contact_id'], 'session_id': row['session_id'],
+                    'messages': json.loads(row['messages_json'])}
+                   for row in db.execute('SELECT * FROM turn_sources ORDER BY turn_id')]
+    recall = _recollect(reopened, inputs['query'], contact=inputs['contact_id'],
+        session=inputs['recall_session'], now=inputs['now'], max_chars=inputs['max_chars'])
+    effects = {'recorded_source_ids': recorded, 'process_invocations': process_calls,
+               'reopened_source_versions_equal': refs_before == refs_after,
+               'source_references': refs_after, 'recall_session': inputs['recall_session'],
+               'supporting_roles': ['judging'],
+               'boundary': 'canonical_source_formation_and_lexical_recollection',
+               'native_request_and_answer': 'not_exercised',
+               'semantic_embedding_and_reranking': 'not_exercised'}
+    context.observe({'boundary': effects['boundary'], 'sources_recorded': len(recorded),
+                     'projection_invocations': process_calls, 'supporting_roles': ['judging']})
+    return {'output': {'sources': sources, 'claims': claims,
+                       'jobs': reopened.status(inputs['contact_id']), 'recall': recall}, 'effects': effects}
+
+
+def _same_text(left, right):
+    return isinstance(left, str) and isinstance(right, str) and left.strip().casefold() == right.strip().casefold()
+
+
+def _fact_matches(row, wanted):
+    """Only oracle-authored complete alternatives, never inferred synonyms."""
+    return (row.get('turn_id') == wanted['source_id']
+        and row.get('memory_quality', {}).get('memory_kind') == wanted['memory_kind']
+        and any(all(_same_text(row.get(key), form[key])
+                    for key in ('subject', 'subject_key', 'predicate', 'value'))
+                for form in wanted['representations']))
+
+
+def memory_outcomes(observed, oracle):
+    """Compare effects with independent source facts; never ask a model to grade."""
+    output, effects = observed['output'], observed['effects']
+    sources = {row['id']: row for row in output['sources']}
+    claims = output['claims']
+    active = [row for row in claims if not row['retracted_by'] and not row['superseded_by']]
+    jobs = output['jobs']
+    completed_sources = {row['turn_id'] for row in jobs
+                         if row['status'] == 'complete' and row['attempts'] == 1}
+    checks = {
+        'all_inputs_retained': set(sources) == set(oracle['source_ids']),
+        'formation_first_attempts_complete': len(jobs) == len(oracle['source_ids']) and all(
+            row['status'] == 'complete' and row['attempts'] == 1 for row in jobs),
+        'state_reopened_without_losing_sources': effects['reopened_source_versions_equal'],
+        'recall_uses_a_later_session': all(row['session_id'] != effects['recall_session'] for row in sources.values()),
+        'promoted_evidence_is_source_grounded': (all(row['turn_id'] in sources and any(
+            row['evidence'] in message['content'] for message in sources[row['turn_id']]['messages'])
+            for row in claims) if claims else None),
+    }
+    no_claim_sources = set(oracle.get('no_claim_sources', []))
+    if no_claim_sources:
+        # Unsubmitted or unfinished sources cannot demonstrate abstention.
+        # An observed bad promotion is still a failure even if another source
+        # was never reached. Do not silently remove either from the denominator.
+        checks['junk_and_fiction_not_promoted'] = (
+            False if any(row['turn_id'] in no_claim_sources for row in claims) else
+            True if no_claim_sources <= (set(sources) & completed_sources) else None)
+    for wanted in oracle['claims']:
+        matches = [row for row in active if _fact_matches(row, wanted)]
+        checks[wanted['name']] = bool(matches)
+        checks[wanted['name']+'_conditions_preserved'] = bool(matches) and all(
+            all(term in row['evidence'] for term in wanted.get('evidence_contains', [])) for row in matches)
+    selected = output['recall']['selected']
+    current_assertions = []
+    by_id = {row['id']: row for row in claims}
+    selected_fields_valid = True
+    for row in selected:
+        if row.get('content_format') == 'source_assertions_v1':
+            card = json.loads(row['content'])
+            for assertion in card.get('assertions', []):
+                retained = by_id.get(assertion.get('claim_id'))
+                if (retained is None or assertion.get('source') != 'turn:' + retained['turn_id']
+                        or assertion.get('value') != retained['value']
+                        or card.get('subject') != retained['subject']
+                        or card.get('predicate') != retained['predicate']):
+                    selected_fields_valid = False
+                else:
+                    current_assertions.append(retained)
+    if oracle.get('correction'):
+        corrected = oracle['correction']
+        old = [row for row in claims if _fact_matches(row, corrected['old_fact'])]
+        new = [row for row in active if _fact_matches(row, corrected['new_fact'])]
+        checks['explicit_correction_retracts_prior_claim'] = bool(old and new) and any(
+            row.get('operation') == 'correct' and row.get('prior_claim_id') == previous['id']
+            and previous['retracted_by'] == row['id'] for row in new for previous in old)
+        checks['corrected_value_selected'] = selected_fields_valid and any(
+            _fact_matches(row, corrected['new_fact']) and row in active for row in current_assertions)
+        checks['obsolete_value_not_selected_as_current'] = (not any(
+            _fact_matches(row, corrected['old_fact']) for row in current_assertions)
+            if selected_fields_valid else None)
+    checks['useful_content_recollected'] = all(term in output['recall']['body'] for term in oracle['recall_contains'])
+    checks['recollection_has_canonical_lineage'] = bool(output['recall']['citations']) and all(
+        row in effects['source_references'] for row in output['recall']['citations'])
+    return checks
+
+
+# Finite independently authored oracle data. Articles and the two equivalent
+# storage predicates are listed here, not guessed from a returned model value.
+_LOCATION_REPRESENTATIONS = {
+    place: [{'subject': subject, 'subject_key': subject_key, 'predicate': predicate, 'value': value}
+            for subject, subject_key in (('spare sensor', 'spare sensor'), ('The spare sensor', 'the spare sensor'))
+            for predicate in ('location', 'storage location') for value in values]
+    for place, values in {'old': ('cedar drawer', 'the cedar drawer'),
+                          'new': ('amber cabinet', 'the amber cabinet')}.items()}
+_OLD_LOCATION = {'source_id': 'turn-a', 'memory_kind': 'personal_context',
+                 'representations': _LOCATION_REPRESENTATIONS['old']}
+_NEW_LOCATION = {'source_id': 'turn-b', 'memory_kind': 'personal_context',
+                 'representations': _LOCATION_REPRESENTATIONS['new']}
+
+
+CASES = [
+    CaseSpec(id='memory.formation-quality', version='4', role='extraction', boundary='cognition_consumer',
+        target_tasks=('source_claim_extraction',),
+        consumer='source_memory', evaluator='memory_outcomes', timeout_seconds=240,
+        inputs={'contact_id': 'person', 'recall_session': 'later-conversation',
+            'query': 'tea preference after evening caffeine', 'now': '2026-04-10T12:00:00+00:00', 'max_chars': 6000,
+            'turns': [
+                {'id': 'turn-a', 'session_id': 'conversation-a', 'occurred_at': '2026-04-01T12:00:00+00:00',
+                 'text': 'I prefer decaffeinated tea after 18:00 because caffeine keeps me awake.'},
+                {'id': 'turn-b', 'session_id': 'conversation-b', 'occurred_at': '2026-04-02T12:00:00+00:00',
+                 'text': 'Thanks, okay.'},
+                {'id': 'turn-c', 'session_id': 'conversation-c', 'occurred_at': '2026-04-03T12:00:00+00:00',
+                 'text': 'In a story I am drafting, I own a silver moonboat.'}]},
+        oracle={'source_ids': ['turn-a', 'turn-b', 'turn-c'], 'no_claim_sources': ['turn-b', 'turn-c'],
+            'claims': [{'name': 'useful_conditional_preference_formed', 'source_id': 'turn-a',
+                'memory_kind': 'preference', 'representations': [
+                    {'subject': 'I', 'subject_key': 'speaker', 'predicate': 'tea preference', 'value': value}
+                    for value in ('decaffeinated', 'decaffeinated tea')],
+                'evidence_contains': ['after 18:00', 'caffeine keeps me awake']}],
+            'recall_contains': ['decaffeinated tea', 'after 18:00']}),
+    CaseSpec(id='memory.corrected-recollection', version='4', role='extraction', boundary='cognition_consumer',
+        target_tasks=('source_claim_extraction',),
+        consumer='source_memory', evaluator='memory_outcomes', timeout_seconds=180,
+        inputs={'contact_id': 'person', 'recall_session': 'later-conversation',
+            'query': 'Where is the spare sensor stored?', 'now': '2026-04-10T12:00:00+00:00', 'max_chars': 6000,
+            'turns': [
+                {'id': 'turn-a', 'session_id': 'conversation-a', 'occurred_at': '2026-04-01T12:00:00+00:00',
+                 'text': 'The spare sensor is stored in the cedar drawer.'},
+                {'id': 'turn-b', 'session_id': 'conversation-b', 'occurred_at': '2026-04-02T12:00:00+00:00',
+                 'text': 'Correction: the spare sensor is stored in the amber cabinet, not the cedar drawer.'}]},
+        oracle={'source_ids': ['turn-a', 'turn-b'],
+            'claims': [{**_NEW_LOCATION, 'name': 'corrected_location_formed',
+                'evidence_contains': ['Correction:', 'not the cedar drawer']}],
+            'correction': {'old_fact': _OLD_LOCATION, 'new_fact': _NEW_LOCATION},
+            'recall_contains': ['amber cabinet']})
+]
+
+CONSUMERS = {'source_memory': source_memory}
+EVALUATORS = {'memory_outcomes': memory_outcomes}

apsimo/qualification/records.py

@@ -0,0 +1,92 @@
+"""Small versioned records. Inputs and independent oracles stay separate."""
+from dataclasses import asdict, dataclass
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import tempfile
+
+SCHEMA = 1
+BOUNDARIES = {'role_completion', 'cognition_consumer', 'native_hermes', 'retrieval', 'speech', 'media_consumer'}
+
+
+def encode(value):
+    return (json.dumps(value, sort_keys=True, indent=2, ensure_ascii=True, allow_nan=False) + '\n').encode()
+
+
+def digest(value):
+    return hashlib.sha256(encode(value)).hexdigest()
+
+
+def read(path):
+    path = Path(path)
+    if path.is_symlink() or path.stat().st_size > 8 * 1024 * 1024:
+        raise ValueError('Invalid qualification record')
+    return json.loads(path.read_bytes())
+
+
+def write_once(path, value):
+    """Publish complete bytes without replacing an earlier outcome."""
+    publish(path, encode(value))
+
+
+def publish(path, raw):
+    """Atomically publish a new private record or derived text report."""
+    path = Path(path)
+    path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(prefix='.record-', dir=path.parent)
+    try:
+        with os.fdopen(fd, 'wb') as stream:
+            stream.write(raw)
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.link(temporary, path)
+        directory = os.open(path.parent, os.O_RDONLY)
+        try:
+            os.fsync(directory)
+        finally:
+            os.close(directory)
+    finally:
+        os.unlink(temporary)
+
+
+@dataclass(frozen=True)
+class CaseSpec:
+    id: str
+    version: str
+    role: str
+    boundary: str
+    consumer: str
+    evaluator: str
+    inputs: dict
+    oracle: dict
+    required_capabilities: tuple[str, ...] = ()
+    timeout_seconds: float = 60
+    max_output_bytes: int = 262144
+    provenance: str = 'public'
+    target_tasks: tuple[str, ...] = ()
+
+    def record(self):
+        if not re.fullmatch(r'[a-zA-Z0-9][a-zA-Z0-9_.-]{0,99}', self.id):
+            raise ValueError('Invalid case ID')
+        if self.boundary not in BOUNDARIES or self.provenance not in {'public', 'private'}:
+            raise ValueError('Invalid case boundary/provenance')
+        if not self.version or not self.role or not self.consumer or not self.evaluator:
+            raise ValueError('Case identity is incomplete')
+        if isinstance(self.timeout_seconds, bool) or not .01 <= self.timeout_seconds <= 600:
+            raise ValueError('Case deadline must be .01..600 seconds')
+        if not 1 <= self.max_output_bytes <= 1024 * 1024:
+            raise ValueError('Case output bound must be 1..1048576 bytes')
+        if not isinstance(self.inputs, dict) or not isinstance(self.oracle, dict):
+            raise ValueError('Case input and oracle must be objects')
+        if (not isinstance(self.target_tasks, tuple) or len(self.target_tasks) > 16
+                or any(not isinstance(task, str) or not re.fullmatch(r'[a-z][a-z0-9_]{0,99}', task)
+                       for task in self.target_tasks)
+                or len(set(self.target_tasks)) != len(self.target_tasks)):
+            raise ValueError('Target tasks must be distinct bounded task names')
+        value = asdict(self)
+        value['required_capabilities'] = list(self.required_capabilities)
+        value['target_tasks'] = list(self.target_tasks)
+        return {**value, 'sha256': digest(value), 'inputs_sha256': digest(self.inputs),
+                'oracle_sha256': digest(self.oracle)}

apsimo/qualification/report.py

@@ -0,0 +1,87 @@
+"""Descriptive results by consumer boundary; all declared cases count."""
+from collections import Counter, defaultdict
+from pathlib import Path
+import statistics
+
+from .records import read
+
+OUTCOMES = {'pass', 'fail', 'unsupported', 'setup_error', 'error', 'timeout', 'interrupted', 'not_run'}
+
+
+def summarize(directory):
+    directory = Path(directory)
+    run = read(directory / 'run.json')
+    rows, groups = [], defaultdict(list)
+    for case in run['cases']:
+        path = directory / 'attempts' / case['id'] / 'result.json'
+        result = read(path) if path.exists() else {'outcome': 'not_run', 'primary_outcome': 'unverified', 'elapsed_ms': None}
+        if path.exists() and (result.get('run_id') != run['id'] or result.get('case_sha256') != case['sha256']):
+            raise ValueError('Result identity does not match declared case')
+        if result['outcome'] not in OUTCOMES:
+            raise ValueError('Unknown outcome')
+        row = {'case_id': case['id'], 'role': case['role'], 'boundary': case['boundary'],
+               'case_sha256': case['sha256'], 'inputs_sha256': case['inputs_sha256'],
+               'oracle_sha256': case['oracle_sha256'],
+               'evaluator_identity': run.get('evaluator_identities', {}).get(case['evaluator']), **result}
+        rows.append(row)
+        groups[(case['role'], case['boundary'])].append(row)
+    summaries = []
+    for (role, boundary), items in sorted(groups.items()):
+        durations = {}
+        for outcome in sorted({r['outcome'] for r in items}):
+            times = [r['elapsed_ms'] for r in items if r['outcome'] == outcome and r.get('elapsed_ms') is not None]
+            durations[outcome] = {'samples': len(times), 'median_ms': statistics.median(times) if times else None}
+        summaries.append({'role': role, 'boundary': boundary, 'declared': len(items),
+            'outcomes': dict(Counter(r['outcome'] for r in items)),
+            'primary_passes': sum(r.get('primary_outcome') == 'pass' for r in items),
+            'duration_by_outcome': durations,
+            'coverage': 'only the listed cases and consumer boundary; not full role qualification'})
+    return {'schema': 1, 'kind': 'run', 'run_id': run['id'], 'recipe': run['recipe'],
+            'suite_sha256': run['suite_sha256'], 'evidence_mode': run['evidence_mode'],
+            'implementation_sha256': run.get('implementation_sha256'),
+            'declared': len(rows), 'groups': summaries, 'cases': rows}
+
+
+def compare(left, right):
+    before, after = summarize(left), summarize(right)
+    old = {r['case_id']: r for r in before['cases']}
+    new = {r['case_id']: r for r in after['cases']}
+    rows = []
+    for identity in sorted(old.keys() | new.keys()):
+        a, b = old.get(identity), new.get(identity)
+        comparable = bool(a and b and a['case_sha256'] == b['case_sha256']
+                          and a.get('evaluator_identity') is not None
+                          and a['evaluator_identity'] == b.get('evaluator_identity')
+                          and before['evidence_mode'] == after['evidence_mode'])
+        rows.append({'case_id': identity, 'comparable': comparable,
+            'grading_changed': bool(a and b and a.get('evaluator_identity') != b.get('evaluator_identity')),
+            'before': a['outcome'] if a else 'absent', 'after': b['outcome'] if b else 'absent',
+            'primary_before': a.get('primary_outcome') if a else None,
+            'primary_after': b.get('primary_outcome') if b else None,
+            'elapsed_ms_delta': (b['elapsed_ms'] - a['elapsed_ms']) if comparable
+                and a['outcome'] == b['outcome'] == 'pass'
+                and a.get('elapsed_ms') is not None and b.get('elapsed_ms') is not None else None})
+    return {'schema': 1, 'kind': 'comparison', 'before': before, 'after': after, 'cases': rows,
+            'basis': 'recipe comparison; no causal weight, serving identity, tail-latency or confidence claim',
+            'system_implementation_changed': before['implementation_sha256'] != after['implementation_sha256'],
+            'same_suite': before['suite_sha256'] == after['suite_sha256']}
+
+
+def markdown(report):
+    if report['kind'] == 'comparison':
+        lines = ['# Recipe comparison', '', report['basis'], '',
+                 '| Case | Comparable | Before | After | Primary before/after | Elapsed change ms |',
+                 '| --- | --- | --- | --- | --- | --- |']
+        for row in report['cases']:
+            lines.append(f"| {row['case_id']} | {row['comparable']} | {row['before']} | {row['after']} | "
+                         f"{row['primary_before']}/{row['primary_after']} | {row['elapsed_ms_delta']} |")
+    else:
+        lines = ['# Model qualification observations', '',
+            f"Mode: {report['evidence_mode']}. Declared cases: {report['declared']}.",
+            'Coverage is limited to the listed cases and consumer boundaries. Unknown observations remain unknown.', '',
+            '| Role | Boundary | All cases | Outcomes | Primary passes | Durations by outcome (median ms, samples) |',
+            '| --- | --- | --- | --- | --- | --- |']
+        for group in report['groups']:
+            lines.append(f"| {group['role']} | {group['boundary']} | {group['declared']} | {group['outcomes']} | "
+                f"{group['primary_passes']} | {group['duration_by_outcome']} |")
+    return '\n'.join(lines) + '\n'