morphtree 2.1.0__py3-none-win_amd64.whl

morphtree/_experimental/product.py

@@ -0,0 +1,156 @@
+"""Installed integrated consumer of explicitly supplied, authenticated models.
+
+Packaged with relative imports; not imported as a workspace learner. SHA is a
+caller-supplied custody identity, not a claim that a model is scientifically good.
+"""
+import copy
+import hashlib
+import os
+from pathlib import Path
+from dataclasses import asdict, replace
+from morphtree.config import HyperParams
+from morphtree.persistence import read_json, LoadLimits
+from .admission import AdmissionError, digest
+from .runtime_io import configure, artifact_path, publish
+from .base import Base, _payloads
+from .facade import Control
+from .roundtrip_v3 import decode
+from .support import FamilyChoice
+from .learning import mutation_admission
+
+class MorphTree(Control):
+    """Unified experimental API. Fresh fit needs a new qualified v2 admission.
+
+    Use load(model, sha256=..., workdir=...) for a supplied retained model.
+    Files remain declarative and bounded; no code or data are loaded implicitly.
+    """
+
+    def fit(self, X, y):
+        self.last_attempt_ = {'status': 'BLOCKED', 'physical_fits': 0, 'trials': 0, 'reason': 'COMPLETION_TRAINING_BOUNDARY_REQUIRES_QUALIFICATION_AND_NEW_OWNER_CONSENT'}
+        raise AdmissionError(self.last_attempt_)
+
+    @classmethod
+    def load(cls, path, *, sha256, workdir):
+        configure(workdir)
+        source = Path(path).resolve()
+        if not source.is_file() or not 0 < source.stat().st_size <= 32 * 1024 ** 2:
+            raise ValueError('Explicit bounded model input required')
+        with source.open('rb') as stream:raw=stream.read(32*1024**2+1)
+        if not 0<len(raw)<=32*1024**2:raise ValueError('Explicit bounded model input required')
+        data = decode(raw, sha256)
+        if type(data) is not dict:raise ValueError('Model input must be a JSON object')
+        folder = artifact_path('.inputs')
+        folder.mkdir(exist_ok=True)
+        snapshot = artifact_path(folder / (sha256 + '.json'))
+        if snapshot.exists():
+            if snapshot.read_bytes() != raw:
+                raise ValueError('Corrupt retained input snapshot')
+        else:
+            pending = artifact_path(str(snapshot) + '.pending')
+            with pending.open('xb') as out:
+                out.write(raw)
+                out.flush()
+                os.fsync(out.fileno())
+            if snapshot.exists():
+                raise FileExistsError('Concurrent input snapshot')
+            os.rename(pending, snapshot)
+        if data.get('schema') in ('morph027-native-control-v1', 'comp002-native-control-v1', 'comp003-native-control-v1', 'comp009-native-control-v1', 'product002-native-control-v1', 'product003-native-control-v1', 'product006-native-control-v1'):
+            old = Control.load(snapshot)
+        else:
+            old = Base.load(snapshot)
+            if old.report_.get('tuning') != 'DISABLED' or old.report_.get('trials'):
+                raise ValueError('Legacy input must have an explicitly untuned report; use verified variants otherwise')
+            old._selected_variant = 'pure'
+            old._variants = {'pure': {'model': old.model_, 'recipe': copy.deepcopy(old.best_recipe_), 'origin': 'PRE_FINAL_NUMERIC_SEARCH', 'evidence': 'EXPLICIT_RETAINED_WEIGHTS_NOT_NEW_FIT', 'exportable': True, 'SEARCH_primary': old.report_.get('SEARCH_primary')}}
+        for variant in old._variants.values():
+            recipe = variant['recipe']
+            expected = HyperParams(**recipe['params'])
+            for member in variant['model'].components:
+                want = expected if recipe['family'] == 'boost' else replace(expected, n_estimators=1, learning_rate=1.0, row_fraction=1.0)
+                if asdict(member.params_) != asdict(want):
+                    raise ValueError('Recipe does not describe actual native parameters')
+        obj = cls(**old.get_params())
+        obj.__dict__.update(old.__dict__)
+        obj.capacity = getattr(old, 'capacity', None)
+        obj._workdir = str(Path(workdir).resolve())
+        obj._input_sha256 = sha256
+        return obj
+
+    def _io(self):
+        self._fitted()
+        configure(self._workdir)
+
+    def predict(self, X):
+        self._io()
+        return super().predict(X)
+
+    def predict_proba(self, X):
+        self._io()
+        return super().predict_proba(X)
+
+    def recipe(self):
+        self._fitted()
+        return copy.deepcopy(self.best_recipe_)
+
+    def cost_report(self):
+        self._fitted()
+        return {'current_physical_fits': 0, 'current_trials': 0, 'current_OOF': 0, 'scope': 'RETAINED_MODEL_CONSUMPTION_ONLY', 'historical_learning_cost': 'SEE_ORIGINAL_REPORT_NOT_REMEASURED', 'source_artifact_sha256': self._input_sha256, 'peak_RSS': 'UNKNOWN', 'performance_qualified': False}
+
+    def save(self, path, *, variant='all'):
+        self._io()
+        return super().save(path, variant=variant)
+
+    def save_recipe(self, path, *, variant='selected'):
+        self._io()
+        if artifact_path(path).exists():
+            raise FileExistsError('Never replace a completed recipe')
+        return super().save_recipe(path, variant=variant)
+
+    def export_python(self, path, *, variant='selected'):
+        self._io()
+        return super().export_python(path, variant=variant)
+
+    def export_bundle(self, directory, *, variants=('pure',)):
+        self._io()
+        return super().export_bundle(directory, variants=variants)
+
+    @classmethod
+    def select_retained(cls, anchor, candidate, *, X, y, ids, role='SEARCH'):
+        """Compare two existing families on their exact historical SEARCH.
+
+        No new labels, new split, fabricated provenance, fit, OOF or tuning.
+        This replay measures selection correctness, never policy utility.
+        """
+        if not isinstance(anchor, cls) or not isinstance(candidate, cls):
+            raise ValueError('Explicit authenticated model objects required')
+        anchor._io()
+        candidate._fitted()
+        a, b = (anchor.recipe(), candidate.recipe())
+        if role != 'SEARCH' or anchor.task_ != candidate.task_ or a['training_identity'] != b['training_identity']:
+            raise ValueError('Same actual TRAIN/SEARCH task identity required')
+        if a['preprocessing'] != b['preprocessing'] or getattr(anchor, 'classes_', ()) != getattr(candidate, 'classes_', ()):
+            raise ValueError('Same encoding and class vocabulary required')
+        if type(ids) is not list or not ids or len(set(ids)) != len(ids) or (len(ids) != len(y)):
+            raise ValueError('Unique aligned SEARCH IDs required')
+        targets = list(y) if anchor.task_ == 'regression' else [list(anchor.classes_).index(v) for v in y]
+        search = {'role': role, 'ids': list(ids), 'X': anchor.encoder_.transform(X), 'y': targets}
+        if digest(search) != a['training_identity'].get('SEARCH'):
+            raise ValueError('Supplied rows/labels/IDs differ from frozen SEARCH')
+        models = {'anchor': anchor, 'candidate': candidate}
+
+        def packet(name, model):
+            r = model.recipe()
+            return {'id': name, 'task': model.task_, 'family': r['family'], 'recipe': r['structure'], 'learning_identity': r['training_identity'], 'params': r['params'], 'artifact_sha256': digest(_payloads(model.model_))}
+        packets = {n: packet(n, m) for n, m in models.items()}
+        events = []
+
+        def verify(p, selection):
+            model = models[p['id']]
+            if p != packets[p['id']] or selection != search or digest(_payloads(model.model_)) != p['artifact_sha256']:
+                raise ValueError('Changed verified candidate/SEARCH')
+            return model.model_.probabilities_or_values(selection['X'])
+        chooser = FamilyChoice(packets['anchor'], search, checkpoint=events.append, verify=verify)
+        result = chooser.run(packets['candidate'])
+        chosen = models[result['packet']['id']]
+        decision = {'schema': 'morphtree-completion-retained-selection-v1', 'selected': result['packet']['id'], 'anchor_loss': result['anchor_loss'], 'selected_loss': result['loss'], 'events': events, 'decision': 'ACCEPT_OTHER_FAMILY' if chosen is candidate else 'RETAIN_ANCHOR', 'morphing': mutation_admission('auto'), 'structure_frozen': chosen.recipe(), 'physical_fits': 0, 'trials': 0, 'GUARD': 0, 'evidence': 'RETAINED_REPLAY_NOT_TRAINED_ADAPTIVE_UTILITY'}
+        return (chosen, decision)

morphtree/_experimental/product_cli.py

@@ -0,0 +1,43 @@
+"""Inference-only CLI for the integrated experimental build."""
+import argparse
+import json
+from pathlib import Path
+from .product import MorphTree
+from .target import _NamedRows
+from .roundtrip_v3 import decode
+import hashlib
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('command', choices=('inspect', 'predict', 'export'))
+    parser.add_argument('--model', required=True)
+    parser.add_argument('--sha256', required=True)
+    parser.add_argument('--workdir', required=True)
+    parser.add_argument('--input')
+    parser.add_argument('--output')
+    args = parser.parse_args(argv)
+    try:
+        model = MorphTree.load(args.model, sha256=args.sha256, workdir=args.workdir)
+        if args.command == 'inspect':
+            value = {'task': model.task_, 'family': model.best_family_,
+                     'recipe': model.recipe(), 'cost': model.cost_report()}
+        elif args.command == 'predict':
+            if not args.input: raise ValueError('--input JSON rows required')
+            path = Path(args.input)
+            if not 0 < path.stat().st_size <= 16*1024**2: raise ValueError('Bounded rows required')
+            raw = path.read_bytes(); rows = decode(raw, hashlib.sha256(raw).hexdigest())
+            if type(rows) is dict and set(rows) == {'columns', 'rows'}:
+                rows = _NamedRows(rows['rows'], rows['columns'])
+            value = {'predictions': model.predict(rows)}
+            if model.task_ != 'regression': value['probabilities'] = model.predict_proba(rows)
+        else:
+            if not args.output: raise ValueError('--output required')
+            model.export_python(args.output); value = {'standalone': args.output, 'new_fits': 0}
+        print(json.dumps(value, ensure_ascii=False, allow_nan=False))
+    except (ValueError, OSError, RuntimeError) as exc:
+        parser.exit(2, type(exc).__name__+': '+str(exc)+'\n')
+
+
+if __name__ == '__main__':
+    main()

morphtree/_experimental/roundtrip_v3.py

@@ -0,0 +1,311 @@
+"""Versioned research-only roundtrip qualification, not a loader or fit permit.
+
+Only the documented native report projections may differ. Every other byte-
+decoded state path is compared. A successful state comparison alone cannot PASS:
+an externally pinned, pre-save behavior witness and verified installed runtime
+are required. Trust comes from the caller's qualified dossier, not this report.
+"""
+import copy
+import hashlib
+import json
+import math
+POLICY = 'PRODUCT002_EXPLICIT_THREE_TASK_GROWTH_ENVELOPE_V3'
+NUMERIC = ('base_search_loss', 'base_guard_loss', 'structure_search_loss', 'structure_guard_loss', 'final_search_loss', 'final_guard_loss', 'structural_gain', 'tuning_gain')
+FIELDS = {'active_parameters': ('B', 'list', 'algorithm.active_parameters'), 'backend': ('C', 'str', 'external installed-runtime dossier; pure/python-stdlib'), 'base_genome': ('C', 'dict', 'historical starting recipe; Base-only scope'), 'base_parameters': ('C', 'dict', 'historical starting parameters; Base-only scope'), 'config': ('C', 'dict', 'external protocol/configuration, never restored wholesale'), 'coverage': ('D', 'dict', 'discovery diagnostics, not a completion receipt'), 'error_profiles': ('D', 'list', 'diagnostic history, empty in this scope'), 'feature_schema': ('B', 'dict', 'top-level n_features/feature_names'), 'final_fingerprint': ('B', 'str', 'algorithm.structural_fingerprint'), 'final_genome': ('B', 'dict', 'algorithm.genome'), 'final_parameters': ('B', 'dict', 'algorithm.parameters'), 'freeze_reason': ('D', 'str', 'diagnostic stop explanation'), 'metrics_scope': ('C', 'str', 'TRAIN-only metric provenance, not GUARD permission'), 'morph_history': ('D', 'list', 'diagnostic history, empty in this scope'), 'plateau_reached': ('D', 'bool', 'diagnostic outcome, not scientific convergence'), 'plateau_scope': ('D', 'str', 'diagnostic scope'), 'python': ('C', 'str', 'historical interpreter; current interpreter externally pinned'), 'refit_full': ('C', 'bool', 'external learning recipe'), 'spec_version': ('B', 'int', 'top-level spec_version'), 'split': ('C', 'dict', 'TRAIN metric partition provenance, not final GUARD'), 'stop_reason': ('D', 'str', 'informational only, never COMPLETE authority'), 'structural_sampling': ('C', 'dict', 'historical TRAIN sampling provenance'), 'timings_seconds': ('D', 'dict', 'operational diagnostics, not performance evidence'), 'tuning': ('C', 'dict', 'historical tuning state, disabled in this scope'), 'version': ('B', 'str', 'top-level producer_version')}
+
+class Unknown(ValueError):
+    """Known-safe equivalence policy does not cover this material."""
+
+def digest(v):
+    return hashlib.sha256(json.dumps(v, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False).encode('utf8')).hexdigest()
+
+def decode(b, expected):
+    if type(b) is not bytes or not 0 < len(b) <= 32 * 1024 ** 2:
+        raise ValueError('Bounded immutable bytes required')
+    if hashlib.sha256(b).hexdigest() != expected:
+        raise ValueError('Artifact SHA mismatch')
+
+    def pairs(items):
+        out = {}
+        for k, v in items:
+            if k in out:
+                raise ValueError('Duplicate JSON key')
+            out[k] = v
+        return out
+
+    def bad(_):
+        raise ValueError('Nonfinite JSON')
+    v = json.loads(b.decode('utf8'), object_pairs_hook=pairs, parse_constant=bad)
+
+    def bounded(x, depth=0):
+        if depth > 64:
+            raise ValueError('JSON depth limit')
+        if isinstance(x, dict):
+            for k, y in x.items():
+                if type(k) is not str:
+                    raise ValueError('String keys required')
+                bounded(y, depth + 1)
+        elif isinstance(x, list):
+            for y in x:
+                bounded(y, depth + 1)
+        elif type(x) is float and (not math.isfinite(x)):
+            raise ValueError('Nonfinite number')
+    bounded(v)
+    digest(v)
+    return v
+
+def differences(a, b, path=''):
+    if type(a) is not type(b):
+        return [{'path': path, 'before': a, 'after': b}]
+    if isinstance(a, dict):
+        out = []
+        for k in sorted(set(a) | set(b)):
+            p = path + '/' + k.replace('~', '~0').replace('/', '~1')
+            if k not in a or k not in b:
+                out.append({'path': p, 'before': a.get(k, 'ABSENT'), 'after': b.get(k, 'ABSENT')})
+            else:
+                out.extend(differences(a[k], b[k], p))
+        return out
+    if isinstance(a, list):
+        if len(a) != len(b):
+            return [{'path': path, 'before': a, 'after': b}]
+        return [d for i, (x, y) in enumerate(zip(a, b)) for d in differences(x, y, path + '/' + str(i))]
+    return [] if a == b else [{'path': path, 'before': a, 'after': b}]
+
+def _finite(v):
+    return type(v) in (int, float) and math.isfinite(v)
+
+def _binary_report(r, native):
+    from morphtree.public import imported_report, PUBLIC_CONFIG_FIELDS
+    from morphtree.config import MorphTreeConfig
+    if type(r) is not dict:
+        raise ValueError('Report object required')
+    unknown = set(r) - set(NUMERIC) - set(FIELDS)
+    if unknown:
+        raise Unknown('Unclassified binary report fields: ' + repr(sorted(unknown)))
+    types = {'str': str, 'dict': dict, 'list': list, 'bool': bool, 'int': int}
+    for k in NUMERIC:
+        if k in r and (not _finite(r[k])):
+            raise ValueError('Nonfinite/invalid metric ' + k)
+    for k, (_, typ, _) in FIELDS.items():
+        if k in r and type(r[k]) is not types[typ]:
+            raise ValueError('Report type ' + k)
+    a = native['algorithm']
+    bindings = {'active_parameters': a['active_parameters'], 'final_genome': a['genome'], 'final_parameters': a['parameters'], 'final_fingerprint': a['structural_fingerprint'], 'spec_version': native['spec_version'], 'version': native['producer_version'], 'refit_full': a['learning_recipe']['refit_full'], 'feature_schema': {'count': native['n_features'], 'names': native['feature_names']}}
+    for k, v in bindings.items():
+        if k in r and r[k] != v:
+            raise ValueError('Contradictory report/authoritative binding ' + k)
+    for k in ('morph_history', 'error_profiles'):
+        if k in r and r[k] != []:
+            raise Unknown('Nonempty unqualified ' + k)
+    for k, target in (('base_genome', 'genome'), ('base_parameters', 'parameters')):
+        if k in r and r[k] != a[target]:
+            raise Unknown('Non-Base historical ' + k)
+    if 'backend' in r and r['backend'] != 'python-stdlib':
+        raise ValueError('Backend report mismatch')
+    if 'config' in r:
+        c = r['config']
+        if not set(c) <= set(PUBLIC_CONFIG_FIELDS):
+            raise Unknown('Unexpected config fields')
+        if c.get('enable_morphing', False) or c.get('enable_tuning', False) or c.get('memory_enabled', False):
+            raise Unknown('Non-Base config outside policy')
+        for k, v in a['learning_recipe'].items():
+            if k in c and c[k] != v:
+                raise ValueError('Report/learning recipe mismatch ' + k)
+        if any((type(v) not in (bool, int, float, str, type(None)) for v in c.values())):
+            raise ValueError('Non-scalar operational config')
+        MorphTreeConfig(**c)
+    schemas = {'coverage': {'local_search_complete', 'scope', 'evaluated_neighbors', 'neighborhood_size', 'reason'}, 'split': {'guard', 'kind', 'refit_rows', 'sealed', 'search', 'train'}, 'structural_sampling': {'policy_active', 'row_fraction', 'sampled_rows', 'scope', 'train_rows'}, 'timings_seconds': {'base', 'discovery', 'refit', 'total_fit', 'tuning'}, 'tuning': {'accepted', 'attempted', 'completed', 'frozen_fingerprint', 'reason', 'trials'}}
+    for k, keys in schemas.items():
+        if k in r and (not set(r[k]) <= keys):
+            raise Unknown('Unclassified nested report ' + k)
+    if r.get('coverage', {}) != {}:
+        raise Unknown('Nonempty discovery coverage outside Base scope')
+    if 'split' in r:
+        s = r['split']
+        if set(s) != schemas['split'] or any((type(s[k]) is not int or s[k] < 0 for k in ('guard', 'refit_rows', 'search', 'train'))):
+            raise ValueError('Split diagnostic types')
+        if s['kind'] != 'baseline_no_selection_metrics_are_training_only' or s['sealed'] != 'not accepted by fit; evaluate externally' or len({s[k] for k in ('guard', 'search', 'refit_rows', 'train')}) != 1:
+            raise Unknown('Unsupported split provenance')
+    for k in ('metrics_scope', 'freeze_reason', 'plateau_scope', 'stop_reason', 'python'):
+        if k in r and len(r[k]) > 4096:
+            raise ValueError('Bounded informational annotation')
+    if 'tuning' in r:
+        t = r['tuning']
+        if set(t) != schemas['tuning'] or type(t['reason']) is not str or len(t['reason']) > 4096:
+            raise ValueError('Exact typed inactive tuning report')
+        if any((t.get(k, False) is not False for k in ('accepted', 'attempted', 'completed'))) or t.get('trials', []) != []:
+            raise Unknown('Active tuning report outside policy')
+        if 'frozen_fingerprint' in t and t['frozen_fingerprint'] != a['structural_fingerprint']:
+            raise ValueError('Tuning fingerprint mismatch')
+    if 'timings_seconds' in r and any((not _finite(v) or v < 0 for v in r['timings_seconds'].values())):
+        raise ValueError('Invalid diagnostic times')
+    if 'structural_sampling' in r:
+        s = r['structural_sampling']
+        if set(s) != schemas['structural_sampling'] or type(s['scope']) is not str or len(s['scope']) > 4096 or (not _finite(s['row_fraction'])):
+            raise ValueError('Exact typed structural sampling')
+        if any((type(s[k]) is not int or s[k] < 1 for k in ('sampled_rows', 'train_rows'))) or s['sampled_rows'] != s['train_rows']:
+            raise ValueError('Unsampled positive integer TRAIN counts required')
+        if 'split' in r and s['train_rows'] != r['split']['train']:
+            raise ValueError('Inconsistent TRAIN report counts')
+        if s.get('policy_active', False) is not False or s.get('row_fraction', 1) != 1:
+            raise Unknown('Sampled structure report outside scope')
+    return imported_report(r)
+
+def report_projection(r, native, task):
+    if task == 'binary':
+        return _binary_report(r, native)
+    from morphtree.public import imported_multiclass_report, imported_regression_report
+    allowed = {'task', 'imported', 'train_metrics', 'validation_metrics'}
+    if task == 'regression':
+        allowed.add('objective')
+    if type(r) is not dict:
+        raise ValueError('Report object required')
+    if set(r) - allowed:
+        raise Unknown('Unclassified task report fields')
+    if r.get('task') != task or r.get('imported') is not True:
+        raise Unknown('Only normalized retained task report qualified')
+    if task == 'regression' and r.get('objective') != 'squared_error':
+        raise ValueError('Objective')
+    metrics = {'rmse', 'mae', 'r2'} if task == 'regression' else {'logloss', 'accuracy', 'macro_f1'}
+    for group in ('train_metrics', 'validation_metrics'):
+        if group not in r:
+            continue
+        if type(r[group]) is not dict or set(r[group]) - metrics:
+            raise Unknown('Unknown metric field')
+        for k, v in r[group].items():
+            if not _finite(v) or (k != 'r2' and v < 0) or (k in ('accuracy', 'macro_f1') and v > 1):
+                raise ValueError('Metric domain')
+    return (imported_regression_report if task == 'regression' else imported_multiclass_report)(r)
+
+def canonical_state(payload):
+    if type(payload) is not dict or set(payload) != {'schema', 'constructor', 'recipe', 'classes', 'report', 'native_payloads'}:
+        raise ValueError('Exact M026 envelope required')
+    if payload['schema'] not in ('morph026-native-estimator-v1', 'comp002-native-estimator-v1', 'comp003-native-estimator-v1', 'comp009-native-estimator-v1', 'product002-native-estimator-v1', 'product003-native-estimator-v1', 'product006-native-estimator-v1'):
+        raise ValueError('Unsupported envelope')
+    if type(payload['constructor']) is not dict or set(payload['constructor']) != {'task', 'families', 'morphing', 'tuning', 'preset', 'metric', 'time_budget', 'n_trials', 'n_jobs', 'random_state', 'verbose'}:
+        raise Unknown('Unknown constructor schema')
+    recipe = payload['recipe']
+    if type(recipe) is not dict or set(recipe) != {'schema', 'task', 'family', 'structure', 'params', 'seed', 'training_identity', 'preprocessing', 'mutations'} or recipe['schema'] != {'morph026-native-estimator-v1': 'morph026-frozen-form-v1', 'comp002-native-estimator-v1': 'comp002-frozen-form-v1', 'comp009-native-estimator-v1': 'comp009-frozen-form-v1', 'comp003-native-estimator-v1': 'comp003-frozen-form-v1', 'product002-native-estimator-v1': 'product002-frozen-form-v1', 'product003-native-estimator-v1': 'product003-frozen-form-v1', 'product006-native-estimator-v1': 'product006-frozen-form-v1'}[payload['schema']]:
+        raise Unknown('Unknown frozen recipe schema')
+    if type(recipe['structure']) is not dict or set(recipe['structure']) != ({'genome', 'classes', 'members', 'encoder', 'lineage'} if payload['schema'] in ('comp003-native-estimator-v1', 'comp009-native-estimator-v1', 'product002-native-estimator-v1', 'product003-native-estimator-v1', 'product006-native-estimator-v1') else {'genome', 'classes', 'members', 'encoder'}) or recipe['mutations'] != []:
+        raise Unknown('Unknown structural recipe')
+    outer_fields = {'GUARD_used', 'SEARCH_primary', 'available_families', 'best_family', 'classes', 'cost_scope', 'descendant_cpu', 'events', 'final_model_role', 'limitations', 'metric', 'morphing', 'optional_dependencies', 'peak_RSS', 'process_cpu_seconds', 'refit_performed', 'schema', 'seed', 'selection', 'stop_reason', 'task', 'trials', 'tried', 'tuning', 'untried', 'wall_seconds', 'worker_admission', 'variants', 'exported_variant'}
+    if type(payload['report']) is not dict or set(payload['report']) - outer_fields:
+        raise Unknown('Unclassified outer report fields')
+    task = payload['recipe']['task']
+    expected = {'binary': 'MorphTreeModel', 'multiclass': 'MorphTreeMulticlassModel', 'regression': 'MorphTreeRegressionModel'}
+    if task not in expected:
+        raise ValueError('Task')
+    p = copy.deepcopy(payload)
+    if type(p['native_payloads']) is not list or not p['native_payloads']:
+        raise ValueError('Native members')
+    for n in p['native_payloads']:
+        if n.get('format') not in ((expected[task], {'regression': 'MorphTreeRegressionGrowthV2', 'multiclass': 'MorphTreeMulticlassGrowthV2'}.get(task)) if p['schema'] in ('product002-native-estimator-v1', 'product003-native-estimator-v1', 'product006-native-estimator-v1') else (expected[task],)) or n.get('spec_version') != 1 or n.get('producer_version') != '0.6.0.dev0':
+            raise ValueError('Native format/version')
+        report_projection(n['report'], n, task)
+        del n['report']
+    return p
+
+def _safe_native_schema(payload):
+    """Use byte-exact production validators, without fitting or inference.
+
+    Needed even when both sides contain the same unexpected field. Equality
+    alone cannot establish schema validity. No quarantine caller uses this API.
+    """
+    from morphtree.model import MorphTreeClassifier, MorphTreeRegressor, MorphTreeMulticlassClassifier
+    classes = {'binary': MorphTreeClassifier, 'regression': MorphTreeRegressor, 'multiclass': MorphTreeMulticlassClassifier}
+    for native in payload['native_payloads']:
+        classes[payload['recipe']['task']]._from_payload(copy.deepcopy(native))
+
+def _behavior(w, task, classes, nrows):
+    if set(w) != ({'prediction', 'probabilities'} if task != 'regression' else {'prediction'}):
+        raise ValueError('Behavior witness schema')
+    p = w['prediction']
+    if type(p) is not list or len(p) != nrows:
+        raise ValueError('Prediction alignment')
+    if task == 'regression':
+        if not all((_finite(v) for v in p)):
+            raise ValueError('Nonfinite predictions')
+        return
+    probs = w['probabilities']
+    if len(probs) != nrows or any((v not in classes for v in p)):
+        raise ValueError('Class alignment')
+    for y, row in zip(p, probs):
+        if type(row) is not list or len(row) != len(classes) or (not all((_finite(v) and 0 <= v <= 1 for v in row))):
+            raise ValueError('Probability domain')
+        if abs(math.fsum(row) - 1) > 1e-12:
+            raise ValueError('Probability normalization')
+        if classes[max(range(len(row)), key=row.__getitem__)] != y:
+            raise ValueError('Class decision rule')
+
+def compare(before_bytes, after_bytes, *, trust, witness):
+    """No IO, fitting, unpickling, eval or generic annotation restoration.
+
+    trust must be made by the qualified caller from verified runtime/dossier,
+    model bytes, immutable query and witness persisted before serialization.
+    This API is a cooperative verifier, not protection against a malicious caller
+    that can rewrite its own trust dossier or execute arbitrary Python in process.
+    """
+    result = {'policy': POLICY, 'status': 'INVALID', 'A': 'UNKNOWN', 'B': 'UNKNOWN', 'C': 'UNKNOWN', 'differences': [], 'accepted_differences': [], 'new_fits': 0, 'new_trials': 0}
+    try:
+        required = {'runtime', 'actual_runtime', 'before_sha256', 'after_sha256', 'witness_sha256', 'query_sha256', 'source_sha256', 'recipe_sha256', 'loader_sha256', 'admission'}
+        if set(trust) != required:
+            raise ValueError('Exact externally pinned trust schema')
+        if trust['admission'] not in ('QUALIFIED_RETAINED_ZERO_FIT', 'QUALIFIED_COMP003_SOFTWARE', 'QUALIFIED_PRODUCT002_SOFTWARE', 'QUALIFIED_COMP003_LIVE_COMPONENTS'):
+            raise Unknown('Quarantine or missing exact admission')
+        if trust['runtime'] != trust['actual_runtime'] or trust['runtime'].get('backend') != 'pure':
+            raise ValueError('Runtime identity changed')
+        a = decode(before_bytes, trust['before_sha256'])
+        b = decode(after_bytes, trust['after_sha256'])
+        result['differences'] = differences(a, b)
+        ca = canonical_state(a)
+        cb = canonical_state(b)
+        _safe_native_schema(a)
+        _safe_native_schema(b)
+        if ca != cb:
+            raise ValueError('Critical predictive/identity state differs')
+        if digest(a['recipe']) != trust['recipe_sha256']:
+            raise ValueError('Recipe identity')
+        task = a['recipe']['task']
+        for i, (x, y) in enumerate(zip(a['native_payloads'], b['native_payloads'])):
+            projection = report_projection(x['report'], x, task)
+            report_projection(y['report'], y, task)
+            if y['report'] != projection:
+                raise ValueError('Restored report not exact safe projection')
+            for d in differences(x['report'], y['report'], f'/native_payloads/{i}/report'):
+                key = d['path'].rsplit('/', 1)[-1]
+                if key not in FIELDS:
+                    raise Unknown('Unclassified report difference')
+                result['accepted_differences'].append({**d, 'category': FIELDS[key][0], 'reason': 'Original safe-loader allowlist deliberately drops this typed annotation', 'authority': FIELDS[key][2]})
+        result.update(B='PASS', C='PASS')
+        if witness is None:
+            raise Unknown('No actual pre-save witness')
+        if digest(witness) != trust['witness_sha256']:
+            raise ValueError('Witness SHA')
+        keys = {'schema', 'runtime', 'source_sha256', 'recipe_sha256', 'query', 'loader_sha256', 'before_state_sha256', 'after_state_sha256', 'before', 'after', 'events'}
+        if set(witness) != keys or witness['schema'] != 'morph032-actual-behavior-v1':
+            raise ValueError('Witness schema')
+        for k in ('runtime', 'source_sha256', 'recipe_sha256', 'loader_sha256'):
+            if witness[k] != trust[k]:
+                raise ValueError('Witness binding ' + k)
+        if digest(witness['query']) != trust['query_sha256']:
+            raise ValueError('Query identity')
+        if witness['before_state_sha256'] != digest(a) or witness['after_state_sha256'] != digest(b):
+            raise ValueError('Witness state binding')
+        ev = witness['events']
+        if set(ev) != {'captured_before_save', 'saved', 'safe_loaded', 'captured_after_load'} or not all((type(v) is int for v in ev.values())):
+            raise ValueError('Witness event schema')
+        if not ev['captured_before_save'] < ev['saved'] < ev['safe_loaded'] < ev['captured_after_load']:
+            raise ValueError('Witness acquired too late or invalid timeline')
+        n = len(witness['query']['rows'])
+        for v in ('before', 'after'):
+            _behavior(witness[v], task, a['classes'], n)
+        if witness['before'] != witness['after']:
+            raise ValueError('Actual predictions/probabilities differ')
+        result.update(status='PASS', A='PASS', reason='Exact math, actual behavior, identity and explicit safe projection')
+    except Unknown as e:
+        result.update(status='UNQUALIFIED', reason=str(e))
+    except (KeyError, TypeError, ValueError, IndexError, RecursionError, OverflowError) as e:
+        result.update(status='REJECTED', reason=type(e).__name__ + ': ' + str(e))
+    return result

morphtree/_experimental/runtime_io.py

@@ -0,0 +1,65 @@
+"""Installed IO glue, not a learner. Copied only into private MORPH030 wheels."""
+import copy
+import json
+import os
+import tempfile
+from contextvars import ContextVar
+from pathlib import Path
+from .admission import CURRENT,AdmissionError,digest,sha,safe_path
+from ..temporary import directory as bounded_directory
+
+WORKDIR=ContextVar('morph030_workdir',default=None)
+
+
+def configure(workdir):
+    p=Path(workdir).resolve()
+    if not p.is_dir():raise ValueError('Explicit existing working directory required')
+    safe_path(p,p);WORKDIR.set(p)
+
+
+def artifact_path(value):
+    root=WORKDIR.get()
+    if root is None:raise ValueError('Call configure(workdir) before explicit IO')
+    return safe_path(root,value)
+
+
+def configure_temp():
+    p=artifact_path('.temporary');p.mkdir(exist_ok=True)
+    # Limited to this explicit experimental process, not Python installation.
+    tempfile.tempdir=str(p)
+    return p
+
+
+def reference(p):
+    p=artifact_path(p)
+    return {'path':p.relative_to(WORKDIR.get()).as_posix(),'sha256':sha(p),'bytes':p.stat().st_size}
+
+
+def publish(path,text,check):
+    path=artifact_path(path);pending=artifact_path(str(path)+'.pending')
+    if path.exists() or pending.exists():raise FileExistsError('Never replace completed or incomplete evidence')
+    with pending.open('x',encoding='utf8') as f:f.write(text);f.flush();os.fsync(f.fileno())
+    check()
+    if path.exists():raise FileExistsError('Concurrent output')
+    os.rename(pending,path)
+
+
+def require(operation):
+    current=CURRENT.get()
+    if current is None or current[1]['operation'] not in operation:raise AdmissionError({'status':'BLOCKED','reason':'EXACT_OPERATION_ADMISSION_REQUIRED','operation':list(operation),'permitted_operations':[]})
+    engine,r=current
+    if engine.stop.exists():raise AdmissionError({'status':'BLOCKED','reason':'STOP_PERMANENT'})
+    return engine,r
+
+
+def explicit_input(path):
+    engine,r=require(('load','replay'))
+    p=artifact_path(path)
+    matches=[v for v in r['components'] if safe_path(engine.q['input_root'],v['path'])==p]
+    if len(matches)!=1:raise ValueError('Exact supplied component reference required')
+    b=p.read_bytes();v=matches[0]
+    import hashlib
+    if len(b)!=v['bytes'] or hashlib.sha256(b).hexdigest()!=v['sha256']:raise ValueError('Changed supplied model bytes')
+    # Original bounded native loader sees a private, stable copy of these bytes.
+    configure_temp();target=artifact_path(Path(bounded_directory())/'input.json');target.write_bytes(b)
+    return target

morphtree/_experimental/sampling.py

@@ -0,0 +1,26 @@
+import random
+
+def class_groups(y):
+    groups={}
+    for i,value in enumerate(y):groups.setdefault(value,[]).append(i)
+    return [groups[value] for value in sorted(groups)]
+def bootstrap(y, task, seed):
+    rng = random.Random(seed)
+    if task == 'regression':
+        return [rng.randrange(len(y)) for _ in y]
+    groups = class_groups(y)
+    ids = [rng.choice(group) for group in groups for _ in group]
+    rng.shuffle(ids)
+    return ids
+
+def oof_folds(y, task, seed):
+    groups = [list(range(len(y)))] if task == 'regression' else class_groups(y)
+    rng = random.Random(seed)
+    folds = [[], []]
+    for group in groups:
+        if len(group) < 2:
+            raise ValueError('OOF requires at least two examples in every TRAIN class')
+        rng.shuffle(group)
+        for j, i in enumerate(group):
+            folds[j % 2].append(i)
+    return [sorted(v) for v in folds]