morphtree 2.1.0__py3-none-win_amd64.whl

morphtree/__init__.py

@@ -0,0 +1,28 @@
+"""MorphTree: own native strategies and a bounded evolving learning program."""
+from .config import AlgorithmGenome,HyperParams,MorphTreeConfig,VERSION as __native_core_version__
+from .model import MorphTreeClassifier as NativeMorphTreeClassifier,MorphTreeRegressor as NativeMorphTreeRegressor,MorphTreeMulticlassClassifier,AutoMorphTree
+from .persistence import LoadLimits
+from ._installed import activate as _activate_installed
+_activate_installed(__file__)
+from .api import MorphTree,TrainingNotEnabledError
+from .estimators import MorphTreeClassifier,MorphTreeRegressor
+__version__='2.1.0'
+__all__=['MorphTreeClassifier','MorphTreeRegressor','MorphTree','load_model','AlgorithmGenome','HyperParams','MorphTreeConfig','LoadLimits','__version__']
+__runtime__={'release_candidate':True,'backend':'automatic-native',
+             'implementation':'Python with verified C kernels',
+             'backend_policy':'C histograms and cached traversal; Python traversal below 128 rows',
+             'native_loading':'Lazy; runtime_info() reports the loaded native library',
+             'qualification':'Documented OS, interpreter, task and scale scope only',
+             'reference_version':'2.1.0 (historical Python implementation)',
+             'publication_approved':False,'scientific_training_authority':0}
+def load_model(path,*,sha256=None):
+    """Load the public model with bounded JSON validation; a digest is optional."""
+    return MorphTree.load(path,sha256=sha256)
+
+# Scalable public task facades; retain the legacy MorphTree/artifact decoder.
+from .scalable import MorphTreeClassifier,MorphTreeRegressor
+__runtime__.update(private_research=False,scope="ORDINARY_USER_WITHOUT_SCIENTIFIC_AUTHORITY")
+def load_model(path,*,sha256=None):
+    """Load current and historical models with validated bounded JSON."""
+    from .scalable import _Scalable
+    return _Scalable.load(path,sha256=sha256)

morphtree/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

morphtree/_experimental/__init__.py

@@ -0,0 +1 @@
+"""Private native runtime; legacy namespace retained for model compatibility."""

morphtree/_experimental/admission.py

@@ -0,0 +1,364 @@
+"""Portable experimental admission, zero learning budget; no checkout imports.
+
+The caller pins a qualification dossier out of band. A self-supplied matching
+name/version/hash is not trust. This cooperative guard is not an OS sandbox.
+"""
+import copy
+import hashlib
+import importlib
+import json
+import math
+import os
+import platform
+import sys
+import time
+from contextvars import ContextVar
+from contextlib import contextmanager
+from pathlib import Path
+
+OPERATIONS = frozenset(('inspect','load','infer','save','export','replay','optuna_import',
+    'fit','trial','OOF','campaign','independent_evaluation','release_prepare'))
+LEARNING = frozenset(('fit','trial','OOF','campaign'))
+STATES = frozenset(('ADMITTED','REJECTED','BLOCKED','INVALID'))
+CURRENT = ContextVar('morph030_exact_operation',default=None)
+
+
+def digest(value):
+    return hashlib.sha256(json.dumps(value,sort_keys=True,separators=(',',':'),ensure_ascii=False,allow_nan=False).encode('utf8')).hexdigest()
+
+
+def sha(path):
+    return hashlib.sha256(Path(path).read_bytes()).hexdigest()
+
+
+def read(path,limit=32*1024**2,*,expected_sha256=None):
+    b=Path(path).read_bytes()
+    if len(b)>limit:raise ValueError('Bounded input exceeded')
+    if expected_sha256 is not None and hashlib.sha256(b).hexdigest()!=expected_sha256:raise ValueError('Changed bound input bytes')
+    def pairs(items):
+        d={}
+        for k,v in items:
+            if k in d:raise ValueError('Duplicate JSON key')
+            d[k]=v
+        return d
+    def bad(_):raise ValueError('Nonfinite JSON')
+    v=json.loads(b.decode('utf8'),object_pairs_hook=pairs,parse_constant=bad)
+    digest(v)
+    return v
+
+
+def safe_path(root,value):
+    root=Path(root).resolve();p=Path(value)
+    p=p if p.is_absolute() else root/p
+    # Reject links/reparse redirections even when their final target is internal.
+    for a in (p,*p.parents):
+        if a.exists() and (a.is_symlink() or getattr(a.stat(),'st_file_attributes',0)&0x400):raise ValueError('Link/reparse path')
+    p=p.resolve()
+    if not p.is_relative_to(root):raise ValueError('Path outside explicit working directory')
+    return p
+
+
+def environment():
+    return {'python':sys.version,'implementation':platform.python_implementation(),
+        'platform':sys.platform,'executable_sha256':sha(sys.executable),'backend':'pure'}
+
+
+def installed_identity(package,qualification):
+    package=Path(package).resolve();q=qualification
+    if q.get('usage')=='ORDINARY_USER_NOT_SCIENTIFIC':
+        from morphtree.api import _user_identity
+        return _user_identity(package,q)
+    if q.get('schema')!='morph030-qualified-installed-v1':raise ValueError('Unknown qualification schema')
+    if q.get('version')!='1.0.0rc4' or q.get('backend')!='pure':raise ValueError('Unqualified version/backend')
+    if environment()!=q['environment']:raise ValueError('Unqualified execution environment')
+    manifest=package/'_experimental/manifest.json'
+    m=read(manifest,expected_sha256=q['manifest_sha256'])
+    if m['version']!=q['version'] or digest(m)!=q['build_identity']:raise ValueError('Incoherent build identity')
+    actual={p.relative_to(package).as_posix() for p in package.rglob('*') if p.is_file() and '__pycache__' not in p.parts}
+    if actual!=set(m['files'])|{'_experimental/manifest.json'}:raise ValueError('Missing/unexpected installed component')
+    for name,h in m['files'].items():
+        if sha(safe_path(package,name))!=h:raise ValueError('Changed installed component: '+name)
+    infos=list(package.parent.glob('morphtree-*.dist-info'))
+    if len(infos)!=1:raise ValueError('Unique experimental distribution metadata required')
+    meta=infos[0]/'METADATA'
+    metadata=meta.read_bytes()
+    if hashlib.sha256(metadata).hexdigest()!=q['metadata_sha256']:raise ValueError('Changed metadata')
+    text=metadata.decode('utf8')
+    if 'Name: morphtree\n' not in text or 'Version: '+q['version']+'\n' not in text:raise ValueError('Metadata name/version mismatch')
+    # local pip records the supplied wheel hash, not merely the package name.
+    direct=read(infos[0]/'direct_url.json')
+    if direct.get('archive_info',{}).get('hashes',{}).get('sha256')!=q['wheel_sha256']:raise ValueError('Installed wheel provenance mismatch')
+    import morphtree
+    if Path(morphtree.__file__).resolve()!=package/'__init__.py':raise ValueError('Shadowed runtime')
+    for name,module in tuple(sys.modules.items()):
+        if name=='morphtree' or name.startswith('morphtree.'):
+            f=getattr(module,'__file__',None)
+            if f and not Path(f).resolve().is_relative_to(package):raise ValueError('Mixed native module origins')
+    return {'runtime':'morphtree-experimental-installed','version':q['version'],
+        'artifact_sha256':q['wheel_sha256'],'build_identity':q['build_identity'],
+        'manifest_sha256':q['manifest_sha256'],'backend':'pure','environment':environment()}
+
+
+def validate_roles(source,expected):
+    """Role is bound to actual row content/order and trusted source declaration.
+
+    Origin/exposure is a provenance assertion pinned by the dossier, never
+    inferred from a filename. These checks cannot discover dishonest originals.
+    """
+    if source!=expected:raise ValueError('Source/role/provenance differs from qualified declaration')
+    if set(source)!={'origin','exposure','sealed','rows','partitions','OOF'}:raise ValueError('Exact source schema required')
+    if source['sealed'] or source['exposure'] not in ('DEV_EXPOSED','SOFTWARE_FIXTURE'):raise ValueError('Sealed/independent source not admitted')
+    rows=source['rows'];ids=[r['id'] for r in rows]
+    if not ids or len(set(ids))!=len(ids):raise ValueError('Unique original row IDs required')
+    # Aliased IDs cannot turn a row into two data roles.
+    contents=[digest({'X':r['X'],'y':r['y']}) for r in rows]
+    if len(set(contents))!=len(contents):raise ValueError('Ambiguous duplicate-row provenance requires another protocol')
+    parts=source['partitions']
+    if set(parts)!={'TRAIN','SEARCH','GUARD_FINAL'} or parts['GUARD_FINAL']:raise ValueError('No final GUARD access in this scope')
+    t,s=parts['TRAIN'],parts['SEARCH']
+    if not t or not s or len(set(t+s))!=len(t+s) or set(t+s)!=set(ids):raise ValueError('Exact disjoint TRAIN/SEARCH coverage required')
+    folds=source['OOF']
+    if folds:
+        held=[]
+        for f in folds:
+            if set(f)!={'fit','held','recipe_sha256'} or set(f['fit'])&set(f['held']) or set(f['fit'])|set(f['held'])!=set(t):raise ValueError('OOF must use disjoint TRAIN-only folds')
+            if len(set(f['fit']))!=len(f['fit']) or len(set(f['held']))!=len(f['held']) or not f['fit'] or not f['held']:raise ValueError('Nonempty unique OOF IDs')
+            held+=f['held']
+        if len(set(held))!=len(held) or set(held)!=set(t) or len({f['recipe_sha256'] for f in folds})!=1:raise ValueError('Fixed recipe/exact held coverage required')
+    return digest(source)
+
+
+def capacity(recipe,task):
+    if recipe.get('schema')=='comp009-evolution-capacity-v1':
+        from morphtree.config import AlgorithmGenome
+        if set(recipe)!={'schema','initial','OOF','child_genome'} or task!='binary':
+            raise ValueError('Exact evolution capacity schema/task')
+        if recipe['OOF']!={'calls':8,'scalar_trees':8,'folds':2,'max_contexts':2} or recipe['child_genome']!=AlgorithmGenome(growth_policy='depthwise').to_dict():
+            raise ValueError('Exact cumulative OOF/child capacity required')
+        initial=recipe['initial']
+        if set(initial)!={'candidates'} or len(initial['candidates'])!=2:
+            raise ValueError('Two exact initial native forms')
+        forms=initial['candidates']
+        expected=[{'family':f,'members':1 if f=='boost' else 2,'stages':2 if f=='boost' else 1,
+                   'classes':2,'genome_sha256':digest(AlgorithmGenome().to_dict())} for f in ('boost','bag')]
+        if forms!=expected or capacity(initial,task)!=4:
+            raise ValueError('Fixed initial native capacity')
+        return 14
+    if set(recipe)=={'candidates','research_genome'}:
+        from morphtree.config import AlgorithmGenome
+        if task!='binary' or recipe['research_genome']!=AlgorithmGenome(growth_policy='depthwise').to_dict():
+            raise ValueError('Exact operative binary research genome required')
+        base=capacity({'candidates':recipe['candidates']},task)
+        return base+max(capacity(form,task) for form in recipe['candidates'])
+    if set(recipe)=={'candidates'}:
+        if type(recipe['candidates']) is not list or not 1<=len(recipe['candidates'])<=2:raise ValueError('Finite one/two native candidate path')
+        families=[p['family'] for p in recipe['candidates']]
+        if len(set(families))!=len(families):raise ValueError('No duplicate family budget alias')
+        return sum(capacity(p,task) for p in recipe['candidates'])
+    if set(recipe)!={'family','members','stages','classes','genome_sha256'} or recipe['family'] not in ('boost','bag'):raise ValueError('Bounded frozen native recipe required')
+    for n in ('members','stages','classes'):
+        if type(recipe[n]) is not int or recipe[n]<1:raise ValueError('Integer actual capacity required')
+    if recipe['family']=='boost' and recipe['members']!=1 or recipe['family']=='bag' and recipe['stages']!=1:raise ValueError('Reset bag versus boost capacity mismatch')
+    if task=='binary' and recipe['classes']!=2 or task=='regression' and recipe['classes']!=1 or task=='multiclass' and not 2<=recipe['classes']<=64:raise ValueError('Task/class capacity mismatch')
+    return recipe['members']*recipe['stages']*(recipe['classes'] if task=='multiclass' else 1)
+
+
+class AdmissionError(RuntimeError):
+    def __init__(self,decision):
+        self.decision=decision
+        super().__init__(decision['status']+': '+decision['reason'])
+
+
+class AdmissionEngine:
+    """One finite, durable, cooperative zero-fit capability ledger.
+
+    Admission authorizes exactly the supplied operation/request hash once. An
+    unfinished INTENT is never silently replayed; physical execution is closed.
+    """
+    def __init__(self,*,package,qualification_path,qualification_sha256,workdir):
+        self.q=read(qualification_path,expected_sha256=qualification_sha256);self.package=Path(package).resolve();self.workdir=Path(workdir).resolve()
+        if not self.workdir.is_dir():raise ValueError('Explicit existing work directory required')
+        self.identity=installed_identity(self.package,self.q);self._q_pin=digest(self.q)
+        self.ledger=safe_path(self.workdir,'ADMISSION.jsonl');self.stop=safe_path(self.workdir,'STOP');self.lock=safe_path(self.workdir,'WRITER.lock')
+        self.started=time.monotonic();self.deadline=self.q['deadline_utc_epoch']
+        public=self.q.get('usage')=='ORDINARY_USER_NOT_SCIENTIFIC'
+        if not public and (self.q['limits']['fits']!=0 or self.q['limits']['trials']!=0):raise ValueError('No training authority in MORPH030')
+        if public and (self.q['limits']['fits']!=128 or type(self.q['limits']['trials']) is not int or not 0<=self.q['limits']['trials']<=4):raise ValueError('Exact user caps required')
+        if self.q['scalar_cap']!=(2048 if public else 32) or not 0<self.q['limits']['operations']<=(256 if public else 100) or not 0<self.q['limits']['seconds']<=(300 if public else 600):raise ValueError('Exact finite MORPH030 profile required')
+
+    @contextmanager
+    def _lease(self):
+        fd=os.open(self.lock,os.O_CREAT|os.O_EXCL|os.O_WRONLY)
+        try:yield
+        finally:os.close(fd);self.lock.unlink()
+
+    def _events(self):
+        if not self.ledger.exists():return []
+        events=[];prev='0'*64
+        for line in self.ledger.read_text('utf8').splitlines():
+            v=json.loads(line);h=v.pop('sha256')
+            if v['previous']!=prev or digest(v)!=h:raise ValueError('Corrupt/partial admission journal; reconcile, never replay')
+            prev=h;events.append(v)
+        return events
+
+    def _event(self,kind,request_id,**values):
+        old=self._events();v={'event':kind,'request_id':request_id,'previous':digest(old[-1]) if old else '0'*64,'utc':time.time(),'monotonic':time.monotonic(),**values};v['sha256']=digest(v)
+        with self.ledger.open('a',encoding='utf8') as f:f.write(json.dumps(v,sort_keys=True,allow_nan=False)+'\n');f.flush();os.fsync(f.fileno())
+
+    def request(self,*,operation,task,recipe,source,role,protocol,authorization,components,requested_cost=None):
+        r={'runtime':copy.deepcopy(self.identity),'operation':operation,'task':task,'recipe':copy.deepcopy(recipe),
+            'effective_capacity':capacity(recipe,task),'source':copy.deepcopy(source),'role':role,
+            'protocol':protocol,'authorization':authorization,'components':copy.deepcopy(components),
+            'budget':copy.deepcopy(self.q['limits']),'deadline':self.deadline,'requested_cost':requested_cost or {'fits':0,'trials':0}}
+        r['id']=digest(r);return r
+
+    def decide(self,r):
+        try:request_hash=digest(r)
+        except (ValueError,TypeError,RecursionError):request_hash=None
+        if type(r) is not dict or request_hash is None:return {'status':'INVALID','reason':'NONCANONICAL_REQUEST','request_sha256':request_hash,'operation':None,'permitted_operations':[],'physical_fits_authorized':0,'trials_authorized':0}
+        def result(status,reason):return {'status':status,'reason':reason,'request_sha256':request_hash,'operation':r.get('operation'),'permitted_operations':[r['operation']] if status=='ADMITTED' else [],'physical_fits_authorized':r.get('requested_cost',{}).get('fits',0) if status=='ADMITTED' and self.q.get('usage')=='ORDINARY_USER_NOT_SCIENTIFIC' else 0,'trials_authorized':r.get('requested_cost',{}).get('trials',0) if status=='ADMITTED' and self.q.get('usage')=='ORDINARY_USER_NOT_SCIENTIFIC' else 0}
+        try:
+            if digest(self.q)!=self._q_pin:return result('INVALID','MUTATED_QUALIFICATION')
+            if installed_identity(self.package,self.q)!=self.identity or r['runtime']!=self.identity:return result('REJECTED','RUNTIME_IDENTITY')
+            if set(r)!={'id','runtime','operation','task','recipe','effective_capacity','source','role','protocol','authorization','components','budget','deadline','requested_cost'}|({'component_key'} if self.q.get('usage')=='ORDINARY_USER_NOT_SCIENTIFIC' and r['operation'] in ('fit','trial') else set()):return result('INVALID','REQUEST_SCHEMA')
+            if r['id']!=digest({k:v for k,v in r.items() if k!='id'}):return result('INVALID','REQUEST_ID')
+            if r['operation'] not in OPERATIONS or r['task'] not in ('binary','multiclass','regression'):return result('INVALID','OPERATION_OR_TASK')
+            if self.stop.exists() or any(e['event']=='STOP' for e in self._events()):return result('BLOCKED','STOP_PERMANENT')
+            if time.time()>=self.deadline or time.monotonic()-self.started>=self.q['limits']['seconds'] or r['deadline']!=self.deadline:return result('BLOCKED','DEADLINE')
+            if r['protocol']!=self.q['protocol'] or r['authorization']!=self.q['authorization']['id']:return result('REJECTED','PROTOCOL_OR_AUTHORIZATION')
+            if digest(r['budget'])!=digest(self.q['limits']):return result('REJECTED','BUDGET_IDENTITY')
+            c=capacity(r['recipe'],r['task'])
+            if c!=r['effective_capacity']:return result('INVALID','MISREPORTED_CAPACITY')
+            if c>self.q['scalar_cap']:return result('REJECTED','CAPACITY_ABOVE_32_QUALIFIED_PROFILE')
+            key=digest(r['source'])
+            if key not in self.q['sources']:return result('REJECTED','UNKNOWN_SOURCE_OR_RELABELLED_ROLE')
+            _user_roles(r['source'],self.q['sources'][key]) if self.q.get('usage')=='ORDINARY_USER_NOT_SCIENTIFIC' else validate_roles(r['source'],self.q['sources'][key])
+            if r['role'] not in ('TRAIN','SEARCH','OOF','DEV'):return result('REJECTED','GUARD_OR_UNKNOWN_ROLE')
+            if r['role']=='OOF' and not r['source']['OOF']:return result('REJECTED','OOF_PROVENANCE_NOT_SUPPLIED')
+            if r['operation'] in LEARNING and r['role']!='TRAIN':return result('REJECTED','TRAINING_REQUIRES_TRAIN')
+            if r['operation']=='replay' and r['role']!='SEARCH':return result('REJECTED','REPLAY_REQUIRES_SEARCH')
+            if r['operation']=='independent_evaluation':return result('REJECTED','DEV_NOT_INDEPENDENT')
+            for ref in r['components']:
+                ev=self._events();completed={e['request_id'] for e in ev if e['event']=='COMPLETE'}
+                produced=[e['artifact'] for e in ev if e['event']=='ARTIFACT' and e['request_id'] in completed]
+                if ref not in self.q['components'] and ref not in produced:return result('REJECTED','UNQUALIFIED_COMPONENT')
+                contracts=self.q['component_contracts']
+                contract=contracts.get(ref['path'])
+                if contract is None:
+                    contract=next((e['contract'] for e in ev if e['event']=='ARTIFACT' and e['artifact']==ref and e['request_id'] in completed),None)
+                if contract!={'task':r['task'],'recipe':r['recipe']}:return result('REJECTED','ACTUAL_COMPONENT_TASK_OR_CAPACITY')
+                p=safe_path(self.q['input_root'],ref['path'])
+                if p.stat().st_size!=ref['bytes'] or sha(p)!=ref['sha256']:return result('REJECTED','COMPONENT_HASH')
+            ev=self._events();used={e['request_id'] for e in ev if e['event']=='INTENT'}
+            if r['id'] in used:return result('BLOCKED','REQUEST_CONSUMED_OR_INCOMPLETE')
+            if len(used)>=self.q['limits']['operations']:return result('BLOCKED','OPERATION_BUDGET_EXHAUSTED')
+            if self.q.get('usage')=='ORDINARY_USER_NOT_SCIENTIFIC' and r['operation'] in ('fit','trial'):
+                cost=r['requested_cost'];wanted={'fits':int(r['operation']=='fit'),'trials':int(r['operation']=='trial')}
+                if cost!=wanted or any(type(v) is not int for v in cost.values()):return result('INVALID','USER_COST')
+                if r['operation'] not in self.q['authorization']['operations']:return result('REJECTED','USER_OPERATION')
+                key=r['component_key']
+                if type(key) is not str or len(key)!=64 or any(c not in '0123456789abcdef' for c in key):return result('INVALID','USER_COMPONENT_ID')
+                for unit in ('fits','trials'):
+                    if sum(e.get(unit,0) for e in ev if e['event']=='INTENT')+cost[unit]>self.q['limits'][unit]:return result('BLOCKED','USER_CUMULATIVE_BUDGET')
+                return result('ADMITTED','EXACT_ORDINARY_USER_OPERATION_NOT_SCIENTIFIC')
+            if digest(r['requested_cost'])!=digest({'fits':0,'trials':0}):return result('BLOCKED','LEARNING_BUDGET_ZERO')
+            if r['operation'] in LEARNING:return result('BLOCKED','NEW_TRAINING_BOUNDARY_UNQUALIFIED_ZERO_BUDGET')
+            if r['operation'] not in self.q['authorization']['operations']:return result('REJECTED','OPERATION_NOT_AUTHORIZED')
+            return result('ADMITTED','EXACT_ZERO_FIT_OPERATION_ONLY')
+        except (KeyError,ValueError,TypeError,OSError) as e:return result('INVALID',type(e).__name__+': '+str(e))
+
+    @contextmanager
+    def operation(self,r):
+        with self._lease():
+            d=self.decide(r)
+            self._event('DECISION',r.get('id','INVALID'),decision=d)
+            if d['status']!='ADMITTED':raise AdmissionError(d)
+            self._event('INTENT',r['id'],request_sha256=digest(r),operation=r['operation'],fits=0,trials=0)
+            token=CURRENT.set((self,copy.deepcopy(r)))
+            try:
+                yield d
+                # Revalidate artifact/deadline/STOP after the callback without
+                # treating our own already persisted intent as a replay.
+                installed_identity(self.package,self.q)
+                if self.stop.exists() or any(e['event']=='STOP' for e in self._events()) or time.time()>=self.deadline or time.monotonic()-self.started>=self.q['limits']['seconds']:raise RuntimeError('STOP/deadline before completion')
+                self._event('COMPLETE',r['id'],fits=0,trials=0)
+            except BaseException as e:
+                self._event('INCOMPLETE',r['id'],error=type(e).__name__,fits=0,trials=0);raise
+            finally:CURRENT.reset(token)
+
+    @contextmanager
+    def fold_operation(self,r):
+        # The outer operation owns WRITER.lock across yield. Reuse that lease
+        # only for this declared TRAIN-fold child, not arbitrary nested access.
+        current=CURRENT.get()
+        if current is None or current[0] is not self:
+            raise ValueError('OOF child requires the active owned parent operation')
+        outer=current[1]
+        if outer['operation']!='inspect' or outer['role']!='TRAIN' or outer['recipe'].get('schema')!='comp009-evolution-capacity-v1' or r['operation']!='inspect' or r['role']!='OOF':
+            raise ValueError('Exact versioned parent/OOF inspect relationship')
+        if any(r['source'][k]!=outer['source'][k] for k in ('origin','rows','partitions','sealed','exposure')):
+            raise ValueError('OOF source differs from actual parent TRAIN/SEARCH')
+        d=self.decide(r);self._event('DECISION',r['id'],decision=d,parent_request=outer['id'])
+        if d['status']!='ADMITTED':raise AdmissionError(d)
+        self._event('INTENT',r['id'],request_sha256=digest(r),operation='inspect',parent_request=outer['id'],fits=0,trials=0)
+        token=CURRENT.set((self,copy.deepcopy(r)))
+        try:
+            yield d
+            installed_identity(self.package,self.q)
+            if self.stop.exists() or time.time()>=self.deadline or time.monotonic()-self.started>=self.q['limits']['seconds']:
+                raise RuntimeError('STOP/deadline before OOF completion')
+            self._event('COMPLETE',r['id'],fits=0,trials=0)
+        except BaseException as e:
+            self._event('INCOMPLETE',r['id'],error=type(e).__name__,fits=0,trials=0);raise
+        finally:CURRENT.reset(token)
+
+    @contextmanager
+    def user_learning(self,r):
+        current=CURRENT.get()
+        if self.q.get('usage')!='ORDINARY_USER_NOT_SCIENTIFIC' or current is None or current[0] is not self or current[1]['operation']!='inspect' or r['operation'] not in ('fit','trial'):
+            raise PermissionError('Exact ordinary-user parent inspect required')
+        d=self.decide(r);self._event('DECISION',r['id'],decision=d)
+        if d['status']!='ADMITTED':raise AdmissionError(d)
+        self._event('INTENT',r['id'],operation=r['operation'],fits=r['requested_cost']['fits'],trials=r['requested_cost']['trials'],domain='ORDINARY_USER_RESERVATION')
+        try:
+            yield d
+            self._event('COMPLETE',r['id'],domain='ORDINARY_USER_RESERVATION')
+        except BaseException as exc:
+            self._event('INCOMPLETE',r['id'],error=type(exc).__name__,domain='ORDINARY_USER_RESERVATION');raise
+
+    def record_output(self,path):
+        current=CURRENT.get()
+        if current is None or current[0] is not self or current[1]['operation'] not in ('save','export'):raise ValueError('Output requires current writer operation')
+        p=safe_path(self.q['input_root'],path)
+        ref={'path':p.relative_to(Path(self.q['input_root']).resolve()).as_posix(),'sha256':sha(p),'bytes':p.stat().st_size}
+        self._event('ARTIFACT',current[1]['id'],artifact=ref,contract={'task':current[1]['task'],'recipe':current[1]['recipe']})
+        return ref
+
+    def stop_permanently(self,reason):
+        with self._lease():
+            if not self.stop.exists():
+                with self.stop.open('x',encoding='utf8') as f:f.write(str(reason));f.flush();os.fsync(f.fileno())
+            self._event('STOP','CONTROLLER',reason=str(reason))
+
+    def optional_optuna(self,r):
+        if r['operation']!='optuna_import':raise ValueError('Import-only capability required')
+        with self.operation(r):
+            lock=self.q['optional_dependency']
+            root=Path(lock['root']).resolve()
+            for name,h in lock['files'].items():
+                if sha(safe_path(root,name))!=h:raise ValueError('Changed optional dependency lock')
+            import importlib.util
+            spec=importlib.util.find_spec('optuna')
+            if spec is None:raise ImportError('Optional Optuna absent')
+            if Path(spec.origin).resolve()!=root/'optuna/__init__.py':raise ValueError('Unknown Optuna origin')
+            module=importlib.import_module('optuna')
+            if module.__version__!=lock['version']:raise ValueError('Incompatible Optuna version')
+            return {'status':'ADMITTED_IMPORT_ONLY','version':module.__version__,'files_verified':len(lock['files']),'new_studies':0,'new_trials':0}
+
+
+def user_qualification(package,root,source,protocol,seconds,user_limits):
+    from morphtree.api import _user_qualification
+    return _user_qualification(package,root,source,protocol,seconds,user_limits)
+def _user_roles(source,expected):
+    from morphtree.api import _user_source
+    return _user_source(source,expected)

morphtree/_experimental/base.py

@@ -0,0 +1,215 @@
+import copy, math, json, tempfile, os
+from ..temporary import OwnedTemporaryDirectory
+from pathlib import Path
+from dataclasses import asdict, replace
+from morphtree.config import AlgorithmGenome, HyperParams
+from morphtree.persistence import read_json, write_json, LoadLimits
+from morphtree.model import MorphTreeClassifier, MorphTreeRegressor, MorphTreeMulticlassClassifier
+from morphtree.metrics import point_losses
+from .admission import digest, sha, AdmissionError
+from .runtime_io import artifact_path, configure_temp, reference, publish
+NATIVE = {'binary': MorphTreeClassifier, 'multiclass': MorphTreeMulticlassClassifier, 'regression': MorphTreeRegressor}
+TASKS = ('binary', 'multiclass', 'regression')
+from .support import FamilyModel
+from .preprocessing import TrainEncoder, _m26_rows, _m26_scalar
+PARAMS = ('task', 'families', 'morphing', 'tuning', 'preset', 'metric', 'time_budget', 'n_trials', 'n_jobs', 'random_state', 'verbose')
+
+def _payloads(model):
+    configure_temp()
+    out = []
+    with OwnedTemporaryDirectory() as tmp:
+        for i, m in enumerate(model.components):
+            p = artifact_path(Path(tmp) / ('member-' + str(i) + '.json'))
+            m.save_model(p)
+            out.append(read_json(p))
+    return out
+
+def _load_members(task, payloads):
+    if task not in NATIVE or type(payloads) is not list or (not 1 <= len(payloads) <= 32):
+        raise ValueError('Bounded known native family required')
+    if any(type(payload) is not dict for payload in payloads):
+        raise ValueError('Each native model payload must be a JSON object')
+    models = [NATIVE[task]._from_payload(p, limits=LoadLimits(max_bytes=16 * 1024 ** 2), backend='pure') for p in payloads]
+    return FamilyModel(task, models, [1 / len(models)] * len(models))
+
+class Base:
+
+    def __init__(self, *, task='auto', families='auto', morphing='auto', tuning='auto', preset='balanced', metric='auto', time_budget='auto', n_trials='auto', n_jobs='auto', random_state=None, verbose=0):
+        for k, v in locals().copy().items():
+            if k != 'self':
+                setattr(self, k, v)
+        self._validate()
+
+    def _validate(self):
+        if self.task not in ('auto', 'classification', 'binary', 'multiclass', 'regression') or self.morphing not in ('auto', 'none') or self.tuning not in ('auto', 'none', 'optuna') or (self.preset != 'balanced'):
+            raise ValueError('Unsupported task/morphing/tuning/preset')
+        if self.families not in ('auto', 'boost', 'bag'):
+            raise ValueError('families must be auto, boost or bag')
+        if self.metric not in ('auto', 'logloss', 'mse'):
+            raise ValueError('Only task-consistent logloss/mse primary metrics are supported')
+        if self.n_jobs != 'auto' and (type(self.n_jobs) is not int or self.n_jobs != 1):
+            raise ValueError('Experimental serial scheduler supports n_jobs=1 or auto only')
+        if self.n_trials != 'auto' and (type(self.n_trials) is not int or not 1 <= self.n_trials <= 4):
+            raise ValueError('n_trials must be auto or 1..4')
+        if self.time_budget != 'auto' and (type(self.time_budget) not in (int, float) or not math.isfinite(self.time_budget) or (not 0 < self.time_budget <= 300)):
+            raise ValueError('Finite 0<time_budget<=300 required')
+        if self.random_state is not None and (type(self.random_state) is not int or not 0 <= self.random_state < 2 ** 31):
+            raise ValueError('random_state must be None or an integer in [0,2**31)')
+        if type(self.verbose) is not int or self.verbose not in (0, 1):
+            raise ValueError('verbose must be 0 or 1')
+
+    def get_params(self, deep=True):
+        return {k: getattr(self, k) for k in PARAMS}
+
+    def set_params(self, **values):
+        if set(values) - set(PARAMS):
+            raise ValueError('Unknown estimator parameter')
+        new = type(self)(**self.get_params() | values)
+        self.__dict__ = new.__dict__
+        return self
+
+    def _fitted(self):
+        if not hasattr(self, 'model_'):
+            raise RuntimeError('MorphTree experimental estimator is not fitted')
+
+    def predict_proba(self, X):
+        self._fitted()
+        from ..source_markers import controlled_source
+        if controlled_source(X):raise ValueError('Controlled sources cannot use ordinary legacy inference')
+        if self.task_ == 'regression':
+            raise ValueError('predict_proba is unavailable for regression')
+        p = self.model_.probabilities_or_values(self.encoder_.transform(X))
+        return [[1 - v, v] for v in p] if self.task_ == 'binary' else p
+
+    def predict(self, X):
+        self._fitted()
+        from ..source_markers import controlled_source
+        if controlled_source(X):raise ValueError('Controlled sources cannot use ordinary legacy inference')
+        if self.task_ == 'regression':
+            return self.model_.probabilities_or_values(self.encoder_.transform(X))
+        return [self.classes_[max(range(len(p)), key=p.__getitem__)] for p in self.predict_proba(X)]
+
+    def report(self):
+        self._fitted()
+        return copy.deepcopy(self.report_)
+
+    def save(self, path):
+        self._fitted()
+        path = artifact_path(path)
+        data = {'schema': {'comp009-frozen-form-v1': 'comp009-native-estimator-v1', 'comp003-frozen-form-v1': 'comp003-native-estimator-v1', 'comp002-frozen-form-v1': 'comp002-native-estimator-v1', 'product002-frozen-form-v1': 'product002-native-estimator-v1', 'product003-frozen-form-v1': 'product003-native-estimator-v1', 'product006-frozen-form-v1': 'product006-native-estimator-v1'}.get(self.best_recipe_['schema'], 'morph026-native-estimator-v1'), 'constructor': self.get_params(), 'recipe': copy.deepcopy(self.best_recipe_), 'classes': list(getattr(self, 'classes_', ())), 'report': self.report(), 'native_payloads': _payloads(self.model_)}
+        if len(json.dumps(data, allow_nan=False).encode('utf8')) > 32 * 1024 ** 2:
+            raise ValueError('Aggregate private model cap is 32MiB')
+        write_json(path, data)
+
+    def save_recipe(self, path):
+        """Declarative frozen form/params, no weights or self-executing training."""
+        self._fitted()
+        write_json(artifact_path(path), self.best_recipe_)
+
+    def export_python(self, path):
+        """Inference only; reuse native standalone runtime and exact preprocessing.
+
+        No fitter, search, Optuna, private TRAIN rows or acquisition credentials
+        are embedded. This is not a retrainable procedure or a pure/tuned pair.
+        """
+        import ast
+        import morphtree.exporting as exporting
+        from . import preprocessing
+        self._fitted()
+        payloads = _payloads(self.model_)
+        class_name = NATIVE[self.task_].__name__
+        text = exporting._bundle_runtime(inference_only=True, task=self.task_) + '\n'
+        from .. import source_markers
+        text += Path(source_markers.__file__).read_text('utf8') + '\n'
+        text += Path(preprocessing.__file__).read_text('utf8') + '\n'
+        text += '_M26_ENCODER = TrainEncoder.from_dict(' + repr(self.encoder_.to_dict()) + ')\n'
+        text += '_M26_TASK = ' + repr(self.task_) + '\n_M26_CLASSES = ' + repr(tuple(getattr(self, 'classes_', ()))) + '\n'
+        text += '_M26_MEMBERS = [' + ','.join((class_name + '._from_payload(json.loads(' + repr(json.dumps(p, allow_nan=False)) + '))' for p in payloads)) + ']\n'
+        text += "\ndef _m26_values(X):\n    if controlled_source(X):raise ValueError('Controlled sources cannot use ordinary legacy standalone inference')\n    rows=_M26_ENCODER.transform(X)\n    cols=[m.predict(rows) if _M26_TASK=='regression' else m.predict_proba(rows) for m in _M26_MEMBERS]\n    if _M26_TASK=='regression':return [math.fsum((1/len(cols))*c[i] for c in cols) for i in range(len(rows))]\n    if _M26_TASK=='binary':\n        p=[math.fsum((1/len(cols))*c[i][1] for c in cols) for i in range(len(rows))]\n        return [[1-v,v] for v in p]\n    return [[math.fsum((1/len(cols))*c[i][j] for c in cols) for j in range(len(_M26_CLASSES))] for i in range(len(rows))]\n\ndef predict_proba(X):\n    if _M26_TASK=='regression':raise ValueError('Regression has no probabilities')\n    return _m26_values(X)\n\ndef predict(X):\n    p=_m26_values(X)\n    return p if _M26_TASK=='regression' else [_M26_CLASSES[max(range(len(row)),key=row.__getitem__)] for row in p]\n"
+        ast.parse(text)
+        if len(text.encode('utf8')) > 32 * 1024 ** 2:
+            raise ValueError('Standalone cap is 32MiB')
+        from .runtime_io import publish
+        publish(path, text, lambda: None)
+
+    @classmethod
+    def load(cls, path):
+        d = read_json(artifact_path(path), LoadLimits(max_bytes=32 * 1024 ** 2))
+        if set(d) != {'schema', 'constructor', 'recipe', 'classes', 'report', 'native_payloads'} or d['schema'] not in ('morph026-native-estimator-v1', 'comp002-native-estimator-v1', 'comp003-native-estimator-v1', 'comp009-native-estimator-v1', 'product002-native-estimator-v1', 'product003-native-estimator-v1', 'product006-native-estimator-v1'):
+            raise ValueError('Exact research estimator schema required')
+        r = d['recipe']
+        expected = {'schema', 'task', 'family', 'structure', 'params', 'seed', 'training_identity', 'preprocessing', 'mutations'}
+        if type(r) is not dict or set(r) != expected or r['schema'] != {'morph026-native-estimator-v1': 'morph026-frozen-form-v1', 'comp002-native-estimator-v1': 'comp002-frozen-form-v1', 'comp009-native-estimator-v1': 'comp009-frozen-form-v1', 'comp003-native-estimator-v1': 'comp003-frozen-form-v1', 'product002-native-estimator-v1': 'product002-frozen-form-v1', 'product003-native-estimator-v1': 'product003-frozen-form-v1', 'product006-native-estimator-v1': 'product006-frozen-form-v1'}[d['schema']] or (r['task'] not in NATIVE) or (r['family'] not in ('boost', 'bag')) or (r['mutations'] != []):
+            raise ValueError('Unsupported private frozen form')
+        if type(r['structure']) is not dict or set(r['structure']) != ({'genome', 'classes', 'members', 'encoder', 'lineage'} if d['schema'] in ('comp003-native-estimator-v1', 'comp009-native-estimator-v1', 'product002-native-estimator-v1', 'product003-native-estimator-v1', 'product006-native-estimator-v1') else {'genome', 'classes', 'members', 'encoder'}) or type(d['report']) is not dict or (not isinstance(d['report'].get('stop_reason'), str)):
+            raise ValueError('Exact private structure/report required')
+        obj = cls(**d['constructor'])
+        encoder = TrainEncoder.from_dict(r['preprocessing'])
+        model = _load_members(r['task'], d['native_payloads'])
+        if any((m.n_features_in_ != encoder.width_ for m in model.components)) or (r['family'] == 'boost' and len(model.components) != 1) or r['structure']['members'] != len(model.components):
+            raise ValueError('Model/encoder/member shape mismatch')
+        HyperParams(**r['params'])
+        classes = d['classes']
+        if type(classes) is not list or any((type(c) not in (str, int, float) or (isinstance(c, str) and len(c) > 1024) for c in classes)) or (r['task'] == 'regression' and classes) or (r['task'] != 'regression' and (not 2 <= len(classes) <= 64 or not (all((isinstance(c, str) for c in classes)) or all((type(c) in (int, float) for c in classes))) or classes != sorted(set(classes)) or (r['task'] == 'binary' and len(classes) != 2))):
+            raise ValueError('Bounded target vocabulary required')
+        if r['task'] == 'multiclass' and any((list(m.classes_) != list(range(len(classes))) for m in model.components)):
+            raise ValueError('Native multiclass column alignment differs')
+        if r['structure'].get('encoder') != r['preprocessing'] or r['structure'].get('classes') != list(range(len(classes))) or (not _qualified_genome(d, model)):
+            raise ValueError('Changed frozen Base form/encoder/columns')
+        obj.model_ = model
+        obj.encoder_ = encoder
+        obj.task_ = r['task']
+        obj.n_features_in_ = len(encoder.state_['columns'])
+        obj.best_recipe_ = copy.deepcopy(r)
+        obj.best_family_ = r['family']
+        obj.best_params_ = copy.deepcopy(r['params'])
+        obj.report_ = copy.deepcopy(d['report'])
+        obj.stop_reason_ = d['report']['stop_reason']
+        if classes:
+            obj.classes_ = tuple(classes)
+        if encoder.state_['names'] is not None:
+            obj.feature_names_in_ = tuple(encoder.state_['names'])
+        return obj
+Base._execution_check = lambda self: None
+Base._record_event = lambda self, event: None
+
+def _qualified_genome(data, model):
+    if data['schema'] in ('product002-native-estimator-v1', 'product003-native-estimator-v1', 'product006-native-estimator-v1'):
+        from morphtree.config import validate_task_genome
+        r = data['recipe']
+        g = validate_task_genome(r['task'], AlgorithmGenome(**r['structure']['genome']))
+        base = AlgorithmGenome()
+        depth = AlgorithmGenome(growth_policy='depthwise')
+        if g not in (base, depth):
+            return False
+        lineage = r['structure']['lineage']
+        if g == base:
+            if lineage != {}:
+                return False
+        else:
+            keys = {'gene', 'parent_packet_sha256', 'from', 'to', 'reason'}
+            if type(lineage) is not dict or set(lineage) != keys or lineage['gene'] != 'growth_policy' or (lineage['from'] != base.to_dict()) or (lineage['to'] != g.to_dict()) or (lineage['reason'] != 'OOF_HEURISTIC_HYPOTHESIS_NOT_CAUSAL_UTILITY') or (type(lineage['parent_packet_sha256']) is not str) or (len(lineage['parent_packet_sha256']) != 64) or any((c not in '0123456789abcdef' for c in lineage['parent_packet_sha256'])):
+                return False
+        return all((m.genome_.to_dict() == g.to_dict() for m in model.components))
+    recipe = data['recipe']
+    g = recipe['structure'].get('genome')
+    allowed = [AlgorithmGenome().to_dict()]
+    if data['schema'] in ('comp002-native-estimator-v1', 'comp003-native-estimator-v1', 'comp009-native-estimator-v1', 'product002-native-estimator-v1', 'product003-native-estimator-v1', 'product006-native-estimator-v1') and recipe['task'] == 'binary':
+        allowed.append(AlgorithmGenome(growth_policy='depthwise').to_dict())
+    if g not in allowed:
+        return False
+    if data['schema'] == 'comp009-native-estimator-v1':
+        lineage = recipe['structure']['lineage']
+        if g == allowed[0]:
+            if lineage != {}:
+                return False
+        elif type(lineage) is not dict or set(lineage) != {'gene', 'parent_packet_sha256', 'from', 'to', 'reason'} or lineage['gene'] != 'growth_policy' or (lineage['from'] != allowed[0]) or (lineage['to'] != g) or (lineage['reason'] != 'OOF_HEURISTIC_HYPOTHESIS_NOT_CAUSAL_UTILITY') or (type(lineage['parent_packet_sha256']) is not str) or (len(lineage['parent_packet_sha256']) != 64) or any((c not in '0123456789abcdef' for c in lineage['parent_packet_sha256'])):
+            return False
+    if data['schema'] == 'comp003-native-estimator-v1':
+        lineage = recipe['structure']['lineage']
+        if g == allowed[0]:
+            if lineage != {}:
+                return False
+        elif type(lineage) is not dict or set(lineage) != {'gene', 'parent_packet_sha256', 'from', 'to', 'reason'} or lineage['gene'] != 'growth_policy' or (lineage['from'] != allowed[0]) or (lineage['to'] != g) or (lineage['reason'] != 'STATIC_TECHNICAL_NOMINATION_NOT_DIAGNOSTIC_UTILITY') or (type(lineage['parent_packet_sha256']) is not str) or (len(lineage['parent_packet_sha256']) != 64) or any((c not in '0123456789abcdef' for c in lineage['parent_packet_sha256'])):
+            return False
+    return all((m.genome_.to_dict() == g for m in model.components)) if recipe['task'] == 'binary' else g == allowed[0]