morphtree 2.1.0__py3-none-win_amd64.whl

morphtree/api.py

@@ -0,0 +1,388 @@
+"""Thin installed Python interface; existing learner/admission/loader only.
+
+This file is packaged as morphtree.api in the experimental wheel. An owned
+execution budget is supplied internally by the existing runner, never inferred
+from a constructor, a mode, a model file or an ordinary user call.
+"""
+import copy
+import hashlib
+import json
+import tempfile
+import threading
+import time
+import uuid
+from .temporary import OwnedTemporaryDirectory
+from contextlib import contextmanager
+from dataclasses import asdict
+from pathlib import Path
+from ._experimental.integrated import MorphTree as Integrated
+from ._experimental.admission import AdmissionError, sha
+from ._experimental.execution_v2 import require_session, ACTIVE, Session, PUBLIC_SCOPE
+
+_USER_RUN_LOCK=threading.Lock()
+PUBLIC_USER_ENABLED=True  # Enabled only by the explicitly versioned product build.
+PUBLIC_USER_RUNTIME='2.1.0'
+
+
+def _user_identity(package, q):
+    """Self-check installed user distribution; NOT scientific qualification."""
+    import base64,csv,hashlib,io,sys
+    from ._experimental import admission as A
+    package=Path(package).resolve()
+    if q.get('usage')!='ORDINARY_USER_NOT_SCIENTIFIC':raise ValueError('Public-user identity domain required')
+    manifest=A.read(package/'_experimental/manifest.json',expected_sha256=q['manifest_sha256'])
+    if manifest['version']!=q['version'] or q['version']!=PUBLIC_USER_RUNTIME or A.digest(manifest)!=q['build_identity']:
+        raise ValueError('User distribution manifest/version differs')
+    files={p.relative_to(package).as_posix() for p in package.rglob('*') if p.is_file() and '__pycache__' not in p.parts}
+    if files!=set(manifest['files'])|{'_experimental/manifest.json'}:raise ValueError('Unexpected/missing installed user component')
+    # One byte snapshot must satisfy BOTH manifest and installed RECORD.
+    # No cross-operation cache:every integrity boundary still reopens files.
+    checked_content={}
+    for n,h in manifest['files'].items():
+        data=A.safe_path(package,n).read_bytes();raw_hash=hashlib.sha256(data).digest()
+        if raw_hash.hex()!=h:raise ValueError('Altered installed user component')
+        checked_content[n]=(raw_hash,len(data))
+    data=(package/'_experimental/manifest.json').read_bytes()
+    if hashlib.sha256(data).hexdigest()!=q['manifest_sha256']:raise ValueError('User manifest changed during verification')
+    checked_content['_experimental/manifest.json']=(hashlib.sha256(data).digest(),len(data))
+    infos=list(package.parent.glob('morphtree-*.dist-info'))
+    if len(infos)!=1:raise ValueError('Unique distribution metadata required')
+    info=infos[0];meta=info/'METADATA';record=info/'RECORD'
+    if A.sha(meta)!=q['metadata_sha256'] or A.sha(record)!=q['record_sha256']:raise ValueError('Altered installed metadata/RECORD')
+    if 'Name: morphtree\n' not in meta.read_text('utf8') or 'Version: '+q['version']+'\n' not in meta.read_text('utf8'):
+        raise ValueError('User metadata name/version mismatch')
+    rows={n:(h,size) for n,h,size in csv.reader(io.StringIO(record.read_text('utf8')))}
+    for n in list(manifest['files'])+['_experimental/manifest.json']:
+        h,size=rows.get('morphtree/'+n,('',0));raw_hash,actual_size=checked_content[n]
+        expected='sha256='+base64.urlsafe_b64encode(raw_hash).decode().rstrip('=')
+        if h!=expected or int(size)!=actual_size:raise ValueError('Installed user RECORD content differs')
+    import morphtree
+    if Path(morphtree.__file__).resolve()!=package/'__init__.py':raise ValueError('Shadowed public package')
+    for n,m in tuple(sys.modules.items()):
+        if n=='morphtree' or n.startswith('morphtree.'):
+            f=getattr(m,'__file__',None)
+            if f and not Path(f).resolve().is_relative_to(package):raise ValueError('Mixed user/source module origins')
+    if q['environment']!=A.environment():raise ValueError('User execution environment changed')
+    return {'runtime':'morphtree-user-installed','version':q['version'],'artifact_sha256':q['record_sha256'],
+        'artifact_identity_kind':'INSTALLED_RECORD_NOT_ORIGINAL_WHEEL_OR_SCIENTIFIC_QUALIFICATION',
+        'build_identity':q['build_identity'],'manifest_sha256':q['manifest_sha256'],'backend':'pure','environment':q['environment']}
+
+
+def _user_qualification(package,root,source,protocol,seconds,user_limits):
+    from ._experimental import admission as A
+    manifest=package/'_experimental/manifest.json';m=A.read(manifest)
+    infos=list(package.parent.glob('morphtree-*.dist-info'))
+    if len(infos)!=1:raise ValueError('Unique installed user distribution required')
+    return {'schema':'morph030-qualified-installed-v1','usage':'ORDINARY_USER_NOT_SCIENTIFIC',
+        'version':m['version'],'backend':'pure','environment':A.environment(),'manifest_sha256':A.sha(manifest),
+        'build_identity':A.digest(m),'metadata_sha256':A.sha(infos[0]/'METADATA'),'record_sha256':A.sha(infos[0]/'RECORD'),
+        'input_root':str(root),'deadline_utc_epoch':time.time()+seconds,
+        'limits':{'fits':128,'trials':user_limits['trials'],'operations':256,'seconds':seconds},'scalar_cap':2048,
+        'sources':{A.digest(source):source},'components':[],'component_contracts':{},'protocol':protocol,
+        'authorization':{'id':'USER-API:'+uuid.uuid4().hex,'operations':['inspect','load','save','export','fit','trial']},
+        'optional_dependency':None}
+
+
+def _user_source(source, expected):
+    """No user-supplied path/role converts this scope into research authority."""
+    from ._experimental import admission as A
+    if source!=expected or set(source)!={'origin','exposure','sealed','rows','partitions','OOF'}:
+        raise ValueError('Exact ordinary user source required')
+    if source['exposure']!='USER_SUPPLIED_NOT_SCIENTIFIC' or source['sealed'] or source['OOF']:
+        raise ValueError('Controlled/OOF/sealed source is not an ordinary input')
+    ids=[r['id'] for r in source['rows']];parts=source['partitions']
+    if not source['origin'].startswith('USER_IN_MEMORY:') or len(ids)!=len(set(ids)) or not ids:
+        raise ValueError('User source identity/rows required')
+    if set(parts)!={'TRAIN','SEARCH','GUARD_FINAL'} or parts['GUARD_FINAL']:
+        raise ValueError('User scope never opens GUARD')
+    union=parts['TRAIN']+parts['SEARCH']
+    if not parts['TRAIN'] or not parts['SEARCH'] or len(union)!=len(set(union)) or set(union)!=set(ids):
+        raise ValueError('Exact disjoint internal user split required')
+    # Do not silently count duplicated observations as independent validation.
+    rows=[A.digest({'X':r['X'],'y':r['y']}) for r in source['rows']]
+    if len(rows)!=len(set(rows)):raise ValueError('Duplicate-row groups need a separately qualified splitting policy')
+    return A.digest(source)
+
+
+def _new_user_session(estimator, X, y, *, started=None):
+    """Internal ordinary-user scope; never an owner/scientific capability.
+
+    Input is supplied in memory, not looked up in a research workspace. The
+    existing runtime/admission/store/executor are reused. No caller can use a
+    failed active scientific context as a request to enter this branch.
+    """
+    from ._experimental import admission as A
+    from ._experimental.target import detect_target, _split, _NamedRows
+    from ._experimental.preprocessing import _m26_rows
+    from ._experimental.producer import PIPE_CURRENT
+    from ._experimental.runtime_io import configure
+    from .config import HyperParams
+    started=time.monotonic() if started is None else started
+    scope=PUBLIC_SCOPE.get()
+    if scope is not None:scope.before_input(estimator)
+    if ACTIVE.get() is not None or A.CURRENT.get() is not None or PIPE_CURRENT.get() is not None:
+        raise AdmissionError({'status':'REJECTED','reason':'RESEARCH_CONTEXT_NEVER_FALLS_THROUGH_TO_PUBLIC'})
+    for obj in (X,y):
+        if any(hasattr(obj,k) for k in ('_morphtree_source_role','_morphtree_scientific_source','_morphtree_sealed')):
+            raise ValueError('Controlled-source objects require their explicit scientific context')
+    estimator._validate()
+    if estimator._research_growth:
+        raise ValueError('Research operator overrides require a scientific execution context')
+    # Dependency availability is checked before source materialization or fits.
+    if estimator.tuning=='optuna':
+        import importlib.util
+        if importlib.util.find_spec('optuna') is None:
+            raise ImportError('Final Optuna is a standard dependency; install optuna==5.0.0 or use_optuna=False skips only numerical tuning')
+    rows,names=_m26_rows(X);task,targets,classes=detect_target(y,estimator.task)
+    if len(rows)!=len(targets):raise ValueError('Aligned X/y required')
+    if sum(4*len(v) if isinstance(v,str) else 8 for row in rows for v in row)>2*1024**2:
+        raise ValueError('Current ordinary-user input exceeds the2MiB estimated payload safety cap')
+    raw_y=list(targets) if task=='regression' else [classes[i] for i in targets]
+    if scope is not None:scope.input_rows(rows,raw_y,names,task)
+    seed=estimator.random_state
+    if seed is None:
+        import random
+        seed=random.SystemRandom().randrange(2**31)
+    ti,si=_split(targets,task,seed)
+    if len(ti)>65536:raise ValueError('Current native component path supports at most65536 TRAIN rows')
+    from ._experimental.preprocessing import TrainEncoder
+    if TrainEncoder().fit([rows[i] for i in ti]).width_>128:
+        raise ValueError('Current native component path supports at most128 encoded features')
+    source={'origin':'USER_IN_MEMORY:'+A.digest({'X':rows,'y':raw_y,'names':names}),
+        'exposure':'USER_SUPPLIED_NOT_SCIENTIFIC','sealed':False,
+        'rows':[{'id':'ROW:'+str(i),'X':r,'y':raw_y[i]} for i,r in enumerate(rows)],
+        'partitions':{'TRAIN':['ROW:'+str(i) for i in ti],'SEARCH':['ROW:'+str(i) for i in si],'GUARD_FINAL':[]},'OOF':[]}
+    seconds=150. if estimator.time_budget=='auto' else float(estimator.time_budget)
+    params=asdict(HyperParams(n_estimators=2,max_leaves=8,max_depth=4,max_bins=15,min_samples_leaf=2,row_fraction=1.))
+    if estimator.capacity:params.update(estimator.capacity)
+    # Original finite learning adapter supports <=8 stages/members and <=4
+    # numerical trials. These are resource limits, not scientific grants.
+    limits={'calls':128,'scalar_trees':8192,'trials':0 if estimator.tuning=='none' else 2 if estimator.n_trials=='auto' else estimator.n_trials,'artifact_bytes':32*1024**2}
+    temporary=OwnedTemporaryDirectory(prefix='morphtree-user-',dir=None if scope is None else scope.calls)
+    root=Path(temporary.name).resolve();configure(root)
+    if scope is not None:scope.retain_temporary(temporary,root)
+    try:
+        protocol={'schema':'morphtree-user-call-v1','purpose':'ORDINARY_USER_NOT_SCIENTIFIC',
+            'id':uuid.uuid4().hex,'source_sha256':A.digest(source),'task':task,'seed':seed,'limits':limits,'seconds':seconds}
+        pp=root/'USER-CALL.json';pp.write_text(json.dumps(protocol,sort_keys=True)+'\n',encoding='utf8')
+        reference={'path':pp.name,'sha256':A.sha(pp)}
+        q=A.user_qualification(Path(__file__).parent,root,source,reference,seconds,limits)
+        preparation=time.monotonic()-started
+        if preparation>=seconds:raise TimeoutError('User budget exhausted during input preparation')
+        q['deadline_utc_epoch']-=preparation
+        qp=root/'USER-ADMISSION.json';qp.write_text(json.dumps(q,sort_keys=True)+'\n',encoding='utf8')
+        engine=A.AdmissionEngine(package=Path(__file__).parent,qualification_path=qp,
+            qualification_sha256=A.sha(qp),workdir=root)
+        plan={'schema':'comp002-v2-session-v1','mode':'PUBLIC_USER','source':source,
+            'protocol':reference,'seed':seed,'params':params,'structural':estimator.morphing=='auto',
+            'max_requests':128,'seconds':seconds,'user_limits':limits}
+        session=Session(engine,root=root,plan=plan)
+        session.started=started;session.engine.started=started
+        session.event('USER_INPUT_PREPARED',seconds_operational=preparation,source_scope='USER_SUPPLIED_NOT_SCIENTIFIC')
+        return temporary,session,_NamedRows(rows,names),raw_y,seed
+    except BaseException as primary:
+        if scope is None:
+            try:temporary.cleanup()
+            except BaseException as cleanup_error:BaseException.add_note(primary,'Owned user workspace cleanup also failed: '+type(cleanup_error).__name__)
+        raise
+
+
+class TrainingNotEnabledError(RuntimeError):
+    """This experimental run has no approved training execution budget."""
+
+
+class MorphTree(Integrated):
+    """COMP007 source: initial native exploration then bounded evolution.
+
+    User code only supplies data and normal estimator options. The qualifying
+    host owns execution authorization; this build does not grant unrestricted
+    training. Adaptive may abstain; research requires an explicit operator and
+    a separately admitted run. Neither choice implies scientific superiority.
+    """
+    def __init__(self, *, mode=None, operators=None, verbosity=None,
+                 use_optuna=None, max_mutations=2, **kwargs):
+        if type(max_mutations) is not int or not 0<=max_mutations<=8:
+            raise ValueError('max_mutations must be an integer in [0,8]')
+        if verbosity is not None:
+            if type(verbosity) is not int or verbosity not in (0,1) or 'verbose' in kwargs:
+                raise ValueError('Choose implemented verbosity 0/1 or legacy verbose, not both')
+            kwargs['verbose']=verbosity
+        if use_optuna is not None and type(use_optuna) is not bool:
+            raise ValueError('use_optuna must be bool')
+        if use_optuna is not None and 'tuning' in kwargs:
+            raise ValueError('Choose use_optuna or legacy tuning, not both')
+        # No silent optional-dependency skip: the existing explicit optuna path
+        # checks the dependency and separate trial admission BEFORE any learning.
+        if 'tuning' not in kwargs:
+            kwargs['tuning']='none' if use_optuna is False else 'optuna'
+        automatic=mode is None and not (set(kwargs)&{'families','morphing'})
+        if automatic:mode='adaptive'
+        if mode is not None:
+            if mode not in ('baseline', 'adaptive', 'research'):
+                raise ValueError('mode must be baseline, adaptive or research')
+            if set(kwargs) & {'families', 'morphing'}:
+                raise ValueError('Choose mode or legacy families/morphing, not both')
+            kwargs.update(families='boost' if mode=='baseline' else 'auto',
+                          morphing='none' if mode=='baseline' else 'auto')
+        if operators is not None and (mode!='research' or tuple(operators)!=('growth_policy:depthwise',)):
+            raise ValueError('Only explicit research growth_policy:depthwise is available')
+        if mode=='research' and operators is None:
+            raise ValueError('research requires explicit operators')
+        super().__init__(**kwargs)
+        if mode=='research': self._research_growth=True
+        self._evolving_program=(not self._research_growth and self.families=='auto' and self.morphing=='auto')
+        self._max_mutations=max_mutations
+
+    def fit(self, X, y):
+        if ACTIVE.get() is not None:
+            # An invalid/expired/STOP scientific context MUST raise here, never
+            # retry as ordinary user data. Original owner checks remain active.
+            require_session()
+            return super().fit(X,y)
+        from ._experimental import admission as A
+        from ._experimental.producer import PIPE_CURRENT
+        from ._experimental.runtime_io import WORKDIR
+        if A.CURRENT.get() is not None or PIPE_CURRENT.get() is not None:
+            raise AdmissionError({'status':'REJECTED','reason':'ORPHANED_OPERATION_NOT_PUBLIC_DATA'})
+        if not PUBLIC_USER_ENABLED:
+            raise TrainingNotEnabledError('This installed version requires its explicitly approved execution context')
+        if not _USER_RUN_LOCK.acquire(blocking=False):
+            raise RuntimeError('Concurrent fit in this process is unsupported; use separate processes')
+        work_token=WORKDIR.set(WORKDIR.get());old_temp=tempfile.tempdir
+        workspace=None;session=None;old_seed=self.random_state
+        before=dict(self.__dict__);closure_attempted=False;started=time.monotonic()
+        try:
+            workspace,session,xx,yy,seed=_new_user_session(self,X,y,started=started)
+            self.random_state=seed
+            with session.activate():result=super().fit(xx,yy)
+            session._user_integrity()
+            closure_attempted=True;session.close_user_run()
+            self._public_user_run=True
+            self._public_workspace=workspace
+            self._public_resource_limits=copy.deepcopy(session.plan['user_limits'])
+            self._cost=session.counters()
+            return result
+        except BaseException as exc:
+            attempt=copy.deepcopy(getattr(self,'last_attempt_',{}))
+            if attempt==before.get('last_attempt_',{}):
+                attempt={'status':'FAIL','phase':'PUBLIC_ADMISSION','physical_fits':getattr(session,'_user_started',0),'trials':getattr(session,'_user_trials',0)}
+            if attempt.get('status')=='PASS':attempt.update(status='FAIL',stop_reason='PUBLIC_FINALIZATION_FAILED_RESTORED_PRIOR_OBJECT')
+            attempt.update(error=type(exc).__name__,message=str(exc))
+            self.__dict__.clear();self.__dict__.update(before);self.last_attempt_=attempt
+            if workspace is not None:self._failed_public_workspace=workspace
+            raise
+        finally:
+            self.random_state=old_seed
+            try:
+                if session is not None and not closure_attempted:
+                    closure_attempted=True;session.close_user_run()
+            except BaseException as exc:
+                self.last_attempt_.update(closure_status='UNKNOWN_FAILED',closure_error=type(exc).__name__)
+                raise
+            finally:
+                tempfile.tempdir=old_temp;WORKDIR.reset(work_token)
+                _USER_RUN_LOCK.release()
+
+    @contextmanager
+    def _public_output(self,path):
+        """Explicit user output destination, never a scientific writer bypass."""
+        from ._experimental import admission as A
+        scope=PUBLIC_SCOPE.get()
+        if scope is not None:scope.output(path)
+        if ACTIVE.get() is not None or A.CURRENT.get() is not None or not (getattr(self,'_public_user_run',False) or getattr(self,'_input_sha256',None)):
+            yield path;return
+        from ._experimental.runtime_io import WORKDIR, configure
+        target=Path(path).resolve();parent=target.parent
+        if not parent.is_dir():raise ValueError('Output parent directory must exist')
+        old=self._workdir;token=WORKDIR.set(WORKDIR.get());old_temp=tempfile.tempdir
+        try:
+            self._workdir=str(parent);configure(parent);yield target
+        finally:
+            self._workdir=old;tempfile.tempdir=old_temp;WORKDIR.reset(token)
+
+    def save(self,path,*,variant='all'):
+        with self._public_output(path) as p:return super().save(p,variant=variant)
+
+    def save_recipe(self,path,*,variant='selected'):
+        with self._public_output(path) as p:return super().save_recipe(p,variant=variant)
+
+    def export_python(self,path,*,variant='selected'):
+        with self._public_output(path) as p:return super().export_python(p,variant=variant)
+
+    def export_algorithm(self,path):
+        with self._public_output(path) as p:return super().export_algorithm(p)
+
+    def set_params(self, **values):
+        """Update implemented public controls without resetting evolution limits.
+
+        The inherited legacy setter reconstructed the object without the
+        public mutation limit/research operator. It also rejected canonical
+        verbosity/use_optuna names. Serialization keeps its legacy constructor
+        projection; executed evolution limits remain in the persisted trace.
+        Validation is transactional: an invalid change preserves fitted state.
+        """
+        options=super().get_params()
+        allowed=set(options)|{'mode','operators','verbosity','use_optuna','max_mutations'}
+        if set(values)-allowed:
+            raise ValueError('Unknown estimator parameter: '+', '.join(sorted(set(values)-allowed)))
+        options['max_mutations']=self._max_mutations
+        if self._research_growth:
+            options.pop('families');options.pop('morphing')
+            options.update(mode='research',operators=('growth_policy:depthwise',))
+        # Remove an inherited legacy spelling only when it was not supplied
+        # explicitly in the same update. Explicit conflicts remain errors.
+        for canonical,legacy in [('verbosity','verbose'),('use_optuna','tuning')]:
+            if canonical in values and legacy not in values:options.pop(legacy,None)
+        if 'mode' in values:
+            for legacy in ('families','morphing'):
+                if legacy not in values:options.pop(legacy,None)
+            if values['mode']!='research' and 'operators' not in values:
+                options.pop('operators',None)
+        options.update(values)
+        replacement=type(self)(**options)
+        self.__dict__=replacement.__dict__
+        return self
+
+    def export_standalone(self, path):
+        """Export the existing dependency-free inference predictor, not training."""
+        return self.export_python(path)
+
+    @classmethod
+    def load(cls, path, *, sha256=None, workdir=None):
+        scope = PUBLIC_SCOPE.get()
+        if scope is not None:
+            scope.output(path)
+            if workdir is not None:
+                scope.output(workdir)
+        source = Path(path).resolve()
+        if not source.is_file() or not 0 < source.stat().st_size <= 32 * 1024 ** 2:
+            raise ValueError('A bounded model file is required')
+        if sha256 is None:
+            with source.open('rb') as stream:
+                raw = stream.read(32 * 1024 ** 2 + 1)
+            if not 0 < len(raw) <= 32 * 1024 ** 2:
+                raise ValueError('A bounded model file is required')
+            expected = hashlib.sha256(raw).hexdigest()
+            del raw
+        else:
+            expected = sha256
+        from ._experimental.runtime_io import WORKDIR
+        token = WORKDIR.set(WORKDIR.get())
+        old_temp = tempfile.tempdir
+        workspace = None
+        try:
+            if workdir is None:
+                workspace = OwnedTemporaryDirectory(prefix='morphtree-model-')
+                workdir = workspace.name
+            model = super().load(source, sha256=expected, workdir=workdir)
+            if workspace is not None:
+                model._loaded_workspace = workspace
+            return model
+        except BaseException as primary:
+            if workspace is not None:
+                try:workspace.cleanup()
+                except BaseException as cleanup_error:BaseException.add_note(primary,'Owned legacy workspace cleanup also failed: '+type(cleanup_error).__name__)
+            raise
+        finally:
+            tempfile.tempdir = old_temp
+            WORKDIR.reset(token)

morphtree/api_compat.py

@@ -0,0 +1,65 @@
+"""Public2.0 call contracts; historical weights keep their original decoder."""
+from pathlib import Path
+import os
+
+
+def validate_workdir(value):
+    if value is None:return None
+    if not isinstance(value,(str,os.PathLike)):
+        raise ValueError('Existing local working directory required')
+    value=os.fspath(value)
+    if not isinstance(value,str) or not value:raise ValueError('Existing local working directory required')
+    from ._experimental.admission import safe_path
+    path=Path(value)
+    # safe_path rejects links/reparse ancestors before resolving them.
+    checked=safe_path(path.absolute(),path.absolute())
+    if not checked.is_dir():raise ValueError('Existing local working directory required')
+    return checked
+
+
+class HistoricalModel:
+    """A real subclass of the new facade with unchanged legacy learned state.
+
+    Public inspection/IO use the original decoder/engine. A new fit uses the
+    scalable controller and only replaces that historical state on success.
+    """
+    _historical_delegates=frozenset(('predict','predict_proba','score','save',
+        'report','get_search_report','get_algorithm_program','get_recipe',
+        'get_mutation_history','cost_report','available_variants','save_recipe',
+        'export_python','export_standalone','export_algorithm','task_','classes_',
+        'n_features_in_','feature_names_in_','best_family_','best_params_',
+        'stop_reason_','_engine','_loaded_workspace'))
+
+    def __getattribute__(self,name):
+        state=object.__getattribute__(self,'__dict__')
+        legacy=state.get('_historical_model')
+        if legacy is not None and name in HistoricalModel._historical_delegates:
+            if name=='_loaded_workspace':return getattr(legacy._engine,name)
+            return getattr(legacy,name)
+        return super().__getattribute__(name)
+
+    def __sklearn_is_fitted__(self):
+        return '_historical_model' in self.__dict__ or super().__sklearn_is_fitted__()
+
+    def fit(self,X,y):
+        historical=self.__dict__.pop('_historical_model',None)
+        try:return super().fit(X,y)
+        except BaseException:
+            if historical is not None:self.__dict__['_historical_model']=historical
+            raise
+
+
+def wrap_historical(legacy,facade,memory_budget):
+    from .scalable import MorphTreeClassifier,MorphTreeRegressor
+    if not hasattr(legacy,'_engine'):
+        from . import estimators
+        kind=estimators.MorphTreeRegressor if legacy.task_=='regression' else estimators.MorphTreeClassifier
+        wrapped=kind.__new__(kind);wrapped._engine=legacy;legacy=wrapped
+    kind=MorphTreeRegressor if legacy.task_=='regression' else MorphTreeClassifier
+    params=legacy.get_params()
+    params.update(memory_budget=memory_budget)
+    # This is an ordinary Python subclass, not __class__ spoofing or a rewrite
+    # of weights. get_params describes its next scalable refit, not old training.
+    cls=type('Historical'+kind.__name__,(HistoricalModel,kind),{'__module__':__name__})
+    obj=cls(**params);obj._historical_model=legacy
+    return obj

morphtree/backend.py

@@ -0,0 +1,162 @@
+"""Reference backend: standard library only, column-major compact bins."""
+from __future__ import annotations
+
+from array import array
+import math
+from .metrics import sigmoid, softmax
+
+
+class PythonBackend:
+    name = "python-stdlib"
+
+    def bin(self, binner, X):
+        return binner.transform(X)
+
+    def gradients(self, raw, y):
+        probabilities = [sigmoid(v) for v in raw]
+        gradient = array("d", (p - label for p, label in zip(probabilities, y)))
+        hessian = array("d", (max(p * (1 - p), 1e-12) for p in probabilities))
+        return probabilities, gradient, hessian
+
+    def probabilities(self, raw):
+        return [sigmoid(v) for v in raw]
+
+    def regression_gradients(self, raw, y):
+        # L = 0.5 * (prediction - target)^2: g = prediction - target, h = 1.
+        return raw, array('d', (v - target for v, target in zip(raw, y))), array('d', [1.0]) * len(y)
+
+    def multiclass_raw(self, prior, n):
+        return [list(prior) for _ in range(n)]
+
+    def multiclass_probabilities(self, raw, strategy, *, normalize=True):
+        if strategy == 'softmax':
+            return [softmax(row) for row in raw]
+        probabilities = [[sigmoid(float(v)) for v in row] for row in raw]
+        if normalize:
+            for row in probabilities:
+                total = math.fsum(row)
+                for j in range(len(row)):
+                    row[j] /= total
+        return probabilities
+
+    def multiclass_gradient_column(self, probabilities, y, column, strategy):
+        # D=2*diag(p*(1-p)) majorizes diag(p)-p*p^T. It is NOT the exact Hessian.
+        factor = 2. if strategy == 'softmax' else 1.
+        return (array('d', (row[column] - (label == column) for row, label in zip(probabilities, y))),
+                array('d', (max(factor * row[column] * (1-row[column]), 1e-12) for row in probabilities)))
+
+    def multiclass_update(self, raw, column, step, prediction):
+        for row, value in zip(raw, prediction):
+            row[column] += step * float(value)
+
+    def center_multiclass(self, raw):
+        for row in raw:
+            mean = math.fsum(row) / len(row)
+            for j in range(len(row)):
+                row[j] -= mean
+
+    def node_statistics(self, Xb, ids, g, h, u, features, need_missing, need_uncertainty):
+        count = max(len(ids), 1)
+        missing = sum(any(Xb[f][i] == 0 for f in features) for i in ids) / count if need_missing else 0.0
+        uncertainty = math.fsum(u[i] for i in ids) / count if need_uncertainty else 0.0
+        return math.fsum(g[i] for i in ids), math.fsum(h[i] for i in ids), uncertainty, missing
+
+    def histogram(self, column, indices, g, h, r, u, size):
+        result = [array("d", [0.0]) * size for _ in range(5)]
+        counts, gs, hs, rs, us = result
+        for i in indices:
+            b = column[i]
+            counts[b] += 1
+            gs[b] += g[i]
+            hs[b] += h[i]
+            rs[b] += r[i]
+            us[b] += u[i]
+        return result
+
+    def subtract(self, parent, child):
+        result = [[a - b for a, b in zip(pa, ch)] for pa, ch in zip(parent, child)]
+        result[0] = [max(0.0, c) for c in result[0]]
+        result[2] = [max(0.0, h) for h in result[2]]
+        return result
+
+    def cumulative(self, histogram):
+        result = []
+        for row in histogram:
+            running, sums = 0.0, []
+            for value in row[1:]:
+                running += value
+                sums.append(running)
+            result.append(sums)
+        return result
+
+    def partition(self, column, ids, cut, missing_left):
+        left, right = [], []
+        for i in ids:
+            b = column[i]
+            goes_left = missing_left if b == 0 else b <= cut
+            (left if goes_left else right).append(i)
+        return left, right
+
+    def predict_tree(self, nodes, Xb):
+        n = len(Xb[0])
+        out = [0.0] * n
+        for row in range(n):
+            node_id = 0
+            for _ in range(len(nodes) + 1):
+                node = nodes[node_id]
+                if node.is_leaf:
+                    out[row] = node.value
+                    break
+                b = Xb[node.feature][row]
+                goes_left = node.missing_left if b == 0 else b <= node.threshold_bin
+                node_id = node.left if goes_left else node.right
+            else:
+                raise ValueError("Invalid cyclic tree")
+        return out
+
+    def add_update(self, raw, step, prediction):
+        return [r + step * v for r, v in zip(raw, prediction)]
+
+    def fast_split(self, hist, total, depth, genome, params):
+        if genome.split_program != ("OBJECTIVE_GAIN",):
+            return NotImplemented
+        # Canonical scalar arithmetic, specialized to the three consumed rows.
+        # Hoist regularization; do not allocate left/right lists for every cut.
+        counts, gradients, hessians = ([float(v) for v in row] for row in hist[:3])
+        n, G, H = total[:3]
+        factor = params.depth_reg_strength if genome.regularization_policy == "depth_scaled" else 0.
+        child_l2 = params.l2 * (1 + factor * (depth + 1))
+        parent_l2 = params.l2 * (1 + factor * depth)
+        l1 = params.l1
+        thresholded = math.copysign(max(abs(G)-l1, 0.), G)
+        parent_score = thresholded * thresholded / max(H+parent_l2, 1e-12)
+        directions = (True, False) if genome.missing_policy == "learned" else (genome.missing_policy == "left",)
+        if len(directions) == 2 and not any(row[0] for row in hist):
+            directions = directions[:1]
+        mn, mg, mh = counts[0], gradients[0], hessians[0]
+        cn = cg = ch = 0.
+        best = None
+        best_rounded = None
+        support, min_hessian, min_gain = params.min_samples_leaf, params.min_hessian, params.min_gain
+        for cut in range(len(counts)):
+            if cut:
+                cn += counts[cut]
+                cg += gradients[cut]
+                ch += hessians[cut]
+            for missing_left in directions:
+                ln = cn + (mn if missing_left else 0.)
+                lg = cg + (mg if missing_left else 0.)
+                lh = ch + (mh if missing_left else 0.)
+                rn, rg, rh = n-ln, G-lg, H-lh
+                if min(ln, rn) < support or min(lh, rh) < min_hessian:
+                    continue
+                lt = math.copysign(max(abs(lg)-l1, 0.), lg)
+                rt = math.copysign(max(abs(rg)-l1, 0.), rg)
+                left_score = lt * lt / max(lh+child_l2, 1e-12)
+                right_score = rt * rt / max(rh+child_l2, 1e-12)
+                gain = 0.5 * (left_score + right_score - parent_score)
+                if math.isfinite(gain) and gain > min_gain:
+                    rounded = round(gain, 12)
+                    if best is None or rounded > best_rounded:
+                        best, best_rounded = (gain, cut, missing_left), rounded
+        return best