morphtree 2.1.0__py3-none-win_amd64.whl

morphtree/scalable_backend.py

@@ -0,0 +1,74 @@
+"""Pure Python native category-equality splits; never order nominal codes.
+
+Numeric operations delegate unchanged to the qualified reference backend.
+Missing and unseen nominal values have separate explicit codes. Each node tests
+one category versus the remainder; arbitrary subsets/target ordering are absent.
+"""
+from array import array
+from bisect import bisect_right
+import math
+from .backend import PythonBackend
+
+
+class Column(array):
+    def __new__(cls,kind,values,nominal=False):return super().__new__(cls,kind,values)
+    def __init__(self,kind,values,nominal=False):self.nominal=nominal
+
+
+class Hist(list):
+    def __init__(self,values,nominal=False):super().__init__(values);self.nominal=nominal
+
+
+class ScalableBackend(PythonBackend):
+    name='private-python-columnar-nominal'
+    def __init__(self,nominal_sizes=None):self.nominal_sizes=nominal_sizes or {}
+    def bin(self,binner,X):
+        if not hasattr(X,'_compact_columns'):return super().bin(binner,X)
+        out=[]
+        for j,cuts in enumerate(binner.thresholds):
+            if j in self.nominal_sizes:out.append(Column('I',X.column(j),True))
+            else:out.append(Column('B' if binner.max_bins<=255 else 'H',(0 if math.isnan(v) else bisect_right(cuts,v)+1 for v in X.column(j))))
+        return out
+    def node_statistics(self,Xb,ids,g,h,u,features,need_missing,need_uncertainty):
+        if not need_missing or not self.nominal_sizes:
+            return super().node_statistics(Xb,ids,g,h,u,features,need_missing,need_uncertainty)
+        # Numeric missing is bin0; native nominal missing is fitted k+1.
+        # Unseen nominal k+2 is separate and must not trigger this local gate.
+        columns=tuple((Xb[f],self.nominal_sizes[f]+1 if f in self.nominal_sizes else 0) for f in features)
+        missing=sum(any(column[i]==code for column,code in columns) for i in ids)/max(len(ids),1)
+        G,H,uncertainty,_=super().node_statistics(Xb,ids,g,h,u,features,False,need_uncertainty)
+        return G,H,uncertainty,missing
+    def histogram(self,column,*args):return Hist(super().histogram(column,*args),getattr(column,'nominal',False))
+    def subtract(self,parent,child):return Hist(super().subtract(parent,child),getattr(parent,'nominal',False))
+    def fast_split(self,hist,total,depth,genome,params):
+        if not getattr(hist,'nominal',False):return super().fast_split(hist,total,depth,genome,params)
+        if genome.split_program not in (('OBJECTIVE_GAIN',),('OBJECTIVE_GAIN','BALANCE','MUL')):raise ValueError('Unsupported private nominal split program')
+        n,G,H=total[:3];counts,gs,hs=hist[:3];l1=params.l1
+        parent_l2=params.l2*(1+(params.depth_reg_strength*depth if genome.regularization_policy=='depth_scaled' else 0))
+        child_l2=params.l2*(1+(params.depth_reg_strength*(depth+1) if genome.regularization_policy=='depth_scaled' else 0))
+        def score(g,h,l2):return max(abs(g)-l1,0.)**2/max(h+l2,1e-12)
+        parent=score(G,H,parent_l2);best=None
+        for code in range(1,len(counts)):
+            ln,lg,lh=counts[code],gs[code],hs[code];rn,rg,rh=n-ln,G-lg,H-lh
+            if min(ln,rn)<params.min_samples_leaf or min(lh,rh)<params.min_hessian:continue
+            gain=.5*(score(lg,lh,child_l2)+score(rg,rh,child_l2)-parent)
+            if len(genome.split_program)>1:gain*=2*min(ln,rn)/n
+            if math.isfinite(gain) and gain>params.min_gain and (best is None or round(gain,12)>round(best[0],12)):best=(gain,code,True)
+        return best
+    def partition(self,column,ids,cut,missing_left):
+        if not getattr(column,'nominal',False):return super().partition(column,ids,cut,missing_left)
+        left=[];right=[]
+        for i in ids:(left if column[i]==cut else right).append(i)
+        return left,right
+    def predict_tree(self,nodes,Xb):
+        out=[]
+        for i in range(len(Xb[0])):
+            k=0
+            for _ in range(len(nodes)+1):
+                node=nodes[k]
+                if node.is_leaf:out.append(node.value);break
+                col=Xb[node.feature];b=col[i]
+                left=(b==node.threshold_bin) if getattr(col,'nominal',False) else node.missing_left if b==0 else b<=node.threshold_bin
+                k=node.left if left else node.right
+            else:raise ValueError('Invalid cyclic native tree')
+        return out

morphtree/scalable_checkpoint.py

@@ -0,0 +1,129 @@
+"""Local atomic fitted-state checkpoints. Recovery never resumes training."""
+import copy
+import hashlib
+import json
+import os
+from pathlib import Path
+import shutil
+import tempfile
+
+from .persistence import atomic_text, decode_json_bytes, LoadLimits, read_regular_bytes
+from .scalable_receipts import verify
+from .temporary import directory as bounded_directory
+
+
+def _safe_directory(path):
+    path = Path(path)
+    if path.is_symlink() or getattr(path, 'is_junction', lambda: False)():
+        raise ValueError('Checkpoint directory cannot be a link')
+    if not path.is_dir():
+        raise ValueError('Checkpoint parent directory must exist')
+    return path.resolve()
+
+
+def _workspace(value, resources):
+    estimate = 8192
+    pending = [iter((value,))]
+    count = 0
+    while pending:
+        try:
+            node = next(pending[-1])
+        except StopIteration:
+            pending.pop()
+            continue
+        count += 1
+        estimate += 512
+        if isinstance(node, str):
+            estimate += len(node) * 32
+        elif isinstance(node, dict):
+            pending.append(iter(node.items()))
+        elif isinstance(node, (list, tuple)):
+            pending.append(iter(node))
+        if count % 128 == 0:
+            resources.reserve(estimate, 'Durable checkpoint workspace')
+    resources.reserve(estimate, 'Durable checkpoint workspace')
+    return estimate
+
+
+def commit(estimator, model, classes, resources, *, final):
+    from .scalable import digest
+    if estimator.checkpoint_dir is None:
+        return None
+    report = estimator.last_attempt_
+    events = report['events']
+    verify(events)
+    if final:
+        if report['status'] != 'COMPLETED' or events[-1]['event'] != 'FINAL_MODEL':
+            raise ValueError('Final checkpoint requires completed finalization')
+    elif digest(model) != report['last_verified_anchor']['model_sha256']:
+        raise ValueError('Checkpoint must match verified TRAIN anchor')
+    wanted = {event['parent_sha256'] for event in events if event['event'] == 'DIAGNOSIS'}
+    evidence = getattr(estimator, 'diagnostic_profiles_', {})
+    resources.reserve(8192, 'Refit configuration header before copying')
+    constructor = estimator._constructor_state()
+    template = dict(model=model, classes=classes, constructor=constructor, constructor_sha256=digest(constructor), report={k: v for k, v in report.items() if k != 'checkpoint' and (final or k != 'selection_checkpoint')}, evidence=dict(diagnostic_profiles={key: evidence[key] for key in wanted}))
+    estimate = _workspace(template, resources)
+    packet = copy.deepcopy(template)
+    packet['schema'] = 'SCALABLE_PRIVATE_MODEL_3'
+    if not final:
+        packet['report']['status'] = 'PRESERVED_VERIFIED_TRAIN_ANCHOR_NOT_FINAL_REFIT'
+    packet['receipt_sha256'] = digest(packet['report'])
+    text = json.dumps(packet, sort_keys=True, allow_nan=False) + '\n'
+    resources.reserve(estimate + len(text) * 8, 'Encoded durable checkpoint workspace')
+    directory = getattr(estimator, '_active_checkpoint_dir_', None)
+    if directory is None:
+        if estimator.checkpoint_dir == 'auto':
+            parent = _safe_directory(tempfile.gettempdir()) / 'morphtree-checkpoints'
+            parent.mkdir(exist_ok=True)
+        else:
+            parent = Path(estimator.checkpoint_dir)
+        parent = _safe_directory(parent)
+        directory = Path(bounded_directory(prefix='fit-', dir=parent))
+        estimator._active_checkpoint_dir_ = directory
+        estimator._checkpoint_sequence_ = 0
+    directory = _safe_directory(directory)
+    report['checkpoint_directory'] = str(directory)
+    if shutil.disk_usage(directory).free < len(text) * 4 + 8192:
+        raise OSError('Insufficient free disk for atomic checkpoint; previous model preserved')
+    name = f"model-{estimator._checkpoint_sequence_:04d}.json"
+    target = directory / name
+    if target.exists():
+        raise FileExistsError('An existing fitted checkpoint cannot be overwritten')
+    atomic_text(target, text)
+    record = dict(schema='SCALABLE_CHECKPOINT_POINTER_1', path=name, sha256=hashlib.sha256(text.encode('ascii')).hexdigest(), call_id=events[0]['call_id'], source_role='TRAIN_FULL' if final and estimator.refit_full else 'TRAIN', final=final)
+    if not final:
+        atomic_text(directory / 'SELECTED.json', json.dumps(record, sort_keys=True) + '\n')
+    atomic_text(directory / 'CURRENT.json', json.dumps(record, sort_keys=True) + '\n')
+    estimator._checkpoint_sequence_ += 1
+    receipt = dict(directory=str(directory), **record)
+    report['checkpoint'] = receipt
+    if not final:
+        report['selection_checkpoint'] = receipt
+    if final:
+        estimator.checkpoint_path_ = str(target)
+    return receipt
+
+
+def read_pointer(directory, resources, *, stage='current'):
+    if type(stage) is not str or stage not in ('current', 'selected'):
+        raise ValueError('Checkpoint stage must be current or selected')
+    directory = _safe_directory(directory)
+    pointer = directory / ('CURRENT.json' if stage == 'current' else 'SELECTED.json')
+    if pointer.is_symlink():
+        raise ValueError('Checkpoint pointer cannot be a link')
+    raw = read_regular_bytes(pointer,8192)
+    record = decode_json_bytes(raw, LoadLimits(max_bytes=8192), reserve=resources.reserve)
+    required = {'schema', 'path', 'sha256', 'call_id', 'source_role', 'final'}
+    if set(record) != required or record['schema'] != 'SCALABLE_CHECKPOINT_POINTER_1':
+        raise ValueError('Exact checkpoint pointer schema required')
+    name = record['path']
+    if type(name) is not str or len(name) != 15 or not name.startswith('model-') or not name.endswith('.json') or not name[6:10].isdigit():
+        raise ValueError('Checkpoint pointer must name a local numbered model')
+    if type(record['sha256']) is not str or len(record['sha256']) != 64 or any(c not in '0123456789abcdef' for c in record['sha256']) or type(record['call_id']) is not str or not record['call_id'] or type(record['final']) is not bool or record['source_role'] not in ('TRAIN', 'TRAIN_FULL') or not record['final'] and record['source_role'] != 'TRAIN':
+        raise ValueError('Invalid checkpoint identity or source role')
+    if stage == 'selected' and record['final']:
+        raise ValueError('Selected checkpoint must be a verified TRAIN anchor')
+    target = directory / name
+    if target.is_symlink() or target.resolve().parent != directory:
+        raise ValueError('Checkpoint model must be a local regular file')
+    return target, record

morphtree/scalable_data.py

@@ -0,0 +1,255 @@
+"""Private scalable representation. No whole DataFrame/ndarray list conversion.
+
+Source dictionaries encode input storage only. Learner vocabularies are fitted
+again on each TRAIN/fold; SEARCH-only categories become unknown at validation.
+"""
+from array import array
+from bisect import bisect_right
+from collections.abc import Sequence
+import json
+import math
+import os
+from numbers import Real
+import sys
+import time
+from .source_markers import controlled_source
+
+
+def scalar(v,*,preserve_integral=False):
+    kind=type(v)
+    if kind is str or v is None:return v
+    if kind is int and preserve_integral:return v
+    if kind is float or kind is int or kind is bool:
+        f=float(v)
+        if math.isnan(f):return None
+        if math.isfinite(f):return f
+        raise ValueError('Finite numeric/string category/None/NaN/nullable scalar required')
+    if type(v).__module__.split('.')[0]=='numpy' and hasattr(v,'item'):v=v.item()
+    if v is None or (sys.modules.get('pandas') is not None and v is getattr(sys.modules['pandas'],'NA',None)):return None
+    if preserve_integral and type(v) is int:return v
+    if isinstance(v,Real):
+        f=float(v)
+        if math.isnan(f):return None
+        if math.isfinite(f):return f
+    if isinstance(v,str):return v
+    raise ValueError('Finite numeric/string category/None/NaN/nullable scalar required')
+
+
+def category_key(v):
+    if not isinstance(v,str):
+        if type(v) is int:v=v if abs(v)>2**53 else float(v)
+        else:
+            v=float(v)
+            if abs(v)>2**53 and v.is_integer():v=int(v)
+        if v==0:v=0.0
+    return json.dumps(['s' if isinstance(v,str) else 'n',v],ensure_ascii=False,allow_nan=False,separators=(',',':'))
+
+
+def canonical_category_key(key):
+    """Exact numeric identity, including compatible legacy float key spelling."""
+    entry=json.loads(key)
+    if type(entry) is not list or len(entry)!=2:raise ValueError('Malformed typed category')
+    tag,value=entry
+    if not (tag=='s' and type(value) is str or tag=='n' and type(value) in (int,float)):raise ValueError('Malformed typed category')
+    value=scalar(value,preserve_integral=True)
+    if value is None:raise ValueError('Missing value is not a vocabulary entry')
+    canonical=category_key(value)
+    try:legacy=json.dumps([tag,value if tag=='s' else float(value)],ensure_ascii=False,allow_nan=False,separators=(',',':'))
+    except OverflowError:legacy=None
+    if key not in (canonical,legacy):raise ValueError('Noncanonical typed category')
+    return canonical
+
+
+def resident_bytes():
+    """Read-only OS accounting; no learner arithmetic outside Python."""
+    try:
+        if os.name=='nt':
+            import ctypes
+            class Counters(ctypes.Structure):_fields_=[('cb',ctypes.c_ulong),('faults',ctypes.c_ulong)]+[(n,ctypes.c_size_t) for n in ('peak','rss','peak_paged','paged','peak_nonpaged','nonpaged','pagefile','peak_pagefile')]
+            c=Counters();c.cb=ctypes.sizeof(c)
+            if not ctypes.WinDLL('psapi',use_last_error=True).GetProcessMemoryInfo(ctypes.c_void_p(-1),ctypes.byref(c),c.cb):return None
+            return c.rss
+        with open('/proc/self/statm',encoding='ascii') as handle:
+            resident_pages=int(handle.read().split()[1])
+        return resident_pages*os.sysconf('SC_PAGE_SIZE')
+    except (OSError,ValueError,AttributeError):return None
+
+
+class Input:
+    """Inspect metadata first, then access one column or bounded block."""
+    def __init__(self,X):
+        if controlled_source(X):raise ValueError('Controlled sources cannot use ordinary input or inference')
+        self.X=X;self.frame=hasattr(X,'columns') and hasattr(X,'iloc');self.fitted_kinds=None
+        if hasattr(X,'format') and hasattr(X,'tocsr'):raise ValueError('Sparse input requires a separate qualified path')
+        if hasattr(X,'shape'):
+            if len(X.shape)!=2:raise ValueError('X must have exactly two dimensions')
+            self.n,self.width=map(int,X.shape)
+        elif isinstance(X,(list,tuple)):
+            self.n=len(X);self.width=len(X[0]) if self.n and isinstance(X[0],(list,tuple,array)) else 0
+            if any(not isinstance(r,(list,tuple,array)) or len(r)!=self.width for r in X):raise ValueError('Rectangular rows required')
+        else:raise ValueError('Supported dense table, ndarray or DataFrame required')
+        if self.n<1 or self.width<1:raise ValueError('X must contain rows and columns')
+        self.names=tuple(X.columns) if self.frame else None
+        if self.names is not None and (len(set(self.names))!=self.width or any(not isinstance(n,str) or not n for n in self.names)):raise ValueError('Unique nonempty string names required')
+    def column(self,j,start=0,stop=None):
+        stop=self.n if stop is None else stop
+        if self.frame:return iter(self.X.iloc[start:stop,j].array)
+        if hasattr(self.X,'shape'):return iter(self.X[start:stop,j])
+        return (self.X[i][j] for i in range(start,stop))
+    def forced_categorical(self,j):return self.fitted_kinds[j]=='categorical' if self.fitted_kinds is not None else self.frame and str(self.X.dtypes.iloc[j])=='category'
+
+
+class Resources:
+    def __init__(self,bytes_limit,seconds,check=None):
+        if type(bytes_limit) is not int or bytes_limit<1 or not math.isfinite(seconds) or seconds<=0:raise ValueError('Positive finite resource allowance')
+        self.bytes_limit=bytes_limit;self.started=time.monotonic();self.deadline=self.started+seconds;self.external=check;self.base_rss=resident_bytes();self.peak_rss=self.base_rss;self.last_memory_check=0.;self.pressure_reader=None;self.base_commit=None;self.peak_commit=None
+    def configure_time(self,seconds):
+        if type(seconds) not in (int,float) or not math.isfinite(seconds) or seconds<=0:raise ValueError('Positive finite resource allowance')
+        self.deadline=self.started+seconds
+    def check(self):
+        if time.monotonic()>self.deadline:raise TimeoutError('Finite scalable operation deadline')
+        if time.monotonic()-self.last_memory_check>.05:
+            self.last_memory_check=time.monotonic();rss=resident_bytes()
+            if rss is not None:
+                self.peak_rss=max(self.peak_rss or 0,rss)
+                if self.base_rss is not None and rss-self.base_rss>self.bytes_limit:raise MemoryError('Observed process RSS increase exceeds operation allowance; no rows were sampled')
+            if self.pressure_reader:
+                observation=self.pressure_reader();commit=observation.get('process_private_bytes')
+                if commit is not None:
+                    self.peak_commit=max(self.peak_commit or 0,commit)
+                    if self.base_commit is not None and commit-self.base_commit>self.bytes_limit:raise MemoryError('Observed process committed-memory increase exceeds operation allowance; no rows were sampled')
+                if observation['effective_available_bytes']<=0:raise MemoryError('Active RAM/commit/address-space headroom exhausted; no rows were sampled')
+        if self.external:self.external()
+    def reserve(self,n,description):
+        if n>self.bytes_limit:raise MemoryError(f'{description} requires estimated {n} bytes, allowance {self.bytes_limit}; no rows were sampled')
+        self.check()
+
+
+class Rows(Sequence):
+    _compact_columns=True
+    def __len__(self):return self.n
+    def __getitem__(self,i):
+        if isinstance(i,slice):return [self[j] for j in range(*i.indices(self.n))]
+        if i<0:i+=self.n
+        if not 0<=i<self.n:raise IndexError(i)
+        return Row(self,i)
+    def column(self,j):return (self.value(i,j) for i in range(self.n))
+
+
+class Row(Sequence):
+    def __init__(self,table,i):self.table=table;self.i=i
+    def __len__(self):return self.table.width
+    def __getitem__(self,j):
+        if isinstance(j,slice):return [self[k] for k in range(*j.indices(len(self)))]
+        if j<0:j+=len(self)
+        if not 0<=j<len(self):raise IndexError(j)
+        return self.table.value(self.i,j)
+
+
+class Table(Rows):
+    def __init__(self,columns,kinds,vocab,n,names,indices=None):
+        self.columns,self.kinds,self.vocab,self.n,self.names,self.indices=columns,kinds,vocab,n,names,indices;self.width=len(columns)
+    @classmethod
+    def read(cls,source,resources,start=0,stop=None):
+        source=source if isinstance(source,Input) else Input(source);stop=source.n if stop is None else stop;n=stop-start
+        if not 0<=start<stop<=source.n:raise ValueError('Nonempty bounded block')
+        name_bytes=sum(sys.getsizeof(name)+8*len(name) for name in source.names or ())
+        base_bytes=n*source.width*10+source.width*256+name_bytes
+        resources.reserve(base_bytes,'Compact source columns and schema names')
+        columns=[];kinds=[];vocab=[];vocab_bytes=0
+        for j in range(source.width):
+            resources.check();first=None
+            for i,value in enumerate(source.column(j,start,stop)):
+                if i%4096==0:resources.check()
+                if isinstance(value,str):resources.reserve(base_bytes+vocab_bytes+len(value)*32+512,'Initial scalar/JSON key workspace')
+                first=scalar(value)
+                if first is not None:break
+            forced=source.forced_categorical(j);cat=forced or isinstance(first,str);a=array('I' if cat else 'd');lookup={};words=[]
+            for i,v in enumerate(source.column(j,start,stop)):
+                if i%4096==0:resources.check()
+                if isinstance(v,str):resources.reserve(base_bytes+vocab_bytes+len(v)*32+512,'Scalar/JSON key workspace before allocation')
+                v=scalar(v,preserve_integral=cat)
+                if cat:
+                    if v is None:a.append(0);continue
+                    if not forced and not isinstance(v,str):raise ValueError('Mixed string/numeric column')
+                    key=category_key(v)
+                    if key not in lookup:
+                        vocab_bytes+=sys.getsizeof(key)+96;resources.reserve(base_bytes+vocab_bytes,'Source columns and category dictionaries')
+                        lookup[key]=len(words)+1;words.append(key)
+                    a.append(lookup[key])
+                else:
+                    if isinstance(v,str):raise ValueError('Mixed string/numeric column')
+                    a.append(math.nan if v is None else v)
+            columns.append(a);kinds.append('categorical' if cat else 'numeric');vocab.append(tuple(words))
+        return cls(tuple(columns),tuple(kinds),tuple(vocab),n,source.names)
+    def take(self,indices):
+        ids=array('I' if len(self.columns[0])<2**32 else 'Q',(self.indices[i] if self.indices is not None else i for i in indices))
+        return Table(self.columns,self.kinds,self.vocab,len(ids),self.names,ids)
+    def value(self,i,j):return self.columns[j][self.indices[i] if self.indices is not None else i]
+    def column(self,j):
+        c=self.columns[j]
+        return iter(c) if self.indices is None else (c[i] for i in self.indices)
+    @property
+    def storage_bytes(self):return sum(sys.getsizeof(c) for c in self.columns)+sum(sys.getsizeof(s) for v in self.vocab for s in v)
+
+
+class Encoder:
+    """One native nominal feature per category column, TRAIN-only vocabulary."""
+    def fit(self,table):
+        self.names=table.names;self.columns=[]
+        for j,kind in enumerate(table.kinds):
+            if kind=='numeric':self.columns.append(dict(kind='numeric'));continue
+            observed=set(table.column(j));vocab=sorted({canonical_category_key(table.vocab[j][i-1]) for i in observed if i})
+            self.columns.append(dict(kind='categorical',vocabulary=vocab))
+        return self
+    def state(self):return dict(schema='SCALABLE_TRAIN_ENCODER_1',names=list(self.names) if self.names is not None else None,columns=self.columns)
+    @classmethod
+    def from_state(cls,state,resources):
+        if type(state) is not dict or set(state)!={'schema','names','columns'} or state['schema']!='SCALABLE_TRAIN_ENCODER_1':raise ValueError('Exact private encoder schema')
+        cols=state['columns'];names=state['names']
+        if type(cols) is not list or not cols:raise ValueError('Nonempty column schema')
+        if names is not None and (type(names) is not list or len(names)!=len(cols) or len(set(names))!=len(names) or any(type(n) is not str or not n for n in names)):raise ValueError('Invalid encoder names')
+        schema_bytes=len(cols)*256+sum(sys.getsizeof(name)+8*len(name) for name in names or ())
+        resources.reserve(schema_bytes,'Decoded encoder metadata before copies')
+        for c in cols:
+            if c==dict(kind='numeric'):continue
+            if type(c) is not dict or set(c)!={'kind','vocabulary'} or c['kind']!='categorical':raise ValueError('Invalid category schema')
+            v=c['vocabulary']
+            if type(v) is not list or any(type(s) is not str for s in v) or any(v[i]>=v[i+1] for i in range(len(v)-1)):raise ValueError('Canonical typed vocabulary required')
+            schema_bytes+=len(v)*128;canonical=set()
+            for key in v:
+                schema_bytes+=sys.getsizeof(key)*3
+                resources.reserve(schema_bytes+len(key)*32+512,'Typed vocabulary validation before JSON allocation')
+                normalized=canonical_category_key(key)
+                if normalized in canonical:raise ValueError('Duplicate equivalent numeric categories')
+                canonical.add(normalized)
+        obj=cls();obj.names=tuple(names) if names is not None else None;obj.columns=cols;return obj
+    def transform(self,table):
+        if self.names is None:
+            if table.names is not None:raise ValueError('Positional TRAIN requires positional prediction')
+            order=list(range(table.width))
+        else:
+            if table.names is None or set(table.names)!=set(self.names):raise ValueError('Feature schema mismatch')
+            positions={name:i for i,name in enumerate(table.names)};order=[positions[n] for n in self.names]
+        if len(order)!=len(self.columns):raise ValueError('Column count mismatch')
+        maps={};nominal={}
+        for j,(source,c) in enumerate(zip(order,self.columns)):
+            if c['kind']!=table.kinds[source]:
+                # An all-missing prediction column has no inferred string type.
+                if not(c['kind']=='categorical' and all(math.isnan(v) for v in table.column(source))):raise ValueError('Feature type mismatch')
+            if c['kind']=='categorical':
+                lookup={canonical_category_key(key):i+1 for i,key in enumerate(c['vocabulary'])};k=len(lookup);nominal[j]=k
+                maps[j]=[k+1]+[lookup.get(canonical_category_key(key),k+2) for key in table.vocab[source]]
+        return Encoded(table,order,maps,nominal)
+
+
+class Encoded(Rows):
+    def __init__(self,table,order,maps,nominal):self.source=table;self.order=order;self.maps=maps;self._nominal_sizes=nominal;self.n=table.n;self.width=len(order)
+    def value(self,i,j):
+        v=self.source.value(i,self.order[j])
+        return (self.maps[j][0] if self.source.kinds[self.order[j]]=='numeric' else self.maps[j][int(v)]) if j in self.maps else v
+    def column(self,j):
+        values=self.source.column(self.order[j])
+        if j not in self.maps:return values
+        if self.source.kinds[self.order[j]]=='numeric':return (self.maps[j][0] for _ in values)
+        return (self.maps[j][int(v)] for v in values)

morphtree/scalable_receipts.py

@@ -0,0 +1,92 @@
+"""Compact operation custody, not an authentication or scientific-usefulness claim.
+
+Dataset buffers stay outside JSON. Digests bind typed source columns, schema,
+ordered views and actual labels. Call-local completed-model caches never admit
+foreign persisted results; chain verification does not authorize budget replay.
+"""
+import hashlib
+import json
+import sys
+import uuid
+
+MAX_OPERATION_BYTES=1024*1024
+ENVELOPE={'receipt_schema','call_id','sequence','previous','sha256'}
+FIELDS={
+ 'CALL_STARTED':{'engine_sha256','scope','requested_task'},
+ 'RESOURCE_PLAN':{'plan'},
+ 'INPUT_PREPARED':{'rows','features','seconds','storage_bytes','input_identity'},
+ 'NATIVE_FIT_STARTED':{'ordinal','phase','rows','source_rows','family','cache_key','component_key','effective_rows_sha256','effective_labels_sha256','seed','params_sha256','preprocessing_sha256','program_sha256','role','reserved_scalar_trees'},
+ 'NATIVE_FIT_COMPLETED':{'ordinal','phase','seconds','component_key','actual_scalar_trees'},
+ 'CACHE_HIT':{'phase','family','rows','cache_key'},
+ 'SCREEN_COMPLETED':{'losses','parent','seconds'},
+ 'OOF_FOLD':{'parent_sha256','program_sha256','fit_rows','held_rows','disjoint','encoder_sha256','fit_indices_sha256','held_indices_sha256'},
+ 'DIAGNOSIS':{'parent_sha256','task','OOF_primary','fold_rows','TRAIN_only','fold_encoder_local','features_examined','region_count','profile_sha256','causal_status','seconds'},
+ 'MUTATION':{'generation','parent_sha256','child_sha256','child_program','parent_SEARCH','child_SEARCH','decision','seconds'},
+ 'PROGRAM_FROZEN':{'genome_sha256'},
+ 'OPTUNA_ASK':{'number','genome_sha256'},
+ 'OPTUNA_COMPLETE':{'number','SEARCH','parameters'},
+ 'BIN_PREPARATION_CACHE':{'limit_bytes','retained_bytes','entries','hits','misses','bypassed','evictions','peak_retained_bytes','reused_rows','preparation_seconds','integrity_seconds'},
+ 'FINAL_MODEL':{'input_rows','selection_TRAIN_rows','SEARCH_rows','final_fit_source_rows','refit_full','SEARCH_primary_before_refit','selection_metric_is_independent','nominal_columns','source_storage_bytes','all_rows_accounted'},
+ 'FIT_STOPPED':{'error_type'},
+}
+
+
+def canonical(v):
+    raw=json.dumps(v,sort_keys=True,allow_nan=False,separators=(',',':')).encode('utf8')
+    if len(raw)>MAX_OPERATION_BYTES:raise ValueError('Operation metadata exceeds finite allowance; data belong in buffers/artifacts')
+    return raw
+
+
+def sha(v):return hashlib.sha256(canonical(v)).hexdigest()
+def framed(h,text):
+    raw=text.encode('utf8');h.update(len(raw).to_bytes(8,'little'));h.update(raw)
+
+
+def schema_digest(names,kinds,vocabularies):
+    h=hashlib.sha256(b'SCALABLE_TYPED_SCHEMA_1')
+    framed(h,'POSITIONAL' if names is None else 'NAMED')
+    if names is not None:
+        for name in names:framed(h,name)
+    for kind,words in zip(kinds,vocabularies):
+        framed(h,kind);h.update(len(words).to_bytes(8,'little'))
+        for word in words:framed(h,word)
+    return h.hexdigest()
+
+
+def encoder_digest(encoder):return schema_digest(encoder.names,[c['kind'] for c in encoder.columns],[c.get('vocabulary',()) for c in encoder.columns])
+
+
+def data_identity(table,labels,role):
+    if role not in ('USER_INPUT','TRAIN','TRAIN_FULL'):raise ValueError('Explicit learning-source role')
+    if len(labels)!=len(table):raise ValueError('Aligned source/labels for identity')
+    h=hashlib.sha256(b'SCALABLE_ARRAY_SOURCE_1')
+    schema=schema_digest(table.names,table.kinds,table.vocab);framed(h,schema);framed(h,sys.byteorder)
+    for c in table.columns:
+        framed(h,c.typecode);h.update(len(c).to_bytes(8,'little'));h.update(memoryview(c))
+    source=h.hexdigest()
+    selected=hashlib.sha256(memoryview(table.indices)).hexdigest() if table.indices is not None else sha({'range_start':0,'range_stop':len(table)})
+    return dict(schema='SCALABLE_DATA_ID_1',source_sha256=source,schema_sha256=schema,effective_indices_sha256=selected,labels_sha256=hashlib.sha256(memoryview(labels)).hexdigest(),rows=len(table),features=table.width,role=role,byteorder=sys.byteorder,label_typecode=labels.typecode)
+
+
+class Ledger:
+    def __init__(self):self.call_id=uuid.uuid4().hex;self.previous='0'*64;self.sequence=0
+    def emit(self,event,fields):
+        if event not in FIELDS or set(fields)!=FIELDS[event]:raise ValueError('Exact operation metadata schema: '+event)
+        v=dict(event=event,**fields,receipt_schema='SCALABLE_OPERATION_1',call_id=self.call_id,sequence=self.sequence,previous=self.previous);v['sha256']=sha(v)
+        self.previous=v['sha256'];self.sequence+=1;return v
+
+
+def verify(events):
+    if type(events) is not list or not events or len(events)>10000:raise ValueError('Bounded nonempty receipt chain')
+    previous='0'*64;call_id=None
+    for i,event in enumerate(events):
+        if type(event) is not dict or event.get('event') not in FIELDS or set(event)!=FIELDS[event['event']]|ENVELOPE|{'event'}:raise ValueError('Invalid receipt fields')
+        if event['receipt_schema']!='SCALABLE_OPERATION_1' or type(event['sequence']) is not int or event['sequence']!=i or event['previous']!=previous:raise ValueError('Broken receipt order')
+        if type(event['call_id']) is not str or len(event['call_id'])!=32 or any(c not in '0123456789abcdef' for c in event['call_id']):raise ValueError('Malformed call identity')
+        if call_id is None:call_id=event['call_id']
+        if event['call_id']!=call_id:raise ValueError('Cross-call receipt replay')
+        body={k:v for k,v in event.items() if k!='sha256'}
+        if event['sha256']!=sha(body):raise ValueError('Changed operation receipt')
+        previous=event['sha256']
+    if events[0]['event']!='CALL_STARTED':raise ValueError('Missing call identity origin')
+    return dict(status='VERIFIED_HASH_CHAIN',events=len(events),call_id=call_id,final_sha256=previous,completion='FINAL_MODEL' if events[-1]['event']=='FINAL_MODEL' else 'PARTIAL_OR_NONTRAINING_CHAIN',authority='Integrity/custody only, not authorization or independent scientific validity')

morphtree/scalable_target.py

@@ -0,0 +1,68 @@
+"""Compact target normalization without ndarray/Series.tolist or regression sort."""
+from array import array
+import sys
+from .scalable_data import scalar
+
+
+def label_scalar(value):
+    if type(value).__module__.split('.')[0]=='numpy' and hasattr(value,'item'):value=value.item()
+    # Class identifiers are not floating-point measurements. Preserve integer
+    # identity above2**53 rather than merging distinct labels through float().
+    return value if type(value) is int else scalar(value)
+
+
+def detect_target(y,task,check,*,reserve=None):
+    if getattr(y,'ndim',1)!=1 or isinstance(y,(str,bytes,dict)) or not hasattr(y,'__len__') or len(y)<4:raise ValueError('At least four one-dimensional scalar targets required')
+    if task not in ('regression','classification','binary','multiclass'):raise ValueError('Explicit classifier/regressor intent required')
+    # Width follows the addressable input,not a small arbitrary class ceiling.
+    code='d' if task=='regression' else 'Q' if len(y)>2**32 else 'I'
+    numeric=None;values=array(code);lookup={};count=0;vocabulary_bytes=0
+    if reserve:reserve(len(y)*values.itemsize*(1 if task=='regression' else 2),'Target buffers before iteration')
+    for value in y:
+        if count%4096==0:check()
+        if reserve and isinstance(value,str):reserve(len(y)*values.itemsize*2+vocabulary_bytes+len(value)*32+512,'Target scalar conversion workspace')
+        value=scalar(value) if task=='regression' else label_scalar(value)
+        if value is None:raise ValueError('Missing target is unsupported')
+        is_numeric=type(value) in (int,float)
+        if numeric is None:numeric=is_numeric
+        if numeric!=is_numeric:raise ValueError('Target must be consistently string or numeric')
+        if task=='regression' and not numeric:raise ValueError('Regression requires finite numeric targets')
+        if task=='regression':values.append(value)
+        else:
+            if value not in lookup:
+                vocabulary_bytes+=sys.getsizeof(value)+256
+                if reserve:reserve(len(y)*values.itemsize*2+vocabulary_bytes,'Target vocabulary/codes/remap workspace')
+                lookup[value]=len(lookup)
+            values.append(lookup[value])
+        count+=1
+    if count!=len(y):raise ValueError('Target iterator/length mismatch')
+    if task=='regression':return task,values,[]
+    classes=sorted(lookup)
+    if len(classes)<2 or task=='binary' and len(classes)!=2:raise ValueError('Distinct task-compatible labels required')
+    task='binary' if len(classes)==2 else 'multiclass'
+    remap=[None]*len(classes)
+    for i,value in enumerate(classes):remap[lookup[value]]=i
+    return task,array(code,(remap[v] for v in values)),classes
+
+
+def compact_oof_train_indices(n, held, resources):
+    """Validated complement in ascending source order,without Python-int sets.
+
+    Index storage follows addressable rows; neither fold nor observations are
+    sampled. Every returned index is unmarked and every held index is marked,
+    which proves the logged disjointness without allocating a second large set.
+    """
+    if type(n) is not int or n < 2:raise ValueError('Positive OOF partition size')
+    code='I' if n < 2**32 else 'Q'
+    resources.reserve(n*(1+array(code).itemsize),'OOF membership mask and compact complement')
+    marked=bytearray(n)
+    for position,index in enumerate(held):
+        if position%4096==0:resources.check()
+        if type(index) is not int or not 0<=index<n or marked[index]:raise ValueError('Unique bounded held indices required')
+        marked[index]=1
+    result=array(code)
+    for index in range(n):
+        if index%4096==0:resources.check()
+        if not marked[index]:result.append(index)
+    if not result or len(result)==n:raise ValueError('OOF needs nonempty disjoint train and held partitions')
+    return result