morphtree 2.1.0__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- morphtree/__init__.py +28 -0
- morphtree/__main__.py +3 -0
- morphtree/_experimental/__init__.py +1 -0
- morphtree/_experimental/admission.py +364 -0
- morphtree/_experimental/base.py +215 -0
- morphtree/_experimental/component_cache_v2.py +263 -0
- morphtree/_experimental/evolution_report.py +307 -0
- morphtree/_experimental/execution_v2.py +697 -0
- morphtree/_experimental/facade.py +289 -0
- morphtree/_experimental/integrated.py +748 -0
- morphtree/_experimental/learning.py +418 -0
- morphtree/_experimental/manifest.json +204 -0
- morphtree/_experimental/numeric_fixture.py +21 -0
- morphtree/_experimental/parameter_contract.py +38 -0
- morphtree/_experimental/postfit.py +51 -0
- morphtree/_experimental/preprocessing.py +133 -0
- morphtree/_experimental/processes.py +191 -0
- morphtree/_experimental/producer.py +542 -0
- morphtree/_experimental/product.py +156 -0
- morphtree/_experimental/product_cli.py +43 -0
- morphtree/_experimental/roundtrip_v3.py +311 -0
- morphtree/_experimental/runtime_io.py +65 -0
- morphtree/_experimental/sampling.py +26 -0
- morphtree/_experimental/support.py +281 -0
- morphtree/_experimental/target.py +59 -0
- morphtree/_experimental/training.py +53 -0
- morphtree/_experimental/worker.py +682 -0
- morphtree/_installed.py +17 -0
- morphtree/_native/__init__.py +1 -0
- morphtree/_native/bin/win_amd64/morphtree_kernels.dll +0 -0
- morphtree/_native/buffers.py +82 -0
- morphtree/_native/loader.py +39 -0
- morphtree/_native/manifest.json +29 -0
- morphtree/_native/runtime.py +48 -0
- morphtree/_native/src/histogram.c +61 -0
- morphtree/_native/src/traversal.c +51 -0
- morphtree/api.py +388 -0
- morphtree/api_compat.py +65 -0
- morphtree/backend.py +162 -0
- morphtree/booster.py +252 -0
- morphtree/cli.py +52 -0
- morphtree/config.py +262 -0
- morphtree/control.py +20 -0
- morphtree/data.py +300 -0
- morphtree/diagnostics.py +158 -0
- morphtree/estimators.py +156 -0
- morphtree/examples.py +25 -0
- morphtree/exporting.py +224 -0
- morphtree/legacy_artifacts.py +32 -0
- morphtree/metrics.py +156 -0
- morphtree/model.py +1010 -0
- morphtree/mutations.py +205 -0
- morphtree/native_lab_backend.py +79 -0
- morphtree/numpy_backend.py +180 -0
- morphtree/optuna_tuner.py +154 -0
- morphtree/persistence.py +414 -0
- morphtree/prediction_output.py +35 -0
- morphtree/prepared_bins.py +82 -0
- morphtree/public.py +106 -0
- morphtree/resource_plan.py +170 -0
- morphtree/scalable.py +781 -0
- morphtree/scalable_backend.py +74 -0
- morphtree/scalable_checkpoint.py +129 -0
- morphtree/scalable_data.py +255 -0
- morphtree/scalable_receipts.py +92 -0
- morphtree/scalable_target.py +68 -0
- morphtree/scalable_variants.py +133 -0
- morphtree/sklearn_view.py +79 -0
- morphtree/source_markers.py +25 -0
- morphtree/species.py +47 -0
- morphtree/temporary.py +65 -0
- morphtree/training_fragment.py +130 -0
- morphtree/tree.py +231 -0
- morphtree/tuning.py +150 -0
- morphtree-2.1.0.dist-info/METADATA +212 -0
- morphtree-2.1.0.dist-info/RECORD +79 -0
- morphtree-2.1.0.dist-info/WHEEL +4 -0
- morphtree-2.1.0.dist-info/entry_points.txt +2 -0
- morphtree-2.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,682 @@
|
|
|
1
|
+
"""Finite installed worker/launcher. No checkout dependency or scheduler.
|
|
2
|
+
|
|
3
|
+
LIVE requires a separately pinned new owner permit; the current directive only
|
|
4
|
+
admits PREFIT and retained REPLAY. Existing ChildGuard/OS leases are reused.
|
|
5
|
+
"""
|
|
6
|
+
import copy
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import subprocess
|
|
10
|
+
import sys
|
|
11
|
+
import time
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from .admission import AdmissionEngine,AdmissionError,read,sha,digest,safe_path,environment
|
|
14
|
+
from .runtime_io import configure,artifact_path
|
|
15
|
+
from .processes import ChildGuard,exclusive_lock
|
|
16
|
+
from .producer import (PROFILE,PHASES,PIPE_CURRENT,Session,PreFitBoundary,prepare,
|
|
17
|
+
bound_reference,validate_permit,atomic_result,live_pipeline,claim_once)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def spawn_guarded(args,*,memory_limits=None,**kwargs):
|
|
21
|
+
"""Bind the existing cleanup job before a Windows venv creates descendants.
|
|
22
|
+
|
|
23
|
+
Only the primary thread of the child just created by this call is resumed.
|
|
24
|
+
No foreign process or thread is modified. Failure is fail-closed.
|
|
25
|
+
"""
|
|
26
|
+
if os.name!='nt':
|
|
27
|
+
child=subprocess.Popen(args,**kwargs);return child,ChildGuard(child,memory_limits=memory_limits)
|
|
28
|
+
import ctypes
|
|
29
|
+
from ctypes import wintypes
|
|
30
|
+
child=subprocess.Popen(args,creationflags=0x00000004,**kwargs) # CREATE_SUSPENDED
|
|
31
|
+
guard=None
|
|
32
|
+
try:
|
|
33
|
+
guard=ChildGuard(child,memory_limits=memory_limits)
|
|
34
|
+
class ThreadEntry(ctypes.Structure):
|
|
35
|
+
_fields_=[('dwSize',wintypes.DWORD),('cntUsage',wintypes.DWORD),('th32ThreadID',wintypes.DWORD),('th32OwnerProcessID',wintypes.DWORD),('tpBasePri',wintypes.LONG),('tpDeltaPri',wintypes.LONG),('dwFlags',wintypes.DWORD)]
|
|
36
|
+
k=ctypes.WinDLL('kernel32',use_last_error=True)
|
|
37
|
+
k.CreateToolhelp32Snapshot.argtypes=[wintypes.DWORD,wintypes.DWORD];k.CreateToolhelp32Snapshot.restype=wintypes.HANDLE
|
|
38
|
+
k.Thread32First.argtypes=[wintypes.HANDLE,ctypes.POINTER(ThreadEntry)];k.Thread32Next.argtypes=k.Thread32First.argtypes
|
|
39
|
+
k.OpenThread.argtypes=[wintypes.DWORD,wintypes.BOOL,wintypes.DWORD];k.OpenThread.restype=wintypes.HANDLE
|
|
40
|
+
k.ResumeThread.argtypes=[wintypes.HANDLE];k.ResumeThread.restype=wintypes.DWORD
|
|
41
|
+
k.CloseHandle.argtypes=[wintypes.HANDLE]
|
|
42
|
+
snapshot=k.CreateToolhelp32Snapshot(4,0)
|
|
43
|
+
if snapshot==ctypes.c_void_p(-1).value:raise ctypes.WinError(ctypes.get_last_error())
|
|
44
|
+
tids=[]
|
|
45
|
+
try:
|
|
46
|
+
entry=ThreadEntry();entry.dwSize=ctypes.sizeof(entry);more=k.Thread32First(snapshot,ctypes.byref(entry))
|
|
47
|
+
while more:
|
|
48
|
+
if entry.th32OwnerProcessID==child.pid:tids.append(entry.th32ThreadID)
|
|
49
|
+
more=k.Thread32Next(snapshot,ctypes.byref(entry))
|
|
50
|
+
finally:k.CloseHandle(snapshot)
|
|
51
|
+
if len(tids)!=1:raise RuntimeError('Owned suspended child must have one primary thread')
|
|
52
|
+
thread=k.OpenThread(2,False,tids[0]) # THREAD_SUSPEND_RESUME, owned child only
|
|
53
|
+
if not thread:raise ctypes.WinError(ctypes.get_last_error())
|
|
54
|
+
try:
|
|
55
|
+
if k.ResumeThread(thread)!=1:raise RuntimeError('Unexpected primary-thread suspension count')
|
|
56
|
+
finally:k.CloseHandle(thread)
|
|
57
|
+
return child,guard
|
|
58
|
+
except BaseException:
|
|
59
|
+
if guard:guard.close()
|
|
60
|
+
else:
|
|
61
|
+
if child.poll() is None:child.kill()
|
|
62
|
+
child.wait()
|
|
63
|
+
raise
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def ref(root,path):
|
|
67
|
+
p=safe_path(root,path);return {'path':p.relative_to(Path(root).resolve()).as_posix(),'sha256':sha(p),'bytes':p.stat().st_size}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def standalone(root,value,check):
|
|
71
|
+
"""Inference-only child, no installed library available under -S."""
|
|
72
|
+
root=Path(root);job={'program':value['program'],'expected':value['expected'],'query':value['query']}
|
|
73
|
+
source='''import sys,json,runpy
|
|
74
|
+
from pathlib import Path
|
|
75
|
+
root=Path(sys.argv[1]);d=json.loads((root/'STANDALONE-INPUT.json').read_text('utf8'))
|
|
76
|
+
class Named:
|
|
77
|
+
columns=d['query']['names']
|
|
78
|
+
def to_numpy(self):return d['query']['rows']
|
|
79
|
+
p=Path(d['program']['path']);p=p if p.is_absolute() else root/p
|
|
80
|
+
query=d['query']['rows'] if d['query']['names'] is None else Named()
|
|
81
|
+
ns=runpy.run_path(str(p));actual={'prediction':ns['predict'](query)}
|
|
82
|
+
if 'probabilities' in d['expected']:actual['probabilities']=ns['predict_proba'](query)
|
|
83
|
+
assert actual==d['expected']
|
|
84
|
+
assert not any(n=='tools' or n.startswith('morphtree') or n.startswith('numpy') or n.startswith('optuna') for n in sys.modules)
|
|
85
|
+
(root/'STANDALONE-RESULT.json').write_text(json.dumps({'status':'PASS','predictions':actual,'package_imports':0,'native_fits':0,'trials':0}),encoding='utf8')
|
|
86
|
+
'''
|
|
87
|
+
# program references are relative to the explicit input root; bind the
|
|
88
|
+
# resolved file into this isolated job, never a checkout path.
|
|
89
|
+
program=safe_path(root,value['program']['path'])
|
|
90
|
+
job['program']={**value['program'],'path':str(program)}
|
|
91
|
+
atomic_result(root,'STANDALONE-INPUT.json',job,check)
|
|
92
|
+
script=safe_path(root,'STANDALONE-CHECK.py')
|
|
93
|
+
from .runtime_io import publish
|
|
94
|
+
publish(script,source,check);check()
|
|
95
|
+
with (root/'STANDALONE.log').open('x',encoding='utf8') as f:
|
|
96
|
+
child,guard=spawn_guarded([sys.executable,'-I','-S','-B',str(script),str(root)],cwd=root,stdout=f,stderr=subprocess.STDOUT,start_new_session=os.name!='nt')
|
|
97
|
+
try:
|
|
98
|
+
while child.poll() is None:
|
|
99
|
+
check();time.sleep(.01)
|
|
100
|
+
finally:guard.close()
|
|
101
|
+
if child.returncode:raise RuntimeError('Standalone failed; original workflow remains failed')
|
|
102
|
+
return read(root/'STANDALONE-RESULT.json')
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def verify_commit(root):
|
|
106
|
+
root=Path(root);receipt=read(root/'COMPLETE.json')
|
|
107
|
+
if receipt.get('schema')!='morph031-installed-result-v2' or receipt.get('status') not in ('QUALIFIED_BEFORE_FIT','QUALIFIED_SOFTWARE_PIPELINE','COMPLETED_LIVE'):raise ValueError('Not a complete installed result')
|
|
108
|
+
result=bound_reference(root,receipt['result'])
|
|
109
|
+
for r in receipt['artifacts']:bound_reference(root,r) if r['path'].endswith('.json') else _verify_bytes(root,r)
|
|
110
|
+
if digest(result)!=receipt['result_digest']:raise ValueError('Changed result identity')
|
|
111
|
+
if receipt['status']!='COMPLETED_LIVE' and result['counts']['physical_fits_started']!=0:raise ValueError('Zero-fit result contains training')
|
|
112
|
+
if receipt['status']=='COMPLETED_LIVE' and not (0<result['counts']['physical_fits_completed']<=13 and result['counts']['physical_fits_completed']==result['counts']['physical_fits_started'] and result['counts']['Optuna_trials']==0 and result.get('pipeline_status')=='ALL_PLANNED_STAGES_COMPLETE'):raise ValueError('No completed actual fit')
|
|
113
|
+
return receipt,result
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _verify_bytes(root,r):
|
|
117
|
+
p=safe_path(root,r['path'])
|
|
118
|
+
if p.stat().st_size!=r['bytes'] or sha(p)!=r['sha256']:raise ValueError('Changed completed artifact')
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def commit(root,status,result,check):
|
|
122
|
+
root=Path(root);check()
|
|
123
|
+
if (root/'COMPLETE.json').exists():raise FileExistsError('Never replace completed result')
|
|
124
|
+
# Even if interrupted after RESULT publication, no COMPLETE means no winner.
|
|
125
|
+
rr=atomic_result(root,'RESULT.json',result,check)
|
|
126
|
+
artifacts=[]
|
|
127
|
+
for p in sorted(root.rglob('*')):
|
|
128
|
+
if p.is_file() and p.name not in ('COMPLETE.json','EXIT.json','WORKER.log') and not p.name.endswith('.pending') and 'leases' not in p.parts and p.name not in ('ADMISSION.jsonl','WRITER.lock','.lock'):
|
|
129
|
+
artifacts.append(ref(root,p))
|
|
130
|
+
done={'schema':'morph031-installed-result-v2','status':status,'result':rr,'result_digest':digest(result),'artifacts':artifacts,'native_counts':result['counts'],'training_claim':status=='COMPLETED_LIVE'}
|
|
131
|
+
atomic_result(root,'COMPLETE.json',done,check)
|
|
132
|
+
verify_commit(root)
|
|
133
|
+
return done
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _worker(root,policy_sha):
|
|
137
|
+
root=Path(root).resolve();configure(root.parent if root.parent.name=='runs' else root)
|
|
138
|
+
policy=read(root/'RUN-POLICY.json',expected_sha256=policy_sha)
|
|
139
|
+
# Bootstrap only: no data/model/Optuna work before actual pipe release.
|
|
140
|
+
atomic_result(root,'HELLO.json',{'pid':os.getpid(),'ppid':os.getppid(),'policy_sha256':policy_sha,'environment':environment()},lambda:None)
|
|
141
|
+
if sys.stdin.buffer.read(1)!=b'1':raise PermissionError('Missing pre-operation pipe admission')
|
|
142
|
+
configure(Path(policy['input_root']))
|
|
143
|
+
# Cooperative audit in the real child: no repository lookup is needed to
|
|
144
|
+
# deny every read outside stdlib/installation/explicit supplied inputs.
|
|
145
|
+
allowed=[Path(policy['input_root']).resolve(),Path(sys.prefix).resolve(),Path(sys.base_prefix).resolve(),Path(policy['authority_root']).resolve()]
|
|
146
|
+
def audit(event,args):
|
|
147
|
+
if event=='import' and (args[0]=='tools' or args[0].startswith('tools.')):raise ImportError('Developer namespace forbidden in installed worker')
|
|
148
|
+
if event=='open' and isinstance(args[0],(str,bytes,os.PathLike)):
|
|
149
|
+
try:p=Path(os.fsdecode(args[0])).resolve()
|
|
150
|
+
except (ValueError,OSError):return
|
|
151
|
+
if not any(p.is_relative_to(a) for a in allowed):raise PermissionError('Outside installed/runtime/explicit-input scope')
|
|
152
|
+
sys.addaudithook(audit)
|
|
153
|
+
engine=AdmissionEngine(package=Path(__file__).resolve().parents[1],qualification_path=policy['qualification_path'],qualification_sha256=policy['qualification_sha256'],workdir=root)
|
|
154
|
+
original_entries=[]
|
|
155
|
+
if policy['mode']!='LIVE':
|
|
156
|
+
# Independent software sentinel: a counter alone cannot prove that the
|
|
157
|
+
# original native function was never entered during a zero-fit test.
|
|
158
|
+
native_file=str(Path(__file__).resolve().parents[1]/'model.py')
|
|
159
|
+
optuna_file=str(Path(__file__).resolve().parents[2]/'optuna/study/study.py')
|
|
160
|
+
def no_native_profile(frame,event,arg):
|
|
161
|
+
if event=='call' and frame.f_code.co_filename==optuna_file and frame.f_code.co_name=='ask':raise AssertionError('Actual Optuna ask forbidden in zero-fit worker')
|
|
162
|
+
if event=='call' and frame.f_code.co_filename==native_file and frame.f_code.co_name=='fit':
|
|
163
|
+
original_entries.append(frame.f_code.co_name)
|
|
164
|
+
raise AssertionError('Original native fit forbidden in zero-fit worker')
|
|
165
|
+
sys.setprofile(no_native_profile)
|
|
166
|
+
if policy['environment']!=environment() or policy['worker_sha256']!=sha(__file__):raise ValueError('Actual installed worker/environment changed')
|
|
167
|
+
plan=bound_reference(engine.q['input_root'],policy['plan'])
|
|
168
|
+
owner=bound_reference(engine.q['input_root'],policy['owner_permit']) if policy['owner_permit'] else None
|
|
169
|
+
token=PIPE_CURRENT.set(digest(policy));session=None
|
|
170
|
+
try:
|
|
171
|
+
session=Session(engine,plan,policy,root,owner_permit=owner,owner_text_sha=policy['owner_text_sha256'])
|
|
172
|
+
if policy['scenario']=='before':
|
|
173
|
+
session.prepared={'task':'binary'}
|
|
174
|
+
raise ValueError('SOFTWARE_BEFORE_CALLBACK_FAULT')
|
|
175
|
+
try:__import__('tools.workspace_paths')
|
|
176
|
+
except ImportError:isolation='CHECKOUT_IMPORT_REFUSED'
|
|
177
|
+
else:raise AssertionError('Developer tools unexpectedly importable')
|
|
178
|
+
if policy['mode']=='FIXTURE':
|
|
179
|
+
value=live_pipeline(session)
|
|
180
|
+
# Generated export locators are input-root relative.
|
|
181
|
+
value['program']={**value['program'],'path':str(safe_path(engine.q['input_root'],value['program']['path']))}
|
|
182
|
+
value.setdefault('standalone',{'status':'ALREADY_VERIFIED_PER_WORKFLOW'})
|
|
183
|
+
status='QUALIFIED_SOFTWARE_PIPELINE'
|
|
184
|
+
elif policy['mode']=='PREFIT':
|
|
185
|
+
try:session.execute()
|
|
186
|
+
except PreFitBoundary as exc:
|
|
187
|
+
atomic_result(root,'PREFIT-READY.json',{'status':'BEFORE_ORIGINAL_NATIVE_FIT','key':session.prepared['key'],'counts':session.counters()},session.check)
|
|
188
|
+
scenario=policy['scenario']
|
|
189
|
+
if scenario=='journal':
|
|
190
|
+
with (session.store.root/'EVENTS.jsonl').open('ab') as f:f.write(b'{')
|
|
191
|
+
raise ValueError('SOFTWARE_JOURNAL_TAIL_FAULT')
|
|
192
|
+
if scenario=='crash':os._exit(17)
|
|
193
|
+
if scenario=='wait':
|
|
194
|
+
while True:session.check();time.sleep(.01)
|
|
195
|
+
if scenario=='filesystem':
|
|
196
|
+
# Intentional conflict at the atomic result destination.
|
|
197
|
+
(root/'RESULT.json.pending').write_text('partial fixture',encoding='utf8')
|
|
198
|
+
value={'kind':'ZERO_FIT_PREFLIGHT','reason':str(exc),'counts':session.counters(),'native_request':session.prepared['request'],'native_key':session.prepared['key'],'training_executed':False}
|
|
199
|
+
status='QUALIFIED_BEFORE_FIT'
|
|
200
|
+
else:raise AssertionError('Qualification reached an original native fit')
|
|
201
|
+
elif policy['mode']=='LIVE':
|
|
202
|
+
value=live_pipeline(session)
|
|
203
|
+
# Native result paths use the configured input-root locator.
|
|
204
|
+
value['program']={**value['program'],'path':str(safe_path(engine.q['input_root'],value['program']['path']))}
|
|
205
|
+
value.setdefault('standalone',{'status':'ALREADY_VERIFIED_PER_WORKFLOW'});status='COMPLETED_LIVE'
|
|
206
|
+
else:raise ValueError('Unknown worker mode')
|
|
207
|
+
session.check();value.update(mode=policy['mode'],plan_id=plan['id'],runtime=engine.identity,actual_worker_pid=os.getpid(),isolation=isolation,original_native_fit_entries=len(original_entries),origins={n:str(Path(m.__file__).resolve()) for n,m in sys.modules.items() if (n=='morphtree' or n.startswith('morphtree.')) and getattr(m,'__file__',None)})
|
|
208
|
+
commit(root,status,value,session.check)
|
|
209
|
+
except BaseException as exc:
|
|
210
|
+
try:failure_counts=session.counters() if session else {'physical_fits_started':0,'physical_fits_completed':0,'Optuna_trials':0}
|
|
211
|
+
except Exception as accounting_error:failure_counts={'status':'UNKNOWN','reason':type(accounting_error).__name__+': '+str(accounting_error),'journal_requires_reconciliation':True}
|
|
212
|
+
if session is not None:engine._event('WORKFLOW_INCOMPLETE',session.prepared.get('key','UNPREPARED') if session.prepared else 'UNPREPARED',error=type(exc).__name__,message=str(exc),counts=failure_counts)
|
|
213
|
+
if not (root/'WORKER-ERROR.json').exists():atomic_result(root,'WORKER-ERROR.json',{'status':'FAIL_OR_INTERRUPTED','error':type(exc).__name__,'message':str(exc),'counts':failure_counts},lambda:None)
|
|
214
|
+
raise
|
|
215
|
+
finally:PIPE_CURRENT.reset(token)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def activate(*,qualification_path,qualification_sha256,plan_ref,root,owner_text,expected_owner_text_sha256):
|
|
219
|
+
"""Explicit owner activation, never called by zero-fit qualification.
|
|
220
|
+
|
|
221
|
+
Freezes the single deadline before creating the detached permit. The
|
|
222
|
+
expected message SHA is supplied by the trusted authorized controller.
|
|
223
|
+
"""
|
|
224
|
+
begin_ns=time.monotonic_ns();epoch=time.time()
|
|
225
|
+
if not expected_owner_text_sha256:raise PermissionError('New explicit owner consent is absent')
|
|
226
|
+
q=read(qualification_path,expected_sha256=qualification_sha256);root=safe_path(q['input_root'],root);configure(q['input_root'])
|
|
227
|
+
if not root.is_dir():raise ValueError('Existing owner-scoped directory required')
|
|
228
|
+
with exclusive_lock(root/'activation-lease'):
|
|
229
|
+
if any((root/n).exists() for n in ('ACTIVATION.json','ATTEMPT.json','RUN-POLICY.json','STOP')):raise RuntimeError('Activation consumed; cannot renew')
|
|
230
|
+
owner=bound_reference(q['input_root'],owner_text)
|
|
231
|
+
if owner_text['sha256']!=expected_owner_text_sha256 or owner.get('authority')!='EXPLICIT_HUMAN_MESSAGE' or owner.get('campaign')!='COMP009-LIVE-001' or owner.get('approved_profile')!=PROFILE:raise PermissionError('Not this new exact owner authorization')
|
|
232
|
+
e=AdmissionEngine(package=Path(__file__).resolve().parents[1],qualification_path=qualification_path,qualification_sha256=qualification_sha256,workdir=root)
|
|
233
|
+
p=bound_reference(q['input_root'],plan_ref);a=prepare(e,p)
|
|
234
|
+
if owner.get('approved_runtime')!=e.identity or owner.get('approved_protocol')!=p['protocol'] or owner.get('authority_root')!=q['authority_root']:raise PermissionError('Owner grant does not bind this exact runtime/protocol/authority scope')
|
|
235
|
+
if a['task']!='multiphase':raise PermissionError('Only exact finite multiphase pilot')
|
|
236
|
+
policy={'schema':'morph031-finite-installed-worker-v2','authority_root':q['authority_root'],'mode':'LIVE','scenario':'normal','input_root':q['input_root'],'qualification_path':str(Path(qualification_path).resolve()),'qualification_sha256':qualification_sha256,'plan':plan_ref,'environment':environment(),'worker_sha256':sha(__file__),
|
|
237
|
+
'created_ns':begin_ns,'deadline_ns':min(begin_ns+int(PROFILE['worker_seconds']*1e9),begin_ns+int(max(0,q['deadline_utc_epoch']-epoch)*1e9)),'deadline_utc':min(epoch+PROFILE['worker_seconds'],q['deadline_utc_epoch'])}
|
|
238
|
+
# INT010 is zero-fit and cannot consume or create a LIVE authority.
|
|
239
|
+
if p['protocol'].get('id')!='COMP-009-LIVE-001' or not e.q.get('training_protocol'):raise PermissionError('Separate frozen COMP003LIVE required before activation')
|
|
240
|
+
physical=bound_reference(q['input_root'],e.q['training_protocol'])
|
|
241
|
+
if physical.get('id')!='COMP-009-LIVE-001' or not physical.get('frozen_before_execution') or physical.get('planned_profile')!=PROFILE:raise PermissionError('Different physical protocol/profile')
|
|
242
|
+
claim_once(q['authority_root'],expected_owner_text_sha256,{'owner_text_sha256':expected_owner_text_sha256,'native_key':a['key'],'output_root':str(root),'policy_sha256':digest(policy)})
|
|
243
|
+
ar=atomic_result(root,'ACTIVATION.json',policy,lambda:None)
|
|
244
|
+
permit={'schema':'morph031-owner-fit-permit-v1','scope':'EXPLICIT_OWNER_APPROVED_COMP009_EVOLUTION','owner_text':owner_text,'runtime':e.identity,'protocol':p['protocol'],'plan_id':p['id'],'native_key':a['key'],'profile':PROFILE,'output_root':str(root),'phases':list(PHASES),'deadline_utc':policy['deadline_utc'],'policy_sha256':digest(policy)}
|
|
245
|
+
pr=atomic_result(root,'OWNER-PERMIT.json',permit,lambda:None)
|
|
246
|
+
return {'activation_ref':ref(q['input_root'],root/ar['path']),'owner_permit':ref(q['input_root'],root/pr['path']),'owner_text_sha256':expected_owner_text_sha256}
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def launch(*,qualification_path,qualification_sha256,plan_ref,root,mode='PREFIT',seconds=20,scenario='normal',control=None,owner_permit=None,owner_text_sha256=None,activation_ref=None):
|
|
250
|
+
"""One bounded actual child. No retries, resume, cached fitting or scheduler."""
|
|
251
|
+
begin_ns=time.monotonic_ns();begin_epoch=time.time();root=Path(root).resolve()
|
|
252
|
+
if mode not in ('PREFIT','FIXTURE','LIVE') or type(seconds) not in (int,float) or not math_finite(seconds) or not 0<seconds<=(PROFILE['worker_seconds'] if mode=='LIVE' else 540):raise ValueError('Finite exact worker mode/time')
|
|
253
|
+
if scenario not in ('normal','wait','crash','filesystem','journal','before','oof') or control not in (None,'stop','terminate'):raise ValueError('Known qualification fault only')
|
|
254
|
+
q=read(qualification_path,expected_sha256=qualification_sha256);safe_path(q['input_root'],root);configure(q['input_root'])
|
|
255
|
+
if not root.is_dir():raise ValueError('Prevalidated existing run directory required')
|
|
256
|
+
with exclusive_lock(root/'parent-lease'):
|
|
257
|
+
if (root/'ATTEMPT.json').exists() or (root/'RUN-POLICY.json').exists() or (root/'STOP').exists():raise RuntimeError('Run already consumed/stopped; no retry or deadline renewal')
|
|
258
|
+
e=AdmissionEngine(package=Path(__file__).resolve().parents[1],qualification_path=qualification_path,qualification_sha256=qualification_sha256,workdir=root)
|
|
259
|
+
plan=bound_reference(q['input_root'],plan_ref);prepared=prepare(e,plan)
|
|
260
|
+
policy={'schema':'morph031-finite-installed-worker-v2','authority_root':q['authority_root'],'mode':mode,'scenario':scenario,'input_root':q['input_root'],'qualification_path':str(Path(qualification_path).resolve()),'qualification_sha256':qualification_sha256,'plan':plan_ref,'environment':environment(),'worker_sha256':sha(__file__),
|
|
261
|
+
'created_ns':begin_ns,'deadline_ns':min(begin_ns+int(seconds*1e9),begin_ns+int(max(0,q['deadline_utc_epoch']-begin_epoch)*1e9)),'deadline_utc':min(begin_epoch+seconds,q['deadline_utc_epoch']),'owner_permit':owner_permit,'owner_text_sha256':owner_text_sha256}
|
|
262
|
+
if mode=='LIVE':
|
|
263
|
+
if scenario!='normal' or control is not None:raise ValueError('No injected faults in a physical pilot')
|
|
264
|
+
if not owner_permit or not owner_text_sha256 or not activation_ref:raise PermissionError('Separate new owner activation/permit absent')
|
|
265
|
+
activation=bound_reference(q['input_root'],activation_ref)
|
|
266
|
+
for k in ('schema','mode','scenario','input_root','qualification_path','qualification_sha256','plan','environment','worker_sha256'):
|
|
267
|
+
if activation[k]!=policy[k]:raise ValueError('Different activation source/environment/request')
|
|
268
|
+
policy={**activation,'owner_permit':owner_permit,'owner_text_sha256':owner_text_sha256}
|
|
269
|
+
begin_ns=policy['created_ns']
|
|
270
|
+
permit=bound_reference(q['input_root'],owner_permit) if owner_permit else None
|
|
271
|
+
validate_permit(permit,expected_owner_text_sha=owner_text_sha256,engine=e,prepared=prepared,policy=policy,workdir=root)
|
|
272
|
+
atomic_result(root,'RUN-POLICY.json',policy,lambda:None)
|
|
273
|
+
policy_file=root/'RUN-POLICY.json';pin=sha(policy_file)
|
|
274
|
+
atomic_result(root,'ATTEMPT.json',{'status':'INTENT_BEFORE_PROCESS','plan_id':plan['id'],'mode':mode,'native_calls':0,'trial_calls':0,'policy_sha256':pin},lambda:None)
|
|
275
|
+
env=os.environ.copy();env.pop('PYTHONPATH',None);env.update(PYTHONUTF8='1',PYTHONDONTWRITEBYTECODE='1',PYTHONNOUSERSITE='1')
|
|
276
|
+
reason=None;hello=None
|
|
277
|
+
with (root/'WORKER.log').open('x',encoding='utf8') as log:
|
|
278
|
+
child,guard=spawn_guarded([sys.executable,'-I','-B','-m','morphtree._experimental.worker',str(root),pin],cwd=root,stdin=subprocess.PIPE,stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=os.name!='nt')
|
|
279
|
+
try:
|
|
280
|
+
while child.poll() is None and not (root/'HELLO.json').exists():
|
|
281
|
+
if time.monotonic_ns()>=policy['deadline_ns']:raise TimeoutError('Bootstrap deadline')
|
|
282
|
+
time.sleep(.01)
|
|
283
|
+
hello=read(root/'HELLO.json')
|
|
284
|
+
if hello['policy_sha256']!=pin or hello['environment']!=environment() or hello['pid']!=child.pid and hello['ppid']!=child.pid:raise ValueError('Unknown bootstrap process identity')
|
|
285
|
+
guard.include_pid(hello['pid'])
|
|
286
|
+
child.stdin.write(b'1');child.stdin.close()
|
|
287
|
+
while child.poll() is None:
|
|
288
|
+
ready=(root/'PREFIT-READY.json').exists()
|
|
289
|
+
if ready and control=='stop':(root/'STOP').write_text('owner-controlled zero-fit test STOP',encoding='utf8');reason='STOP';break
|
|
290
|
+
if ready and control=='terminate':reason='CONTROLLED_PROCESS_INTERRUPTION';break
|
|
291
|
+
if time.monotonic_ns()>=policy['deadline_ns'] or time.time()>=policy['deadline_utc']:reason='DEADLINE';break
|
|
292
|
+
if (root/'STOP').exists():reason='STOP';break
|
|
293
|
+
time.sleep(.01)
|
|
294
|
+
except Exception as exc:reason='BOOTSTRAP_OR_PROCESS:'+type(exc).__name__+': '+str(exc)
|
|
295
|
+
finally:guard.close()
|
|
296
|
+
rc=child.wait(timeout=5)
|
|
297
|
+
completed=False
|
|
298
|
+
if rc==0 and reason is None:
|
|
299
|
+
try:receipt,value=verify_commit(root);completed=True
|
|
300
|
+
except Exception as exc:reason='RESULT_VERIFICATION:'+type(exc).__name__+': '+str(exc)
|
|
301
|
+
exit={'status':'SOFTWARE_QUALIFIED' if completed and mode!='LIVE' else 'COMPLETED_LIVE' if completed else 'INCOMPLETE','returncode':rc,'reason':reason,'mode':mode,'actual_worker_pid':hello['pid'] if hello else None,'launcher_pid':child.pid,'guard_closed':True,'native_training_claim':completed and mode=='LIVE','operational_seconds_with_cleanup':time.monotonic_ns()/1e9-begin_ns/1e9,'retry':False}
|
|
302
|
+
atomic_result(root,'EXIT.json',exit,lambda:None)
|
|
303
|
+
if not (root/'STOP').exists():(root/'STOP').write_text('terminal finite run',encoding='utf8')
|
|
304
|
+
return exit
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def math_finite(value):
|
|
308
|
+
import math
|
|
309
|
+
return math.isfinite(value)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
PUBLIC_PROFILE={'calls':51,'scalar_trees':90,'trials':6,'global_seconds':600,
|
|
313
|
+
'worker_seconds':540,'call_seconds':150,'component_seconds':60,'workers':1,
|
|
314
|
+
'artifact_bytes':128*1024**2,'process_commit_bytes':768*1024**2,'job_commit_bytes':1024**3,
|
|
315
|
+
'retries':0,'refits':0,'calibrators':0,'GUARD':0,'HTTP':0}
|
|
316
|
+
PUBLIC_CAMPAIGN='PRODUCT007-DEFAULT-LIVE-001'
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def public_artifact_bytes(root):
|
|
320
|
+
"""Checked size,not a disk quota;renamed serializer temporaries may vanish."""
|
|
321
|
+
import stat
|
|
322
|
+
total=0
|
|
323
|
+
for p in Path(root).rglob('*'):
|
|
324
|
+
try:s=p.lstat()
|
|
325
|
+
except FileNotFoundError:continue
|
|
326
|
+
if stat.S_ISLNK(s.st_mode) or getattr(s,'st_file_attributes',0)&0x400:
|
|
327
|
+
raise ValueError('Public campaign artifact reparse/symlink escape')
|
|
328
|
+
if stat.S_ISREG(s.st_mode):total+=s.st_size
|
|
329
|
+
return total
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def public_resource_availability(root):
|
|
333
|
+
"""One safety snapshot,not an exclusive host or performance admission."""
|
|
334
|
+
import ctypes,shutil
|
|
335
|
+
if os.name!='nt':raise RuntimeError('Owned private-commit resource profile qualified on Windows only')
|
|
336
|
+
class MemoryStatus(ctypes.Structure):
|
|
337
|
+
_fields_=[('length',ctypes.c_ulong),('load',ctypes.c_ulong)]+[(n,ctypes.c_uint64) for n in ('total_physical','available_physical','total_commit','available_commit','total_virtual','available_virtual','extended_virtual')]
|
|
338
|
+
value=MemoryStatus();value.length=ctypes.sizeof(value)
|
|
339
|
+
kernel=ctypes.WinDLL('kernel32',use_last_error=True)
|
|
340
|
+
kernel.GlobalMemoryStatusEx.argtypes=[ctypes.POINTER(MemoryStatus)]
|
|
341
|
+
if not kernel.GlobalMemoryStatusEx(ctypes.byref(value)):raise OSError('Cannot read resource availability')
|
|
342
|
+
disk=shutil.disk_usage(root)
|
|
343
|
+
if disk.free<4*PUBLIC_PROFILE['artifact_bytes'] or min(value.available_physical,value.available_commit)<PUBLIC_PROFILE['job_commit_bytes']:
|
|
344
|
+
raise RuntimeError('Insufficient point-in-time disk or memory availability;no retry')
|
|
345
|
+
return {'available_physical_bytes':int(value.available_physical),'available_commit_bytes':int(value.available_commit),
|
|
346
|
+
'free_disk_bytes':disk.free,'exclusive_host':False,'foreign_processes_inspected_or_modified':False,
|
|
347
|
+
'performance_admission':False,'scope':'Point-in-time safety observation;resources are not reserved against unrelated workloads'}
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _public_protocol(path,pin,*,verify_inputs=True):
|
|
351
|
+
"""Exact owned qualification profile,not an ordinary-user requirement."""
|
|
352
|
+
import math
|
|
353
|
+
from morphtree import api
|
|
354
|
+
from . import admission as A
|
|
355
|
+
path=Path(path).resolve();p=read(path,expected_sha256=pin)
|
|
356
|
+
keys={'schema','id','profile','runtime','wheel','optional','sources','prefit_models','authority_root','not_after'}
|
|
357
|
+
if set(p)!=keys or p['schema']!='product007-public-protocol-v1' or p['id']!=PUBLIC_CAMPAIGN or p['profile']!=PUBLIC_PROFILE:
|
|
358
|
+
raise ValueError('Exact public qualification protocol/profile required')
|
|
359
|
+
if type(p['not_after']) not in (int,float) or not math.isfinite(p['not_after']) or time.time()>=p['not_after']:
|
|
360
|
+
raise PermissionError('Public qualification protocol expired')
|
|
361
|
+
if p['authority_root']!='authorities':raise ValueError('Single canonical authority root required')
|
|
362
|
+
package=Path(api.__file__).parent
|
|
363
|
+
q={'usage':'ORDINARY_USER_NOT_SCIENTIFIC',**p['runtime']}
|
|
364
|
+
identity=A.installed_identity(package,q)
|
|
365
|
+
if api.PUBLIC_USER_RUNTIME!='1.0.0rc4' or identity['version']!=api.PUBLIC_USER_RUNTIME:
|
|
366
|
+
raise ValueError('Exact qualified product004 runtime required')
|
|
367
|
+
# A wheel is binary: bind it directly, never parse it as JSON.
|
|
368
|
+
wheel=p['wheel'];wf=safe_path(path.parent,wheel['path'])
|
|
369
|
+
if set(wheel)!={'path','sha256','bytes'} or wf.stat().st_size!=wheel['bytes'] or sha(wf)!=wheel['sha256']:
|
|
370
|
+
raise ValueError('Qualified wheel bytes changed')
|
|
371
|
+
if set(p['sources'])!={'binary','multiclass','regression'} or set(p['prefit_models'])!=set(p['sources']):
|
|
372
|
+
raise ValueError('Three exact DEV sources and retained IO models required')
|
|
373
|
+
optional=p['optional']
|
|
374
|
+
if set(optional)!={'version','files'} or optional['version']!='5.0.0' or not 1<=len(optional['files'])<=4096:
|
|
375
|
+
raise ValueError('Exact optional dependency graph required')
|
|
376
|
+
for n,h in optional['files'].items():
|
|
377
|
+
if Path(n).parts[0]=='morphtree' or sha(safe_path(package.parent,n))!=h:raise ValueError('Changed optional dependency')
|
|
378
|
+
if verify_inputs:
|
|
379
|
+
from .target import detect_target,_split
|
|
380
|
+
for task,s in p['sources'].items():
|
|
381
|
+
if set(s)!={'data','exposure','split'} or s['exposure']!='DEV_EXPOSED':raise PermissionError('Only frozen exposed DEV source')
|
|
382
|
+
raw=bound_reference(path.parent,s['data']);actual,y,classes=detect_target(raw['y'],task)
|
|
383
|
+
ti,si=_split(y,task,42)
|
|
384
|
+
if actual!=task or len(raw['X'])!=48 or any(len(row)!=3 for row in raw['X']) or s['split']!={'TRAIN':ti,'SEARCH':si,'GUARD':[]}:
|
|
385
|
+
raise ValueError('Changed DEV rows/task/split')
|
|
386
|
+
model=safe_path(path.parent,p['prefit_models'][task]['path']);r=p['prefit_models'][task]
|
|
387
|
+
if model.stat().st_size!=r['bytes'] or sha(model)!=r['sha256']:raise ValueError('Changed retained IO model')
|
|
388
|
+
return p,identity
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
class PublicCallBoundary:
|
|
392
|
+
"""Existing worker's shared resource/custody boundary;no learning policy."""
|
|
393
|
+
def __init__(self,root,policy,protocol):
|
|
394
|
+
from . import component_cache_v2 as cache
|
|
395
|
+
self.root=Path(root).resolve();self.policy=copy.deepcopy(policy);self.protocol=copy.deepcopy(protocol)
|
|
396
|
+
self.pin=digest(self.policy);self.protocol_pin=digest(protocol);self.closed=False;self.cursor=0
|
|
397
|
+
self.calls=safe_path(self.root,'calls');self.calls.mkdir(exist_ok=True)
|
|
398
|
+
self.sessions=[];self.temporary_roots=[]
|
|
399
|
+
configure(self.root);cache.configure(self.root,Path(__file__).parents[1])
|
|
400
|
+
self.store=cache.MultitaskStore(safe_path(self.root,'global-store'),max_compute_jobs=PUBLIC_PROFILE['calls'],seconds=PUBLIC_PROFILE['worker_seconds'])
|
|
401
|
+
self.check()
|
|
402
|
+
|
|
403
|
+
def check(self):
|
|
404
|
+
if self.closed or (self.root/'STOP').exists():raise RuntimeError('Public campaign STOP permanent')
|
|
405
|
+
if time.time()>=self.policy['deadline_utc'] or time.monotonic_ns()>=self.policy['deadline_ns']:
|
|
406
|
+
raise TimeoutError('One immutable global campaign deadline')
|
|
407
|
+
|
|
408
|
+
def integrity(self):
|
|
409
|
+
self.check()
|
|
410
|
+
if digest(self.policy)!=self.pin or digest(self.protocol)!=self.protocol_pin or sha(self.root/'PUBLIC-POLICY.json')!=self.policy['policy_file_sha256']:
|
|
411
|
+
raise ValueError('Mutated campaign binding')
|
|
412
|
+
if sha(Path(self.policy['protocol']['path']))!=self.policy['protocol']['sha256']:
|
|
413
|
+
raise ValueError('Mutated campaign protocol')
|
|
414
|
+
if public_artifact_bytes(self.root)>PUBLIC_PROFILE['artifact_bytes']:
|
|
415
|
+
raise RuntimeError('Checked artifact budget exceeded;not a disk quota')
|
|
416
|
+
|
|
417
|
+
def _global(self):
|
|
418
|
+
from contextlib import contextmanager
|
|
419
|
+
from .runtime_io import WORKDIR
|
|
420
|
+
@contextmanager
|
|
421
|
+
def scope():
|
|
422
|
+
token=WORKDIR.set(self.root)
|
|
423
|
+
try:yield
|
|
424
|
+
finally:WORKDIR.reset(token)
|
|
425
|
+
return scope()
|
|
426
|
+
|
|
427
|
+
def event(self,kind,**fields):
|
|
428
|
+
with self._global():self.store.event(kind,**fields)
|
|
429
|
+
|
|
430
|
+
def used(self):
|
|
431
|
+
with self._global():rows=self.store.events()
|
|
432
|
+
intents=[r for r in rows if r['event']=='PUBLIC_LEARNING_INTENT']
|
|
433
|
+
return {'calls':sum(r['operation']=='fit' for r in intents),'scalar_trees':sum(r['trees'] for r in intents),'trials':sum(r['operation']=='trial' for r in intents)}
|
|
434
|
+
|
|
435
|
+
def reserve(self,session,*,operation,trees,key=None):
|
|
436
|
+
self.integrity()
|
|
437
|
+
if session not in self.sessions or operation not in ('fit','trial') or type(trees) is not int or trees<0 or operation=='trial' and trees:
|
|
438
|
+
raise PermissionError('Unknown nested operation/session')
|
|
439
|
+
if session.phase not in ('FAMILY','OOF','STRUCTURE','NUMERIC') or operation=='trial' and (session.phase!='NUMERIC' or session.frozen is None):
|
|
440
|
+
raise PermissionError('No numerical reservation before structural freeze')
|
|
441
|
+
with self._global(),exclusive_lock(safe_path(self.root,'public-budget-lock')):
|
|
442
|
+
used=self.used();delta={'calls':int(operation=='fit'),'scalar_trees':trees,'trials':int(operation=='trial')}
|
|
443
|
+
if any(used[k]+v>PUBLIC_PROFILE[k] for k,v in delta.items()):raise RuntimeError('Shared public campaign budget exhausted')
|
|
444
|
+
if key is None and self.policy['mode']=='PREFIT':key=digest({'software_only':used,'operation':operation,'call':str(session.root)})
|
|
445
|
+
if type(key) is not str or len(key)!=64 or any(c not in '0123456789abcdef' for c in key):raise ValueError('Canonical nested operation key required')
|
|
446
|
+
self.store.event('PUBLIC_LEARNING_INTENT',operation=operation,key=key,trees=trees,call_root=str(session.root),phase=session.phase,
|
|
447
|
+
evidence='PREFIT_RESERVATION_NOT_LEARNING' if self.policy['mode']=='PREFIT' else 'OWNER_BOUND_LIVE_RESERVATION_NOT_COMPLETION')
|
|
448
|
+
|
|
449
|
+
def before_input(self,model):
|
|
450
|
+
self.integrity()
|
|
451
|
+
if self.cursor>=3 or model.task!='auto' or (model.families,model.morphing,model.tuning)!=('auto','auto','optuna') or model.random_state!=42 or model.time_budget!='auto' or model.n_trials!='auto' or model.n_jobs not in ('auto',1) or model.capacity is not None or model._max_mutations!=2:
|
|
452
|
+
raise PermissionError('Exact next public API call required')
|
|
453
|
+
|
|
454
|
+
def input_rows(self,rows,y,names,task):
|
|
455
|
+
self.integrity();expected=('binary','multiclass','regression')[self.cursor]
|
|
456
|
+
r=self.protocol['sources'][expected]['data'];raw=bound_reference(Path(self.policy['protocol']['path']).parent,r)
|
|
457
|
+
if task!=expected or rows!=raw['X'] or y!=raw['y'] or names is not None:
|
|
458
|
+
raise ValueError('Public qualification input differs from frozen plain lists')
|
|
459
|
+
|
|
460
|
+
def retain_temporary(self,temporary,root):
|
|
461
|
+
self.integrity();root=safe_path(self.calls,root)
|
|
462
|
+
# Qualification-owned custody only:normal user calls keep normal cleanup.
|
|
463
|
+
temporary._finalizer.detach();self.temporary_roots.append(str(root))
|
|
464
|
+
self.event('PUBLIC_CALL_DIRECTORY_RETAINED',path=str(root),call_index=self.cursor)
|
|
465
|
+
|
|
466
|
+
def bind_session(self,session):
|
|
467
|
+
self.integrity()
|
|
468
|
+
if str(session.root) not in self.temporary_roots or len(self.sessions)!=self.cursor:
|
|
469
|
+
raise PermissionError('Unexpected concurrent or unretained user Session')
|
|
470
|
+
self.sessions.append(session)
|
|
471
|
+
self.event('PUBLIC_SESSION_BOUND',call_index=self.cursor,session_sha256=session.policy_sha,source_sha256=digest(session.source))
|
|
472
|
+
|
|
473
|
+
def output(self,path):
|
|
474
|
+
self.integrity();return safe_path(self.root,path)
|
|
475
|
+
|
|
476
|
+
def end_call(self,*,status):
|
|
477
|
+
self.integrity();s=self.sessions[-1]
|
|
478
|
+
if not s._closed:raise RuntimeError('Public call not closed')
|
|
479
|
+
self.event('PUBLIC_CALL_CLOSED',call_index=self.cursor,status=status,counts=s.counters())
|
|
480
|
+
self.cursor+=1
|
|
481
|
+
|
|
482
|
+
def close(self,reason):
|
|
483
|
+
if self.closed:return
|
|
484
|
+
try:self.event('PUBLIC_CAMPAIGN_CLOSED',reason=reason,reservations=self.used(),authority_residual=0)
|
|
485
|
+
finally:
|
|
486
|
+
self.closed=True
|
|
487
|
+
if not (self.root/'STOP').exists():(self.root/'STOP').write_text(reason,encoding='utf8')
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def run_public_qualification(root,protocol_path,protocol_sha256,*,mode='PREFIT',owner_text=None,expected_owner_sha256=None,scenario='normal'):
|
|
491
|
+
"""Single owned worker;no valid LIVE entry without separate fresh owner SHA.
|
|
492
|
+
|
|
493
|
+
The expected SHA comes from the trusted human-facing caller,not a permit or
|
|
494
|
+
a model. This cooperative boundary cannot authenticate hostile Python code.
|
|
495
|
+
"""
|
|
496
|
+
from .producer import _original_claim_once
|
|
497
|
+
from .runtime_io import WORKDIR
|
|
498
|
+
if mode not in ('PREFIT','LIVE') or scenario not in ('normal','wait','crash') or mode=='LIVE' and scenario!='normal':
|
|
499
|
+
raise ValueError('Known finite qualification mode only')
|
|
500
|
+
if mode=='LIVE' and (not owner_text or not expected_owner_sha256):raise PermissionError('Separate fresh human authorization absent')
|
|
501
|
+
entry=time.monotonic_ns();epoch=time.time();protocol_path=Path(protocol_path).resolve()
|
|
502
|
+
# Reject before source rows are materialized. A valid fresh human token is
|
|
503
|
+
# consumed before full preflight;preflight failure cannot recycle it.
|
|
504
|
+
root=safe_path(protocol_path.parent,root)
|
|
505
|
+
authority=safe_path(protocol_path.parent,'authorities');authority.mkdir(exist_ok=True)
|
|
506
|
+
if mode=='LIVE':
|
|
507
|
+
token=expected_owner_sha256
|
|
508
|
+
else:token=digest({'software_only':str(root),'protocol':protocol_sha256})
|
|
509
|
+
previous=WORKDIR.get();configure(authority)
|
|
510
|
+
try:claim=_original_claim_once(authority,token,{'schema':'product007-public-consumption-v1','mode':mode,'campaign':PUBLIC_CAMPAIGN,'protocol_sha256':protocol_sha256,'output_root':str(root),'owner_sha256':expected_owner_sha256 if mode=='LIVE' else None})
|
|
511
|
+
finally:
|
|
512
|
+
if previous is not None:configure(previous)
|
|
513
|
+
if root.exists():
|
|
514
|
+
configure(authority)
|
|
515
|
+
atomic_result(authority/'.native-authorities'/token,'CLOSED.json',{'status':'INCOMPLETE','reason':'OUTPUT_ALREADY_EXISTS','authority_residual':0},lambda:None)
|
|
516
|
+
raise FileExistsError('Never replay qualification output;binding now consumed')
|
|
517
|
+
root.mkdir();configure(root);child=None;guard=None;reason=None;statistics=None;hello=None
|
|
518
|
+
try:
|
|
519
|
+
owner_record=None
|
|
520
|
+
if mode=='LIVE':
|
|
521
|
+
owner=safe_path(protocol_path.parent,owner_text)
|
|
522
|
+
owner_record=read(owner,expected_sha256=expected_owner_sha256)
|
|
523
|
+
required={'schema','authority','campaign','protocol_sha256','runtime_manifest_sha256','profile','message','created_utc_epoch','expires_utc_epoch'}
|
|
524
|
+
if set(owner_record)!=required or owner_record['schema']!='product007-explicit-owner-v1' or owner_record['authority']!='EXPLICIT_HUMAN_MESSAGE' or owner_record['campaign']!=PUBLIC_CAMPAIGN or owner_record['protocol_sha256']!=protocol_sha256 or owner_record['profile']!=PUBLIC_PROFILE or not isinstance(owner_record['message'],str) or not owner_record['message']:
|
|
525
|
+
raise PermissionError('Exact fresh owner scope absent;metadata is not human authorization')
|
|
526
|
+
import math
|
|
527
|
+
if type(owner_record['created_utc_epoch']) not in (int,float) or type(owner_record['expires_utc_epoch']) not in (int,float) or not math.isfinite(owner_record['created_utc_epoch']) or not math.isfinite(owner_record['expires_utc_epoch']) or not owner_record['created_utc_epoch']<=time.time()<owner_record['expires_utc_epoch']:
|
|
528
|
+
raise PermissionError('Owner grant expired or not yet valid')
|
|
529
|
+
p,identity=_public_protocol(protocol_path,protocol_sha256)
|
|
530
|
+
resources=public_resource_availability(root)
|
|
531
|
+
if mode=='LIVE' and owner_record['runtime_manifest_sha256']!=p['runtime']['manifest_sha256']:
|
|
532
|
+
raise PermissionError('Owner grant binds another installed runtime')
|
|
533
|
+
remaining=min(PUBLIC_PROFILE['worker_seconds'],p['not_after']-epoch,
|
|
534
|
+
PUBLIC_PROFILE['global_seconds'] if owner_record is None else owner_record['expires_utc_epoch']-epoch)
|
|
535
|
+
policy={'schema':'product004-owned-public-policy-v1','mode':mode,'scenario':scenario,'protocol':{'path':str(protocol_path),'sha256':protocol_sha256},
|
|
536
|
+
'profile':PUBLIC_PROFILE,'runtime':identity,'resources_before_worker':resources,'created_ns':entry,'deadline_ns':entry+int(remaining*1e9),
|
|
537
|
+
'deadline_utc':min(epoch+PUBLIC_PROFILE['global_seconds'],p['not_after'],float('inf') if owner_record is None else owner_record['expires_utc_epoch']),'owner_sha256':expected_owner_sha256 if mode=='LIVE' else None,
|
|
538
|
+
'owner_text':None if mode=='PREFIT' else {'path':str(owner),'sha256':expected_owner_sha256,'bytes':owner.stat().st_size},
|
|
539
|
+
'claim':{'path':str(authority/'.native-authorities'/token/'CLAIM.json'),'sha256':sha(authority/'.native-authorities'/token/'CLAIM.json')}}
|
|
540
|
+
atomic_result(root,'PUBLIC-POLICY.json',policy,lambda:None);pin=sha(root/'PUBLIC-POLICY.json')
|
|
541
|
+
env=os.environ.copy();env.pop('PYTHONPATH',None);env.update(PYTHONUTF8='1',PYTHONDONTWRITEBYTECODE='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1')
|
|
542
|
+
with (root/'WORKER.log').open('x',encoding='utf8') as log:
|
|
543
|
+
child,guard=spawn_guarded([sys.executable,'-I','-B','-m','morphtree._experimental.worker','public-child',str(root),pin],cwd=root,stdin=subprocess.PIPE,stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=os.name!='nt',
|
|
544
|
+
memory_limits={k:PUBLIC_PROFILE[k] for k in ('process_commit_bytes','job_commit_bytes')})
|
|
545
|
+
while not (root/'PUBLIC-HELLO.json').exists():
|
|
546
|
+
if child.poll() is not None:raise RuntimeError('Public bootstrap failed')
|
|
547
|
+
if time.monotonic_ns()>=policy['deadline_ns']:raise TimeoutError('Bootstrap deadline')
|
|
548
|
+
time.sleep(.02)
|
|
549
|
+
hello=read(root/'PUBLIC-HELLO.json')
|
|
550
|
+
if hello['policy_sha256']!=pin or hello['pid']!=child.pid and hello['ppid']!=child.pid:raise ValueError('Unknown actual worker')
|
|
551
|
+
guard.include_pid(hello['pid']);statistics=guard.statistics()
|
|
552
|
+
child.stdin.write(b'1');child.stdin.close()
|
|
553
|
+
while child.poll() is None:
|
|
554
|
+
if (root/'STOP').exists() and not (root/'PUBLIC-RESULT.json').exists():raise RuntimeError('STOP')
|
|
555
|
+
if time.monotonic_ns()>=policy['deadline_ns'] or time.time()>=policy['deadline_utc']:raise TimeoutError('Global deadline')
|
|
556
|
+
if public_artifact_bytes(root)>PUBLIC_PROFILE['artifact_bytes']:raise RuntimeError('Checked artifact cap')
|
|
557
|
+
statistics=guard.statistics()
|
|
558
|
+
if statistics['peak_over_job_policy_bytes']:
|
|
559
|
+
raise RuntimeError('Observed private commit peak exceeded allocation policy;no absolute peak guarantee')
|
|
560
|
+
time.sleep(.05)
|
|
561
|
+
statistics=guard.statistics()
|
|
562
|
+
if child.returncode:raise RuntimeError('Public worker failed;no retry')
|
|
563
|
+
result=read(root/'PUBLIC-RESULT.json')
|
|
564
|
+
if result['status']!='COMPLETE' or result['mode']!=mode or len(result['cases'])!=3:
|
|
565
|
+
raise ValueError('Incomplete public result')
|
|
566
|
+
from types import SimpleNamespace
|
|
567
|
+
from . import component_cache_v2 as cache
|
|
568
|
+
rows=cache.MultitaskStore.events(SimpleNamespace(root=root/'global-store'))
|
|
569
|
+
starts=trees=asked=0
|
|
570
|
+
for row in rows:
|
|
571
|
+
if row['event']!='PUBLIC_SESSION_BOUND':continue
|
|
572
|
+
# Directory is present before bind and never cleaned by GC.
|
|
573
|
+
index=row['call_index'];directory=next(r['path'] for r in rows if r['event']=='PUBLIC_CALL_DIRECTORY_RETAINED' and r['call_index']==index)
|
|
574
|
+
folder=safe_path(root,directory)
|
|
575
|
+
if sha(folder/'SESSION.json')!=row['session_sha256']:raise ValueError('Changed retained session')
|
|
576
|
+
events=cache.MultitaskStore.events(SimpleNamespace(root=folder/'store'))
|
|
577
|
+
starts+=sum(e['event']=='NATIVE_FIT_STARTED' for e in events)
|
|
578
|
+
trees+=sum(e.get('scalar_trees',0) for e in events if e['event']=='NATIVE_FIT_STARTED')
|
|
579
|
+
asked+=sum(e['event']=='NUMERIC_TRIAL_ASKED' and e.get('real',False) for e in events)
|
|
580
|
+
if mode=='PREFIT' and (starts or trees or asked):raise ValueError('PREFIT performed learning')
|
|
581
|
+
if mode=='LIVE' and (not 0<starts<=PUBLIC_PROFILE['calls'] or not 0<trees<=PUBLIC_PROFILE['scalar_trees'] or not 0<=asked<=PUBLIC_PROFILE['trials']):
|
|
582
|
+
raise ValueError('Incomplete actual three-task learning/trials')
|
|
583
|
+
atomic_result(root,'PUBLIC-ACCOUNTING.json',{'native_fits_started':starts,'scalar_trees_started':trees,'actual_Optuna_trials_created':asked,'reservations':result['global_reservations'],'mode':mode},lambda:None)
|
|
584
|
+
except BaseException as exc:reason=type(exc).__name__+': '+str(exc)
|
|
585
|
+
finally:
|
|
586
|
+
if guard is not None:
|
|
587
|
+
try:statistics=guard.statistics()
|
|
588
|
+
except Exception as exc:
|
|
589
|
+
statistics={'status':'UNKNOWN','reason':type(exc).__name__+': '+str(exc)}
|
|
590
|
+
reason=reason or 'RESOURCE_ACCOUNTING_UNKNOWN'
|
|
591
|
+
finally:guard.close()
|
|
592
|
+
elif child is not None and child.poll() is None:child.kill();child.wait()
|
|
593
|
+
configure(root)
|
|
594
|
+
for call in (root/'calls').glob('morphtree-user-*') if (root/'calls').exists() else []:
|
|
595
|
+
call=safe_path(root,call)
|
|
596
|
+
if call.is_dir() and not (call/'STOP').exists():(call/'STOP').write_text('Owned campaign terminated;no resumed learning',encoding='utf8')
|
|
597
|
+
closure={'schema':'product007-public-closure-v1','status':'PASS_PREFIT' if reason is None and mode=='PREFIT' else 'COMPLETED_LIVE' if reason is None else 'INCOMPLETE',
|
|
598
|
+
'mode':mode,'reason':reason,'returncode':None if child is None else child.returncode,'owned_guard_closed':guard is not None,'resource_observation':statistics,
|
|
599
|
+
'authority_residual':0,'retry':False,'elapsed_operational_seconds':(time.monotonic_ns()-entry)/1e9,'worker':hello}
|
|
600
|
+
atomic_result(root,'PUBLIC-EXIT.json',closure,lambda:None)
|
|
601
|
+
if not (root/'STOP').exists():(root/'STOP').write_text('terminal campaign;no authority remains',encoding='utf8')
|
|
602
|
+
# The canonical claim already makes interruption nonrenewable.Closure
|
|
603
|
+
# copies are additional evidence;they never reset/replace that claim.
|
|
604
|
+
configure(authority)
|
|
605
|
+
atomic_result(authority/'.native-authorities'/token,'CLOSED.json',closure,lambda:None)
|
|
606
|
+
configure(root)
|
|
607
|
+
return closure
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _public_worker(root,pin):
|
|
611
|
+
from contextlib import ExitStack
|
|
612
|
+
from types import SimpleNamespace
|
|
613
|
+
from unittest.mock import patch
|
|
614
|
+
from morphtree import MorphTree,api
|
|
615
|
+
from morphtree.booster import BinaryHistogramBooster
|
|
616
|
+
from morphtree.tree import HistogramTree
|
|
617
|
+
from . import execution_v2 as E
|
|
618
|
+
from .postfit import finish
|
|
619
|
+
root=Path(root).resolve();configure(root);policy=read(root/'PUBLIC-POLICY.json',expected_sha256=pin)
|
|
620
|
+
atomic_result(root,'PUBLIC-HELLO.json',{'pid':os.getpid(),'ppid':os.getppid(),'policy_sha256':pin},lambda:None)
|
|
621
|
+
if sys.stdin.read(1)!='1':raise PermissionError('No owned parent GO')
|
|
622
|
+
p,identity=_public_protocol(policy['protocol']['path'],policy['protocol']['sha256'])
|
|
623
|
+
if identity!=policy['runtime'] or policy['profile']!=PUBLIC_PROFILE or policy['mode'] not in ('PREFIT','LIVE'):
|
|
624
|
+
raise ValueError('Different public policy/runtime')
|
|
625
|
+
claim=read(policy['claim']['path'],expected_sha256=policy['claim']['sha256'])
|
|
626
|
+
if claim['mode']!=policy['mode'] or claim['protocol_sha256']!=policy['protocol']['sha256'] or claim['output_root']!=str(root) or claim['owner_sha256']!=policy['owner_sha256']:
|
|
627
|
+
raise PermissionError('Actual consumed claim does not match')
|
|
628
|
+
if policy['mode']=='LIVE':
|
|
629
|
+
if not policy['owner_sha256'] or not policy.get('owner_text') or claim['campaign']!=PUBLIC_CAMPAIGN:
|
|
630
|
+
raise PermissionError('Actual owner binding missing')
|
|
631
|
+
owner=policy['owner_text'];path=safe_path(Path(policy['protocol']['path']).parent,owner['path'])
|
|
632
|
+
if path.stat().st_size!=owner['bytes'] or sha(path)!=owner['sha256'] or owner['sha256']!=policy['owner_sha256']:
|
|
633
|
+
raise PermissionError('Actual owner text changed')
|
|
634
|
+
permission=read(path,expected_sha256=owner['sha256'])
|
|
635
|
+
if permission['authority']!='EXPLICIT_HUMAN_MESSAGE' or permission['campaign']!=PUBLIC_CAMPAIGN or permission['protocol_sha256']!=policy['protocol']['sha256'] or permission['runtime_manifest_sha256']!=p['runtime']['manifest_sha256'] or permission['profile']!=PUBLIC_PROFILE or not permission['created_utc_epoch']<=time.time()<permission['expires_utc_epoch']:
|
|
636
|
+
raise PermissionError('Actual owner scope expired or differs')
|
|
637
|
+
policy['policy_file_sha256']=pin;scope=PublicCallBoundary(root,policy,p);results=[];models=[]
|
|
638
|
+
token=E.PUBLIC_SCOPE.set(scope)
|
|
639
|
+
try:
|
|
640
|
+
with ExitStack() as stack:
|
|
641
|
+
if policy['mode']=='PREFIT':
|
|
642
|
+
import optuna
|
|
643
|
+
def stop(session,model,X,y,**kwargs):
|
|
644
|
+
session._user_integrity();scope.event('PUBLIC_PREFIT_BOUNDARY',task=session.prepared['task'])
|
|
645
|
+
if policy['scenario']=='crash':os._exit(17)
|
|
646
|
+
if policy['scenario']=='wait':
|
|
647
|
+
while True:scope.check();time.sleep(.02)
|
|
648
|
+
raise E.PreFitBoundary('STOP_BEFORE_ORIGINAL_NATIVE_FIT')
|
|
649
|
+
stack.enter_context(patch.object(E.Session,'execute_native',stop))
|
|
650
|
+
stack.enter_context(patch.object(BinaryHistogramBooster,'fit',side_effect=AssertionError('NO_NATIVE_FIT_IN_PREFIT')))
|
|
651
|
+
stack.enter_context(patch.object(HistogramTree,'fit',side_effect=AssertionError('NO_TREE_FIT_IN_PREFIT')))
|
|
652
|
+
stack.enter_context(patch.object(optuna,'create_study',side_effect=AssertionError('NO_STUDY_IN_PREFIT')))
|
|
653
|
+
for task in ('binary','multiclass','regression'):
|
|
654
|
+
scope.check();source=p['sources'][task]
|
|
655
|
+
raw=bound_reference(Path(policy['protocol']['path']).parent,source['data'])
|
|
656
|
+
model=MorphTree(random_state=42,verbosity=0);models.append(model)
|
|
657
|
+
try:model.fit(raw['X'],raw['y'])
|
|
658
|
+
except E.PreFitBoundary:
|
|
659
|
+
if policy['mode']!='PREFIT':raise
|
|
660
|
+
session=scope.sessions[-1]
|
|
661
|
+
output=safe_path(root,'result-'+task);output.mkdir()
|
|
662
|
+
if policy['mode']=='PREFIT':
|
|
663
|
+
import shutil
|
|
664
|
+
f=p['prefit_models'][task];src=safe_path(Path(policy['protocol']['path']).parent,f['path']);dest=output/'RETAINED-INPUT.json'
|
|
665
|
+
shutil.copyfile(src,dest);model=MorphTree.load(dest,sha256=f['sha256'],workdir=root)
|
|
666
|
+
else:raw={**raw,'names':None} # ordinary fit used plain lists
|
|
667
|
+
configure(root)
|
|
668
|
+
io=finish(SimpleNamespace(check=scope.check,engine=session.engine,source=session.source,root=root,mode='PUBLIC_POSTFIT_IO',counters=session.counters),model,output,raw)
|
|
669
|
+
scope.end_call(status='PREFIT_AND_RETAINED_IO' if policy['mode']=='PREFIT' else 'LIVE_AND_POSTFIT_IO')
|
|
670
|
+
results.append({'task':task,'training':'NOT_EXECUTED' if policy['mode']=='PREFIT' else 'ACTUAL_PUBLIC_API','counts':session.counters(),'postfit':io})
|
|
671
|
+
configure(root)
|
|
672
|
+
atomic_result(root,'PUBLIC-RESULT.json',{'status':'COMPLETE','mode':policy['mode'],'cases':results,'global_reservations':scope.used(),'functional_complete':False,'scientific_adaptive_benefit':'NOT_EVALUATED'},scope.check)
|
|
673
|
+
except BaseException as exc:
|
|
674
|
+
scope.event('PUBLIC_WORKER_ERROR',error=type(exc).__name__,message=str(exc));raise
|
|
675
|
+
finally:
|
|
676
|
+
try:scope.close('terminal public qualification')
|
|
677
|
+
finally:E.PUBLIC_SCOPE.reset(token)
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
if __name__=='__main__':
|
|
681
|
+
if sys.argv[1]=='public-child':_public_worker(sys.argv[2],sys.argv[3])
|
|
682
|
+
else:_worker(sys.argv[1],sys.argv[2])
|