morphtree 2.1.0__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- morphtree/__init__.py +28 -0
- morphtree/__main__.py +3 -0
- morphtree/_experimental/__init__.py +1 -0
- morphtree/_experimental/admission.py +364 -0
- morphtree/_experimental/base.py +215 -0
- morphtree/_experimental/component_cache_v2.py +263 -0
- morphtree/_experimental/evolution_report.py +307 -0
- morphtree/_experimental/execution_v2.py +697 -0
- morphtree/_experimental/facade.py +289 -0
- morphtree/_experimental/integrated.py +748 -0
- morphtree/_experimental/learning.py +418 -0
- morphtree/_experimental/manifest.json +204 -0
- morphtree/_experimental/numeric_fixture.py +21 -0
- morphtree/_experimental/parameter_contract.py +38 -0
- morphtree/_experimental/postfit.py +51 -0
- morphtree/_experimental/preprocessing.py +133 -0
- morphtree/_experimental/processes.py +191 -0
- morphtree/_experimental/producer.py +542 -0
- morphtree/_experimental/product.py +156 -0
- morphtree/_experimental/product_cli.py +43 -0
- morphtree/_experimental/roundtrip_v3.py +311 -0
- morphtree/_experimental/runtime_io.py +65 -0
- morphtree/_experimental/sampling.py +26 -0
- morphtree/_experimental/support.py +281 -0
- morphtree/_experimental/target.py +59 -0
- morphtree/_experimental/training.py +53 -0
- morphtree/_experimental/worker.py +682 -0
- morphtree/_installed.py +17 -0
- morphtree/_native/__init__.py +1 -0
- morphtree/_native/bin/win_amd64/morphtree_kernels.dll +0 -0
- morphtree/_native/buffers.py +82 -0
- morphtree/_native/loader.py +39 -0
- morphtree/_native/manifest.json +29 -0
- morphtree/_native/runtime.py +48 -0
- morphtree/_native/src/histogram.c +61 -0
- morphtree/_native/src/traversal.c +51 -0
- morphtree/api.py +388 -0
- morphtree/api_compat.py +65 -0
- morphtree/backend.py +162 -0
- morphtree/booster.py +252 -0
- morphtree/cli.py +52 -0
- morphtree/config.py +262 -0
- morphtree/control.py +20 -0
- morphtree/data.py +300 -0
- morphtree/diagnostics.py +158 -0
- morphtree/estimators.py +156 -0
- morphtree/examples.py +25 -0
- morphtree/exporting.py +224 -0
- morphtree/legacy_artifacts.py +32 -0
- morphtree/metrics.py +156 -0
- morphtree/model.py +1010 -0
- morphtree/mutations.py +205 -0
- morphtree/native_lab_backend.py +79 -0
- morphtree/numpy_backend.py +180 -0
- morphtree/optuna_tuner.py +154 -0
- morphtree/persistence.py +414 -0
- morphtree/prediction_output.py +35 -0
- morphtree/prepared_bins.py +82 -0
- morphtree/public.py +106 -0
- morphtree/resource_plan.py +170 -0
- morphtree/scalable.py +781 -0
- morphtree/scalable_backend.py +74 -0
- morphtree/scalable_checkpoint.py +129 -0
- morphtree/scalable_data.py +255 -0
- morphtree/scalable_receipts.py +92 -0
- morphtree/scalable_target.py +68 -0
- morphtree/scalable_variants.py +133 -0
- morphtree/sklearn_view.py +79 -0
- morphtree/source_markers.py +25 -0
- morphtree/species.py +47 -0
- morphtree/temporary.py +65 -0
- morphtree/training_fragment.py +130 -0
- morphtree/tree.py +231 -0
- morphtree/tuning.py +150 -0
- morphtree-2.1.0.dist-info/METADATA +212 -0
- morphtree-2.1.0.dist-info/RECORD +79 -0
- morphtree-2.1.0.dist-info/WHEEL +4 -0
- morphtree-2.1.0.dist-info/entry_points.txt +2 -0
- morphtree-2.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Explicit software driver, never an Optuna study/trial or a fit."""
|
|
2
|
+
from types import SimpleNamespace
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def module(fault=None):
|
|
6
|
+
class Trial:
|
|
7
|
+
def __init__(self,number):self.number=number
|
|
8
|
+
def suggest_int(self,name,low,high):return low+1+self.number
|
|
9
|
+
def suggest_float(self,name,low,high,log=False):return low if self.number==0 else high
|
|
10
|
+
class Study:
|
|
11
|
+
def __init__(self):self.n=0
|
|
12
|
+
def ask(self):
|
|
13
|
+
if fault=='ask':raise RuntimeError('SIMULATED_ASK_FAILURE')
|
|
14
|
+
trial=Trial(self.n);self.n+=1;return trial
|
|
15
|
+
def tell(self,*a,**k):
|
|
16
|
+
if fault=='tell':raise RuntimeError('SIMULATED_TELL_FAILURE')
|
|
17
|
+
class Sampler:
|
|
18
|
+
def __init__(self,**kwargs):self.parameters=kwargs
|
|
19
|
+
return SimpleNamespace(samplers=SimpleNamespace(TPESampler=Sampler),
|
|
20
|
+
trial=SimpleNamespace(TrialState=SimpleNamespace(FAIL='SIMULATED_FAIL')),
|
|
21
|
+
create_study=lambda **kwargs:Study())
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Research-only numeric-space admission, reusing actual native activity rules.
|
|
2
|
+
|
|
3
|
+
Potentially active is not guaranteed sensitive on every dataset. Frozen topology,
|
|
4
|
+
binning and learning-budget dimensions are not final numeric opportunities here.
|
|
5
|
+
No historical tuning space or production API is modified.
|
|
6
|
+
"""
|
|
7
|
+
from dataclasses import asdict,fields
|
|
8
|
+
from morphtree.config import AlgorithmGenome,HyperParams
|
|
9
|
+
|
|
10
|
+
FROZEN={'max_depth','max_leaves','max_bins','n_estimators'}
|
|
11
|
+
|
|
12
|
+
def schema(family,task,genome):
|
|
13
|
+
if family not in ('boost','bag') or task not in ('binary','multiclass','regression') or not isinstance(genome,AlgorithmGenome):raise ValueError('Explicit native family/task/genome')
|
|
14
|
+
if task!='binary' and genome!=AlgorithmGenome():raise ValueError('Current native nonbinary API supports fixed Base genome only')
|
|
15
|
+
active=set(genome.active_parameters());known={f.name for f in fields(HyperParams)};frozen=set(FROZEN)
|
|
16
|
+
if family=='bag':
|
|
17
|
+
frozen.add('learning_rate')
|
|
18
|
+
# Reset members have one stage: decay multiplies iteration0, hence zero.
|
|
19
|
+
active.discard('update_decay')
|
|
20
|
+
return {name:{'status':'INACTIVE' if name not in active else 'FROZEN_FORM_OR_BUDGET' if name in frozen else 'POTENTIALLY_ACTIVE','condition':'AlgorithmGenome.active_parameters + fixed-form/budget boundary','data_dependent':True} for name in sorted(known)}
|
|
21
|
+
|
|
22
|
+
def admit_space(space,*,family,task,genome,drop_inactive=False):
|
|
23
|
+
if not isinstance(space,dict) or not space or type(drop_inactive) is not bool:raise ValueError('Nonempty explicit future space')
|
|
24
|
+
rules=schema(family,task,genome);accepted={};excluded={}
|
|
25
|
+
for name,spec in space.items():
|
|
26
|
+
if name not in rules:raise ValueError('Unknown parameter: '+str(name))
|
|
27
|
+
status=rules[name]['status']
|
|
28
|
+
if status=='FROZEN_FORM_OR_BUDGET':raise ValueError('Structural/budget dimension is frozen: '+name)
|
|
29
|
+
if status=='INACTIVE':
|
|
30
|
+
if not drop_inactive:raise ValueError('Inactive under actual genome: '+name)
|
|
31
|
+
excluded[name]='INACTIVE_UNDER_FROZEN_GENOME';continue
|
|
32
|
+
if type(spec) not in (list,tuple) or len(spec)!=2:raise ValueError('Bound pair required')
|
|
33
|
+
lo,hi=spec
|
|
34
|
+
if type(lo) not in (int,float) or type(hi) not in (int,float) or not lo<hi:raise ValueError('Increasing finite native parameter bounds')
|
|
35
|
+
for v in (lo,hi):HyperParams(**(asdict(HyperParams())|{name:v}))
|
|
36
|
+
accepted[name]=list(spec)
|
|
37
|
+
if not accepted:raise ValueError('No active numeric opportunity remains')
|
|
38
|
+
return {'space':accepted,'excluded':excluded,'mode':'RESEARCH_ONLY_NOT_INSTALLED_OR_HISTORICAL_SPACE_CHANGE'}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Existing safe-loader/V3/standalone sequence for a returned installed model.
|
|
2
|
+
|
|
3
|
+
No training or selector here; actual pre-save observations are persisted first.
|
|
4
|
+
"""
|
|
5
|
+
import json
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from .admission import digest,sha
|
|
9
|
+
from .runtime_io import artifact_path,reference
|
|
10
|
+
from .base import Base
|
|
11
|
+
from .target import _NamedRows
|
|
12
|
+
from . import roundtrip_v3 as v3
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def finish(session,model,root,raw):
|
|
16
|
+
session.check();query={'rows':raw['predict_X'],'names':raw['names']}
|
|
17
|
+
X=_NamedRows(query['rows'],query['names']);obj=model._variant('selected')
|
|
18
|
+
def behavior(m):
|
|
19
|
+
value={'prediction':m.predict(X)}
|
|
20
|
+
if m.task_!='regression':value['probabilities']=m.predict_proba(X)
|
|
21
|
+
return value
|
|
22
|
+
before=behavior(obj);t0=time.perf_counter_ns();source_sha=digest(session.source)
|
|
23
|
+
pre=root/'PRE-SAVE-WITNESS.json'
|
|
24
|
+
pre.write_text(json.dumps({'before':before,'query':query,'source_sha256':source_sha,'captured_before_save':t0},sort_keys=True)+'\n',encoding='utf8')
|
|
25
|
+
p=root/'NATIVE-BEFORE.json';obj.save(p);t1=time.perf_counter_ns()
|
|
26
|
+
loaded=Base.load(p);t2=time.perf_counter_ns();after=behavior(loaded);t3=time.perf_counter_ns()
|
|
27
|
+
q=root/'NATIVE-AFTER.json';loaded.save(q)
|
|
28
|
+
a,b=p.read_bytes(),q.read_bytes();sa,sb=json.loads(a),json.loads(b)
|
|
29
|
+
runtime=session.engine.identity;recipe=digest(obj.best_recipe_);loader=sha(session.engine.package/'model.py')
|
|
30
|
+
witness={'schema':'morph032-actual-behavior-v1','runtime':runtime,'source_sha256':source_sha,'recipe_sha256':recipe,
|
|
31
|
+
'query':query,'loader_sha256':loader,'before_state_sha256':digest(sa),'after_state_sha256':digest(sb),
|
|
32
|
+
'before':before,'after':after,'events':dict(zip(('captured_before_save','saved','safe_loaded','captured_after_load'),(t0,t1,t2,t3)))}
|
|
33
|
+
trust={'runtime':runtime,'actual_runtime':runtime,'before_sha256':sha(p),'after_sha256':sha(q),'witness_sha256':digest(witness),
|
|
34
|
+
'query_sha256':digest(query),'source_sha256':source_sha,'recipe_sha256':recipe,'loader_sha256':loader,
|
|
35
|
+
'admission':('QUALIFIED_RETAINED_ZERO_FIT' if session.mode=='PUBLIC_POSTFIT_IO' else
|
|
36
|
+
'QUALIFIED_COMP003_LIVE_COMPONENTS' if session.mode=='LIVE' else 'QUALIFIED_COMP003_SOFTWARE')}
|
|
37
|
+
comparison=v3.compare(a,b,trust=trust,witness=witness)
|
|
38
|
+
(root/'V3.json').write_text(json.dumps({'result':comparison,'witness':witness,'trust':trust},sort_keys=True,indent=2)+'\n',encoding='utf8')
|
|
39
|
+
if comparison['status']!='PASS':raise ValueError('V3 failed: '+str(comparison))
|
|
40
|
+
path=root/'MODEL.json';model.save(path)
|
|
41
|
+
from .integrated import MorphTree
|
|
42
|
+
restored=MorphTree.load(path,sha256=sha(path),workdir=session.root)
|
|
43
|
+
if behavior(restored)!=before or restored.recipe()!=model.recipe():raise ValueError('Actual final wrapper/recipe changed')
|
|
44
|
+
program=root/'PREDICTOR.py';restored.export_python(program)
|
|
45
|
+
from .worker import standalone
|
|
46
|
+
value={'program':{'path':'PREDICTOR.py','sha256':sha(program),'bytes':program.stat().st_size},'expected':before,'query':query}
|
|
47
|
+
result=standalone(root,value,session.check)
|
|
48
|
+
session.check()
|
|
49
|
+
return {'model':reference(path),'program':reference(program),'expected':before,'query':query,
|
|
50
|
+
'V3':reference(root/'V3.json'),'standalone':result,'task':model.task_,'family':model.best_family_,
|
|
51
|
+
'scope':session.mode,'report':model.report(),'counts':session.counters()}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""Bounded TRAIN-local schema shared by the public runtime and standalone export.
|
|
2
|
+
|
|
3
|
+
String categoricals use sorted one-hot columns plus separate missing/unseen
|
|
4
|
+
channels. Numeric missing values stay None for the native missing split rule.
|
|
5
|
+
No optional dependency is imported. pandas.NA is recognized only when pandas is
|
|
6
|
+
already loaded by the caller. Each validation fold needs its own instance.
|
|
7
|
+
"""
|
|
8
|
+
import copy
|
|
9
|
+
import math
|
|
10
|
+
import sys
|
|
11
|
+
from numbers import Real
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _m26_missing(v):
|
|
15
|
+
if v is None:
|
|
16
|
+
return True
|
|
17
|
+
if isinstance(v, Real):
|
|
18
|
+
return math.isnan(float(v))
|
|
19
|
+
# Never evaluate NA's truth value or import pandas in a standalone runtime.
|
|
20
|
+
pandas = sys.modules.get('pandas')
|
|
21
|
+
return pandas is not None and v is getattr(pandas, 'NA', None)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _m26_scalar(v):
|
|
25
|
+
if _m26_missing(v):
|
|
26
|
+
return None
|
|
27
|
+
if isinstance(v, str):
|
|
28
|
+
if len(v) > 1024:
|
|
29
|
+
raise ValueError('Category string exceeds 1024 characters')
|
|
30
|
+
return v
|
|
31
|
+
if isinstance(v, Real):
|
|
32
|
+
z = float(v)
|
|
33
|
+
if math.isfinite(z):
|
|
34
|
+
return z
|
|
35
|
+
raise ValueError('Expected finite numeric, string category, None, numeric NaN or pandas.NA; infinity/objects are unsupported')
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _m26_rows(X):
|
|
39
|
+
names = list(X.columns) if hasattr(X, 'columns') else None
|
|
40
|
+
if hasattr(X, 'to_numpy'):
|
|
41
|
+
X = X.to_numpy()
|
|
42
|
+
if hasattr(X, 'tolist'):
|
|
43
|
+
X = X.tolist()
|
|
44
|
+
if not isinstance(X, (list, tuple)) or not 1 <= len(X) <= 100000:
|
|
45
|
+
raise ValueError('X must be a nonempty dense 2D table, at most 100000 rows')
|
|
46
|
+
if not isinstance(X[0], (list, tuple)) or not 1 <= len(X[0]) <= 4096:
|
|
47
|
+
raise ValueError('X must have 1..4096 columns')
|
|
48
|
+
n = len(X[0])
|
|
49
|
+
if len(X)*n > 2000000 or any(not isinstance(r, (list, tuple)) or len(r) != n for r in X):
|
|
50
|
+
raise ValueError('Rectangular X required, at most 2000000 input cells')
|
|
51
|
+
if names is not None and (len(names) != n or any(not isinstance(v, str) or not v or len(v)>1024 for v in names) or len(set(names)) != n):
|
|
52
|
+
raise ValueError('Unique nonempty string column names required')
|
|
53
|
+
return [[_m26_scalar(v) for v in r] for r in X], names
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class TrainEncoder:
|
|
57
|
+
"""Immutable-by-copy serializable schema; fit accepts TRAIN rows only."""
|
|
58
|
+
def fit(self, X):
|
|
59
|
+
rows, names = _m26_rows(X)
|
|
60
|
+
columns = []
|
|
61
|
+
for col in zip(*rows):
|
|
62
|
+
present = [v for v in col if v is not None]
|
|
63
|
+
if any(isinstance(v, str) for v in present):
|
|
64
|
+
if any(not isinstance(v, str) for v in present):
|
|
65
|
+
raise ValueError('Mixed numeric/string column; supply a consistent type')
|
|
66
|
+
vocab = sorted(set(present))
|
|
67
|
+
if len(vocab) > 64:
|
|
68
|
+
raise ValueError('Experimental categorical vocabulary cap is 64 per TRAIN column')
|
|
69
|
+
columns.append({'kind': 'categorical', 'vocabulary': vocab})
|
|
70
|
+
else:
|
|
71
|
+
columns.append({'kind': 'numeric'})
|
|
72
|
+
state = {'schema': 'morph026-train-encoder-v1', 'names': names, 'columns': columns}
|
|
73
|
+
checked = self.from_dict(state) # Validate before replacing a fitted schema.
|
|
74
|
+
self.state_, self.width_ = checked.state_, checked.width_
|
|
75
|
+
return self
|
|
76
|
+
|
|
77
|
+
def to_dict(self):
|
|
78
|
+
if not hasattr(self, 'state_'):
|
|
79
|
+
raise RuntimeError('Encoder is not fitted')
|
|
80
|
+
return copy.deepcopy(self.state_)
|
|
81
|
+
|
|
82
|
+
@classmethod
|
|
83
|
+
def from_dict(cls, state):
|
|
84
|
+
if type(state) is not dict or set(state) != {'schema', 'names', 'columns'} or state['schema'] != 'morph026-train-encoder-v1':
|
|
85
|
+
raise ValueError('Exact encoder schema required')
|
|
86
|
+
cols, names = state['columns'], state['names']
|
|
87
|
+
if type(cols) is not list or not 1 <= len(cols) <= 4096:
|
|
88
|
+
raise ValueError('Bounded column schema required')
|
|
89
|
+
if names is not None and (type(names) is not list or len(names)!=len(cols) or any(not isinstance(n,str) or not n or len(n)>1024 for n in names) or len(set(names))!=len(names)):
|
|
90
|
+
raise ValueError('Invalid column names')
|
|
91
|
+
width = 0
|
|
92
|
+
for c in cols:
|
|
93
|
+
if c == {'kind': 'numeric'}:
|
|
94
|
+
width += 1
|
|
95
|
+
elif type(c) is dict and set(c)=={'kind','vocabulary'} and c['kind']=='categorical':
|
|
96
|
+
v = c['vocabulary']
|
|
97
|
+
if type(v) is not list or not 1<=len(v)<=64 or any(not isinstance(s,str) or len(s)>1024 for s in v) or v!=sorted(set(v)):
|
|
98
|
+
raise ValueError('Bounded canonical TRAIN vocabulary required')
|
|
99
|
+
width += len(v)+2
|
|
100
|
+
else:
|
|
101
|
+
raise ValueError('Unknown encoder column rule')
|
|
102
|
+
if width>4096:
|
|
103
|
+
raise ValueError('Encoded feature cap is 4096')
|
|
104
|
+
obj=cls(); obj.state_=copy.deepcopy(state); obj.width_=width
|
|
105
|
+
return obj
|
|
106
|
+
|
|
107
|
+
def transform(self, X):
|
|
108
|
+
s=self.to_dict(); rows,names=_m26_rows(X)
|
|
109
|
+
if len(rows[0])!=len(s['columns']):
|
|
110
|
+
raise ValueError('Missing or extra input columns')
|
|
111
|
+
if s['names'] is not None:
|
|
112
|
+
if names is None or set(names)!=set(s['names']):
|
|
113
|
+
raise ValueError('Named training schema requires the same named columns')
|
|
114
|
+
order=[names.index(n) for n in s['names']]
|
|
115
|
+
rows=[[r[i] for i in order] for r in rows]
|
|
116
|
+
elif names is not None:
|
|
117
|
+
raise ValueError('Training used positional columns; named prediction input is ambiguous')
|
|
118
|
+
result=[]
|
|
119
|
+
for row in rows:
|
|
120
|
+
out=[]
|
|
121
|
+
for value,c in zip(row,s['columns']):
|
|
122
|
+
if c['kind']=='numeric':
|
|
123
|
+
if isinstance(value,str):
|
|
124
|
+
raise ValueError('String supplied to numeric TRAIN column')
|
|
125
|
+
out.append(value)
|
|
126
|
+
else:
|
|
127
|
+
if value is not None and not isinstance(value,str):
|
|
128
|
+
raise ValueError('Numeric supplied to string categorical column')
|
|
129
|
+
vocab=c['vocabulary']; z=[0.0]*(len(vocab)+2)
|
|
130
|
+
i=len(vocab) if value is None else vocab.index(value) if value in vocab else len(vocab)+1
|
|
131
|
+
z[i]=1.0;out.extend(z)
|
|
132
|
+
result.append(out)
|
|
133
|
+
return result
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import os,stat,time,signal,subprocess
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from contextlib import contextmanager
|
|
4
|
+
from .runtime_io import artifact_path
|
|
5
|
+
class ContinuityError(ValueError):
|
|
6
|
+
"""Invalid plan, changed evidence, or an unsafe recovery request."""
|
|
7
|
+
|
|
8
|
+
class LockBusy(ContinuityError):
|
|
9
|
+
"""Another live supervisor owns this state directory."""
|
|
10
|
+
|
|
11
|
+
def _safe(path):
|
|
12
|
+
path = Path(os.path.abspath(path))
|
|
13
|
+
for item in reversed((path, *path.parents)):
|
|
14
|
+
try:
|
|
15
|
+
info = item.lstat()
|
|
16
|
+
except FileNotFoundError:
|
|
17
|
+
continue
|
|
18
|
+
if stat.S_ISLNK(info.st_mode) or getattr(info, 'st_file_attributes', 0) & 1024:
|
|
19
|
+
raise ContinuityError('Symlinks and reparse points are forbidden')
|
|
20
|
+
return path
|
|
21
|
+
|
|
22
|
+
def _artifact(path):
|
|
23
|
+
try:
|
|
24
|
+
return artifact_path(path)
|
|
25
|
+
except ValueError as exc:
|
|
26
|
+
raise ContinuityError(str(exc)) from exc
|
|
27
|
+
|
|
28
|
+
@contextmanager
|
|
29
|
+
def exclusive_lock(state_dir):
|
|
30
|
+
"""Advisory kernel lock; closure or process death releases ownership."""
|
|
31
|
+
root = _artifact(state_dir)
|
|
32
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
path = _safe(root / '.lock')
|
|
34
|
+
with path.open('a+b') as stream:
|
|
35
|
+
if path.stat().st_size == 0:
|
|
36
|
+
stream.write(b'\x00')
|
|
37
|
+
stream.flush()
|
|
38
|
+
stream.seek(0)
|
|
39
|
+
try:
|
|
40
|
+
if os.name == 'nt':
|
|
41
|
+
import msvcrt
|
|
42
|
+
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
|
43
|
+
else:
|
|
44
|
+
import fcntl
|
|
45
|
+
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
46
|
+
except OSError as exc:
|
|
47
|
+
raise LockBusy('Another live supervisor owns the state directory') from exc
|
|
48
|
+
try:
|
|
49
|
+
yield root
|
|
50
|
+
finally:
|
|
51
|
+
stream.seek(0)
|
|
52
|
+
if os.name == 'nt':
|
|
53
|
+
import msvcrt
|
|
54
|
+
msvcrt.locking(stream.fileno(), msvcrt.LK_UNLCK, 1)
|
|
55
|
+
else:
|
|
56
|
+
import fcntl
|
|
57
|
+
fcntl.flock(stream.fileno(), fcntl.LOCK_UN)
|
|
58
|
+
|
|
59
|
+
class ChildGuard:
|
|
60
|
+
"""Windows job kills descendants on parent death; POSIX uses a process group."""
|
|
61
|
+
|
|
62
|
+
def __init__(self, child, *, memory_limits=None):
|
|
63
|
+
self.child = child
|
|
64
|
+
self.job = None
|
|
65
|
+
self._owned_pids = {child.pid}
|
|
66
|
+
self._working_set_peaks = {}
|
|
67
|
+
self.memory_limits = memory_limits
|
|
68
|
+
if memory_limits is not None:
|
|
69
|
+
if os.name != 'nt' or type(memory_limits) is not dict or set(memory_limits) != {'process_commit_bytes', 'job_commit_bytes'} or any((type(v) is not int or v < 32 * 1024 ** 2 for v in memory_limits.values())) or (memory_limits['process_commit_bytes'] > memory_limits['job_commit_bytes']):
|
|
70
|
+
if child.poll() is None:
|
|
71
|
+
child.kill()
|
|
72
|
+
child.wait()
|
|
73
|
+
raise ContinuityError('Windows private-commit memory profile unavailable/invalid; not an RSS limit')
|
|
74
|
+
if os.name == 'nt':
|
|
75
|
+
import ctypes
|
|
76
|
+
from ctypes import wintypes
|
|
77
|
+
|
|
78
|
+
class Basic(ctypes.Structure):
|
|
79
|
+
_fields_ = [('PerProcessUserTimeLimit', ctypes.c_int64), ('PerJobUserTimeLimit', ctypes.c_int64), ('LimitFlags', wintypes.DWORD), ('MinimumWorkingSetSize', ctypes.c_size_t), ('MaximumWorkingSetSize', ctypes.c_size_t), ('ActiveProcessLimit', wintypes.DWORD), ('Affinity', ctypes.c_size_t), ('PriorityClass', wintypes.DWORD), ('SchedulingClass', wintypes.DWORD)]
|
|
80
|
+
|
|
81
|
+
class IO(ctypes.Structure):
|
|
82
|
+
_fields_ = [(name, ctypes.c_uint64) for name in ('ReadOperationCount', 'WriteOperationCount', 'OtherOperationCount', 'ReadTransferCount', 'WriteTransferCount', 'OtherTransferCount')]
|
|
83
|
+
|
|
84
|
+
class Extended(ctypes.Structure):
|
|
85
|
+
_fields_ = [('BasicLimitInformation', Basic), ('IoInfo', IO), ('ProcessMemoryLimit', ctypes.c_size_t), ('JobMemoryLimit', ctypes.c_size_t), ('PeakProcessMemoryUsed', ctypes.c_size_t), ('PeakJobMemoryUsed', ctypes.c_size_t)]
|
|
86
|
+
|
|
87
|
+
class Accounting(ctypes.Structure):
|
|
88
|
+
_fields_ = [(name, ctypes.c_int64) for name in ('TotalUserTime', 'TotalKernelTime', 'ThisPeriodTotalUserTime', 'ThisPeriodTotalKernelTime')] + [(name, wintypes.DWORD) for name in ('TotalPageFaultCount', 'TotalProcesses', 'ActiveProcesses', 'TotalTerminatedProcesses')]
|
|
89
|
+
kernel = ctypes.WinDLL('kernel32', use_last_error=True)
|
|
90
|
+
kernel.CreateJobObjectW.argtypes = [ctypes.c_void_p, wintypes.LPCWSTR]
|
|
91
|
+
kernel.CreateJobObjectW.restype = wintypes.HANDLE
|
|
92
|
+
kernel.SetInformationJobObject.argtypes = [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD]
|
|
93
|
+
kernel.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE]
|
|
94
|
+
kernel.IsProcessInJob.argtypes = [wintypes.HANDLE, wintypes.HANDLE, ctypes.POINTER(wintypes.BOOL)]
|
|
95
|
+
kernel.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
|
|
96
|
+
kernel.OpenProcess.restype = wintypes.HANDLE
|
|
97
|
+
kernel.TerminateJobObject.argtypes = [wintypes.HANDLE, wintypes.UINT]
|
|
98
|
+
kernel.QueryInformationJobObject.argtypes = [wintypes.HANDLE, ctypes.c_int, ctypes.c_void_p, wintypes.DWORD, ctypes.c_void_p]
|
|
99
|
+
kernel.CloseHandle.argtypes = [wintypes.HANDLE]
|
|
100
|
+
self.kernel = kernel
|
|
101
|
+
self.accounting = Accounting
|
|
102
|
+
self.extended = Extended
|
|
103
|
+
self.job = kernel.CreateJobObjectW(None, None)
|
|
104
|
+
limit = Extended()
|
|
105
|
+
limit.BasicLimitInformation.LimitFlags = 8192
|
|
106
|
+
if memory_limits is not None:
|
|
107
|
+
limit.BasicLimitInformation.LimitFlags |= 256 | 512
|
|
108
|
+
limit.ProcessMemoryLimit = memory_limits['process_commit_bytes']
|
|
109
|
+
limit.JobMemoryLimit = memory_limits['job_commit_bytes']
|
|
110
|
+
if not self.job or not kernel.SetInformationJobObject(self.job, 9, ctypes.byref(limit), ctypes.sizeof(limit)) or (not kernel.AssignProcessToJobObject(self.job, int(child._handle))):
|
|
111
|
+
if self.job:
|
|
112
|
+
kernel.CloseHandle(self.job)
|
|
113
|
+
self.job = None
|
|
114
|
+
child.kill()
|
|
115
|
+
child.wait()
|
|
116
|
+
raise ContinuityError('Cannot establish Windows process-tree cleanup job')
|
|
117
|
+
|
|
118
|
+
def statistics(self):
|
|
119
|
+
"""Owned Job accounting only. Commit peaks are NOT RSS/working-set peaks."""
|
|
120
|
+
if self.job is None:
|
|
121
|
+
return {'scope': 'UNAVAILABLE', 'job_private_commit_peak_bytes': None, 'CPU_job_seconds': None}
|
|
122
|
+
import ctypes
|
|
123
|
+
limit = self.extended()
|
|
124
|
+
account = self.accounting()
|
|
125
|
+
if not self.kernel.QueryInformationJobObject(self.job, 9, ctypes.byref(limit), ctypes.sizeof(limit), None) or not self.kernel.QueryInformationJobObject(self.job, 1, ctypes.byref(account), ctypes.sizeof(account), None):
|
|
126
|
+
raise ContinuityError('Cannot verify owned Job limits/accounting')
|
|
127
|
+
actual = {'process_commit_bytes': int(limit.ProcessMemoryLimit), 'job_commit_bytes': int(limit.JobMemoryLimit)}
|
|
128
|
+
flags = int(limit.BasicLimitInformation.LimitFlags)
|
|
129
|
+
if self.memory_limits is not None and (actual != self.memory_limits or flags & 768 != 768):
|
|
130
|
+
raise ContinuityError('Owned Job memory limits differ')
|
|
131
|
+
from ctypes import wintypes
|
|
132
|
+
|
|
133
|
+
class Memory(ctypes.Structure):
|
|
134
|
+
_fields_ = [('cb', wintypes.DWORD), ('PageFaultCount', wintypes.DWORD)] + [(n, ctypes.c_size_t) for n in ('PeakWorkingSetSize', 'WorkingSetSize', 'QuotaPeakPagedPoolUsage', 'QuotaPagedPoolUsage', 'QuotaPeakNonPagedPoolUsage', 'QuotaNonPagedPoolUsage', 'PagefileUsage', 'PeakPagefileUsage', 'PrivateUsage')]
|
|
135
|
+
psapi = ctypes.WinDLL('psapi', use_last_error=True)
|
|
136
|
+
psapi.GetProcessMemoryInfo.argtypes = [wintypes.HANDLE, ctypes.c_void_p, wintypes.DWORD]
|
|
137
|
+
for pid in self._owned_pids:
|
|
138
|
+
handle = self.kernel.OpenProcess(1024 | 16, False, pid)
|
|
139
|
+
if not handle:
|
|
140
|
+
continue
|
|
141
|
+
try:
|
|
142
|
+
member = wintypes.BOOL()
|
|
143
|
+
if not self.kernel.IsProcessInJob(handle, self.job, ctypes.byref(member)) or not member.value:
|
|
144
|
+
continue
|
|
145
|
+
memory = Memory()
|
|
146
|
+
memory.cb = ctypes.sizeof(memory)
|
|
147
|
+
if psapi.GetProcessMemoryInfo(handle, ctypes.byref(memory), ctypes.sizeof(memory)):
|
|
148
|
+
self._working_set_peaks[pid] = max(self._working_set_peaks.get(pid, 0), int(memory.PeakWorkingSetSize))
|
|
149
|
+
finally:
|
|
150
|
+
self.kernel.CloseHandle(handle)
|
|
151
|
+
return {'scope': 'WINDOWS_OWNED_JOB_PRIVATE_COMMIT_NOT_RSS', 'memory_limits': actual, 'private_commit_allocation_policy_verified': self.memory_limits is not None, 'hard_private_commit_cap_verified': False, 'hard_peak_private_commit_cap': False, 'peak_over_job_policy_bytes': max(0, int(limit.PeakJobMemoryUsed) - actual['job_commit_bytes']) if self.memory_limits else None, 'memory_boundary': 'Configured Windows allocation policy;observed refusal is not proof of an absolute peak ceiling', 'job_private_commit_peak_bytes': int(limit.PeakJobMemoryUsed), 'process_private_commit_peak_bytes': int(limit.PeakProcessMemoryUsed), 'CPU_job_seconds': (account.TotalUserTime + account.TotalKernelTime) / 10000000.0, 'active_processes': int(account.ActiveProcesses), 'total_processes': int(account.TotalProcesses), 'owned_process_peak_working_sets_bytes': dict(self._working_set_peaks), 'working_set_scope': 'Sampled OS-maintained peaks of verified owned launcher/worker PIDs;not aggregate job RSS', 'peak_RSS': 'NOT_A_PORTABLE_OR_AGGREGATE_RSS_MEASURE', 'aggregate_RSS_cap': False}
|
|
152
|
+
|
|
153
|
+
def include_pid(self, pid):
|
|
154
|
+
"""Also attach an interpreter already started by a Windows venv launcher."""
|
|
155
|
+
if self.job is None:
|
|
156
|
+
return
|
|
157
|
+
import ctypes
|
|
158
|
+
from ctypes import wintypes
|
|
159
|
+
handle = self.kernel.OpenProcess(256 | 1 | 4096, False, pid)
|
|
160
|
+
if not handle:
|
|
161
|
+
raise ContinuityError('Cannot identify the bootstrap interpreter')
|
|
162
|
+
try:
|
|
163
|
+
member = wintypes.BOOL()
|
|
164
|
+
if not self.kernel.IsProcessInJob(handle, self.job, ctypes.byref(member)):
|
|
165
|
+
raise ContinuityError('Cannot verify bootstrap job membership')
|
|
166
|
+
if not member.value and (not self.kernel.AssignProcessToJobObject(self.job, handle)):
|
|
167
|
+
raise ContinuityError('Cannot attach bootstrap interpreter to cleanup job')
|
|
168
|
+
self._owned_pids.add(pid)
|
|
169
|
+
finally:
|
|
170
|
+
self.kernel.CloseHandle(handle)
|
|
171
|
+
|
|
172
|
+
def close(self):
|
|
173
|
+
if self.job is not None:
|
|
174
|
+
import ctypes
|
|
175
|
+
self.kernel.TerminateJobObject(self.job, 1)
|
|
176
|
+
deadline = time.monotonic() + 5
|
|
177
|
+
while time.monotonic() < deadline:
|
|
178
|
+
counters = self.accounting()
|
|
179
|
+
if not self.kernel.QueryInformationJobObject(self.job, 1, ctypes.byref(counters), ctypes.sizeof(counters), None) or counters.ActiveProcesses == 0:
|
|
180
|
+
break
|
|
181
|
+
time.sleep(0.01)
|
|
182
|
+
self.kernel.CloseHandle(self.job)
|
|
183
|
+
self.job = None
|
|
184
|
+
elif os.name != 'nt':
|
|
185
|
+
try:
|
|
186
|
+
os.killpg(self.child.pid, signal.SIGTERM)
|
|
187
|
+
except ProcessLookupError:
|
|
188
|
+
pass
|
|
189
|
+
if self.child.poll() is None:
|
|
190
|
+
self.child.kill()
|
|
191
|
+
self.child.wait()
|