pythagoras 0.20.51__py3-none-any.whl → 0.21.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pythagoras/_020_ordinary_code_portals/code_normalizer.py +11 -3
- pythagoras/_020_ordinary_code_portals/ordinary_portal_core_classes.py +9 -6
- pythagoras/_030_data_portals/__init__.py +2 -2
- pythagoras/_030_data_portals/data_portal_core_classes.py +70 -56
- pythagoras/_040_logging_code_portals/kw_args.py +9 -5
- pythagoras/_040_logging_code_portals/logging_portal_core_classes.py +15 -16
- pythagoras/_060_autonomous_code_portals/autonomous_portal_core_classes.py +46 -28
- pythagoras/_070_protected_code_portals/GPU_pre_validators.py +0 -0
- pythagoras/_070_protected_code_portals/RAM_pre_validators.py +58 -0
- pythagoras/_070_protected_code_portals/__init__.py +4 -3
- pythagoras/_070_protected_code_portals/fn_arg_names_checker.py +3 -2
- pythagoras/_070_protected_code_portals/protected_decorators.py +13 -12
- pythagoras/_070_protected_code_portals/protected_portal_core_classes.py +95 -78
- pythagoras/{_090_swarming_portals → _070_protected_code_portals}/system_utils.py +29 -1
- pythagoras/_070_protected_code_portals/validator_fn_classes.py +95 -0
- pythagoras/_080_pure_code_portals/pure_core_classes.py +16 -18
- pythagoras/_080_pure_code_portals/pure_decorator.py +12 -12
- pythagoras/_090_swarming_portals/swarming_portals.py +4 -10
- pythagoras/__init__.py +14 -14
- {pythagoras-0.20.51.dist-info → pythagoras-0.21.0.dist-info}/METADATA +5 -2
- {pythagoras-0.20.51.dist-info → pythagoras-0.21.0.dist-info}/RECORD +23 -22
- {pythagoras-0.20.51.dist-info → pythagoras-0.21.0.dist-info}/WHEEL +1 -1
- pythagoras/_070_protected_code_portals/GPU_guards.py +0 -2
- pythagoras/_070_protected_code_portals/RAM_guards.py +0 -33
- /pythagoras/_070_protected_code_portals/{python_packages_guards.py → python_packages_pre_validators.py} +0 -0
|
@@ -5,6 +5,8 @@ from typing import Callable, Any, List
|
|
|
5
5
|
|
|
6
6
|
from persidict import PersiDict, Joker, KEEP_CURRENT
|
|
7
7
|
|
|
8
|
+
from .validator_fn_classes import ValidatorFn, PreValidatorFn, PostValidatorFn, SimplePreValidatorFn, \
|
|
9
|
+
ComplexPreValidatorFn
|
|
8
10
|
from .._030_data_portals import DataPortal
|
|
9
11
|
from .._040_logging_code_portals import KwArgs
|
|
10
12
|
from .._030_data_portals import ValueAddr
|
|
@@ -32,44 +34,43 @@ class ProtectedCodePortal(AutonomousCodePortal):
|
|
|
32
34
|
|
|
33
35
|
class ProtectedFn(AutonomousFn):
|
|
34
36
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
_pre_validators_cached: list[AutonomousFn] | None
|
|
38
|
+
_post_validators_cached: list[AutonomousFn] | None
|
|
39
|
+
_pre_validators_addrs: list[ValueAddr]
|
|
40
|
+
_post_validators_addrs: list[ValueAddr]
|
|
39
41
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
+
post_validators_arg_names = ["packed_kwargs", "fn_addr", "result"]
|
|
43
|
+
pre_validators_arg_names = ["packed_kwargs", "fn_addr"]
|
|
42
44
|
|
|
43
45
|
def __init__(self, fn: Callable | str
|
|
44
|
-
,
|
|
45
|
-
,
|
|
46
|
+
, pre_validators: list[AutonomousFn] | List[Callable] | None = None
|
|
47
|
+
, post_validators: list[AutonomousFn] | List[Callable] | None = None
|
|
46
48
|
, excessive_logging: bool|None = KEEP_CURRENT
|
|
47
49
|
, fixed_kwargs: dict | None = None
|
|
48
|
-
, portal:
|
|
49
|
-
|
|
50
|
-
,fn=fn
|
|
50
|
+
, portal: ProtectedCodePortal | None = None):
|
|
51
|
+
super().__init__(fn=fn
|
|
51
52
|
, portal = portal
|
|
52
53
|
, fixed_kwargs=fixed_kwargs
|
|
53
54
|
, excessive_logging = excessive_logging)
|
|
54
55
|
|
|
56
|
+
pre_validators = self._normalize_validators(pre_validators, PreValidatorFn)
|
|
57
|
+
post_validators = self._normalize_validators(post_validators, PostValidatorFn)
|
|
58
|
+
|
|
55
59
|
if isinstance(fn, ProtectedFn):
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
return
|
|
60
|
+
pre_validators += fn.pre_validators
|
|
61
|
+
post_validators += fn.post_validators
|
|
59
62
|
|
|
60
|
-
self.
|
|
61
|
-
|
|
62
|
-
self.
|
|
63
|
-
|
|
64
|
-
self._guards_addrs = [ValueAddr(g, store=False) for g in self._guards]
|
|
65
|
-
self._validators_addrs = [ValueAddr(v, store=False) for v in self._validators]
|
|
63
|
+
self._pre_validators_cached = self._normalize_validators(pre_validators, PreValidatorFn)
|
|
64
|
+
self._post_validators_cached = self._normalize_validators(post_validators, PostValidatorFn)
|
|
65
|
+
self._pre_validators_addrs = [ValueAddr(g, store=False) for g in self._pre_validators_cached]
|
|
66
|
+
self._post_validators_addrs = [ValueAddr(v, store=False) for v in self._post_validators_cached]
|
|
66
67
|
|
|
67
68
|
|
|
68
69
|
def __getstate__(self):
|
|
69
70
|
"""This method is called when the object is pickled."""
|
|
70
71
|
state = super().__getstate__()
|
|
71
|
-
state["
|
|
72
|
-
state["
|
|
72
|
+
state["pre_validators_addrs"] = self._pre_validators_addrs
|
|
73
|
+
state["post_validators_addrs"] = self._post_validators_addrs
|
|
73
74
|
return state
|
|
74
75
|
|
|
75
76
|
|
|
@@ -77,53 +78,60 @@ class ProtectedFn(AutonomousFn):
|
|
|
77
78
|
"""This method is called when the object is unpickled."""
|
|
78
79
|
self._invalidate_cache()
|
|
79
80
|
super().__setstate__(state)
|
|
80
|
-
self.
|
|
81
|
-
self.
|
|
81
|
+
self._pre_validators_addrs = state["pre_validators_addrs"]
|
|
82
|
+
self._post_validators_addrs = state["post_validators_addrs"]
|
|
82
83
|
|
|
83
84
|
|
|
84
85
|
def _first_visit_to_portal(self, portal: DataPortal) -> None:
|
|
86
|
+
"""Register an object in a portal that the object has not seen before."""
|
|
85
87
|
super()._first_visit_to_portal(portal)
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
88
|
+
|
|
89
|
+
if hasattr(self,"_pre_validators_cached"):
|
|
90
|
+
with portal:
|
|
91
|
+
_ = [ValueAddr(g) for g in self._pre_validators_cached]
|
|
92
|
+
|
|
93
|
+
if hasattr(self,"_post_validators_cached"):
|
|
94
|
+
with portal:
|
|
95
|
+
_ = [ValueAddr(g) for g in self._post_validators_cached]
|
|
93
96
|
|
|
94
97
|
|
|
95
98
|
@property
|
|
96
|
-
def
|
|
97
|
-
if not hasattr(self, "
|
|
98
|
-
self.
|
|
99
|
-
return self.
|
|
99
|
+
def pre_validators(self) -> list[AutonomousFn]:
|
|
100
|
+
if not hasattr(self, "_pre_validators_cached"):
|
|
101
|
+
self._pre_validators_cached = [addr.get() for addr in self._pre_validators_addrs]
|
|
102
|
+
return self._pre_validators_cached
|
|
100
103
|
|
|
101
104
|
|
|
102
105
|
@property
|
|
103
|
-
def
|
|
104
|
-
if not hasattr(self, "
|
|
105
|
-
self.
|
|
106
|
-
return self.
|
|
106
|
+
def post_validators(self) -> list[AutonomousFn]:
|
|
107
|
+
if not hasattr(self, "_post_validators_cached"):
|
|
108
|
+
self._post_validators_cached = [addr.get() for addr in self._post_validators_addrs]
|
|
109
|
+
return self._post_validators_cached
|
|
107
110
|
|
|
108
111
|
|
|
109
112
|
def can_be_executed(self, kw_args: KwArgs) -> bool:
|
|
110
113
|
with self.portal as portal:
|
|
111
114
|
kw_args = kw_args.pack()
|
|
112
|
-
|
|
113
|
-
portal.entropy_infuser.shuffle(
|
|
114
|
-
for
|
|
115
|
-
|
|
115
|
+
pre_validators = copy(self.pre_validators)
|
|
116
|
+
portal.entropy_infuser.shuffle(pre_validators)
|
|
117
|
+
for pre_validator in pre_validators:
|
|
118
|
+
pre_validation_result = None
|
|
119
|
+
if isinstance(pre_validator, SimplePreValidatorFn):
|
|
120
|
+
pre_validation_result = pre_validator()
|
|
121
|
+
else:
|
|
122
|
+
pre_validation_result = pre_validator(packed_kwargs=kw_args, fn_addr = self.addr)
|
|
123
|
+
if pre_validation_result is not OK:
|
|
116
124
|
return False
|
|
117
125
|
return True
|
|
118
126
|
|
|
119
127
|
|
|
120
|
-
def
|
|
128
|
+
def validate_execution_result(self, kw_args: KwArgs, result: Any) -> bool:
|
|
121
129
|
with self.portal as portal:
|
|
122
130
|
kw_args = kw_args.pack()
|
|
123
|
-
|
|
124
|
-
portal.entropy_infuser.shuffle(
|
|
125
|
-
for
|
|
126
|
-
if
|
|
131
|
+
post_validators = copy(self.post_validators)
|
|
132
|
+
portal.entropy_infuser.shuffle(post_validators)
|
|
133
|
+
for post_validator in post_validators:
|
|
134
|
+
if post_validator(packed_kwargs=kw_args, fn_addr = self.addr
|
|
127
135
|
, result=result) is not OK:
|
|
128
136
|
return False
|
|
129
137
|
return True
|
|
@@ -134,38 +142,44 @@ class ProtectedFn(AutonomousFn):
|
|
|
134
142
|
kw_args = KwArgs(**kwargs)
|
|
135
143
|
assert self.can_be_executed(kw_args)
|
|
136
144
|
result = super().execute(**kwargs)
|
|
137
|
-
assert self.
|
|
145
|
+
assert self.validate_execution_result(kw_args, result)
|
|
138
146
|
return result
|
|
139
147
|
|
|
140
148
|
|
|
141
|
-
def
|
|
142
|
-
,
|
|
143
|
-
,
|
|
144
|
-
) -> list[
|
|
145
|
-
"""Return list of
|
|
149
|
+
def _normalize_validators(self
|
|
150
|
+
, validators: list[ValidatorFn] | None
|
|
151
|
+
, validator_type: type
|
|
152
|
+
) -> list[ValidatorFn]:
|
|
153
|
+
"""Return list of validators in a normalized form.
|
|
146
154
|
|
|
147
|
-
All the functions-
|
|
155
|
+
All the functions-validators are converted to AutonomousFn objects,
|
|
148
156
|
and returned as a list, sorted by functions' hash signatures.
|
|
149
157
|
"""
|
|
150
|
-
|
|
158
|
+
assert validator_type in {PreValidatorFn, PostValidatorFn}
|
|
159
|
+
if validators is None:
|
|
151
160
|
return []
|
|
152
|
-
if not isinstance(
|
|
153
|
-
if callable(
|
|
154
|
-
|
|
155
|
-
assert isinstance(
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
for
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
161
|
+
if not isinstance(validators, list):
|
|
162
|
+
if callable(validators) or isinstance(validators, ValidatorFn) or isinstance(validators, str):
|
|
163
|
+
validators = [validators]
|
|
164
|
+
assert isinstance(validators, list)
|
|
165
|
+
validators = flatten_list(validators)
|
|
166
|
+
new_validators = []
|
|
167
|
+
for validator in validators:
|
|
168
|
+
if not isinstance(validator, validator_type):
|
|
169
|
+
if validator_type is PreValidatorFn:
|
|
170
|
+
try:
|
|
171
|
+
validator = ComplexPreValidatorFn(validator)
|
|
172
|
+
except:
|
|
173
|
+
validator = SimplePreValidatorFn(validator)
|
|
174
|
+
elif validator_type is PostValidatorFn:
|
|
175
|
+
validator = PostValidatorFn(validator)
|
|
176
|
+
else:
|
|
177
|
+
raise TypeError(f"Unknown type {validator_type}")
|
|
178
|
+
new_validators.append(validator)
|
|
179
|
+
validators = {f.hash_signature: f for f in new_validators}
|
|
180
|
+
validators = sort_dict_by_keys(validators)
|
|
181
|
+
validators = list(validators.values())
|
|
182
|
+
return validators
|
|
169
183
|
|
|
170
184
|
|
|
171
185
|
@property
|
|
@@ -173,8 +187,11 @@ class ProtectedFn(AutonomousFn):
|
|
|
173
187
|
return AutonomousFn.portal.__get__(self)
|
|
174
188
|
|
|
175
189
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
190
|
+
def _invalidate_cache(self):
|
|
191
|
+
super()._invalidate_cache()
|
|
192
|
+
if hasattr(self, "_post_validators_cached"):
|
|
193
|
+
assert hasattr(self, "_post_validators_addrs"), "Premature cache invalidation: _post_validators_addrs is missing."
|
|
194
|
+
del self._post_validators_cached
|
|
195
|
+
if hasattr(self, "_pre_validators_cached"):
|
|
196
|
+
assert hasattr(self, "_pre_validators_addrs"), "Premature cache invalidation: _pre_validators_addrs is missing."
|
|
197
|
+
del self._pre_validators_cached
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import psutil
|
|
3
|
+
import pynvml
|
|
3
4
|
|
|
4
5
|
def get_available_ram_mb() -> int:
|
|
5
6
|
"""Returns the amount of available RAM in MB. """
|
|
@@ -7,7 +8,7 @@ def get_available_ram_mb() -> int:
|
|
|
7
8
|
return int(free_ram)
|
|
8
9
|
|
|
9
10
|
|
|
10
|
-
def
|
|
11
|
+
def get_unused_cpu_cores() -> float:
|
|
11
12
|
"""Returns the free (logical) CPU capacity"""
|
|
12
13
|
|
|
13
14
|
cnt = psutil.cpu_count(logical=True) or 1
|
|
@@ -50,3 +51,30 @@ def get_current_process_start_time() -> int:
|
|
|
50
51
|
return get_process_start_time(get_current_process_id())
|
|
51
52
|
|
|
52
53
|
|
|
54
|
+
def get_unused_nvidia_gpus() -> float:
|
|
55
|
+
"""Returns the total unused GPU capacity as a float value.
|
|
56
|
+
|
|
57
|
+
The function calculates the sum of unused capacity across all available NVIDIA GPUs.
|
|
58
|
+
For example, 2.0 means two completely unused GPUs
|
|
59
|
+
0 is returned if no GPUs or on error.
|
|
60
|
+
"""
|
|
61
|
+
try:
|
|
62
|
+
pynvml.nvmlInit()
|
|
63
|
+
device_count = pynvml.nvmlDeviceGetCount()
|
|
64
|
+
unused_capacity = 0.0
|
|
65
|
+
|
|
66
|
+
for i in range(device_count):
|
|
67
|
+
handle = pynvml.nvmlDeviceGetHandleByIndex(i)
|
|
68
|
+
utilization = pynvml.nvmlDeviceGetUtilizationRates(handle)
|
|
69
|
+
unused_capacity += max(0.0, 100.0 - utilization.gpu) # Clamp to avoid negative values
|
|
70
|
+
|
|
71
|
+
return unused_capacity / 100.0
|
|
72
|
+
|
|
73
|
+
except pynvml.NVMLError as e:
|
|
74
|
+
# Return 0.0 on any NVML error (no GPUs, driver issues, etc.)
|
|
75
|
+
return 0.0
|
|
76
|
+
finally:
|
|
77
|
+
try:
|
|
78
|
+
pynvml.nvmlShutdown()
|
|
79
|
+
except:
|
|
80
|
+
pass # Safe cleanup even if initialization failed
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
from typing import Callable
|
|
2
|
+
|
|
3
|
+
from persidict import KEEP_CURRENT, Joker
|
|
4
|
+
|
|
5
|
+
from pythagoras._060_autonomous_code_portals import AutonomousFn, AutonomousCodePortal
|
|
6
|
+
from .fn_arg_names_checker import check_if_fn_accepts_args
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ValidatorFn(AutonomousFn):
|
|
10
|
+
def __init__(self, fn: Callable | str | AutonomousFn
|
|
11
|
+
, fixed_kwargs: dict | None = None
|
|
12
|
+
, excessive_logging: bool | Joker = KEEP_CURRENT
|
|
13
|
+
, portal: AutonomousCodePortal | None = None):
|
|
14
|
+
super().__init__(
|
|
15
|
+
fn=fn
|
|
16
|
+
, fixed_kwargs=fixed_kwargs
|
|
17
|
+
, excessive_logging=excessive_logging
|
|
18
|
+
, portal=portal)
|
|
19
|
+
|
|
20
|
+
check_if_fn_accepts_args(self.get_allowed_kwargs_names(), self.source_code)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def get_allowed_kwargs_names(cls)->set[str]:
|
|
25
|
+
raise NotImplementedError("This method must be overridden")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def execute(self,**kwargs):
|
|
29
|
+
assert set(kwargs) == self.get_allowed_kwargs_names()
|
|
30
|
+
return super().execute(**kwargs)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class PreValidatorFn(ValidatorFn):
|
|
34
|
+
def __init__(self, fn: Callable | str | AutonomousFn
|
|
35
|
+
, fixed_kwargs: dict | None = None
|
|
36
|
+
, excessive_logging: bool | Joker = KEEP_CURRENT
|
|
37
|
+
, portal: AutonomousCodePortal | None = None):
|
|
38
|
+
super().__init__(
|
|
39
|
+
fn=fn
|
|
40
|
+
, fixed_kwargs=fixed_kwargs
|
|
41
|
+
, excessive_logging=excessive_logging
|
|
42
|
+
, portal=portal)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class SimplePreValidatorFn(PreValidatorFn):
|
|
46
|
+
def __init__(self, fn: Callable | str | AutonomousFn
|
|
47
|
+
, fixed_kwargs: dict | None = None
|
|
48
|
+
, excessive_logging: bool | Joker = KEEP_CURRENT
|
|
49
|
+
, portal: AutonomousCodePortal | None = None):
|
|
50
|
+
super().__init__(
|
|
51
|
+
fn=fn
|
|
52
|
+
, fixed_kwargs=fixed_kwargs
|
|
53
|
+
, excessive_logging=excessive_logging
|
|
54
|
+
, portal=portal)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@classmethod
|
|
58
|
+
def get_allowed_kwargs_names(cls) -> set[str]:
|
|
59
|
+
"""Simple pre-validators do not take any inputs."""
|
|
60
|
+
return set()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class ComplexPreValidatorFn(PreValidatorFn):
|
|
64
|
+
def __init__(self, fn: Callable | str | AutonomousFn
|
|
65
|
+
, fixed_kwargs: dict | None = None
|
|
66
|
+
, excessive_logging: bool | Joker = KEEP_CURRENT
|
|
67
|
+
, portal: AutonomousCodePortal | None = None):
|
|
68
|
+
super().__init__(
|
|
69
|
+
fn=fn
|
|
70
|
+
, fixed_kwargs=fixed_kwargs
|
|
71
|
+
, excessive_logging=excessive_logging
|
|
72
|
+
, portal=portal)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@classmethod
|
|
76
|
+
def get_allowed_kwargs_names(cls) -> set[str]:
|
|
77
|
+
"""Complex pre-validators use info about the function and its input arguments."""
|
|
78
|
+
return {"packed_kwargs", "fn_addr"}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class PostValidatorFn(ValidatorFn):
|
|
82
|
+
def __init__(self, fn: Callable | str | AutonomousFn
|
|
83
|
+
, fixed_kwargs: dict | None = None
|
|
84
|
+
, excessive_logging: bool | Joker = KEEP_CURRENT
|
|
85
|
+
, portal: AutonomousCodePortal | None = None):
|
|
86
|
+
super().__init__(
|
|
87
|
+
fn=fn
|
|
88
|
+
, fixed_kwargs=fixed_kwargs
|
|
89
|
+
, excessive_logging=excessive_logging
|
|
90
|
+
, portal=portal)
|
|
91
|
+
|
|
92
|
+
@classmethod
|
|
93
|
+
def get_allowed_kwargs_names(cls) -> set[str]:
|
|
94
|
+
"""Post-validators use info about the function, its input arguments and returned value."""
|
|
95
|
+
return {"packed_kwargs", "fn_addr", "result" }
|
|
@@ -93,18 +93,18 @@ class PureCodePortal(ProtectedCodePortal):
|
|
|
93
93
|
class PureFn(ProtectedFn):
|
|
94
94
|
|
|
95
95
|
def __init__(self, fn: Callable | str
|
|
96
|
-
,
|
|
97
|
-
,
|
|
96
|
+
, pre_validators: list[AutonomousFn] | List[Callable] | None = None
|
|
97
|
+
, post_validators: list[AutonomousFn] | List[Callable] | None = None
|
|
98
98
|
, excessive_logging: bool | Joker = KEEP_CURRENT
|
|
99
99
|
, fixed_kwargs: dict | None = None
|
|
100
100
|
, portal: PureCodePortal | None = None):
|
|
101
101
|
ProtectedFn.__init__(self
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
102
|
+
, fn=fn
|
|
103
|
+
, portal = portal
|
|
104
|
+
, fixed_kwargs=fixed_kwargs
|
|
105
|
+
, excessive_logging = excessive_logging
|
|
106
|
+
, pre_validators=pre_validators
|
|
107
|
+
, post_validators=post_validators)
|
|
108
108
|
|
|
109
109
|
|
|
110
110
|
def get_address(self, **kwargs) -> PureFnExecutionResultAddr:
|
|
@@ -261,13 +261,13 @@ class PureFnExecutionResultAddr(HashAddr):
|
|
|
261
261
|
assert isinstance(fn, PureFn)
|
|
262
262
|
with fn.portal as portal:
|
|
263
263
|
self._kwargs_cache = KwArgs(**arguments)
|
|
264
|
+
self._fn_cache = fn
|
|
264
265
|
signature = LoggingFnCallSignature(fn, self._kwargs_cache)
|
|
265
266
|
self._call_signature_cache = signature
|
|
266
267
|
tmp = ValueAddr(signature)
|
|
267
|
-
|
|
268
|
+
new_descriptor = fn.name +"_result_addr"
|
|
268
269
|
new_hash_signature = tmp.hash_signature
|
|
269
|
-
super().__init__(
|
|
270
|
-
self._fn_cache = fn
|
|
270
|
+
super().__init__(new_descriptor, new_hash_signature)
|
|
271
271
|
|
|
272
272
|
|
|
273
273
|
def _invalidate_cache(self):
|
|
@@ -289,13 +289,11 @@ class PureFnExecutionResultAddr(HashAddr):
|
|
|
289
289
|
del self._call_signature_cache
|
|
290
290
|
|
|
291
291
|
|
|
292
|
-
def get_ValueAddr(self):
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
prefix = self.prefix.removesuffix("_result_addr")
|
|
296
|
-
prefix += "_" + LoggingFnCallSignature.__name__.lower()
|
|
292
|
+
def get_ValueAddr(self):
|
|
293
|
+
descriptor = self.descriptor.removesuffix("_result_addr")
|
|
294
|
+
descriptor += "_" + LoggingFnCallSignature.__name__.lower()
|
|
297
295
|
return ValueAddr.from_strings( # TODO: refactor this
|
|
298
|
-
|
|
296
|
+
descriptor= descriptor
|
|
299
297
|
, hash_signature=self.hash_signature)
|
|
300
298
|
|
|
301
299
|
|
|
@@ -465,7 +463,7 @@ class PureFnExecutionResultAddr(HashAddr):
|
|
|
465
463
|
def can_be_executed(self) -> bool: #*#*#
|
|
466
464
|
"""Indicates if the function can be executed in the current session.
|
|
467
465
|
|
|
468
|
-
The function should be refactored once we start fully supporting
|
|
466
|
+
TODO: The function should be refactored once we start fully supporting
|
|
469
467
|
guards
|
|
470
468
|
"""
|
|
471
469
|
with self.fn.portal:
|
|
@@ -10,26 +10,26 @@ from persidict import KEEP_CURRENT, Joker
|
|
|
10
10
|
class pure(protected):
|
|
11
11
|
|
|
12
12
|
def __init__(self
|
|
13
|
-
,
|
|
14
|
-
,
|
|
13
|
+
, pre_validators: list[AutonomousFn] | None = None
|
|
14
|
+
, post_validators: list[AutonomousFn] | None = None
|
|
15
15
|
, fixed_kwargs: dict | None = None
|
|
16
16
|
, excessive_logging: bool | Joker = KEEP_CURRENT
|
|
17
17
|
, portal: PureCodePortal | None = None
|
|
18
18
|
):
|
|
19
19
|
protected.__init__(self=self
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
20
|
+
, portal=portal
|
|
21
|
+
, excessive_logging=excessive_logging
|
|
22
|
+
, fixed_kwargs=fixed_kwargs
|
|
23
|
+
, pre_validators=pre_validators
|
|
24
|
+
, post_validators=post_validators)
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
def __call__(self, fn:Callable|str) -> PureFn:
|
|
28
28
|
wrapper = PureFn(fn
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
29
|
+
, portal=self._portal
|
|
30
|
+
, pre_validators=self._pre_validators
|
|
31
|
+
, fixed_kwargs=self._fixed_kwargs
|
|
32
|
+
, post_validators=self._post_validators
|
|
33
|
+
, excessive_logging=self._excessive_logging)
|
|
34
34
|
return wrapper
|
|
35
35
|
|
|
@@ -1,23 +1,17 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import atexit
|
|
4
|
-
import os
|
|
5
4
|
from time import sleep
|
|
6
|
-
import random
|
|
7
5
|
|
|
8
6
|
import pandas as pd
|
|
9
7
|
import parameterizable
|
|
10
8
|
from parameterizable import sort_dict_by_keys
|
|
11
9
|
from persidict import PersiDict, Joker, KEEP_CURRENT
|
|
12
10
|
|
|
13
|
-
from .._010_basic_portals import
|
|
14
|
-
from
|
|
15
|
-
from .system_utils import (
|
|
16
|
-
get_current_process_id, process_is_active,
|
|
17
|
-
get_process_start_time, get_current_process_start_time, get_available_cpu_cores, get_available_ram_mb)
|
|
11
|
+
from .._010_basic_portals import get_all_known_portals
|
|
12
|
+
from pythagoras._070_protected_code_portals.system_utils import *
|
|
18
13
|
from .._040_logging_code_portals.logging_portal_core_classes import build_execution_environment_summary
|
|
19
14
|
from .._010_basic_portals.basic_portal_core_classes import _describe_runtime_characteristic
|
|
20
|
-
from .._800_signatures_and_converters.random_signatures import get_random_signature
|
|
21
15
|
from persidict import OverlappingMultiDict
|
|
22
16
|
from .._080_pure_code_portals.pure_core_classes import (
|
|
23
17
|
PureCodePortal, PureFnExecutionResultAddr)
|
|
@@ -136,7 +130,7 @@ class SwarmingPortal(PureCodePortal):
|
|
|
136
130
|
n = self._get_config_setting("max_n_workers")
|
|
137
131
|
if n in (None, KEEP_CURRENT):
|
|
138
132
|
n = 10
|
|
139
|
-
n = min(n,
|
|
133
|
+
n = min(n, get_unused_cpu_cores())
|
|
140
134
|
n = min(n, get_available_ram_mb() / 500)
|
|
141
135
|
n = int(n)+1
|
|
142
136
|
self._max_n_workers_cache = n
|
|
@@ -269,7 +263,7 @@ def _process_random_execution_request(**portal_init_params):
|
|
|
269
263
|
portal._randomly_delay_execution()
|
|
270
264
|
continue
|
|
271
265
|
new_address = PureFnExecutionResultAddr.from_strings(
|
|
272
|
-
|
|
266
|
+
descriptor=addr[2], hash_signature=addr[0]+addr[1]+addr[3]
|
|
273
267
|
,assert_readiness=False)
|
|
274
268
|
if not new_address.needs_execution:
|
|
275
269
|
continue
|
pythagoras/__init__.py
CHANGED
|
@@ -21,19 +21,19 @@ from ._100_top_level_API import *
|
|
|
21
21
|
from ._800_signatures_and_converters import *
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
primary_decorators = {d.__name__:d for d in [
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
all_decorators = {d.__name__:d for d in [
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
]}
|
|
24
|
+
# primary_decorators = {d.__name__:d for d in [
|
|
25
|
+
# autonomous
|
|
26
|
+
# , pure
|
|
27
|
+
# ]}
|
|
28
|
+
#
|
|
29
|
+
# all_decorators = {d.__name__:d for d in [
|
|
30
|
+
# ordinary
|
|
31
|
+
# , storable
|
|
32
|
+
# , logging
|
|
33
|
+
# , safe
|
|
34
|
+
# , autonomous
|
|
35
|
+
# , protected
|
|
36
|
+
# , pure
|
|
37
|
+
# ]}
|
|
38
38
|
|
|
39
39
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: pythagoras
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.21.0
|
|
4
4
|
Summary: Planet-scale distributed computing in Python.
|
|
5
5
|
Keywords: cloud,ML,AI,serverless,distributed,parallel,machine-learning,deep-learning,pythagoras
|
|
6
6
|
Author: Volodymyr (Vlad) Pavlov
|
|
@@ -30,6 +30,7 @@ Requires-Dist: pytest
|
|
|
30
30
|
Requires-Dist: boto3
|
|
31
31
|
Requires-Dist: moto
|
|
32
32
|
Requires-Dist: uv
|
|
33
|
+
Requires-Dist: nvidia-ml-py
|
|
33
34
|
Requires-Dist: persidict[aws] ; extra == 'aws'
|
|
34
35
|
Requires-Dist: boto3 ; extra == 'aws'
|
|
35
36
|
Requires-Dist: moto ; extra == 'aws'
|
|
@@ -52,6 +53,8 @@ Description-Content-Type: text/markdown
|
|
|
52
53
|
|
|
53
54
|
Planet-scale distributed computing in Python.
|
|
54
55
|
|
|
56
|
+
!!! RESEARCH PREVIEW !!!
|
|
57
|
+
|
|
55
58
|
## What is it?
|
|
56
59
|
|
|
57
60
|
`Pythagoras` is an easy-to-use framework for cost-efficient
|
|
@@ -69,7 +72,7 @@ with intelligent data and code change tracking;
|
|
|
69
72
|
## Usage
|
|
70
73
|
|
|
71
74
|
* `Introduction to Pythagoras` notebook:
|
|
72
|
-
https://colab.research.google.com/drive/
|
|
75
|
+
https://colab.research.google.com/drive/117rAtaWxjfSubzGx0shgqNjONY3E5jgd
|
|
73
76
|
|
|
74
77
|
## How to get it?
|
|
75
78
|
|