pythagoras 0.22.1__py3-none-any.whl → 0.23.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pythagoras/_010_basic_portals/basic_portal_core_classes.py +3 -3
- pythagoras/_020_ordinary_code_portals/ordinary_portal_core_classes.py +4 -4
- pythagoras/_030_data_portals/__init__.py +11 -8
- pythagoras/_030_data_portals/data_portal_core_classes.py +32 -28
- pythagoras/_030_data_portals/ready_and_get.py +21 -8
- pythagoras/_040_logging_code_portals/__init__.py +7 -13
- pythagoras/_040_logging_code_portals/exception_processing_tracking.py +6 -4
- pythagoras/_040_logging_code_portals/kw_args.py +1 -1
- pythagoras/_040_logging_code_portals/logging_portal_core_classes.py +3 -5
- pythagoras/_050_safe_code_portals/__init__.py +13 -0
- pythagoras/_050_safe_code_portals/safe_decorator.py +1 -1
- pythagoras/_050_safe_code_portals/safe_portal_core_classes.py +13 -7
- pythagoras/_060_autonomous_code_portals/__init__.py +14 -25
- pythagoras/_060_autonomous_code_portals/autonomous_decorators.py +21 -40
- pythagoras/_060_autonomous_code_portals/autonomous_portal_core_classes.py +18 -6
- pythagoras/_060_autonomous_code_portals/names_usage_analyzer.py +1 -1
- pythagoras/_070_protected_code_portals/__init__.py +17 -1
- pythagoras/_070_protected_code_portals/basic_pre_validators.py +8 -8
- pythagoras/_070_protected_code_portals/package_manager.py +0 -1
- pythagoras/_070_protected_code_portals/protected_decorators.py +2 -2
- pythagoras/_070_protected_code_portals/protected_portal_core_classes.py +55 -21
- pythagoras/_070_protected_code_portals/{OK_const.py → validation_succesful_const.py} +4 -2
- pythagoras/_080_pure_code_portals/__init__.py +4 -18
- pythagoras/_080_pure_code_portals/pure_core_classes.py +32 -19
- pythagoras/_080_pure_code_portals/pure_decorator.py +27 -6
- pythagoras/_090_swarming_portals/__init__.py +11 -0
- pythagoras/_090_swarming_portals/swarming_portals.py +18 -0
- pythagoras/_100_top_level_API/default_local_portal.py +4 -2
- pythagoras/_900_project_stats_collector/__init__.py +1 -0
- pythagoras/_900_project_stats_collector/project_analyzer.py +1 -0
- pythagoras/__init__.py +13 -21
- pythagoras/core/__init__.py +1 -1
- {pythagoras-0.22.1.dist-info → pythagoras-0.23.0.dist-info}/METADATA +1 -1
- {pythagoras-0.22.1.dist-info → pythagoras-0.23.0.dist-info}/RECORD +35 -35
- {pythagoras-0.22.1.dist-info → pythagoras-0.23.0.dist-info}/WHEEL +0 -0
|
@@ -5,7 +5,7 @@ from typing import Callable, Any
|
|
|
5
5
|
|
|
6
6
|
from persidict import PersiDict, Joker, KEEP_CURRENT
|
|
7
7
|
from .._020_ordinary_code_portals.code_normalizer import _pythagoras_decorator_names
|
|
8
|
-
from .. import DataPortal
|
|
8
|
+
from .._030_data_portals import DataPortal
|
|
9
9
|
from .._040_logging_code_portals import KwArgs
|
|
10
10
|
|
|
11
11
|
from .._060_autonomous_code_portals.names_usage_analyzer import (
|
|
@@ -14,9 +14,6 @@ from .._060_autonomous_code_portals.names_usage_analyzer import (
|
|
|
14
14
|
from .._050_safe_code_portals.safe_portal_core_classes import (
|
|
15
15
|
SafeFn, SafeCodePortal)
|
|
16
16
|
|
|
17
|
-
import pythagoras as pth
|
|
18
|
-
|
|
19
|
-
|
|
20
17
|
class AutonomousCodePortal(SafeCodePortal):
|
|
21
18
|
|
|
22
19
|
def __init__(self
|
|
@@ -36,7 +33,7 @@ class AutonomousFn(SafeFn):
|
|
|
36
33
|
_fixed_kwargs_packed: KwArgs | None
|
|
37
34
|
|
|
38
35
|
def __init__(self, fn: Callable|str|SafeFn
|
|
39
|
-
, fixed_kwargs: dict|None = None
|
|
36
|
+
, fixed_kwargs: dict[str,Any]|None = None
|
|
40
37
|
, excessive_logging: bool|Joker = KEEP_CURRENT
|
|
41
38
|
, portal: AutonomousCodePortal|None = None):
|
|
42
39
|
super().__init__(fn=fn
|
|
@@ -106,6 +103,14 @@ class AutonomousFn(SafeFn):
|
|
|
106
103
|
|
|
107
104
|
|
|
108
105
|
def fix_kwargs(self, **kwargs) -> AutonomousFn:
|
|
106
|
+
"""Create a new function by pre-filling some arguments.
|
|
107
|
+
|
|
108
|
+
This is called a partial application in functional programming
|
|
109
|
+
It allows creating specialized functions from general ones by
|
|
110
|
+
transforming a function with multiple parameters
|
|
111
|
+
into another function with fewer parameters by fixing some arguments.
|
|
112
|
+
"""
|
|
113
|
+
|
|
109
114
|
overlapping_keys = set(kwargs.keys()) & set(self.fixed_kwargs.keys())
|
|
110
115
|
assert len(overlapping_keys) == 0
|
|
111
116
|
new_fixed_kwargs = {**self.fixed_kwargs,**kwargs}
|
|
@@ -139,7 +144,14 @@ class AutonomousFn(SafeFn):
|
|
|
139
144
|
|
|
140
145
|
|
|
141
146
|
def _invalidate_cache(self):
|
|
147
|
+
"""Invalidate the function's attribute cache.
|
|
148
|
+
|
|
149
|
+
If the function's attribute named ATTR is cached,
|
|
150
|
+
its cached value will be stored in an attribute named _ATTR_cache
|
|
151
|
+
This method should delete all such attributes.
|
|
152
|
+
"""
|
|
142
153
|
super()._invalidate_cache()
|
|
143
154
|
if hasattr(self, "_fixed_kwargs_cached"):
|
|
144
|
-
assert hasattr(self, "_fixed_kwargs_packed")
|
|
155
|
+
assert (hasattr(self, "_fixed_kwargs_packed")
|
|
156
|
+
, "Premature cache invalidation: fixed_kwargs_packed is missing.")
|
|
145
157
|
del self._fixed_kwargs_cached
|
|
@@ -14,7 +14,7 @@ class NamesUsedInFunction:
|
|
|
14
14
|
self.accessible = set() # all names, currently accessable within the function
|
|
15
15
|
|
|
16
16
|
class NamesUsageAnalyzer(ast.NodeVisitor):
|
|
17
|
-
"""Collect data needed to analyze function
|
|
17
|
+
"""Collect data needed to analyze function autonomicity.
|
|
18
18
|
|
|
19
19
|
This class is a visitor of an AST (Abstract Syntax Tree) that collects data
|
|
20
20
|
needed to analyze function autonomy.
|
|
@@ -1,4 +1,20 @@
|
|
|
1
|
-
|
|
1
|
+
"""Classes and functions that allow protected execution of code.
|
|
2
|
+
|
|
3
|
+
Protected functions are functions that can be executed only if
|
|
4
|
+
certain conditions are met before the execution; also, certain conditions
|
|
5
|
+
must be met after the execution in order for the system to accept
|
|
6
|
+
and use execution results. These conditions are called validators
|
|
7
|
+
(pre-validators and post-validators). A protected function can have many
|
|
8
|
+
pre-validators and post-validators.
|
|
9
|
+
|
|
10
|
+
Validators can be passive (e.g., check if the node has enough RAM)
|
|
11
|
+
or active (e.g., check if some external library is installed, and,
|
|
12
|
+
if not, try to install it). Validators can be rather complex
|
|
13
|
+
(e.g., check if the result, returned by the function, is a valid image).
|
|
14
|
+
Under the hood, validators are autonomous functions.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from .validation_succesful_const import *
|
|
2
18
|
from .protected_portal_core_classes import *
|
|
3
19
|
from .protected_decorators import *
|
|
4
20
|
from .system_utils import *
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
from .._070_protected_code_portals import SimplePreValidatorFn
|
|
2
|
-
from .
|
|
2
|
+
from .validation_succesful_const import ValidationSuccessClass
|
|
3
3
|
|
|
4
4
|
|
|
5
|
-
def _at_least_X_CPU_cores_free_check(n:int)->
|
|
5
|
+
def _at_least_X_CPU_cores_free_check(n:int)-> ValidationSuccessClass | None:
|
|
6
6
|
cores = pth.get_unused_cpu_cores()
|
|
7
7
|
if cores >= n-0.1:
|
|
8
|
-
return pth.
|
|
8
|
+
return pth.VALIDATION_SUCCESSFUL
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def unused_cpu(cores:int) -> SimplePreValidatorFn:
|
|
@@ -14,10 +14,10 @@ def unused_cpu(cores:int) -> SimplePreValidatorFn:
|
|
|
14
14
|
return SimplePreValidatorFn(_at_least_X_CPU_cores_free_check).fix_kwargs(n=cores)
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
def _at_least_X_G_RAM_free_check(x:int)->
|
|
17
|
+
def _at_least_X_G_RAM_free_check(x:int)-> ValidationSuccessClass | None:
|
|
18
18
|
ram = pth.get_unused_ram_mb() / 1024
|
|
19
19
|
if ram >= x-0.1:
|
|
20
|
-
return pth.
|
|
20
|
+
return pth.VALIDATION_SUCCESSFUL
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def unused_ram(Gb:int) -> SimplePreValidatorFn:
|
|
@@ -26,15 +26,15 @@ def unused_ram(Gb:int) -> SimplePreValidatorFn:
|
|
|
26
26
|
return SimplePreValidatorFn(_at_least_X_G_RAM_free_check).fix_kwargs(x=Gb)
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
def _check_python_package_and_install_if_needed(package_name)->
|
|
29
|
+
def _check_python_package_and_install_if_needed(package_name)-> ValidationSuccessClass | None:
|
|
30
30
|
assert isinstance(package_name, str)
|
|
31
31
|
import importlib
|
|
32
32
|
try:
|
|
33
33
|
importlib.import_module(package_name)
|
|
34
|
-
return pth.
|
|
34
|
+
return pth.VALIDATION_SUCCESSFUL
|
|
35
35
|
except:
|
|
36
36
|
pth.install_package(package_name)
|
|
37
|
-
return pth.
|
|
37
|
+
return pth.VALIDATION_SUCCESSFUL
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
def installed_packages(*args) -> list[SimplePreValidatorFn]:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Support for work with protected functions."""
|
|
2
2
|
|
|
3
|
-
from typing import Callable
|
|
3
|
+
from typing import Callable, Any
|
|
4
4
|
|
|
5
5
|
from .validator_fn_classes import ValidatorFn
|
|
6
6
|
from .._060_autonomous_code_portals import autonomous
|
|
@@ -15,7 +15,7 @@ class protected(autonomous):
|
|
|
15
15
|
def __init__(self
|
|
16
16
|
, pre_validators: list[ValidatorFn] | None = None
|
|
17
17
|
, post_validators: list[ValidatorFn] | None = None
|
|
18
|
-
, fixed_kwargs: dict | None = None
|
|
18
|
+
, fixed_kwargs: dict[str,Any] | None = None
|
|
19
19
|
, excessive_logging: bool|Joker = KEEP_CURRENT
|
|
20
20
|
, portal: ProtectedCodePortal | None = None
|
|
21
21
|
):
|
|
@@ -1,23 +1,37 @@
|
|
|
1
|
+
"""Classes and functions that allow protected execution of code.
|
|
2
|
+
|
|
3
|
+
Protected functions are functions that can be executed only if
|
|
4
|
+
certain conditions are met before the execution; also, certain conditions
|
|
5
|
+
must be met after the execution in order for the system to accept
|
|
6
|
+
and use execution results. These conditions are called validators
|
|
7
|
+
(pre-validators and post-validators). A protected function can have many
|
|
8
|
+
pre-validators and post-validators.
|
|
9
|
+
|
|
10
|
+
Validators can be passive (e.g., check if the node has enough RAM)
|
|
11
|
+
or active (e.g., check if some external library is installed, and,
|
|
12
|
+
if not, try to install it). Validators can be rather complex
|
|
13
|
+
(e.g., check if the result, returned by the function, is a valid image).
|
|
14
|
+
Under the hood, validators are autonomous functions.
|
|
15
|
+
"""
|
|
16
|
+
|
|
1
17
|
from __future__ import annotations
|
|
2
18
|
|
|
3
19
|
from copy import copy
|
|
4
|
-
from typing import Callable, Any
|
|
20
|
+
from typing import Callable, Any
|
|
5
21
|
|
|
6
22
|
from persidict import PersiDict, Joker, KEEP_CURRENT
|
|
7
23
|
|
|
8
|
-
from .validator_fn_classes import
|
|
9
|
-
ComplexPreValidatorFn
|
|
24
|
+
from .validator_fn_classes import *
|
|
10
25
|
from .._030_data_portals import DataPortal
|
|
11
26
|
from .._040_logging_code_portals import KwArgs
|
|
12
27
|
from .._030_data_portals import ValueAddr
|
|
13
28
|
from parameterizable import sort_dict_by_keys
|
|
14
29
|
from .list_flattener import flatten_list
|
|
15
|
-
from .
|
|
30
|
+
from .validation_succesful_const import VALIDATION_SUCCESSFUL
|
|
16
31
|
|
|
17
32
|
|
|
18
33
|
from .._060_autonomous_code_portals import (
|
|
19
34
|
AutonomousCodePortal, AutonomousFn)
|
|
20
|
-
from .fn_arg_names_checker import check_if_fn_accepts_args
|
|
21
35
|
|
|
22
36
|
|
|
23
37
|
class ProtectedCodePortal(AutonomousCodePortal):
|
|
@@ -34,8 +48,8 @@ class ProtectedCodePortal(AutonomousCodePortal):
|
|
|
34
48
|
|
|
35
49
|
class ProtectedFn(AutonomousFn):
|
|
36
50
|
|
|
37
|
-
_pre_validators_cached: list[
|
|
38
|
-
_post_validators_cached: list[
|
|
51
|
+
_pre_validators_cached: list[ValidatorFn] | None
|
|
52
|
+
_post_validators_cached: list[ValidatorFn] | None
|
|
39
53
|
_pre_validators_addrs: list[ValueAddr]
|
|
40
54
|
_post_validators_addrs: list[ValueAddr]
|
|
41
55
|
|
|
@@ -43,27 +57,39 @@ class ProtectedFn(AutonomousFn):
|
|
|
43
57
|
pre_validators_arg_names = ["packed_kwargs", "fn_addr"]
|
|
44
58
|
|
|
45
59
|
def __init__(self, fn: Callable | str
|
|
46
|
-
, pre_validators: list[
|
|
47
|
-
, post_validators: list[
|
|
48
|
-
, excessive_logging: bool|
|
|
49
|
-
, fixed_kwargs: dict | None = None
|
|
60
|
+
, pre_validators: list[ValidatorFn] | list[Callable] | None = None
|
|
61
|
+
, post_validators: list[ValidatorFn] | list[Callable] | None = None
|
|
62
|
+
, excessive_logging: bool | Joker = KEEP_CURRENT
|
|
63
|
+
, fixed_kwargs: dict[str,Any] | None = None
|
|
50
64
|
, portal: ProtectedCodePortal | None = None):
|
|
51
65
|
super().__init__(fn=fn
|
|
52
66
|
, portal = portal
|
|
53
67
|
, fixed_kwargs=fixed_kwargs
|
|
54
68
|
, excessive_logging = excessive_logging)
|
|
55
69
|
|
|
56
|
-
pre_validators
|
|
57
|
-
|
|
70
|
+
if pre_validators is None:
|
|
71
|
+
pre_validators = list()
|
|
72
|
+
else:
|
|
73
|
+
pre_validators = copy(pre_validators)
|
|
74
|
+
|
|
75
|
+
if post_validators is None:
|
|
76
|
+
post_validators = list()
|
|
77
|
+
else:
|
|
78
|
+
post_validators = copy(post_validators)
|
|
58
79
|
|
|
59
80
|
if isinstance(fn, ProtectedFn):
|
|
60
81
|
pre_validators += fn.pre_validators
|
|
61
82
|
post_validators += fn.post_validators
|
|
62
83
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
self.
|
|
84
|
+
pre_validators = self._normalize_validators(pre_validators, PreValidatorFn)
|
|
85
|
+
post_validators = self._normalize_validators(post_validators, PostValidatorFn)
|
|
86
|
+
|
|
87
|
+
self._pre_validators_cached = pre_validators
|
|
88
|
+
self._post_validators_cached = post_validators
|
|
89
|
+
self._pre_validators_addrs = [ValueAddr(g, store=False)
|
|
90
|
+
for g in self._pre_validators_cached]
|
|
91
|
+
self._post_validators_addrs = [ValueAddr(v, store=False)
|
|
92
|
+
for v in self._post_validators_cached]
|
|
67
93
|
|
|
68
94
|
|
|
69
95
|
def __getstate__(self):
|
|
@@ -120,7 +146,7 @@ class ProtectedFn(AutonomousFn):
|
|
|
120
146
|
pre_validation_result = pre_validator()
|
|
121
147
|
else:
|
|
122
148
|
pre_validation_result = pre_validator(packed_kwargs=kw_args, fn_addr = self.addr)
|
|
123
|
-
if pre_validation_result is not
|
|
149
|
+
if pre_validation_result is not VALIDATION_SUCCESSFUL:
|
|
124
150
|
return False
|
|
125
151
|
return True
|
|
126
152
|
|
|
@@ -132,7 +158,7 @@ class ProtectedFn(AutonomousFn):
|
|
|
132
158
|
portal.entropy_infuser.shuffle(post_validators)
|
|
133
159
|
for post_validator in post_validators:
|
|
134
160
|
if post_validator(packed_kwargs=kw_args, fn_addr = self.addr
|
|
135
|
-
, result=result) is not
|
|
161
|
+
, result=result) is not VALIDATION_SUCCESSFUL:
|
|
136
162
|
return False
|
|
137
163
|
return True
|
|
138
164
|
|
|
@@ -188,10 +214,18 @@ class ProtectedFn(AutonomousFn):
|
|
|
188
214
|
|
|
189
215
|
|
|
190
216
|
def _invalidate_cache(self):
|
|
217
|
+
"""Invalidate the function's attribute cache.
|
|
218
|
+
|
|
219
|
+
If the function's attribute named ATTR is cached,
|
|
220
|
+
its cached value will be stored in an attribute named _ATTR_cache
|
|
221
|
+
This method should delete all such attributes.
|
|
222
|
+
"""
|
|
191
223
|
super()._invalidate_cache()
|
|
192
224
|
if hasattr(self, "_post_validators_cached"):
|
|
193
|
-
assert hasattr(self, "_post_validators_addrs")
|
|
225
|
+
assert (hasattr(self, "_post_validators_addrs")
|
|
226
|
+
, "Premature cache invalidation: _post_validators_addrs is missing.")
|
|
194
227
|
del self._post_validators_cached
|
|
195
228
|
if hasattr(self, "_pre_validators_cached"):
|
|
196
|
-
assert hasattr(self, "_pre_validators_addrs")
|
|
229
|
+
assert (hasattr(self, "_pre_validators_addrs")
|
|
230
|
+
, "Premature cache invalidation: _pre_validators_addrs is missing.")
|
|
197
231
|
del self._pre_validators_cached
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
class
|
|
1
|
+
class ValidationSuccessClass:
|
|
2
|
+
"""Singleton class to represent a successful validation."""
|
|
3
|
+
|
|
2
4
|
_instance = None
|
|
3
5
|
|
|
4
6
|
def __new__(cls):
|
|
@@ -6,4 +8,4 @@ class OKClass:
|
|
|
6
8
|
cls._instance = super().__new__(cls)
|
|
7
9
|
return cls._instance
|
|
8
10
|
|
|
9
|
-
|
|
11
|
+
VALIDATION_SUCCESSFUL = ValidationSuccessClass()
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"""Decorators and utilities to work with pure functions.
|
|
2
2
|
|
|
3
|
-
A pure function is
|
|
3
|
+
A pure function is a protected function that has no side effects and
|
|
4
4
|
always returns the same result if it is called multiple times
|
|
5
5
|
with the same arguments.
|
|
6
6
|
|
|
7
|
-
This
|
|
7
|
+
This subpackage defines a decorator which is used to inform Pythagoras that
|
|
8
8
|
a function is intended to be pure: @pure().
|
|
9
9
|
|
|
10
10
|
Pythagoras persistently caches results, produced by a pure function, so that
|
|
@@ -16,22 +16,8 @@ While caching the results of a pure function, Pythagoras also tracks
|
|
|
16
16
|
changes in the source code of the function. If the source code of a pure
|
|
17
17
|
function changes, the function is executed again on the next call.
|
|
18
18
|
However, the previously cached results are still available
|
|
19
|
-
for the old version of the function.
|
|
20
|
-
|
|
21
|
-
A pure function must be autonomous. Pythagoras tracks source code changes
|
|
22
|
-
for the pure function as well other autonomous functions it is using,
|
|
23
|
-
provided they belong to the same island. Pythagoras does not track source code
|
|
24
|
-
changes for functions from other islands, even if they are used
|
|
25
|
-
by the pure function. Pythagoras also does not track any other
|
|
26
|
-
source code changes (e.g. changes in the imported packages).
|
|
27
|
-
|
|
28
|
-
Pythagoras provides infrastructure for remote execution of
|
|
29
|
-
pure functions in distributed environments. Pythagoras employs
|
|
30
|
-
an asynchronous execution model called 'swarming':
|
|
31
|
-
you do not know when your function will be executed,
|
|
32
|
-
what machine will execute it, and how many times it will be executed.
|
|
33
|
-
Pythagoras ensures that the function will be eventually executed
|
|
34
|
-
at least once, but does not offer any further guarantees.
|
|
19
|
+
for the old version of the function. Only changes in the function's
|
|
20
|
+
source code are tracked.
|
|
35
21
|
"""
|
|
36
22
|
|
|
37
23
|
from .pure_core_classes import (
|
|
@@ -1,3 +1,25 @@
|
|
|
1
|
+
"""Classes to work with pure functions.
|
|
2
|
+
|
|
3
|
+
A pure function is a protected function that has no side effects and
|
|
4
|
+
always returns the same result if it is called multiple times
|
|
5
|
+
with the same arguments.
|
|
6
|
+
|
|
7
|
+
This subpackage defines a decorator which is used to inform Pythagoras that
|
|
8
|
+
a function is intended to be pure: @pure().
|
|
9
|
+
|
|
10
|
+
Pythagoras persistently caches results, produced by a pure function, so that
|
|
11
|
+
if the function is called multiple times with the same arguments,
|
|
12
|
+
the function is executed only once, and the cached result is returned
|
|
13
|
+
for all the subsequent executions.
|
|
14
|
+
|
|
15
|
+
While caching the results of a pure function, Pythagoras also tracks
|
|
16
|
+
changes in the source code of the function. If the source code of a pure
|
|
17
|
+
function changes, the function is executed again on the next call.
|
|
18
|
+
However, the previously cached results are still available
|
|
19
|
+
for the old version of the function. Only changes in the function's
|
|
20
|
+
source code are tracked.
|
|
21
|
+
"""
|
|
22
|
+
|
|
1
23
|
from __future__ import annotations
|
|
2
24
|
|
|
3
25
|
import time
|
|
@@ -9,22 +31,19 @@ import pandas as pd
|
|
|
9
31
|
|
|
10
32
|
from persidict import PersiDict, Joker, KEEP_CURRENT
|
|
11
33
|
|
|
12
|
-
from
|
|
13
|
-
|
|
34
|
+
from persidict import WriteOnceDict
|
|
35
|
+
|
|
36
|
+
from .._010_basic_portals import *
|
|
14
37
|
from .._010_basic_portals.basic_portal_core_classes import (
|
|
15
38
|
_describe_persistent_characteristic)
|
|
16
|
-
from persidict import WriteOnceDict
|
|
17
|
-
from .._040_logging_code_portals.logging_portal_core_classes import (
|
|
18
|
-
LoggingFnCallSignature)
|
|
19
39
|
|
|
20
40
|
from .._030_data_portals import HashAddr, ValueAddr
|
|
21
|
-
|
|
22
|
-
from .._060_autonomous_code_portals.autonomous_portal_core_classes import (
|
|
23
|
-
AutonomousFn)
|
|
41
|
+
from .._040_logging_code_portals import *
|
|
24
42
|
|
|
25
43
|
|
|
26
|
-
from ..
|
|
44
|
+
from .._070_protected_code_portals import *
|
|
27
45
|
|
|
46
|
+
# from .._040_logging_code_portals.kw_args import KwArgs
|
|
28
47
|
|
|
29
48
|
ASupportingFunc:TypeAlias = str | AutonomousFn
|
|
30
49
|
|
|
@@ -232,23 +251,16 @@ class PureFn(ProtectedFn):
|
|
|
232
251
|
return ProtectedFn.portal.__get__(self)
|
|
233
252
|
|
|
234
253
|
|
|
235
|
-
# @portal.setter
|
|
236
|
-
# def portal(self, new_portal: PureCodePortal) -> None: #*#*#
|
|
237
|
-
# if not isinstance(new_portal, PureCodePortal):
|
|
238
|
-
# raise TypeError("portal must be a PureCodePortal instance")
|
|
239
|
-
# ProtectedFn.portal.__set__(self, new_portal)
|
|
240
|
-
|
|
241
|
-
|
|
242
254
|
class PureFnExecutionResultAddr(HashAddr):
|
|
243
|
-
"""An address of
|
|
255
|
+
"""An address of a (future) result of pure function execution.
|
|
244
256
|
|
|
245
257
|
This class is used to point to the result of an execution of a pure
|
|
246
|
-
function in a portal. The address is used to request an execution
|
|
258
|
+
function in a portal. The address is used to request an execution or
|
|
247
259
|
to retrieve the result (if available) from the portal.
|
|
248
260
|
|
|
249
261
|
The address also provides access to various logs and records of the
|
|
250
262
|
function execution, such as environmental contexts of the execution attempts,
|
|
251
|
-
outputs printed, exceptions thrown and events emitted.
|
|
263
|
+
outputs printed, exceptions thrown, and events emitted.
|
|
252
264
|
"""
|
|
253
265
|
_fn_cache: PureFn | None
|
|
254
266
|
_call_signature_cache: LoggingFnCallSignature | None
|
|
@@ -287,6 +299,7 @@ class PureFnExecutionResultAddr(HashAddr):
|
|
|
287
299
|
del self._kwargs_cache
|
|
288
300
|
if hasattr(self, "_call_signature_cache"):
|
|
289
301
|
del self._call_signature_cache
|
|
302
|
+
super()._invalidate_cache()
|
|
290
303
|
|
|
291
304
|
|
|
292
305
|
def get_ValueAddr(self):
|
|
@@ -1,7 +1,28 @@
|
|
|
1
|
-
|
|
1
|
+
"""Decorator to work with pure functions.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
A pure function is a protected function that has no side effects and
|
|
4
|
+
always returns the same result if it is called multiple times
|
|
5
|
+
with the same arguments.
|
|
6
|
+
|
|
7
|
+
This subpackage defines a decorator which is used to inform Pythagoras that
|
|
8
|
+
a function is intended to be pure: @pure().
|
|
9
|
+
|
|
10
|
+
Pythagoras persistently caches results, produced by a pure function, so that
|
|
11
|
+
if the function is called multiple times with the same arguments,
|
|
12
|
+
the function is executed only once, and the cached result is returned
|
|
13
|
+
for all the subsequent executions.
|
|
14
|
+
|
|
15
|
+
While caching the results of a pure function, Pythagoras also tracks
|
|
16
|
+
changes in the source code of the function. If the source code of a pure
|
|
17
|
+
function changes, the function is executed again on the next call.
|
|
18
|
+
However, the previously cached results are still available
|
|
19
|
+
for the old version of the function. Only changes in the function's
|
|
20
|
+
source code are tracked.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from typing import Callable, Any
|
|
24
|
+
|
|
25
|
+
from .._070_protected_code_portals import protected, ValidatorFn
|
|
5
26
|
from .._080_pure_code_portals.pure_core_classes import (
|
|
6
27
|
PureCodePortal, PureFn)
|
|
7
28
|
|
|
@@ -10,9 +31,9 @@ from persidict import KEEP_CURRENT, Joker
|
|
|
10
31
|
class pure(protected):
|
|
11
32
|
|
|
12
33
|
def __init__(self
|
|
13
|
-
, pre_validators: list[
|
|
14
|
-
, post_validators: list[
|
|
15
|
-
, fixed_kwargs: dict | None = None
|
|
34
|
+
, pre_validators: list[ValidatorFn] | None = None
|
|
35
|
+
, post_validators: list[ValidatorFn] | None = None
|
|
36
|
+
, fixed_kwargs: dict[str, Any] | None = None
|
|
16
37
|
, excessive_logging: bool | Joker = KEEP_CURRENT
|
|
17
38
|
, portal: PureCodePortal | None = None
|
|
18
39
|
):
|
|
@@ -1 +1,12 @@
|
|
|
1
|
+
""" Classes and functions that enable swarming algorithm.
|
|
2
|
+
|
|
3
|
+
Pythagoras provides infrastructure for remote execution of
|
|
4
|
+
pure functions in distributed environments. Pythagoras employs
|
|
5
|
+
an asynchronous execution model called 'swarming':
|
|
6
|
+
you do not know when your function will be executed,
|
|
7
|
+
what machine will execute it, and how many times it will be executed.
|
|
8
|
+
Pythagoras ensures that the function will be eventually executed
|
|
9
|
+
at least once but does not offer any further guarantees.
|
|
10
|
+
"""
|
|
11
|
+
|
|
1
12
|
from .swarming_portals import *
|
|
@@ -1,3 +1,14 @@
|
|
|
1
|
+
""" Classes and functions that enable swarming algorithm.
|
|
2
|
+
|
|
3
|
+
Pythagoras provides infrastructure for remote execution of
|
|
4
|
+
pure functions in distributed environments. Pythagoras employs
|
|
5
|
+
an asynchronous execution model called 'swarming':
|
|
6
|
+
you do not know when your function will be executed,
|
|
7
|
+
what machine will execute it, and how many times it will be executed.
|
|
8
|
+
Pythagoras ensures that the function will be eventually executed
|
|
9
|
+
at least once but does not offer any further guarantees.
|
|
10
|
+
"""
|
|
11
|
+
|
|
1
12
|
from __future__ import annotations
|
|
2
13
|
|
|
3
14
|
import atexit
|
|
@@ -176,8 +187,15 @@ class SwarmingPortal(PureCodePortal):
|
|
|
176
187
|
|
|
177
188
|
|
|
178
189
|
def _invalidate_cache(self):
|
|
190
|
+
"""Invalidate the object's attribute cache.
|
|
191
|
+
|
|
192
|
+
If the object's attribute named ATTR is cached,
|
|
193
|
+
its cached value will be stored in an attribute named _ATTR_cache
|
|
194
|
+
This method should delete all such attributes.
|
|
195
|
+
"""
|
|
179
196
|
if hasattr(self, "_max_n_workers_cache"):
|
|
180
197
|
del self._max_n_workers_cache
|
|
198
|
+
super()._invalidate_cache()
|
|
181
199
|
|
|
182
200
|
parameterizable.register_parameterizable_class(SwarmingPortal)
|
|
183
201
|
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
from ..
|
|
1
|
+
from .._010_basic_portals import get_active_portal
|
|
2
2
|
from .._010_basic_portals import get_default_portal_base_dir
|
|
3
|
+
from .._090_swarming_portals import SwarmingPortal
|
|
3
4
|
|
|
4
5
|
|
|
5
|
-
def
|
|
6
|
+
def _instantiate_default_local_portal():
|
|
7
|
+
#NB: there is a hidden circular dependency from get_active_portal()
|
|
6
8
|
SwarmingPortal(root_dict = get_default_portal_base_dir())
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""This subpackage is only needed for development. No runtime usage."""
|
pythagoras/__init__.py
CHANGED
|
@@ -1,9 +1,17 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Distributed serverless compute at a global scale.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
Pythagoras is a framework for distributed compute in Python.
|
|
4
|
+
|
|
5
|
+
It offers 3 main advantages:
|
|
6
|
+
|
|
7
|
+
- Global scale: parallelize your algorithms to scale to millions of nodes.
|
|
8
|
+
- Low maintenance: no need to manage servers and infrastructure,
|
|
9
|
+
we replace expensive compute with cheap storage.
|
|
10
|
+
- High performance: 'compute once, reuse forever' strategy
|
|
11
|
+
significanty accelerates long-running workflows.
|
|
12
|
+
|
|
13
|
+
Pythagoras is able to affer these benefits as it's the first framework
|
|
14
|
+
to fully implement the Functional Programming 2.0 paradigm.
|
|
7
15
|
"""
|
|
8
16
|
|
|
9
17
|
|
|
@@ -21,19 +29,3 @@ from ._100_top_level_API import *
|
|
|
21
29
|
from ._800_signatures_and_converters import *
|
|
22
30
|
|
|
23
31
|
|
|
24
|
-
# primary_decorators = {d.__name__:d for d in [
|
|
25
|
-
# autonomous
|
|
26
|
-
# , pure
|
|
27
|
-
# ]}
|
|
28
|
-
#
|
|
29
|
-
# all_decorators = {d.__name__:d for d in [
|
|
30
|
-
# ordinary
|
|
31
|
-
# , storable
|
|
32
|
-
# , logging
|
|
33
|
-
# , safe
|
|
34
|
-
# , autonomous
|
|
35
|
-
# , protected
|
|
36
|
-
# , pure
|
|
37
|
-
# ]}
|
|
38
|
-
|
|
39
|
-
|
pythagoras/core/__init__.py
CHANGED
|
@@ -3,4 +3,4 @@ from .._060_autonomous_code_portals import autonomous
|
|
|
3
3
|
from .._080_pure_code_portals import pure
|
|
4
4
|
from .._090_swarming_portals import SwarmingPortal
|
|
5
5
|
from .._100_top_level_API import get_portal
|
|
6
|
-
from .._070_protected_code_portals import
|
|
6
|
+
from .._070_protected_code_portals.basic_pre_validators import *
|