pythagoras 0.22.2__py3-none-any.whl → 0.23.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pythagoras/_010_basic_portals/basic_portal_core_classes.py +1 -1
- pythagoras/_020_ordinary_code_portals/ordinary_portal_core_classes.py +2 -2
- pythagoras/_030_data_portals/__init__.py +4 -4
- pythagoras/_030_data_portals/data_portal_core_classes.py +9 -6
- pythagoras/_040_logging_code_portals/__init__.py +7 -13
- pythagoras/_040_logging_code_portals/exception_processing_tracking.py +6 -4
- pythagoras/_040_logging_code_portals/kw_args.py +1 -1
- pythagoras/_040_logging_code_portals/logging_portal_core_classes.py +1 -3
- pythagoras/_050_safe_code_portals/__init__.py +13 -0
- pythagoras/_050_safe_code_portals/safe_decorator.py +1 -1
- pythagoras/_050_safe_code_portals/safe_portal_core_classes.py +13 -7
- pythagoras/_060_autonomous_code_portals/__init__.py +14 -25
- pythagoras/_060_autonomous_code_portals/autonomous_decorators.py +21 -40
- pythagoras/_060_autonomous_code_portals/autonomous_portal_core_classes.py +12 -6
- pythagoras/_060_autonomous_code_portals/names_usage_analyzer.py +1 -1
- pythagoras/_070_protected_code_portals/__init__.py +17 -1
- pythagoras/_070_protected_code_portals/basic_pre_validators.py +8 -8
- pythagoras/_070_protected_code_portals/package_manager.py +0 -1
- pythagoras/_070_protected_code_portals/protected_decorators.py +2 -2
- pythagoras/_070_protected_code_portals/protected_portal_core_classes.py +45 -19
- pythagoras/_070_protected_code_portals/{OK_const.py → validation_succesful_const.py} +4 -2
- pythagoras/_080_pure_code_portals/__init__.py +4 -18
- pythagoras/_080_pure_code_portals/pure_core_classes.py +31 -19
- pythagoras/_080_pure_code_portals/pure_decorator.py +27 -6
- pythagoras/_090_swarming_portals/__init__.py +11 -0
- pythagoras/_090_swarming_portals/swarming_portals.py +11 -0
- pythagoras/_100_top_level_API/default_local_portal.py +4 -2
- pythagoras/_900_project_stats_collector/__init__.py +1 -0
- pythagoras/_900_project_stats_collector/project_analyzer.py +1 -0
- pythagoras/__init__.py +17 -21
- pythagoras/core/__init__.py +1 -1
- {pythagoras-0.22.2.dist-info → pythagoras-0.23.1.dist-info}/METADATA +1 -1
- {pythagoras-0.22.2.dist-info → pythagoras-0.23.1.dist-info}/RECORD +34 -34
- {pythagoras-0.22.2.dist-info → pythagoras-0.23.1.dist-info}/WHEEL +0 -0
|
@@ -123,7 +123,7 @@ def get_active_portal() -> BasicPortal:
|
|
|
123
123
|
return _active_portals_stack[-1]
|
|
124
124
|
|
|
125
125
|
if _most_recently_created_portal is None:
|
|
126
|
-
sys.modules["pythagoras"].
|
|
126
|
+
sys.modules["pythagoras"]._instantiate_default_local_portal()
|
|
127
127
|
|
|
128
128
|
_active_portals_stack.append(_most_recently_created_portal)
|
|
129
129
|
_active_portals_counters_stack.append(1)
|
|
@@ -15,7 +15,7 @@ from .._010_basic_portals.basic_portal_core_classes import (
|
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
def get_normalized_function_source(a_func: OrdinaryFn | Callable | str) -> str:
|
|
18
|
-
"""Return function's source code in a 'canonical' form.
|
|
18
|
+
"""Return a function's source code in a 'canonical' form.
|
|
19
19
|
|
|
20
20
|
Remove all comments, docstrings and empty lines;
|
|
21
21
|
standardize code formatting based on PEP 8.
|
|
@@ -263,7 +263,7 @@ class OrdinaryFn(PortalAwareClass):
|
|
|
263
263
|
|
|
264
264
|
def __setstate__(self, state):
|
|
265
265
|
"""This method is called when the object is unpickled."""
|
|
266
|
-
|
|
266
|
+
super().__setstate__(state)
|
|
267
267
|
self._source_code = state["source_code"]
|
|
268
268
|
|
|
269
269
|
|
|
@@ -22,10 +22,10 @@ This is done to address limitations of some file systems
|
|
|
22
22
|
and to optimize work sith cloud storage (e.g. S3).
|
|
23
23
|
|
|
24
24
|
Typically, a DataPortal is implemented as
|
|
25
|
-
a shared directory on a file system (e.g
|
|
26
|
-
or as a shared bucket in a cloud storage (e.g
|
|
27
|
-
In this case, a ValueAddr becomes a part of file path
|
|
28
|
-
or a URL (e.g
|
|
25
|
+
a shared directory on a file system (e.g., Amazon EFS),
|
|
26
|
+
or as a shared bucket in a cloud storage (e.g., Amazon S3).
|
|
27
|
+
In this case, a ValueAddr becomes a part of a file path
|
|
28
|
+
or a URL (e.g., a hash serves as a filename,
|
|
29
29
|
and a prefix is a folder name).
|
|
30
30
|
"""
|
|
31
31
|
|
|
@@ -419,7 +419,7 @@ class ValueAddr(HashAddr):
|
|
|
419
419
|
ValueAddr is a universal global identifier of any (constant) value.
|
|
420
420
|
|
|
421
421
|
Using only the value's hash should (theoretically) be enough to
|
|
422
|
-
uniquely address all possible data objects that
|
|
422
|
+
uniquely address all possible data objects that humanity will create
|
|
423
423
|
in the foreseeable future (see, for example, ipfs.io).
|
|
424
424
|
|
|
425
425
|
However, an address also includes a descriptor with an optional suffix.
|
|
@@ -437,8 +437,8 @@ class ValueAddr(HashAddr):
|
|
|
437
437
|
descriptor = data_value_addr.descriptor
|
|
438
438
|
hash_signature = data_value_addr.hash_signature
|
|
439
439
|
HashAddr.__init__(self
|
|
440
|
-
|
|
441
|
-
|
|
440
|
+
, descriptor=descriptor
|
|
441
|
+
, hash_signature=hash_signature)
|
|
442
442
|
return
|
|
443
443
|
|
|
444
444
|
assert not isinstance(data, HashAddr), (
|
|
@@ -448,8 +448,8 @@ class ValueAddr(HashAddr):
|
|
|
448
448
|
descriptor = self._build_descriptor(data)
|
|
449
449
|
hash_signature = self._build_hash_signature(data)
|
|
450
450
|
HashAddr.__init__(self
|
|
451
|
-
|
|
452
|
-
|
|
451
|
+
, descriptor=descriptor
|
|
452
|
+
, hash_signature=hash_signature)
|
|
453
453
|
|
|
454
454
|
self._value_cache = data
|
|
455
455
|
|
|
@@ -584,7 +584,10 @@ class ValueAddr(HashAddr):
|
|
|
584
584
|
) -> HashAddr:
|
|
585
585
|
"""(Re)construct address from text representations of descriptor and hash"""
|
|
586
586
|
|
|
587
|
-
address = super().from_strings(
|
|
587
|
+
address = super().from_strings(
|
|
588
|
+
descriptor=descriptor
|
|
589
|
+
, hash_signature=hash_signature
|
|
590
|
+
, assert_readiness=False)
|
|
588
591
|
address._containing_portals = set()
|
|
589
592
|
if assert_readiness:
|
|
590
593
|
if not address.ready:
|
|
@@ -1,24 +1,18 @@
|
|
|
1
1
|
"""Classes and functions to work with application-level logging.
|
|
2
2
|
|
|
3
|
-
The main class in this sub-package is LoggingCodePortal, which extends
|
|
3
|
+
The main class in this sub-package is LoggingCodePortal, which extends DataPortal
|
|
4
4
|
to provide application-level logging capabilities for events and exceptions.
|
|
5
5
|
'Application-level' means that the events and exceptions are logged into
|
|
6
|
-
location(s) that is(are) the same across the entire application
|
|
7
|
-
and does(do) not depend on the specific function from which
|
|
8
|
-
the even or exception is originated.
|
|
6
|
+
location(s) that is(are) the same across the entire application.
|
|
9
7
|
|
|
10
|
-
|
|
11
|
-
which are persistent dictionaries (PersiDict-s) that store
|
|
12
|
-
the exceptions
|
|
8
|
+
LoggingCodePortal provides three attributes, _run_history, _crash_history,
|
|
9
|
+
and _event_history`, which are persistent dictionaries (PersiDict-s) that store
|
|
10
|
+
the exceptions and logged / recorded events.
|
|
13
11
|
|
|
14
|
-
|
|
12
|
+
Functions log_exception() and `log_event() are provided to log
|
|
15
13
|
exceptions and events. These methods are designed to be
|
|
16
14
|
called from anywhere in the application, and they will log the exception
|
|
17
|
-
or event into
|
|
18
|
-
those that have been registered with the current
|
|
19
|
-
stack of nested 'with' statements.
|
|
20
|
-
|
|
21
|
-
The class also supports logging uncaught exceptions globally.
|
|
15
|
+
or event into the best suitable LoggingPortal.
|
|
22
16
|
"""
|
|
23
17
|
|
|
24
18
|
from .logging_portal_core_classes import *
|
|
@@ -1,22 +1,24 @@
|
|
|
1
1
|
def _exception_needs_to_be_processed(exc_type, exc_value, trace_back) -> bool:
|
|
2
|
+
"""Chack if the exception needs to be logged by Pythagoras"""
|
|
2
3
|
if exc_type is None:
|
|
3
4
|
return False
|
|
4
5
|
|
|
5
6
|
if hasattr(exc_value, "__notes__"):
|
|
6
|
-
if "
|
|
7
|
+
if "__suppress_pythagoras_logging__" in exc_value.__notes__:
|
|
7
8
|
return False
|
|
8
9
|
else:
|
|
9
10
|
return True
|
|
10
11
|
|
|
11
|
-
if hasattr(exc_value, "
|
|
12
|
+
if hasattr(exc_value, "__suppress_pythagoras_logging__"):
|
|
12
13
|
return False
|
|
13
14
|
|
|
14
15
|
return True
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
def _mark_exception_as_processed(exc_type, exc_value, trace_back) -> None:
|
|
19
|
+
"""Mark the exception as processed by Pythagoras"""
|
|
18
20
|
if hasattr(exc_value, "add_note"):
|
|
19
21
|
exc_value.add_note(
|
|
20
|
-
"
|
|
22
|
+
"__suppress_pythagoras_logging__")
|
|
21
23
|
else:
|
|
22
|
-
exc_value.
|
|
24
|
+
exc_value.__suppress_pythagoras_logging__ = True
|
|
@@ -7,7 +7,7 @@ from parameterizable import sort_dict_by_keys
|
|
|
7
7
|
class KwArgs(dict):
|
|
8
8
|
""" A class that encapsulates keyword arguments for a function call.
|
|
9
9
|
|
|
10
|
-
It allows
|
|
10
|
+
It allows "normalizing" the dictionary by sorting the keys
|
|
11
11
|
and replacing values with their hash addresses
|
|
12
12
|
in order to always get the same hash values
|
|
13
13
|
for the same lists of arguments.
|
|
@@ -75,7 +75,6 @@ class LoggingFn(StorableFn):
|
|
|
75
75
|
return StorableFn.portal.__get__(self)
|
|
76
76
|
|
|
77
77
|
|
|
78
|
-
|
|
79
78
|
class LoggingFnCallSignature:
|
|
80
79
|
"""A signature of a call to a (logging) function.
|
|
81
80
|
|
|
@@ -389,7 +388,6 @@ class LoggingFnExecutionRecord(NotPicklable):
|
|
|
389
388
|
f"{self.call_signature.fn_name} execution results.")
|
|
390
389
|
|
|
391
390
|
|
|
392
|
-
|
|
393
391
|
class LoggingFnExecutionFrame(NotPicklable):
|
|
394
392
|
call_stack: list[LoggingFnExecutionFrame] = []
|
|
395
393
|
|
|
@@ -512,7 +510,7 @@ class LoggingCodePortal(DataPortal):
|
|
|
512
510
|
from which the event or exception is raised.
|
|
513
511
|
|
|
514
512
|
The class provides two dictionaries, `_crash_history` and `event_log`,
|
|
515
|
-
to store the
|
|
513
|
+
to store the exception history and event log respectively.
|
|
516
514
|
|
|
517
515
|
Static methods `log_exception` and `log_event` are provided to log
|
|
518
516
|
exceptions and events. These methods are designed to be
|
|
@@ -1,2 +1,15 @@
|
|
|
1
|
+
"""Classes and functions that allow safe execution of code.
|
|
2
|
+
|
|
3
|
+
The main classes in this sub-package are SafeCodePortal and SafeFn,
|
|
4
|
+
which extend LoggingCodePortal and LoggingFn
|
|
5
|
+
to provide safe execution capabilities for logging functions.
|
|
6
|
+
|
|
7
|
+
SafeFn functions can't access (hence can't harm) any data/devices outside
|
|
8
|
+
the function's local scope and the portal.
|
|
9
|
+
|
|
10
|
+
This functionality has not been implemented yet.
|
|
11
|
+
It will be done soon by integrating https://pypi.org/project/RestrictedPython/
|
|
12
|
+
"""
|
|
13
|
+
|
|
1
14
|
from .safe_portal_core_classes import *
|
|
2
15
|
from .safe_decorator import *
|
|
@@ -1,3 +1,16 @@
|
|
|
1
|
+
"""Classes and functions that allow safe execution of code.
|
|
2
|
+
|
|
3
|
+
The main classes in this sub-package are SafeCodePortal and SafeFn,
|
|
4
|
+
which extend LoggingCodePortal and LoggingFn
|
|
5
|
+
to provide safe execution capabilities for logging functions.
|
|
6
|
+
|
|
7
|
+
SafeFn functions can't access (hence can't harm) any data/devices outside
|
|
8
|
+
the function's local scope and the portal.
|
|
9
|
+
|
|
10
|
+
This functionality has not been implemented yet.
|
|
11
|
+
It will be done soon by integrating https://pypi.org/project/RestrictedPython/
|
|
12
|
+
"""
|
|
13
|
+
|
|
1
14
|
from __future__ import annotations
|
|
2
15
|
|
|
3
16
|
from typing import Callable
|
|
@@ -49,11 +62,4 @@ class SafeFn(LoggingFn):
|
|
|
49
62
|
return LoggingFn.portal.__get__(self)
|
|
50
63
|
|
|
51
64
|
|
|
52
|
-
# @portal.setter
|
|
53
|
-
# def portal(self, new_portal: SafeCodePortal) -> None:
|
|
54
|
-
# if not isinstance(new_portal, SafeCodePortal):
|
|
55
|
-
# raise TypeError("portal must be a LoggingCodePortal instance")
|
|
56
|
-
# LoggingFn.portal.__set__(self, new_portal)
|
|
57
|
-
|
|
58
|
-
|
|
59
65
|
register_parameterizable_class(SafeCodePortal)
|
|
@@ -1,46 +1,35 @@
|
|
|
1
1
|
"""Classes and utilities to work with autonomous functions.
|
|
2
2
|
|
|
3
|
-
In
|
|
3
|
+
In essence, an 'autonomous' function contains self-sufficient code
|
|
4
4
|
that does not depend on external imports or definitions. All required
|
|
5
|
-
imports should be done inside the function body.
|
|
6
|
-
can be autonomous.
|
|
5
|
+
imports should be done inside the function body.
|
|
6
|
+
Only ordinary functions can be autonomous.
|
|
7
7
|
|
|
8
8
|
Autonomous functions are always allowed to use the built-in objects
|
|
9
9
|
(functions, types, variables), as well as global objects,
|
|
10
10
|
explicitly imported inside the function body. An autonomous function
|
|
11
|
-
is allowed to use other autonomous functions
|
|
12
|
-
|
|
13
|
-
if they belong to the same island.
|
|
14
|
-
|
|
15
|
-
An island is namespace that groups autonomous functions together.
|
|
16
|
-
An autonomous function can use other autonomous functions from the same island
|
|
17
|
-
without explicitly importing them.
|
|
11
|
+
is allowed to use other autonomous functions if they are passed as
|
|
12
|
+
input arguments to the function.
|
|
18
13
|
|
|
19
14
|
Autonomous functions are not allowed to:
|
|
20
15
|
|
|
21
16
|
* use global objects, imported or defined outside the function body
|
|
22
|
-
(except built-in objects
|
|
23
|
-
from the same island);
|
|
17
|
+
(except built-in objects);
|
|
24
18
|
* use yield (yield from) statements;
|
|
25
19
|
* use nonlocal variables, referencing the outside objects.
|
|
26
20
|
|
|
27
|
-
If an autonomous function is using other autonomous functions
|
|
28
|
-
from the same island, it is called "loosely autonomous function".
|
|
29
|
-
Otherwise, it is called "strictly autonomous function".
|
|
30
|
-
|
|
31
21
|
Autonomous functions can have nested functions and classes.
|
|
32
22
|
|
|
33
|
-
Only
|
|
34
|
-
class methods and lambda functions cannot be autonomous.
|
|
23
|
+
Only ordinary functions can be autonomous. Asynchronous functions, closures,
|
|
24
|
+
class methods, and lambda functions cannot be autonomous.
|
|
35
25
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
@autonomous() and @strictly_autonomous() .
|
|
26
|
+
Autonomous functions support partial application of arguments:
|
|
27
|
+
the process of pre-filling some arguments of a function,
|
|
28
|
+
producing a new autonomous function that takes the remaining arguments.
|
|
40
29
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
30
|
+
This module defines a decorator which is used to
|
|
31
|
+
inform Pythagoras that a function is intended to be autonomous
|
|
32
|
+
and to enforce autonomicity requirements.
|
|
44
33
|
|
|
45
34
|
Applying a decorator to a function ensures both static and runtime autonomicity
|
|
46
35
|
checks are performed for the function. Static checks happen at the time
|
|
@@ -1,33 +1,39 @@
|
|
|
1
1
|
"""Support for work with autonomous functions.
|
|
2
2
|
|
|
3
|
-
In essence, an autonomous function contains self-sufficient code
|
|
4
|
-
that does not depend on external imports or definitions.
|
|
3
|
+
In essence, an 'autonomous' function contains self-sufficient code
|
|
4
|
+
that does not depend on external imports or definitions. All required
|
|
5
|
+
imports should be done inside the function body.
|
|
6
|
+
Only ordinary functions can be autonomous.
|
|
5
7
|
|
|
6
8
|
Autonomous functions are always allowed to use the built-in objects
|
|
7
9
|
(functions, types, variables), as well as global objects,
|
|
8
10
|
explicitly imported inside the function body. An autonomous function
|
|
9
|
-
|
|
10
|
-
|
|
11
|
+
is allowed to use other autonomous functions if they are passed as
|
|
12
|
+
input arguments to the function.
|
|
11
13
|
|
|
12
14
|
Autonomous functions are not allowed to:
|
|
15
|
+
|
|
13
16
|
* use global objects, imported or defined outside the function body
|
|
14
|
-
(except built-in objects
|
|
15
|
-
other autonomous functions from the same island);
|
|
17
|
+
(except built-in objects);
|
|
16
18
|
* use yield (yield from) statements;
|
|
17
19
|
* use nonlocal variables, referencing the outside objects.
|
|
18
20
|
|
|
19
|
-
If an autonomous function is allowed to use other autonomous functions,
|
|
20
|
-
it is called "loosely autonomous function". Otherwise, it is called
|
|
21
|
-
"strictly autonomous function".
|
|
22
|
-
|
|
23
21
|
Autonomous functions can have nested functions and classes.
|
|
24
22
|
|
|
25
|
-
Only ordinary functions can be autonomous. Asynchronous functions,
|
|
26
|
-
class methods and lambda functions cannot be autonomous.
|
|
23
|
+
Only ordinary functions can be autonomous. Asynchronous functions, closures,
|
|
24
|
+
class methods, and lambda functions cannot be autonomous.
|
|
25
|
+
|
|
26
|
+
Autonomous functions support partial application of arguments:
|
|
27
|
+
the process of pre-filling some arguments of a function,
|
|
28
|
+
producing a new autonomous function that takes the remaining arguments.
|
|
27
29
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
and to enforce autonomicity requirements
|
|
30
|
+
This module defines a decorator which is used to
|
|
31
|
+
inform Pythagoras that a function is intended to be autonomous
|
|
32
|
+
and to enforce autonomicity requirements.
|
|
33
|
+
|
|
34
|
+
Applying a decorator to a function ensures both static and runtime autonomicity
|
|
35
|
+
checks are performed for the function. Static checks happen at the time
|
|
36
|
+
of decoration, while runtime checks happen at the time of function execution.
|
|
31
37
|
"""
|
|
32
38
|
from typing import Callable
|
|
33
39
|
|
|
@@ -59,31 +65,6 @@ class autonomous(safe):
|
|
|
59
65
|
|
|
60
66
|
|
|
61
67
|
def __call__(self, fn: Callable|str) -> AutonomousFn:
|
|
62
|
-
"""Decorator for autonomous functions.
|
|
63
|
-
|
|
64
|
-
It does both static and dynamic checks for autonomous functions.
|
|
65
|
-
|
|
66
|
-
Static checks: it checks whether the function uses any global
|
|
67
|
-
non-built-in objects which do not have associated import statements
|
|
68
|
-
inside the function. If allow_idempotent==True,
|
|
69
|
-
global idempotent functions are also allowed.
|
|
70
|
-
The decorator also checks whether the function is using
|
|
71
|
-
any non-local objects variables, and whether the function
|
|
72
|
-
has yield / yield from statements in its code. If static checks fail,
|
|
73
|
-
the decorator throws a FunctionAutonomicityError exception.
|
|
74
|
-
|
|
75
|
-
Dynamic checks: during the execution time it hides all the global
|
|
76
|
-
and non-local objects from the function, except the built-in ones
|
|
77
|
-
(and idempotent ones, if allow_idempotent==True).
|
|
78
|
-
If a function tries to use a non-built-in
|
|
79
|
-
(and non-idempotent, if allow_idempotent==True)
|
|
80
|
-
object without explicitly importing it inside the function body,
|
|
81
|
-
it will result in raising an exception.
|
|
82
|
-
|
|
83
|
-
Currently, neither static nor dynamic checks are guaranteed to catch
|
|
84
|
-
all possible violations of function autonomy requirements.
|
|
85
|
-
"""
|
|
86
|
-
|
|
87
68
|
wrapper = AutonomousFn(fn
|
|
88
69
|
,portal=self._portal
|
|
89
70
|
,fixed_kwargs=self._fixed_kwargs
|
|
@@ -5,7 +5,7 @@ from typing import Callable, Any
|
|
|
5
5
|
|
|
6
6
|
from persidict import PersiDict, Joker, KEEP_CURRENT
|
|
7
7
|
from .._020_ordinary_code_portals.code_normalizer import _pythagoras_decorator_names
|
|
8
|
-
from .. import DataPortal
|
|
8
|
+
from .._030_data_portals import DataPortal
|
|
9
9
|
from .._040_logging_code_portals import KwArgs
|
|
10
10
|
|
|
11
11
|
from .._060_autonomous_code_portals.names_usage_analyzer import (
|
|
@@ -14,9 +14,6 @@ from .._060_autonomous_code_portals.names_usage_analyzer import (
|
|
|
14
14
|
from .._050_safe_code_portals.safe_portal_core_classes import (
|
|
15
15
|
SafeFn, SafeCodePortal)
|
|
16
16
|
|
|
17
|
-
import pythagoras as pth
|
|
18
|
-
|
|
19
|
-
|
|
20
17
|
class AutonomousCodePortal(SafeCodePortal):
|
|
21
18
|
|
|
22
19
|
def __init__(self
|
|
@@ -36,7 +33,7 @@ class AutonomousFn(SafeFn):
|
|
|
36
33
|
_fixed_kwargs_packed: KwArgs | None
|
|
37
34
|
|
|
38
35
|
def __init__(self, fn: Callable|str|SafeFn
|
|
39
|
-
, fixed_kwargs: dict|None = None
|
|
36
|
+
, fixed_kwargs: dict[str,Any]|None = None
|
|
40
37
|
, excessive_logging: bool|Joker = KEEP_CURRENT
|
|
41
38
|
, portal: AutonomousCodePortal|None = None):
|
|
42
39
|
super().__init__(fn=fn
|
|
@@ -106,6 +103,14 @@ class AutonomousFn(SafeFn):
|
|
|
106
103
|
|
|
107
104
|
|
|
108
105
|
def fix_kwargs(self, **kwargs) -> AutonomousFn:
|
|
106
|
+
"""Create a new function by pre-filling some arguments.
|
|
107
|
+
|
|
108
|
+
This is called a partial application in functional programming
|
|
109
|
+
It allows creating specialized functions from general ones by
|
|
110
|
+
transforming a function with multiple parameters
|
|
111
|
+
into another function with fewer parameters by fixing some arguments.
|
|
112
|
+
"""
|
|
113
|
+
|
|
109
114
|
overlapping_keys = set(kwargs.keys()) & set(self.fixed_kwargs.keys())
|
|
110
115
|
assert len(overlapping_keys) == 0
|
|
111
116
|
new_fixed_kwargs = {**self.fixed_kwargs,**kwargs}
|
|
@@ -147,5 +152,6 @@ class AutonomousFn(SafeFn):
|
|
|
147
152
|
"""
|
|
148
153
|
super()._invalidate_cache()
|
|
149
154
|
if hasattr(self, "_fixed_kwargs_cached"):
|
|
150
|
-
assert hasattr(self, "_fixed_kwargs_packed")
|
|
155
|
+
assert (hasattr(self, "_fixed_kwargs_packed")
|
|
156
|
+
, "Premature cache invalidation: fixed_kwargs_packed is missing.")
|
|
151
157
|
del self._fixed_kwargs_cached
|
|
@@ -14,7 +14,7 @@ class NamesUsedInFunction:
|
|
|
14
14
|
self.accessible = set() # all names, currently accessable within the function
|
|
15
15
|
|
|
16
16
|
class NamesUsageAnalyzer(ast.NodeVisitor):
|
|
17
|
-
"""Collect data needed to analyze function
|
|
17
|
+
"""Collect data needed to analyze function autonomicity.
|
|
18
18
|
|
|
19
19
|
This class is a visitor of an AST (Abstract Syntax Tree) that collects data
|
|
20
20
|
needed to analyze function autonomy.
|
|
@@ -1,4 +1,20 @@
|
|
|
1
|
-
|
|
1
|
+
"""Classes and functions that allow protected execution of code.
|
|
2
|
+
|
|
3
|
+
Protected functions are functions that can be executed only if
|
|
4
|
+
certain conditions are met before the execution; also, certain conditions
|
|
5
|
+
must be met after the execution in order for the system to accept
|
|
6
|
+
and use execution results. These conditions are called validators
|
|
7
|
+
(pre-validators and post-validators). A protected function can have many
|
|
8
|
+
pre-validators and post-validators.
|
|
9
|
+
|
|
10
|
+
Validators can be passive (e.g., check if the node has enough RAM)
|
|
11
|
+
or active (e.g., check if some external library is installed, and,
|
|
12
|
+
if not, try to install it). Validators can be rather complex
|
|
13
|
+
(e.g., check if the result, returned by the function, is a valid image).
|
|
14
|
+
Under the hood, validators are autonomous functions.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from .validation_succesful_const import *
|
|
2
18
|
from .protected_portal_core_classes import *
|
|
3
19
|
from .protected_decorators import *
|
|
4
20
|
from .system_utils import *
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
from .._070_protected_code_portals import SimplePreValidatorFn
|
|
2
|
-
from .
|
|
2
|
+
from .validation_succesful_const import ValidationSuccessClass
|
|
3
3
|
|
|
4
4
|
|
|
5
|
-
def _at_least_X_CPU_cores_free_check(n:int)->
|
|
5
|
+
def _at_least_X_CPU_cores_free_check(n:int)-> ValidationSuccessClass | None:
|
|
6
6
|
cores = pth.get_unused_cpu_cores()
|
|
7
7
|
if cores >= n-0.1:
|
|
8
|
-
return pth.
|
|
8
|
+
return pth.VALIDATION_SUCCESSFUL
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def unused_cpu(cores:int) -> SimplePreValidatorFn:
|
|
@@ -14,10 +14,10 @@ def unused_cpu(cores:int) -> SimplePreValidatorFn:
|
|
|
14
14
|
return SimplePreValidatorFn(_at_least_X_CPU_cores_free_check).fix_kwargs(n=cores)
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
def _at_least_X_G_RAM_free_check(x:int)->
|
|
17
|
+
def _at_least_X_G_RAM_free_check(x:int)-> ValidationSuccessClass | None:
|
|
18
18
|
ram = pth.get_unused_ram_mb() / 1024
|
|
19
19
|
if ram >= x-0.1:
|
|
20
|
-
return pth.
|
|
20
|
+
return pth.VALIDATION_SUCCESSFUL
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def unused_ram(Gb:int) -> SimplePreValidatorFn:
|
|
@@ -26,15 +26,15 @@ def unused_ram(Gb:int) -> SimplePreValidatorFn:
|
|
|
26
26
|
return SimplePreValidatorFn(_at_least_X_G_RAM_free_check).fix_kwargs(x=Gb)
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
def _check_python_package_and_install_if_needed(package_name)->
|
|
29
|
+
def _check_python_package_and_install_if_needed(package_name)-> ValidationSuccessClass | None:
|
|
30
30
|
assert isinstance(package_name, str)
|
|
31
31
|
import importlib
|
|
32
32
|
try:
|
|
33
33
|
importlib.import_module(package_name)
|
|
34
|
-
return pth.
|
|
34
|
+
return pth.VALIDATION_SUCCESSFUL
|
|
35
35
|
except:
|
|
36
36
|
pth.install_package(package_name)
|
|
37
|
-
return pth.
|
|
37
|
+
return pth.VALIDATION_SUCCESSFUL
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
def installed_packages(*args) -> list[SimplePreValidatorFn]:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Support for work with protected functions."""
|
|
2
2
|
|
|
3
|
-
from typing import Callable
|
|
3
|
+
from typing import Callable, Any
|
|
4
4
|
|
|
5
5
|
from .validator_fn_classes import ValidatorFn
|
|
6
6
|
from .._060_autonomous_code_portals import autonomous
|
|
@@ -15,7 +15,7 @@ class protected(autonomous):
|
|
|
15
15
|
def __init__(self
|
|
16
16
|
, pre_validators: list[ValidatorFn] | None = None
|
|
17
17
|
, post_validators: list[ValidatorFn] | None = None
|
|
18
|
-
, fixed_kwargs: dict | None = None
|
|
18
|
+
, fixed_kwargs: dict[str,Any] | None = None
|
|
19
19
|
, excessive_logging: bool|Joker = KEEP_CURRENT
|
|
20
20
|
, portal: ProtectedCodePortal | None = None
|
|
21
21
|
):
|
|
@@ -1,23 +1,37 @@
|
|
|
1
|
+
"""Classes and functions that allow protected execution of code.
|
|
2
|
+
|
|
3
|
+
Protected functions are functions that can be executed only if
|
|
4
|
+
certain conditions are met before the execution; also, certain conditions
|
|
5
|
+
must be met after the execution in order for the system to accept
|
|
6
|
+
and use execution results. These conditions are called validators
|
|
7
|
+
(pre-validators and post-validators). A protected function can have many
|
|
8
|
+
pre-validators and post-validators.
|
|
9
|
+
|
|
10
|
+
Validators can be passive (e.g., check if the node has enough RAM)
|
|
11
|
+
or active (e.g., check if some external library is installed, and,
|
|
12
|
+
if not, try to install it). Validators can be rather complex
|
|
13
|
+
(e.g., check if the result, returned by the function, is a valid image).
|
|
14
|
+
Under the hood, validators are autonomous functions.
|
|
15
|
+
"""
|
|
16
|
+
|
|
1
17
|
from __future__ import annotations
|
|
2
18
|
|
|
3
19
|
from copy import copy
|
|
4
|
-
from typing import Callable, Any
|
|
20
|
+
from typing import Callable, Any
|
|
5
21
|
|
|
6
22
|
from persidict import PersiDict, Joker, KEEP_CURRENT
|
|
7
23
|
|
|
8
|
-
from .validator_fn_classes import
|
|
9
|
-
ComplexPreValidatorFn
|
|
24
|
+
from .validator_fn_classes import *
|
|
10
25
|
from .._030_data_portals import DataPortal
|
|
11
26
|
from .._040_logging_code_portals import KwArgs
|
|
12
27
|
from .._030_data_portals import ValueAddr
|
|
13
28
|
from parameterizable import sort_dict_by_keys
|
|
14
29
|
from .list_flattener import flatten_list
|
|
15
|
-
from .
|
|
30
|
+
from .validation_succesful_const import VALIDATION_SUCCESSFUL
|
|
16
31
|
|
|
17
32
|
|
|
18
33
|
from .._060_autonomous_code_portals import (
|
|
19
34
|
AutonomousCodePortal, AutonomousFn)
|
|
20
|
-
from .fn_arg_names_checker import check_if_fn_accepts_args
|
|
21
35
|
|
|
22
36
|
|
|
23
37
|
class ProtectedCodePortal(AutonomousCodePortal):
|
|
@@ -34,8 +48,8 @@ class ProtectedCodePortal(AutonomousCodePortal):
|
|
|
34
48
|
|
|
35
49
|
class ProtectedFn(AutonomousFn):
|
|
36
50
|
|
|
37
|
-
_pre_validators_cached: list[
|
|
38
|
-
_post_validators_cached: list[
|
|
51
|
+
_pre_validators_cached: list[ValidatorFn] | None
|
|
52
|
+
_post_validators_cached: list[ValidatorFn] | None
|
|
39
53
|
_pre_validators_addrs: list[ValueAddr]
|
|
40
54
|
_post_validators_addrs: list[ValueAddr]
|
|
41
55
|
|
|
@@ -43,27 +57,39 @@ class ProtectedFn(AutonomousFn):
|
|
|
43
57
|
pre_validators_arg_names = ["packed_kwargs", "fn_addr"]
|
|
44
58
|
|
|
45
59
|
def __init__(self, fn: Callable | str
|
|
46
|
-
, pre_validators: list[
|
|
47
|
-
, post_validators: list[
|
|
48
|
-
, excessive_logging: bool|
|
|
49
|
-
, fixed_kwargs: dict | None = None
|
|
60
|
+
, pre_validators: list[ValidatorFn] | list[Callable] | None = None
|
|
61
|
+
, post_validators: list[ValidatorFn] | list[Callable] | None = None
|
|
62
|
+
, excessive_logging: bool | Joker = KEEP_CURRENT
|
|
63
|
+
, fixed_kwargs: dict[str,Any] | None = None
|
|
50
64
|
, portal: ProtectedCodePortal | None = None):
|
|
51
65
|
super().__init__(fn=fn
|
|
52
66
|
, portal = portal
|
|
53
67
|
, fixed_kwargs=fixed_kwargs
|
|
54
68
|
, excessive_logging = excessive_logging)
|
|
55
69
|
|
|
56
|
-
pre_validators
|
|
57
|
-
|
|
70
|
+
if pre_validators is None:
|
|
71
|
+
pre_validators = list()
|
|
72
|
+
else:
|
|
73
|
+
pre_validators = copy(pre_validators)
|
|
74
|
+
|
|
75
|
+
if post_validators is None:
|
|
76
|
+
post_validators = list()
|
|
77
|
+
else:
|
|
78
|
+
post_validators = copy(post_validators)
|
|
58
79
|
|
|
59
80
|
if isinstance(fn, ProtectedFn):
|
|
60
81
|
pre_validators += fn.pre_validators
|
|
61
82
|
post_validators += fn.post_validators
|
|
62
83
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
self.
|
|
84
|
+
pre_validators = self._normalize_validators(pre_validators, PreValidatorFn)
|
|
85
|
+
post_validators = self._normalize_validators(post_validators, PostValidatorFn)
|
|
86
|
+
|
|
87
|
+
self._pre_validators_cached = pre_validators
|
|
88
|
+
self._post_validators_cached = post_validators
|
|
89
|
+
self._pre_validators_addrs = [ValueAddr(g, store=False)
|
|
90
|
+
for g in self._pre_validators_cached]
|
|
91
|
+
self._post_validators_addrs = [ValueAddr(v, store=False)
|
|
92
|
+
for v in self._post_validators_cached]
|
|
67
93
|
|
|
68
94
|
|
|
69
95
|
def __getstate__(self):
|
|
@@ -120,7 +146,7 @@ class ProtectedFn(AutonomousFn):
|
|
|
120
146
|
pre_validation_result = pre_validator()
|
|
121
147
|
else:
|
|
122
148
|
pre_validation_result = pre_validator(packed_kwargs=kw_args, fn_addr = self.addr)
|
|
123
|
-
if pre_validation_result is not
|
|
149
|
+
if pre_validation_result is not VALIDATION_SUCCESSFUL:
|
|
124
150
|
return False
|
|
125
151
|
return True
|
|
126
152
|
|
|
@@ -132,7 +158,7 @@ class ProtectedFn(AutonomousFn):
|
|
|
132
158
|
portal.entropy_infuser.shuffle(post_validators)
|
|
133
159
|
for post_validator in post_validators:
|
|
134
160
|
if post_validator(packed_kwargs=kw_args, fn_addr = self.addr
|
|
135
|
-
, result=result) is not
|
|
161
|
+
, result=result) is not VALIDATION_SUCCESSFUL:
|
|
136
162
|
return False
|
|
137
163
|
return True
|
|
138
164
|
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
class
|
|
1
|
+
class ValidationSuccessClass:
|
|
2
|
+
"""Singleton class to represent a successful validation."""
|
|
3
|
+
|
|
2
4
|
_instance = None
|
|
3
5
|
|
|
4
6
|
def __new__(cls):
|
|
@@ -6,4 +8,4 @@ class OKClass:
|
|
|
6
8
|
cls._instance = super().__new__(cls)
|
|
7
9
|
return cls._instance
|
|
8
10
|
|
|
9
|
-
|
|
11
|
+
VALIDATION_SUCCESSFUL = ValidationSuccessClass()
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"""Decorators and utilities to work with pure functions.
|
|
2
2
|
|
|
3
|
-
A pure function is
|
|
3
|
+
A pure function is a protected function that has no side effects and
|
|
4
4
|
always returns the same result if it is called multiple times
|
|
5
5
|
with the same arguments.
|
|
6
6
|
|
|
7
|
-
This
|
|
7
|
+
This subpackage defines a decorator which is used to inform Pythagoras that
|
|
8
8
|
a function is intended to be pure: @pure().
|
|
9
9
|
|
|
10
10
|
Pythagoras persistently caches results, produced by a pure function, so that
|
|
@@ -16,22 +16,8 @@ While caching the results of a pure function, Pythagoras also tracks
|
|
|
16
16
|
changes in the source code of the function. If the source code of a pure
|
|
17
17
|
function changes, the function is executed again on the next call.
|
|
18
18
|
However, the previously cached results are still available
|
|
19
|
-
for the old version of the function.
|
|
20
|
-
|
|
21
|
-
A pure function must be autonomous. Pythagoras tracks source code changes
|
|
22
|
-
for the pure function as well other autonomous functions it is using,
|
|
23
|
-
provided they belong to the same island. Pythagoras does not track source code
|
|
24
|
-
changes for functions from other islands, even if they are used
|
|
25
|
-
by the pure function. Pythagoras also does not track any other
|
|
26
|
-
source code changes (e.g. changes in the imported packages).
|
|
27
|
-
|
|
28
|
-
Pythagoras provides infrastructure for remote execution of
|
|
29
|
-
pure functions in distributed environments. Pythagoras employs
|
|
30
|
-
an asynchronous execution model called 'swarming':
|
|
31
|
-
you do not know when your function will be executed,
|
|
32
|
-
what machine will execute it, and how many times it will be executed.
|
|
33
|
-
Pythagoras ensures that the function will be eventually executed
|
|
34
|
-
at least once, but does not offer any further guarantees.
|
|
19
|
+
for the old version of the function. Only changes in the function's
|
|
20
|
+
source code are tracked.
|
|
35
21
|
"""
|
|
36
22
|
|
|
37
23
|
from .pure_core_classes import (
|
|
@@ -1,3 +1,25 @@
|
|
|
1
|
+
"""Classes to work with pure functions.
|
|
2
|
+
|
|
3
|
+
A pure function is a protected function that has no side effects and
|
|
4
|
+
always returns the same result if it is called multiple times
|
|
5
|
+
with the same arguments.
|
|
6
|
+
|
|
7
|
+
This subpackage defines a decorator which is used to inform Pythagoras that
|
|
8
|
+
a function is intended to be pure: @pure().
|
|
9
|
+
|
|
10
|
+
Pythagoras persistently caches results, produced by a pure function, so that
|
|
11
|
+
if the function is called multiple times with the same arguments,
|
|
12
|
+
the function is executed only once, and the cached result is returned
|
|
13
|
+
for all the subsequent executions.
|
|
14
|
+
|
|
15
|
+
While caching the results of a pure function, Pythagoras also tracks
|
|
16
|
+
changes in the source code of the function. If the source code of a pure
|
|
17
|
+
function changes, the function is executed again on the next call.
|
|
18
|
+
However, the previously cached results are still available
|
|
19
|
+
for the old version of the function. Only changes in the function's
|
|
20
|
+
source code are tracked.
|
|
21
|
+
"""
|
|
22
|
+
|
|
1
23
|
from __future__ import annotations
|
|
2
24
|
|
|
3
25
|
import time
|
|
@@ -9,22 +31,19 @@ import pandas as pd
|
|
|
9
31
|
|
|
10
32
|
from persidict import PersiDict, Joker, KEEP_CURRENT
|
|
11
33
|
|
|
12
|
-
from
|
|
13
|
-
|
|
34
|
+
from persidict import WriteOnceDict
|
|
35
|
+
|
|
36
|
+
from .._010_basic_portals import *
|
|
14
37
|
from .._010_basic_portals.basic_portal_core_classes import (
|
|
15
38
|
_describe_persistent_characteristic)
|
|
16
|
-
from persidict import WriteOnceDict
|
|
17
|
-
from .._040_logging_code_portals.logging_portal_core_classes import (
|
|
18
|
-
LoggingFnCallSignature)
|
|
19
39
|
|
|
20
40
|
from .._030_data_portals import HashAddr, ValueAddr
|
|
21
|
-
|
|
22
|
-
from .._060_autonomous_code_portals.autonomous_portal_core_classes import (
|
|
23
|
-
AutonomousFn)
|
|
41
|
+
from .._040_logging_code_portals import *
|
|
24
42
|
|
|
25
43
|
|
|
26
|
-
from ..
|
|
44
|
+
from .._070_protected_code_portals import *
|
|
27
45
|
|
|
46
|
+
# from .._040_logging_code_portals.kw_args import KwArgs
|
|
28
47
|
|
|
29
48
|
ASupportingFunc:TypeAlias = str | AutonomousFn
|
|
30
49
|
|
|
@@ -232,23 +251,16 @@ class PureFn(ProtectedFn):
|
|
|
232
251
|
return ProtectedFn.portal.__get__(self)
|
|
233
252
|
|
|
234
253
|
|
|
235
|
-
# @portal.setter
|
|
236
|
-
# def portal(self, new_portal: PureCodePortal) -> None: #*#*#
|
|
237
|
-
# if not isinstance(new_portal, PureCodePortal):
|
|
238
|
-
# raise TypeError("portal must be a PureCodePortal instance")
|
|
239
|
-
# ProtectedFn.portal.__set__(self, new_portal)
|
|
240
|
-
|
|
241
|
-
|
|
242
254
|
class PureFnExecutionResultAddr(HashAddr):
|
|
243
|
-
"""An address of
|
|
255
|
+
"""An address of a (future) result of pure function execution.
|
|
244
256
|
|
|
245
257
|
This class is used to point to the result of an execution of a pure
|
|
246
|
-
function in a portal. The address is used to request an execution
|
|
258
|
+
function in a portal. The address is used to request an execution or
|
|
247
259
|
to retrieve the result (if available) from the portal.
|
|
248
260
|
|
|
249
261
|
The address also provides access to various logs and records of the
|
|
250
262
|
function execution, such as environmental contexts of the execution attempts,
|
|
251
|
-
outputs printed, exceptions thrown and events emitted.
|
|
263
|
+
outputs printed, exceptions thrown, and events emitted.
|
|
252
264
|
"""
|
|
253
265
|
_fn_cache: PureFn | None
|
|
254
266
|
_call_signature_cache: LoggingFnCallSignature | None
|
|
@@ -1,7 +1,28 @@
|
|
|
1
|
-
|
|
1
|
+
"""Decorator to work with pure functions.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
A pure function is a protected function that has no side effects and
|
|
4
|
+
always returns the same result if it is called multiple times
|
|
5
|
+
with the same arguments.
|
|
6
|
+
|
|
7
|
+
This subpackage defines a decorator which is used to inform Pythagoras that
|
|
8
|
+
a function is intended to be pure: @pure().
|
|
9
|
+
|
|
10
|
+
Pythagoras persistently caches results, produced by a pure function, so that
|
|
11
|
+
if the function is called multiple times with the same arguments,
|
|
12
|
+
the function is executed only once, and the cached result is returned
|
|
13
|
+
for all the subsequent executions.
|
|
14
|
+
|
|
15
|
+
While caching the results of a pure function, Pythagoras also tracks
|
|
16
|
+
changes in the source code of the function. If the source code of a pure
|
|
17
|
+
function changes, the function is executed again on the next call.
|
|
18
|
+
However, the previously cached results are still available
|
|
19
|
+
for the old version of the function. Only changes in the function's
|
|
20
|
+
source code are tracked.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from typing import Callable, Any
|
|
24
|
+
|
|
25
|
+
from .._070_protected_code_portals import protected, ValidatorFn
|
|
5
26
|
from .._080_pure_code_portals.pure_core_classes import (
|
|
6
27
|
PureCodePortal, PureFn)
|
|
7
28
|
|
|
@@ -10,9 +31,9 @@ from persidict import KEEP_CURRENT, Joker
|
|
|
10
31
|
class pure(protected):
|
|
11
32
|
|
|
12
33
|
def __init__(self
|
|
13
|
-
, pre_validators: list[
|
|
14
|
-
, post_validators: list[
|
|
15
|
-
, fixed_kwargs: dict | None = None
|
|
34
|
+
, pre_validators: list[ValidatorFn] | None = None
|
|
35
|
+
, post_validators: list[ValidatorFn] | None = None
|
|
36
|
+
, fixed_kwargs: dict[str, Any] | None = None
|
|
16
37
|
, excessive_logging: bool | Joker = KEEP_CURRENT
|
|
17
38
|
, portal: PureCodePortal | None = None
|
|
18
39
|
):
|
|
@@ -1 +1,12 @@
|
|
|
1
|
+
""" Classes and functions that enable swarming algorithm.
|
|
2
|
+
|
|
3
|
+
Pythagoras provides infrastructure for remote execution of
|
|
4
|
+
pure functions in distributed environments. Pythagoras employs
|
|
5
|
+
an asynchronous execution model called 'swarming':
|
|
6
|
+
you do not know when your function will be executed,
|
|
7
|
+
what machine will execute it, and how many times it will be executed.
|
|
8
|
+
Pythagoras ensures that the function will be eventually executed
|
|
9
|
+
at least once but does not offer any further guarantees.
|
|
10
|
+
"""
|
|
11
|
+
|
|
1
12
|
from .swarming_portals import *
|
|
@@ -1,3 +1,14 @@
|
|
|
1
|
+
""" Classes and functions that enable swarming algorithm.
|
|
2
|
+
|
|
3
|
+
Pythagoras provides infrastructure for remote execution of
|
|
4
|
+
pure functions in distributed environments. Pythagoras employs
|
|
5
|
+
an asynchronous execution model called 'swarming':
|
|
6
|
+
you do not know when your function will be executed,
|
|
7
|
+
what machine will execute it, and how many times it will be executed.
|
|
8
|
+
Pythagoras ensures that the function will be eventually executed
|
|
9
|
+
at least once but does not offer any further guarantees.
|
|
10
|
+
"""
|
|
11
|
+
|
|
1
12
|
from __future__ import annotations
|
|
2
13
|
|
|
3
14
|
import atexit
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
from ..
|
|
1
|
+
from .._010_basic_portals import get_active_portal
|
|
2
2
|
from .._010_basic_portals import get_default_portal_base_dir
|
|
3
|
+
from .._090_swarming_portals import SwarmingPortal
|
|
3
4
|
|
|
4
5
|
|
|
5
|
-
def
|
|
6
|
+
def _instantiate_default_local_portal():
|
|
7
|
+
#NB: there is a hidden circular dependency from get_active_portal()
|
|
6
8
|
SwarmingPortal(root_dict = get_default_portal_base_dir())
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""This subpackage is only needed for development. No runtime usage."""
|
pythagoras/__init__.py
CHANGED
|
@@ -1,9 +1,21 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""Distributed serverless compute at a global scale.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
Pythagoras is a framework for distributed computations in Python.
|
|
4
|
+
|
|
5
|
+
It offers 3 main advantages:
|
|
6
|
+
|
|
7
|
+
- Global scale: parallelize your algorithms to scale to millions of nodes.
|
|
8
|
+
- Low maintenance: no need to manage servers and infrastructure,
|
|
9
|
+
Pythagoras replaces expensive compute with cheap storage.
|
|
10
|
+
- High performance: 'compute once, reuse forever' strategy
|
|
11
|
+
significanty accelerates long-running workflows.
|
|
12
|
+
|
|
13
|
+
Pythagoras is able to offer these benefits as it's the first framework
|
|
14
|
+
to fully implement the Functional Programming 2.0 paradigm.
|
|
15
|
+
|
|
16
|
+
Pythagoras excels at optimizing complex, long-running,
|
|
17
|
+
resource-demanding computations. It’s not the best choice for real-time,
|
|
18
|
+
latency-sensitive workflows.
|
|
7
19
|
"""
|
|
8
20
|
|
|
9
21
|
|
|
@@ -21,19 +33,3 @@ from ._100_top_level_API import *
|
|
|
21
33
|
from ._800_signatures_and_converters import *
|
|
22
34
|
|
|
23
35
|
|
|
24
|
-
# primary_decorators = {d.__name__:d for d in [
|
|
25
|
-
# autonomous
|
|
26
|
-
# , pure
|
|
27
|
-
# ]}
|
|
28
|
-
#
|
|
29
|
-
# all_decorators = {d.__name__:d for d in [
|
|
30
|
-
# ordinary
|
|
31
|
-
# , storable
|
|
32
|
-
# , logging
|
|
33
|
-
# , safe
|
|
34
|
-
# , autonomous
|
|
35
|
-
# , protected
|
|
36
|
-
# , pure
|
|
37
|
-
# ]}
|
|
38
|
-
|
|
39
|
-
|
pythagoras/core/__init__.py
CHANGED
|
@@ -3,4 +3,4 @@ from .._060_autonomous_code_portals import autonomous
|
|
|
3
3
|
from .._080_pure_code_portals import pure
|
|
4
4
|
from .._090_swarming_portals import SwarmingPortal
|
|
5
5
|
from .._100_top_level_API import get_portal
|
|
6
|
-
from .._070_protected_code_portals import
|
|
6
|
+
from .._070_protected_code_portals.basic_pre_validators import *
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
pythagoras/.DS_Store,sha256=40d35c2d6b7fca6728a8be89c610686ceb83b21e32d6fe0ddedcf1b3f58bfdaf,6148
|
|
2
2
|
pythagoras/_010_basic_portals/__init__.py,sha256=111d2457fb61ee16dae275d78e83fe8e837e72a9143281f889148280a50fef96,1703
|
|
3
|
-
pythagoras/_010_basic_portals/basic_portal_core_classes.py,sha256=
|
|
3
|
+
pythagoras/_010_basic_portals/basic_portal_core_classes.py,sha256=9dc857d35e03503710fabb118dea250ac344d19c6865f973d3bdcbdc2c89f841,18996
|
|
4
4
|
pythagoras/_010_basic_portals/exceptions.py,sha256=a15bab585769159db1bedf123bc92e920695f14faa3ad1f4ab0f863b7742b391,237
|
|
5
5
|
pythagoras/_010_basic_portals/long_infoname.py,sha256=9742b2bf023389704c88628a8ed81d1c82ad6eb15a0a27c9cddadbfae1e9cf2d,650
|
|
6
6
|
pythagoras/_010_basic_portals/not_picklable_class.py,sha256=2f7405a32c530a68fae64c5e06731b69c44dea1fc0611e72284d5b13dffd77c0,352
|
|
@@ -10,45 +10,45 @@ pythagoras/_020_ordinary_code_portals/__init__.py,sha256=a77912a9a4188f4c65864f4
|
|
|
10
10
|
pythagoras/_020_ordinary_code_portals/code_normalizer.py,sha256=ecbdcfe40699292f483eaac812fe5425cdc71c33f8f1fc3a1064165d973142d3,5004
|
|
11
11
|
pythagoras/_020_ordinary_code_portals/function_processing.py,sha256=ae5841d1da5a533b6fe92336c2551d6947444dc576a55508098b9f56b40245c7,3623
|
|
12
12
|
pythagoras/_020_ordinary_code_portals/ordinary_decorator.py,sha256=278ab1d37344120a585af8380fc52400bd0f76d02ddac43230dd68a7a2cc16c0,1028
|
|
13
|
-
pythagoras/_020_ordinary_code_portals/ordinary_portal_core_classes.py,sha256=
|
|
14
|
-
pythagoras/_030_data_portals/__init__.py,sha256=
|
|
15
|
-
pythagoras/_030_data_portals/data_portal_core_classes.py,sha256=
|
|
13
|
+
pythagoras/_020_ordinary_code_portals/ordinary_portal_core_classes.py,sha256=6d8d87cf2d7d9537b3bfde9d49cd65e52b1a1b13643133aaf08b1f9e4755d706,9743
|
|
14
|
+
pythagoras/_030_data_portals/__init__.py,sha256=7ff17d0c29ae5603cc833c118ee363e8523add2de85eeee58f7cd4eba370ec77,1427
|
|
15
|
+
pythagoras/_030_data_portals/data_portal_core_classes.py,sha256=34f1e5235a37747b585d503bcc613f79ab61c4af99fafdfbf707005927a3c2bd,20411
|
|
16
16
|
pythagoras/_030_data_portals/ready_and_get.py,sha256=838004869475dd877a1158a7cb6c20f2f922b97e1ac174286ce30ce287fb6520,2002
|
|
17
17
|
pythagoras/_030_data_portals/storable_decorator.py,sha256=97c5b71a156622c7208e80931aaded99d7a12860cb2159c56d333e196fb51b88,578
|
|
18
|
-
pythagoras/_040_logging_code_portals/__init__.py,sha256=
|
|
19
|
-
pythagoras/_040_logging_code_portals/exception_processing_tracking.py,sha256=
|
|
18
|
+
pythagoras/_040_logging_code_portals/__init__.py,sha256=ab6855c8e56013ef51bb7c9c8a52bdf184bd45ab7cb5273a7ab77b02d1b1a5a0,996
|
|
19
|
+
pythagoras/_040_logging_code_portals/exception_processing_tracking.py,sha256=b14f8509acafa6917a81a7c14402ac25c302d0930172708288bceedf2166c620,778
|
|
20
20
|
pythagoras/_040_logging_code_portals/execution_environment_summary.py,sha256=853bd36da75802d8ac6781fcaeafa7fe1b0a3c24b7f23b12cfc970fc0d4322a6,2268
|
|
21
|
-
pythagoras/_040_logging_code_portals/kw_args.py,sha256=
|
|
21
|
+
pythagoras/_040_logging_code_portals/kw_args.py,sha256=cf513ce2e8621e13f552e449930c57d4867279f3aa3e7ca0d796197751eae9d7,2604
|
|
22
22
|
pythagoras/_040_logging_code_portals/logging_decorator.py,sha256=d3bf70dbf6791e15c52eb8325d07a48ee39bcbd09d5201541916d14f9969ba35,891
|
|
23
|
-
pythagoras/_040_logging_code_portals/logging_portal_core_classes.py,sha256=
|
|
23
|
+
pythagoras/_040_logging_code_portals/logging_portal_core_classes.py,sha256=ebc0a18b3f4e85caad69e47c3a244c62e9c42d5aa262c1e34d380620910127e2,21869
|
|
24
24
|
pythagoras/_040_logging_code_portals/notebook_checker.py,sha256=e654090c80f38512114a65f54fcf2701110c10ca030c57f438829cbd3a678739,541
|
|
25
25
|
pythagoras/_040_logging_code_portals/output_capturer.py,sha256=a210a9eaaab12fb22e2467e716e088823e4e7381e60bc73bb99184feecd69199,4216
|
|
26
26
|
pythagoras/_040_logging_code_portals/uncaught_exceptions.py,sha256=0776cfbd7e679c9271e098cb486765d715ce054da9aa8dc23fe32898901f5da1,3121
|
|
27
|
-
pythagoras/_050_safe_code_portals/__init__.py,sha256=
|
|
28
|
-
pythagoras/_050_safe_code_portals/safe_decorator.py,sha256=
|
|
29
|
-
pythagoras/_050_safe_code_portals/safe_portal_core_classes.py,sha256=
|
|
30
|
-
pythagoras/_060_autonomous_code_portals/__init__.py,sha256=
|
|
31
|
-
pythagoras/_060_autonomous_code_portals/autonomous_decorators.py,sha256=
|
|
32
|
-
pythagoras/_060_autonomous_code_portals/autonomous_portal_core_classes.py,sha256=
|
|
33
|
-
pythagoras/_060_autonomous_code_portals/names_usage_analyzer.py,sha256=
|
|
34
|
-
pythagoras/_070_protected_code_portals/
|
|
35
|
-
pythagoras/_070_protected_code_portals/
|
|
36
|
-
pythagoras/_070_protected_code_portals/basic_pre_validators.py,sha256=742771f96dd635a8d132ac3da90fd17689fe4b38e06160c8019be4fce7ef1602,1441
|
|
27
|
+
pythagoras/_050_safe_code_portals/__init__.py,sha256=611f95e96d96675ed28ea993c98db1758d7ac5354412e2ecd8c75d263fd60995,557
|
|
28
|
+
pythagoras/_050_safe_code_portals/safe_decorator.py,sha256=653b08326b5bdde180592a7074c6174662326898aa57a99d15b07f32a9e0efce,804
|
|
29
|
+
pythagoras/_050_safe_code_portals/safe_portal_core_classes.py,sha256=0f414a7a6f280cddb9a9b55dbb86f7039fcc4e558147d1c1a2704df640de9ecf,2026
|
|
30
|
+
pythagoras/_060_autonomous_code_portals/__init__.py,sha256=7308e1db007167f22096e0c59701784a4ecc5d5fb776ae016a50dc3b6d100285,1747
|
|
31
|
+
pythagoras/_060_autonomous_code_portals/autonomous_decorators.py,sha256=762597d378d96889dce798c0837f75651061f92b797b12512c36624bb76a62f8,2895
|
|
32
|
+
pythagoras/_060_autonomous_code_portals/autonomous_portal_core_classes.py,sha256=89ccd81b5583f263ccde98b73b12ab0d5eeaf19ee31d2c3bad186155f4dc44f8,5968
|
|
33
|
+
pythagoras/_060_autonomous_code_portals/names_usage_analyzer.py,sha256=9aaaf3010c37b46c4f617b0c03c1347e1848a49a29bd116bce47353154ac3d74,7474
|
|
34
|
+
pythagoras/_070_protected_code_portals/__init__.py,sha256=4ef19c25acf6d10aac9aff26da903876e06d167fc274229f9126ce485a094bc9,989
|
|
35
|
+
pythagoras/_070_protected_code_portals/basic_pre_validators.py,sha256=1d63cc3464c602a022e26b74a255c9539ecdb00437c48bb50a41e1e5e443f8f6,1595
|
|
37
36
|
pythagoras/_070_protected_code_portals/fn_arg_names_checker.py,sha256=e858ce5099868030a39057177f93a893e13d7a210582e6ab636aaaccec898fb0,1230
|
|
38
37
|
pythagoras/_070_protected_code_portals/list_flattener.py,sha256=9a54b512ad9dc201db348e7908f5ca5f36d171ae3da433511d38023a6ba8d4b5,331
|
|
39
|
-
pythagoras/_070_protected_code_portals/package_manager.py,sha256=
|
|
40
|
-
pythagoras/_070_protected_code_portals/protected_decorators.py,sha256=
|
|
41
|
-
pythagoras/_070_protected_code_portals/protected_portal_core_classes.py,sha256=
|
|
38
|
+
pythagoras/_070_protected_code_portals/package_manager.py,sha256=4111d71886adcf1e8d28f3c2d3a08c8a352601297f9ab3f08a4541586ed9d376,1965
|
|
39
|
+
pythagoras/_070_protected_code_portals/protected_decorators.py,sha256=956ed45870004246d77d6e364015b3a0d936bd8b919dfa9a56c97d69713a9b80,1627
|
|
40
|
+
pythagoras/_070_protected_code_portals/protected_portal_core_classes.py,sha256=a1c4451fa648e05da0376e8ccf27e0e731cfb1d8ea707b33a4ca1f313622de94,9308
|
|
42
41
|
pythagoras/_070_protected_code_portals/system_utils.py,sha256=878a22110140c852b3aaf774cb0c0023458360a4ae221fef89ff3a5d02176301,2360
|
|
42
|
+
pythagoras/_070_protected_code_portals/validation_succesful_const.py,sha256=602cd3598dd43a71421eb547f1e80f411e30df616deb26912a741a2898d06754,300
|
|
43
43
|
pythagoras/_070_protected_code_portals/validator_fn_classes.py,sha256=4afa4bf0629f68e4b1623003272ffd7111ac9cbb739e81fde601049d19b84269,3232
|
|
44
|
-
pythagoras/_080_pure_code_portals/__init__.py,sha256=
|
|
45
|
-
pythagoras/_080_pure_code_portals/pure_core_classes.py,sha256=
|
|
46
|
-
pythagoras/_080_pure_code_portals/pure_decorator.py,sha256=
|
|
47
|
-
pythagoras/_090_swarming_portals/__init__.py,sha256=
|
|
44
|
+
pythagoras/_080_pure_code_portals/__init__.py,sha256=aa8564346e1701934310dd5eca92424f5784e0be49c6bad97459f4bab1bdad73,1082
|
|
45
|
+
pythagoras/_080_pure_code_portals/pure_core_classes.py,sha256=5227a65ca50143b0234ce8845d45fad38a7b0b80d7b15ad7f441e594944b59f7,19457
|
|
46
|
+
pythagoras/_080_pure_code_portals/pure_decorator.py,sha256=587650ce6cb1802a402c7aea7de88e32b33a4c391972fd18d9bef9e9ecf84369,2279
|
|
47
|
+
pythagoras/_090_swarming_portals/__init__.py,sha256=4ee035ecf7ed4c1b9da6d01b94db41943e3a06a5228ad92c3ad7f7cb4d502a6d,514
|
|
48
48
|
pythagoras/_090_swarming_portals/output_suppressor.py,sha256=83e6cc9bcc62a226babb1165912ef5095ea948499ce5136a7516ac8b54522607,626
|
|
49
|
-
pythagoras/_090_swarming_portals/swarming_portals.py,sha256=
|
|
49
|
+
pythagoras/_090_swarming_portals/swarming_portals.py,sha256=51e1a4149659cc1ef62761338ae9190736a89ecefc5c10281f20b805f4e4c1d4,12485
|
|
50
50
|
pythagoras/_100_top_level_API/__init__.py,sha256=b392edc2c918da7c2444f14accfd0fac2cd0d5cf6849c64ed2433dfdb58b8b75,64
|
|
51
|
-
pythagoras/_100_top_level_API/default_local_portal.py,sha256=
|
|
51
|
+
pythagoras/_100_top_level_API/default_local_portal.py,sha256=4a7ca44e94d78352ae4f5ab00e7ac067ade562c84ccbed27362520a0e54cc42b,339
|
|
52
52
|
pythagoras/_100_top_level_API/top_level_API.py,sha256=4b63575b86df2fdf28e93c67e8d33411e05bd67cf016d3d297ec9635ebc04081,906
|
|
53
53
|
pythagoras/_800_signatures_and_converters/__init__.py,sha256=d9420254db0c62903fe5419612cef536f219ec319e81b7efc3e783a113aac93b,167
|
|
54
54
|
pythagoras/_800_signatures_and_converters/base_16_32_convertors.py,sha256=66356d2a38e33203201d484cfb543da8b6160832a1965aaaf73e1880a2f4a828,1236
|
|
@@ -56,10 +56,10 @@ pythagoras/_800_signatures_and_converters/current_date_gmt_str.py,sha256=2beac61
|
|
|
56
56
|
pythagoras/_800_signatures_and_converters/hash_signatures.py,sha256=02c5080ff67bc9e174e899a4f2428015722ec6817484dd0ec0e4b54f99eeeff9,989
|
|
57
57
|
pythagoras/_800_signatures_and_converters/node_signatures.py,sha256=2de43f2cc4c3d62f56240fc26fc70668fde809a65f023502fb773ccafae3224b,557
|
|
58
58
|
pythagoras/_800_signatures_and_converters/random_signatures.py,sha256=c174f28c04ccf08c0c5eadaf1e1a00aba4fec88ec370def64afcadd39123141a,323
|
|
59
|
-
pythagoras/_900_project_stats_collector/__init__.py,sha256=
|
|
60
|
-
pythagoras/_900_project_stats_collector/project_analyzer.py,sha256=
|
|
61
|
-
pythagoras/__init__.py,sha256=
|
|
62
|
-
pythagoras/core/__init__.py,sha256=
|
|
63
|
-
pythagoras-0.
|
|
64
|
-
pythagoras-0.
|
|
65
|
-
pythagoras-0.
|
|
59
|
+
pythagoras/_900_project_stats_collector/__init__.py,sha256=11a82df8184f3ed046829332c31da590b0cad7ad3763db7f40176d1ca5071c56,71
|
|
60
|
+
pythagoras/_900_project_stats_collector/project_analyzer.py,sha256=ba1c9c14a8d421712961c6589da937ca7e0916172197ea19ef0b7ab291718686,3574
|
|
61
|
+
pythagoras/__init__.py,sha256=4cc3ed25d4a2fd64a10a9267b1555d3b8f1672fb3bf0631b522e601dcd5e30bc,1233
|
|
62
|
+
pythagoras/core/__init__.py,sha256=11c3616471c79550d69a0d3b3b49a45b90087919b82477db5de99c8ebc6ba5c0,298
|
|
63
|
+
pythagoras-0.23.1.dist-info/WHEEL,sha256=607c46fee47e440c91332c738096ff0f5e54ca3b0818ee85462dd5172a38e793,79
|
|
64
|
+
pythagoras-0.23.1.dist-info/METADATA,sha256=4f420f2685e10acb52d4ac4e38ba364e9f5e55be98cc6f222931e34172ce201c,4241
|
|
65
|
+
pythagoras-0.23.1.dist-info/RECORD,,
|
|
File without changes
|