pythagoras 0.20.51__py3-none-any.whl → 0.21.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pythagoras/_020_ordinary_code_portals/code_normalizer.py +11 -3
- pythagoras/_020_ordinary_code_portals/ordinary_portal_core_classes.py +9 -6
- pythagoras/_030_data_portals/__init__.py +2 -2
- pythagoras/_030_data_portals/data_portal_core_classes.py +70 -56
- pythagoras/_040_logging_code_portals/kw_args.py +9 -5
- pythagoras/_040_logging_code_portals/logging_portal_core_classes.py +15 -16
- pythagoras/_060_autonomous_code_portals/autonomous_portal_core_classes.py +46 -28
- pythagoras/_070_protected_code_portals/GPU_pre_validators.py +0 -0
- pythagoras/_070_protected_code_portals/RAM_pre_validators.py +58 -0
- pythagoras/_070_protected_code_portals/__init__.py +4 -3
- pythagoras/_070_protected_code_portals/fn_arg_names_checker.py +3 -2
- pythagoras/_070_protected_code_portals/protected_decorators.py +13 -12
- pythagoras/_070_protected_code_portals/protected_portal_core_classes.py +95 -78
- pythagoras/{_090_swarming_portals → _070_protected_code_portals}/system_utils.py +29 -1
- pythagoras/_070_protected_code_portals/validator_fn_classes.py +95 -0
- pythagoras/_080_pure_code_portals/pure_core_classes.py +16 -18
- pythagoras/_080_pure_code_portals/pure_decorator.py +12 -12
- pythagoras/_090_swarming_portals/swarming_portals.py +4 -10
- pythagoras/__init__.py +14 -14
- {pythagoras-0.20.51.dist-info → pythagoras-0.21.0.dist-info}/METADATA +5 -2
- {pythagoras-0.20.51.dist-info → pythagoras-0.21.0.dist-info}/RECORD +23 -22
- {pythagoras-0.20.51.dist-info → pythagoras-0.21.0.dist-info}/WHEEL +1 -1
- pythagoras/_070_protected_code_portals/GPU_guards.py +0 -2
- pythagoras/_070_protected_code_portals/RAM_guards.py +0 -33
- /pythagoras/_070_protected_code_portals/{python_packages_guards.py → python_packages_pre_validators.py} +0 -0
|
@@ -10,7 +10,15 @@ from .function_processing import get_function_name_from_source
|
|
|
10
10
|
from .._010_basic_portals.long_infoname import get_long_infoname
|
|
11
11
|
from .function_processing import assert_ordinarity
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
_pythagoras_decorator_names = {
|
|
14
|
+
"ordinary"
|
|
15
|
+
, "storable"
|
|
16
|
+
, "logging"
|
|
17
|
+
, "safe"
|
|
18
|
+
, "autonomous"
|
|
19
|
+
, "protected"
|
|
20
|
+
, "pure"
|
|
21
|
+
}
|
|
14
22
|
|
|
15
23
|
def _get_normalized_function_source_impl(
|
|
16
24
|
a_func: Callable | str
|
|
@@ -72,13 +80,13 @@ def _get_normalized_function_source_impl(
|
|
|
72
80
|
f"Function {a_func_name} can't have multiple decorators,"
|
|
73
81
|
+ " only one decorator is allowed.")
|
|
74
82
|
|
|
75
|
-
all_decorators = pth.all_decorators
|
|
83
|
+
# all_decorators = pth.all_decorators
|
|
76
84
|
# all_decorators = sys.modules['pythagoras'].all_decorators
|
|
77
85
|
|
|
78
86
|
if drop_pth_decorators and len(decorator_list):
|
|
79
87
|
decorator = decorator_list[0].func
|
|
80
88
|
pth_dec_counter = 0
|
|
81
|
-
for candidate in
|
|
89
|
+
for candidate in _pythagoras_decorator_names:
|
|
82
90
|
try:
|
|
83
91
|
if decorator.id == candidate:
|
|
84
92
|
pth_dec_counter += 1
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
from copy import deepcopy
|
|
3
4
|
from typing import Callable, Any
|
|
4
5
|
|
|
5
6
|
import pandas as pd
|
|
@@ -111,7 +112,9 @@ class OrdinaryFn(PortalAwareClass):
|
|
|
111
112
|
):
|
|
112
113
|
PortalAwareClass.__init__(self, portal=portal)
|
|
113
114
|
if isinstance(fn, OrdinaryFn):
|
|
114
|
-
self.__setstate__(fn.__getstate__())
|
|
115
|
+
self.__setstate__(deepcopy(fn.__getstate__()))
|
|
116
|
+
if self._linked_portal_at_init is None:
|
|
117
|
+
self._linked_portal_at_init = fn._linked_portal_at_init
|
|
115
118
|
#TODO: check this logic
|
|
116
119
|
else:
|
|
117
120
|
if not (callable(fn) or isinstance(fn, str)):
|
|
@@ -264,8 +267,8 @@ class OrdinaryFn(PortalAwareClass):
|
|
|
264
267
|
self._source_code = state["source_code"]
|
|
265
268
|
|
|
266
269
|
|
|
267
|
-
def
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
return
|
|
270
|
+
def __hash_signature_descriptor__(self) -> str:
|
|
271
|
+
descriptor = self.name
|
|
272
|
+
descriptor += "_" + self.__class__.__name__
|
|
273
|
+
descriptor = descriptor.lower()
|
|
274
|
+
return descriptor
|
|
@@ -12,8 +12,8 @@ A ValueAddr is a unique identifier for an immutable value.
|
|
|
12
12
|
Two objects with exactly the same type and value will always have
|
|
13
13
|
exactly the same ValueAddr-es.
|
|
14
14
|
|
|
15
|
-
A ValueAddr consists of 2 strings: a
|
|
16
|
-
A
|
|
15
|
+
A ValueAddr consists of 2 strings: a descriptor, and a hash.
|
|
16
|
+
A descriptor contains human-readable information about an object's type.
|
|
17
17
|
A hash string contains the object's hash signature.
|
|
18
18
|
|
|
19
19
|
Typically, a DataPortal is implemented as
|
|
@@ -205,6 +205,8 @@ class StorableFn(OrdinaryFn):
|
|
|
205
205
|
def _first_visit_to_portal(self, portal: DataPortal) -> None:
|
|
206
206
|
super()._first_visit_to_portal(portal)
|
|
207
207
|
self._persist_initial_config_params(portal)
|
|
208
|
+
with portal:
|
|
209
|
+
_ = ValueAddr(self)
|
|
208
210
|
|
|
209
211
|
|
|
210
212
|
def _persist_initial_config_params(self, portal:DataPortal) -> None:
|
|
@@ -217,11 +219,11 @@ class StorableFn(OrdinaryFn):
|
|
|
217
219
|
return OrdinaryFn.portal.__get__(self)
|
|
218
220
|
|
|
219
221
|
|
|
220
|
-
@portal.setter
|
|
221
|
-
def portal(self, new_portal: DataPortal) -> None:
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
222
|
+
# @portal.setter
|
|
223
|
+
# def portal(self, new_portal: DataPortal) -> None:
|
|
224
|
+
# if not isinstance(new_portal, DataPortal):
|
|
225
|
+
# raise TypeError("portal must be a DataPortal instance")
|
|
226
|
+
# OrdinaryFn.portal.__set__(self, new_portal)
|
|
225
227
|
|
|
226
228
|
|
|
227
229
|
def _get_config_setting(self, key: SafeStrTuple, portal:DataPortal) -> Any:
|
|
@@ -284,81 +286,93 @@ class HashAddr(SafeStrTuple):
|
|
|
284
286
|
Two objects with exactly the same type and value will always have
|
|
285
287
|
exactly the same HashAddr-es.
|
|
286
288
|
|
|
287
|
-
A HashAddr consists of 2 components: a
|
|
288
|
-
A
|
|
289
|
-
A hash string contains the object's hash signature. It may
|
|
290
|
-
an optional
|
|
289
|
+
A HashAddr consists of 2 components: a descriptor, and a hash.
|
|
290
|
+
A descriptor contains human-readable information about an object's type.
|
|
291
|
+
A hash string contains the object's hash signature. It may contain
|
|
292
|
+
an optional suffix, which provides additional human-readable
|
|
291
293
|
information about the object's structure / value.
|
|
292
294
|
"""
|
|
293
295
|
|
|
294
|
-
def __init__(self,
|
|
296
|
+
def __init__(self, descriptor:str
|
|
295
297
|
, hash_signature:str):
|
|
296
|
-
if not isinstance(
|
|
297
|
-
raise TypeError("
|
|
298
|
-
if len(
|
|
299
|
-
raise ValueError("
|
|
300
|
-
SafeStrTuple.__init__(self,
|
|
298
|
+
if not isinstance(descriptor, str) or not isinstance(hash_signature, str):
|
|
299
|
+
raise TypeError("descriptor and hash_signature must be strings")
|
|
300
|
+
if len(descriptor) == 0 or len(hash_signature) == 0:
|
|
301
|
+
raise ValueError("descriptor and hash_signature must not be empty")
|
|
302
|
+
SafeStrTuple.__init__(self,hash_signature[:3], hash_signature[3:6]
|
|
303
|
+
,descriptor, hash_signature[6:])
|
|
301
304
|
|
|
302
305
|
|
|
303
306
|
@property
|
|
304
|
-
def
|
|
307
|
+
def shard(self)->str:
|
|
305
308
|
return self.strings[0]
|
|
306
309
|
|
|
307
310
|
@property
|
|
308
|
-
def
|
|
311
|
+
def subshard(self)->str:
|
|
309
312
|
return self.strings[1]
|
|
310
313
|
|
|
314
|
+
@property
|
|
315
|
+
def descriptor(self) -> str:
|
|
316
|
+
return self.strings[2]
|
|
317
|
+
|
|
318
|
+
@property
|
|
319
|
+
def hash_tail(self)->str:
|
|
320
|
+
return self.strings[3]
|
|
321
|
+
|
|
322
|
+
@property
|
|
323
|
+
def hash_signature(self) -> str:
|
|
324
|
+
return self.shard+self.subshard+self.hash_tail
|
|
325
|
+
|
|
311
326
|
@staticmethod
|
|
312
|
-
def
|
|
327
|
+
def _build_descriptor(x: Any) -> str:
|
|
313
328
|
"""Create a short human-readable summary of an object."""
|
|
314
329
|
|
|
315
|
-
if (hasattr(x, "
|
|
316
|
-
and callable(x.
|
|
317
|
-
|
|
330
|
+
if (hasattr(x, "__hash_signature_descriptor__")
|
|
331
|
+
and callable(x.__hash_signature_descriptor__)):
|
|
332
|
+
descriptor = x.__hash_signature_descriptor__()
|
|
318
333
|
else:
|
|
319
|
-
|
|
334
|
+
descriptor = x.__class__.__name__.lower()
|
|
335
|
+
if (hasattr(x, "shape") and hasattr(x.shape, "__iter__")
|
|
336
|
+
and callable(x.shape.__iter__) and not callable(x.shape)):
|
|
337
|
+
suffix, connector = "_shape_", "_x_"
|
|
338
|
+
for n in x.shape:
|
|
339
|
+
suffix += str(n) + connector
|
|
340
|
+
suffix = suffix[:-len(connector)]
|
|
341
|
+
elif hasattr(x, "__len__") and callable(x.__len__):
|
|
342
|
+
suffix = "_len_" + str(len(x))
|
|
343
|
+
else:
|
|
344
|
+
suffix = ""
|
|
345
|
+
|
|
346
|
+
suffix = replace_unsafe_chars(suffix, replace_with="_")
|
|
347
|
+
descriptor = descriptor + suffix
|
|
320
348
|
|
|
321
|
-
return
|
|
349
|
+
return descriptor
|
|
322
350
|
|
|
323
351
|
|
|
324
352
|
@staticmethod
|
|
325
353
|
def _build_hash_signature(x: Any) -> str:
|
|
326
354
|
"""Create a URL-safe hashdigest for an object."""
|
|
327
|
-
|
|
328
|
-
if (hasattr(x, "shape") and hasattr(x.shape, "__iter__")
|
|
329
|
-
and callable(x.shape.__iter__) and not callable(x.shape)):
|
|
330
|
-
descriptor, connector = "shape_", "_x_"
|
|
331
|
-
for n in x.shape:
|
|
332
|
-
descriptor += str(n) + connector
|
|
333
|
-
descriptor = descriptor[:-len(connector)] + "_"
|
|
334
|
-
elif hasattr(x, "__len__") and callable(x.__len__):
|
|
335
|
-
descriptor = "len_" + str(len(x)) + "_"
|
|
336
|
-
else:
|
|
337
|
-
descriptor = ""
|
|
338
|
-
|
|
339
|
-
descriptor = replace_unsafe_chars(descriptor, replace_with="_")
|
|
340
|
-
raw_hash_signature = get_hash_signature(x)
|
|
341
|
-
hash_signature = descriptor + raw_hash_signature
|
|
342
|
-
|
|
355
|
+
hash_signature = get_hash_signature(x)
|
|
343
356
|
return hash_signature
|
|
344
357
|
|
|
345
358
|
|
|
346
359
|
@classmethod
|
|
347
360
|
def from_strings(cls, *
|
|
348
|
-
,
|
|
361
|
+
, descriptor:str
|
|
349
362
|
, hash_signature:str
|
|
350
363
|
, assert_readiness:bool=True
|
|
351
364
|
) -> HashAddr:
|
|
352
|
-
"""(Re)construct address from text representations of
|
|
365
|
+
"""(Re)construct address from text representations of descriptor and hash"""
|
|
353
366
|
|
|
354
|
-
if not isinstance(
|
|
355
|
-
raise TypeError("
|
|
367
|
+
if not isinstance(descriptor, str) or not isinstance(hash_signature, str):
|
|
368
|
+
raise TypeError("descriptor and hash_signature must be strings")
|
|
356
369
|
|
|
357
|
-
if len(
|
|
358
|
-
raise ValueError("
|
|
370
|
+
if len(descriptor) == 0 or len(hash_signature) == 0:
|
|
371
|
+
raise ValueError("descriptor and hash_signature must not be empty")
|
|
359
372
|
|
|
360
373
|
address = cls.__new__(cls)
|
|
361
|
-
super(cls, address).__init__(
|
|
374
|
+
super(cls, address).__init__(descriptor=descriptor
|
|
375
|
+
, hash_signature=hash_signature)
|
|
362
376
|
if assert_readiness:
|
|
363
377
|
if not address.ready:
|
|
364
378
|
raise ValueError("Address is not ready for retrieving data")
|
|
@@ -407,7 +421,7 @@ class ValueAddr(HashAddr):
|
|
|
407
421
|
uniquely address all possible data objects that the humanity will create
|
|
408
422
|
in the foreseeable future (see, for example ipfs.io).
|
|
409
423
|
|
|
410
|
-
However, an address also includes a
|
|
424
|
+
However, an address also includes a descriptor with an optional suffix.
|
|
411
425
|
It makes it easier for humans to interpret an address,
|
|
412
426
|
and further decreases collision risk.
|
|
413
427
|
"""
|
|
@@ -419,22 +433,22 @@ class ValueAddr(HashAddr):
|
|
|
419
433
|
|
|
420
434
|
if hasattr(data, "get_ValueAddr"):
|
|
421
435
|
data_value_addr = data.get_ValueAddr()
|
|
422
|
-
|
|
436
|
+
descriptor = data_value_addr.descriptor
|
|
423
437
|
hash_signature = data_value_addr.hash_signature
|
|
424
438
|
HashAddr.__init__(self
|
|
425
|
-
|
|
426
|
-
|
|
439
|
+
, descriptor=descriptor
|
|
440
|
+
, hash_signature=hash_signature)
|
|
427
441
|
return
|
|
428
442
|
|
|
429
443
|
assert not isinstance(data, HashAddr), (
|
|
430
444
|
"get_ValueAddr is the only way to "
|
|
431
445
|
+ "convert HashAddr into ValueAddr")
|
|
432
446
|
|
|
433
|
-
|
|
447
|
+
descriptor = self._build_descriptor(data)
|
|
434
448
|
hash_signature = self._build_hash_signature(data)
|
|
435
449
|
HashAddr.__init__(self
|
|
436
|
-
|
|
437
|
-
|
|
450
|
+
, descriptor=descriptor
|
|
451
|
+
, hash_signature=hash_signature)
|
|
438
452
|
|
|
439
453
|
self._value_cache = data
|
|
440
454
|
|
|
@@ -563,13 +577,13 @@ class ValueAddr(HashAddr):
|
|
|
563
577
|
|
|
564
578
|
@classmethod
|
|
565
579
|
def from_strings(cls, *
|
|
566
|
-
,
|
|
580
|
+
, descriptor: str
|
|
567
581
|
, hash_signature: str
|
|
568
582
|
, assert_readiness: bool = True
|
|
569
583
|
) -> HashAddr:
|
|
570
|
-
"""(Re)construct address from text representations of
|
|
584
|
+
"""(Re)construct address from text representations of descriptor and hash"""
|
|
571
585
|
|
|
572
|
-
address = super().from_strings(
|
|
586
|
+
address = super().from_strings(descriptor=descriptor, hash_signature=hash_signature, assert_readiness=False)
|
|
573
587
|
address._containing_portals = set()
|
|
574
588
|
if assert_readiness:
|
|
575
589
|
if not address.ready:
|
|
@@ -59,13 +59,17 @@ class KwArgs(dict):
|
|
|
59
59
|
return unpacked_copy
|
|
60
60
|
|
|
61
61
|
|
|
62
|
-
def pack(self) -> PackedKwArgs:
|
|
62
|
+
def pack(self, store = True) -> PackedKwArgs:
|
|
63
63
|
""" Replace values with their hash addresses."""
|
|
64
|
-
portal = get_active_portal()
|
|
65
64
|
packed_copy = dict()
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
65
|
+
if store:
|
|
66
|
+
portal = get_active_portal()
|
|
67
|
+
with portal:
|
|
68
|
+
for k,v in self.items():
|
|
69
|
+
packed_copy[k] = ValueAddr(v,store=True)
|
|
70
|
+
else:
|
|
71
|
+
for k, v in self.items():
|
|
72
|
+
packed_copy[k] = ValueAddr(v, store=False)
|
|
69
73
|
packed_copy = PackedKwArgs(**packed_copy)
|
|
70
74
|
return packed_copy
|
|
71
75
|
|
|
@@ -40,11 +40,16 @@ class LoggingFn(StorableFn):
|
|
|
40
40
|
, excessive_logging: bool|Joker = KEEP_CURRENT
|
|
41
41
|
, portal: LoggingCodePortal | None = None
|
|
42
42
|
):
|
|
43
|
+
super().__init__(fn=fn, portal=portal)
|
|
44
|
+
|
|
43
45
|
if not isinstance(excessive_logging, (bool, Joker)):
|
|
44
46
|
raise TypeError(
|
|
45
47
|
"excessive_logging must be a boolean or Joker, "
|
|
46
48
|
f"got {type(excessive_logging)}")
|
|
47
|
-
|
|
49
|
+
|
|
50
|
+
if excessive_logging is KEEP_CURRENT and isinstance(fn, LoggingFn):
|
|
51
|
+
excessive_logging = fn._ephemeral_config_params_at_init["excessive_logging"]
|
|
52
|
+
|
|
48
53
|
self._ephemeral_config_params_at_init[
|
|
49
54
|
"excessive_logging"] = excessive_logging
|
|
50
55
|
|
|
@@ -70,12 +75,6 @@ class LoggingFn(StorableFn):
|
|
|
70
75
|
return StorableFn.portal.__get__(self)
|
|
71
76
|
|
|
72
77
|
|
|
73
|
-
# @portal.setter
|
|
74
|
-
# def portal(self, new_portal: LoggingCodePortal) -> None:
|
|
75
|
-
# if not isinstance(new_portal, LoggingCodePortal):
|
|
76
|
-
# raise TypeError("portal must be a LoggingCodePortal instance")
|
|
77
|
-
# StorableFn.portal.__set__(self, new_portal)
|
|
78
|
-
|
|
79
78
|
|
|
80
79
|
class LoggingFnCallSignature:
|
|
81
80
|
"""A signature of a call to a (logging) function.
|
|
@@ -113,16 +112,16 @@ class LoggingFnCallSignature:
|
|
|
113
112
|
def __getstate__(self):
|
|
114
113
|
"""This method is called when the object is pickled."""
|
|
115
114
|
state = dict(
|
|
116
|
-
|
|
117
|
-
,
|
|
115
|
+
fn_addr=self._fn_addr
|
|
116
|
+
, kwargs_addr=self._kwargs_addr)
|
|
118
117
|
return state
|
|
119
118
|
|
|
120
119
|
|
|
121
120
|
def __setstate__(self, state):
|
|
122
121
|
"""This method is called when the object is unpickled."""
|
|
123
122
|
self._invalidate_cache()
|
|
124
|
-
self._fn_addr = state["
|
|
125
|
-
self._kwargs_addr = state["
|
|
123
|
+
self._fn_addr = state["fn_addr"]
|
|
124
|
+
self._kwargs_addr = state["kwargs_addr"]
|
|
126
125
|
|
|
127
126
|
|
|
128
127
|
def _invalidate_cache(self):
|
|
@@ -181,11 +180,11 @@ class LoggingFnCallSignature:
|
|
|
181
180
|
return self.fn.excessive_logging
|
|
182
181
|
|
|
183
182
|
|
|
184
|
-
def
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
return
|
|
183
|
+
def __hash_signature_descriptor__(self) -> str:
|
|
184
|
+
descriptor = self.fn_name
|
|
185
|
+
descriptor += "_" + self.__class__.__name__
|
|
186
|
+
descriptor = descriptor.lower()
|
|
187
|
+
return descriptor
|
|
189
188
|
|
|
190
189
|
|
|
191
190
|
def execute(self) -> Any:
|
|
@@ -4,7 +4,8 @@ import builtins
|
|
|
4
4
|
from typing import Callable, Any
|
|
5
5
|
|
|
6
6
|
from persidict import PersiDict, Joker, KEEP_CURRENT
|
|
7
|
-
|
|
7
|
+
from .._020_ordinary_code_portals.code_normalizer import _pythagoras_decorator_names
|
|
8
|
+
from .. import DataPortal
|
|
8
9
|
from .._040_logging_code_portals import KwArgs
|
|
9
10
|
|
|
10
11
|
from .._060_autonomous_code_portals.names_usage_analyzer import (
|
|
@@ -31,34 +32,37 @@ class AutonomousCodePortal(SafeCodePortal):
|
|
|
31
32
|
|
|
32
33
|
class AutonomousFn(SafeFn):
|
|
33
34
|
|
|
34
|
-
|
|
35
|
+
_fixed_kwargs_cached: KwArgs | None
|
|
36
|
+
_fixed_kwargs_packed: KwArgs | None
|
|
35
37
|
|
|
36
38
|
def __init__(self, fn: Callable|str|SafeFn
|
|
37
39
|
, fixed_kwargs: dict|None = None
|
|
38
40
|
, excessive_logging: bool|Joker = KEEP_CURRENT
|
|
39
41
|
, portal: AutonomousCodePortal|None = None):
|
|
40
|
-
|
|
41
|
-
,fn=fn
|
|
42
|
+
super().__init__(fn=fn
|
|
42
43
|
, portal = portal
|
|
43
44
|
, excessive_logging = excessive_logging)
|
|
44
45
|
|
|
46
|
+
fixed_kwargs = dict() if fixed_kwargs is None else fixed_kwargs
|
|
47
|
+
fixed_kwargs = KwArgs(**fixed_kwargs)
|
|
48
|
+
fixed_kwargs_packed = fixed_kwargs.pack(store=False)
|
|
49
|
+
|
|
45
50
|
if isinstance(fn, AutonomousFn):
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
51
|
+
self._fixed_kwargs_packed.update(fixed_kwargs_packed)
|
|
52
|
+
self._fixed_kwargs_cached = KwArgs(**{**fn.fixed_kwargs, **fixed_kwargs})
|
|
53
|
+
else:
|
|
54
|
+
self._fixed_kwargs_cached = fixed_kwargs
|
|
55
|
+
self._fixed_kwargs_packed = fixed_kwargs_packed
|
|
49
56
|
|
|
50
57
|
fn_name = self.name
|
|
51
58
|
|
|
52
|
-
fixed_kwargs = dict() if fixed_kwargs is None else fixed_kwargs
|
|
53
|
-
self._fixed_kwargs = KwArgs(fixed_kwargs)
|
|
54
|
-
|
|
55
59
|
analyzer = analyze_names_in_function(self.source_code)
|
|
56
60
|
normalized_source = analyzer["normalized_source"]
|
|
57
61
|
analyzer = analyzer["analyzer"]
|
|
58
62
|
assert self.source_code == normalized_source
|
|
59
63
|
|
|
60
64
|
nonlocal_names = analyzer.names.explicitly_nonlocal_unbound_deep
|
|
61
|
-
all_decorators =
|
|
65
|
+
all_decorators = _pythagoras_decorator_names
|
|
62
66
|
# all_decorators = sys.modules["pythagoras"].all_decorators
|
|
63
67
|
nonlocal_names -= set(all_decorators) #????????????
|
|
64
68
|
|
|
@@ -71,7 +75,8 @@ class AutonomousFn(SafeFn):
|
|
|
71
75
|
|
|
72
76
|
import_required = analyzer.names.explicitly_global_unbound_deep
|
|
73
77
|
import_required |= analyzer.names.unclassified_deep
|
|
74
|
-
import_required -= set(pth.primary_decorators)
|
|
78
|
+
# import_required -= set(pth.primary_decorators)
|
|
79
|
+
import_required -= {"pure", "autonomous"}
|
|
75
80
|
builtin_names = set(dir(builtins))
|
|
76
81
|
import_required -= builtin_names
|
|
77
82
|
pth_names = set(self._available_names())
|
|
@@ -84,35 +89,48 @@ class AutonomousFn(SafeFn):
|
|
|
84
89
|
+ f" without importing them inside the function body")
|
|
85
90
|
|
|
86
91
|
|
|
92
|
+
@property
|
|
93
|
+
def fixed_kwargs(self) -> KwArgs:
|
|
94
|
+
if not hasattr(self, "_fixed_kwargs_cached"):
|
|
95
|
+
with self.portal:
|
|
96
|
+
self._fixed_kwargs_cached = self._fixed_kwargs_packed.unpack()
|
|
97
|
+
return self._fixed_kwargs_cached
|
|
98
|
+
|
|
99
|
+
|
|
87
100
|
def execute(self, **kwargs) -> Any:
|
|
88
101
|
with self.portal:
|
|
89
|
-
overlapping_keys = set(kwargs.keys()) & set(self.
|
|
102
|
+
overlapping_keys = set(kwargs.keys()) & set(self.fixed_kwargs.keys())
|
|
90
103
|
assert len(overlapping_keys) == 0
|
|
91
|
-
kwargs.update(self.
|
|
92
|
-
return
|
|
104
|
+
kwargs.update(self.fixed_kwargs)
|
|
105
|
+
return super().execute(**kwargs)
|
|
93
106
|
|
|
94
107
|
|
|
95
108
|
def fix_kwargs(self, **kwargs) -> AutonomousFn:
|
|
96
|
-
overlapping_keys = set(kwargs.keys()) & set(self.
|
|
109
|
+
overlapping_keys = set(kwargs.keys()) & set(self.fixed_kwargs.keys())
|
|
97
110
|
assert len(overlapping_keys) == 0
|
|
98
|
-
new_fixed_kwargs = self.
|
|
99
|
-
|
|
100
|
-
new_fn = AutonomousFn(self.source_code
|
|
101
|
-
, fixed_kwargs=new_fixed_kwargs
|
|
102
|
-
, portal=self._linked_portal)
|
|
111
|
+
new_fixed_kwargs = {**self.fixed_kwargs,**kwargs}
|
|
112
|
+
new_fn = type(self)(fn=self, fixed_kwargs=new_fixed_kwargs)
|
|
103
113
|
return new_fn
|
|
104
114
|
|
|
105
115
|
|
|
116
|
+
def _first_visit_to_portal(self, portal: DataPortal) -> None:
|
|
117
|
+
super()._first_visit_to_portal(portal)
|
|
118
|
+
if hasattr(self,"_fixed_kwargs_cached"):
|
|
119
|
+
with portal:
|
|
120
|
+
_ = self._fixed_kwargs_cached.pack()
|
|
121
|
+
|
|
122
|
+
|
|
106
123
|
def __getstate__(self):
|
|
107
124
|
"""This method is called when the object is pickled."""
|
|
108
125
|
state = super().__getstate__()
|
|
109
|
-
state["
|
|
126
|
+
state["fixed_kwargs_packed"] = self._fixed_kwargs_packed
|
|
110
127
|
return state
|
|
111
128
|
|
|
129
|
+
|
|
112
130
|
def __setstate__(self, state):
|
|
113
131
|
"""This method is called when the object is unpickled."""
|
|
114
132
|
super().__setstate__(state)
|
|
115
|
-
self.
|
|
133
|
+
self._fixed_kwargs_packed = state["fixed_kwargs_packed"]
|
|
116
134
|
|
|
117
135
|
|
|
118
136
|
@property
|
|
@@ -120,8 +138,8 @@ class AutonomousFn(SafeFn):
|
|
|
120
138
|
return SafeFn.portal.__get__(self)
|
|
121
139
|
|
|
122
140
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
141
|
+
def _invalidate_cache(self):
|
|
142
|
+
super()._invalidate_cache()
|
|
143
|
+
if hasattr(self, "_fixed_kwargs_cached"):
|
|
144
|
+
assert hasattr(self, "_fixed_kwargs_packed"), "Premature cache invalidation: fixed_kwargs_packed is missing."
|
|
145
|
+
del self._fixed_kwargs_cached
|
|
File without changes
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from .._060_autonomous_code_portals import autonomous
|
|
2
|
+
from .OK_const import OK, OKClass
|
|
3
|
+
from .system_utils import *
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
# def free_ram_bytes(packed_kwargs, fn_addr, required_memory):
|
|
7
|
+
# import psutil
|
|
8
|
+
# mem_info = psutil.virtual_memory()
|
|
9
|
+
# if mem_info.available >= required_memory:
|
|
10
|
+
# return OK
|
|
11
|
+
#
|
|
12
|
+
#
|
|
13
|
+
# def RAM_K(limit:int) -> AutonomousFn:
|
|
14
|
+
# global free_ram_bytes
|
|
15
|
+
# if not isinstance(free_ram_bytes, AutonomousFn):
|
|
16
|
+
# free_ram_bytes = autonomous()(free_ram_bytes)
|
|
17
|
+
# limit = int(limit/1024)
|
|
18
|
+
# assert limit > 0
|
|
19
|
+
# return free_ram_bytes.fix_kwargs(required_memory = limit)
|
|
20
|
+
#
|
|
21
|
+
#
|
|
22
|
+
# def RAM_M(limit:int) -> AutonomousFn:
|
|
23
|
+
# limit = int(limit/1024)
|
|
24
|
+
# return RAM_K(limit)
|
|
25
|
+
#
|
|
26
|
+
#
|
|
27
|
+
# def RAM_G(limit:int) -> AutonomousFn:
|
|
28
|
+
# limit = int(limit/1024)
|
|
29
|
+
# return RAM_M(limit)
|
|
30
|
+
#
|
|
31
|
+
#
|
|
32
|
+
# def RAM_T(limit:int) -> AutonomousFn:
|
|
33
|
+
# limit = int(limit/1024)
|
|
34
|
+
# return RAM_G(limit)
|
|
35
|
+
|
|
36
|
+
@autonomous()
|
|
37
|
+
def at_least_X_G_RAM_free_check(x:int)->bool|OKClass:
|
|
38
|
+
ram = pth.get_available_ram_mb()/1024
|
|
39
|
+
if ram >= x:
|
|
40
|
+
return pth.OK
|
|
41
|
+
else:
|
|
42
|
+
return False
|
|
43
|
+
|
|
44
|
+
def at_least_X_G_RAM_free(x:int):
|
|
45
|
+
assert isinstance(x, int)
|
|
46
|
+
assert x > 0
|
|
47
|
+
return at_least_X_G_RAM_free_check.fix_kwargs(x=x)
|
|
48
|
+
|
|
49
|
+
at_least_1_G_RAM_free = at_least_X_G_RAM_free(x=1)
|
|
50
|
+
at_least_2_G_RAM_free = at_least_X_G_RAM_free(x=2)
|
|
51
|
+
at_least_4_G_RAM_free = at_least_X_G_RAM_free(x=4)
|
|
52
|
+
at_least_8_G_RAM_free = at_least_X_G_RAM_free(x=8)
|
|
53
|
+
at_least_16_G_RAM_free = at_least_X_G_RAM_free(x=16)
|
|
54
|
+
at_least_32_G_RAM_free = at_least_X_G_RAM_free(x=32)
|
|
55
|
+
at_least_64_G_RAM_free = at_least_X_G_RAM_free(x=64)
|
|
56
|
+
at_least_128_G_RAM_free = at_least_X_G_RAM_free(x=128)
|
|
57
|
+
at_least_256_G_RAM_free = at_least_X_G_RAM_free(x=256)
|
|
58
|
+
at_least_512_G_RAM_free = at_least_X_G_RAM_free(x=512)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from .OK_const import *
|
|
2
2
|
from .protected_portal_core_classes import *
|
|
3
3
|
from .protected_decorators import *
|
|
4
|
-
from .
|
|
5
|
-
from .
|
|
6
|
-
from .
|
|
4
|
+
from .GPU_pre_validators import *
|
|
5
|
+
from .python_packages_pre_validators import *
|
|
6
|
+
from .RAM_pre_validators import *
|
|
7
|
+
from .system_utils import *
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import ast
|
|
2
|
-
from typing import List
|
|
2
|
+
from typing import List, Set
|
|
3
3
|
|
|
4
|
-
|
|
4
|
+
|
|
5
|
+
def check_if_fn_accepts_args(required_arg_names: List[str]|Set[str], fn: str) -> bool:
|
|
5
6
|
"""
|
|
6
7
|
Analyzes the source code (string) `fn` of a Python function and determines
|
|
7
8
|
if it can accept the arguments named in `required_arg_names`.
|
|
@@ -2,18 +2,19 @@
|
|
|
2
2
|
|
|
3
3
|
from typing import Callable
|
|
4
4
|
|
|
5
|
-
from
|
|
5
|
+
from .validator_fn_classes import ValidatorFn
|
|
6
|
+
from .._060_autonomous_code_portals import autonomous
|
|
6
7
|
from .protected_portal_core_classes import ProtectedFn, ProtectedCodePortal
|
|
7
8
|
from persidict import Joker, KEEP_CURRENT
|
|
8
9
|
|
|
9
10
|
class protected(autonomous):
|
|
10
11
|
|
|
11
|
-
|
|
12
|
-
|
|
12
|
+
_pre_validators: list[ValidatorFn] | None
|
|
13
|
+
_post_validators: list[ValidatorFn] | None
|
|
13
14
|
|
|
14
15
|
def __init__(self
|
|
15
|
-
,
|
|
16
|
-
,
|
|
16
|
+
, pre_validators: list[ValidatorFn] | None = None
|
|
17
|
+
, post_validators: list[ValidatorFn] | None = None
|
|
17
18
|
, fixed_kwargs: dict | None = None
|
|
18
19
|
, excessive_logging: bool|Joker = KEEP_CURRENT
|
|
19
20
|
, portal: ProtectedCodePortal | None = None
|
|
@@ -24,15 +25,15 @@ class protected(autonomous):
|
|
|
24
25
|
, portal=portal
|
|
25
26
|
, excessive_logging=excessive_logging
|
|
26
27
|
, fixed_kwargs=fixed_kwargs)
|
|
27
|
-
self.
|
|
28
|
-
self.
|
|
28
|
+
self._pre_validators = pre_validators
|
|
29
|
+
self._post_validators = post_validators
|
|
29
30
|
|
|
30
31
|
|
|
31
32
|
def __call__(self, fn: Callable|str) -> ProtectedFn:
|
|
32
33
|
wrapper = ProtectedFn(fn
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
34
|
+
, portal=self._portal
|
|
35
|
+
, pre_validators=self._pre_validators
|
|
36
|
+
, fixed_kwargs=self._fixed_kwargs
|
|
37
|
+
, post_validators=self._post_validators
|
|
38
|
+
, excessive_logging=self._excessive_logging)
|
|
38
39
|
return wrapper
|