pythagoras 0.20.52__py3-none-any.whl → 0.21.0__py3-none-any.whl

pythagoras/_070_protected_code_portals/protected_portal_core_classes.py

@@ -5,6 +5,8 @@ from typing import Callable, Any, List
 
 from persidict import PersiDict, Joker, KEEP_CURRENT
 
+from .validator_fn_classes import ValidatorFn, PreValidatorFn, PostValidatorFn, SimplePreValidatorFn, \
+    ComplexPreValidatorFn
 from .._030_data_portals import DataPortal
 from .._040_logging_code_portals import KwArgs
 from .._030_data_portals import ValueAddr
@@ -32,44 +34,43 @@ class ProtectedCodePortal(AutonomousCodePortal):
 
 class ProtectedFn(AutonomousFn):
 
-    _guards: list[AutonomousFn] | None
-    _validators: list[AutonomousFn] | None
-    _guards_addrs: list[ValueAddr]
-    _validators_addrs: list[ValueAddr]
+    _pre_validators_cached: list[AutonomousFn] | None
+    _post_validators_cached: list[AutonomousFn] | None
+    _pre_validators_addrs: list[ValueAddr]
+    _post_validators_addrs: list[ValueAddr]
 
-    validators_arg_names = ["packed_kwargs", "fn_addr", "result"]
-    guards_arg_names = ["packed_kwargs", "fn_addr"]
+    post_validators_arg_names = ["packed_kwargs", "fn_addr", "result"]
+    pre_validators_arg_names = ["packed_kwargs", "fn_addr"]
 
     def __init__(self, fn: Callable | str
-                 , guards: list[AutonomousFn] | List[Callable] | None = None
-                 , validators: list[AutonomousFn] | List[Callable] | None = None
+                 , pre_validators: list[AutonomousFn] | List[Callable] | None = None
+                 , post_validators: list[AutonomousFn] | List[Callable] | None = None
                  , excessive_logging: bool|None = KEEP_CURRENT
                  , fixed_kwargs: dict | None = None
-                 , portal: AutonomousCodePortal | None = None):
-        AutonomousFn.__init__(self
-            ,fn=fn
+                 , portal: ProtectedCodePortal | None = None):
+        super().__init__(fn=fn
             , portal = portal
             , fixed_kwargs=fixed_kwargs
             , excessive_logging = excessive_logging)
 
+        pre_validators = self._normalize_validators(pre_validators, PreValidatorFn)
+        post_validators = self._normalize_validators(post_validators, PostValidatorFn)
+
         if isinstance(fn, ProtectedFn):
-            assert guards is None
-            assert validators is None
-            return
+            pre_validators += fn.pre_validators
+            post_validators += fn.post_validators
 
-        self._guards = self._normalize_protectors(
-            guards, ProtectedFn.guards_arg_names)
-        self._validators = self._normalize_protectors(
-            validators, ProtectedFn.validators_arg_names)
-        self._guards_addrs = [ValueAddr(g, store=False) for g in self._guards]
-        self._validators_addrs = [ValueAddr(v, store=False) for v in self._validators]
+        self._pre_validators_cached = self._normalize_validators(pre_validators, PreValidatorFn)
+        self._post_validators_cached = self._normalize_validators(post_validators, PostValidatorFn)
+        self._pre_validators_addrs = [ValueAddr(g, store=False) for g in self._pre_validators_cached]
+        self._post_validators_addrs = [ValueAddr(v, store=False) for v in self._post_validators_cached]
 
 
     def __getstate__(self):
         """This method is called when the object is pickled."""
         state = super().__getstate__()
-        state["guards_addrs"] = self._guards_addrs
-        state["validators_addrs"] = self._validators_addrs
+        state["pre_validators_addrs"] = self._pre_validators_addrs
+        state["post_validators_addrs"] = self._post_validators_addrs
         return state
 
 
@@ -77,53 +78,60 @@ class ProtectedFn(AutonomousFn):
         """This method is called when the object is unpickled."""
         self._invalidate_cache()
         super().__setstate__(state)
-        self._guards_addrs = state["guards_addrs"]
-        self._validators_addrs = state["validators_addrs"]
+        self._pre_validators_addrs = state["pre_validators_addrs"]
+        self._post_validators_addrs = state["post_validators_addrs"]
 
 
     def _first_visit_to_portal(self, portal: DataPortal) -> None:
+        """Register an object in a portal that the object has not seen before."""
         super()._first_visit_to_portal(portal)
-        with portal:
-            if hasattr(self, "_guards") and self._guards is not None:
-                new_guards_addrs = [ValueAddr(g) for g in self._guards]
-                assert self._guards_addrs == new_guards_addrs
-            if hasattr(self, "_validators") and self._validators is not None:
-                new_validators_addrs = [ValueAddr(v) for v in self._validators]
-                assert self._validators_addrs == new_validators_addrs
+
+        if hasattr(self,"_pre_validators_cached"):
+            with portal:
+                _ = [ValueAddr(g) for g in self._pre_validators_cached]
+
+        if hasattr(self,"_post_validators_cached"):
+            with portal:
+                _ = [ValueAddr(g) for g in self._post_validators_cached]
 
 
     @property
-    def guards(self) -> list[AutonomousFn]:
-        if not hasattr(self, "_guards") or self._guards is None:
-            self._guards = [addr.get() for addr in self._guards_addrs]
-        return self._guards
+    def pre_validators(self) -> list[AutonomousFn]:
+        if not hasattr(self, "_pre_validators_cached"):
+            self._pre_validators_cached = [addr.get() for addr in self._pre_validators_addrs]
+        return self._pre_validators_cached
 
 
     @property
-    def validators(self) -> list[AutonomousFn]:
-        if not hasattr(self, "_validators") or self._validators is None:
-            self._validators = [addr.get() for addr in self._validators_addrs]
-        return self._validators
+    def post_validators(self) -> list[AutonomousFn]:
+        if not hasattr(self, "_post_validators_cached"):
+            self._post_validators_cached = [addr.get() for addr in self._post_validators_addrs]
+        return self._post_validators_cached
 
 
     def can_be_executed(self, kw_args: KwArgs) -> bool:
         with self.portal as portal:
             kw_args = kw_args.pack()
-            guards = copy(self.guards)
-            portal.entropy_infuser.shuffle(guards)
-            for guard in guards:
-                if guard(packed_kwargs=kw_args, fn_addr = self.addr) is not OK:
+            pre_validators = copy(self.pre_validators)
+            portal.entropy_infuser.shuffle(pre_validators)
+            for pre_validator in pre_validators:
+                pre_validation_result = None
+                if isinstance(pre_validator, SimplePreValidatorFn):
+                    pre_validation_result = pre_validator()
+                else:
+                    pre_validation_result = pre_validator(packed_kwargs=kw_args, fn_addr = self.addr)
+                if pre_validation_result is not OK:
                     return False
             return True
 
 
-    def validate_result(self, kw_args: KwArgs,  result: Any) -> bool:
+    def validate_execution_result(self, kw_args: KwArgs, result: Any) -> bool:
         with self.portal as portal:
             kw_args = kw_args.pack()
-            validators = copy(self.validators)
-            portal.entropy_infuser.shuffle(validators)
-            for validator in validators:
-                if validator(packed_kwargs=kw_args, fn_addr = self.addr
+            post_validators = copy(self.post_validators)
+            portal.entropy_infuser.shuffle(post_validators)
+            for post_validator in post_validators:
+                if post_validator(packed_kwargs=kw_args, fn_addr = self.addr
                         , result=result) is not OK:
                     return False
             return True
@@ -134,38 +142,44 @@ class ProtectedFn(AutonomousFn):
             kw_args = KwArgs(**kwargs)
             assert self.can_be_executed(kw_args)
             result = super().execute(**kwargs)
-            assert self.validate_result(kw_args, result)
+            assert self.validate_execution_result(kw_args, result)
             return result
 
 
-    def _normalize_protectors(self
-            , protectors: list[AutonomousFn] | None
-            , required_arg_names: list[str]
-            ) -> list[AutonomousFn]:
-        """Return list of protectors in a normalized form.
+    def _normalize_validators(self
+            , validators: list[ValidatorFn] | None
+            , validator_type: type
+            ) -> list[ValidatorFn]:
+        """Return list of validators in a normalized form.
 
-        All the functions-protectors are converted to AutonomousFn objects,
+        All the functions-validators are converted to AutonomousFn objects,
         and returned as a list, sorted by functions' hash signatures.
         """
-        if protectors is None:
+        assert validator_type in {PreValidatorFn, PostValidatorFn}
+        if validators is None:
             return []
-        if not isinstance(protectors, list):
-            if callable(protectors) or isinstance(protectors, str):
-                protectors = [protectors]
-        assert isinstance(protectors, list)
-        protectors = flatten_list(protectors)
-        new_protectors = []
-        for protector in protectors:
-            protector = AutonomousFn(fn=protector, excessive_logging= KEEP_CURRENT
-                , portal=None, fixed_kwargs=None)
-            assert isinstance(protector, AutonomousFn)
-            assert check_if_fn_accepts_args(
-                required_arg_names, protector.source_code)
-            new_protectors.append(protector)
-        protectors = {f.hash_signature: f for f in new_protectors}
-        protectors = sort_dict_by_keys(protectors)
-        protectors = list(protectors.values())
-        return protectors
+        if not isinstance(validators, list):
+            if callable(validators) or isinstance(validators, ValidatorFn) or isinstance(validators, str):
+                validators = [validators]
+        assert isinstance(validators, list)
+        validators = flatten_list(validators)
+        new_validators = []
+        for validator in validators:
+            if not isinstance(validator, validator_type):
+                if validator_type is PreValidatorFn:
+                    try:
+                        validator = ComplexPreValidatorFn(validator)
+                    except:
+                        validator = SimplePreValidatorFn(validator)
+                elif validator_type is PostValidatorFn:
+                    validator = PostValidatorFn(validator)
+                else:
+                    raise TypeError(f"Unknown type {validator_type}")
+            new_validators.append(validator)
+        validators = {f.hash_signature: f for f in new_validators}
+        validators = sort_dict_by_keys(validators)
+        validators = list(validators.values())
+        return validators
 
 
     @property
@@ -173,8 +187,11 @@ class ProtectedFn(AutonomousFn):
         return AutonomousFn.portal.__get__(self)
 
 
-    # @portal.setter
-    # def portal(self, new_portal: ProtectedCodePortal) -> None:
-    #     if not isinstance(new_portal, ProtectedCodePortal):
-    #         raise TypeError("portal must be a ProtectedCodePortal instance")
-    #     AutonomousFn.portal.__set__(self, new_portal)
+    def _invalidate_cache(self):
+        super()._invalidate_cache()
+        if hasattr(self, "_post_validators_cached"):
+            assert hasattr(self, "_post_validators_addrs"), "Premature cache invalidation: _post_validators_addrs is missing."
+            del self._post_validators_cached
+        if hasattr(self, "_pre_validators_cached"):
+            assert hasattr(self, "_pre_validators_addrs"), "Premature cache invalidation: _pre_validators_addrs is missing."
+            del self._pre_validators_cached

pythagoras/{_090_swarming_portals → _070_protected_code_portals}/system_utils.py

@@ -1,5 +1,6 @@
 import os
 import psutil
+import pynvml
 
 def get_available_ram_mb() -> int:
     """Returns the amount of available RAM in MB. """
@@ -7,7 +8,7 @@ def get_available_ram_mb() -> int:
     return int(free_ram)
 
 
-def get_available_cpu_cores() -> float:
+def get_unused_cpu_cores() -> float:
     """Returns the free (logical) CPU capacity"""
 
     cnt = psutil.cpu_count(logical=True) or 1
@@ -50,3 +51,30 @@ def get_current_process_start_time() -> int:
     return get_process_start_time(get_current_process_id())
 
 
+def get_unused_nvidia_gpus() -> float:
+    """Returns the total unused GPU capacity as a float value.
+
+    The function calculates the sum of unused capacity across all available NVIDIA GPUs.
+    For example, 2.0 means two completely unused GPUs
+    0 is returned if no GPUs or on error.
+    """
+    try:
+        pynvml.nvmlInit()
+        device_count = pynvml.nvmlDeviceGetCount()
+        unused_capacity = 0.0
+
+        for i in range(device_count):
+            handle = pynvml.nvmlDeviceGetHandleByIndex(i)
+            utilization = pynvml.nvmlDeviceGetUtilizationRates(handle)
+            unused_capacity += max(0.0, 100.0 - utilization.gpu)  # Clamp to avoid negative values
+
+        return unused_capacity / 100.0
+
+    except pynvml.NVMLError as e:
+        # Return 0.0 on any NVML error (no GPUs, driver issues, etc.)
+        return 0.0
+    finally:
+        try:
+            pynvml.nvmlShutdown()
+        except:
+            pass  # Safe cleanup even if initialization failed

pythagoras/_070_protected_code_portals/validator_fn_classes.py

@@ -0,0 +1,95 @@
+from typing import Callable
+
+from persidict import KEEP_CURRENT, Joker
+
+from pythagoras._060_autonomous_code_portals import AutonomousFn, AutonomousCodePortal
+from .fn_arg_names_checker import check_if_fn_accepts_args
+
+
+class ValidatorFn(AutonomousFn):
+    def __init__(self, fn: Callable | str | AutonomousFn
+        , fixed_kwargs: dict | None = None
+        , excessive_logging: bool | Joker = KEEP_CURRENT
+        , portal: AutonomousCodePortal | None = None):
+        super().__init__(
+            fn=fn
+            , fixed_kwargs=fixed_kwargs
+            , excessive_logging=excessive_logging
+            , portal=portal)
+
+        check_if_fn_accepts_args(self.get_allowed_kwargs_names(), self.source_code)
+
+
+    @classmethod
+    def get_allowed_kwargs_names(cls)->set[str]:
+        raise NotImplementedError("This method must be overridden")
+
+
+    def execute(self,**kwargs):
+        assert set(kwargs) == self.get_allowed_kwargs_names()
+        return super().execute(**kwargs)
+
+
+class PreValidatorFn(ValidatorFn):
+    def __init__(self, fn: Callable | str | AutonomousFn
+        , fixed_kwargs: dict | None = None
+        , excessive_logging: bool | Joker = KEEP_CURRENT
+        , portal: AutonomousCodePortal | None = None):
+        super().__init__(
+            fn=fn
+            , fixed_kwargs=fixed_kwargs
+            , excessive_logging=excessive_logging
+            , portal=portal)
+
+
+class SimplePreValidatorFn(PreValidatorFn):
+    def __init__(self, fn: Callable | str | AutonomousFn
+        , fixed_kwargs: dict | None = None
+        , excessive_logging: bool | Joker = KEEP_CURRENT
+        , portal: AutonomousCodePortal | None = None):
+        super().__init__(
+            fn=fn
+            , fixed_kwargs=fixed_kwargs
+            , excessive_logging=excessive_logging
+            , portal=portal)
+
+
+    @classmethod
+    def get_allowed_kwargs_names(cls) -> set[str]:
+        """Simple pre-validators do not take any inputs."""
+        return set()
+
+
+class ComplexPreValidatorFn(PreValidatorFn):
+    def __init__(self, fn: Callable | str | AutonomousFn
+        , fixed_kwargs: dict | None = None
+        , excessive_logging: bool | Joker = KEEP_CURRENT
+        , portal: AutonomousCodePortal | None = None):
+        super().__init__(
+            fn=fn
+            , fixed_kwargs=fixed_kwargs
+            , excessive_logging=excessive_logging
+            , portal=portal)
+
+
+    @classmethod
+    def get_allowed_kwargs_names(cls) -> set[str]:
+        """Complex pre-validators use info about the function and its input arguments."""
+        return {"packed_kwargs", "fn_addr"}
+
+
+class PostValidatorFn(ValidatorFn):
+    def __init__(self, fn: Callable | str | AutonomousFn
+        , fixed_kwargs: dict | None = None
+        , excessive_logging: bool | Joker = KEEP_CURRENT
+        , portal: AutonomousCodePortal | None = None):
+        super().__init__(
+            fn=fn
+            , fixed_kwargs=fixed_kwargs
+            , excessive_logging=excessive_logging
+            , portal=portal)
+
+    @classmethod
+    def get_allowed_kwargs_names(cls) -> set[str]:
+        """Post-validators use info about the function, its input arguments and returned value."""
+        return {"packed_kwargs", "fn_addr", "result" }

pythagoras/_080_pure_code_portals/pure_core_classes.py

@@ -93,18 +93,18 @@ class PureCodePortal(ProtectedCodePortal):
 class PureFn(ProtectedFn):
 
     def __init__(self, fn: Callable | str
-                 , guards: list[AutonomousFn] | List[Callable] | None = None
-                 , validators: list[AutonomousFn] | List[Callable] | None = None
+                 , pre_validators: list[AutonomousFn] | List[Callable] | None = None
+                 , post_validators: list[AutonomousFn] | List[Callable] | None = None
                  , excessive_logging: bool | Joker = KEEP_CURRENT
                  , fixed_kwargs: dict | None = None
                  , portal: PureCodePortal | None = None):
         ProtectedFn.__init__(self
-            ,fn=fn
-            , portal = portal
-            , fixed_kwargs=fixed_kwargs
-            , excessive_logging = excessive_logging
-            , guards=guards
-            , validators=validators)
+                             , fn=fn
+                             , portal = portal
+                             , fixed_kwargs=fixed_kwargs
+                             , excessive_logging = excessive_logging
+                             , pre_validators=pre_validators
+                             , post_validators=post_validators)
 
 
     def get_address(self, **kwargs) -> PureFnExecutionResultAddr:
@@ -261,13 +261,13 @@ class PureFnExecutionResultAddr(HashAddr):
         assert isinstance(fn, PureFn)
         with fn.portal as portal:
             self._kwargs_cache = KwArgs(**arguments)
+            self._fn_cache = fn
             signature = LoggingFnCallSignature(fn, self._kwargs_cache)
             self._call_signature_cache = signature
             tmp = ValueAddr(signature)
-            new_prefix = fn.name +"_result_addr"
+            new_descriptor = fn.name +"_result_addr"
             new_hash_signature = tmp.hash_signature
-            super().__init__(new_prefix, new_hash_signature)
-            self._fn_cache = fn
+            super().__init__(new_descriptor, new_hash_signature)
 
 
     def _invalidate_cache(self):
@@ -289,13 +289,11 @@ class PureFnExecutionResultAddr(HashAddr):
             del self._call_signature_cache
 
 
-    def get_ValueAddr(self): #?????????????
-        # prefix = self.fn.name.lower() + "_"
-        # prefix += LoggingFnCallSignature.__name__.lower()
-        prefix = self.prefix.removesuffix("_result_addr")
-        prefix += "_" + LoggingFnCallSignature.__name__.lower()
+    def get_ValueAddr(self):
+        descriptor = self.descriptor.removesuffix("_result_addr")
+        descriptor += "_" + LoggingFnCallSignature.__name__.lower()
         return ValueAddr.from_strings(  # TODO: refactor this
-            prefix = prefix
+            descriptor= descriptor
             , hash_signature=self.hash_signature)
 
 
@@ -465,7 +463,7 @@ class PureFnExecutionResultAddr(HashAddr):
     def can_be_executed(self) -> bool: #*#*#
         """Indicates if the function can be executed in the current session.
 
-        The function should be refactored once we start fully supporting
+        TODO: The function should be refactored once we start fully supporting
         guards
         """
         with self.fn.portal:

pythagoras/_080_pure_code_portals/pure_decorator.py

@@ -10,26 +10,26 @@ from persidict import KEEP_CURRENT, Joker
 class pure(protected):
 
     def __init__(self
-                 , guards: list[AutonomousFn] | None = None
-                 , validators: list[AutonomousFn] | None = None
+                 , pre_validators: list[AutonomousFn] | None = None
+                 , post_validators: list[AutonomousFn] | None = None
                  , fixed_kwargs: dict | None = None
                  , excessive_logging: bool | Joker = KEEP_CURRENT
                  , portal: PureCodePortal | None = None
                  ):
         protected.__init__(self=self
-            , portal=portal
-            , excessive_logging=excessive_logging
-            , fixed_kwargs=fixed_kwargs
-            , guards=guards
-            , validators=validators)
+                           , portal=portal
+                           , excessive_logging=excessive_logging
+                           , fixed_kwargs=fixed_kwargs
+                           , pre_validators=pre_validators
+                           , post_validators=post_validators)
 
 
     def __call__(self, fn:Callable|str) -> PureFn:
         wrapper = PureFn(fn
-            , portal=self._portal
-            , guards=self._guards
-            , fixed_kwargs=self._fixed_kwargs
-            , validators=self._validators
-            , excessive_logging=self._excessive_logging)
+                         , portal=self._portal
+                         , pre_validators=self._pre_validators
+                         , fixed_kwargs=self._fixed_kwargs
+                         , post_validators=self._post_validators
+                         , excessive_logging=self._excessive_logging)
         return wrapper
 

pythagoras/_090_swarming_portals/swarming_portals.py

@@ -1,23 +1,17 @@
 from __future__ import annotations
 
 import atexit
-import os
 from time import sleep
-import random
 
 import pandas as pd
 import parameterizable
 from parameterizable import sort_dict_by_keys
 from persidict import PersiDict, Joker, KEEP_CURRENT
 
-from .._010_basic_portals import get_number_of_known_portals, get_all_known_portals
-from .._010_basic_portals import BasicPortal
-from .system_utils import (
-    get_current_process_id, process_is_active,
-    get_process_start_time, get_current_process_start_time, get_available_cpu_cores, get_available_ram_mb)
+from .._010_basic_portals import get_all_known_portals
+from pythagoras._070_protected_code_portals.system_utils import *
 from .._040_logging_code_portals.logging_portal_core_classes import build_execution_environment_summary
 from .._010_basic_portals.basic_portal_core_classes import _describe_runtime_characteristic
-from .._800_signatures_and_converters.random_signatures import get_random_signature
 from persidict import OverlappingMultiDict
 from .._080_pure_code_portals.pure_core_classes import (
     PureCodePortal, PureFnExecutionResultAddr)
@@ -136,7 +130,7 @@ class SwarmingPortal(PureCodePortal):
             n = self._get_config_setting("max_n_workers")
             if n in (None, KEEP_CURRENT):
                 n = 10
-            n = min(n, get_available_cpu_cores())
+            n = min(n, get_unused_cpu_cores())
             n = min(n, get_available_ram_mb() / 500)
             n = int(n)+1
             self._max_n_workers_cache = n
@@ -269,7 +263,7 @@ def _process_random_execution_request(**portal_init_params):
                 portal._randomly_delay_execution()
                 continue
             new_address = PureFnExecutionResultAddr.from_strings(
-                prefix=addr[0], hash_signature=addr[1]
+                descriptor=addr[2], hash_signature=addr[0]+addr[1]+addr[3]
                 ,assert_readiness=False)
             if not new_address.needs_execution:
                 continue

pythagoras/__init__.py

@@ -21,19 +21,19 @@ from ._100_top_level_API import *
 from ._800_signatures_and_converters import *
 
 
-primary_decorators = {d.__name__:d for d in [
-    autonomous
-    , pure
-    ]}
-
-all_decorators = {d.__name__:d for d in [
-    ordinary
-    , storable
-    , logging
-    , safe
-    , autonomous
-    , protected
-    , pure
-]}
+# primary_decorators = {d.__name__:d for d in [
+#     autonomous
+#     , pure
+#     ]}
+#
+# all_decorators = {d.__name__:d for d in [
+#     ordinary
+#     , storable
+#     , logging
+#     , safe
+#     , autonomous
+#     , protected
+#     , pure
+# ]}
 
 

{pythagoras-0.20.52.dist-info → pythagoras-0.21.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pythagoras
-Version: 0.20.52
+Version: 0.21.0
 Summary: Planet-scale distributed computing in Python.
 Keywords: cloud,ML,AI,serverless,distributed,parallel,machine-learning,deep-learning,pythagoras
 Author: Volodymyr (Vlad) Pavlov
@@ -30,6 +30,7 @@ Requires-Dist: pytest
 Requires-Dist: boto3
 Requires-Dist: moto
 Requires-Dist: uv
+Requires-Dist: nvidia-ml-py
 Requires-Dist: persidict[aws] ; extra == 'aws'
 Requires-Dist: boto3 ; extra == 'aws'
 Requires-Dist: moto ; extra == 'aws'