pythagoras 0.24.6__tar.gz → 0.24.7__tar.gz

{pythagoras-0.24.6 → pythagoras-0.24.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pythagoras
-Version: 0.24.6
+Version: 0.24.7
 Summary: Planet-scale distributed computing in Python.
 Keywords: cloud,ML,AI,serverless,distributed,parallel,machine-learning,deep-learning,pythagoras
 Author: Volodymyr (Vlad) Pavlov

{pythagoras-0.24.6 → pythagoras-0.24.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "pythagoras"
-version = "0.24.6"
+version = "0.24.7"
 authors = [
     {name = "Volodymyr (Vlad) Pavlov", email = "vlpavlov@ieee.org"},
 ]

pythagoras-0.24.7/src/pythagoras/_070_protected_code_portals/basic_pre_validators.py

@@ -0,0 +1,172 @@
+"""Basic pre-validation utilities for protected code portals.
+
+This module contains small, composable validators used by protected portals
+before executing user functions. Each public factory returns a
+SimplePreValidatorFn configured with fixed arguments, so validators can be
+attached declaratively to protected functions.
+
+Execution context:
+- When executed by a portal, validator functions run with two names injected
+  into their global namespace: `self` (the ValidatorFn instance) and `pth`
+  (the pythagoras package). This allows them to access portal services such as
+  system introspection and package management.
+
+Conventions:
+- Return ValidationSuccessFlag (VALIDATION_SUCCESSFUL) to indicate the check
+  passed; return None to indicate the check did not pass.
+"""
+
+from .._070_protected_code_portals import SimplePreValidatorFn
+from .validation_succesful_const import ValidationSuccessFlag
+
+
+def _at_least_X_CPU_cores_free_check(n: int) -> ValidationSuccessFlag | None:
+    """Pass if at least ``n`` logical CPU cores are currently free.
+
+    This is a lightweight runtime check based on a heuristic estimation of
+    unused logical CPU capacity (see pth.get_unused_cpu_cores).
+
+    Args:
+        n (int): Minimum number of free logical CPU cores required.
+
+    Returns:
+        ValidationSuccessFlag | None: VALIDATION_SUCCESSFUL if the estimated
+        number of free cores is >= n (within a small 0.1 tolerance);
+        otherwise None.
+
+    Notes:
+        - The tolerance (0.1) helps account for fluctuations in the estimator.
+        - Uses instantaneous/short-horizon metrics; momentary spikes may affect
+          the outcome.
+    """
+    cores = pth.get_unused_cpu_cores()
+    if cores >= n - 0.1:
+        return pth.VALIDATION_SUCCESSFUL
+
+
+def unused_cpu(cores: int) -> SimplePreValidatorFn:
+    """Create a validator that requires at least the given free CPU cores.
+
+    Args:
+        cores (int): Minimum number of free logical CPU cores required (> 0).
+
+    Returns:
+        SimplePreValidatorFn: A pre-validator that succeeds only when
+        the system has at least ``cores`` free logical CPU cores.
+
+    Raises:
+        TypeError: If ``cores`` is not an integer.
+        ValueError: If ``cores`` is not greater than 0.
+    """
+    if not isinstance(cores, int):
+        raise TypeError("cores must be an int")
+    if cores <= 0:
+        raise ValueError("cores must be > 0")
+    return SimplePreValidatorFn(_at_least_X_CPU_cores_free_check).fix_kwargs(n=cores)
+
+
+def _at_least_X_G_RAM_free_check(x: int) -> ValidationSuccessFlag | None:
+    """Pass if at least ``x`` GiB of RAM are currently available.
+
+    The check uses pth.get_unused_ram_mb() divided by 1024 to obtain gibibytes
+    (GiB) and compares with a small tolerance.
+
+    Args:
+        x (int): Minimum amount of free RAM in GiB required.
+
+    Returns:
+        ValidationSuccessFlag | None: VALIDATION_SUCCESSFUL if the estimated
+        free RAM in GiB is >= x (within a 0.1 tolerance); otherwise None.
+    """
+    ram = pth.get_unused_ram_mb() / 1024
+    if ram >= x - 0.1:
+        return pth.VALIDATION_SUCCESSFUL
+
+
+def unused_ram(Gb: int) -> SimplePreValidatorFn:
+    """Create a validator that requires at least the given free RAM (GiB).
+
+    Args:
+        Gb (int): Minimum free memory required in GiB (> 0).
+
+    Returns:
+        SimplePreValidatorFn: A pre-validator that succeeds only when
+        the system has at least ``Gb`` GiB of RAM available.
+
+    Raises:
+        TypeError: If ``Gb`` is not an integer.
+        ValueError: If ``Gb`` is not greater than 0.
+    """
+    if not isinstance(Gb, int):
+        raise TypeError("Gb must be an int")
+    if Gb <= 0:
+        raise ValueError("Gb must be > 0")
+    return SimplePreValidatorFn(_at_least_X_G_RAM_free_check).fix_kwargs(x=Gb)
+
+
+def _check_python_package_and_install_if_needed(
+        package_name: str) -> ValidationSuccessFlag | None:
+    """Ensure a Python package is importable, attempting installation if not.
+
+    This validator tries to import the given package. If import fails, it will
+    throttle installation attempts to at most once every 10 minutes per
+    (node, package) pair and then invoke pth.install_package(package_name).
+
+    Args:
+        package_name (str): The importable package/module name to check.
+
+    Returns:
+        ValidationSuccessFlag | None: VALIDATION_SUCCESSFUL if the package is
+        already importable or was successfully installed; otherwise None.
+
+    Notes:
+        - The function relies on names injected by the portal: ``self`` (the
+          validator instance) and ``pth`` (pythagoras package).
+        - Throttling key is a tuple of (node_signature, package_name,
+          "installation_attempt") stored in portal._config_settings.
+        - Installation is performed synchronously and may take time.
+    """
+    if not isinstance(package_name, str):
+        raise TypeError("package_name must be a str")
+    import importlib, time
+    try:
+        importlib.import_module(package_name)
+        return pth.VALIDATION_SUCCESSFUL
+    except:
+        portal = self.portal
+        address = (pth.get_node_signature()
+                    , package_name
+                    , "installation_attempt")
+        # allow installation retries every 10 minutes
+        if (not address in portal._config_settings
+                or portal._config_settings[address] < time.time() - 600):
+            portal._config_settings[address] = time.time()
+            pth.install_package(package_name)
+            return pth.VALIDATION_SUCCESSFUL
+
+
+def installed_packages(*args) -> list[SimplePreValidatorFn]:
+    """Create validators ensuring each named package is available.
+
+    For each provided package name, this returns a SimplePreValidatorFn that
+    checks the package is importable and installs it if needed (with throttling).
+
+    Args:
+        *args: One or more package names as strings.
+
+    Returns:
+        list[SimplePreValidatorFn]: A list of pre-validators, one per package
+        name, preserving the original order.
+
+    Raises:
+        TypeError: If any of the provided arguments is not a string.
+    """
+    validators = []
+    for package_name in args:
+        if not isinstance(package_name, str):
+            raise TypeError("All package names must be strings")
+        # TODO: check if the package is available on pypi.org
+        new_validator = SimplePreValidatorFn(_check_python_package_and_install_if_needed)
+        new_validator = new_validator.fix_kwargs(package_name=package_name)
+        validators.append(new_validator)
+    return validators

pythagoras-0.24.7/src/pythagoras/_070_protected_code_portals/fn_arg_names_checker.py

@@ -0,0 +1,41 @@
+import ast
+from typing import List, Set
+
+
+def check_if_fn_accepts_args(required_arg_names: List[str]|Set[str], fn: str) -> bool:
+    """Determine whether a function can accept specific keyword argument names.
+
+    Analyzes the source code of a single Python function and checks whether
+    all required names could be passed as keyword arguments.
+
+    Args:
+      required_arg_names: Iterable of parameter names that must be accepted.
+      fn: Source code string containing exactly one function definition.
+
+    Returns:
+      True if the function has **kwargs or explicitly defines all required names
+      as keyword-acceptable parameters; otherwise False.
+
+    Raises:
+      ValueError: If no function definition is found or if multiple functions are present.
+    """
+
+    tree = ast.parse(fn)
+
+    func_def_nodes = [node for node in tree.body if isinstance(node, ast.FunctionDef)]
+    if not func_def_nodes:
+        raise ValueError("No function definition found in the provided source code.")
+    if not len(func_def_nodes) == 1:
+        raise ValueError("Multiple function definitions found in the provided source code.")
+    func_def = func_def_nodes[0]
+    args = func_def.args
+
+    # If there's a **kwargs (args.kwarg != None), it can accept any named argument
+    if args.kwarg is not None:
+        return True
+
+    # Collect all explicitly named parameters (excluding positional-only)
+    # Note: args.posonlyargs are NOT included as they cannot be passed by keyword
+    param_names = {arg.arg for arg in args.args + args.kwonlyargs}
+
+    return set(required_arg_names).issubset(param_names)

pythagoras-0.24.7/src/pythagoras/_070_protected_code_portals/protected_decorators.py

@@ -0,0 +1,97 @@
+"""Decorators for building protected functions.
+
+This module provides the protected decorator which wraps callables
+into ProtectedFn objects. A ProtectedFn coordinates pre- and post-execution
+validation using ValidatorFn instances and executes within a
+ProtectedCodePortal context.
+
+The decorator is a thin, declarative layer over the underlying core classes,
+allowing you to attach validators at definition
+time while keeping function logic clean and focused.
+"""
+
+from typing import Callable, Any
+
+# from .validator_fn_classes import ValidatorFn
+from .._060_autonomous_code_portals import autonomous
+from .protected_portal_core_classes import *
+from persidict import Joker, KEEP_CURRENT
+
+class protected(autonomous):
+    """Decorator for protected functions with pre/post validation.
+
+    This decorator wraps a target callable into a ProtectedFn that enforces
+    a sequence of pre- and post-execution validators. It builds on the
+    autonomous decorator, adding validator support to it.
+
+    Typical usage:
+        @protected(pre_validators=[...], post_validators=[...])
+        def fn(...):
+            ...
+
+    See Also:
+        ProtectedFn: The runtime wrapper that performs validation and execution.
+        ProtectedCodePortal: Portal coordinating protected function execution.
+
+    Attributes:
+        _pre_validators (list[ValidatorFn] | None): Validators executed before
+            the target function.
+        _post_validators (list[ValidatorFn] | None): Validators executed after
+            the target function.
+    """
+
+    _pre_validators: list[ValidatorFn] | None
+    _post_validators: list[ValidatorFn] | None
+
+    def __init__(self
+                 , pre_validators: list[ValidatorFn] | None = None
+                 , post_validators: list[ValidatorFn] | None = None
+                 , fixed_kwargs: dict[str,Any] | None = None
+                 , excessive_logging: bool|Joker = KEEP_CURRENT
+                 , portal: ProtectedCodePortal | None = None
+                 ):
+        """Initialize the protected decorator.
+
+        Args:
+            pre_validators (list[ValidatorFn] | None): Pre-execution validators
+                to apply. Each item is either a ValidatorFn or a callable that
+                can be wrapped into a PreValidatorFn by ProtectedFn.
+            post_validators (list[ValidatorFn] | None): Post-execution validators
+                to apply. Each item is either a ValidatorFn or a callable that
+                can be wrapped into a PostValidatorFn by ProtectedFn.
+            fixed_kwargs (dict[str, Any] | None): Keyword arguments to pre-bind
+                to the wrapped function for every call.
+            excessive_logging (bool | Joker): Enables verbose logging for the
+                wrapped function and its validators. Use KEEP_CURRENT to inherit
+                the current setting from the portal/context.
+            portal (ProtectedCodePortal | None): Optional portal instance to
+                bind the wrapped function to. If None, a suitable portal will be
+                inferred when fuction is called.
+        """
+        assert isinstance(portal, ProtectedCodePortal) or portal is None
+        assert isinstance(fixed_kwargs, dict) or fixed_kwargs is None
+        autonomous.__init__(self=self
+            , portal=portal
+            , excessive_logging=excessive_logging
+            , fixed_kwargs=fixed_kwargs)
+        self._pre_validators = pre_validators
+        self._post_validators = post_validators
+
+
+    def __call__(self, fn: Callable|str) -> ProtectedFn:
+        """Wrap the given function into a ProtectedFn.
+
+        Args:
+            fn (Callable | str): The target function or its source code string.
+
+        Returns:
+            ProtectedFn: A wrapper that performs pre/post validation and then
+            executes the function.
+        """
+        wrapper = ProtectedFn(fn
+                              , portal=self._portal
+                              , pre_validators=self._pre_validators
+                              , fixed_kwargs=self._fixed_kwargs
+                              , post_validators=self._post_validators
+                              , excessive_logging=self._excessive_logging)
+        return wrapper

{pythagoras-0.24.6 → pythagoras-0.24.7}/src/pythagoras/_070_protected_code_portals/protected_portal_core_classes.py

@@ -54,6 +54,16 @@ class ProtectedCodePortal(AutonomousCodePortal):
             , p_consistency_checks: float|Joker = KEEP_CURRENT
             , excessive_logging: bool|Joker = KEEP_CURRENT
             ):
+        """Initialize the portal.
+
+        Args:
+            root_dict (PersiDict | str | None): Backing storage or its path.
+                If None, use default.
+            p_consistency_checks (float | Joker): Probability for internal
+                consistency checks (or KEEP_CURRENT to inherit).
+            excessive_logging (bool | Joker): Verbose logging flag
+                (KEEP_CURRENT to inherit).
+        """
         super().__init__(root_dict=root_dict
             , p_consistency_checks=p_consistency_checks
             , excessive_logging=excessive_logging)
@@ -220,6 +230,16 @@ class ProtectedFn(AutonomousFn):
     def validate_execution_result(self
             , kw_args: KwArgs
             , result: Any) -> ValidationSuccessFlag|None:
+        """Run post-validators to confirm the execution result is acceptable.
+
+        Args:
+            kw_args (KwArgs): Arguments that were passed to the protected function.
+            result (Any): The value returned by the protected function.
+
+        Returns:
+            ValidationSuccessFlag | None: VALIDATION_SUCCESSFUL if all
+                post-validators pass, otherwise None.
+        """
         with self.portal as portal:
             kw_args = kw_args.pack()
             post_validators = copy(self.post_validators)
@@ -232,6 +252,24 @@ class ProtectedFn(AutonomousFn):
 
 
     def execute(self, **kwargs) -> Any:
+        """Execute the protected function with validation.
+
+        This method performs the following loop:
+        - Runs pre-validators. If a pre-validator returns a
+          ProtectedFnCallSignature, that signature is executed and validation is
+          reattempted. If any pre-validator fails, an AssertionError is raised.
+        - Executes the wrapped function.
+        - Runs post-validators and asserts they all succeed.
+
+        Args:
+            **kwargs: Keyword arguments to pass to the wrapped function.
+
+        Returns:
+            Any: The result returned by the wrapped function.
+
+        Raises:
+            AssertionError: If pre- or post-validation fails.
+        """
         with (self.portal):
             kw_args = KwArgs(**kwargs)
             while True:
@@ -252,8 +290,21 @@ class ProtectedFn(AutonomousFn):
             ) -> list[ValidatorFn]:
         """Return list of validators in a normalized form.
 
-        All the functions-validators are converted to AutonomousFn objects,
-        and returned as a list, sorted by functions' hash signatures.
+        - Wraps plain callables/strings into appropriate ValidatorFn subclasses.
+        - Flattens nested lists.
+        - Removes duplicates while inforcing deterministic
+          order via sort_dict_by_keys.
+
+        Args:
+            validators (list[ValidatorFn] | ValidatorFn | None): Validators in
+                any supported representation (single, list, nested lists, etc.).
+            validator_type (type): Either PreValidatorFn or PostValidatorFn.
+
+        Returns:
+            list[ValidatorFn]: A sorted list of validator instances.
+
+        Raises:
+            TypeError: If an unexpected validator_type is provided.
         """
         assert validator_type in {PreValidatorFn, PostValidatorFn}
         if validators is None:
@@ -286,6 +337,12 @@ class ProtectedFn(AutonomousFn):
 
     @property
     def portal(self) -> ProtectedCodePortal:
+        """Return the bound ProtectedCodePortal.
+
+        Returns:
+            ProtectedCodePortal: The portal controlling execution context and
+            storage for this protected function.
+        """
         return super().portal
 
 
@@ -310,14 +367,33 @@ class ProtectedFn(AutonomousFn):
 
 
     def get_signature(self, arguments:dict) -> ProtectedFnCallSignature:
+        """Create a call signature for this protected function.
+
+        Args:
+            arguments (dict): Arguments to bind into the call signature.
+
+        Returns:
+            ProtectedFnCallSignature: Signature object representing a
+            particular call to this function.
+        """
         return ProtectedFnCallSignature(self, arguments)
 
 
 class ProtectedFnCallSignature(AutonomousFnCallSignature):
-    """A signature of a call to a pure function"""
+    """Invocation signature for a protected function.
+
+    Encapsulates a function reference and bound arguments that can be executed
+    later via execute().
+    """
     _fn_cache: ProtectedFn | None
 
     def __init__(self, fn: ProtectedFn, arguments: dict):
+        """Initialize the signature.
+
+        Args:
+            fn (ProtectedFn): The protected function to call.
+            arguments (dict): Keyword arguments to be passed at execution time.
+        """
         assert isinstance(fn, ProtectedFn)
         assert isinstance(arguments, dict)
         super().__init__(fn, arguments)
@@ -329,10 +405,26 @@ class ProtectedFnCallSignature(AutonomousFnCallSignature):
 
 
 class ValidatorFn(AutonomousFn):
+    """Base class for validator wrappers.
+
+    A ValidatorFn ensures the wrapped callable accepts exactly the keyword
+    arguments declared by get_allowed_kwargs_names(). Subclasses define the
+    specific interface for pre/post validation phases.
+    """
     def __init__(self, fn: Callable | str | AutonomousFn
         , fixed_kwargs: dict | None = None
         , excessive_logging: bool | Joker = KEEP_CURRENT
         , portal: AutonomousCodePortal | None = None):
+        """Initialize a validator function wrapper.
+
+        Args:
+            fn (Callable | str | AutonomousFn): The validator implementation or
+                its source code.
+            fixed_kwargs (dict | None): Keyword arguments fixed for every
+                validation call.
+            excessive_logging (bool | Joker): Controls verbose logging.
+            portal (AutonomousCodePortal | None): Optional portal binding.
+        """
         super().__init__(
             fn=fn
             , fixed_kwargs=fixed_kwargs
@@ -344,20 +436,52 @@ class ValidatorFn(AutonomousFn):
 
     @classmethod
     def get_allowed_kwargs_names(cls)->set[str]:
+        """Return the exact set of allowed keyword argument names.
+
+        Subclasses must override to declare their interface.
+
+        Returns:
+            set[str]: Names of keyword arguments accepted by execute().
+        """
         raise NotImplementedError("This method must be overridden")
 
 
     def execute(self,**kwargs) \
             -> ProtectedFnCallSignature | ValidationSuccessFlag | None:
+        """Execute the validator after verifying keyword arguments.
+
+        Args:
+            **kwargs: Must exactly match get_allowed_kwargs_names().
+
+        Returns:
+            ProtectedFnCallSignature | ValidationSuccessFlag | None: Depending
+            on the validator type and outcome.
+        """
         assert set(kwargs) == self.get_allowed_kwargs_names()
         return super().execute(**kwargs)
 
 
 class PreValidatorFn(ValidatorFn):
+    """Base class for pre-execution validators.
+
+    Pre-validators are executed before the protected function. They may return:
+    - VALIDATION_SUCCESSFUL to indicate execution can proceed;
+    - ProtectedFnCallSignature to request execution of an auxiliary action
+      prior to re-validating;
+    - None to indicate failure.
+    """
     def __init__(self, fn: Callable | str | AutonomousFn
         , fixed_kwargs: dict | None = None
         , excessive_logging: bool | Joker = KEEP_CURRENT
         , portal: AutonomousCodePortal | None = None):
+        """Initialize a pre-execution validator wrapper.
+
+        Args:
+            fn (Callable | str | AutonomousFn): The pre-validator implementation.
+            fixed_kwargs (dict | None): Keyword arguments fixed for every call.
+            excessive_logging (bool | Joker): Controls verbose logging.
+            portal (AutonomousCodePortal | None): Optional portal binding.
+        """
         super().__init__(
             fn=fn
             , fixed_kwargs=fixed_kwargs
@@ -366,10 +490,22 @@ class PreValidatorFn(ValidatorFn):
 
 
 class SimplePreValidatorFn(PreValidatorFn):
+    """A pre-validator that takes no runtime inputs.
+
+    The wrapped callable must accept no parameters; use fixed_kwargs only.
+    """
     def __init__(self, fn: Callable | str | AutonomousFn
         , fixed_kwargs: dict | None = None
         , excessive_logging: bool | Joker = KEEP_CURRENT
         , portal: AutonomousCodePortal | None = None):
+        """Initialize a simple pre-validator.
+
+        Args:
+            fn (Callable | str | AutonomousFn): The implementation.
+            fixed_kwargs (dict | None): Fixed keyword arguments, if any.
+            excessive_logging (bool | Joker): Controls verbose logging.
+            portal (AutonomousCodePortal | None): Optional portal binding.
+        """
         super().__init__(
             fn=fn
             , fixed_kwargs=fixed_kwargs
@@ -384,10 +520,23 @@ class SimplePreValidatorFn(PreValidatorFn):
 
 
 class ComplexPreValidatorFn(PreValidatorFn):
+    """A pre-validator that can inspect inputs and the function address.
+
+    The callable must accept the keyword arguments named
+    packed_kwargs and fn_addr.
+    """
     def __init__(self, fn: Callable | str | AutonomousFn
         , fixed_kwargs: dict | None = None
         , excessive_logging: bool | Joker = KEEP_CURRENT
         , portal: AutonomousCodePortal | None = None):
+        """Initialize a complex pre-validator.
+
+        Args:
+            fn (Callable | str | AutonomousFn): The implementation.
+            fixed_kwargs (dict | None): Fixed keyword arguments, if any.
+            excessive_logging (bool | Joker): Controls verbose logging.
+            portal (AutonomousCodePortal | None): Optional portal binding.
+        """
         super().__init__(
             fn=fn
             , fixed_kwargs=fixed_kwargs
@@ -402,10 +551,22 @@ class ComplexPreValidatorFn(PreValidatorFn):
 
 
 class PostValidatorFn(ValidatorFn):
+    """Post-execution validator wrapper.
+
+    The callable must accept packed_kwargs, fn_addr, and result.
+    """
     def __init__(self, fn: Callable | str | AutonomousFn
         , fixed_kwargs: dict | None = None
         , excessive_logging: bool | Joker = KEEP_CURRENT
         , portal: AutonomousCodePortal | None = None):
+        """Initialize a post-execution validator.
+
+        Args:
+            fn (Callable | str | AutonomousFn): The implementation.
+            fixed_kwargs (dict | None): Fixed keyword arguments, if any.
+            excessive_logging (bool | Joker): Controls verbose logging.
+            portal (AutonomousCodePortal | None): Optional portal binding.
+        """
         super().__init__(
             fn=fn
             , fixed_kwargs=fixed_kwargs
@@ -414,5 +575,9 @@ class PostValidatorFn(ValidatorFn):
 
     @classmethod
     def get_allowed_kwargs_names(cls) -> set[str]:
-        """Post-validators use info about the function, its input arguments and returned value."""
+        """Post-validators use function metadata, inputs, and the result.
+
+        Returns:
+            set[str]: {"packed_kwargs", "fn_addr", "result"}
+        """
         return {"packed_kwargs", "fn_addr", "result" }