pythagoras 0.24.0__tar.gz → 0.24.2__tar.gz

{pythagoras-0.24.0 → pythagoras-0.24.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pythagoras
-Version: 0.24.0
+Version: 0.24.2
 Summary: Planet-scale distributed computing in Python.
 Keywords: cloud,ML,AI,serverless,distributed,parallel,machine-learning,deep-learning,pythagoras
 Author: Volodymyr (Vlad) Pavlov
@@ -16,8 +16,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Dist: parameterizable
-Requires-Dist: persidict
+Requires-Dist: persidict==0.104.1
 Requires-Dist: lz4
 Requires-Dist: joblib
 Requires-Dist: scikit-learn
@@ -32,6 +31,7 @@ Requires-Dist: boto3
 Requires-Dist: moto
 Requires-Dist: uv
 Requires-Dist: nvidia-ml-py
+Requires-Dist: parameterizable==0.100.0
 Requires-Dist: persidict[aws] ; extra == 'aws'
 Requires-Dist: boto3 ; extra == 'aws'
 Requires-Dist: moto ; extra == 'aws'

{pythagoras-0.24.0 → pythagoras-0.24.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "pythagoras"
-version = "0.24.0"
+version = "0.24.2"
 authors = [
     {name = "Volodymyr (Vlad) Pavlov", email = "vlpavlov@ieee.org"},
 ]
@@ -26,8 +26,7 @@ classifiers = [
 ]
 keywords = ["cloud", "ML", "AI", "serverless", "distributed", "parallel", "machine-learning", "deep-learning", "pythagoras"]
 dependencies = [
-    "parameterizable",
-    "persidict",
+    "persidict==0.104.1",
     "lz4",
     "joblib",
     "scikit-learn",
@@ -42,6 +41,7 @@ dependencies = [
     "moto",
     "uv",
     "nvidia-ml-py",
+    "parameterizable==0.100.0",
 ]
 
 [project.urls]

{pythagoras-0.24.0 → pythagoras-0.24.2}/src/pythagoras/_010_basic_portals/basic_portal_core_classes.py

@@ -14,6 +14,7 @@ from abc import abstractmethod
 from pathlib import Path
 from typing import TypeVar, Type, Any, NewType
 import pandas as pd
+import parameterizable
 from parameterizable import ParameterizableClass, sort_dict_by_keys
 
 from persidict import PersiDict, FileDirDict, SafeStrTuple
@@ -358,15 +359,15 @@ class BasicPortal(NotPicklable,ParameterizableClass, metaclass = PostInitMeta):
         return sorted_params
 
 
-    @property
-    def _ephemeral_param_names(self) -> set[str]:
-        """Get the names of the portal's ephemeral parameters.
-
-        Portal's ephemeral parameters are not stored persistently.
-        They affect behaviour of a portal object in an application session,
-        but they do not affect behaviour of the actual portal across multiple runs.
-        """
-        return set()
+    # @property
+    # def auxiliary_param_names(self) -> set[str]:
+    #     """Get the names of the portal's ephemeral parameters.
+    #
+    #     Portal's ephemeral parameters are not stored persistently.
+    #     They affect behaviour of a portal object in an application session,
+    #     but they do not affect behaviour of the actual portal across multiple runs.
+    #     """
+    #     return set()
 
 
     @property
@@ -376,12 +377,8 @@ class BasicPortal(NotPicklable,ParameterizableClass, metaclass = PostInitMeta):
         It's an internal hash used by Pythagoras and is different from .__hash__()
         """
         if not hasattr(self,"_str_id_cache"):
-            params = self.get_portable_params()
-            ephemeral_names = self._ephemeral_param_names
-            nonephemeral_params = {k:params[k] for k in params
-                if k not in ephemeral_names}
             self._str_id_cache = PortalStrID(
-                get_hash_signature(nonephemeral_params))
+                get_hash_signature(self.get_essential_jsparams()))
         return self._str_id_cache
 
 

{pythagoras-0.24.0 → pythagoras-0.24.2}/src/pythagoras/_030_data_portals/data_portal_core_classes.py

@@ -61,7 +61,7 @@ class DataPortal(OrdinaryCodePortal):
     _config_settings: PersiDict | None
     _config_settings_cache: dict
 
-    _ephemeral_config_params_at_init: dict[str, Any] | None
+    _auxiliary_config_params_at_init: dict[str, Any] | None
 
     def __init__(self
             , root_dict: PersiDict|str|None = None
@@ -69,7 +69,7 @@ class DataPortal(OrdinaryCodePortal):
             ):
         OrdinaryCodePortal.__init__(self, root_dict = root_dict)
         del root_dict
-        self._ephemeral_config_params_at_init = dict()
+        self._auxiliary_config_params_at_init = dict()
         self._config_settings_cache = dict()
 
         config_settings_prototype = self._root_dict.get_subdict("config_settings")
@@ -84,7 +84,7 @@ class DataPortal(OrdinaryCodePortal):
             raise ValueError("p_consistency_checks must be a float in [0,1] "
                 +f"or a Joker, but got {p_consistency_checks}")
 
-        self._ephemeral_config_params_at_init["p_consistency_checks"
+        self._auxiliary_config_params_at_init["p_consistency_checks"
             ] = p_consistency_checks
 
         value_store_prototype = self._root_dict.get_subdict("value_store")
@@ -97,7 +97,7 @@ class DataPortal(OrdinaryCodePortal):
 
 
     def _persist_initial_config_params(self) -> None:
-        for key, value in self._ephemeral_config_params_at_init.items():
+        for key, value in self._auxiliary_config_params_at_init.items():
             self._set_config_setting(key, value)
 
 
@@ -111,15 +111,15 @@ class DataPortal(OrdinaryCodePortal):
     def get_params(self) -> dict:
         """Get the portal's configuration parameters"""
         params = super().get_params()
-        params.update(self._ephemeral_config_params_at_init)
+        params.update(self._auxiliary_config_params_at_init)
         sorted_params = sort_dict_by_keys(params)
         return sorted_params
 
 
     @property
-    def _ephemeral_param_names(self) -> set[str]:
-        names = super()._ephemeral_param_names
-        names.update(self._ephemeral_config_params_at_init)
+    def auxiliary_param_names(self) -> set[str]:
+        names = super().auxiliary_param_names
+        names.update(self._auxiliary_config_params_at_init)
         return names
 
 
@@ -186,7 +186,7 @@ class DataPortal(OrdinaryCodePortal):
         """Clear the portal's state"""
         self._value_store = None
         self._config_settings = None
-        self._ephemeral_config_params_at_init = None
+        self._auxiliary_config_params_at_init = None
         self._invalidate_cache()
         super()._clear()
 
@@ -195,14 +195,14 @@ class StorableFn(OrdinaryFn):
     """An ordinary function that can be persistently stored in a DataPortal."""
 
     _addr_cache: ValueAddr
-    _ephemeral_config_params_at_init: dict[str, Any] | None
+    _auxiliary_config_params_at_init: dict[str, Any] | None
 
     def __init__(self
         , fn: Callable | str
         , portal: DataPortal | None = None
         ):
         OrdinaryFn.__init__(self, fn=fn, portal=portal)
-        self._ephemeral_config_params_at_init = dict()
+        self._auxiliary_config_params_at_init = dict()
 
 
     def _first_visit_to_portal(self, portal: DataPortal) -> None:
@@ -213,7 +213,7 @@ class StorableFn(OrdinaryFn):
 
 
     def _persist_initial_config_params(self, portal:DataPortal) -> None:
-        for key, value in self._ephemeral_config_params_at_init.items():
+        for key, value in self._auxiliary_config_params_at_init.items():
             self._set_config_setting(key, value, portal)
 
 
@@ -268,7 +268,7 @@ class StorableFn(OrdinaryFn):
     def __setstate__(self, state):
         """This method is called when the object is unpickled."""
         super().__setstate__(state)
-        self._ephemeral_config_params_at_init = dict()
+        self._auxiliary_config_params_at_init = dict()
 
 
     def __getstate__(self):

{pythagoras-0.24.0 → pythagoras-0.24.2}/src/pythagoras/_040_logging_code_portals/logging_portal_core_classes.py

@@ -5,7 +5,7 @@ import traceback
 from typing import Callable, Any
 
 import pandas as pd
-from parameterizable import register_parameterizable_class
+# from parameterizable import register_parameterizable_class
 from persidict import PersiDict, KEEP_CURRENT, Joker
 
 from .._010_basic_portals import NotPicklable, get_active_portal
@@ -48,9 +48,9 @@ class LoggingFn(StorableFn):
                 f"got {type(excessive_logging)}")
 
         if excessive_logging is KEEP_CURRENT and isinstance(fn, LoggingFn):
-            excessive_logging = fn._ephemeral_config_params_at_init["excessive_logging"]
+            excessive_logging = fn._auxiliary_config_params_at_init["excessive_logging"]
 
-        self._ephemeral_config_params_at_init[
+        self._auxiliary_config_params_at_init[
             "excessive_logging"] = excessive_logging
 
 
@@ -545,7 +545,7 @@ class LoggingCodePortal(DataPortal):
                 "excessive_logging must be a boolean or Joker, "
                 f"got {type(excessive_logging)}")
 
-        self._ephemeral_config_params_at_init["excessive_logging"
+        self._auxiliary_config_params_at_init["excessive_logging"
             ] = excessive_logging
 
         crash_history_prototype = self._root_dict.get_subdict("crash_history")
@@ -611,7 +611,7 @@ class LoggingCodePortal(DataPortal):
         super()._clear()
 
 
-register_parameterizable_class(LoggingCodePortal)
+# register_parameterizable_class(LoggingCodePortal)
 
 
 def log_exception() -> None:

{pythagoras-0.24.0 → pythagoras-0.24.2}/src/pythagoras/_050_safe_code_portals/safe_portal_core_classes.py

@@ -15,7 +15,7 @@ from __future__ import annotations
 
 from typing import Callable
 
-from parameterizable import register_parameterizable_class
+# from parameterizable import register_parameterizable_class
 from persidict import PersiDict, Joker, KEEP_CURRENT
 
 from .._040_logging_code_portals.logging_portal_core_classes import *
@@ -80,4 +80,4 @@ class SafeFn(LoggingFn):
         return SafeFnCallSignature(fn=self, arguments=arguments)
 
 
-register_parameterizable_class(SafeCodePortal)
+# register_parameterizable_class(SafeCodePortal)

pythagoras-0.24.2/src/pythagoras/_090_swarming_portals/output_suppressor.py

@@ -0,0 +1,18 @@
+import os
+from contextlib import ExitStack, redirect_stderr, redirect_stdout
+
+
+
+class OutputSuppressor:
+    """A context manager that suppresses stdout and stderr output."""
+
+    def __enter__(self):
+        """Redirect stdout and stderr to os.devnull."""
+        self._stack = ExitStack()
+        devnull = self._stack.enter_context(open(os.devnull, "w"))
+        self._stack.enter_context(redirect_stdout(devnull))
+        self._stack.enter_context(redirect_stderr(devnull))
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return self._stack.__exit__(exc_type, exc_val, exc_tb)

{pythagoras-0.24.0 → pythagoras-0.24.2}/src/pythagoras/_090_swarming_portals/swarming_portals.py

@@ -12,13 +12,19 @@ at least once but does not offer any further guarantees.
 from __future__ import annotations
 
 import atexit
+import json
 from time import sleep
 
 import pandas as pd
 import parameterizable
-from parameterizable import sort_dict_by_keys
+from parameterizable import (
+    sort_dict_by_keys,
+    update_jsparams,
+    access_jsparams)
 from persidict import PersiDict, Joker, KEEP_CURRENT
 
+from parameterizable.json_processor import _Markers
+
 from .. import VALIDATION_SUCCESSFUL
 from .._010_basic_portals import get_all_known_portals
 from .._070_protected_code_portals.system_utils import get_unused_ram_mb, get_unused_cpu_cores, process_is_active, \
@@ -66,7 +72,7 @@ class SwarmingPortal(PureCodePortal):
         if parent_process_id is None or parent_process_start_time is None:
             assert parent_process_id is None
             assert parent_process_start_time is None
-            self._ephemeral_config_params_at_init["max_n_workers"
+            self._auxiliary_config_params_at_init["max_n_workers"
                 ] = max_n_workers
         else:
             assert max_n_workers == 0
@@ -88,6 +94,14 @@ class SwarmingPortal(PureCodePortal):
         self._child_process = None
 
 
+    def get_params(self) -> dict:
+        params = super().get_params()
+        params["parent_process_id"] = self._parent_process_id
+        params["parent_process_start_time"] = self._parent_process_start_time
+        sorted_params = sort_dict_by_keys(params)
+        return sorted_params
+
+
     @property
     def is_parent(self) -> bool:
         """Check if this portal is the parent process."""
@@ -106,14 +120,14 @@ class SwarmingPortal(PureCodePortal):
         if self.is_parent:
             if self.max_n_workers > 0:
 
-                portal_init_params = self.get_portable_params()
-                portal_init_params["max_n_workers"] = self.max_n_workers
-                portal_init_params = sort_dict_by_keys(portal_init_params)
+                portal_init_jsparams = parameterizable.dumpjs(self)
+                portal_init_jsparams = update_jsparams(portal_init_jsparams,
+                    max_n_workers = self.max_n_workers)
 
                 ctx = get_context("spawn")
                 self._child_process = ctx.Process(
                     target=_launch_many_background_workers
-                    , kwargs=portal_init_params)
+                    , args=(portal_init_jsparams,))
                 self._child_process.start()
 
 
@@ -198,22 +212,22 @@ class SwarmingPortal(PureCodePortal):
             del self._max_n_workers_cache
         super()._invalidate_cache()
 
-parameterizable.register_parameterizable_class(SwarmingPortal)
+# parameterizable.register_parameterizable_class(SwarmingPortal)
 
 
-def _launch_many_background_workers(**portal_init_params) -> None:
+def _launch_many_background_workers(portal_init_jsparams:JsonSerializedParams) -> None:
     """Launch many background worker processes."""
-    n_workers_to_launch = portal_init_params["max_n_workers"]
+
+
+    n_workers_to_launch = access_jsparams(portal_init_jsparams
+        , "max_n_workers")["max_n_workers"]
     n_workers_to_launch = int(n_workers_to_launch)
 
-    portal_init_params["max_n_workers"] = 0
-    current_process_id = get_current_process_id()
-    portal_init_params["parent_process_id"] = current_process_id
-    portal_init_params["parent_process_start_time"
-        ] = get_current_process_start_time()
-    portal_init_params = sort_dict_by_keys(portal_init_params)
-    portal = parameterizable.get_object_from_portable_params(
-        portal_init_params)
+    portal_init_jsparams = update_jsparams(portal_init_jsparams,
+        max_n_workers=0, parent_process_id = get_current_process_id(),
+        parent_process_start_time = get_current_process_start_time())
+
+    portal = parameterizable.loadjs(portal_init_jsparams)
     assert isinstance(portal, SwarmingPortal)
 
     summary = build_execution_environment_summary()
@@ -226,7 +240,7 @@ def _launch_many_background_workers(**portal_init_params) -> None:
             portal._randomly_delay_execution(p=1)
             ctx = get_context("spawn")
             try:
-                p = ctx.Process(target=_background_worker, kwargs=portal_init_params)
+                p = ctx.Process(target=_background_worker, args=(portal_init_jsparams,))
                 p.start()
                 list_of_all_workers.append(p)
             except Exception as e:
@@ -243,7 +257,7 @@ def _launch_many_background_workers(**portal_init_params) -> None:
                     portal._randomly_delay_execution(p=1)
                     ctx = get_context("spawn")
                     try:
-                        p = ctx.Process(target=_background_worker, kwargs=portal_init_params)
+                        p = ctx.Process(target=_background_worker, args=(portal_init_jsparams,))
                         p.start()
                         new_list_of_all_workers.append(p)
                     except Exception as e:
@@ -251,10 +265,9 @@ def _launch_many_background_workers(**portal_init_params) -> None:
             list_of_all_workers = new_list_of_all_workers
 
 
-def _background_worker(**portal_init_params) -> None:
+def _background_worker(portal_init_jsparams:JsonSerializedParams) -> None:
     """Background worker that keeps processing random execution requests."""
-    portal = parameterizable.get_object_from_portable_params(
-        portal_init_params)
+    portal = parameterizable.loadjs(portal_init_jsparams)
     assert isinstance(portal, SwarmingPortal)
     with portal:
         ctx = get_context("spawn")
@@ -264,16 +277,19 @@ def _background_worker(**portal_init_params) -> None:
                     return
                 p = ctx.Process(
                     target=_process_random_execution_request
-                    , kwargs=portal_init_params)
+                    , args=(portal_init_jsparams,))
                 p.start()
                 p.join()
                 portal._randomly_delay_execution()
 
 
-def _process_random_execution_request(**portal_init_params):
+def _process_random_execution_request(portal_init_jsparams:JsonSerializedParams):
     """Process one random execution request."""
-    portal = parameterizable.get_object_from_portable_params(
-        portal_init_params)
+    # portal = parameterizable.get_object_from_portable_params(
+    #     portal_init_params)
+    portal_init_jsparams = update_jsparams(
+        portal_init_jsparams, max_n_workers=0)
+    portal = parameterizable.loadjs(portal_init_jsparams)
     assert isinstance(portal, SwarmingPortal)
     with portal:
         call_signature:PureFnCallSignature|None = None

pythagoras-0.24.0/src/pythagoras/_090_swarming_portals/output_suppressor.py

@@ -1,23 +0,0 @@
-import os
-import sys
-
-
-class OutputSuppressor:
-    """A context manager that suppresses stdout and stderr output."""
-    def __init__(self):
-        pass
-
-    def __enter__(self):
-        """Redirect stdout and stderr to os.devnull."""
-        self.old_stdout = sys.stdout
-        self.old_stderr = sys.stderr
-        self.devnull = open(os.devnull, 'w')
-        sys.stdout = self.devnull
-        sys.stderr = self.devnull
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        sys.stdout = self.old_stdout
-        sys.stderr = self.old_stderr
-        if self.devnull:
-            self.devnull.close()