pythagoras 0.23.19__tar.gz → 0.24.1__tar.gz

{pythagoras-0.23.19 → pythagoras-0.24.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pythagoras
-Version: 0.23.19
+Version: 0.24.1
 Summary: Planet-scale distributed computing in Python.
 Keywords: cloud,ML,AI,serverless,distributed,parallel,machine-learning,deep-learning,pythagoras
 Author: Volodymyr (Vlad) Pavlov
@@ -16,8 +16,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Dist: parameterizable
-Requires-Dist: persidict
+Requires-Dist: persidict==0.104.1
 Requires-Dist: lz4
 Requires-Dist: joblib
 Requires-Dist: scikit-learn
@@ -32,6 +31,7 @@ Requires-Dist: boto3
 Requires-Dist: moto
 Requires-Dist: uv
 Requires-Dist: nvidia-ml-py
+Requires-Dist: parameterizable==0.100.0
 Requires-Dist: persidict[aws] ; extra == 'aws'
 Requires-Dist: boto3 ; extra == 'aws'
 Requires-Dist: moto ; extra == 'aws'

{pythagoras-0.23.19 → pythagoras-0.24.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "pythagoras"
-version = "0.23.19"
+version = "0.24.1"
 authors = [
     {name = "Volodymyr (Vlad) Pavlov", email = "vlpavlov@ieee.org"},
 ]
@@ -26,8 +26,7 @@ classifiers = [
 ]
 keywords = ["cloud", "ML", "AI", "serverless", "distributed", "parallel", "machine-learning", "deep-learning", "pythagoras"]
 dependencies = [
-    "parameterizable",
-    "persidict",
+    "persidict==0.104.1",
     "lz4",
     "joblib",
     "scikit-learn",
@@ -42,6 +41,7 @@ dependencies = [
     "moto",
     "uv",
     "nvidia-ml-py",
+    "parameterizable==0.100.0",
 ]
 
 [project.urls]

{pythagoras-0.23.19 → pythagoras-0.24.1}/src/pythagoras/_010_basic_portals/basic_portal_core_classes.py

@@ -14,6 +14,7 @@ from abc import abstractmethod
 from pathlib import Path
 from typing import TypeVar, Type, Any, NewType
 import pandas as pd
+import parameterizable
 from parameterizable import ParameterizableClass, sort_dict_by_keys
 
 from persidict import PersiDict, FileDirDict, SafeStrTuple
@@ -376,7 +377,10 @@ class BasicPortal(NotPicklable,ParameterizableClass, metaclass = PostInitMeta):
         It's an internal hash used by Pythagoras and is different from .__hash__()
         """
         if not hasattr(self,"_str_id_cache"):
-            params = self.get_portable_params()
+            params = self.get_params()
+            jsparams = parameterizable.dumpjs(params)
+            param_names = set(params.keys())
+            params = parameterizable.access_jsparams(jsparams, *param_names)
             ephemeral_names = self._ephemeral_param_names
             nonephemeral_params = {k:params[k] for k in params
                 if k not in ephemeral_names}

{pythagoras-0.23.19 → pythagoras-0.24.1}/src/pythagoras/_030_data_portals/data_portal_core_classes.py

@@ -75,7 +75,7 @@ class DataPortal(OrdinaryCodePortal):
         config_settings_prototype = self._root_dict.get_subdict("config_settings")
         config_settings_params = config_settings_prototype.get_params()
         config_settings_params.update(
-            digest_len=0, immutable_items=False, file_type="pkl")
+            digest_len=0, append_only=False, serialization_format="pkl")
         config_settings = type(self._root_dict)(**config_settings_params)
         self._config_settings = config_settings
 
@@ -90,7 +90,7 @@ class DataPortal(OrdinaryCodePortal):
         value_store_prototype = self._root_dict.get_subdict("value_store")
         value_store_params = value_store_prototype.get_params()
         value_store_params.update(
-            digest_len=0, immutable_items=True, file_type = "pkl")
+            digest_len=0, append_only=True, serialization_format = "pkl")
         value_store = type(self._root_dict)(**value_store_params)
         value_store = WriteOnceDict(value_store, 0)
         self._value_store = value_store

{pythagoras-0.23.19 → pythagoras-0.24.1}/src/pythagoras/_040_logging_code_portals/logging_portal_core_classes.py

@@ -5,7 +5,7 @@ import traceback
 from typing import Callable, Any
 
 import pandas as pd
-from parameterizable import register_parameterizable_class
+# from parameterizable import register_parameterizable_class
 from persidict import PersiDict, KEEP_CURRENT, Joker
 
 from .._010_basic_portals import NotPicklable, get_active_portal
@@ -551,13 +551,13 @@ class LoggingCodePortal(DataPortal):
         crash_history_prototype = self._root_dict.get_subdict("crash_history")
         crash_history_params = crash_history_prototype.get_params()
         crash_history_params.update(
-            dict(file_type="json", immutable_items=True , digest_len=0))
+            dict(serialization_format="json", append_only=True , digest_len=0))
         self._crash_history = type(self._root_dict)(**crash_history_params)
 
         event_history_prototype = self._root_dict.get_subdict("event_history")
         event_history_params = event_history_prototype.get_params()
         event_history_params.update(
-            dict(file_type="json", immutable_items=True, digest_len=0))
+            dict(serialization_format="json", append_only=True, digest_len=0))
         self._event_history = type(self._root_dict)(**event_history_params)
 
         run_history_prototype = self._root_dict.get_subdict("run_history")
@@ -566,10 +566,10 @@ class LoggingCodePortal(DataPortal):
         run_history = OverlappingMultiDict(
             dict_type=dict_type
             , shared_subdicts_params=run_history_shared_params
-            , json=dict(immutable_items=True)
-            , py=dict(base_class_for_values=str, immutable_items=False) # Immutable items????
-            , txt=dict(base_class_for_values=str, immutable_items=True)
-            , pkl=dict(immutable_items=True)
+            , json=dict(append_only=True)
+            , py=dict(base_class_for_values=str, append_only=False) # Immutable items????
+            , txt=dict(base_class_for_values=str, append_only=True)
+            , pkl=dict(append_only=True)
         )
         self._run_history = run_history
 
@@ -611,7 +611,7 @@ class LoggingCodePortal(DataPortal):
         super()._clear()
 
 
-register_parameterizable_class(LoggingCodePortal)
+# register_parameterizable_class(LoggingCodePortal)
 
 
 def log_exception() -> None:

{pythagoras-0.23.19 → pythagoras-0.24.1}/src/pythagoras/_050_safe_code_portals/safe_portal_core_classes.py

@@ -15,7 +15,7 @@ from __future__ import annotations
 
 from typing import Callable
 
-from parameterizable import register_parameterizable_class
+# from parameterizable import register_parameterizable_class
 from persidict import PersiDict, Joker, KEEP_CURRENT
 
 from .._040_logging_code_portals.logging_portal_core_classes import *
@@ -80,4 +80,4 @@ class SafeFn(LoggingFn):
         return SafeFnCallSignature(fn=self, arguments=arguments)
 
 
-register_parameterizable_class(SafeCodePortal)
+# register_parameterizable_class(SafeCodePortal)

{pythagoras-0.23.19 → pythagoras-0.24.1}/src/pythagoras/_080_pure_code_portals/pure_core_classes.py

@@ -68,7 +68,7 @@ class PureCodePortal(ProtectedCodePortal):
         results_dict_prototype = self._root_dict.get_subdict(
             "execution_results")
         results_dict_params = results_dict_prototype.get_params()
-        results_dict_params.update(immutable_items=True,  file_type = "pkl")
+        results_dict_params.update(append_only=True,  serialization_format = "pkl")
         execution_results = type(self._root_dict)(**results_dict_params)
         execution_results = WriteOnceDict(execution_results, 0)
         self._execution_results = execution_results
@@ -76,7 +76,7 @@ class PureCodePortal(ProtectedCodePortal):
         requests_dict_prototype = self._root_dict.get_subdict(
             "execution_requests")
         requests_dict_params = requests_dict_prototype.get_params()
-        requests_dict_params.update(immutable_items=False, file_type="pkl")
+        requests_dict_params.update(append_only=False, serialization_format="pkl")
         execution_requests = type(self._root_dict)(**requests_dict_params)
         self._execution_requests = execution_requests
 

{pythagoras-0.23.19 → pythagoras-0.24.1}/src/pythagoras/_080_pure_code_portals/recursion_pre_validator.py

@@ -20,12 +20,17 @@ def _recursion_pre_validator(
     param_value = unpacked_kwargs[param_name]
     assert isinstance(param_value, int)
     assert param_value >= 0
-    if param_value in {0,1}:
+    if param_value in {0,1,2,3,4}:
         return pth.VALIDATION_SUCCESSFUL
-    
+    unpacked_kwargs[param_name] = param_value - 1
+    result_addr = pth.PureFnExecutionResultAddr(
+        fn=fn, arguments=unpacked_kwargs)
+    if result_addr.ready:
+        return pth.VALIDATION_SUCCESSFUL
+    result = result_addr.call_signature
+
     # Binary search to find the smallest n where result_addr.ready is not True
-    left, right = 2, param_value - 1
-    result = pth.VALIDATION_SUCCESSFUL  # Default result if all are ready
+    left, right = 2, param_value - 2
     
     while left <= right:
         mid = (left + right) // 2
@@ -42,7 +47,6 @@ def _recursion_pre_validator(
             # Result is ready, search in the right half
             left = mid + 1
 
-    
     return result
 
 

pythagoras-0.24.1/src/pythagoras/_090_swarming_portals/output_suppressor.py

@@ -0,0 +1,18 @@
+import os
+from contextlib import ExitStack, redirect_stderr, redirect_stdout
+
+
+
+class OutputSuppressor:
+    """A context manager that suppresses stdout and stderr output."""
+
+    def __enter__(self):
+        """Redirect stdout and stderr to os.devnull."""
+        self._stack = ExitStack()
+        devnull = self._stack.enter_context(open(os.devnull, "w"))
+        self._stack.enter_context(redirect_stdout(devnull))
+        self._stack.enter_context(redirect_stderr(devnull))
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        return self._stack.__exit__(exc_type, exc_val, exc_tb)

{pythagoras-0.23.19 → pythagoras-0.24.1}/src/pythagoras/_090_swarming_portals/swarming_portals.py

@@ -12,13 +12,19 @@ at least once but does not offer any further guarantees.
 from __future__ import annotations
 
 import atexit
+import json
 from time import sleep
 
 import pandas as pd
 import parameterizable
-from parameterizable import sort_dict_by_keys
+from parameterizable import (
+    sort_dict_by_keys,
+    update_jsparams,
+    access_jsparams)
 from persidict import PersiDict, Joker, KEEP_CURRENT
 
+from parameterizable.json_processor import _Markers
+
 from .. import VALIDATION_SUCCESSFUL
 from .._010_basic_portals import get_all_known_portals
 from .._070_protected_code_portals.system_utils import get_unused_ram_mb, get_unused_cpu_cores, process_is_active, \
@@ -77,8 +83,8 @@ class SwarmingPortal(PureCodePortal):
         compute_nodes = OverlappingMultiDict(
             dict_type=dict_type
             , shared_subdicts_params=compute_nodes_shared_params
-            , json=dict(immutable_items=False)
-            , pkl=dict(immutable_items=False)
+            , json=dict(append_only=False)
+            , pkl=dict(append_only=False)
             )
         self._compute_nodes = compute_nodes
 
@@ -88,6 +94,14 @@ class SwarmingPortal(PureCodePortal):
         self._child_process = None
 
 
+    def get_params(self) -> dict:
+        params = super().get_params()
+        params["parent_process_id"] = self._parent_process_id
+        params["parent_process_start_time"] = self._parent_process_start_time
+        sorted_params = sort_dict_by_keys(params)
+        return sorted_params
+
+
     @property
     def is_parent(self) -> bool:
         """Check if this portal is the parent process."""
@@ -106,14 +120,14 @@ class SwarmingPortal(PureCodePortal):
         if self.is_parent:
             if self.max_n_workers > 0:
 
-                portal_init_params = self.get_portable_params()
-                portal_init_params["max_n_workers"] = self.max_n_workers
-                portal_init_params = sort_dict_by_keys(portal_init_params)
+                portal_init_jsparams = parameterizable.dumpjs(self)
+                portal_init_jsparams = update_jsparams(portal_init_jsparams,
+                    max_n_workers = self.max_n_workers)
 
                 ctx = get_context("spawn")
                 self._child_process = ctx.Process(
                     target=_launch_many_background_workers
-                    , kwargs=portal_init_params)
+                    , args=(portal_init_jsparams,))
                 self._child_process.start()
 
 
@@ -198,22 +212,22 @@ class SwarmingPortal(PureCodePortal):
             del self._max_n_workers_cache
         super()._invalidate_cache()
 
-parameterizable.register_parameterizable_class(SwarmingPortal)
+# parameterizable.register_parameterizable_class(SwarmingPortal)
 
 
-def _launch_many_background_workers(**portal_init_params) -> None:
+def _launch_many_background_workers(portal_init_jsparams:JsonSerializedParams) -> None:
     """Launch many background worker processes."""
-    n_workers_to_launch = portal_init_params["max_n_workers"]
+
+
+    n_workers_to_launch = access_jsparams(portal_init_jsparams
+        , "max_n_workers")["max_n_workers"]
     n_workers_to_launch = int(n_workers_to_launch)
 
-    portal_init_params["max_n_workers"] = 0
-    current_process_id = get_current_process_id()
-    portal_init_params["parent_process_id"] = current_process_id
-    portal_init_params["parent_process_start_time"
-        ] = get_current_process_start_time()
-    portal_init_params = sort_dict_by_keys(portal_init_params)
-    portal = parameterizable.get_object_from_portable_params(
-        portal_init_params)
+    portal_init_jsparams = update_jsparams(portal_init_jsparams,
+        max_n_workers=0, parent_process_id = get_current_process_id(),
+        parent_process_start_time = get_current_process_start_time())
+
+    portal = parameterizable.loadjs(portal_init_jsparams)
     assert isinstance(portal, SwarmingPortal)
 
     summary = build_execution_environment_summary()
@@ -226,7 +240,7 @@ def _launch_many_background_workers(**portal_init_params) -> None:
             portal._randomly_delay_execution(p=1)
             ctx = get_context("spawn")
             try:
-                p = ctx.Process(target=_background_worker, kwargs=portal_init_params)
+                p = ctx.Process(target=_background_worker, args=(portal_init_jsparams,))
                 p.start()
                 list_of_all_workers.append(p)
             except Exception as e:
@@ -243,7 +257,7 @@ def _launch_many_background_workers(**portal_init_params) -> None:
                     portal._randomly_delay_execution(p=1)
                     ctx = get_context("spawn")
                     try:
-                        p = ctx.Process(target=_background_worker, kwargs=portal_init_params)
+                        p = ctx.Process(target=_background_worker, args=(portal_init_jsparams,))
                         p.start()
                         new_list_of_all_workers.append(p)
                     except Exception as e:
@@ -251,10 +265,9 @@ def _launch_many_background_workers(**portal_init_params) -> None:
             list_of_all_workers = new_list_of_all_workers
 
 
-def _background_worker(**portal_init_params) -> None:
+def _background_worker(portal_init_jsparams:JsonSerializedParams) -> None:
     """Background worker that keeps processing random execution requests."""
-    portal = parameterizable.get_object_from_portable_params(
-        portal_init_params)
+    portal = parameterizable.loadjs(portal_init_jsparams)
     assert isinstance(portal, SwarmingPortal)
     with portal:
         ctx = get_context("spawn")
@@ -264,16 +277,19 @@ def _background_worker(**portal_init_params) -> None:
                     return
                 p = ctx.Process(
                     target=_process_random_execution_request
-                    , kwargs=portal_init_params)
+                    , args=(portal_init_jsparams,))
                 p.start()
                 p.join()
                 portal._randomly_delay_execution()
 
 
-def _process_random_execution_request(**portal_init_params):
+def _process_random_execution_request(portal_init_jsparams:JsonSerializedParams):
     """Process one random execution request."""
-    portal = parameterizable.get_object_from_portable_params(
-        portal_init_params)
+    # portal = parameterizable.get_object_from_portable_params(
+    #     portal_init_params)
+    portal_init_jsparams = update_jsparams(
+        portal_init_jsparams, max_n_workers=0)
+    portal = parameterizable.loadjs(portal_init_jsparams)
     assert isinstance(portal, SwarmingPortal)
     with portal:
         call_signature:PureFnCallSignature|None = None

pythagoras-0.23.19/src/pythagoras/_090_swarming_portals/output_suppressor.py

@@ -1,23 +0,0 @@
-import os
-import sys
-
-
-class OutputSuppressor:
-    """A context manager that suppresses stdout and stderr output."""
-    def __init__(self):
-        pass
-
-    def __enter__(self):
-        """Redirect stdout and stderr to os.devnull."""
-        self.old_stdout = sys.stdout
-        self.old_stderr = sys.stderr
-        self.devnull = open(os.devnull, 'w')
-        sys.stdout = self.devnull
-        sys.stderr = self.devnull
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        sys.stdout = self.old_stdout
-        sys.stderr = self.old_stderr
-        if self.devnull:
-            self.devnull.close()