pythagoras 0.8.5__tar.gz → 0.8.9__tar.gz

{pythagoras-0.8.5/pythagoras.egg-info → pythagoras-0.8.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pythagoras
-Version: 0.8.5
+Version: 0.8.9
 Summary: Simple framework for planet-scale idempotent computations in Python.
 Home-page: https://github.com/vladlpavlov/pythagoras
 Author: Volodymyr (Vlad) Pavlov
@@ -34,6 +34,9 @@ Requires-Dist: boto3
 Requires-Dist: moto
 Requires-Dist: pytest
 Requires-Dist: autopep8
+Requires-Dist: torch
+Requires-Dist: keras
+Requires-Dist: tensorflow
 
 # Pythagoras
 
@@ -74,6 +77,9 @@ at the Python package index at: https://pypi.org/project/pythagoras
 * [jsonpickle](https://jsonpickle.github.io)
 * [scikit-learn](https://scikit-learn.org)
 * [autopep8](https://pypi.org/project/autopep8)
+* [pytorch](https://pytorch.org)
+* [tensorflow](https://www.tensorflow.org)
+* [keras](https://keras.io)
 
 ## Key Contacts
 

{pythagoras-0.8.5 → pythagoras-0.8.9}/README.md

@@ -37,6 +37,9 @@ at the Python package index at: https://pypi.org/project/pythagoras
 * [jsonpickle](https://jsonpickle.github.io)
 * [scikit-learn](https://scikit-learn.org)
 * [autopep8](https://pypi.org/project/autopep8)
+* [pytorch](https://pytorch.org)
+* [tensorflow](https://www.tensorflow.org)
+* [keras](https://keras.io)
 
 ## Key Contacts
 

{pythagoras-0.8.5 → pythagoras-0.8.9}/pythagoras/_01_foundational_objects/hash_addresses.py

@@ -79,10 +79,11 @@ class HashAddress(SafeStrTuple, ABC):
         address = cls.__new__(cls)
         super(cls, address).__init__(prefix, hash_value)
         if assert_readiness:
-            assert address.ready()
+            assert address.ready
         return address
 
 
+    @property
     @abstractmethod
     def ready(self) -> bool:
         """Check if address points to a value that is ready to be retrieved."""

{pythagoras-0.8.5 → pythagoras-0.8.9}/pythagoras/_01_foundational_objects/value_addresses.py

@@ -38,17 +38,18 @@ class ValueAddress(HashAddress):
 
         super().__init__(prefix, hash_value)
 
-        if push_to_cloud and not (self in pth.value_store):
-            pth.value_store[self] = data
+        if push_to_cloud and not (self in pth.global_value_store):
+            pth.global_value_store[self] = data
 
+    @property
     def ready(self):
         """Check if address points to a value that is ready to be retrieved."""
-        return self in pth.value_store
+        return self in pth.global_value_store
 
 
     def get(self, timeout:Optional[int] = None) -> Any:
         """Retrieve value, referenced by the address"""
-        return pth.value_store[self]
+        return pth.global_value_store[self]
 
     def get_typed(self
             ,expected_type:Type[T]

{pythagoras-0.8.5 → pythagoras-0.8.9}/pythagoras/_02_ordinary_functions/ordinary_funcs.py

@@ -38,4 +38,8 @@ class OrdinaryFunction:
 
     @property
     def decorator(self):
-        return "@pth.ordinary()"
+        return "@pth.ordinary()"
+
+    @property
+    def f_name(self):
+        return self.name

{pythagoras-0.8.5 → pythagoras-0.8.9}/pythagoras/_03_autonomous_functions/autonomous_decorators.py

@@ -35,7 +35,7 @@ from pythagoras._03_autonomous_functions.default_island_singleton import (
     DefaultIslandType, DefaultIsland)
 from pythagoras._03_autonomous_functions.autonomous_funcs import (
      AutonomousFunction)
-from pythagoras._05_mission_control.global_state_management import (
+from pythagoras._06_mission_control.global_state_management import (
     is_fully_unitialized)
 
 

{pythagoras-0.8.5 → pythagoras-0.8.9}/pythagoras/_03_autonomous_functions/autonomous_funcs.py

@@ -17,12 +17,16 @@ from pythagoras._03_autonomous_functions.call_graph_explorer import (
 
 from pythagoras._03_autonomous_functions.names_usage_analyzer import (
     analyze_names_in_function)
-from pythagoras._05_mission_control.events_and_exceptions_core import \
-    EventLogger
 
-from pythagoras._05_mission_control.global_state_management import (
+from pythagoras._05_events_and_exceptions.find_in_callstack import (
+    find_local_var_in_callstack)
+
+from pythagoras._05_events_and_exceptions.event_logger import (
+    log_exception)
+
+from pythagoras._06_mission_control.global_state_management import (
     is_correctly_initialized)
-from pythagoras._99_misc_utils.find_in_callstack import find_in_callstack
+
 
 
 class AutonomousFunction(OrdinaryFunction):
@@ -139,7 +143,7 @@ class AutonomousFunction(OrdinaryFunction):
         return True
 
 
-    def __call__(self, **kwargs) -> Any:
+    def execute(self, **kwargs) -> Any:
         try:
             assert self.runtime_checks()
             names_dict = dict()
@@ -163,6 +167,9 @@ class AutonomousFunction(OrdinaryFunction):
             log_exception()
             raise e
 
+    def __call__(self, **kwargs) -> Any:
+        return self.execute(**kwargs)
+
     def __getstate__(self):
         draft_state = dict(name = self.name
             , naked_source_code = self.naked_source_code
@@ -226,21 +233,6 @@ def register_autonomous_function(f: AutonomousFunction) -> None:
         assert not hasattr(f, "_runtime_checks_passed")
         assert not hasattr(f, "_dependencies")
 
-def log_exception():
-    callers = find_in_callstack(name_to_find="self"
-        , class_to_find=AutonomousFunction)
-    caller_name = ""
-    if len(callers) > 0:
-        caller_name = callers[0].name + "_"
-    (exc_type, exc_value, trace_back) = sys.exc_info()
-    exception_description = traceback.format_exception(
-        exc_type, exc_value, trace_back)
-    logger = EventLogger(event_log = pth.crash_history
-        , prefix = caller_name + exc_type.__name__
-        , save_context = True)
-    logger.log_event(exception=exc_value
-        , exception_description=exception_description)
-
 
 class EventPosterFactory:
     def __init__(self, silent:bool = False):
@@ -260,8 +252,8 @@ class EventPoster:
         self.silent = silent
 
     def __call__(self, **event_args)-> None:
-        callers = find_in_callstack(name_to_find="self"
-            , class_to_find=AutonomousFunction)
+        callers = find_local_var_in_callstack(name_to_find="self"
+                                              , class_to_find=AutonomousFunction)
         caller_name = ""
         if len(callers) > 0:
             caller_name = callers[0].name
@@ -272,11 +264,10 @@ class EventPoster:
             caller_prefix = caller_name
             caller_type = "AutonomousFunction"
         prefix = caller_prefix + self.label
-        logger = EventLogger(event_log=pth.event_log
+        logger = EventLogger(event_log=pth.global_event_log
             , prefix=prefix, save_context=False)
-        logger.log_event(**event_args)
-        if not self.silent:
 
+        if not self.silent:
             print(30*"~")
             print(f"Event '{self.label}' "
                   + f"inside an {caller_type} '{caller_name}':")
@@ -284,5 +275,17 @@ class EventPoster:
                 print(f"    {key} = {value}")
             print()
 
+        label_arg_name = "event_label"
+        while label_arg_name in event_args:
+            label_arg_name += "_"
+        event_args[label_arg_name] = self.label
+
+        caller_arg_name = "caller_name"
+        while caller_arg_name in event_args:
+            caller_arg_name += "_"
+        event_args[caller_arg_name] = caller_name
+
+        logger.log_event(**event_args)
+
 post_event: EventPosterFactory = EventPosterFactory(silent=True)
 print_event: EventPosterFactory = EventPosterFactory(silent=False)

{pythagoras-0.8.5 → pythagoras-0.8.9}/pythagoras/_04_idempotent_functions/__init__.py

@@ -2,7 +2,7 @@ from pythagoras._04_idempotent_functions.kw_args import (
     SortedKwArgs, PackedKwArgs, UnpackedKwArgs)
 
 from pythagoras._04_idempotent_functions.idempotent_func_and_address import (
-    IdempotentFunction)
+    IdempotentFunction, FuncOutputAddress)
 
 from pythagoras._04_idempotent_functions.idempotent_decorator import (
     idempotent)

{pythagoras-0.8.5 → pythagoras-0.8.9}/pythagoras/_04_idempotent_functions/idempotent_decorator.py

@@ -8,7 +8,7 @@ from pythagoras._03_autonomous_functions.default_island_singleton import (
 from pythagoras._04_idempotent_functions.idempotent_func_and_address import (
     IdempotentFunction)
 
-from pythagoras._05_mission_control.global_state_management import (
+from pythagoras._06_mission_control.global_state_management import (
     is_fully_unitialized)
 
 

{pythagoras-0.8.5 → pythagoras-0.8.9}/pythagoras/_04_idempotent_functions/idempotent_func_and_address.py

@@ -4,12 +4,13 @@ import time
 from typing import Callable, Any
 
 import pythagoras as pth
+from pythagoras import get_random_safe_str
 
 from pythagoras._01_foundational_objects.hash_addresses import HashAddress
 from pythagoras._01_foundational_objects.value_addresses import ValueAddress
 
 from pythagoras._03_autonomous_functions.autonomous_funcs import (
-    AutonomousFunction, register_autonomous_function, log_exception)
+    AutonomousFunction, register_autonomous_function)
 
 from pythagoras._02_ordinary_functions.ordinary_funcs import (
     OrdinaryFunction)
@@ -21,6 +22,8 @@ from pythagoras._04_idempotent_functions.kw_args import (
     UnpackedKwArgs, PackedKwArgs, SortedKwArgs)
 from pythagoras._04_idempotent_functions.process_augmented_func_src import (
     process_augmented_func_src)
+from pythagoras._05_events_and_exceptions.context_utils import build_context
+
 
 class IdempotentFunction(AutonomousFunction):
     augmented_code_checked: bool
@@ -105,16 +108,45 @@ class IdempotentFunction(AutonomousFunction):
 
         self.augmented_code_checked = True
 
-    def __call__(self, **kwargs) -> Any:
+
+    def execute(self, **kwargs) -> Any:
         packed_kwargs = PackedKwArgs(**kwargs)
         output_address = FuncOutputAddress(self, packed_kwargs)
-        if output_address.ready():
+        _pth_f_addr_ = output_address
+        if output_address.ready:
             return output_address.get()
+        output_address.request_execution()
+        registration_addr = (output_address[0]
+            , output_address[1], get_random_safe_str())
+        pth.function_execution_attempts[registration_addr] = build_context()
         unpacked_kwargs = UnpackedKwArgs(**packed_kwargs)
-        result = super().__call__(**unpacked_kwargs)
+        result = super().execute(**unpacked_kwargs)
         pth.function_output_store[output_address] = ValueAddress(result)
+        pth.function_execution_requests.delete_if_exists(output_address)
         return result
 
+    def list_execute(self, list_of_kwargs:list[dict]) -> Any:
+        assert isinstance(list_of_kwargs, (list, tuple))
+        for kwargs in list_of_kwargs:
+            assert isinstance(kwargs, dict)
+        addrs = []
+        for kwargs in list_of_kwargs:
+            new_addr = FuncOutputAddress(self, kwargs)
+            new_addr.request_execution()
+            addrs.append(new_addr)
+        addrs_indexed = list(zip(range(len(addrs)), addrs))
+        pth.entropy_infuser.shuffle(addrs_indexed)
+        results_dict = dict()
+        for n, an_addr in addrs_indexed:
+            results_dict[n] = an_addr.function.execute(**an_addr.arguments)
+        results_list = [results_dict[n] for n in range(len(addrs))]
+        return results_list
+
+    def execution_attempts(self, **kwargs) -> list:
+        packed_kwargs = PackedKwArgs(**kwargs)
+        output_address = FuncOutputAddress(self, packed_kwargs)
+        return output_address.execution_attempts
+
 
 def register_idempotent_function(f: IdempotentFunction) -> None:
     """Register an idempotent function in the Pythagoras system."""
@@ -143,8 +175,18 @@ class FuncOutputAddress(HashAddress):
         tmp = ValueAddress(signature)
         super().__init__(tmp.prefix, tmp.hash_value)
 
+    @property
     def ready(self):
-        return self in pth.function_output_store
+        result =  self in pth.function_output_store
+        return result
+
+    def request_execution(self):
+        if self in pth.function_output_store:
+            if self in pth.function_execution_requests:
+                del pth.function_execution_requests[self]
+        else:
+            if self not in pth.function_execution_requests:
+                pth.function_execution_requests[self] = True
 
     def get(self, timeout: int = None):
         """Retrieve value, referenced by the address.
@@ -152,14 +194,20 @@ class FuncOutputAddress(HashAddress):
         If the value is not immediately available, backoff exponentially
         till timeout is exceeded. If timeout is None, keep trying forever.
         """
+        if self.ready:
+            return pth.global_value_store[pth.function_output_store[self]]
+        self.request_execution()
+
         start_time, backoff_period = time.time(), 1.0
         stop_time = (start_time + timeout) if timeout else None
         # start_time, stop_time and backoff_period are in seconds
+
         while True:
-            try:
-                address = pth.function_output_store[self]
-                return pth.value_store[address]
-            except:
+            if self.ready:
+                result = pth.global_value_store[pth.function_output_store[self]]
+                pth.function_execution_requests.delete_if_exists(self)
+                return result
+            else:
                 time.sleep(backoff_period)
                 backoff_period *= 2.0
                 backoff_period += pth.entropy_infuser.uniform(-0.5, 0.5)
@@ -185,9 +233,62 @@ class FuncOutputAddress(HashAddress):
         signature = signature_addr.get()
         return signature.f_name
 
+    @property
+    def island_name(self) -> str:
+        return self.function.island_name
+
     @property
     def arguments(self) -> SortedKwArgs:
         signature_addr = ValueAddress.from_strings(
             prefix=self.prefix, hash_value=self.hash_value)
         signature = signature_addr.get()
-        return signature.args_addr.get()
+        return signature.args_addr.get()
+
+    @property
+    def can_be_executed(self) -> bool:
+        """Indicates if the function can be executed in the current session.
+
+        Currently, it's just a placeholder that always returns True.
+
+        The function should fe refactored once we start supporting
+        VALIDATORS, CORRECTORS and SEQUENCERS
+        """
+        return True
+
+    @property
+    def needs_execution(self) -> bool:
+        """Indicates if the function is a good candidate for execution.
+
+        Returns False if the result is already available, or if some other
+        process is currently working on it. Otherwise, returns True.
+        """
+        DEFAULT_EXECUTION_TIME = 10
+        MAX_EXECUTION_ATTEMPTS = 5
+        # TODO: these should not be constants
+        if self.ready:
+            return False
+        past_attempts = pth.function_execution_attempts.get_subdict(self)
+        n_past_attempts = len(past_attempts)
+        if n_past_attempts == 0:
+            return True
+        if n_past_attempts > MAX_EXECUTION_ATTEMPTS:
+            #TODO: log this event. Should we have DLQ?
+            return False
+        most_recent_timestamp = max(
+            past_attempts.mtimestamp(a) for a in past_attempts)
+        current_timestamp = time.time()
+        if (current_timestamp - most_recent_timestamp
+                > DEFAULT_EXECUTION_TIME*(2**n_past_attempts)):
+            return True
+        return False
+
+    @property
+    def execution_attempts(self) -> list:
+        attempts = pth.function_execution_attempts.get_subdict(self)
+        attemps_timed = {-attempts.mtimestamp(a):attempts[a] for a in attempts}
+        times = sorted(attemps_timed)
+        result = []
+        for t in times:
+            result.append(attemps_timed[t])
+        return result
+

{pythagoras-0.8.5 → pythagoras-0.8.9}/pythagoras/_04_idempotent_functions/kw_args.py

@@ -25,7 +25,7 @@ class SortedKwArgs(dict):
         unpacked_copy = dict()
         for k,v in self.items():
             if isinstance(v, ValueAddress):
-                unpacked_copy[k] = pth.value_store[v]
+                unpacked_copy[k] = pth.global_value_store[v]
             else:
                 unpacked_copy[k] = v
         return unpacked_copy

pythagoras-0.8.9/pythagoras/_05_events_and_exceptions/__init__.py

@@ -0,0 +1,2 @@
+from pythagoras._05_events_and_exceptions.notebook_checker import (
+    is_executed_in_notebook)

pythagoras-0.8.5/pythagoras/_99_misc_utils/context_builder.py → pythagoras-0.8.9/pythagoras/_05_events_and_exceptions/context_utils.py

@@ -5,7 +5,8 @@ import socket
 from typing import Dict
 from getpass import getuser
 from datetime import datetime
-from pythagoras._99_misc_utils.notebook_checker import is_executed_in_notebook
+import torch
+from pythagoras._05_events_and_exceptions.notebook_checker import is_executed_in_notebook
 
 def build_context()-> Dict:
     """Capture core information about execution environment.
@@ -14,7 +15,6 @@ def build_context()-> Dict:
     to help debug (distributed) applications.
     """
     cwd = os.getcwd()
-    is_in_notebook = is_executed_in_notebook()
 
     context = dict(
         hostname = socket.gethostname()
@@ -26,12 +26,20 @@ def build_context()-> Dict:
         ,processor = platform.processor()
         ,cpu_count = psutil.cpu_count()
         ,cpu_load_avg = psutil.getloadavg()
+        ,cuda_gpu_count=torch.cuda.device_count()
         ,disk_usage = psutil.disk_usage(cwd)
         ,virtual_memory = psutil.virtual_memory()
-        ,cwd = os.getcwd()
         ,working_directory = cwd
         ,local_timezone = datetime.now().astimezone().tzname()
-        ,is_in_notebook = is_in_notebook
+        ,is_in_notebook = is_executed_in_notebook()
         )
 
-    return context
+    return context
+
+
+def add_context(**kwargs):
+    context_param_name = "context"
+    while context_param_name in kwargs:
+        context_param_name += "_"
+    kwargs[context_param_name] = build_context()
+    return kwargs

pythagoras-0.8.9/pythagoras/_05_events_and_exceptions/event_logger.py

@@ -0,0 +1,88 @@
+import sys
+import traceback
+from copy import copy
+
+from pythagoras._05_events_and_exceptions.context_utils import add_context
+from pythagoras._05_events_and_exceptions.current_date_gmt_str import (
+    current_date_gmt_string)
+from pythagoras._05_events_and_exceptions.find_in_callstack import (
+    find_local_var_in_callstack)
+from pythagoras._05_events_and_exceptions.type_retrievers import (
+    retrieve_AutonomousFunction_class, retrieve_FuncOutputAddress_class)
+from pythagoras._99_misc_utils.random_safe_str_creator import (
+    get_random_safe_str)
+
+from persidict import PersiDict
+import pythagoras as pth
+
+
+def log_one_entry(event_log_dict:PersiDict
+        , path:list[str] | None
+        , prefixes: list[str] | None
+        , name:str
+        , event:any):
+    if prefixes is not None and len(prefixes) > 0:
+        name = "_".join(prefixes + [name])
+    if path is None or len(path) == 0:
+        address = [name]
+    else:
+        address = path + [name]
+    event_log_dict[address] = event
+
+def log_two_mirrored_entries(
+        global_event_log_dict:PersiDict | None
+        , local_event_log_dict:PersiDict | None
+        , prefix: str | None
+        , event: any):
+    func_addrs = find_local_var_in_callstack(name_to_find="_pth_f_addr_"
+            , class_to_find = retrieve_FuncOutputAddress_class())
+    if len(func_addrs) > 0:
+        func_pointer = func_addrs[0]
+        local_path = list(func_pointer)
+    else:
+        funcs = find_local_var_in_callstack(name_to_find="self"
+            , class_to_find=retrieve_AutonomousFunction_class())
+        if len(funcs) > 0:
+            func_pointer = funcs[0]
+            local_path = None
+        else:
+            assert False, "No function pointer found in callstack"
+
+    island_name = func_pointer.island_name
+    function_name = func_pointer.f_name
+    event_name = get_random_safe_str()
+
+    if prefix is not None:
+        prefixes = [prefix]
+    else:
+        prefixes = []
+
+    if global_event_log_dict is not None:
+        if island_name is not None:
+            global_prefixes = [island_name] + prefixes
+        else:
+            global_prefixes = copy(prefixes)
+        global_prefixes = [function_name] + global_prefixes
+        global_path = [current_date_gmt_string()]
+        log_one_entry(global_event_log_dict
+            , global_path, global_prefixes, event_name, event)
+
+    if local_event_log_dict is not None and local_path is not None:
+        local_prefixes = copy(prefixes)
+        log_one_entry(local_event_log_dict
+            , local_path, local_prefixes, event_name, event)
+
+
+def log_exception():
+    (exc_type, exc_value, trace_back) = sys.exc_info()
+    description = traceback.format_exception(
+        exc_type, exc_value, trace_back)
+    event = add_context(
+        exception_description=description
+        ,exception=exc_value
+        ,trace_back= trace_back)
+    log_two_mirrored_entries(
+        global_event_log_dict=pth.global_crash_history
+        , local_event_log_dict=pth.function_crash_history
+        , prefix=exc_type.__name__
+        , event=event)

{pythagoras-0.8.5/pythagoras/_99_misc_utils → pythagoras-0.8.9/pythagoras/_05_events_and_exceptions}/find_in_callstack.py

@@ -1,11 +1,11 @@
 import inspect
 from typing import List, Any
 
-def find_in_callstack(
+def find_local_var_in_callstack(
         name_to_find: str
-        , class_to_find: None | type = None
+        , class_to_find: None | type | str = None
         ) -> List[Any]:
-    """ Search the entire call stack for objects with the specified name/type.
+    """ Search the entire call stack for local objects with the specified name/type.
 
     If the callstack contains objects
     with name name_to_find and type class_to_find,
@@ -19,16 +19,18 @@ def find_in_callstack(
 
     Parameters:
     name_to_find (str): The name of the object to search for.
-    class_to_find (None | type): The type of the object to search for.
+    class_to_find (None | type | str): The type of the object to search for.
 
     Returns:
-    List[Any]: A list of objects found in the stack
+    List[Any]: A list of local objects found in the stack
     with the specified name/type.
     The list contains no duplicates.
     """
 
     assert isinstance(name_to_find, str) and len(name_to_find)
-    assert class_to_find is None or inspect.isclass(class_to_find)
+    assert (class_to_find is None
+            or inspect.isclass(class_to_find)
+            or isinstance(class_to_find, str))
 
     found_objects = []
 
@@ -36,11 +38,12 @@ def find_in_callstack(
         frame = frame_info.frame
         if name_to_find in frame.f_locals:
             candidate = frame.f_locals[name_to_find]
-            if class_to_find is None or isinstance(candidate, class_to_find):
+            if class_to_find is None:
                 found_objects.append(candidate)
-        if name_to_find in frame.f_globals:
-            candidate = frame.f_globals[name_to_find]
-            if class_to_find is None or isinstance(candidate, class_to_find):
+            elif isinstance(class_to_find, str):
+                if candidate.__class__.__name__ == class_to_find:
+                    found_objects.append(candidate)
+            elif isinstance(candidate, class_to_find):
                 found_objects.append(candidate)
 
     dedup_dict = dict()

pythagoras-0.8.9/pythagoras/_05_events_and_exceptions/type_retrievers.py

@@ -0,0 +1,30 @@
+import sys
+
+def retrieve_IdempotentFunction_class() -> type:
+    """Return the IdempotentFunction class.
+
+    This is for-internal-use-only function, created to
+    avoid circular import dependencies.
+    """
+    return sys.modules['pythagoras'].IdempotentFunction
+
+
+def retrieve_AutonomousFunction_class() -> type:
+    """Return the AutonomousFunction class.
+
+    This is for-internal-use-only function, created to
+    avoid circular import dependencies.
+    """
+    return sys.modules['pythagoras'].AutonomousFunction
+
+
+def retrieve_FuncOutputAddress_class() -> type:
+    """Return the FuncOutputAddress class.
+
+    This is for-internal-use-only function, created to
+    avoid circular import dependencies.
+    """
+    return sys.modules['pythagoras'].FuncOutputAddress
+
+
+