pythagoras 0.24.4__py3-none-any.whl → 0.24.7__py3-none-any.whl

pythagoras/_090_swarming_portals/swarming_portals.py

@@ -45,6 +45,29 @@ BACKGROUND_WORKERS_TXT = "Background workers"
 
 
 class SwarmingPortal(PureCodePortal):
+    """Portal for asynchronous swarming execution of pure functions.
+
+    The SwarmingPortal distributes execution of registered pure functions
+    across background worker processes (potentially on different machines)
+    in a best-effort, eventually-executed manner. It manages a child process
+    that maintains a pool of background workers and randomly dispatches
+    execution requests. No strong guarantees are provided regarding which
+    worker runs a task, how many times it runs, or when it will run; only that
+    eligible requests will eventually be executed at least once.
+
+    Notes:
+    - Parent/child: The portal instance created by user code is the parent.
+      It may spawn a separate child process whose responsibility is to keep
+      background workers alive. Child processes created by the portal do not
+      spawn further workers (max_n_workers is forced to 0 in children).
+    - Resources: The effective number of workers is automatically bounded by
+      currently available CPU cores and RAM at runtime.
+    - Environment logging: Runtime environment summary is stored under compute_nodes
+      for debugging purposes to help describe where the parent process is running.
+
+    See also: OutputSuppressor for silencing worker output, PureCodePortal for
+    the base API and lifecycle management, and tests in tests/_090_swarming_portals.
+    """
     _compute_nodes: OverlappingMultiDict | None
     _node_id: str | None
 
@@ -62,6 +85,31 @@ class SwarmingPortal(PureCodePortal):
                  , parent_process_id: int | None = None
                  , parent_process_start_time: float | None = None
                  ):
+        """Initialize a swarming portal.
+
+        Args:
+            root_dict: Persistent dictionary or path used to back the portal's
+                state. If None, a default dictionary will be used.
+            p_consistency_checks: Probability or Joker controlling internal
+                consistency checks. Passed to PureCodePortal.
+            excessive_logging: Whether to enable verbose logging. Passed to
+                PureCodePortal.
+            max_n_workers: Desired maximum number of background workers for the
+                parent process. Children must pass 0 here.
+                The effective value may be reduced based on available CPUs and
+                RAM at runtime.
+            parent_process_id: ID of the parent process when this portal is
+                constructed inside a child process. For parent portals, it
+                must be None.
+            parent_process_start_time: Start time of the parent process, used to
+                detect PID reuse. For parents, it must be None.
+
+        Notes:
+            - When parent_process_id or parent_process_start_time is provided,
+              both must be provided and max_n_workers must be 0.
+            - Initializes compute_nodes storage and captures a unique
+              node signature for this runtime.
+        """
         PureCodePortal.__init__(self
             , root_dict=root_dict
             , p_consistency_checks=p_consistency_checks
@@ -95,6 +143,13 @@ class SwarmingPortal(PureCodePortal):
 
 
     def get_params(self) -> dict:
+        """Return portal parameters including parent process metadata.
+
+        Returns:
+            dict: Sorted dictionary of portal parameters inherited from
+            PureCodePortal plus parent_process_id and
+            parent_process_start_time.
+        """
         params = super().get_params()
         params["parent_process_id"] = self._parent_process_id
         params["parent_process_start_time"] = self._parent_process_start_time
@@ -104,13 +159,26 @@ class SwarmingPortal(PureCodePortal):
 
     @property
     def is_parent(self) -> bool:
-        """Check if this portal is the parent process."""
+        """Whether this portal instance represents the parent process.
+
+        Returns:
+            bool: True if this process created the portal instance on the node
+            (no parent metadata set); False if this is a child portal
+            instantiated inside the background worker process.
+        """
         if self._parent_process_id is None:
             return True
         return False
 
 
     def _post_init_hook(self) -> None:
+        """Lifecycle hook invoked after initialization.
+
+        Registers a global atexit handler once per process to terminate any
+        child processes spawned by portals. For parent portals with a positive
+        max_n_workers, spawns a child process that manages the pool of
+        background worker processes.
+        """
         super()._post_init_hook()
 
         if not SwarmingPortal._atexit_is_registered:
@@ -132,7 +200,12 @@ class SwarmingPortal(PureCodePortal):
 
 
     def _terminate_child_process(self):
-        """Terminate the child process if it is running."""
+        """Terminate the child process if it is running.
+
+        This method is idempotent and safe to call from atexit handlers.
+        It attempts a graceful termination, then escalates to kill if the
+        child does not stop within a short grace period.
+        """
         if self._child_process is not None:
             if self._child_process.is_alive():
                 self._child_process.terminate()
@@ -145,14 +218,28 @@ class SwarmingPortal(PureCodePortal):
 
     @property
     def _execution_environment_address(self) -> list[str]: #TODO: move to Logs
-        """Get the address of the execution environment in the compute nodes."""
+        """Address path for storing execution environment summary.
+
+        Returns:
+            list[str]: A hierarchical key used in the compute_nodes storage of
+            the form [node_id, parent_pid_and_start, "execution_environment"].
+        """
         s = str(self._parent_process_id) + "_" + str(self._parent_process_start_time)
         return [self._node_id, s, "execution_environment"]
 
 
     @property
     def max_n_workers(self) -> int:
-        """Get the maximum number of background workers"""
+        """Effective cap on background worker processes.
+
+        The configured max_n_workers value is adjusted down by runtime
+        resource availability: currently unused CPU cores and available RAM.
+        The result is cached in RAM for the life of the portal process
+        until the cache is invalidated.
+
+        Returns:
+            int: Effective maximum number of worker processes to use.
+        """
         if not hasattr(self, "_max_n_workers_cache"):
             n = self._get_config_setting("max_n_workers")
             if n in (None, KEEP_CURRENT):
@@ -166,7 +253,13 @@ class SwarmingPortal(PureCodePortal):
 
 
     def describe(self) -> pd.DataFrame:
-        """Get a DataFrame describing the portal's current state"""
+        """Describe the current portal configuration and runtime.
+
+        Returns:
+            pandas.DataFrame: A table combining PureCodePortal description with
+            swarming-specific characteristics such as effective number of
+            background workers.
+        """
         all_params = [super().describe()]
         all_params.append(_describe_runtime_characteristic(
             BACKGROUND_WORKERS_TXT, self.max_n_workers))
@@ -177,6 +270,12 @@ class SwarmingPortal(PureCodePortal):
 
 
     def parent_runtime_is_live(self):
+        """Check that the recorded parent process is still alive.
+
+        Returns:
+            bool: True if the parent PID exists and its start time matches the
+            recorded start time (to avoid PID reuse issues); False otherwise.
+        """
         if not process_is_active(self._parent_process_id):
             return False
         if get_process_start_time(self._parent_process_id) != self._parent_process_start_time:
@@ -185,6 +284,12 @@ class SwarmingPortal(PureCodePortal):
 
 
     def _clear(self):
+        """Release resources and clear internal state.
+
+        Side Effects:
+            Terminates the child process if present and clears references to
+            compute node metadata before delegating to the base implementation.
+        """
         self._compute_nodes = None
         self._terminate_child_process()
         super()._clear()
@@ -195,18 +300,28 @@ class SwarmingPortal(PureCodePortal):
             , min_delay:float = 0.02
             , max_delay:float = 0.22
             ) -> None:
-        """Randomly delay execution by a given probability."""
+        """Introduce randomized backoff to reduce contention.
+
+        With probability p, sleeps for a random delay uniformly drawn from
+        [min_delay, max_delay]. Uses the portal's entropy_infuser to remain
+        deterministic when seeded in tests.
+
+        Args:
+            p: Probability of applying the delay.
+            min_delay: Minimum sleep duration in seconds.
+            max_delay: Maximum sleep duration in seconds.
+        """
         if self.entropy_infuser.uniform(0, 1) < p:
             delay = self.entropy_infuser.uniform(min_delay, max_delay)
             sleep(delay)
 
 
     def _invalidate_cache(self):
-        """Invalidate the object's attribute cache.
+        """Drop cached computed attributes and delegate to base class.
 
-        If the object's attribute named ATTR is cached,
-        its cached value will be stored in an attribute named _ATTR_cache
-        This method should delete all such attributes.
+        This implementation removes any attributes used as local caches by this
+        class (currently _max_n_workers_cache) and then calls the base
+        implementation to allow it to clear its own caches.
         """
         if hasattr(self, "_max_n_workers_cache"):
             del self._max_n_workers_cache
@@ -214,7 +329,19 @@ class SwarmingPortal(PureCodePortal):
 
 
 def _launch_many_background_workers(portal_init_jsparams:JsonSerializedObject) -> None:
-    """Launch many background worker processes."""
+    """Spawn and maintain a pool of background worker processes.
+
+    This function is executed inside a dedicated child process created by the
+    parent portal. It spawns up to max_n_workers worker processes, restarts
+    any that exit unexpectedly, and records an execution environment summary
+    under the portal's compute_nodes.
+
+    Args:
+        portal_init_jsparams: Serialized initialization parameters for
+            reconstructing a SwarmingPortal. The parameters are adjusted to
+            indicate this is a child context (max_n_workers=0) and to record
+            the parent process metadata.
+    """
 
 
     n_workers_to_launch = access_jsparams(portal_init_jsparams
@@ -264,7 +391,16 @@ def _launch_many_background_workers(portal_init_jsparams:JsonSerializedObject) -
 
 
 def _background_worker(portal_init_jsparams:JsonSerializedObject) -> None:
-    """Background worker that keeps processing random execution requests."""
+    """Worker loop that processes random execution requests serially.
+
+    Runs indefinitely until the parent process is detected as dead.
+    Within the loop, each individual request is handled in a subprocess to
+    isolate failures and to reduce the risk of state leakage.
+
+    Args:
+        portal_init_jsparams: Serialized initialization parameters for
+            reconstructing a SwarmingPortal in child context.
+    """
     portal = parameterizable.loadjs(portal_init_jsparams)
     assert isinstance(portal, SwarmingPortal)
     with portal:
@@ -282,9 +418,18 @@ def _background_worker(portal_init_jsparams:JsonSerializedObject) -> None:
 
 
 def _process_random_execution_request(portal_init_jsparams:JsonSerializedObject):
-    """Process one random execution request."""
-    # portal = parameterizable.get_object_from_portable_params(
-    #     portal_init_params)
+    """Process a single pending execution request, if any.
+
+    The function reconstructs a child-context portal, selects a random pending
+    execution request (if available), and validates readiness. If validation
+    yields a PureFnCallSignature, it continues with it; otherwise, it executes
+    the request when validation returns VALIDATION_SUCCESSFUL. Output during
+    execution is suppressed by the caller to keep workers quiet.
+
+    Args:
+        portal_init_jsparams: Serialized initialization parameters for
+            reconstructing a SwarmingPortal in child context.
+    """
     portal_init_jsparams = update_jsparams(
         portal_init_jsparams, max_n_workers=0)
     portal = parameterizable.loadjs(portal_init_jsparams)
@@ -326,13 +471,14 @@ def _process_random_execution_request(portal_init_jsparams:JsonSerializedObject)
 
 
 def _terminate_all_portals_child_processes():
-    """ Clean runtime id.
+    """Terminate child processes for all known portals.
 
-    This function is called at the end of the program execution.
-    It deletes the principal_runtime_id record from all portals.
+    Registered with atexit the first time a SwarmingPortal is initialized.
+    Ensures that any child processes are terminated to avoid orphaned workers.
     """
     for portal in get_all_known_portals():
         try:
             portal._terminate_child_process()
-        except:
+        except Exception:
+            # Best-effort cleanup; ignore errors during shutdown.
             pass

pythagoras/_100_top_level_API/__init__.py

@@ -1,2 +1,13 @@
+"""Top-level, user-facing API shortcuts.
+
+This subpackage exposes the easiest entry points for application authors,
+primarily constructors and helpers to obtain a portal and interact with it.
+
+Exports:
+  top_level_API: Functions that construct and return a portal (e.g., get_portal).
+  default_local_portal: Defaults and helpers for creating a local portal
+  when not explicitly created/provided by an application that uses Pythagoras.
+"""
+
 from .top_level_API import *
 from .default_local_portal import *

pythagoras/_800_signatures_and_converters/__init__.py

@@ -1,3 +1,20 @@
+"""Signatures and conversion utilities.
+
+This subpackage provides helpers for generating stable identifiers and
+converting between common textual/byte representations used across
+Pythagoras. It re-exports frequently used helpers for convenience.
+
+The modules exposed here are intentionally lightweight and side-effect free
+so they can be used in hashing and address computations.
+
+Exports:
+  base_16_32_convertors: Base-16/32 encoding and decoding helpers.
+  current_date_gmt_str: Utilities to format current date/time in GMT.
+  hash_signatures: Functions to compute content hash/signature strings.
+  node_signature: Functions to derive signatures for the current node.
+  random_signatures: Helpers to generate random, collision-resistant IDs.
+"""
+
 from .base_16_32_convertors import *
 from .current_date_gmt_str import *
 from .hash_signatures import *

pythagoras/_800_signatures_and_converters/base_16_32_convertors.py

@@ -1,48 +1,83 @@
+from typing import Final
 
-base32_alphabet = '0123456789abcdefghijklmnopqrstuv'
-base32_alphabet_map = {char:index for index,char in enumerate(base32_alphabet)}
+_BASE32_ALPHABET: Final[str] = '0123456789abcdefghijklmnopqrstuv'
+_BASE32_ALPHABET_MAP: Final[dict[str, int]] = {
+    char:index for index,char in enumerate(_BASE32_ALPHABET)}
 
 
 def convert_base16_to_base32(hexdigest: str) -> str:
-    """
-    Convert a hexadecimal (base 16) string to a base 32 string.
+    """Convert a hexadecimal (base16) string to this project's base32.
+
+    Args:
+        hexdigest (str): A hexadecimal string (case-insensitive). May be an
+            empty string or "0" to represent zero.
 
-    :name hexdigest: A string representing a hexadecimal number.
-    :return: A string representing the equivalent number in base 32.
+    Returns:
+        str: The corresponding value encoded with the custom base32 alphabet
+        (digits 0-9 then letters a-v).
+
+    Examples:
+        >>> convert_base16_to_base32("ff")
+        '7v'
     """
 
-    if not hexdigest:
-        return '0'
-    num = int(hexdigest,16)
+    try:
+        num = int(hexdigest, 16)
+    except ValueError as e:
+        raise ValueError(f"Invalid hexadecimal string: {hexdigest}") from e
+
     base32_str = convert_int_to_base32(num)
 
     return base32_str
 
 
 def convert_int_to_base32(n: int) -> str:
-    """
-    Convert an integer to a base 32 string.
+    """Convert a non-negative integer to Pythagoras' base32 string.
+
+    Args:
+        n (int): Non-negative integer to encode.
+
+    Returns:
+        str: The base32 representation.
 
-    :name n: An integer.
-    :return: A string representing the equivalent number in base 32.
+    Raises:
+        ValueError: If n is negative.
     """
+    if n < 0:
+        raise ValueError("n must be non-negative")
+
+    if n == 0:
+        return "0"
+
     base32_str = ''
     while n > 0:
-        base32_str = base32_alphabet[n & 31] + base32_str
+        base32_str = _BASE32_ALPHABET[n & 31] + base32_str
         n >>= 5
 
     return base32_str
 
 def convert_base_32_to_int(digest: str) -> int:
-    """
-    Convert a base 32 string to an integer.
+    """Convert a base32 string (custom alphabet) to an integer.
 
-    :name digest: A string representing a number in base 32.
-    :return: An integer representing the equivalent number.
+    Args:
+        digest (str): String encoded with Pythagoras' base32 alphabet.
+
+    Returns:
+        int: The decoded non-negative integer value.
+
+    Raises:
+        KeyError: If digest contains a character outside the supported
+            base32 alphabet (0-9, a-v).
     """
+    if not digest:
+        raise ValueError("Digest cannot be empty")
+
     digest = digest.lower()
     num = 0
-    for char in digest:
-        num = num * 32 + base32_alphabet_map[char]
+    try:
+        for char in digest:
+            num = num * 32 + _BASE32_ALPHABET_MAP[char]
+    except KeyError as e:
+        raise ValueError(f"Invalid character '{e.args[0]}' in base32 digest: {digest}") from e
     return num
 

pythagoras/_800_signatures_and_converters/current_date_gmt_str.py

@@ -1,11 +1,26 @@
 from datetime import datetime, timezone
+from typing import Final
 
+_MONTH_ABBREVIATIONS: Final[tuple[str, ...]] = (
+    "Jan", "Feb", "Mar", "Apr", "May", "Jun",
+    "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")
 
-def current_date_gmt_string():
-    """ Return the current date and time in the format '2024_12Jan_22_utc'
+
+def current_date_gmt_string() -> str:
+    """Get the current UTC date as a compact string.
+
+    Produces an underscore-delimited UTC date string suitable for
+    stable file names and log records.
+
+    The format is: "YYYY_MMMonAbbrev_dd_utc" (e.g., "2024_12Dec_11_utc").
+
+    Returns:
+        str: The formatted UTC date string, for the current moment.
     """
 
     utc_now = datetime.now(timezone.utc)
-    date_str = utc_now.strftime("%Y_%m%b_%d_utc")
-
-    return date_str
+    month_abbrev = _MONTH_ABBREVIATIONS[utc_now.month - 1]
+    # locale-dependent month abbreviation
+    result =  (f"{utc_now.year}_{utc_now.month:02d}{month_abbrev}" +
+              f"_{utc_now.day:02d}_utc")
+    return result

pythagoras/_800_signatures_and_converters/hash_signatures.py

@@ -1,33 +1,69 @@
 import sys
-from typing import Any
+from typing import Any, Final
 
 import joblib.hashing
 
 from .base_16_32_convertors import convert_base16_to_base32
 
 
-hash_type: str = "sha256"
-max_signature_length: int = 22
+_HASH_TYPE: Final[str] = "sha256"
+_MAX_SIGNATURE_LENGTH: Final[int] = 22
 
 def get_base16_hash_signature(x:Any) -> str:
-    """Return base16 hash signature of an object.
+    """Compute a hexadecimal (base16) hash for an arbitrary Python object.
 
-    Uses joblib's Hasher (or NumpyHasher). It uses Pickle for serialization,
-    except for NumPy arrays, which use optimized custom routines.
+    This function delegates to joblib's hashing utilities. If NumPy is
+    imported in the current process, it uses NumpyHasher for efficient and
+    stable hashing of NumPy arrays; otherwise it uses the generic Hasher.
+
+    Args:
+        x (Any): The object to hash. Must be picklable by joblib unless a
+            specialized routine (e.g., for NumPy arrays) is available.
+
+    Returns:
+        str: A hexadecimal string digest computed with the configured
+            algorithm (sha256 by default).
+
+    Notes:
+        - joblib relies on pickle for most Python objects; ensure that custom
+          objects are picklable for stable results.
+        - The digest is deterministic for the same object content.
     """
     if 'numpy' in sys.modules:
-        hasher = joblib.hashing.NumpyHasher(hash_name=hash_type)
+        hasher = joblib.hashing.NumpyHasher(hash_name=_HASH_TYPE)
     else:
-        hasher = joblib.hashing.Hasher(hash_name=hash_type)
+        hasher = joblib.hashing.Hasher(hash_name=_HASH_TYPE)
     hash_signature = hasher.hash(x)
     return str(hash_signature)
 
 def get_base32_hash_signature(x:Any) -> str:
-    """Return base32 hash signature of an object"""
+    """Compute a base32-encoded hash for an arbitrary Python object.
+
+    Internally computes a hexadecimal digest first, then converts it to the
+    custom base32 alphabet used by Pythagoras.
+
+    Args:
+        x (Any): The object to hash.
+
+    Returns:
+        str: The full-length base32 digest string (not truncated).
+    """
     base_16_hash = get_base16_hash_signature(x)
     base_32_hash = convert_base16_to_base32(base_16_hash)
     return base_32_hash
 
 def get_hash_signature(x:Any) -> str:
-    return get_base32_hash_signature(x)[:max_signature_length]
+    """Compute a short, URL-safe hash signature for an object.
+
+    This is a convenience wrapper that returns the first max_signature_length
+    characters of the base32 digest, which is typically sufficient for
+    collision-resistant identifiers in logs and filenames.
+
+    Args:
+        x (Any): The object to hash.
+
+    Returns:
+        str: The truncated base32 digest string.
+    """
+    return get_base32_hash_signature(x)[:_MAX_SIGNATURE_LENGTH]
 

pythagoras/_800_signatures_and_converters/node_signature.py

@@ -1,4 +1,6 @@
-import uuid, platform, getpass
+import getpass
+import platform
+import uuid
 from functools import cache
 
 from .hash_signatures import get_hash_signature
@@ -6,15 +8,28 @@ from .hash_signatures import get_hash_signature
 
 @cache
 def get_node_signature() -> str:
-    """Returns a globally-unique signature for the current computing node.
+    """Return a stable signature for the current computing node and user.
+
+    The signature is derived from a concatenation of multiple system- and
+    user-specific attributes (MAC address, OS info, CPU, username) and then
+    hashed using Pythagoras' short base32 digest. The result is intended to
+    uniquely identify the node within logs and distributed systems.
+
+    Caching:
+        The result is cached for the lifetime of the process using
+        functools.cache, as the underlying attributes are not expected to
+        change while the process is running.
+
+    Returns:
+        str: A short base32 signature string representing this node.
     """
-    mac = uuid.getnode()
-    system = platform.system()
-    release = platform.release()
-    version = platform.version()
-    machine = platform.machine()
-    processor = platform.processor()
-    user = getpass.getuser()
-    id_string = f"{mac}{system}{release}{version}"
-    id_string += f"{machine}{processor}{user}"
-    return get_hash_signature(id_string)
+    id_parts = [
+        str(uuid.getnode()),
+        platform.system(),
+        platform.release(),
+        platform.version(),
+        platform.machine(),
+        platform.processor(),
+        getpass.getuser(),
+    ]
+    return get_hash_signature("".join(id_parts))

pythagoras/_800_signatures_and_converters/random_signatures.py

@@ -1,11 +1,22 @@
 import uuid
+from typing import Final
 
 from .base_16_32_convertors import convert_int_to_base32
 
-max_signature_length: int = 22
+_MAX_SIGNATURE_LENGTH: Final[int] = 22
 
 def get_random_signature() -> str:
-    # random_int = uuid.uuid4().int + uuid.uuid1().int
+    """Generate a short, random base32 signature string.
+
+    The randomness is sourced from uuid.uuid4(), which uses a cryptographically
+    strong RNG provided by the OS. The resulting large integer is encoded with
+    Pythagoras' base32 alphabet and truncated to max_signature_length
+    characters.
+
+    Returns:
+        str: A random, URL-safe base32 string of length up to
+        max_signature_length.
+    """
     random_int = uuid.uuid4().int
     random_str = convert_int_to_base32(random_int)
-    return random_str[:max_signature_length]
+    return random_str[:_MAX_SIGNATURE_LENGTH]