persidict 0.34.2__py3-none-any.whl → 0.34.3__py3-none-any.whl

persidict/s3_dict.py

@@ -48,27 +48,32 @@ class S3Dict(PersiDict):
                  , digest_len:int = 8
                  , base_class_for_values:Optional[type] = None
                  ,*args ,**kwargs):
-        """A constructor defines location of the store and object format to use.
-
-        bucket_name and region define an S3 location of the storage
-        that will contain all the objects in the S3_Dict.
-        If the bucket does not exist, it will be created.
-
-        root_prefix is a common S3 prefix for all objectnames in a dictionary.
-
-        _base_dir is a local directory that will be used to store tmp files.
-
-        base_class_for_values constraints the type of values that can be
-        stored in the dictionary. If specified, it will be used to
-        check types of values in the dictionary. If not specified,
-        no type checking will be performed and all types will be allowed.
-
-        file_type is an extension, which will be used for all files in the dictionary.
-        If file_type has one of two values: "lz4" or "json", it defines
-        which file format will be used by FileDirDict to store values.
-        For all other values of file_type, the file format will always be plain
-        text. "lz4" or "json" allow storing arbitrary Python objects,
-        while all other file_type-s only work with str objects.
+        """Initialize an S3-backed persistent dictionary.
+
+        Args:
+            bucket_name (str): Name of the S3 bucket to use. The bucket will be
+                created if it does not already exist.
+            region (str | None): AWS region of the bucket. If None, the default
+                client region is used.
+            root_prefix (str): Common S3 key prefix under which all objects are
+                stored. A trailing slash is added if missing.
+            base_dir (str): Local directory used for temporary files and a
+                small on-disk cache.
+            file_type (str): Extension/format for stored values. "pkl" or
+                "json" store arbitrary Python objects; other values imply plain
+                text and only allow str values.
+            immutable_items (bool): If True, disallow changing existing items.
+            digest_len (int): Number of base32 MD5 characters appended to key
+                elements to avoid case-insensitive collisions. Use 0 to disable.
+            base_class_for_values (type | None): Optional base class that all
+                values must inherit from. If provided and not str, file_type
+                must be "pkl" or "json".
+            *args: Ignored; reserved for compatibility.
+            **kwargs: Ignored; reserved for compatibility.
+
+        Raises:
+            ValueError: If file_type is "__etag__" (reserved) or configuration
+                is inconsistent with base_class_for_values.
         """
 
         super().__init__(immutable_items = immutable_items, digest_len = 0)
@@ -107,6 +112,11 @@ class S3Dict(PersiDict):
 
         This method is needed to support Parameterizable API.
         The method is absent in the original dict API.
+
+        Returns:
+            dict: A mapping of parameter names to their configured values,
+            including region, bucket_name, and root_prefix combined with
+            parameters from the local cache.
         """
         params = self.local_cache.get_params()
         params["region"] = self.region
@@ -118,9 +128,12 @@ class S3Dict(PersiDict):
 
     @property
     def base_url(self):
-        """Return dictionary's URl.
+        """Return the S3 URL prefix of this dictionary.
 
         This property is absent in the original dict API.
+
+        Returns:
+            str: The base S3 URL in the form "s3://<bucket>/<root_prefix>".
         """
         return f"s3://{self.bucket_name}/{self.root_prefix}"
 
@@ -130,12 +143,22 @@ class S3Dict(PersiDict):
         """Return dictionary's base directory in the local filesystem.
 
         This property is absent in the original dict API.
+
+        Returns:
+            str: Path to the local on-disk cache directory used by S3Dict.
         """
         return self.local_cache.base_dir
 
 
     def _build_full_objectname(self, key:PersiDictKey) -> str:
-        """ Convert PersiDictKey into an S3 objectname. """
+        """Convert a key into a full S3 object key (object name).
+
+        Args:
+            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+
+        Returns:
+            str: The full S3 key under root_prefix with file_type suffix applied.
+        """
         key = SafeStrTuple(key)
         key = sign_safe_str_tuple(key, self.digest_len)
         objectname = self.root_prefix +  "/".join(key)+ "." + self.file_type
@@ -143,7 +166,14 @@ class S3Dict(PersiDict):
 
 
     def __contains__(self, key:PersiDictKey) -> bool:
-        """True if the dictionary has the specified key, else False. """
+        """Return True if the specified key exists in S3.
+
+        Args:
+            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+
+        Returns:
+            bool: True if the object exists (or is cached when immutable), else False.
+        """
         key = SafeStrTuple(key)
         if self.immutable_items:
             file_name = self.local_cache._build_full_path(
@@ -159,7 +189,12 @@ class S3Dict(PersiDict):
 
 
     def _write_etag_file(self, file_name: str, etag: str):
-        """Atomically write the ETag to its cache file."""
+        """Atomically write the ETag to its cache file.
+
+        Args:
+            file_name (str): Path to the cached data file (without the ETag suffix).
+            etag (str): The S3 ETag value to persist alongside the cached file.
+        """
         if not etag:
             return
         etag_file_name = file_name + ".__etag__"
@@ -187,7 +222,18 @@ class S3Dict(PersiDict):
 
 
     def __getitem__(self, key:PersiDictKey) -> Any:
-        """X.__getitem__(y) is an equivalent to X[y]. """
+        """Retrieve the value stored for a key from S3 or local cache.
+
+        If immutable_items is True and a local cached file exists, that cache is
+        returned. Otherwise, the object is fetched from S3, with conditional
+        requests used when possible.
+
+        Args:
+            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+
+        Returns:
+            Any: The stored value.
+        """
 
         key = SafeStrTuple(key)
         file_name = self.local_cache._build_full_path(key, create_subdirs=True)
@@ -257,7 +303,23 @@ class S3Dict(PersiDict):
 
 
     def __setitem__(self, key:PersiDictKey, value:Any):
-        """Set self[key] to value. """
+        """Store a value for a key in S3 and update the local cache.
+
+        Interprets joker values KEEP_CURRENT and DELETE_CURRENT accordingly.
+        Validates a value type if base_class_for_values is set, then writes to the
+        local cache and uploads to S3. If possible, caches the S3 ETag locally to
+        enable conditional GETs later.
+
+        Args:
+            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+            value (Any): Value to store, or a joker command.
+
+        Raises:
+            KeyError: If attempting to modify an existing item when
+                immutable_items is True.
+            TypeError: If value is a PersiDict or does not match
+                base_class_for_values when it is set.
+        """
 
         if value is KEEP_CURRENT:
             return
@@ -302,7 +364,14 @@ class S3Dict(PersiDict):
 
 
     def __delitem__(self, key:PersiDictKey):
-        """Delete self[key]. """
+        """Delete the stored value for a key from S3 and local cache.
+
+        Args:
+            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+
+        Raises:
+            KeyError: If immutable_items is True.
+        """
 
         key = SafeStrTuple(key)
         if self.immutable_items:
@@ -322,6 +391,9 @@ class S3Dict(PersiDict):
         WARNING: This operation can be very slow and costly on large S3 buckets
         as it needs to iterate over all objects in the dictionary's prefix.
         Avoid using it in performance-sensitive code.
+
+        Returns:
+            int: Number of stored items under this dictionary's root_prefix.
         """
 
         num_files = 0
@@ -344,7 +416,24 @@ class S3Dict(PersiDict):
 
 
     def _generic_iter(self, result_type: str):
-        """Underlying implementation for .items()/.keys()/.values() iterators"""
+        """Underlying implementation for .items()/.keys()/.values() iterators.
+
+        Iterates over S3 objects under the configured root_prefix and yields
+        keys, values, and/or timestamps according to the requested result_type.
+        Keys are mapped to SafeStrTuple by removing the file extension and
+        unsigning based on digest_len.
+
+        Args:
+            result_type (set[str]): Any non-empty subset of {"keys", "values",
+                "timestamps"} specifying which fields to yield.
+
+        Returns:
+            Iterator: A generator yielding:
+                - SafeStrTuple if result_type == {"keys"}
+                - Any if result_type == {"values"}
+                - tuple[SafeStrTuple, Any] if result_type == {"keys", "values"}
+                - tuple[..., float] including POSIX timestamp if "timestamps" is requested.
+        """
 
         assert isinstance(result_type, set)
         assert 1 <= len(result_type) <= 3
@@ -356,11 +445,20 @@ class S3Dict(PersiDict):
         prefix_len = len(self.root_prefix)
 
         def splitter(full_name: str) -> SafeStrTuple:
+            """Convert an S3 object key into a SafeStrTuple without the suffix.
+
+            Args:
+                full_name (str): Full S3 object key (including root_prefix).
+
+            Returns:
+                SafeStrTuple: The parsed key parts, still signed.
+            """
             assert full_name.startswith(self.root_prefix)
             result = full_name[prefix_len:-ext_len].split(sep="/")
             return SafeStrTuple(result)
 
         def step():
+            """Generator that pages through S3 and yields entries based on result_type."""
             paginator = self.s3_client.get_paginator("list_objects_v2")
             page_iterator = paginator.paginate(
                 Bucket=self.bucket_name, Prefix = self.root_prefix)
@@ -400,9 +498,16 @@ class S3Dict(PersiDict):
     def get_subdict(self, key:PersiDictKey) -> S3Dict:
         """Get a subdictionary containing items with the same prefix key.
 
-        For non-existing prefix key, an empty sub-dictionary is returned.
-
+        For a non-existing prefix key, an empty sub-dictionary is returned.
         This method is absent in the original dict API.
+
+        Args:
+            key (PersiDictKey): A common prefix (string or sequence of strings)
+                used to scope items stored under this dictionary.
+
+        Returns:
+            S3Dict: A new S3Dict instance rooted at the given prefix, sharing
+            the same bucket, region, serialization, and immutability settings.
         """
 
         key = SafeStrTuple(key)
@@ -430,11 +535,18 @@ class S3Dict(PersiDict):
 
 
     def timestamp(self,key:PersiDictKey) -> float:
-        """Get last modification time (in seconds, Unix epoch time).
+        """Get last modification time (Unix epoch seconds) for a key.
 
         This method is absent in the original dict API.
+
+        Args:
+            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+
+        Returns:
+            float: POSIX timestamp (seconds since the Unix epoch) of the last
+            modification time as reported by S3 for the object.
         """
-        #TODO: check work with timezones
+        # TODO: check work with timezones
         key = SafeStrTuple(key)
         obj_name = self._build_full_objectname(key)
         response = self.s3_client.head_object(Bucket=self.bucket_name, Key=obj_name)

persidict/safe_chars.py

@@ -4,11 +4,30 @@ SAFE_CHARS_SET = set(string.ascii_letters + string.digits + "()_-~.=")
 SAFE_STRING_MAX_LENGTH = 254
 
 def get_safe_chars() -> set[str]:
-    """Return a set of allowed characters."""
+    """Get the set of allowed characters.
+
+    Returns:
+        set[str]: A copy of the set of characters considered safe for
+            building file names and URL components. Includes ASCII letters,
+            digits, and the characters ()_-~.= .
+    """
     return SAFE_CHARS_SET.copy()
 
-def replace_unsafe_chars(a_str:str, replace_with:str) -> str :
-    """ Replace unsafe (special) characters with allowed (safe) ones."""
+def replace_unsafe_chars(a_str: str, replace_with: str) -> str:
+    """Replace unsafe characters in a string.
+
+    Replaces any character not present in the safe-character set with a
+    replacement substring.
+
+    Args:
+        a_str (str): Input string that may contain unsafe characters.
+        replace_with (str): The substring to use for every unsafe character
+            encountered in a_str.
+
+    Returns:
+        str: The transformed string where all unsafe characters are replaced
+        by the provided replacement substring.
+    """
     safe_chars = get_safe_chars()
     result_list = [(c if c in safe_chars else replace_with) for c in a_str]
     result_str = "".join(result_list)

persidict/safe_str_tuple.py

@@ -1,4 +1,9 @@
-"""SafeStrTuple: an immutable flat tuple of non-emtpy URL/filename-safe strings.
+"""Utilities for strict, flat tuples of URL/filename-safe strings.
+
+This module defines SafeStrTuple, an immutable, hashable, flat tuple of non-empty
+strings restricted to a predefined safe character set and bounded length. It is
+useful for constructing keys and paths that must be portable and safe for URLs
+and filesystems.
 """
 from __future__ import annotations
 from collections.abc import Sequence, Mapping, Hashable
@@ -6,33 +11,69 @@ from typing import Any
 from .safe_chars import SAFE_CHARS_SET, SAFE_STRING_MAX_LENGTH
 
 
-def _is_sequence_not_mapping(obj:Any) -> bool:
-    """Check if obj is a sequence (e.g. list) but not a mapping (e.g. dict)."""
+def _is_sequence_not_mapping(obj: Any) -> bool:
+    """Return True if the object looks like a sequence but not a mapping.
+
+    This function prefers ABC checks but falls back to duck-typing to handle
+    some custom/typed collections.
+
+    Args:
+        obj: Object to inspect.
+
+    Returns:
+        bool: True if obj is a sequence (e.g., list, tuple) and not a mapping
+        (e.g., dict); otherwise False.
+    """
     if isinstance(obj, Sequence) and not isinstance(obj, Mapping):
         return True
     elif hasattr(obj, "keys") and callable(obj.keys):
         return False
-    elif (hasattr(obj, "__getitem__") and callable(obj.__getitem__)
-        and hasattr(obj, "__len__") and callable(obj.__len__)
-        and hasattr(obj, "__iter__") and callable(obj.__iter__)):
+    elif (
+        hasattr(obj, "__getitem__")
+        and callable(obj.__getitem__)
+        and hasattr(obj, "__len__")
+        and callable(obj.__len__)
+        and hasattr(obj, "__iter__")
+        and callable(obj.__iter__)
+    ):
         return True
     else:
         return False
 
+
 class SafeStrTuple(Sequence, Hashable):
-    """An immutable sequence of non-emtpy URL/filename-safe strings.
+    """An immutable sequence of non-empty URL/filename-safe strings.
+
+    The sequence is flat (no nested structures) and hashable, making it suitable
+    for use as a dictionary key. All strings are validated to contain only
+    characters from SAFE_CHARS_SET and to have length less than
+    SAFE_STRING_MAX_LENGTH.
     """
 
     strings: tuple[str, ...]
 
     def __init__(self, *args, **kwargs):
-        """Create a SafeStrTuple from a sequence/tree of strings.
-
-        The constructor accepts a sequence (list, tuple, etc.) of objects,
-        each of which can be a string or a nested sequence of
-        objects with similar structure. The input tree of strings is flattened.
-        Each string must be non-empty and contain
-        only URL/filename-safe characters.
+        """Initialize from strings or nested sequences of strings.
+
+        The constructor accepts one or more arguments which may be:
+        - a SafeStrTuple
+        - a single string
+        - a sequence (list/tuple/etc.) containing any of the above recursively
+
+        The input is flattened left-to-right into a single tuple of validated
+        strings. Empty strings and strings with characters outside
+        SAFE_CHARS_SET are rejected. Strings must also be shorter than
+        SAFE_STRING_MAX_LENGTH.
+
+        Args:
+            *args: One or more inputs (strings, sequences, or SafeStrTuple) that
+                will be flattened into a tuple of safe strings.
+            **kwargs: Not supported.
+
+        Raises:
+            AssertionError: If kwargs are provided; if no args are provided; if
+                any string is empty, too long, or contains disallowed chars; or
+                if an argument has an invalid type.
         """
         assert len(kwargs) == 0
         assert len(args) > 0
@@ -54,28 +95,60 @@ class SafeStrTuple(Sequence, Hashable):
 
     @property
     def str_chain(self) -> tuple[str, ...]:
-        """for backward compatibility"""
+        """Alias for strings for backward compatibility.
+
+        Returns:
+            tuple[str, ...]: The underlying tuple of strings.
+        """
         return self.strings
 
-    def __getitem__(self, key:int)-> str:
-        """Return a string at position key."""
+    def __getitem__(self, key: int) -> str:
+        """Return the string at the given index.
+
+        Args:
+            key: Zero-based index.
+
+        Returns:
+            str: The string at the specified position.
+        """
         return self.strings[key]
 
     def __len__(self) -> int:
-        """Return the number of strings in the tuple."""
+        """Return the number of strings in the tuple.
+
+        Returns:
+            int: The number of elements.
+        """
         return len(self.strings)
 
     def __hash__(self):
-        """Return a hash of the tuple."""
+        """Compute the hash of the underlying tuple.
+
+        Returns:
+            int: A hash value suitable for dict/set usage.
+        """
         return hash(self.strings)
 
     def __repr__(self) -> str:
-        """Return repr(self)."""
-        return f"{type(self).__name__}({self.strings})"
+        """Return a developer-friendly representation.
 
+        Returns:
+            str: A representation including the class name and contents.
+        """
+        return f"{type(self).__name__}({self.strings})"
 
     def __eq__(self, other) -> bool:
-        """Return self == other."""
+        """Compare two SafeStrTuple-compatible objects for equality.
+
+        If other is not a SafeStrTuple, it will be coerced using the same
+        validation rules.
+
+        Args:
+            other: Another SafeStrTuple or compatible input.
+
+        Returns:
+            bool: True if both contain the same sequence of strings.
+        """
         if isinstance(other, SafeStrTuple):
             if type(self).__eq__ != type(other).__eq__:
                 return other.__eq__(self)
@@ -84,25 +157,53 @@ class SafeStrTuple(Sequence, Hashable):
 
         return self.strings == other.strings
 
-
     def __add__(self, other) -> SafeStrTuple:
-        """Return self + other."""
+        """Concatenate with another SafeStrTuple-compatible object.
+
+        Args:
+            other: Another SafeStrTuple or compatible input.
+
+        Returns:
+            SafeStrTuple: A new instance containing elements of self then other.
+        """
         other = SafeStrTuple(other)
         return SafeStrTuple(*(self.strings + other.strings))
 
     def __radd__(self, other) -> SafeStrTuple:
-        """Return other + self."""
+        """Concatenate with another object in reversed order (other + self).
+
+        Args:
+            other: Another SafeStrTuple or compatible input.
+
+        Returns:
+            SafeStrTuple: A new instance containing elements of other then self.
+        """
         other = SafeStrTuple(other)
         return SafeStrTuple(*(other.strings + self.strings))
 
     def __iter__(self):
-        """Return iter(self)."""
+        """Return an iterator over the strings.
+
+        Returns:
+            Iterator[str]: An iterator over the internal tuple.
+        """
         return iter(self.strings)
 
     def __contains__(self, item) -> bool:
-        """Return item in self."""
+        """Check membership.
+
+        Args:
+            item: String to check for presence.
+
+        Returns:
+            bool: True if item is present.
+        """
         return item in self.strings
 
     def __reversed__(self) -> SafeStrTuple:
-        """Return a reversed SafeStrTuple."""
+        """Return a reversed SafeStrTuple.
+
+        Returns:
+            SafeStrTuple: A new instance with elements in reverse order.
+        """
         return SafeStrTuple(*reversed(self.strings))