persidict 0.34.2__py3-none-any.whl → 0.35.0__py3-none-any.whl

persidict/file_dir_dict.py

@@ -1,11 +1,11 @@
-""" Persistent dictionaries that store key-value pairs on local disks.
-
-This functionality is implemented by the class FileDirDict
-(inherited from PersiDict): a dictionary that
-stores key-value pairs as files on a local hard-drive.
-A key is used to compose a filename, while a value is stored in the file
-as a binary, or as a json object, or as a plain text
-(depends on configuration parameters).
+"""Persistent dictionary implementation backed by local files.
+
+FileDirDict stores each key-value pair in a separate file under a base
+directory. Keys determine directory structure and filename; values are
+serialized depending on ``file_type``.
+
+- file_type="pkl" or "json": arbitrary Python objects via pickle/jsonpickle.
+- any other value: strings are stored as plain text.
 """
 from __future__ import annotations
 
@@ -56,29 +56,35 @@ class FileDirDict(PersiDict):
                  , immutable_items:bool = False
                  , digest_len:int = 8
                  , base_class_for_values: Optional[type] = None):
-        """A constructor defines location of the store and file format to use.
-
-        _base_dir is a directory that will contain all the files in
-        the FileDirDict. If the directory does not exist, it will be created.
-
-        base_class_for_values constraints the type of values that can be
-        stored in the dictionary. If specified, it will be used to
-        check types of values in the dictionary. If not specified,
-        no type checking will be performed and all types will be allowed.
-
-        file_type is extension, which will be used for all files in the dictionary.
-        If file_type has one of two values: "pkl" or "json", it defines
-        which file format will be used by FileDirDict to store values.
-        For all other values of file_type, the file format will always be plain
-        text. "pkl" and "json" allow to store arbitrary Python objects,
-        while all other file_type-s only work with str objects.
+        """Initialize a filesystem-backed persistent dictionary.
+
+        Args:
+            base_dir (str): Base directory where all files are stored. Created
+                if it does not exist.
+            file_type (str): File extension/format to use for stored values.
+                - "pkl" or "json": arbitrary Python objects are supported.
+                - any other value: only strings are supported and stored as text.
+            immutable_items (bool): If True, existing items cannot be modified
+                or deleted.
+            digest_len (int): Length of a hash suffix appended to each key path
+                element to avoid case-insensitive collisions. Use 0 to disable.
+            base_class_for_values (Optional[type]): Optional base class that all
+                stored values must be instances of. If provided and not ``str``,
+                then file_type must be either "pkl" or "json".
+
+        Raises:
+            ValueError: If base_dir points to a file; if file_type is "__etag__";
+                if file_type contains unsafe characters; or if configuration is
+                inconsistent (e.g., non-str values with unsupported file_type).
+            RuntimeError: If base_dir cannot be created or is not a directory.
         """
 
         super().__init__(immutable_items = immutable_items
                 ,digest_len = digest_len
                 ,base_class_for_values = base_class_for_values)
 
-        assert file_type == replace_unsafe_chars(file_type, "")
+        if file_type != replace_unsafe_chars(file_type, ""):
+            raise ValueError("file_type contains unsafe characters")
         self.file_type = file_type
         if self.file_type == "__etag__":
             raise ValueError(
@@ -87,8 +93,8 @@ class FileDirDict(PersiDict):
 
         if (base_class_for_values is None or
                 not issubclass(base_class_for_values,str)):
-            assert file_type in {"json", "pkl"}, ("For non-string values"
-                + " file_type must be either 'pkl' or 'json'.")
+            if file_type not in {"json", "pkl"}:
+                raise ValueError("For non-string values file_type must be either 'pkl' or 'json'.")
 
         base_dir = str(base_dir)
 
@@ -96,7 +102,8 @@ class FileDirDict(PersiDict):
             raise ValueError(f"{base_dir} is a file, not a directory.")
 
         os.makedirs(base_dir, exist_ok=True)
-        assert os.path.isdir(base_dir)
+        if not os.path.isdir(base_dir):
+            raise RuntimeError(f"Failed to create or access directory: {base_dir}")
 
         # self.base_dir_param = _base_dir
         self._base_dir = os.path.abspath(base_dir)
@@ -105,8 +112,12 @@ class FileDirDict(PersiDict):
     def get_params(self):
         """Return configuration parameters of the dictionary.
 
-        This method is needed to support Parameterizable API.
-        The method is absent in the original dict API.
+        This method is needed to support the Parameterizable API and is absent
+        in the standard dict API.
+
+        Returns:
+            dict: A mapping of parameter names to values including base_dir and
+                file_type merged with the base PersiDict parameters.
         """
         params = PersiDict.get_params(self)
         additional_params = dict(
@@ -122,6 +133,9 @@ class FileDirDict(PersiDict):
         """Return dictionary's URL.
 
         This property is absent in the original dict API.
+
+        Returns:
+            str: URL of the underlying storage in the form "file://<abs_path>".
         """
         return f"file://{self._base_dir}"
 
@@ -131,16 +145,25 @@ class FileDirDict(PersiDict):
         """Return dictionary's base directory.
 
         This property is absent in the original dict API.
+
+        Returns:
+            str: Absolute path to the base directory used by this dictionary.
         """
         return self._base_dir
 
 
     def __len__(self) -> int:
-        """ Get the number of key-value pairs in the dictionary.
+        """Return the number of key-value pairs in the dictionary.
+
+        This performs a recursive traversal of the base directory.
 
-        WARNING: This operation can be slow on large dictionaries as it
-        needs to recursively walk the entire base directory.
-        Avoid using it in performance-sensitive code.
+        Returns:
+            int: Count of stored items.
+
+        Note:
+            This operation can be slow on large dictionaries as it walks the
+            entire directory tree. Avoid using it in performance-sensitive
+            code paths.
         """
 
         suffix = "." + self.file_type
@@ -149,7 +172,11 @@ class FileDirDict(PersiDict):
 
 
     def clear(self) -> None:
-        """ Remove all elements from the dictionary."""
+        """Remove all elements from the dictionary.
+
+        Raises:
+            KeyError: If immutable_items is True.
+        """
 
         if self.immutable_items:
             raise KeyError("Can't clear a dict that contains immutable items")
@@ -172,7 +199,26 @@ class FileDirDict(PersiDict):
                          , key:SafeStrTuple
                          , create_subdirs:bool=False
                          , is_file_path:bool=True) -> str:
-        """Convert a key into a filesystem path."""
+        """Convert a key into an absolute filesystem path.
+
+        Transforms a SafeStrTuple into either a directory path or a file path
+        inside this dictionary's base directory. When is_file_path is True, the
+        final component is treated as a filename with the configured file_type
+        extension. When create_subdirs is True, missing intermediate directories
+        are created.
+
+        Args:
+            key (SafeStrTuple): The key to convert. It will be temporarily
+                signed according to digest_len to produce collision-safe names.
+            create_subdirs (bool): If True, create any missing intermediate
+                directories.
+            is_file_path (bool): If True, return a file path ending with
+                ".{file_type}"; otherwise return just the directory path for
+                the key prefix.
+
+        Returns:
+            str: An absolute path within base_dir corresponding to the key.
+        """
 
         key = sign_safe_str_tuple(key, self.digest_len)
         key = [self._base_dir] + list(key.strings)
@@ -190,7 +236,22 @@ class FileDirDict(PersiDict):
 
 
     def _build_key_from_full_path(self, full_path:str)->SafeStrTuple:
-        """Convert a filesystem path back into a key."""
+        """Convert an absolute filesystem path back into a SafeStrTuple key.
+
+        This function reverses _build_full_path, stripping base_dir, removing the
+        file_type extension if the path points to a file, and unsigning the key
+        components according to digest_len.
+
+        Args:
+            full_path (str): Absolute path within the dictionary's base
+                directory.
+
+        Returns:
+            SafeStrTuple: The reconstructed (unsigned) key.
+
+        Raises:
+            ValueError: If full_path is not located under base_dir.
+        """
 
         # Ensure we're working with absolute paths
         full_path = os.path.abspath(full_path)
@@ -225,8 +286,15 @@ class FileDirDict(PersiDict):
         """Get a subdictionary containing items with the same prefix key.
 
         For non-existing prefix key, an empty sub-dictionary is returned.
-
         This method is absent in the original dict API.
+
+        Args:
+            key (PersiDictKey): Prefix key (string or sequence of strings) that
+                identifies the subdirectory.
+
+        Returns:
+            FileDirDict: A new FileDirDict instance rooted at the specified
+                subdirectory, sharing the same parameters as this dictionary.
         """
         key = SafeStrTuple(key)
         full_dir_path = self._build_full_path(
@@ -240,7 +308,14 @@ class FileDirDict(PersiDict):
 
 
     def _read_from_file_impl(self, file_name:str) -> Any:
-        """Read a value from a file. """
+        """Read a value from a single file without retries.
+
+        Args:
+            file_name (str): Absolute path to the file to read.
+
+        Returns:
+            Any: The deserialized value according to file_type.
+        """
 
         if self.file_type == "pkl":
             with open(file_name, 'rb') as f:
@@ -255,7 +330,22 @@ class FileDirDict(PersiDict):
 
 
     def _read_from_file(self,file_name:str) -> Any:
-        """Read a value from a file. """
+        """Read a value from a file with retry/backoff for concurrency.
+
+        Validates that the configured file_type is compatible with the allowed
+        value types, then attempts to read the file using an exponential backoff
+        to better tolerate concurrent writers.
+
+        Args:
+            file_name (str): Absolute path of the file to read.
+
+        Returns:
+            Any: The deserialized value according to file_type.
+
+        Raises:
+            ValueError: If file_type is incompatible with non-string values.
+            Exception: Propagates the last exception if all retries fail.
+        """
 
         if not (self.file_type in {"pkl", "json"} or issubclass(
             self.base_class_for_values, str)):
@@ -275,7 +365,15 @@ class FileDirDict(PersiDict):
 
 
     def _save_to_file_impl(self, file_name:str, value:Any) -> None:
-        """Save a value to a file. """
+        """Write a single value to a file atomically (no retries).
+
+        Uses a temporary file and atomic rename to avoid partial writes and to
+        reduce the chance of readers observing corrupted data.
+
+        Args:
+            file_name (str): Absolute destination file path.
+            value (Any): Value to serialize and save.
+        """
 
         dir_name = os.path.dirname(file_name)
         # Use a temporary file and atomic rename to prevent data corruption
@@ -313,7 +411,20 @@ class FileDirDict(PersiDict):
             raise
 
     def _save_to_file(self, file_name:str, value:Any) -> None:
-        """Save a value to a file. """
+        """Save a value to a file with retry/backoff.
+
+        Ensures the configured file_type is compatible with value types and then
+        writes the value using an exponential backoff to better tolerate
+        concurrent readers/writers.
+
+        Args:
+            file_name (str): Absolute destination file path.
+            value (Any): Value to serialize and save.
+
+        Raises:
+            ValueError: If file_type is incompatible with non-string values.
+            Exception: Propagates the last exception if all retries fail.
+        """
 
         if not (self.file_type in {"pkl", "json"} or issubclass(
             self.base_class_for_values, str)):
@@ -334,14 +445,36 @@ class FileDirDict(PersiDict):
 
 
     def __contains__(self, key:PersiDictKey) -> bool:
-        """True if the dictionary has the specified key, else False. """
+        """Check whether a key exists in the dictionary.
+
+        Args:
+            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+
+        Returns:
+            bool: True if a file for the key exists; False otherwise.
+        """
         key = SafeStrTuple(key)
         filename = self._build_full_path(key)
         return os.path.isfile(filename)
 
 
     def __getitem__(self, key:PersiDictKey) -> Any:
-        """ Implementation for x[y] syntax. """
+        """Retrieve the value stored for a key.
+
+        Equivalent to obj[key]. Reads the corresponding file from the disk and
+        deserializes according to file_type.
+
+        Args:
+            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+
+        Returns:
+            Any: The stored value.
+
+        Raises:
+            KeyError: If the file for the key does not exist.
+            TypeError: If the deserialized value does not match base_class_for_values
+                when it is set.
+        """
         key = SafeStrTuple(key)
         filename = self._build_full_path(key)
         if not os.path.isfile(filename):
@@ -356,7 +489,22 @@ class FileDirDict(PersiDict):
 
 
     def __setitem__(self, key:PersiDictKey, value:Any):
-        """Set self[key] to value."""
+        """Store a value for a key on the disk.
+
+        Interprets joker values KEEP_CURRENT and DELETE_CURRENT accordingly.
+        Validates value type if base_class_for_values is set, then serializes
+        and writes to a file determined by the key and file_type.
+
+        Args:
+            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+            value (Any): Value to store, or a joker command.
+
+        Raises:
+            KeyError: If attempting to modify an existing item when
+                immutable_items is True.
+            TypeError: If the value is a PersiDict or does not match
+                base_class_for_values when it is set.
+        """
 
         if value is KEEP_CURRENT:
             return
@@ -384,9 +532,17 @@ class FileDirDict(PersiDict):
 
 
     def __delitem__(self, key:PersiDictKey) -> None:
-        """Delete self[key]."""
+        """Delete the stored value for a key.
+
+        Args:
+            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+
+        Raises:
+            KeyError: If immutable_items is True or if the key does not exist.
+        """
         key = SafeStrTuple(key)
-        assert not self.immutable_items, "Can't delete immutable items"
+        if self.immutable_items:
+            raise KeyError("Can't delete immutable items")
         filename = self._build_full_path(key)
         if not os.path.isfile(filename):
             raise KeyError(f"File {filename} does not exist")
@@ -394,22 +550,54 @@ class FileDirDict(PersiDict):
 
 
     def _generic_iter(self, result_type: set[str]):
-        """Underlying implementation for .items()/.keys()/.values() iterators"""
-        assert isinstance(result_type, set)
-        assert 1 <= len(result_type) <= 3
-        assert len(result_type | {"keys", "values", "timestamps"}) == 3
-        assert 1 <= len(result_type & {"keys", "values", "timestamps"}) <= 3
+        """Underlying implementation for .items()/.keys()/.values() iterators.
+
+        Produces generators over keys, values, and/or timestamps by traversing
+        the directory tree under base_dir. Keys are converted back from paths by
+        removing the file extension and unsigning according to digest_len.
+
+        Args:
+            result_type (set[str]): Any non-empty subset of {"keys", "values",
+                "timestamps"} specifying which fields to yield.
+
+        Returns:
+            Iterator: A generator yielding:
+                - SafeStrTuple if result_type == {"keys"}
+                - Any if result_type == {"values"}
+                - tuple[SafeStrTuple, Any] if result_type == {"keys", "values"}
+                - tuple[..., float] including POSIX timestamp if "timestamps" is requested.
+
+        Raises:
+            TypeError: If result_type is not a set.
+            ValueError: If result_type is empty or contains unsupported labels.
+        """
+        if not isinstance(result_type, set):
+            raise TypeError("result_type must be a set")
+        if not (1 <= len(result_type) <= 3):
+            raise ValueError("result_type must be a non-empty subset of {'keys','values','timestamps'}")
+        allowed = {"keys", "values", "timestamps"}
+        invalid = result_type - allowed
+        if invalid:
+            raise ValueError(f"Unsupported result_type entries: {sorted(invalid)}; allowed: {sorted(allowed)}")
 
         walk_results = os.walk(self._base_dir)
         ext_len = len(self.file_type) + 1
 
         def splitter(dir_path: str):
-            """Transform a dirname into a PersiDictKey key"""
+            """Transform a relative dirname into SafeStrTuple components.
+
+            Args:
+                dir_path (str): Relative path under base_dir (e.g., "a/b").
+
+            Returns:
+                list[str]: List of safe string components (may be empty).
+            """
             if dir_path == ".":
                 return []
             return dir_path.split(os.sep)
 
         def step():
+            """Generator that yields entries based on result_type."""
             suffix = "." + self.file_type
             for dir_name, _, files in walk_results:
                 for f in files:
@@ -448,6 +636,15 @@ class FileDirDict(PersiDict):
         """Get last modification time (in seconds, Unix epoch time).
 
         This method is absent in the original dict API.
+
+        Args:
+            key (PersiDictKey): Key whose timestamp to return.
+
+        Returns:
+            float: POSIX timestamp of the underlying file.
+
+        Raises:
+            FileNotFoundError: If the key does not exist.
         """
         key = SafeStrTuple(key)
         filename = self._build_full_path(key)
@@ -455,6 +652,15 @@ class FileDirDict(PersiDict):
 
 
     def random_key(self) -> PersiDictKey | None:
+        """Return a uniformly random key from the dictionary, or None if empty.
+
+        Performs a full directory traversal using reservoir sampling
+        (k=1) to select a random file matching the configured file_type without
+        loading all keys into memory.
+
+        Returns:
+            PersiDictKey | None: A random key if any items exist; otherwise None.
+        """
         # canonicalise extension once
         ext = None
         if self.file_type:

persidict/jokers.py

@@ -1,8 +1,19 @@
-"""A singleton constant to indicate no change in a value.
+"""Special singleton markers used to modify values in PersiDict without data payload.
 
-When updating a value in a persistent dictionary,
-use KEEP_CURRENT as the new value to indicate that
-the existing value should remain unchanged.
+This module defines two singleton flags used as "joker" values when writing to
+persistent dictionaries:
+
+- KEEP_CURRENT: keep the current value unchanged.
+- DELETE_CURRENT: delete the current value if it exists.
+
+These flags are intended to be passed as the value part in dict-style
+assignments (e.g., d[key] = KEEP_CURRENT) and are interpreted by PersiDict
+implementations.
+
+Examples:
+    >>> from persidict.jokers import KEEP_CURRENT, DELETE_CURRENT
+    >>> d[key] = KEEP_CURRENT  # Do not alter existing value
+    >>> d[key] = DELETE_CURRENT  # Remove key if present
 """
 from typing import Any
 
@@ -12,32 +23,61 @@ from parameterizable import (
 
 
 class Joker(ParameterizableClass):
+    """Base class for singleton joker flags.
+
+    Implements a per-subclass singleton pattern and integrates with the
+    parameterizable framework. Subclasses represent value-less commands that
+    alter persistence behavior when assigned to a key.
+
+    Returns:
+        Joker: The singleton instance for the subclass when instantiated.
+    """
     _instances = {}
 
     def get_params(self) -> dict[str, Any]:
+        """Return parameters for parameterizable API.
+
+        Returns:
+            dict[str, Any]: Always an empty dict for joker flags.
+        """
         return {}
 
     def __new__(cls):
+        """Create or return the singleton instance for the subclass."""
         if cls not in Joker._instances:
             Joker._instances[cls] = super().__new__(cls)
         return Joker._instances[cls]
 
 
 class KeepCurrentFlag(Joker):
-    """A singleton constant to indicate no change in a value.
+    """Flag instructing PersiDict to keep the current value unchanged.
+
+    Usage:
+        Assign this flag instead of a real value to indicate that an existing
+        value should not be modified.
+
+    Examples:
+        >>> d[key] = KEEP_CURRENT
 
-    When updating a value in a persistent dictionary,
-    use KeepCurrent as the new value to indicate that
-    the existing value (if any) should remain unchanged.
+    Note:
+        This is a singleton class; constructing it repeatedly returns the same
+        instance.
     """
     pass
 
 class DeleteCurrentFlag(Joker):
-    """A singleton constant to indicate that the current value should be deleted.
+    """Flag instructing PersiDict to delete the current value for a key.
+
+    Usage:
+        Assign this flag instead of a real value to remove the key if it
+        exists. If the key is absent, implementations will typically no-op.
+
+    Examples:
+        >>> d[key] = DELETE_CURRENT
 
-    When updating a value in a persistent dictionary,
-    use DeleteCurrentFlag as the new value to indicate that
-    the existing value (if any) should be removed from the dictionary.
+    Note:
+        This is a singleton class; constructing it repeatedly returns the same
+        instance.
     """
     pass