persidict 0.38.0__py3-none-any.whl → 0.103.0__py3-none-any.whl

persidict/file_dir_dict.py

@@ -2,15 +2,14 @@
 
 FileDirDict stores each key-value pair in a separate file under a base
 directory. Keys determine directory structure and filename; values are
-serialized depending on ``file_type``.
+serialized depending on ``serialization_format``.
 
-- file_type="pkl" or "json": arbitrary Python objects via pickle/jsonpickle.
+- serialization_format="pkl" or "json": arbitrary Python objects via pickle/jsonpickle.
 - any other value: strings are stored as plain text.
 """
 from __future__ import annotations
 
 import os
-import pathlib
 import random
 import tempfile
 import time
@@ -23,12 +22,10 @@ import jsonpickle.ext.pandas as jsonpickle_pandas
 import parameterizable
 from parameterizable import sort_dict_by_keys
 
-from .jokers import Joker
-from .safe_chars import replace_unsafe_chars
-from .safe_str_tuple import SafeStrTuple
+from .singletons import Joker, EXECUTION_IS_COMPLETE
+from .safe_str_tuple import SafeStrTuple, NonEmptySafeStrTuple
 from .safe_str_tuple_signing import sign_safe_str_tuple, unsign_safe_str_tuple
-from .persi_dict import PersiDict, PersiDictKey, non_empty_persidict_key
-
+from .persi_dict import PersiDict, PersiDictKey, NonEmptyPersiDictKey
 
 if os.name == 'nt':
     import msvcrt
@@ -52,30 +49,42 @@ if os.name == 'nt':
 
     def add_long_path_prefix(path: str) -> str:
         """Add the '\\\\?\\' prefix to a path on Windows to support long paths.
+        
+        Handles both regular paths and UNC paths correctly.
 
         Args:
             path (str): The original file or directory path.
 
         Returns:
-            str: The modified path with the '\\\\?\\' prefix if on Windows
+            str: The modified path with the appropriate prefix if on Windows
                 and not already present; otherwise, the original path.
+                UNC paths get '\\\\?\\UNC\\' prefix, regular paths get '\\\\?\\'.
         """
-        if  not path.startswith('\\\\?\\'):
-            return f'\\\\?\\{path}'
-        else:
+        if path.startswith('\\\\?\\'):
             return path
+        elif path.startswith('\\\\'):
+            # UNC path: \\server\share -> \\?\UNC\server\share
+            return f'\\\\?\\UNC\\{path[2:]}'
+        else:
+            return f'\\\\?\\{path}'
 
     def drop_long_path_prefix(path: str) -> str:
         """Remove the '\\\\?\\' prefix from a path on Windows if present.
+        
+        Handles both regular paths and UNC paths correctly.
 
         Args:
             path (str): The file or directory path, possibly with the '\\\\?\\' prefix.
 
         Returns:
             str: The path without the '\\\\?\\' prefix if it was present; otherwise,
-                the original path.
+                the original path. UNC paths are converted back from '\\\\?\\UNC\\' 
+                format to '\\\\' format.
         """
-        if path.startswith('\\\\?\\'):
+        if path.startswith('\\\\?\\UNC\\'):
+            # UNC path: \\?\UNC\server\share -> \\server\share
+            return f'\\\\{path[8:]}'
+        elif path.startswith('\\\\?\\'):
             return path[4:]
         else:
             return path
@@ -106,50 +115,45 @@ class FileDirDict(PersiDict):
     """
 
     _base_dir:str
-    file_type:str
+    digest_len:int
 
     def __init__(self
                  , base_dir: str = FILEDIRDICT_DEFAULT_BASE_DIR
-                 , file_type: str = "pkl"
-                 , immutable_items:bool = False
-                 , digest_len:int = 8
+                 , serialization_format: str = "pkl"
+                 , append_only:bool = False
+                 , digest_len:int = 1
                  , base_class_for_values: Optional[type] = None):
         """Initialize a filesystem-backed persistent dictionary.
 
         Args:
             base_dir (str): Base directory where all files are stored. Created
                 if it does not exist.
-            file_type (str): File extension/format to use for stored values.
+            serialization_format (str): File extension/format to use for stored values.
                 - "pkl" or "json": arbitrary Python objects are supported.
                 - any other value: only strings are supported and stored as text.
-            immutable_items (bool): If True, existing items cannot be modified
+            append_only (bool): If True, existing items cannot be modified
                 or deleted.
             digest_len (int): Length of a hash suffix appended to each key path
                 element to avoid case-insensitive collisions. Use 0 to disable.
                 If you decide to enable it (not 0), we recommend at least 4.
             base_class_for_values (Optional[type]): Optional base class that all
                 stored values must be instances of. If provided and not ``str``,
-                then file_type must be either "pkl" or "json".
+                then serialization_format must be either "pkl" or "json".
 
         Raises:
-            ValueError: If file_type contains unsafe characters; or
+            ValueError: If serialization_format contains unsafe characters; or
                 if configuration is inconsistent (e.g., non-str values
-                with unsupported file_type).
+                with unsupported serialization_format).
             RuntimeError: If base_dir cannot be created or is not a directory.
         """
 
-        super().__init__(immutable_items = immutable_items
-                ,digest_len = digest_len
-                ,base_class_for_values = base_class_for_values)
-
-        if file_type != replace_unsafe_chars(file_type, ""):
-            raise ValueError("file_type contains unsafe characters")
-        self.file_type = file_type
+        super().__init__(append_only=append_only,
+                         base_class_for_values=base_class_for_values,
+                         serialization_format=serialization_format)
 
-        if (base_class_for_values is None or
-                not issubclass(base_class_for_values,str)):
-            if file_type not in {"json", "pkl"}:
-                raise ValueError("For non-string values file_type must be either 'pkl' or 'json'.")
+        if digest_len < 0:
+            raise ValueError("digest_len must be non-negative")
+        self.digest_len = digest_len
 
         base_dir = str(base_dir)
         self._base_dir = os.path.abspath(base_dir)
@@ -170,32 +174,20 @@ class FileDirDict(PersiDict):
         in the standard dict API.
 
         Returns:
-            dict: A mapping of parameter names to values including base_dir and
-                file_type merged with the base PersiDict parameters.
+            dict: A mapping of parameter names to values including base_dir
+                merged with the base PersiDict parameters.
         """
         params = PersiDict.get_params(self)
         additional_params = dict(
-            base_dir=self.base_dir
-            , file_type=self.file_type)
+            base_dir=self.base_dir,
+            digest_len=self.digest_len)
         params.update(additional_params)
         sorted_params = sort_dict_by_keys(params)
         return sorted_params
 
 
     @property
-    def base_url(self) -> str:
-        """Return dictionary's URL.
-
-        This property is absent in the original dict API.
-
-        Returns:
-            str: URL of the underlying storage in the form "file://<abs_path>".
-        """
-        return pathlib.Path(self._base_dir).resolve().as_uri()
-
-
-    @property
-    def base_dir(self) -> str:
+    def base_dir(self) -> str|None:
         """Return dictionary's base directory.
 
         This property is absent in the original dict API.
@@ -220,7 +212,7 @@ class FileDirDict(PersiDict):
             code paths.
         """
 
-        suffix = "." + self.file_type
+        suffix = "." + self.serialization_format
         return sum(1 for _, _, files in os.walk(self._base_dir)
                    for f in files if f.endswith(suffix))
 
@@ -229,21 +221,24 @@ class FileDirDict(PersiDict):
         """Remove all elements from the dictionary.
 
         Raises:
-            KeyError: If immutable_items is True.
+            KeyError: If append_only is True.
         """
 
-        if self.immutable_items:
+        if self.append_only:
             raise KeyError("Can't clear a dict that contains immutable items")
 
         # we can't use shutil.rmtree() because
         # there may be overlapping dictionaries
-        # with different file_type-s
+        # with different serialization_format-s
         for subdir_info in os.walk(self._base_dir, topdown=False):
             (subdir_name, _, files) = subdir_info
-            suffix = "." + self.file_type
+            suffix = "." + self.serialization_format
             for f in files:
                 if f.endswith(suffix):
-                    os.remove(os.path.join(subdir_name, f))
+                    try:
+                        os.remove(os.path.join(subdir_name, f))
+                    except OSError:
+                        continue
             if (subdir_name != self._base_dir) and (
                     len(os.listdir(subdir_name)) == 0 ):
                 try:
@@ -262,7 +257,7 @@ class FileDirDict(PersiDict):
 
         Transforms a SafeStrTuple into either a directory path or a file path
         inside this dictionary's base directory. When is_file_path is True, the
-        final component is treated as a filename with the configured file_type
+        final component is treated as a filename with the configured serialization_format
         extension. When create_subdirs is True, missing intermediate directories
         are created.
 
@@ -272,7 +267,7 @@ class FileDirDict(PersiDict):
             create_subdirs (bool): If True, create any missing intermediate
                 directories.
             is_file_path (bool): If True, return a file path ending with
-                ".{file_type}"; otherwise return just the directory path for
+                ".{serialization_format}"; otherwise return just the directory path for
                 the key prefix.
 
         Returns:
@@ -293,7 +288,7 @@ class FileDirDict(PersiDict):
             os.makedirs(path_for_makedirs, exist_ok=True)
 
         if is_file_path:
-            file_name = key_components[-1] + "." + self.file_type
+            file_name = key_components[-1] + "." + self.serialization_format
             final_path = os.path.join(dir_path, file_name)
         else:
             final_path = dir_path
@@ -305,7 +300,7 @@ class FileDirDict(PersiDict):
             """Convert an absolute filesystem path back into a SafeStrTuple key.
 
             This function reverses _build_full_path, stripping base_dir, removing the
-            file_type extension if the path points to a file, and unsigning the key
+            serialization_format extension if the path points to a file, and unsigning the key
             components according to digest_len.
 
             Args:
@@ -321,7 +316,8 @@ class FileDirDict(PersiDict):
 
             # Remove the base directory from the path
             if not full_path.startswith(self._base_dir):
-                raise ValueError(f"Path {full_path} is not within base directory {self._base_dir}")
+                raise ValueError(f"Path {full_path} is not "
+                                 f"within base directory {self._base_dir}")
 
             # Get the relative path
             rel_path = os.path.relpath(
@@ -336,7 +332,7 @@ class FileDirDict(PersiDict):
             path_components = rel_path.split(os.sep)
 
             # If it's a file path, remove the file extension from the last component
-            suffix = "." + self.file_type
+            suffix = "." + self.serialization_format
             if path_components[-1].endswith(suffix):
                 path_components[-1] = path_components[-1][:-len(suffix)]
 
@@ -353,6 +349,7 @@ class FileDirDict(PersiDict):
         """Get a subdictionary containing items with the same prefix key.
 
         For non-existing prefix key, an empty sub-dictionary is returned.
+        If the prefix is empty, the entire dictionary is returned.
         This method is absent in the original dict API.
 
         Args:
@@ -370,8 +367,8 @@ class FileDirDict(PersiDict):
             is_file_path = False)
         return FileDirDict(
             base_dir= full_dir_path
-            , file_type=self.file_type
-            , immutable_items= self.immutable_items
+            , serialization_format=self.serialization_format
+            , append_only= self.append_only
             , digest_len=self.digest_len
             , base_class_for_values=self.base_class_for_values)
 
@@ -383,9 +380,9 @@ class FileDirDict(PersiDict):
             file_name (str): Absolute path to the file to read.
 
         Returns:
-            Any: The deserialized value according to file_type.
+            Any: The deserialized value according to serialization_format.
         """
-        file_open_mode = 'rb' if self.file_type == "pkl" else 'r'
+        file_open_mode = 'rb' if self.serialization_format == "pkl" else 'r'
         if os.name == 'nt':
             handle = CreateFileW(file_name, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE | FILE_SHARE_WRITE, None, OPEN_EXISTING, 0, None)
             if int(handle) == INVALID_HANDLE_VALUE:
@@ -394,27 +391,27 @@ class FileDirDict(PersiDict):
 
             fd = None
             try:
-                if self.file_type == "pkl":
+                if self.serialization_format == "pkl":
                     fd_open_mode = os.O_RDONLY | os.O_BINARY
                 else:
                     fd_open_mode = os.O_RDONLY
                 fd = msvcrt.open_osfhandle(int(handle),fd_open_mode)
-            except Exception:
+            except:
                 CloseHandle(handle)
                 raise
 
             try:
                 f = os.fdopen(fd, file_open_mode)
                 fd = None
-            except Exception:
+            except:
                 if fd is not None:
                     os.close(fd)
                 raise
 
             with f:
-                if self.file_type == "pkl":
+                if self.serialization_format == "pkl":
                     result = joblib.load(f)
-                elif self.file_type == "json":
+                elif self.serialization_format == "json":
                     result = jsonpickle.loads(f.read())
                 else:
                     result = f.read()
@@ -422,9 +419,9 @@ class FileDirDict(PersiDict):
             return result
         else:
             with open(file_name, file_open_mode) as f:
-                if self.file_type == "pkl":
+                if self.serialization_format == "pkl":
                     result = joblib.load(f)
-                elif self.file_type == "json":
+                elif self.serialization_format == "json":
                     result = jsonpickle.loads(f.read())
                 else:
                     result = f.read()
@@ -434,7 +431,7 @@ class FileDirDict(PersiDict):
     def _read_from_file(self,file_name:str) -> Any:
         """Read a value from a file with retry/backoff for concurrency.
 
-        Validates that the configured file_type is compatible with the allowed
+        Validates that the configured serialization_format is compatible with the allowed
         value types, then attempts to read the file using an exponential backoff
         to better tolerate concurrent writers.
 
@@ -442,17 +439,17 @@ class FileDirDict(PersiDict):
             file_name (str): Absolute path of the file to read.
 
         Returns:
-            Any: The deserialized value according to file_type.
+            Any: The deserialized value according to serialization_format.
 
         Raises:
-            ValueError: If file_type is incompatible with non-string values.
+            ValueError: If serialization_format is incompatible with non-string values.
             Exception: Propagates the last exception if all retries fail.
         """
 
-        if not (self.file_type in {"pkl", "json"} or issubclass(
+        if not (self.serialization_format in {"pkl", "json"} or issubclass(
             self.base_class_for_values, str)):
             raise ValueError("When base_class_for_values is not str,"
-                + " file_type must be pkl or json.")
+                + " serialization_format must be pkl or json.")
 
         n_retries = 12
         # extra protections to better handle concurrent writes
@@ -461,7 +458,7 @@ class FileDirDict(PersiDict):
                 return self._read_from_file_impl(file_name)
             except Exception as e:
                 if i < n_retries - 1:
-                    time.sleep(random.uniform(0.01, 0.2) * (1.5 ** i))
+                    time.sleep(random.uniform(0.01, 0.2) * (1.75 ** i))
                 else:
                     raise e
 
@@ -482,12 +479,12 @@ class FileDirDict(PersiDict):
         fd, temp_path = tempfile.mkstemp(dir=dir_name, prefix=".__tmp__")
 
         try:
-            if self.file_type == "pkl":
+            if self.serialization_format == "pkl":
                 with open(fd, 'wb') as f:
                     joblib.dump(value, f, compress='lz4')
                     f.flush()
                     os.fsync(f.fileno())
-            elif self.file_type == "json":
+            elif self.serialization_format == "json":
                 with open(fd, 'w') as f:
                     f.write(jsonpickle.dumps(value, indent=4))
                     f.flush()
@@ -506,8 +503,8 @@ class FileDirDict(PersiDict):
                     finally:
                         os.close(dir_fd)
                 elif os.name == 'nt':
-                # On Windows, try to flush directory metadata
-                # This is less reliable than on POSIX systems
+                    # On Windows, try to flush directory metadata
+                    # This is less reliable than on POSIX systems
                     try:
                         handle = CreateFileW(
                             dir_name,
@@ -539,7 +536,7 @@ class FileDirDict(PersiDict):
     def _save_to_file(self, file_name:str, value:Any) -> None:
         """Save a value to a file with retry/backoff.
 
-        Ensures the configured file_type is compatible with value types and then
+        Ensures the configured serialization_format is compatible with value types and then
         writes the value using an exponential backoff to better tolerate
         concurrent readers/writers.
 
@@ -548,14 +545,14 @@ class FileDirDict(PersiDict):
             value (Any): Value to serialize and save.
 
         Raises:
-            ValueError: If file_type is incompatible with non-string values.
+            ValueError: If serialization_format is incompatible with non-string values.
             Exception: Propagates the last exception if all retries fail.
         """
 
-        if not (self.file_type in {"pkl", "json"} or issubclass(
+        if not (self.serialization_format in {"pkl", "json"} or issubclass(
             self.base_class_for_values, str)):
             raise ValueError("When base_class_for_values is not str,"
-                + " file_type must be pkl or json.")
+                + " serialization_format must be pkl or json.")
 
         n_retries = 12
         # extra protections to better handle concurrent writes
@@ -565,33 +562,35 @@ class FileDirDict(PersiDict):
                 return
             except Exception as e:
                 if i < n_retries - 1:
-                    time.sleep(random.uniform(0.01, 0.2) * (1.5 ** i))
+                    time.sleep(random.uniform(0.01, 0.2) * (1.75 ** i))
                 else:
                     raise e
 
 
-    def __contains__(self, key:PersiDictKey) -> bool:
+    def __contains__(self, key:NonEmptyPersiDictKey) -> bool:
         """Check whether a key exists in the dictionary.
 
         Args:
-            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+            key (NonEmptyPersiDictKey): Key (string or sequence of strings
+                or NonEmptySafeStrTuple).
 
         Returns:
             bool: True if a file for the key exists; False otherwise.
         """
-        key = non_empty_persidict_key(key)
+        key = NonEmptySafeStrTuple(key)
         filename = self._build_full_path(key)
         return os.path.isfile(filename)
 
 
-    def __getitem__(self, key:PersiDictKey) -> Any:
+    def __getitem__(self, key:NonEmptyPersiDictKey) -> Any:
         """Retrieve the value stored for a key.
 
         Equivalent to obj[key]. Reads the corresponding file from the disk and
-        deserializes according to file_type.
+        deserializes according to serialization_format.
 
         Args:
-            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+            key (NonEmptyPersiDictKey): Key (string or sequence of strings
+                or NonEmptySafeStrTuple).
 
         Returns:
             Any: The stored value.
@@ -601,7 +600,7 @@ class FileDirDict(PersiDict):
             TypeError: If the deserialized value does not match base_class_for_values
                 when it is set.
         """
-        key = non_empty_persidict_key(key)
+        key = NonEmptySafeStrTuple(key)
         filename = self._build_full_path(key)
         if not os.path.isfile(filename):
             raise KeyError(f"File {filename} does not exist")
@@ -614,44 +613,45 @@ class FileDirDict(PersiDict):
         return result
 
 
-    def __setitem__(self, key:PersiDictKey, value:Any):
+    def __setitem__(self, key:NonEmptyPersiDictKey, value:Any):
         """Store a value for a key on the disk.
 
         Interprets joker values KEEP_CURRENT and DELETE_CURRENT accordingly.
         Validates value type if base_class_for_values is set, then serializes
-        and writes to a file determined by the key and file_type.
+        and writes to a file determined by the key and serialization_format.
 
         Args:
-            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+            key (NonEmptyPersiDictKey): Key (string or sequence of strings
+                or NonEmptySafeStrTuple).
             value (Any): Value to store, or a joker command.
 
         Raises:
             KeyError: If attempting to modify an existing item when
-                immutable_items is True.
+                append_only is True.
             TypeError: If the value is a PersiDict or does not match
                 base_class_for_values when it is set.
         """
 
-        key = non_empty_persidict_key(key)
-        PersiDict.__setitem__(self, key, value)
-        if isinstance(value, Joker):
-            # processed by base class
-            return
+        key = NonEmptySafeStrTuple(key)
+        if self._process_setitem_args(key, value) is EXECUTION_IS_COMPLETE:
+            return None
 
         filename = self._build_full_path(key, create_subdirs=True)
         self._save_to_file(filename, value)
 
 
-    def __delitem__(self, key:PersiDictKey) -> None:
+    def __delitem__(self, key:NonEmptyPersiDictKey) -> None:
         """Delete the stored value for a key.
 
         Args:
-            key (PersiDictKey): Key (string or sequence of strings) or SafeStrTuple.
+            key (NonEmptyPersiDictKey): Key (string or sequence of strings
+                or NonEmptySafeStrTuple).
 
         Raises:
-            KeyError: If immutable_items is True or if the key does not exist.
+            KeyError: If append_only is True or if the key does not exist.
         """
-        key = non_empty_persidict_key(key)
+        key = NonEmptySafeStrTuple(key)
+        self._process_delitem_args(key)
         filename = self._build_full_path(key)
         if not os.path.isfile(filename):
             raise KeyError(f"File {filename} does not exist")
@@ -681,9 +681,9 @@ class FileDirDict(PersiDict):
             ValueError: If result_type is empty or contains unsupported labels.
         """
 
-        PersiDict._generic_iter(self, result_type)
+        self._process_generic_iter_args(result_type)
         walk_results = os.walk(self._base_dir)
-        ext_len = len(self.file_type) + 1
+        ext_len = len(self.serialization_format) + 1
 
         def splitter(dir_path: str):
             """Transform a relative dirname into SafeStrTuple components.
@@ -700,7 +700,7 @@ class FileDirDict(PersiDict):
 
         def step():
             """Generator that yields entries based on result_type."""
-            suffix = "." + self.file_type
+            suffix = "." + self.serialization_format
             for dir_name, _, files in walk_results:
                 for f in files:
                     if f.endswith(suffix):
@@ -714,13 +714,21 @@ class FileDirDict(PersiDict):
                         to_return = []
 
                         if "keys" in result_type:
-                            key_to_return= unsign_safe_str_tuple(
+                            key_to_return = unsign_safe_str_tuple(
                                 result_key, self.digest_len)
                             to_return.append(key_to_return)
 
                         if "values" in result_type:
+                            # The file can be deleted between listing and fetching.
+                            # Skip such races instead of raising to make iteration robust.
                             full_path = os.path.join(dir_name, f)
-                            value_to_return = self._read_from_file(full_path)
+                            try:
+                                value_to_return = self._read_from_file(full_path)
+                            except:
+                                if not os.path.isfile(full_path):
+                                    continue
+                                else:
+                                    raise
                             to_return.append(value_to_return)
 
                         if len(result_type) == 1:
@@ -735,13 +743,13 @@ class FileDirDict(PersiDict):
         return step()
 
 
-    def timestamp(self, key:PersiDictKey) -> float:
+    def timestamp(self, key:NonEmptyPersiDictKey) -> float:
         """Get last modification time (in seconds, Unix epoch time).
 
         This method is absent in the original dict API.
 
         Args:
-            key (PersiDictKey): Key whose timestamp to return.
+            key (NonEmptyPersiDictKey): Key whose timestamp to return.
 
         Returns:
             float: POSIX timestamp of the underlying file.
@@ -749,25 +757,25 @@ class FileDirDict(PersiDict):
         Raises:
             FileNotFoundError: If the key does not exist.
         """
-        key = non_empty_persidict_key(key)
+        key = NonEmptySafeStrTuple(key)
         filename = self._build_full_path(key)
         return os.path.getmtime(filename)
 
 
-    def random_key(self) -> PersiDictKey | None:
+    def random_key(self) -> NonEmptySafeStrTuple | None:
         """Return a uniformly random key from the dictionary, or None if empty.
 
         Performs a full directory traversal using reservoir sampling
-        (k=1) to select a random file matching the configured file_type without
+        (k=1) to select a random file matching the configured serialization_format without
         loading all keys into memory.
 
         Returns:
-            PersiDictKey | None: A random key if any items exist; otherwise None.
+            NonEmptySafeStrTuple | None: A random key if any items exist; otherwise None.
         """
         # canonicalise extension once
         ext = None
-        if self.file_type:
-            ext = self.file_type
+        if self.serialization_format:
+            ext = self.serialization_format
             if not ext.startswith("."):
                 ext = "." + ext